[infinispan-commits] Infinispan SVN: r1303 - in trunk/lucene-directory/src: test/java/org/infinispan/lucene and 1 other directory.
infinispan-commits at lists.jboss.org
infinispan-commits at lists.jboss.org
Wed Dec 16 10:02:15 EST 2009
Author: sannegrinovero
Date: 2009-12-16 10:02:14 -0500 (Wed, 16 Dec 2009)
New Revision: 1303
Modified:
trunk/lucene-directory/src/main/java/org/infinispan/lucene/InfinispanIndexIO.java
trunk/lucene-directory/src/test/java/org/infinispan/lucene/InfinispanDirectoryIOTest.java
Log:
[ISPN-276] (Expensive to read a single byte in Lucene Directory)
Modified: trunk/lucene-directory/src/main/java/org/infinispan/lucene/InfinispanIndexIO.java
===================================================================
--- trunk/lucene-directory/src/main/java/org/infinispan/lucene/InfinispanIndexIO.java 2009-12-15 16:58:25 UTC (rev 1302)
+++ trunk/lucene-directory/src/main/java/org/infinispan/lucene/InfinispanIndexIO.java 2009-12-16 15:02:14 UTC (rev 1303)
@@ -36,6 +36,7 @@
*
* @since 4.0
* @author Lukasz Moren
+ * @author Davide Di Somma
* @see org.apache.lucene.store.Directory
* @see org.apache.lucene.store.IndexInput
* @see org.apache.lucene.store.IndexOutput
@@ -43,7 +44,7 @@
public class InfinispanIndexIO {
// used as default chunk size if not provided in conf
- // each Lucene index is splitted into parts with default size defined here
+ // each Lucene index segment is splitted into parts with default size defined here
public final static int DEFAULT_BUFFER_SIZE = 16 * 1024;
private static byte[] getChunkFromPosition(Map<CacheKey, Object> cache, FileCacheKey fileKey, int pos, int bufferSize) {
@@ -52,11 +53,11 @@
return (byte[]) cache.get(key);
}
- private static int getPositionInBuffer(int pos, int bufferSize) {
+ private static final int getPositionInBuffer(int pos, int bufferSize) {
return (pos % bufferSize);
}
- private static int getChunkNumberFromPosition(int pos, int bufferSize) {
+ private static final int getChunkNumberFromPosition(int pos, int bufferSize) {
return ((pos) / (bufferSize));
}
@@ -76,6 +77,7 @@
private byte[] buffer;
private int bufferPosition = 0;
private int filePosition = 0;
+ private int lastChunkNumberLoaded = -1;
public InfinispanIndexInput(Cache<CacheKey, Object> cache, FileCacheKey fileKey) throws IOException {
this(cache, fileKey, InfinispanIndexIO.DEFAULT_BUFFER_SIZE);
@@ -110,25 +112,20 @@
}
private byte[] getChunkFromPosition(Cache<CacheKey, Object> cache, FileCacheKey fileKey, int pos, int bufferSize) {
- Object object = InfinispanIndexIO.getChunkFromPosition(cache, fileKey, pos, bufferSize);
- if (object == null) {
- object = InfinispanIndexIO.getChunkFromPosition(localCache, fileKey, pos, bufferSize);
- }
- return (byte[]) object;
+ if(lastChunkNumberLoaded != getChunkNumberFromPosition(filePosition, bufferSize)) {
+ Object object = InfinispanIndexIO.getChunkFromPosition(cache, fileKey, pos, bufferSize);
+ if (object == null) {
+ object = InfinispanIndexIO.getChunkFromPosition(localCache, fileKey, pos, bufferSize);
+ }
+ lastChunkNumberLoaded = getChunkNumberFromPosition(filePosition, bufferSize);
+ return (byte[]) object;
+ } else {
+ return buffer;
+ }
+
}
public byte readByte() throws IOException {
- if (file == null) {
- throw new IOException("File " + fileKey + " does not exist");
- }
-
- if (filePosition == 0 && file.getSize() == 0) {
- if (log.isTraceEnabled()) {
- log.trace("pointer and file sizes are both 0; returning -1");
- }
- return -1;
- }
-
buffer = getChunkFromPosition(cache, fileKey, filePosition, bufferSize);
if (buffer == null) {
throw new IOException("Chunk id = [ " + getChunkNumberFromPosition(filePosition, bufferSize)
@@ -141,17 +138,6 @@
}
public void readBytes(byte[] b, int offset, int len) throws IOException {
-
- if (file == null) {
- throw new IOException("(null): File does not exist");
- }
-
- if (filePosition == 0 && file.getSize() == 0) {
- if (log.isTraceEnabled()) {
- log.trace("pointer and file sizes are both 0; returning -1");
- }
- }
-
int bytesToRead = len;
while (bytesToRead > 0) {
buffer = getChunkFromPosition(cache, fileKey, filePosition, bufferSize);
@@ -173,6 +159,7 @@
public void close() throws IOException {
filePosition = 0;
bufferPosition = 0;
+ lastChunkNumberLoaded = -1;
buffer = null;
localCache = null;
if (log.isDebugEnabled()) {
Modified: trunk/lucene-directory/src/test/java/org/infinispan/lucene/InfinispanDirectoryIOTest.java
===================================================================
--- trunk/lucene-directory/src/test/java/org/infinispan/lucene/InfinispanDirectoryIOTest.java 2009-12-15 16:58:25 UTC (rev 1302)
+++ trunk/lucene-directory/src/test/java/org/infinispan/lucene/InfinispanDirectoryIOTest.java 2009-12-16 15:02:14 UTC (rev 1303)
@@ -22,23 +22,226 @@
package org.infinispan.lucene;
import java.io.ByteArrayOutputStream;
+import java.io.IOException;
import java.util.Arrays;
import java.util.HashSet;
+import java.util.Random;
import java.util.Set;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.infinispan.Cache;
+import org.infinispan.lucene.testutils.RepeatableLongByteSequence;
import org.testng.annotations.Test;
/**
* @author Lukasz Moren
+ * @author Davide Di Somma
* @author Sanne Grinovero
*/
@Test(groups = "functional", testName = "lucene.InfinispanDirectoryIOTest")
public class InfinispanDirectoryIOTest {
- @Test(enabled=false)
+ @Test
+ public void testReadWholeFile() throws IOException {
+ final int BUFFER_SIZE = 64;
+
+ Cache<CacheKey, Object> cache = CacheTestSupport.createTestCacheManager().getCache();
+ InfinispanDirectory dir = new InfinispanDirectory(cache, "index", BUFFER_SIZE);
+
+ try {
+ final int SHORT_FILE_SIZE = 61;
+ assert BUFFER_SIZE > SHORT_FILE_SIZE;
+ createFileWithRepeatableContent(dir, "SingleChunk.txt", SHORT_FILE_SIZE);
+ assertReadByteWorkingCorrectly(dir, "SingleChunk.txt", SHORT_FILE_SIZE);
+ assertReadBytesWorkingCorrectly(dir, "SingleChunk.txt", SHORT_FILE_SIZE, 15);
+
+ final int VERY_BIG_FILE_SIZE = 10000;
+ assert BUFFER_SIZE < VERY_BIG_FILE_SIZE;
+ createFileWithRepeatableContent(dir, "MultipleChunks.txt", VERY_BIG_FILE_SIZE);
+ assertReadByteWorkingCorrectly(dir, "MultipleChunks.txt", VERY_BIG_FILE_SIZE);
+ assertReadBytesWorkingCorrectly(dir, "MultipleChunks.txt", VERY_BIG_FILE_SIZE, 33);
+
+ final int LAST_CHUNK_COMPLETELY_FILLED_FILE_SIZE = 256;
+ assert (LAST_CHUNK_COMPLETELY_FILLED_FILE_SIZE % BUFFER_SIZE) == 0;
+ createFileWithRepeatableContent(dir, "LastChunkFilled.txt",
+ LAST_CHUNK_COMPLETELY_FILLED_FILE_SIZE);
+ assertReadByteWorkingCorrectly(dir, "LastChunkFilled.txt",
+ LAST_CHUNK_COMPLETELY_FILLED_FILE_SIZE);
+ assertReadBytesWorkingCorrectly(dir, "LastChunkFilled.txt",
+ LAST_CHUNK_COMPLETELY_FILLED_FILE_SIZE, 11);
+ assert 4 == getChunksNumber(cache, "index", "LastChunkFilled.txt");
+
+ final int LAST_CHUNK_WITH_LONELY_BYTE_FILE_SIZE = 257;
+ assert (LAST_CHUNK_WITH_LONELY_BYTE_FILE_SIZE % BUFFER_SIZE) == 1;
+ createFileWithRepeatableContent(dir, "LonelyByteInLastChunk.txt",
+ LAST_CHUNK_WITH_LONELY_BYTE_FILE_SIZE);
+ assertReadByteWorkingCorrectly(dir, "LonelyByteInLastChunk.txt",
+ LAST_CHUNK_WITH_LONELY_BYTE_FILE_SIZE);
+ assertReadBytesWorkingCorrectly(dir, "LonelyByteInLastChunk.txt",
+ LAST_CHUNK_WITH_LONELY_BYTE_FILE_SIZE, 12);
+ assert 5 == getChunksNumber(cache, "index", "LonelyByteInLastChunk.txt");
+
+ } finally {
+ for (String fileName : dir.listAll()) {
+ dir.deleteFile(fileName);
+ }
+ cache.getCacheManager().stop();
+ dir.close();
+ }
+ }
+
+
+ @Test
+ public void testReadRandomSampleFile() throws IOException {
+ final int BUFFER_SIZE = 64;
+
+ Cache<CacheKey, Object> cache = CacheTestSupport.createTestCacheManager().getCache();
+ InfinispanDirectory dir = new InfinispanDirectory(cache, "index", BUFFER_SIZE);
+
+ try {
+ final int FILE_SIZE = 1000;
+ assert BUFFER_SIZE < FILE_SIZE;
+ createFileWithRepeatableContent(dir, "RandomSampleFile.txt", FILE_SIZE);
+
+ IndexInput indexInput = dir.openInput("RandomSampleFile.txt");
+ assert indexInput.length() == FILE_SIZE;
+ RepeatableLongByteSequence bytesGenerator = new RepeatableLongByteSequence();
+
+ Random r = new Random();
+ long seekPoint = 0;
+ //Now it reads some random byte and it compares to the expected byte
+ for (int i = 0; i < FILE_SIZE; i++) {
+ if(seekPoint == i) {
+ assert bytesGenerator.nextByte() == indexInput.readByte();
+ seekPoint = indexInput.getFilePointer() + r.nextInt(10);
+ indexInput.seek(seekPoint);
+ } else {
+ bytesGenerator.nextByte();
+ }
+
+ }
+ indexInput.close();
+
+ } finally {
+ for (String fileName : dir.listAll()) {
+ dir.deleteFile(fileName);
+ }
+ cache.getCacheManager().stop();
+ dir.close();
+ }
+ }
+
+
+
+ /**
+ * Used to verify that IndexInput.readBytes method reads correctly the whole file content comparing the
+ * result with the expected sequence of bytes
+ *
+ * @param dir
+ * The Directory containing the file to verify
+ * @param fileName
+ * The file name to read
+ * @param contentFileSizeExpected
+ * The size content file expected
+ * @param arrayLengthToRead
+ * The length of byte array to read
+ * @throws IOException
+ */
+ private void assertReadBytesWorkingCorrectly(InfinispanDirectory dir, String fileName,
+ final int contentFileSizeExpected, final int arrayLengthToRead) throws IOException {
+ IndexInput indexInput = dir.openInput(fileName);
+ assert indexInput.length() == contentFileSizeExpected;
+
+ RepeatableLongByteSequence bytesGenerator = new RepeatableLongByteSequence();
+
+ byte[] readBytes = new byte[arrayLengthToRead];
+ byte[] expectedBytes = new byte[arrayLengthToRead];
+
+ long toRead = contentFileSizeExpected;
+ while (toRead > 0) {
+ // the condition is satisfied when the file is close to the end
+ if (toRead < arrayLengthToRead) {
+ readBytes = new byte[(int) toRead];
+ expectedBytes = new byte[(int) toRead];
+ }
+ int nextBytesToRead = (int) Math.min(toRead, arrayLengthToRead);
+
+ bytesGenerator.nextBytes(expectedBytes);
+ indexInput.readBytes(readBytes, 0, nextBytesToRead);
+
+ assert Arrays.equals(expectedBytes, readBytes);
+
+ toRead = toRead - nextBytesToRead;
+
+ }
+ indexInput.close();
+ }
+
+ /**
+ * Used to verify that IndexInput.readByte method reads correctly the whole file content comparing the
+ * result with the expected sequence of bytes
+ *
+ * @param dir
+ * The Directory containing the file to verify
+ * @param fileName
+ * The file name to read
+ * @param contentFileSizeExpected
+ * The size content file expected
+ * @throws IOException
+ */
+ private void assertReadByteWorkingCorrectly(InfinispanDirectory dir, String fileName,
+ final int contentFileSizeExpected) throws IOException {
+ IndexInput indexInput = dir.openInput(fileName);
+ assert indexInput.length() == contentFileSizeExpected;
+ RepeatableLongByteSequence bytesGenerator = new RepeatableLongByteSequence();
+
+ for (int i = 0; i < contentFileSizeExpected; i++) {
+ assert bytesGenerator.nextByte() == indexInput.readByte();
+ }
+ indexInput.close();
+ }
+
+ /**
+ * It returns the number of chunks of file which is divided
+ *
+ * @param cache
+ * @param index
+ * @param fileName
+ * @return
+ */
+ private int getChunksNumber(Cache<CacheKey, Object> cache, String index, String fileName) {
+ int chunksNumber = 0;
+ while (cache.containsKey(new ChunkCacheKey(index, fileName, chunksNumber))) {
+ chunksNumber++;
+ }
+ return chunksNumber;
+ }
+
+ /**
+ * It creates a file with fixed size using a RepeatableLongByteSequence object to generate a
+ * repeatable content
+ *
+ * @param dir
+ * The Directory containing the file to create
+ * @param fileName
+ * The file name to create
+ * @param contentFileSize
+ * The size content file to create
+ * @throws IOException
+ */
+ private void createFileWithRepeatableContent(InfinispanDirectory dir, String fileName,
+ final int contentFileSize) throws IOException {
+ IndexOutput indexOutput = dir.createOutput(fileName);
+ RepeatableLongByteSequence bytesGenerator = new RepeatableLongByteSequence();
+ for (int i = 0; i < contentFileSize; i++) {
+ indexOutput.writeByte(bytesGenerator.nextByte());
+ }
+ indexOutput.close();
+ }
+
+
+ @Test(enabled = false)
public void testReadChunks() throws Exception {
final int BUFFER_SIZE = 64;
@@ -72,7 +275,7 @@
String part2 = new String(buf);
// make sure the generated bytes do add up!
- assert worldText.equals( part1 + part2.trim() );
+ assert worldText.equals(part1 + part2.trim());
file1.setSize(helloText.length());
file2.setSize(worldText.length());
@@ -199,7 +402,7 @@
io.close();
assert dir.fileExists("MyNewFile.txt");
- assert null!=cache.get(new ChunkCacheKey("index", "MyNewFile.txt", 0));
+ assert null != cache.get(new ChunkCacheKey("index", "MyNewFile.txt", 0));
// test contents by reading:
byte[] buf = new byte[9];
@@ -215,8 +418,8 @@
// now compare.
byte[] chunk1 = (byte[]) cache.get(new ChunkCacheKey("index", "MyNewFile.txt", 0));
byte[] chunk2 = (byte[]) cache.get(new ChunkCacheKey("index", "MyNewFile.txt", 1));
- assert null!=chunk1;
- assert null!=chunk2;
+ assert null != chunk1;
+ assert null != chunk2;
assert testText.equals(new String(chunk1) + new String(chunk2).trim());
@@ -239,14 +442,14 @@
io.writeBytes(testTextAsBytes, testTextAsBytes.length);
io.close();
- assert null!=cache.get(new FileCacheKey("index", "MyNewFile.txt"));
- assert null!=cache.get(new ChunkCacheKey("index", "MyNewFile.txt", 0));
+ assert null != cache.get(new FileCacheKey("index", "MyNewFile.txt"));
+ assert null != cache.get(new ChunkCacheKey("index", "MyNewFile.txt", 0));
// test contents by reading:
IndexInput ii = dir.openInput("MyNewFile.txt");
- assert ii.readByte()== 1;
- assert ii.readByte()== 2;
- assert ii.readByte()== 3;
+ assert ii.readByte() == 1;
+ assert ii.readByte() == 2;
+ assert ii.readByte() == 3;
byte[] buf = new byte[32];
ii.readBytes(buf, 0, testTextAsBytes.length);
@@ -256,4 +459,5 @@
cache.getCacheManager().stop();
dir.close();
}
+
}
More information about the infinispan-commits
mailing list