[infinispan-commits] Infinispan SVN: r2145 - in trunk/lucene-directory/src: test/java/org/infinispan/lucene and 1 other directories.
infinispan-commits at lists.jboss.org
infinispan-commits at lists.jboss.org
Tue Aug 3 14:08:33 EDT 2010
Author: sannegrinovero
Date: 2010-08-03 14:08:32 -0400 (Tue, 03 Aug 2010)
New Revision: 2145
Modified:
trunk/lucene-directory/src/main/java/org/infinispan/lucene/InfinispanIndexOutput.java
trunk/lucene-directory/src/test/java/org/infinispan/lucene/DirectoryIntegrityCheck.java
trunk/lucene-directory/src/test/java/org/infinispan/lucene/InfinispanDirectoryIOTest.java
trunk/lucene-directory/src/test/java/org/infinispan/lucene/testutils/RepeatableLongByteSequence.java
Log:
[ISPN-523] (reduce Lucene Directory memory usage: store only initialized buffer ranges) trunk
Modified: trunk/lucene-directory/src/main/java/org/infinispan/lucene/InfinispanIndexOutput.java
===================================================================
--- trunk/lucene-directory/src/main/java/org/infinispan/lucene/InfinispanIndexOutput.java 2010-08-03 18:07:01 UTC (rev 2144)
+++ trunk/lucene-directory/src/main/java/org/infinispan/lucene/InfinispanIndexOutput.java 2010-08-03 18:08:32 UTC (rev 2145)
@@ -64,9 +64,20 @@
}
}
- private static byte[] getChunkById(AdvancedCache<CacheKey, Object> cache, FileCacheKey fileKey, int chunkNumber) {
+ private static byte[] getChunkById(AdvancedCache<CacheKey, Object> cache, FileCacheKey fileKey, int chunkNumber, int bufferSize) {
CacheKey key = new ChunkCacheKey(fileKey.getIndexName(), fileKey.getFileName(), chunkNumber);
- return (byte[]) cache.withFlags(Flag.SKIP_LOCKING).get(key);
+ byte[] readBuffer = (byte[]) cache.withFlags(Flag.SKIP_LOCKING).get(key);
+ if (readBuffer==null) {
+ return new byte[bufferSize];
+ }
+ else if (readBuffer.length==bufferSize) {
+ return readBuffer;
+ }
+ else {
+ byte[] newBuffer = new byte[bufferSize];
+ System.arraycopy(readBuffer, 0, newBuffer, 0, readBuffer.length);
+ return newBuffer;
+ }
}
@@ -80,11 +91,9 @@
private void newChunk() throws IOException {
flush();// save data first
+ currentChunkNumber++;
// check if we have to create new chunk, or get already existing in cache for modification
- currentChunkNumber++;
- if ((buffer = getChunkById(cache, fileKey, currentChunkNumber)) == null) {
- buffer = new byte[bufferSize];
- }
+ buffer = getChunkById(cache, fileKey, currentChunkNumber, bufferSize);
positionInBuffer = 0;
}
@@ -101,7 +110,7 @@
public void writeBytes(byte[] b, int offset, int length) throws IOException {
int writtenBytes = 0;
while (writtenBytes < length) {
- int pieceLength = Math.min(buffer.length - positionInBuffer, length - writtenBytes);
+ int pieceLength = Math.min(bufferSize - positionInBuffer, length - writtenBytes);
System.arraycopy(b, offset + writtenBytes, buffer, positionInBuffer, pieceLength);
positionInBuffer += pieceLength;
filePosition += pieceLength;
@@ -123,9 +132,20 @@
// size changed, apply change to file header
file.touch();
resizeFileIfNeeded();
+ if (file.getSize() < filePosition) {
+ file.setSize(filePosition);
+ }
+ byte[] bufferToFlush = buffer;
+ if (isWritingOnLastChunk()) {
+ int newBufferSize = (int) (file.getSize() % bufferSize);
+ if (newBufferSize != 0) {
+ bufferToFlush = new byte[newBufferSize];
+ System.arraycopy(buffer, 0, bufferToFlush, 0, newBufferSize);
+ }
+ }
cache.startBatch();
// add chunk to cache
- cache.withFlags(Flag.SKIP_REMOTE_LOOKUP).put(key, buffer);
+ cache.withFlags(Flag.SKIP_REMOTE_LOOKUP).put(key, bufferToFlush);
// override existing file header with new size and last time access
cache.withFlags(Flag.SKIP_REMOTE_LOOKUP).put(fileKey, file);
cache.endBatch(true);
@@ -164,7 +184,7 @@
}
if (requestedChunkNumber != currentChunkNumber) {
flush();
- buffer = getChunkById(cache, fileKey, requestedChunkNumber);
+ buffer = getChunkById(cache, fileKey, requestedChunkNumber, bufferSize);
currentChunkNumber = requestedChunkNumber;
}
positionInBuffer = getPositionInBuffer(pos, bufferSize);
@@ -176,4 +196,9 @@
return file.getSize();
}
-}
\ No newline at end of file
+
+ private boolean isWritingOnLastChunk() {
+ int lastChunkNumber = (int) (file.getSize() / bufferSize);
+ return currentChunkNumber == lastChunkNumber;
+ }
+}
Modified: trunk/lucene-directory/src/test/java/org/infinispan/lucene/DirectoryIntegrityCheck.java
===================================================================
--- trunk/lucene-directory/src/test/java/org/infinispan/lucene/DirectoryIntegrityCheck.java 2010-08-03 18:07:01 UTC (rev 2144)
+++ trunk/lucene-directory/src/test/java/org/infinispan/lucene/DirectoryIntegrityCheck.java 2010-08-03 18:08:32 UTC (rev 2145)
@@ -77,7 +77,7 @@
FileMetadata metadata = (FileMetadata) value;
long totalFileSize = metadata.getSize();
long actualFileSize = deepCountFileSize(fileCacheKey, cache);
-// Assert.assertEquals(actualFileSize, totalFileSize); Depends on ISPN-523
+ Assert.assertEquals(actualFileSize, totalFileSize);
} else if (key instanceof FileListCacheKey) {
fileListCacheKeyInstances++;
Assert.assertEquals(1, fileListCacheKeyInstances);
Modified: trunk/lucene-directory/src/test/java/org/infinispan/lucene/InfinispanDirectoryIOTest.java
===================================================================
--- trunk/lucene-directory/src/test/java/org/infinispan/lucene/InfinispanDirectoryIOTest.java 2010-08-03 18:07:01 UTC (rev 2144)
+++ trunk/lucene-directory/src/test/java/org/infinispan/lucene/InfinispanDirectoryIOTest.java 2010-08-03 18:08:32 UTC (rev 2145)
@@ -69,6 +69,70 @@
}
@Test
+ public void testWriteUsingSeekMethod() throws IOException {
+ final int BUFFER_SIZE = 64;
+
+ Cache<CacheKey, Object> cache = cacheManager.getCache();
+ InfinispanDirectory dir = new InfinispanDirectory(cache, INDEXNAME, BUFFER_SIZE);
+
+ String fileName = "SomeText.txt";
+ IndexOutput io = dir.createOutput(fileName);
+ RepeatableLongByteSequence bytesGenerator = new RepeatableLongByteSequence();
+ //It writes repeatable text
+ int REPEATABLE_BUFFER_SIZE = 1501;
+ for (int i = 0; i < REPEATABLE_BUFFER_SIZE; i++) {
+ io.writeByte(bytesGenerator.nextByte());
+ }
+ io.flush();
+
+ //Text to write on file with repeatable text
+ String someText = "This is some text";
+ byte[] someTextAsBytes = someText.getBytes();
+ //4 points in random order where writing someText: at begin of file, at end of file, within a single chunk,
+ //between 2 chunks
+ int[] pointers = {0, 635, REPEATABLE_BUFFER_SIZE, 135};
+
+ for(int i=0; i < pointers.length; i++) {
+ io.seek(pointers[i]);
+ io.writeBytes(someTextAsBytes, someTextAsBytes.length);
+ }
+
+ io.close();
+ bytesGenerator.reset();
+ final long finalSize = REPEATABLE_BUFFER_SIZE + someTextAsBytes.length;
+ assert io.length() == finalSize;
+
+ int indexPointer = 0;
+ Arrays.sort(pointers);
+ byte[] buffer = null;
+ int chunkIndex = 0;
+ //now testing the stream is equal to the produced repeatable but including the edits at pointed positions
+ for (int i = 0; i < REPEATABLE_BUFFER_SIZE + someTextAsBytes.length; i++) {
+ if(i % BUFFER_SIZE == 0) {
+ buffer = (byte[])cache.get(new ChunkCacheKey(INDEXNAME, fileName, chunkIndex++));
+ }
+
+ byte predictableByte = bytesGenerator.nextByte();
+ if(i < pointers[indexPointer]) {
+ //Assert predictable text
+ Assert.assertEquals(predictableByte, buffer[i % BUFFER_SIZE]);
+ } else if(pointers[indexPointer] <= i && i < pointers[indexPointer] + someTextAsBytes.length){
+ //Assert someText
+ Assert.assertEquals(someTextAsBytes[i - pointers[indexPointer]], buffer[i % BUFFER_SIZE]);
+ }
+
+ if(i == pointers[indexPointer] + someTextAsBytes.length) {
+ //Change pointer
+ indexPointer++;
+ }
+ }
+
+ dir.close();
+ DirectoryIntegrityCheck.verifyDirectoryStructure(cache, INDEXNAME);
+ }
+
+
+ @Test
public void testReadWholeFile() throws IOException {
final int BUFFER_SIZE = 64;
@@ -438,6 +502,7 @@
DirectoryIntegrityCheck.verifyDirectoryStructure(cache, INDEXNAME);
}
+ @Test
public void testWriteChunksDefaultChunks() throws Exception {
Cache<CacheKey, Object> cache = cacheManager.getCache();
InfinispanDirectory dir = new InfinispanDirectory(cache, INDEXNAME);
Modified: trunk/lucene-directory/src/test/java/org/infinispan/lucene/testutils/RepeatableLongByteSequence.java
===================================================================
--- trunk/lucene-directory/src/test/java/org/infinispan/lucene/testutils/RepeatableLongByteSequence.java 2010-08-03 18:07:01 UTC (rev 2144)
+++ trunk/lucene-directory/src/test/java/org/infinispan/lucene/testutils/RepeatableLongByteSequence.java 2010-08-03 18:08:32 UTC (rev 2145)
@@ -70,6 +70,13 @@
buffer[i] = nextByte();
}
}
+
+ public void reset() {
+ lastUsedValue = -1;
+ currentMax = (byte) 1;
+ currentMin = (byte) -1;
+ rising = true;
+ }
@Test(description="To verify the RepeatableLongByteSequence meets the requirement of producing "
+ "always the same values when using the single nextByte()")
More information about the infinispan-commits
mailing list