[infinispan-commits] Infinispan SVN: r1968 - in trunk/lucene-directory/src: test/java/org/infinispan/lucene and 1 other directory.
infinispan-commits at lists.jboss.org
infinispan-commits at lists.jboss.org
Sun Jul 4 05:12:09 EDT 2010
Author: sannegrinovero
Date: 2010-07-04 05:12:08 -0400 (Sun, 04 Jul 2010)
New Revision: 1968
Added:
trunk/lucene-directory/src/test/java/org/infinispan/lucene/DirectoryIntegrityCheck.java
Modified:
trunk/lucene-directory/src/main/java/org/infinispan/lucene/InfinispanIndexOutput.java
trunk/lucene-directory/src/test/java/org/infinispan/lucene/InfinispanDirectoryIOTest.java
trunk/lucene-directory/src/test/java/org/infinispan/lucene/SimpleLuceneTest.java
Log:
[ISPN-523] (reduce Lucene Directory memory usage: store only initialized buffer ranges) - trunk
Modified: trunk/lucene-directory/src/main/java/org/infinispan/lucene/InfinispanIndexOutput.java
===================================================================
--- trunk/lucene-directory/src/main/java/org/infinispan/lucene/InfinispanIndexOutput.java 2010-07-04 09:02:06 UTC (rev 1967)
+++ trunk/lucene-directory/src/main/java/org/infinispan/lucene/InfinispanIndexOutput.java 2010-07-04 09:12:08 UTC (rev 1968)
@@ -66,7 +66,18 @@
private static byte[] getChunkFromPosition(AdvancedCache<CacheKey, Object> cache, FileCacheKey fileKey, int pos, int bufferSize) {
CacheKey key = new ChunkCacheKey(fileKey.getIndexName(), fileKey.getFileName(), getChunkNumberFromPosition(pos, bufferSize));
- return (byte[]) cache.withFlags(Flag.SKIP_LOCKING).get(key);
+ byte[] readBuffer = (byte[]) cache.withFlags(Flag.SKIP_LOCKING).get(key);
+ if (readBuffer==null) {
+ return new byte[bufferSize];
+ }
+ else if (readBuffer.length==bufferSize) {
+ return readBuffer;
+ }
+ else {
+ byte[] newBuffer = new byte[bufferSize];
+ System.arraycopy(readBuffer, 0, newBuffer, 0, readBuffer.length);
+ return newBuffer;
+ }
}
private static int getPositionInBuffer(int pos, int bufferSize) {
@@ -80,9 +91,7 @@
private void newChunk() throws IOException {
flush();// save data first
// check if we have to create new chunk, or get already existing in cache for modification
- if ((buffer = getChunkFromPosition(cache, fileKey, filePosition, bufferSize)) == null) {
- buffer = new byte[bufferSize];
- }
+ buffer = getChunkFromPosition(cache, fileKey, filePosition, bufferSize);
bufferPosition = 0;
}
@@ -97,7 +106,7 @@
public void writeBytes(byte[] b, int offset, int length) throws IOException {
int writedBytes = 0;
while (writedBytes < length) {
- int pieceLength = Math.min(buffer.length - bufferPosition, length - writedBytes);
+ int pieceLength = Math.min(bufferSize - bufferPosition, length - writedBytes);
System.arraycopy(b, offset + writedBytes, buffer, bufferPosition, pieceLength);
bufferPosition += pieceLength;
filePosition += pieceLength;
@@ -122,9 +131,17 @@
if (file.getSize() < filePosition) {
file.setSize(filePosition);
}
+ int newBufferSize = (int) (file.getSize() % bufferSize);
+ byte[] shortedBuffer;
+ if (newBufferSize != 0) {
+ shortedBuffer = new byte[newBufferSize];
+ System.arraycopy(buffer, 0, shortedBuffer, 0, newBufferSize);
+ } else {
+ shortedBuffer = buffer;
+ }
cache.startBatch();
// add chunk to cache
- cache.withFlags(Flag.SKIP_REMOTE_LOOKUP).put(key, buffer);
+ cache.withFlags(Flag.SKIP_REMOTE_LOOKUP).put(key, shortedBuffer);
// override existing file header with new size and last time access
cache.withFlags(Flag.SKIP_REMOTE_LOOKUP).put(fileKey, file);
cache.endBatch(true);
Added: trunk/lucene-directory/src/test/java/org/infinispan/lucene/DirectoryIntegrityCheck.java
===================================================================
--- trunk/lucene-directory/src/test/java/org/infinispan/lucene/DirectoryIntegrityCheck.java (rev 0)
+++ trunk/lucene-directory/src/test/java/org/infinispan/lucene/DirectoryIntegrityCheck.java 2010-07-04 09:12:08 UTC (rev 1968)
@@ -0,0 +1,114 @@
+/*
+ * JBoss, Home of Professional Open Source.
+ * Copyright 2009, Red Hat Middleware LLC, and individual contributors
+ * as indicated by the @author tags. See the copyright.txt file in the
+ * distribution for a full listing of individual contributors.
+ *
+ * This is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this software; if not, write to the Free
+ * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+ * 02110-1301 USA, or see the FSF site: http://www.fsf.org.
+ */
+package org.infinispan.lucene;
+
+import java.util.Set;
+
+import org.infinispan.Cache;
+import org.infinispan.lucene.CacheKey;
+import org.infinispan.lucene.ChunkCacheKey;
+import org.infinispan.lucene.FileCacheKey;
+import org.infinispan.lucene.FileListCacheKey;
+import org.infinispan.lucene.FileMetadata;
+import org.testng.Assert;
+
+/**
+ * DirectoryIntegrityCheck contains helpers to assert assumptions we make on the structure of an
+ * index as stored in an Infinispan cache.
+ *
+ * @author Sanne Grinovero
+ * @since 4.1
+ */
+public class DirectoryIntegrityCheck {
+
+ /**
+ * Verifies that no garbage elements are left over in the cache and that for each type of object
+ * the expected value is stored. Also asserts for proper size metadata comparing to actual bytes
+ * used in chunks. It's assumed that only one index is stored in the inspected cache, and that
+ * the index is not being used by IndexReaders or IndexWriters.
+ *
+ * @param cache
+ * The cache to inspect
+ * @param indexName
+ * The name of the unique index stored in the cache
+ */
+ public static void verifyDirectoryStructure(Cache<CacheKey, Object> cache, String indexName) {
+ Set<String> fileList = (Set<String>) cache.get(new FileListCacheKey(indexName));
+ Assert.assertNotNull(fileList);
+ int fileListCacheKeyInstances = 0;
+ for (Object key : cache.keySet()) {
+ if (key instanceof ChunkCacheKey) {
+ ChunkCacheKey existingChunkKey = (ChunkCacheKey) key;
+ String filename = existingChunkKey.getFileName();
+ Assert.assertEquals(existingChunkKey.getIndexName(), indexName);
+ Assert.assertTrue(fileList.contains(filename));
+ Object value = cache.get(existingChunkKey);
+ Assert.assertNotNull(value);
+ Assert.assertTrue(value instanceof byte[]);
+ byte[] buffer = (byte[]) cache.get(existingChunkKey);
+ Assert.assertTrue(buffer.length != 0);
+ } else if (key instanceof FileCacheKey) {
+ FileCacheKey fileCacheKey = (FileCacheKey) key;
+ Assert.assertEquals(fileCacheKey.getIndexName(), indexName);
+ Assert.assertFalse(fileCacheKey.isLockKey());
+ Assert.assertTrue(fileList.contains(fileCacheKey.getFileName()), fileCacheKey + " should not have existed");
+ Object value = cache.get(fileCacheKey);
+ Assert.assertNotNull(value);
+ Assert.assertTrue(value instanceof FileMetadata);
+ FileMetadata metadata = (FileMetadata) value;
+ long totalFileSize = metadata.getSize();
+ long actualFileSize = deepCountFileSize(fileCacheKey, cache);
+ Assert.assertEquals(actualFileSize, totalFileSize);
+ } else if (key instanceof FileListCacheKey) {
+ fileListCacheKeyInstances++;
+ Assert.assertEquals(1, fileListCacheKeyInstances);
+ } else {
+ Assert.fail("an unexpected key was found in the cache having key type " + key.getClass() + " toString:" + key);
+ }
+ }
+ }
+
+ /**
+ * For a given FileCacheKey return the total size of all chunks related to the file.
+ *
+ * @param fileCacheKey
+ * the key to the file to inspect
+ * @param cache
+ * the cache storing the chunks
+ * @return the total size adding all found chunks up
+ */
+ public static long deepCountFileSize(FileCacheKey fileCacheKey, Cache<CacheKey, Object> cache) {
+ String indexName = fileCacheKey.getIndexName();
+ String fileName = fileCacheKey.getFileName();
+ long accumulator = 0;
+ for (int i = 0;; i++) {
+ ChunkCacheKey chunkKey = new ChunkCacheKey(indexName, fileName, i);
+ byte[] buffer = (byte[]) cache.get(chunkKey);
+ if (buffer == null) {
+ return accumulator;
+ } else {
+ accumulator += buffer.length;
+ }
+ }
+ }
+
+}
Modified: trunk/lucene-directory/src/test/java/org/infinispan/lucene/InfinispanDirectoryIOTest.java
===================================================================
--- trunk/lucene-directory/src/test/java/org/infinispan/lucene/InfinispanDirectoryIOTest.java 2010-07-04 09:02:06 UTC (rev 1967)
+++ trunk/lucene-directory/src/test/java/org/infinispan/lucene/InfinispanDirectoryIOTest.java 2010-07-04 09:12:08 UTC (rev 1968)
@@ -35,6 +35,7 @@
import org.infinispan.Cache;
import org.infinispan.lucene.testutils.RepeatableLongByteSequence;
import org.infinispan.manager.CacheContainer;
+import org.testng.annotations.AfterMethod;
import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
@@ -58,6 +59,11 @@
public void killCacheManager() {
cacheManager.stop();
}
+
+ @AfterMethod
+ public void clearCache() {
+ cacheManager.getCache().clear();
+ }
@Test
public void testReadWholeFile() throws IOException {
@@ -91,6 +97,9 @@
assertReadByteWorkingCorrectly(dir, "LonelyByteInLastChunk.txt", LAST_CHUNK_WITH_LONELY_BYTE_FILE_SIZE);
assertReadBytesWorkingCorrectly(dir, "LonelyByteInLastChunk.txt", LAST_CHUNK_WITH_LONELY_BYTE_FILE_SIZE, 12);
assert 5 == getChunksNumber(cache, "index", "LonelyByteInLastChunk.txt");
+
+ dir.close();
+ DirectoryIntegrityCheck.verifyDirectoryStructure(cache, "index");
}
@Test
@@ -124,6 +133,8 @@
}
indexInput.close();
+ dir.close();
+ DirectoryIntegrityCheck.verifyDirectoryStructure(cache, "index");
}
/**
@@ -376,6 +387,7 @@
assert new String(baos.toByteArray()).equals(worldText);
dir.close();
+ DirectoryIntegrityCheck.verifyDirectoryStructure(cache, "index");
}
public void testWriteChunks() throws Exception {
@@ -389,7 +401,7 @@
io.writeByte((byte) 66);
io.writeByte((byte) 69);
- io.close();
+ io.flush();
assert dir.fileExists("MyNewFile.txt");
assert null != cache.get(new ChunkCacheKey("index", "MyNewFile.txt", 0));
@@ -414,14 +426,15 @@
assert testText.equals(new String(chunk1) + new String(chunk2).trim());
dir.close();
+ DirectoryIntegrityCheck.verifyDirectoryStructure(cache, "index");
}
public void testWriteChunksDefaultChunks() throws Exception {
Cache<CacheKey, Object> cache = cacheManager.getCache();
InfinispanDirectory dir = new InfinispanDirectory(cache, "index");
- String testText = "This is some rubbish";
- byte[] testTextAsBytes = testText.getBytes();
+ final String testText = "This is some rubbish";
+ final byte[] testTextAsBytes = testText.getBytes();
IndexOutput io = dir.createOutput("MyNewFile.txt");
@@ -430,8 +443,12 @@
io.writeByte((byte) 3);
io.writeBytes(testTextAsBytes, testTextAsBytes.length);
io.close();
+ DirectoryIntegrityCheck.verifyDirectoryStructure(cache, "index");
- assert null != cache.get(new FileCacheKey("index", "MyNewFile.txt"));
+ FileCacheKey fileCacheKey = new FileCacheKey("index", "MyNewFile.txt");
+ assert null != cache.get(fileCacheKey);
+ FileMetadata metadata = (FileMetadata) cache.get(fileCacheKey);
+ Assert.assertEquals(testTextAsBytes.length + 3, metadata.getSize());
assert null != cache.get(new ChunkCacheKey("index", "MyNewFile.txt", 0));
// test contents by reading:
@@ -439,13 +456,14 @@
assert ii.readByte() == 1;
assert ii.readByte() == 2;
assert ii.readByte() == 3;
- byte[] buf = new byte[32];
+ byte[] buf = new byte[testTextAsBytes.length];
ii.readBytes(buf, 0, testTextAsBytes.length);
assert testText.equals(new String(buf).trim());
dir.close();
+ DirectoryIntegrityCheck.verifyDirectoryStructure(cache, "index");
}
}
Modified: trunk/lucene-directory/src/test/java/org/infinispan/lucene/SimpleLuceneTest.java
===================================================================
--- trunk/lucene-directory/src/test/java/org/infinispan/lucene/SimpleLuceneTest.java 2010-07-04 09:02:06 UTC (rev 1967)
+++ trunk/lucene-directory/src/test/java/org/infinispan/lucene/SimpleLuceneTest.java 2010-07-04 09:12:08 UTC (rev 1968)
@@ -75,6 +75,8 @@
assertTextIsFoundInIds(dirB, "node", 1);
dirA.close();
dirB.close();
+ DirectoryIntegrityCheck.verifyDirectoryStructure(cache(0, "lucene"), "indexName");
+ DirectoryIntegrityCheck.verifyDirectoryStructure(cache(1, "lucene"), "indexName");
}
@Test(description="Verifies the caches can be reused after a Directory close")
More information about the infinispan-commits
mailing list