[hibernate-commits] Hibernate SVN: r14770 - in search/trunk/src: test/org/hibernate/search/test/reader/functionality and 1 other directories.

Author: sannegrinovero Date: 2008-06-16 11:43:48 -0400 (Mon, 16 Jun 2008) New Revision: 14770 Added: search/trunk/src/test/org/hibernate/search/test/reader/functionality/FilterOnDirectoryTest.java Modified: search/trunk/src/java/org/hibernate/search/reader/SharingBufferReaderProvider.java search/trunk/src/test/org/hibernate/search/test/reader/functionality/SharingBufferIndexProviderTest.java search/trunk/src/test/org/hibernate/search/test/reader/functionality/TestableSharingBufferReaderProvider.java search/trunk/src/test/org/hibernate/search/test/reader/performance/ReaderPerformance.java Log: HSEARCH-212 : a new ReaderProvider Modified: search/trunk/src/java/org/hibernate/search/reader/SharingBufferReaderProvider.java =================================================================== --- search/trunk/src/java/org/hibernate/search/reader/SharingBufferReaderProvider.java 2008-06-16 15:08:45 UTC (rev 14769) +++ search/trunk/src/java/org/hibernate/search/reader/SharingBufferReaderProvider.java 2008-06-16 15:43:48 UTC (rev 14770) @@ -1,17 +1,19 @@ package org.hibernate.search.reader; -import static org.hibernate.search.reader.ReaderProviderHelper.buildMultiReader; -import static org.hibernate.search.reader.ReaderProviderHelper.clean; - import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; import java.util.Map; import java.util.Properties; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.MultiReader; +import org.hibernate.annotations.common.AssertionFailure; import org.hibernate.search.SearchException; import org.hibernate.search.engine.SearchFactoryImplementor; import org.hibernate.search.store.DirectoryProvider; @@ -19,182 +21,209 @@ import org.slf4j.LoggerFactory; /** + * As does SharedReaderProvider this also shares IndexReaders as long as they are "current"; + * main difference with SharedReaderProvider is the way to update the Readers when needed: + * this uses IndexReader.reopen() which should improve performance on larger indexes + * as it shares buffers with previous IndexReader generation for the segments which didn't change. + * Current drawbacks are: need of Lucene > 2.3.0 and less mature (experimental). + * * @author Sanne Grinovero */ public class SharingBufferReaderProvider implements ReaderProvider { - //contains last updated Reader; protected by lockOnOpenLatest. - private volatile ReaderUsagePair current; + /** + * contains all Readers (most current per DP and all unclosed old) + */ + //TODO ConcurrentHashMap's constructor could benefit from some hints as arguments. + protected final Map<IndexReader,ReaderUsagePair> allReaders = new ConcurrentHashMap<IndexReader,ReaderUsagePair>( 100 ); - private final Lock lockOnOpenLatest = new ReentrantLock(); + /** + * contains last updated Reader; protected by lockOnOpenLatest (in the values) + */ + protected Map<DirectoryProvider,PerDirectoryLatestReader> currentReaders; - //contains all older Readers: - protected final Map<IndexReader,ReaderUsagePair> oldReaders = new ConcurrentHashMap<IndexReader,ReaderUsagePair>(); - - private final Logger log = LoggerFactory.getLogger ( SharingBufferReaderProvider.class ); + private final Logger log = LoggerFactory.getLogger( SharingBufferReaderProvider.class ); - public void closeReader(IndexReader reader) { - if ( reader == current.reader ) { - boolean closeit; - lockOnOpenLatest.lock(); - try { - if ( reader == current.reader ){ - current.usageCounter.getAndDecrement(); - closeit = false; - } - else { - closeit = true; - } - } - finally { - lockOnOpenLatest.unlock(); - } - if ( closeit ) { - closeOldReader( reader ); - } + public void closeReader(IndexReader multiReader) { + if ( multiReader == null ) return; + IndexReader[] readers; + if ( multiReader instanceof MultiReader ) { + readers = (IndexReader[]) ReaderProviderHelper.getSubReadersFromMultiReader( (MultiReader) multiReader ); } else { - closeOldReader( reader ); + throw new AssertionFailure( "Everything should be wrapped in a MultiReader" ); } - printState(); + log.trace( "Closing MultiReader: {}", multiReader ); + for ( IndexReader reader : readers ) { + ReaderUsagePair container = allReaders.get( reader ); + container.close();//virtual + } + log.trace( "IndexReader closed." ); } - private void closeOldReader(IndexReader reader) { - try { - ReaderUsagePair pair = oldReaders.get( reader ); - boolean closed = pair.close(); //also testing "assert pair!=null"; - if ( closed ) { - //not longer needed, so remove references: - oldReaders.remove( reader ); - log.trace( "IndexReader closed." ); + public void initialize(Properties props, SearchFactoryImplementor searchFactoryImplementor) { + Map<DirectoryProvider,PerDirectoryLatestReader> map = new HashMap<DirectoryProvider,PerDirectoryLatestReader>(); + Set<DirectoryProvider> providers = searchFactoryImplementor.getLockableDirectoryProviders().keySet(); + for ( DirectoryProvider provider : providers ) { + try { + map.put( provider, new PerDirectoryLatestReader( provider ) ); + } catch (IOException e) { + throw new SearchException( "Unable to open Lucene IndexReader", e ); } - else { - log.trace( "Closing of IndexReader skipped: still being used." ); - } } - catch (IOException e) { - log.warn( "Unable to close Lucene IndexReader", e ); - //remove references anyway: - oldReaders.remove( reader ); - } + //FIXME I'm not convinced this non-final fields are safe without locks, but I may be wrong. + currentReaders = Collections.unmodifiableMap( map ); } - public void initialize(Properties props, SearchFactoryImplementor searchFactoryImplementor) { - //FIXME initialize currentReaderContainer here instead of lazy - } - public IndexReader openReader(DirectoryProvider... directoryProviders) { boolean trace = log.isTraceEnabled(); - if ( trace ) log.trace( "Opening IndexReader for directoryProviders: {}", directoryProviders ); - IndexReader toReturn; - lockOnOpenLatest.lock(); - try { - if ( current == null ) { //FIXME move this case to initialize - current = initialReaderOpening( directoryProviders ); - log.trace( "IndexReader initialized." ); + int length = directoryProviders.length; + IndexReader[] readers = new IndexReader[length]; + if ( trace ) log.trace( "Opening IndexReader for directoryProviders: {}", length ); + for (int index = 0; index < length; index++) { + DirectoryProvider directoryProvider = directoryProviders[index]; + if ( trace ) log.trace( "Opening IndexReader from {}", directoryProvider.getDirectory() ); + PerDirectoryLatestReader directoryLatestReader = currentReaders.get( directoryProvider ); + readers[index] = directoryLatestReader.refreshAndGet(); + } + // don't use ReaderProviderHelper.buildMultiReader as we need our own cleanup. + if ( length == 0 ) { + return null; + } + else { + try { + return new CacheableMultiReader( readers ); } - else { - reopenIndexreader(); + catch (Exception e) { + //Lucene 2.2 used to throw IOExceptions here + for ( IndexReader ir : readers ) { + ReaderUsagePair readerUsagePair = allReaders.get( ir ); + readerUsagePair.close(); + } + throw new SearchException( "Unable to open a MultiReader", e ); } - toReturn = current.reader; //choose reader before unlock - } finally { - lockOnOpenLatest.unlock(); } - printState(); - return toReturn; } - private void reopenIndexreader() { - // we own the lock - IndexReader before = current.reader; - IndexReader updatedReader; - try { - updatedReader = before.reopen(); - } catch (IOException e) { - throw new SearchException( "Unable to reopen IndexReader", e ); + //overridable method for testability: + protected IndexReader readerFactory(DirectoryProvider provider) throws IOException { + return IndexReader.open( provider.getDirectory() ); + } + + /** + * Container for the couple IndexReader,UsageCounter. + */ + protected final class ReaderUsagePair { + + public final IndexReader reader; + /** + * When reaching 0 (always test on change) the reader should be really + * closed and then discarded. + * Starts at 2 because: + * first usage token is artificial: means "current" is not to be closed (+1) + * additionally when creating it will be used (+1) + */ + protected final AtomicInteger usageCounter = new AtomicInteger( 2 ); + + ReaderUsagePair(IndexReader r) { + reader = r; } - if ( before == updatedReader ) { - current.incrementUseCounter(); - } - else { //store the old one for close() functionality. - int useCount = current.usageCounter.get(); - if ( useCount != 0 ) { - oldReaders.put( before, current ); - } - else { - //or close it if nobody uses. + + /** + * closes the IndexReader if no other resource is using it; + * in this case the reference to this container will also be removed. + */ + public void close() { + int refCount = usageCounter.decrementAndGet(); + if ( refCount==0 ) { + //TODO I've been experimenting with the idea of an async-close: didn't appear to have an interesting benefit, + //so discarded the code. should try with bigger indexes to see if the effect gets more impressive. + ReaderUsagePair removed = allReaders.remove( reader );//remove ourself try { - current.reader.close(); + reader.close(); } catch (IOException e) { log.warn( "Unable to close Lucene IndexReader", e ); } + assert removed != null; } - current = new ReaderUsagePair( updatedReader ); - } - } - - public final void printState(){ - if ( log.isTraceEnabled()) - log.trace( "Current "+ current + " older:" + oldReaders.values() ); - } - - private ReaderUsagePair initialReaderOpening(DirectoryProvider[] directoryProviders) { - // we own the lock - final int length = directoryProviders.length; - IndexReader[] readers = new IndexReader[length]; - try { - for (int index = 0; index < length; index++) { - readers[index] = IndexReader.open( directoryProviders[index].getDirectory() ); + else if ( refCount<0 ) { + //doesn't happen with current code, could help spotting future bugs? + throw new AssertionFailure( "Closing an IndexReader for which you didn't own a lock-token, or somebody else which didn't own closed already." ); } } - catch (IOException e) { - //TODO more contextual info - clean( new SearchException( "Unable to open one of the Lucene indexes", e ), readers ); + + public String toString(){ + return "Reader:" + this.hashCode() + " ref.count=" + usageCounter.get(); } - IndexReader iR = readerFactory( length, readers ); - return new ReaderUsagePair( iR ); + } - //overridable method for testability: - protected IndexReader readerFactory(int length, IndexReader[] readers) { - return buildMultiReader( length, readers ); - } - - protected static class ReaderUsagePair { - protected final IndexReader reader; - protected final AtomicInteger usageCounter; + /** + * An instance for each DirectoryProvider, + * establishing the association between "current" ReaderUsagePair + * for a DirectoryProvider and it's lock. + */ + protected final class PerDirectoryLatestReader { - ReaderUsagePair(IndexReader r) { - reader = r; - usageCounter = new AtomicInteger( 1 ); - } + /** + * Reference to the most current IndexReader for a DirectoryProvider; + * guarded by lockOnReplaceCurrent; + */ + public ReaderUsagePair current; //guarded by lockOnReplaceCurrent + private final Lock lockOnReplaceCurrent = new ReentrantLock(); - void incrementUseCounter() { - usageCounter.incrementAndGet(); + /** + * @param provider The DirectoryProvider for which we manage the IndexReader. + * @throws IOException when the index initialization fails. + */ + public PerDirectoryLatestReader(DirectoryProvider provider) throws IOException { + IndexReader reader = readerFactory( provider ); + ReaderUsagePair initialPair = new ReaderUsagePair( reader ); + initialPair.usageCounter.set( 1 );//a token to mark as active (preventing real close). + lockOnReplaceCurrent.lock();//no harm, just ensuring safe publishing. + current = initialPair; + lockOnReplaceCurrent.unlock(); + allReaders.put( reader, initialPair ); } - - public int getUsageCount(){ - return usageCounter.get(); - } /** - * @return true when really closing the underlying IndexReader - * @throws IOException + * Gets an updated IndexReader for the current DirectoryProvider; + * the index status will be checked. + * @return the current IndexReader if it's in sync with underlying index, a new one otherwise. */ - private boolean close() throws IOException { - int count = usageCounter.decrementAndGet(); - if ( count == 0 ) { - reader.close(); - return true; + public IndexReader refreshAndGet() { + ReaderUsagePair previousCurrent; + IndexReader updatedReader; + lockOnReplaceCurrent.lock(); + try { + IndexReader beforeUpdateReader = current.reader; + try { + updatedReader = beforeUpdateReader.reopen(); + } catch (IOException e) { + throw new SearchException( "Unable to reopen IndexReader", e ); + } + if ( beforeUpdateReader == updatedReader ) { + previousCurrent = null; + current.usageCounter.incrementAndGet(); + } + else { + ReaderUsagePair newPair = new ReaderUsagePair( updatedReader ); + //no need to increment usageCounter in newPair, as it is constructed with correct number 2. + assert newPair.usageCounter.get() == 2; + previousCurrent = current; + current = newPair; + allReaders.put( updatedReader, newPair );//unfortunately still needs lock + } + } finally { + lockOnReplaceCurrent.unlock(); } - assert count >= 0; - return false; + // doesn't need lock: + if ( previousCurrent != null ) { + previousCurrent.close();// release a token as it's not the current any more. + } + return updatedReader; } - public String toString(){ - return "Reader:"+this.hashCode()+" count="+usageCounter.get(); - } - } - + } Added: search/trunk/src/test/org/hibernate/search/test/reader/functionality/FilterOnDirectoryTest.java =================================================================== --- search/trunk/src/test/org/hibernate/search/test/reader/functionality/FilterOnDirectoryTest.java (rev 0) +++ search/trunk/src/test/org/hibernate/search/test/reader/functionality/FilterOnDirectoryTest.java 2008-06-16 15:43:48 UTC (rev 14770) @@ -0,0 +1,70 @@ +package org.hibernate.search.test.reader.functionality; + +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.hibernate.Session; +import org.hibernate.Transaction; +import org.hibernate.search.Environment; +import org.hibernate.search.FullTextQuery; +import org.hibernate.search.FullTextSession; +import org.hibernate.search.Search; +import org.hibernate.search.reader.SharingBufferReaderProvider; +import org.hibernate.search.test.SearchTestCase; +import org.hibernate.search.test.reader.Detective; +import org.hibernate.search.test.reader.Suspect; + +public class FilterOnDirectoryTest extends SearchTestCase { + + public void testFilteredClasses() throws Exception { + createDoeFamily(); + FullTextSession fts = Search.createFullTextSession( openSession() ); + Transaction tx = fts.beginTransaction(); + Query q = new TermQuery( new Term( "name", "doe" ) ); + + assertEquals( 2, fts.createFullTextQuery( q ).getResultSize() ); + assertEquals( 2, fts.createFullTextQuery( q, Detective.class, Suspect.class ).getResultSize() ); + + FullTextQuery detectiveQuery = fts.createFullTextQuery( q, Detective.class ); + assertEquals( 1, detectiveQuery.getResultSize() ); + assertTrue( detectiveQuery.list().get(0) instanceof Detective ); + + FullTextQuery suspectQuery = fts.createFullTextQuery( q, Suspect.class ); + assertEquals( 1, suspectQuery.getResultSize() ); + assertTrue( suspectQuery.list().get(0) instanceof Suspect ); + + assertEquals( 2, fts.createFullTextQuery( q ).getResultSize() ); + assertEquals( 2, fts.createFullTextQuery( q, Detective.class, Suspect.class ).getResultSize() ); + + tx.commit(); + fts.close(); + } + + private void createDoeFamily() { + Session s = openSession( ); + Transaction tx = s.beginTransaction(); + Detective detective = new Detective(); + detective.setName( "John Doe" ); + s.persist( detective ); + Suspect suspect = new Suspect(); + suspect.setName( "Jane Doe" ); + s.persist( suspect ); + tx.commit(); + s.close(); + } + + protected void configure(org.hibernate.cfg.Configuration cfg) { + super.configure( cfg ); + cfg.setProperty( Environment.ANALYZER_CLASS, StandardAnalyzer.class.getName() ); + cfg.setProperty( Environment.READER_STRATEGY, SharingBufferReaderProvider.class.getName() ); + } + + protected Class[] getMappings() { + return new Class[] { + Detective.class, + Suspect.class + }; + } + +} Modified: search/trunk/src/test/org/hibernate/search/test/reader/functionality/SharingBufferIndexProviderTest.java =================================================================== --- search/trunk/src/test/org/hibernate/search/test/reader/functionality/SharingBufferIndexProviderTest.java 2008-06-16 15:08:45 UTC (rev 14769) +++ search/trunk/src/test/org/hibernate/search/test/reader/functionality/SharingBufferIndexProviderTest.java 2008-06-16 15:43:48 UTC (rev 14770) @@ -1,12 +1,18 @@ package org.hibernate.search.test.reader.functionality; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executors; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import org.apache.lucene.index.IndexReader; +import org.hibernate.search.store.DirectoryProvider; import org.hibernate.search.test.reader.functionality.TestableSharingBufferReaderProvider.MockIndexReader; +import org.hibernate.search.test.reader.functionality.TestableSharingBufferReaderProvider.TestManipulatorPerDP; import junit.framework.TestCase; @@ -21,24 +27,27 @@ private final Runnable changeTask = new ChangeTask(); private final AtomicInteger countDoneSearches = new AtomicInteger(); private final AtomicInteger countDoneIndexmods = new AtomicInteger(); - private static final int SEARCHES_NUM = 5000; + private static final int SEARCHES_NUM = 50000; + private static final Random random = new Random(); public void testStressingMock() throws InterruptedException { + readerProvider.initialize(null, null); ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool( 200 );//much chaos for ( int i = 0; i < SEARCHES_NUM; i++ ) { executor.execute( makeTask( i ) ); } executor.shutdown(); startSignal.countDown(); - executor.awaitTermination( 15, TimeUnit.SECONDS ); - assertTrue( "memory leak: holding a reference to some unused IndexReader", readerProvider.isMapEmpty() ); - MockIndexReader openReader = readerProvider.fakeOpenReader(); + executor.awaitTermination( 500, TimeUnit.SECONDS ); + assertTrue( "memory leak: holding a reference to some unused IndexReader", readerProvider.areAllOldReferencesGone() ); for ( MockIndexReader reader : readerProvider.getCreatedIndexReaders() ) { - if ( reader != openReader ) { + if ( readerProvider.isReaderCurrent( reader ) ) { + assertTrue( "the most current reader should be open", ! reader.isClosed() ); + } + else { assertTrue( "an IndexReader is still open", reader.isClosed() ); } } - assertTrue( "the most current reader should be open", ! openReader.isClosed() ); assertEquals( SEARCHES_NUM, countDoneSearches.get() ); assertEquals( SEARCHES_NUM/10, countDoneIndexmods.get() ); } @@ -52,6 +61,18 @@ } } + private DirectoryProvider[] getRandomEvailableDPs() { + int arraySize = random.nextInt( readerProvider.manipulators.size() - 1 ) + 1; + DirectoryProvider[] array = new DirectoryProvider[arraySize]; + List<DirectoryProvider> availableDPs = new ArrayList<DirectoryProvider>( readerProvider.manipulators.keySet() ); + for (int i=0; i<arraySize; i++){ + int chosenDpIndex = random.nextInt( availableDPs.size() ); + array[i] = availableDPs.get( chosenDpIndex ); + availableDPs.remove( array[i] ); + } + return array; + } + private class SearchTask implements Runnable { public void run() { try { @@ -60,7 +81,7 @@ //manage termination: return; } - MockIndexReader fakeOpenReader = readerProvider.fakeOpenReader(); + IndexReader fakeOpenReader = readerProvider.openReader( getRandomEvailableDPs() ); Thread.yield(); readerProvider.closeReader( fakeOpenReader ); countDoneSearches.incrementAndGet(); @@ -71,7 +92,11 @@ public void run() { super.run(); Thread.yield(); - readerProvider.setToDirtyState(); + DirectoryProvider[] randomEvailableDPs = getRandomEvailableDPs(); + for ( DirectoryProvider dp : randomEvailableDPs ) { + TestManipulatorPerDP testManipulatorPerDP = readerProvider.manipulators.get( dp ); + testManipulatorPerDP.setIndexChanged(); + } countDoneIndexmods.incrementAndGet(); } } Modified: search/trunk/src/test/org/hibernate/search/test/reader/functionality/TestableSharingBufferReaderProvider.java =================================================================== --- search/trunk/src/test/org/hibernate/search/test/reader/functionality/TestableSharingBufferReaderProvider.java 2008-06-16 15:08:45 UTC (rev 14769) +++ search/trunk/src/test/org/hibernate/search/test/reader/functionality/TestableSharingBufferReaderProvider.java 2008-06-16 15:43:48 UTC (rev 14770) @@ -2,69 +2,129 @@ import java.io.IOException; import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Properties; import java.util.Vector; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.lucene.document.Document; import org.apache.lucene.document.FieldSelector; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.MultiReader; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermDocs; import org.apache.lucene.index.TermEnum; import org.apache.lucene.index.TermFreqVector; import org.apache.lucene.index.TermPositions; import org.apache.lucene.index.TermVectorMapper; +import org.hibernate.search.SearchException; +import org.hibernate.search.engine.SearchFactoryImplementor; +import org.hibernate.search.reader.ReaderProviderHelper; import org.hibernate.search.reader.SharingBufferReaderProvider; import org.hibernate.search.store.DirectoryProvider; +import org.hibernate.search.store.RAMDirectoryProvider; /** * @author Sanne Grinovero */ public class TestableSharingBufferReaderProvider extends SharingBufferReaderProvider { - private final AtomicBoolean isIndexReaderCurrent = new AtomicBoolean( false );//starts at true, see MockIndexReader contructor - private final AtomicBoolean factoryCalled = new AtomicBoolean( false ); + private static final int NUM_DIRECTORY_PROVIDERS = 4; private final Vector<MockIndexReader> createdReadersHistory = new Vector<MockIndexReader>( 500 ); - private final MockIndexReader firstIndexReader = new MockIndexReader(); + final Map<DirectoryProvider,TestManipulatorPerDP> manipulators = new ConcurrentHashMap<DirectoryProvider,TestManipulatorPerDP>(); + public TestableSharingBufferReaderProvider() { + for (int i=0; i<NUM_DIRECTORY_PROVIDERS; i++) { + TestManipulatorPerDP tm = new TestManipulatorPerDP( i ); + manipulators.put( tm.dp, tm ); + } + } + + public static class TestManipulatorPerDP { + private final AtomicBoolean isIndexReaderCurrent = new AtomicBoolean( false );//starts at true, see MockIndexReader contructor + private final AtomicBoolean isReaderCreated = new AtomicBoolean( false ); + private final DirectoryProvider dp = new RAMDirectoryProvider(); + + public TestManipulatorPerDP( int seed ) { + dp.initialize( "dp" + seed, null, null ); + dp.start(); + } + + public void setIndexChanged() { + isIndexReaderCurrent.set( false ); + } + + } + + public boolean isReaderCurrent(MockIndexReader reader) { + //avoid usage of allReaders or test would be useless + for (PerDirectoryLatestReader latest : super.currentReaders.values() ) { + IndexReader latestReader = latest.current.reader; + if ( latestReader == reader) { + return true; + } + } + return false; + } + @Override - protected IndexReader readerFactory(int length, IndexReader[] readers) { - if ( factoryCalled.compareAndSet( false, true) ) { - return firstIndexReader; + protected IndexReader readerFactory(DirectoryProvider provider) { + TestManipulatorPerDP manipulatorPerDP = manipulators.get( provider ); + if ( ! manipulatorPerDP.isReaderCreated.compareAndSet( false, true ) ) { + throw new IllegalStateException( "IndexReader1 created twice" ); } else { - throw new IllegalStateException( "factory for reader called more than once" ); + return new MockIndexReader( manipulatorPerDP.isIndexReaderCurrent ); } } - public void setToDirtyState() { - isIndexReaderCurrent.set( false ); + @Override + public void initialize(Properties props, SearchFactoryImplementor searchFactoryImplementor) { + Map<DirectoryProvider,PerDirectoryLatestReader> map = new HashMap<DirectoryProvider,PerDirectoryLatestReader>(); + try { + for ( DirectoryProvider dp : manipulators.keySet() ) { + map.put( dp, new PerDirectoryLatestReader( dp ) ); + } + } catch (IOException e) { + throw new SearchException( "Unable to open Lucene IndexReader", e ); + } + currentReaders = Collections.unmodifiableMap( map ); } - public boolean isMapEmpty(){ - return super.oldReaders.isEmpty(); + public boolean areAllOldReferencesGone() { + int numReferencesReaders = super.allReaders.size(); + int numExpectedActiveReaders = manipulators.size(); + return numReferencesReaders == numExpectedActiveReaders; } public List<MockIndexReader> getCreatedIndexReaders(){ return createdReadersHistory; } - public MockIndexReader fakeOpenReader() { -// System.out.println( "tracking "+oldReaders.size() + " old readers." ); - return (MockIndexReader) super.openReader( new DirectoryProvider[0] ); + public MockIndexReader getCurrentMockReaderPerDP(DirectoryProvider dp) { + IndexReader[] indexReaders = ReaderProviderHelper.getSubReadersFromMultiReader( (MultiReader) super.openReader( new DirectoryProvider[]{ dp } ) ); + if ( indexReaders.length != 1 ){ + throw new IllegalStateException( "Expecting one reader" ); + } + return (MockIndexReader) indexReaders[0]; } public class MockIndexReader extends IndexReader { private final AtomicBoolean closed = new AtomicBoolean( false ); private final AtomicBoolean hasAlreadyBeenReOpened = new AtomicBoolean( false ); + private final AtomicBoolean isIndexReaderCurrent; - MockIndexReader(){ - createdReadersHistory.add( this ); + MockIndexReader(AtomicBoolean isIndexReaderCurrent) { + this.isIndexReaderCurrent = isIndexReaderCurrent; if ( ! isIndexReaderCurrent.compareAndSet(false, true) ) { throw new IllegalStateException( "Unnecessarily reopened" ); } + createdReadersHistory.add( this ); } public final boolean isClosed() { @@ -73,11 +133,11 @@ @Override protected void doClose() throws IOException { - boolean okToClose = closed.compareAndSet(false, true); + boolean okToClose = closed.compareAndSet( false, true ); if ( ! okToClose ) { throw new IllegalStateException( "Attempt to close a closed IndexReader" ); } - if ( ! hasAlreadyBeenReOpened.get() ){ + if ( ! hasAlreadyBeenReOpened.get() ) { throw new IllegalStateException( "Attempt to close the most current IndexReader" ); } } @@ -89,7 +149,7 @@ } else { if ( hasAlreadyBeenReOpened.compareAndSet( false, true) ) { - return new MockIndexReader(); + return new MockIndexReader( isIndexReaderCurrent ); } else throw new IllegalStateException( "Attempt to reopen an old IndexReader more than once" ); @@ -153,7 +213,7 @@ @Override public boolean hasDeletions() { - throw new UnsupportedOperationException(); + return false;//just something to make MultiReader constructor happy } @Override @@ -163,7 +223,7 @@ @Override public int maxDoc() { - throw new UnsupportedOperationException(); + return 10;//just something to make MultiReader constructor happy } @Override Modified: search/trunk/src/test/org/hibernate/search/test/reader/performance/ReaderPerformance.java =================================================================== --- search/trunk/src/test/org/hibernate/search/test/reader/performance/ReaderPerformance.java 2008-06-16 15:08:45 UTC (rev 14769) +++ search/trunk/src/test/org/hibernate/search/test/reader/performance/ReaderPerformance.java 2008-06-16 15:43:48 UTC (rev 14770) @@ -31,11 +31,11 @@ //more iterations for more reliable measures: private static final int TOTAL_WORK_BATCHES = 1000; //the next 3 define the kind of workload mix to test on: - private static final int SEARCHERS_PER_BATCH = 20; + private static final int SEARCHERS_PER_BATCH = 10; private static final int UPDATES_PER_BATCH = 2; private static final int INSERTIONS_PER_BATCH = 1; - private static final int WORKER_THREADS = 20; + private static final int WORKER_THREADS = 30; protected void setUp() throws Exception { baseIndexDir.mkdir(); @@ -78,7 +78,7 @@ // FileHelper.delete( baseIndexDir ); } - protected final void configure(org.hibernate.cfg.Configuration cfg) { + protected void configure(org.hibernate.cfg.Configuration cfg) { super.configure( cfg ); cfg.setProperty( "hibernate.search.default.directory_provider", FSDirectoryProvider.class.getName() ); cfg.setProperty( "hibernate.search.default.indexBase", baseIndexDir.getAbsolutePath() ); @@ -89,8 +89,8 @@ protected abstract String getReaderStrategyName(); - //this test is disabled as it is very slow (and you should read the resulting numbers) - public final void no_testPerformance() throws InterruptedException{ + //this test is disabled as it is very slow (and someone should read the output) + public final void disabled_testPerformance() throws InterruptedException{ ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool( WORKER_THREADS ); CountDownLatch startSignal = new CountDownLatch(1); InsertActivity insertionTask = new InsertActivity( getSessions(), startSignal );