[hibernate-commits] Hibernate SVN: r14765 - in search/trunk/src: test/org/hibernate/search/test/reader and 2 other directories.

Author: sannegrinovero Date: 2008-06-13 18:42:10 -0400 (Fri, 13 Jun 2008) New Revision: 14765 Added: search/trunk/src/java/org/hibernate/search/reader/SharingBufferReaderProvider.java search/trunk/src/test/org/hibernate/search/test/reader/functionality/ search/trunk/src/test/org/hibernate/search/test/reader/functionality/SharingBufferIndexProviderTest.java search/trunk/src/test/org/hibernate/search/test/reader/functionality/TestableSharingBufferReaderProvider.java search/trunk/src/test/org/hibernate/search/test/reader/performance/ search/trunk/src/test/org/hibernate/search/test/reader/performance/AbstractActivity.java search/trunk/src/test/org/hibernate/search/test/reader/performance/BufferSharingReaderPerfTest.java search/trunk/src/test/org/hibernate/search/test/reader/performance/IndexFillRunnable.java search/trunk/src/test/org/hibernate/search/test/reader/performance/InsertActivity.java search/trunk/src/test/org/hibernate/search/test/reader/performance/NotSharedReaderPerfTest.java search/trunk/src/test/org/hibernate/search/test/reader/performance/ReaderPerformance.java search/trunk/src/test/org/hibernate/search/test/reader/performance/SearchActivity.java search/trunk/src/test/org/hibernate/search/test/reader/performance/SharedReaderPerfTest.java search/trunk/src/test/org/hibernate/search/test/reader/performance/UpdateActivity.java Log: HSEARCH-212 : Added a new very efficient ReaderProvider Added: search/trunk/src/java/org/hibernate/search/reader/SharingBufferReaderProvider.java =================================================================== --- search/trunk/src/java/org/hibernate/search/reader/SharingBufferReaderProvider.java (rev 0) +++ search/trunk/src/java/org/hibernate/search/reader/SharingBufferReaderProvider.java 2008-06-13 22:42:10 UTC (rev 14765) @@ -0,0 +1,200 @@ +package org.hibernate.search.reader; + +import static org.hibernate.search.reader.ReaderProviderHelper.buildMultiReader; +import static org.hibernate.search.reader.ReaderProviderHelper.clean; + +import java.io.IOException; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +import org.apache.lucene.index.IndexReader; +import org.hibernate.search.SearchException; +import org.hibernate.search.engine.SearchFactoryImplementor; +import org.hibernate.search.store.DirectoryProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author Sanne Grinovero + */ +public class SharingBufferReaderProvider implements ReaderProvider { + + //contains last updated Reader; protected by lockOnOpenLatest. + private volatile ReaderUsagePair current; + + private final Lock lockOnOpenLatest = new ReentrantLock(); + + //contains all older Readers: + protected final Map<IndexReader,ReaderUsagePair> oldReaders = new ConcurrentHashMap<IndexReader,ReaderUsagePair>(); + + private final Logger log = LoggerFactory.getLogger ( SharingBufferReaderProvider.class ); + + public void closeReader(IndexReader reader) { + if ( reader == current.reader ) { + boolean closeit; + lockOnOpenLatest.lock(); + try { + if ( reader == current.reader ){ + current.usageCounter.getAndDecrement(); + closeit = false; + } + else { + closeit = true; + } + } + finally { + lockOnOpenLatest.unlock(); + } + if ( closeit ) { + closeOldReader( reader ); + } + } + else { + closeOldReader( reader ); + } + printState(); + } + + private void closeOldReader(IndexReader reader) { + try { + ReaderUsagePair pair = oldReaders.get( reader ); + boolean closed = pair.close(); //also testing "assert pair!=null"; + if ( closed ) { + //not longer needed, so remove references: + oldReaders.remove( reader ); + log.trace( "IndexReader closed." ); + } + else { + log.trace( "Closing of IndexReader skipped: still being used." ); + } + } + catch (IOException e) { + log.warn( "Unable to close Lucene IndexReader", e ); + //remove references anyway: + oldReaders.remove( reader ); + } + } + + public void initialize(Properties props, SearchFactoryImplementor searchFactoryImplementor) { + //FIXME initialize currentReaderContainer here instead of lazy + } + + public IndexReader openReader(DirectoryProvider... directoryProviders) { + boolean trace = log.isTraceEnabled(); + if ( trace ) log.trace( "Opening IndexReader for directoryProviders: {}", directoryProviders ); + IndexReader toReturn; + lockOnOpenLatest.lock(); + try { + if ( current == null ) { //FIXME move this case to initialize + current = initialReaderOpening( directoryProviders ); + log.trace( "IndexReader initialized." ); + } + else { + reopenIndexreader(); + } + toReturn = current.reader; //choose reader before unlock + } finally { + lockOnOpenLatest.unlock(); + } + printState(); + return toReturn; + } + + private void reopenIndexreader() { + // we own the lock + IndexReader before = current.reader; + IndexReader updatedReader; + try { + updatedReader = before.reopen(); + } catch (IOException e) { + throw new SearchException( "Unable to reopen IndexReader", e ); + } + if ( before == updatedReader ) { + current.incrementUseCounter(); + } + else { //store the old one for close() functionality. + int useCount = current.usageCounter.get(); + if ( useCount != 0 ) { + oldReaders.put( before, current ); + } + else { + //or close it if nobody uses. + try { + current.reader.close(); + } catch (IOException e) { + log.warn( "Unable to close Lucene IndexReader", e ); + } + } + current = new ReaderUsagePair( updatedReader ); + } + } + + public final void printState(){ + if ( log.isTraceEnabled()) + log.trace( "Current "+ current + " older:" + oldReaders.values() ); + } + + private ReaderUsagePair initialReaderOpening(DirectoryProvider[] directoryProviders) { + // we own the lock + final int length = directoryProviders.length; + IndexReader[] readers = new IndexReader[length]; + try { + for (int index = 0; index < length; index++) { + readers[index] = IndexReader.open( directoryProviders[index].getDirectory() ); + } + } + catch (IOException e) { + //TODO more contextual info + clean( new SearchException( "Unable to open one of the Lucene indexes", e ), readers ); + } + IndexReader iR = readerFactory( length, readers ); + return new ReaderUsagePair( iR ); + } + + //overridable method for testability: + protected IndexReader readerFactory(int length, IndexReader[] readers) { + return buildMultiReader( length, readers ); + } + + protected static class ReaderUsagePair { + protected final IndexReader reader; + protected final AtomicInteger usageCounter; + + ReaderUsagePair(IndexReader r) { + reader = r; + usageCounter = new AtomicInteger( 1 ); + } + + void incrementUseCounter() { + usageCounter.incrementAndGet(); + } + + public int getUsageCount(){ + return usageCounter.get(); + } + + /** + * @return true when really closing the underlying IndexReader + * @throws IOException + */ + private boolean close() throws IOException { + int count = usageCounter.decrementAndGet(); + if ( count == 0 ) { + reader.close(); + return true; + } + assert count >= 0; + return false; + } + + public String toString(){ + return "Reader:"+this.hashCode()+" count="+usageCounter.get(); + } + + } + +} Added: search/trunk/src/test/org/hibernate/search/test/reader/functionality/SharingBufferIndexProviderTest.java =================================================================== --- search/trunk/src/test/org/hibernate/search/test/reader/functionality/SharingBufferIndexProviderTest.java (rev 0) +++ search/trunk/src/test/org/hibernate/search/test/reader/functionality/SharingBufferIndexProviderTest.java 2008-06-13 22:42:10 UTC (rev 14765) @@ -0,0 +1,79 @@ +package org.hibernate.search.test.reader.functionality; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import org.hibernate.search.test.reader.functionality.TestableSharingBufferReaderProvider.MockIndexReader; + +import junit.framework.TestCase; + +/** + * @author Sanne Grinovero + */ +public class SharingBufferIndexProviderTest extends TestCase { + + private final TestableSharingBufferReaderProvider readerProvider = new TestableSharingBufferReaderProvider(); + private final CountDownLatch startSignal = new CountDownLatch(1); + private final Runnable searchTask = new SearchTask(); + private final Runnable changeTask = new ChangeTask(); + private final AtomicInteger countDoneSearches = new AtomicInteger(); + private final AtomicInteger countDoneIndexmods = new AtomicInteger(); + private static final int SEARCHES_NUM = 5000; + + public void testStressingMock() throws InterruptedException { + ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool( 200 );//much chaos + for ( int i = 0; i < SEARCHES_NUM; i++ ) { + executor.execute( makeTask( i ) ); + } + executor.shutdown(); + startSignal.countDown(); + executor.awaitTermination( 15, TimeUnit.SECONDS ); + assertTrue( "memory leak: holding a reference to some unused IndexReader", readerProvider.isMapEmpty() ); + MockIndexReader openReader = readerProvider.fakeOpenReader(); + for ( MockIndexReader reader : readerProvider.getCreatedIndexReaders() ) { + if ( reader != openReader ) { + assertTrue( "an IndexReader is still open", reader.isClosed() ); + } + } + assertTrue( "the most current reader should be open", ! openReader.isClosed() ); + assertEquals( SEARCHES_NUM, countDoneSearches.get() ); + assertEquals( SEARCHES_NUM/10, countDoneIndexmods.get() ); + } + + private Runnable makeTask(int i) { + if ( i % 10 == 0) { + return changeTask; + } + else { + return searchTask; + } + } + + private class SearchTask implements Runnable { + public void run() { + try { + startSignal.await(); + } catch (InterruptedException e) { + //manage termination: + return; + } + MockIndexReader fakeOpenReader = readerProvider.fakeOpenReader(); + Thread.yield(); + readerProvider.closeReader( fakeOpenReader ); + countDoneSearches.incrementAndGet(); + } + } + + private class ChangeTask extends SearchTask { + public void run() { + super.run(); + Thread.yield(); + readerProvider.setToDirtyState(); + countDoneIndexmods.incrementAndGet(); + } + } + +} Added: search/trunk/src/test/org/hibernate/search/test/reader/functionality/TestableSharingBufferReaderProvider.java =================================================================== --- search/trunk/src/test/org/hibernate/search/test/reader/functionality/TestableSharingBufferReaderProvider.java (rev 0) +++ search/trunk/src/test/org/hibernate/search/test/reader/functionality/TestableSharingBufferReaderProvider.java 2008-06-13 22:42:10 UTC (rev 14765) @@ -0,0 +1,206 @@ +package org.hibernate.search.test.reader.functionality; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Vector; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.FieldSelector; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermDocs; +import org.apache.lucene.index.TermEnum; +import org.apache.lucene.index.TermFreqVector; +import org.apache.lucene.index.TermPositions; +import org.apache.lucene.index.TermVectorMapper; +import org.hibernate.search.reader.SharingBufferReaderProvider; +import org.hibernate.search.store.DirectoryProvider; + +/** + * @author Sanne Grinovero + */ +public class TestableSharingBufferReaderProvider extends SharingBufferReaderProvider { + + private final AtomicBoolean isIndexReaderCurrent = new AtomicBoolean( false );//starts at true, see MockIndexReader contructor + private final AtomicBoolean factoryCalled = new AtomicBoolean( false ); + private final Vector<MockIndexReader> createdReadersHistory = new Vector<MockIndexReader>( 500 ); + private final MockIndexReader firstIndexReader = new MockIndexReader(); + + @Override + protected IndexReader readerFactory(int length, IndexReader[] readers) { + if ( factoryCalled.compareAndSet( false, true) ) { + return firstIndexReader; + } + else { + throw new IllegalStateException( "factory for reader called more than once" ); + } + } + + public void setToDirtyState() { + isIndexReaderCurrent.set( false ); + } + + public boolean isMapEmpty(){ + return super.oldReaders.isEmpty(); + } + + public List<MockIndexReader> getCreatedIndexReaders(){ + return createdReadersHistory; + } + + public MockIndexReader fakeOpenReader() { +// System.out.println( "tracking "+oldReaders.size() + " old readers." ); + return (MockIndexReader) super.openReader( new DirectoryProvider[0] ); + } + + public class MockIndexReader extends IndexReader { + + private final AtomicBoolean closed = new AtomicBoolean( false ); + private final AtomicBoolean hasAlreadyBeenReOpened = new AtomicBoolean( false ); + + MockIndexReader(){ + createdReadersHistory.add( this ); + if ( ! isIndexReaderCurrent.compareAndSet(false, true) ) { + throw new IllegalStateException( "Unnecessarily reopened" ); + } + } + + public final boolean isClosed() { + return closed.get(); + } + + @Override + protected void doClose() throws IOException { + boolean okToClose = closed.compareAndSet(false, true); + if ( ! okToClose ) { + throw new IllegalStateException( "Attempt to close a closed IndexReader" ); + } + if ( ! hasAlreadyBeenReOpened.get() ){ + throw new IllegalStateException( "Attempt to close the most current IndexReader" ); + } + } + + @Override + public synchronized IndexReader reopen(){ + if ( isIndexReaderCurrent.get() ) { + return this; + } + else { + if ( hasAlreadyBeenReOpened.compareAndSet( false, true) ) { + return new MockIndexReader(); + } + else + throw new IllegalStateException( "Attempt to reopen an old IndexReader more than once" ); + } + } + + @Override + protected void doCommit() { + throw new UnsupportedOperationException(); + } + + @Override + protected void doDelete(int docNum) { + throw new UnsupportedOperationException(); + } + + @Override + protected void doSetNorm(int doc, String field, byte value) { + throw new UnsupportedOperationException(); + } + + @Override + protected void doUndeleteAll() { + throw new UnsupportedOperationException(); + } + + @Override + public int docFreq(Term t) { + throw new UnsupportedOperationException(); + } + + @Override + public Document document(int n, FieldSelector fieldSelector) { + throw new UnsupportedOperationException(); + } + + @Override + public Collection getFieldNames(FieldOption fldOption) { + throw new UnsupportedOperationException(); + } + + @Override + public TermFreqVector getTermFreqVector(int docNumber, String field) { + throw new UnsupportedOperationException(); + } + + @Override + public void getTermFreqVector(int docNumber, String field, TermVectorMapper mapper) { + throw new UnsupportedOperationException(); + } + + @Override + public void getTermFreqVector(int docNumber, TermVectorMapper mapper) { + throw new UnsupportedOperationException(); + } + + @Override + public TermFreqVector[] getTermFreqVectors(int docNumber) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean hasDeletions() { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isDeleted(int n) { + throw new UnsupportedOperationException(); + } + + @Override + public int maxDoc() { + throw new UnsupportedOperationException(); + } + + @Override + public byte[] norms(String field) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public void norms(String field, byte[] bytes, int offset) { + throw new UnsupportedOperationException(); + } + + @Override + public int numDocs() { + throw new UnsupportedOperationException(); + } + + @Override + public TermDocs termDocs() { + throw new UnsupportedOperationException(); + } + + @Override + public TermPositions termPositions() { + throw new UnsupportedOperationException(); + } + + @Override + public TermEnum terms() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public TermEnum terms(Term t) throws IOException { + throw new UnsupportedOperationException(); + } + + } + +} Added: search/trunk/src/test/org/hibernate/search/test/reader/performance/AbstractActivity.java =================================================================== --- search/trunk/src/test/org/hibernate/search/test/reader/performance/AbstractActivity.java (rev 0) +++ search/trunk/src/test/org/hibernate/search/test/reader/performance/AbstractActivity.java 2008-06-13 22:42:10 UTC (rev 14765) @@ -0,0 +1,80 @@ +package org.hibernate.search.test.reader.performance; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.queryParser.MultiFieldQueryParser; +import org.apache.lucene.queryParser.ParseException; +import org.apache.lucene.queryParser.QueryParser; +import org.apache.lucene.search.Query; +import org.hibernate.Session; +import org.hibernate.SessionFactory; +import org.hibernate.Transaction; +import org.hibernate.search.FullTextQuery; +import org.hibernate.search.FullTextSession; +import org.hibernate.search.Search; + +/** + * @author Sanne Grinovero + */ +public abstract class AbstractActivity implements Runnable { + + private final ThreadLocal<QueryParser> parsers = new ThreadLocal<QueryParser>(){ + @Override + protected QueryParser initialValue(){ + return new MultiFieldQueryParser( + new String[] {"name", "physicalDescription", "suspectCharge"}, + new StandardAnalyzer() ); + } + }; + + private final SessionFactory sf; + private final AtomicInteger jobSeed = new AtomicInteger(); + private final CountDownLatch startSignal; + + AbstractActivity(SessionFactory sf, CountDownLatch startSignal) { + this.startSignal = startSignal; + this.sf = sf; + } + + public final void run() { + try { + startSignal.await(); + } catch (InterruptedException e) { + e.printStackTrace(); + return; + } + Session s = sf.openSession(); + try { + FullTextSession fts = Search.createFullTextSession( s ); + Transaction tx = s.beginTransaction(); + boolean ok = false; + try { + doAction( fts, jobSeed.getAndIncrement() ); + ok = true; + } finally { + if (ok) + tx.commit(); + else + tx.rollback(); + } + } finally { + s.close(); + } + } + + protected FullTextQuery getQuery(String queryString, FullTextSession s, Class... classes) { + Query luceneQuery = null; + try { + luceneQuery = parsers.get().parse(queryString); + } + catch (ParseException e) { + e.printStackTrace(); + } + return s.createFullTextQuery( luceneQuery, classes ); + } + + protected abstract void doAction(FullTextSession s, int jobSeed); + +} Added: search/trunk/src/test/org/hibernate/search/test/reader/performance/BufferSharingReaderPerfTest.java =================================================================== --- search/trunk/src/test/org/hibernate/search/test/reader/performance/BufferSharingReaderPerfTest.java (rev 0) +++ search/trunk/src/test/org/hibernate/search/test/reader/performance/BufferSharingReaderPerfTest.java 2008-06-13 22:42:10 UTC (rev 14765) @@ -0,0 +1,15 @@ +package org.hibernate.search.test.reader.performance; + +import org.hibernate.search.reader.SharingBufferReaderProvider; + +/** + * @author Sanne Grinovero + */ +public class BufferSharingReaderPerfTest extends ReaderPerformance { + + @Override + protected String getReaderStrategyName() { + return SharingBufferReaderProvider.class.getName(); + } + +} Added: search/trunk/src/test/org/hibernate/search/test/reader/performance/IndexFillRunnable.java =================================================================== --- search/trunk/src/test/org/hibernate/search/test/reader/performance/IndexFillRunnable.java (rev 0) +++ search/trunk/src/test/org/hibernate/search/test/reader/performance/IndexFillRunnable.java 2008-06-13 22:42:10 UTC (rev 14765) @@ -0,0 +1,37 @@ +package org.hibernate.search.test.reader.performance; + +import java.io.IOException; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.Field.Index; +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.index.IndexWriter; + +/** + * @author Sanne Grinovero + */ +public class IndexFillRunnable implements Runnable { + + private volatile int jobSeed = 0; + private final IndexWriter iw; + + public IndexFillRunnable(IndexWriter iw) { + super(); + this.iw = iw; + } + + public void run() { + Field f1 = new Field("name", "Some One " + jobSeed++, Store.NO, Index.TOKENIZED ); + Field f2 = new Field("physicalDescription", " just more people sitting around and filling my index... ", Store.NO, Index.TOKENIZED ); + Document d = new Document(); + d.add( f1 ); + d.add( f2 ); + try { + iw.addDocument( d ); + } catch (IOException e) { + e.printStackTrace(); + } + } + +} Added: search/trunk/src/test/org/hibernate/search/test/reader/performance/InsertActivity.java =================================================================== --- search/trunk/src/test/org/hibernate/search/test/reader/performance/InsertActivity.java (rev 0) +++ search/trunk/src/test/org/hibernate/search/test/reader/performance/InsertActivity.java 2008-06-13 22:42:10 UTC (rev 14765) @@ -0,0 +1,38 @@ +package org.hibernate.search.test.reader.performance; + +import java.util.concurrent.CountDownLatch; + +import org.hibernate.SessionFactory; +import org.hibernate.search.FullTextSession; +import org.hibernate.search.test.reader.Detective; +import org.hibernate.search.test.reader.Suspect; + +/** + * @author Emmanuel Bernard + * @author Sanne Grinovero + */ +public class InsertActivity extends AbstractActivity { + + InsertActivity(SessionFactory sf, CountDownLatch startSignal) { + super(sf, startSignal); + } + + @Override + protected void doAction(FullTextSession s, int jobSeed) { + Detective detective = new Detective(); + detective.setName("John Doe " + jobSeed); + detective.setBadge("123455" + jobSeed); + detective.setPhysicalDescription("Blond green eye etc etc"); + s.persist(detective); + Suspect suspect = new Suspect(); + suspect.setName("Jane Doe " + jobSeed); + suspect.setPhysicalDescription("brunette, short, 30-ish"); + if (jobSeed % 20 == 0) { + suspect.setSuspectCharge("thief liar "); + } else { + suspect.setSuspectCharge(" It's 1875 in London. The police have captured career criminal Montmorency. In the process he has been grievously wounded and it is up to a young surgeon to treat his wounds. During his recovery Montmorency learns of the city's new sewer system and sees in it the perfect underground highway for his thievery. Washington Post columnist John Kelly recommends this title for middle schoolers, especially to be read aloud."); + } + s.persist(suspect); + } + +} Added: search/trunk/src/test/org/hibernate/search/test/reader/performance/NotSharedReaderPerfTest.java =================================================================== --- search/trunk/src/test/org/hibernate/search/test/reader/performance/NotSharedReaderPerfTest.java (rev 0) +++ search/trunk/src/test/org/hibernate/search/test/reader/performance/NotSharedReaderPerfTest.java 2008-06-13 22:42:10 UTC (rev 14765) @@ -0,0 +1,13 @@ +package org.hibernate.search.test.reader.performance; + +/** + * @author Sanne Grinovero + */ +public class NotSharedReaderPerfTest extends ReaderPerformance { + + @Override + protected String getReaderStrategyName() { + return "not-shared"; + } + +} Added: search/trunk/src/test/org/hibernate/search/test/reader/performance/ReaderPerformance.java =================================================================== --- search/trunk/src/test/org/hibernate/search/test/reader/performance/ReaderPerformance.java (rev 0) +++ search/trunk/src/test/org/hibernate/search/test/reader/performance/ReaderPerformance.java 2008-06-13 22:42:10 UTC (rev 14765) @@ -0,0 +1,119 @@ +package org.hibernate.search.test.reader.performance; + +import java.io.File; +import java.io.IOException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import org.apache.lucene.analysis.SimpleAnalyzer; +import org.apache.lucene.analysis.StopAnalyzer; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.LockObtainFailedException; +import org.hibernate.search.Environment; +import org.hibernate.search.store.FSDirectoryProvider; +import org.hibernate.search.test.SearchTestCase; +import org.hibernate.search.test.reader.Detective; +import org.hibernate.search.test.reader.Suspect; +import org.hibernate.search.util.FileHelper; + +/** + * To enable performance tests: de-comment buildBigIndex(); in setUp() and rename no_testPerformance + * @author Sanne Grinovero + */ +public abstract class ReaderPerformance extends SearchTestCase { + + private static final File baseIndexDir = new File( new File( "." ), "indextemp" ); + + //more iterations for more reliable measures: + private static final int TOTAL_WORK_BATCHES = 1000; + //the next 3 define the kind of workload mix to test on: + private static final int SEARCHERS_PER_BATCH = 20; + private static final int UPDATES_PER_BATCH = 2; + private static final int INSERTIONS_PER_BATCH = 1; + + private static final int WORKER_THREADS = 20; + + protected void setUp() throws Exception { + baseIndexDir.mkdir(); + File[] files = baseIndexDir.listFiles(); + for ( File file : files ) { + FileHelper.delete( file ); + } + super.setUp(); + //enable this line: +// buildBigIndex(); + } + + public void testFakeTest(){ + //to make JUnit happy when disabling performance test + } + + private void buildBigIndex() throws InterruptedException, CorruptIndexException, LockObtainFailedException, IOException { + FSDirectory directory = FSDirectory.getDirectory(new File(baseIndexDir, Detective.class.getCanonicalName())); + IndexWriter iw = new IndexWriter( directory, new SimpleAnalyzer(), true ); + IndexFillRunnable filler = new IndexFillRunnable( iw ); + ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool( WORKER_THREADS ); + for (int batch=0; batch<=100000; batch++){ + executor.execute( filler ); + } + executor.shutdown(); + executor.awaitTermination( 600, TimeUnit.SECONDS ); + iw.close(); + System.out.println( "Index created." ); + } + + protected Class[] getMappings() { + return new Class[] { + Detective.class, + Suspect.class + }; + } + + protected void tearDown() throws Exception { + super.tearDown(); +// FileHelper.delete( baseIndexDir ); + } + + protected final void configure(org.hibernate.cfg.Configuration cfg) { + super.configure( cfg ); + cfg.setProperty( "hibernate.search.default.directory_provider", FSDirectoryProvider.class.getName() ); + cfg.setProperty( "hibernate.search.default.indexBase", baseIndexDir.getAbsolutePath() ); + cfg.setProperty( "hibernate.search.default.optimizer.transaction_limit.max", "10" ); // workaround too many open files + cfg.setProperty( Environment.ANALYZER_CLASS, StopAnalyzer.class.getName() ); + cfg.setProperty( Environment.READER_STRATEGY, getReaderStrategyName() ); + } + + protected abstract String getReaderStrategyName(); + + //this test is disabled as it is very slow (and you should read the resulting numbers) + public final void no_testPerformance() throws InterruptedException{ + ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool( WORKER_THREADS ); + CountDownLatch startSignal = new CountDownLatch(1); + InsertActivity insertionTask = new InsertActivity( getSessions(), startSignal ); + SearchActivity searchTask = new SearchActivity( getSessions(), startSignal ); + UpdateActivity updateTask = new UpdateActivity( getSessions(), startSignal ); + //we declare needed activities in order, scheduler will "mix": + for (int batch=0; batch<=TOTAL_WORK_BATCHES; batch++){ + for ( int inserters=0; inserters<INSERTIONS_PER_BATCH; inserters++) + executor.execute( insertionTask ); + for ( int searchers=0; searchers<SEARCHERS_PER_BATCH; searchers++) + executor.execute( searchTask ); + for ( int updaters=0; updaters<UPDATES_PER_BATCH; updaters++) + executor.execute( updateTask ); + } + executor.shutdown(); + long startTime = System.currentTimeMillis(); + startSignal.countDown();//start! + executor.awaitTermination( 600, TimeUnit.SECONDS ); + long endTime = System.currentTimeMillis(); + System.out.println( "Performance test for " + getReaderStrategyName() + ": " + (endTime - startTime) +"ms. (" + + (TOTAL_WORK_BATCHES*SEARCHERS_PER_BATCH) + " searches, " + + (TOTAL_WORK_BATCHES*INSERTIONS_PER_BATCH) + " insertions, " + + (TOTAL_WORK_BATCHES*UPDATES_PER_BATCH) + " updates)" ); + } + +} Added: search/trunk/src/test/org/hibernate/search/test/reader/performance/SearchActivity.java =================================================================== --- search/trunk/src/test/org/hibernate/search/test/reader/performance/SearchActivity.java (rev 0) +++ search/trunk/src/test/org/hibernate/search/test/reader/performance/SearchActivity.java 2008-06-13 22:42:10 UTC (rev 14765) @@ -0,0 +1,27 @@ +package org.hibernate.search.test.reader.performance; + +import java.util.concurrent.CountDownLatch; + +import org.hibernate.SessionFactory; +import org.hibernate.search.FullTextQuery; +import org.hibernate.search.FullTextSession; +import org.hibernate.search.test.reader.Detective; + +/** + * @author Emmanuel Bernard + * @author Sanne Grinovero + */ +public class SearchActivity extends AbstractActivity { + + SearchActivity(SessionFactory sf, CountDownLatch startSignal) { + super(sf, startSignal); + } + + @Override + protected void doAction(FullTextSession s, int jobSeed) { + FullTextQuery q = getQuery( "John Doe", s, Detective.class); + q.setMaxResults( 10 ); + q.getResultSize(); + } + +} Added: search/trunk/src/test/org/hibernate/search/test/reader/performance/SharedReaderPerfTest.java =================================================================== --- search/trunk/src/test/org/hibernate/search/test/reader/performance/SharedReaderPerfTest.java (rev 0) +++ search/trunk/src/test/org/hibernate/search/test/reader/performance/SharedReaderPerfTest.java 2008-06-13 22:42:10 UTC (rev 14765) @@ -0,0 +1,13 @@ +package org.hibernate.search.test.reader.performance; + +/** + * @author Sanne Grinovero + */ +public class SharedReaderPerfTest extends ReaderPerformance { + + @Override + protected String getReaderStrategyName() { + return "shared"; + } + +} Added: search/trunk/src/test/org/hibernate/search/test/reader/performance/UpdateActivity.java =================================================================== --- search/trunk/src/test/org/hibernate/search/test/reader/performance/UpdateActivity.java (rev 0) +++ search/trunk/src/test/org/hibernate/search/test/reader/performance/UpdateActivity.java 2008-06-13 22:42:10 UTC (rev 14765) @@ -0,0 +1,30 @@ +package org.hibernate.search.test.reader.performance; + +import java.util.List; +import java.util.concurrent.CountDownLatch; + +import org.hibernate.SessionFactory; +import org.hibernate.search.FullTextQuery; +import org.hibernate.search.FullTextSession; +import org.hibernate.search.test.reader.Detective; + +/** + * @author Sanne Grinovero + */ +public class UpdateActivity extends AbstractActivity { + + UpdateActivity(SessionFactory sf, CountDownLatch startSignal) { + super(sf, startSignal); + } + + @Override + protected void doAction(FullTextSession s, int jobSeed) { + FullTextQuery q = getQuery( "John", s, Detective.class ); + List list = q.setMaxResults( 1 ).list(); + for ( Object o : list){ + Detective detective = (Detective) o; + detective.setPhysicalDescription( "old" ); + } + } + +}