Author: sannegrinovero Date: 2009-10-30 07:49:13 -0400 (Fri, 30 Oct 2009) New Revision: 17882 Added: search/trunk/src/main/java/org/hibernate/search/backend/impl/lucene/CloseIndexRunnable.java Modified: search/trunk/src/main/docbook/en-US/modules/batchindex.xml search/trunk/src/main/docbook/en-US/modules/configuration.xml search/trunk/src/main/java/org/hibernate/search/Environment.java search/trunk/src/main/java/org/hibernate/search/backend/impl/lucene/LuceneBackendQueueProcessorFactory.java search/trunk/src/main/java/org/hibernate/search/backend/impl/lucene/PerDPQueueProcessor.java search/trunk/src/main/java/org/hibernate/search/backend/impl/lucene/PerDPResources.java search/trunk/src/main/java/org/hibernate/search/backend/impl/lucene/QueueProcessors.java search/trunk/src/main/java/org/hibernate/search/backend/impl/lucene/works/DeleteExtWorkDelegate.java search/trunk/src/main/java/org/hibernate/search/engine/SearchFactoryImplementor.java search/trunk/src/main/java/org/hibernate/search/impl/SearchFactoryImpl.java search/trunk/src/main/java/org/hibernate/search/store/DirectoryProviderFactory.java search/trunk/src/test/java/org/hibernate/search/test/reader/performance/ReaderPerformance.java Log: HSEARCH-327 Capability to reuse IndexWriter instances across transactions Modified: search/trunk/src/main/docbook/en-US/modules/batchindex.xml =================================================================== --- search/trunk/src/main/docbook/en-US/modules/batchindex.xml 2009-10-30 10:52:10 UTC (rev 17881) +++ search/trunk/src/main/docbook/en-US/modules/batchindex.xml 2009-10-30 11:49:13 UTC (rev 17882) @@ -238,12 +238,12 @@ performance if the main entity is relating to enum-like data included in the index.</para> </example> - <tip>The "sweet spot" of number of threads to achieve best performance is + <tip><para>The "sweet spot" of number of threads to achieve best performance is highly dependent on your overall architecture, database design and even data values. To find out the best number of threads for your application it is recommended to use a profiler: all internal thread groups have meaningful names - to be easily identified with most tools. + to be easily identified with most tools.</para> </tip> <note> @@ -256,11 +256,15 @@ </section> - <para>Other parameters which also can affect indexing time and memory + <para>Other parameters which also affect indexing time and memory consumption are:</para> <itemizedlist> <listitem> + <literal>hibernate.search.[default|&lt;indexname&gt;].enable_greedy_locking</literal> + </listitem> + + <listitem> <literal>hibernate.search.[default|&lt;indexname&gt;].indexwriter.batch.max_buffered_docs</literal> </listitem> @@ -285,8 +289,8 @@ </listitem> </itemizedlist> - <para>These parameters are Lucene specific and Hibernate Search is just - passing these parameters through - see <xref + <para>All <literal>.indexwriter</literal> parameters are Lucene specific and + Hibernate Search is just passing these parameters through - see <xref linkend="lucene-indexing-performance" /> for more details.</para> </section> Modified: search/trunk/src/main/docbook/en-US/modules/configuration.xml =================================================================== --- search/trunk/src/main/docbook/en-US/modules/configuration.xml 2009-10-30 10:52:10 UTC (rev 17881) +++ search/trunk/src/main/docbook/en-US/modules/configuration.xml 2009-10-30 11:49:13 UTC (rev 17882) @@ -762,6 +762,18 @@ </thead> <tbody> + + <row> + <entry><literal>hibernate.search.[default|&lt;indexname&gt;].enable_greedy_locking</literal></entry> + + <entry><para>Set to <literal>true</literal> when no other + application uses the same index, this will enable + Hibernate Search to work in exlusive mode on the index and + improve performance in writing changes to the index.</para></entry> + + <entry><literal>false</literal> (releases locks as soon as possible)</entry> + </row> + <row> <entry><literal>hibernate.search.[default|&lt;indexname&gt;].indexwriter.[transaction|batch].max_buffered_delete_terms</literal></entry> @@ -878,6 +890,11 @@ </tbody> </tgroup> </table> + + <tip><para>When your architecture permits it, always set + <literal>hibernate.search.default.enable_greedy_locking=true</literal> + as it greatly improves efficiency in index writing.</para> + </tip> <para>To tune the indexing speed it might be useful to time the object loading from database in isolation from the writes to the index. To Modified: search/trunk/src/main/java/org/hibernate/search/Environment.java =================================================================== --- search/trunk/src/main/java/org/hibernate/search/Environment.java 2009-10-30 10:52:10 UTC (rev 17881) +++ search/trunk/src/main/java/org/hibernate/search/Environment.java 2009-10-30 11:49:13 UTC (rev 17882) @@ -102,4 +102,13 @@ */ public static final String BATCH_BACKEND = "hibernate.search.batchbackend"; + /** + * When set to true a lock on the index will not be released until the + * SearchFactory (or SessionFactory) is closed. + * This improves performance in applying changes to the index, but no other application + * can access the index in write mode while Hibernate Search is running. + * This is an index-scoped property and defaults to false. + */ + public static final String ENABLE_GREEDY_LOCKING = "enable_greedy_locking"; + } Added: search/trunk/src/main/java/org/hibernate/search/backend/impl/lucene/CloseIndexRunnable.java =================================================================== --- search/trunk/src/main/java/org/hibernate/search/backend/impl/lucene/CloseIndexRunnable.java (rev 0) +++ search/trunk/src/main/java/org/hibernate/search/backend/impl/lucene/CloseIndexRunnable.java 2009-10-30 11:49:13 UTC (rev 17882) @@ -0,0 +1,52 @@ +/* $Id$ + * + * Hibernate, Relational Persistence for Idiomatic Java + * + * Copyright (c) 2009, Red Hat, Inc. and/or its affiliates or third-party contributors as + * indicated by the @author tags or express copyright attribution + * statements applied by the authors. All third-party contributions are + * distributed under license by Red Hat, Inc. + * + * This copyrighted material is made available to anyone wishing to use, modify, + * copy, or redistribute it subject to the terms and conditions of the GNU + * Lesser General Public License, as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this distribution; if not, write to: + * Free Software Foundation, Inc. + * 51 Franklin Street, Fifth Floor + * Boston, MA 02110-1301 USA + */ +package org.hibernate.search.backend.impl.lucene; + +import org.slf4j.Logger; +import org.hibernate.search.backend.Workspace; +import org.hibernate.search.util.LoggerFactory; + +/** + * Runnable to close the IndexWriter of a Workspace, if open. + * Used when a close operation needs scheduling for after other Runnables in an executor queue. + * + * @author Sanne Grinovero + */ +public class CloseIndexRunnable implements Runnable { + + private static final Logger log = LoggerFactory.make(); + private final Workspace workspace; + + public CloseIndexRunnable(Workspace workspace) { + this.workspace = workspace; + } + + public void run() { + log.debug( "Closing IndexWriter if needed" ); + workspace.closeIndexWriter(); + } + +} + Property changes on: search/trunk/src/main/java/org/hibernate/search/backend/impl/lucene/CloseIndexRunnable.java ___________________________________________________________________ Name: svn:keywords + Id Name: svn:eol-style + LF Modified: search/trunk/src/main/java/org/hibernate/search/backend/impl/lucene/LuceneBackendQueueProcessorFactory.java =================================================================== --- search/trunk/src/main/java/org/hibernate/search/backend/impl/lucene/LuceneBackendQueueProcessorFactory.java 2009-10-30 10:52:10 UTC (rev 17881) +++ search/trunk/src/main/java/org/hibernate/search/backend/impl/lucene/LuceneBackendQueueProcessorFactory.java 2009-10-30 11:49:13 UTC (rev 17882) @@ -78,10 +78,9 @@ } public void close() { - // needs to stop all used ThreadPools + // needs to stop all used ThreadPools and cleanup locks for (PerDPResources res : resourcesMap.values() ) { - ExecutorService executor = res.getExecutor(); - executor.shutdown(); + res.shutdown(); } } Modified: search/trunk/src/main/java/org/hibernate/search/backend/impl/lucene/PerDPQueueProcessor.java =================================================================== --- search/trunk/src/main/java/org/hibernate/search/backend/impl/lucene/PerDPQueueProcessor.java 2009-10-30 10:52:10 UTC (rev 17881) +++ search/trunk/src/main/java/org/hibernate/search/backend/impl/lucene/PerDPQueueProcessor.java 2009-10-30 11:49:13 UTC (rev 17882) @@ -50,6 +50,7 @@ private final Workspace workspace; private final LuceneWorkVisitor worker; private final ExecutorService executor; + private final boolean useGreedyLocks; private final List<LuceneWork> workOnWriter = new ArrayList<LuceneWork>(); // if any work needs batchmode, set corresponding flag to true: @@ -63,6 +64,7 @@ this.worker = resources.getVisitor(); this.workspace = resources.getWorkspace(); this.executor = resources.getExecutor(); + this.useGreedyLocks = resources.isLockingGreedy(); } /** @@ -91,11 +93,11 @@ lw.getWorkDelegate( worker ).performWork( lw, indexWriter ); } workspace.commitIndexWriter(); - //TODO skip this when indexing in batches: performOptimizations(); } finally { - workspace.closeIndexWriter(); + if ( ! useGreedyLocks ) + workspace.closeIndexWriter(); } } Modified: search/trunk/src/main/java/org/hibernate/search/backend/impl/lucene/PerDPResources.java =================================================================== --- search/trunk/src/main/java/org/hibernate/search/backend/impl/lucene/PerDPResources.java 2009-10-30 10:52:10 UTC (rev 17881) +++ search/trunk/src/main/java/org/hibernate/search/backend/impl/lucene/PerDPResources.java 2009-10-30 11:49:13 UTC (rev 17882) @@ -43,11 +43,13 @@ private final ExecutorService executor; private final LuceneWorkVisitor visitor; private final Workspace workspace; + private final boolean lockingIsGreedy; - PerDPResources(SearchFactoryImplementor searchFactoryImp, DirectoryProvider dp) { + PerDPResources(SearchFactoryImplementor searchFactoryImp, DirectoryProvider<?> dp) { workspace = new Workspace( searchFactoryImp, dp ); visitor = new LuceneWorkVisitor( workspace ); executor = Executors.newFixedThreadPool( 1 ); + lockingIsGreedy = searchFactoryImp.isLockingGreedy( dp ); } public ExecutorService getExecutor() { @@ -61,5 +63,17 @@ public Workspace getWorkspace() { return workspace; } + + public boolean isLockingGreedy() { + return lockingIsGreedy; + } + + public void shutdown() { + //sets the index to be closed after all current jobs are processed: + if ( lockingIsGreedy ) { + executor.execute( new CloseIndexRunnable( workspace ) ); + } + executor.shutdown(); + } } Modified: search/trunk/src/main/java/org/hibernate/search/backend/impl/lucene/QueueProcessors.java =================================================================== --- search/trunk/src/main/java/org/hibernate/search/backend/impl/lucene/QueueProcessors.java 2009-10-30 10:52:10 UTC (rev 17881) +++ search/trunk/src/main/java/org/hibernate/search/backend/impl/lucene/QueueProcessors.java 2009-10-30 11:49:13 UTC (rev 17882) @@ -74,7 +74,7 @@ } /** - * Runs all PerDPQueueProcessor and don't wait fot them to finish. + * Runs all PerDPQueueProcessor and don't wait for them to finish. */ private void runAllAsync() { // execute all work in parallel on each DirectoryProvider; Modified: search/trunk/src/main/java/org/hibernate/search/backend/impl/lucene/works/DeleteExtWorkDelegate.java =================================================================== --- search/trunk/src/main/java/org/hibernate/search/backend/impl/lucene/works/DeleteExtWorkDelegate.java 2009-10-30 10:52:10 UTC (rev 17881) +++ search/trunk/src/main/java/org/hibernate/search/backend/impl/lucene/works/DeleteExtWorkDelegate.java 2009-10-30 11:49:13 UTC (rev 17882) @@ -53,9 +53,6 @@ DeleteExtWorkDelegate(Workspace workspace) { super( workspace ); - if ( workspace.getEntitiesInDirectory().size() != 1 ) { - throw new AssertionFailure( "Can't use this delegate on shared indexes" ); - } managedType = workspace.getEntitiesInDirectory().iterator().next(); builder = workspace.getDocumentBuilder( managedType ); } Modified: search/trunk/src/main/java/org/hibernate/search/engine/SearchFactoryImplementor.java =================================================================== --- search/trunk/src/main/java/org/hibernate/search/engine/SearchFactoryImplementor.java 2009-10-30 10:52:10 UTC (rev 17881) +++ search/trunk/src/main/java/org/hibernate/search/engine/SearchFactoryImplementor.java 2009-10-30 11:49:13 UTC (rev 17882) @@ -74,7 +74,7 @@ void close(); - void addClassToDirectoryProvider(Class<?> clazz, DirectoryProvider<?> directoryProvider); + void addClassToDirectoryProvider(Class<?> clazz, DirectoryProvider<?> directoryProvider, boolean useGreedyLocking); Set<Class<?>> getClassesInDirectoryProvider(DirectoryProvider<?> directoryProvider); @@ -82,7 +82,7 @@ ReentrantLock getDirectoryProviderLock(DirectoryProvider<?> dp); - void addDirectoryProvider(DirectoryProvider<?> provider); + void addDirectoryProvider(DirectoryProvider<?> provider, boolean usesGreedyLocking); int getFilterCacheBitResultsSize(); @@ -91,4 +91,6 @@ BatchBackend makeBatchBackend(MassIndexerProgressMonitor progressMonitor); Similarity getSimilarity(DirectoryProvider<?> directoryProvider); + + boolean isLockingGreedy(DirectoryProvider<?> provider); } Modified: search/trunk/src/main/java/org/hibernate/search/impl/SearchFactoryImpl.java =================================================================== --- search/trunk/src/main/java/org/hibernate/search/impl/SearchFactoryImpl.java 2009-10-30 10:52:10 UTC (rev 17881) +++ search/trunk/src/main/java/org/hibernate/search/impl/SearchFactoryImpl.java 2009-10-30 11:49:13 UTC (rev 17882) @@ -252,7 +252,7 @@ } } - public void addClassToDirectoryProvider(Class<?> clazz, DirectoryProvider<?> directoryProvider) { + public void addClassToDirectoryProvider(Class<?> clazz, DirectoryProvider<?> directoryProvider, boolean useGreedyLocking) { //no need to set a read barrier, we only use this class in the init thread DirectoryProviderData data = dirProviderData.get( directoryProvider ); if ( data == null ) { @@ -260,6 +260,7 @@ dirProviderData.put( directoryProvider, data ); } data.classes.add( clazz ); + data.usesGreedyLocking = useGreedyLocking; } public Set<Class<?>> getClassesInDirectoryProvider(DirectoryProvider<?> directoryProvider) { @@ -556,6 +557,7 @@ public OptimizerStrategy optimizerStrategy; public final Set<Class<?>> classes = new HashSet<Class<?>>( 2 ); public Similarity similarity = null; + private boolean usesGreedyLocking; } public ReentrantLock getDirectoryProviderLock(DirectoryProvider<?> dp) { @@ -564,9 +566,11 @@ return this.dirProviderData.get( dp ).dirLock; } - public void addDirectoryProvider(DirectoryProvider<?> provider) { + public void addDirectoryProvider(DirectoryProvider<?> provider, boolean usesGreedyLocking) { //no need to set a barrier we use this method in the init thread - this.dirProviderData.put( provider, new DirectoryProviderData() ); + DirectoryProviderData dirConfiguration = new DirectoryProviderData(); + dirConfiguration.usesGreedyLocking = usesGreedyLocking; + this.dirProviderData.put( provider, dirConfiguration ); } public int getFilterCacheBitResultsSize() { @@ -653,4 +657,10 @@ return similarity; } + public boolean isLockingGreedy(DirectoryProvider<?> provider) { + if ( barrier != 0 ) { + } //read barrier + return dirProviderData.get( provider ).usesGreedyLocking; + } + } Modified: search/trunk/src/main/java/org/hibernate/search/store/DirectoryProviderFactory.java =================================================================== --- search/trunk/src/main/java/org/hibernate/search/store/DirectoryProviderFactory.java 2009-10-30 10:52:10 UTC (rev 17881) +++ search/trunk/src/main/java/org/hibernate/search/store/DirectoryProviderFactory.java 2009-10-30 11:49:13 UTC (rev 17882) @@ -31,6 +31,7 @@ import org.hibernate.annotations.common.reflection.ReflectionManager; import org.hibernate.annotations.common.reflection.XClass; import org.hibernate.annotations.common.reflection.java.JavaReflectionManager; +import org.hibernate.search.Environment; import org.hibernate.search.SearchException; import org.hibernate.search.annotations.Indexed; import org.hibernate.search.backend.LuceneIndexingParameters; @@ -131,20 +132,18 @@ throw new SearchException( "Unable to initialize directory provider: " + directoryProviderName, e ); } int index = providers.indexOf( provider ); + boolean useGreedyLocking = isGreedyLockingEnabled( directoryProviderName, indexProps ); if ( index != -1 ) { //share the same Directory provider for the same underlying store final DirectoryProvider<?> directoryProvider = providers.get( index ); - searchFactoryImplementor.addClassToDirectoryProvider( entity, directoryProvider ); + searchFactoryImplementor.addClassToDirectoryProvider( entity, directoryProvider, useGreedyLocking); return directoryProvider; } else { configureOptimizerStrategy( searchFactoryImplementor, indexProps, provider ); configureIndexingParameters( searchFactoryImplementor, indexProps, provider ); providers.add( provider ); - searchFactoryImplementor.addClassToDirectoryProvider( entity, provider ); - if ( !searchFactoryImplementor.getDirectoryProviders().contains( provider ) ) { - searchFactoryImplementor.addDirectoryProvider( provider ); - } + searchFactoryImplementor.addClassToDirectoryProvider( entity, provider, useGreedyLocking ); return provider; } } @@ -265,5 +264,12 @@ return providers; } } + + private static boolean isGreedyLockingEnabled(String directoryProviderName, Properties indexProps) { + String usesGreedyLockingProperty = indexProps.getProperty( Environment.ENABLE_GREEDY_LOCKING, "false" ); + boolean usesGreedyLocking = ConfigurationParseHelper.parseBoolean( usesGreedyLockingProperty, + "Illegal value for property " + Environment.ENABLE_GREEDY_LOCKING + " on index " + directoryProviderName ); + return usesGreedyLocking; + } } Modified: search/trunk/src/test/java/org/hibernate/search/test/reader/performance/ReaderPerformance.java =================================================================== --- search/trunk/src/test/java/org/hibernate/search/test/reader/performance/ReaderPerformance.java 2009-10-30 10:52:10 UTC (rev 17881) +++ search/trunk/src/test/java/org/hibernate/search/test/reader/performance/ReaderPerformance.java 2009-10-30 11:49:13 UTC (rev 17882) @@ -71,10 +71,6 @@ super.setUp(); } - public void testFakeTest(){ - //to make JUnit happy when disabling performance test - } - private void buildBigIndex() throws InterruptedException, CorruptIndexException, LockObtainFailedException, IOException { System.out.println( "Going to create fake index..." ); FSDirectory directory = FSDirectory.getDirectory(new File(getBaseIndexDir(), Detective.class.getCanonicalName())); @@ -111,14 +107,14 @@ cfg.setProperty( "hibernate.search.default.directory_provider", FSDirectoryProvider.class.getName() ); cfg.setProperty( "hibernate.search.default.indexBase", getBaseIndexDir().getAbsolutePath() ); cfg.setProperty( "hibernate.search.default.optimizer.transaction_limit.max", "10" ); // workaround too many open files + cfg.setProperty( "hibernate.search.default." + Environment.ENABLE_GREEDY_LOCKING, "true" ); cfg.setProperty( Environment.ANALYZER_CLASS, StopAnalyzer.class.getName() ); cfg.setProperty( Environment.READER_STRATEGY, getReaderStrategyName() ); } protected abstract String getReaderStrategyName(); - //this test is disabled as it is very slow (and someone should read the output) - public final void disabled_testPerformance() throws InterruptedException, CorruptIndexException, LockObtainFailedException, IOException { + public final void testPerformance() throws InterruptedException, CorruptIndexException, LockObtainFailedException, IOException { buildBigIndex(); for (int i=0; i<WARMUP_CYCLES; i++) { timeMs();