[hibernate-commits] Hibernate SVN: r12845 - in trunk/HibernateExt/search/src/java/org/hibernate/search: backend/impl/lucene and 3 other directories.

Author: epbernard Date: 2007-07-29 11:10:49 -0400 (Sun, 29 Jul 2007) New Revision: 12845 Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/backend/Workspace.java trunk/HibernateExt/search/src/java/org/hibernate/search/backend/impl/lucene/LuceneBackendQueueProcessor.java trunk/HibernateExt/search/src/java/org/hibernate/search/backend/impl/lucene/LuceneWorker.java trunk/HibernateExt/search/src/java/org/hibernate/search/engine/SearchFactoryImplementor.java trunk/HibernateExt/search/src/java/org/hibernate/search/query/FullTextQueryImpl.java trunk/HibernateExt/search/src/java/org/hibernate/search/store/DirectoryProviderFactory.java trunk/HibernateExt/search/src/java/org/hibernate/search/store/DirectoryProviderShardingStrategy.java trunk/HibernateExt/search/src/java/org/hibernate/search/store/IdHashShardingStrategy.java trunk/HibernateExt/search/src/java/org/hibernate/search/store/NotShardedStrategy.java Log: HSEARCH-86 introduce Index Sharding, including sharding strategy Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/backend/Workspace.java =================================================================== --- trunk/HibernateExt/search/src/java/org/hibernate/search/backend/Workspace.java 2007-07-29 14:56:47 UTC (rev 12844) +++ trunk/HibernateExt/search/src/java/org/hibernate/search/backend/Workspace.java 2007-07-29 15:10:49 UTC (rev 12845) @@ -2,43 +2,39 @@ package org.hibernate.search.backend; import java.io.IOException; -import java.io.Serializable; -import java.util.HashMap; -import java.util.Map; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; -import java.util.Set; -import java.util.HashSet; +import java.util.Map; import java.util.concurrent.locks.ReentrantLock; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriter; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.SimpleAnalyzer; -import org.apache.lucene.document.Document; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.hibernate.annotations.common.AssertionFailure; +import org.hibernate.search.SearchException; import org.hibernate.search.engine.DocumentBuilder; import org.hibernate.search.engine.SearchFactoryImplementor; -import org.hibernate.search.SearchException; import org.hibernate.search.store.DirectoryProvider; import org.hibernate.search.store.optimization.OptimizerStrategy; -import org.hibernate.annotations.common.AssertionFailure; /** * Lucene workspace. - * <p> + * <p/> * <b>This is not intended to be used in a multithreaded environment</b>. * <p/> * <ul> - * <li>One cannot execute modification through an IndexReader when an IndexWriter has been acquired - * on the same underlying directory + * <li>One cannot execute modification through an IndexReader when an IndexWriter has been acquired + * on the same underlying directory * </li> - * <li>One cannot get an IndexWriter when an IndexReader have been acquired and modificed on the same - * underlying directory + * <li>One cannot get an IndexWriter when an IndexReader have been acquired and modificed on the same + * underlying directory * </li> * <li>The recommended approach is to execute all the modifications on the IndexReaders, {@link #clean()}, and acquire the - * index writers + * index writers * </li> * </ul> * @@ -53,13 +49,13 @@ private List<DirectoryProvider> lockedProviders = new ArrayList<DirectoryProvider>(); private Map<DirectoryProvider, DPStatistics> dpStatistics = new HashMap<DirectoryProvider, DPStatistics>(); private SearchFactoryImplementor searchFactoryImplementor; - + /** * Flag indicating if the current work should be executed the Lucene parameters for batch indexing. */ private boolean isBatch = false; - + public Workspace(SearchFactoryImplementor searchFactoryImplementor) { this.searchFactoryImplementor = searchFactoryImplementor; } @@ -75,11 +71,11 @@ public IndexReader getIndexReader(DirectoryProvider provider, Class entity) { //one cannot access a reader for update after a writer has been accessed if ( writers.containsKey( provider ) ) - throw new AssertionFailure("Tries to read for update an index while a writer is accessed" + entity); + throw new AssertionFailure( "Tries to read for update an index while a writer is accessed" + entity ); IndexReader reader = readers.get( provider ); if ( reader != null ) return reader; lockProvider( provider ); - dpStatistics.get(provider).operations++; + dpStatistics.get( provider ).operations++; try { reader = IndexReader.open( provider.getDirectory() ); readers.put( provider, reader ); @@ -114,32 +110,30 @@ IndexWriter writer = writers.get( provider ); if ( writer != null ) return writer; lockProvider( provider ); - if (modificationOperation) dpStatistics.get(provider).operations++; + if ( modificationOperation ) dpStatistics.get( provider ).operations++; try { Analyzer analyzer = entity != null ? searchFactoryImplementor.getDocumentBuilders().get( entity ).getAnalyzer() : new SimpleAnalyzer(); //never used writer = new IndexWriter( provider.getDirectory(), analyzer, false ); //has been created at init time - LuceneIndexingParameters indexingParams = searchFactoryImplementor.getIndexingParameters(provider); - if(isBatch) - { - writer.setMergeFactor(indexingParams.getBatchMergeFactor()); - writer.setMaxMergeDocs(indexingParams.getBatchMaxMergeDocs()); - writer.setMaxBufferedDocs(indexingParams.getBatchMaxBufferedDocs()); - } - else - { - writer.setMergeFactor(indexingParams.getMergeFactor()); - writer.setMaxMergeDocs(indexingParams.getMaxMergeDocs()); - writer.setMaxBufferedDocs(indexingParams.getMaxBufferedDocs()); + LuceneIndexingParameters indexingParams = searchFactoryImplementor.getIndexingParameters( provider ); + if ( isBatch ) { + writer.setMergeFactor( indexingParams.getBatchMergeFactor() ); + writer.setMaxMergeDocs( indexingParams.getBatchMaxMergeDocs() ); + writer.setMaxBufferedDocs( indexingParams.getBatchMaxBufferedDocs() ); } - + else { + writer.setMergeFactor( indexingParams.getMergeFactor() ); + writer.setMaxMergeDocs( indexingParams.getMaxMergeDocs() ); + writer.setMaxBufferedDocs( indexingParams.getMaxBufferedDocs() ); + } + writers.put( provider, writer ); } catch (IOException e) { cleanUp( - new SearchException( "Unable to open IndexWriter" + ( entity != null ? " for " + entity : "" ), e ) + new SearchException( "Unable to open IndexWriter" + ( entity != null ? " for " + entity : "" ), e ) ); } return writer; @@ -159,7 +153,7 @@ private void cleanUp(SearchException originalException) { //release all readers and writers, then release locks SearchException raisedException = originalException; - for ( IndexReader reader : readers.values() ) { + for (IndexReader reader : readers.values()) { try { reader.close(); } @@ -175,16 +169,16 @@ readers.clear(); //TODO release lock of all indexes that do not need optimization early //don't optimze if there is a failure - if (raisedException == null) { - for ( DirectoryProvider provider : lockedProviders ) { - Workspace.DPStatistics stats = dpStatistics.get(provider); - if (!stats.optimizationForced) { + if ( raisedException == null ) { + for (DirectoryProvider provider : lockedProviders) { + Workspace.DPStatistics stats = dpStatistics.get( provider ); + if ( !stats.optimizationForced ) { OptimizerStrategy optimizerStrategy = searchFactoryImplementor.getOptimizerStrategy( provider ); optimizerStrategy.addTransaction( stats.operations ); try { optimizerStrategy.optimize( this ); } - catch( SearchException e ) { + catch (SearchException e) { raisedException = new SearchException( "Exception while optimizing directoryProvider: " + provider.getDirectory().toString(), e ); break; //no point in continuing @@ -192,7 +186,7 @@ } } } - for ( IndexWriter writer : writers.values() ) { + for (IndexWriter writer : writers.values()) { try { writer.close(); } @@ -205,7 +199,7 @@ } } } - for ( DirectoryProvider provider : lockedProviders ) { + for (DirectoryProvider provider : lockedProviders) { searchFactoryImplementor.getLockableDirectoryProviders().get( provider ).unlock(); } writers.clear(); @@ -221,18 +215,9 @@ cleanUp( null ); } - public void optimize(Class entity) { - DocumentBuilder documentBuilders = searchFactoryImplementor.getDocumentBuilders().get( entity ); - DirectoryProvider[] providers = documentBuilders.getDirectoryProviderSelectionStrategy() - .getDirectoryProvidersForOptimization( entity ); - for (DirectoryProvider provider: providers) { - optimize( provider ); - } - } - - private void optimize(DirectoryProvider provider) { + public void optimize(DirectoryProvider provider) { OptimizerStrategy optimizerStrategy = searchFactoryImplementor.getOptimizerStrategy( provider ); - dpStatistics.get(provider).optimizationForced = true; + dpStatistics.get( provider ).optimizationForced = true; optimizerStrategy.optimizationForced(); } Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/backend/impl/lucene/LuceneBackendQueueProcessor.java =================================================================== --- trunk/HibernateExt/search/src/java/org/hibernate/search/backend/impl/lucene/LuceneBackendQueueProcessor.java 2007-07-29 14:56:47 UTC (rev 12844) +++ trunk/HibernateExt/search/src/java/org/hibernate/search/backend/impl/lucene/LuceneBackendQueueProcessor.java 2007-07-29 15:10:49 UTC (rev 12845) @@ -71,9 +71,7 @@ } } else if ( OptimizeLuceneWork.class.isAssignableFrom( work.getClass() ) ) { - DirectoryProvider[] providers = shardingStrategy.getDirectoryProvidersForOptimization( - work.getEntityClass() - ); + DirectoryProvider[] providers = shardingStrategy.getDirectoryProvidersForAllShards(); for (DirectoryProvider provider : providers) { queueWithFlatDPs.add( new LuceneWorker.WorkWithPayload(work, provider) ); } Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/backend/impl/lucene/LuceneWorker.java =================================================================== --- trunk/HibernateExt/search/src/java/org/hibernate/search/backend/impl/lucene/LuceneWorker.java 2007-07-29 14:56:47 UTC (rev 12844) +++ trunk/HibernateExt/search/src/java/org/hibernate/search/backend/impl/lucene/LuceneWorker.java 2007-07-29 15:10:49 UTC (rev 12845) @@ -13,13 +13,13 @@ import org.apache.lucene.index.TermDocs; import org.hibernate.annotations.common.AssertionFailure; import org.hibernate.search.SearchException; -import org.hibernate.search.store.DirectoryProvider; import org.hibernate.search.backend.AddLuceneWork; import org.hibernate.search.backend.DeleteLuceneWork; import org.hibernate.search.backend.LuceneWork; +import org.hibernate.search.backend.OptimizeLuceneWork; import org.hibernate.search.backend.Workspace; -import org.hibernate.search.backend.OptimizeLuceneWork; import org.hibernate.search.engine.DocumentBuilder; +import org.hibernate.search.store.DirectoryProvider; /** * Stateless implementation that performs a unit of work. @@ -105,11 +105,11 @@ throw new SearchException( "Unable to remove from Lucene index: " + entity + "#" + id, e ); } finally { - if (termDocs != null) try { + if ( termDocs != null ) try { termDocs.close(); } catch (IOException e) { - log.warn( "Unable to close termDocs properly", e); + log.warn( "Unable to close termDocs properly", e ); } } @@ -122,7 +122,7 @@ IndexWriter writer = workspace.getIndexWriter( provider, entity, false ); try { writer.optimize(); - workspace.optimize(entity); + workspace.optimize( provider ); } catch (IOException e) { throw new SearchException( "Unable to optimize Lucene index: " + entity, e ); Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/engine/SearchFactoryImplementor.java =================================================================== --- trunk/HibernateExt/search/src/java/org/hibernate/search/engine/SearchFactoryImplementor.java 2007-07-29 14:56:47 UTC (rev 12844) +++ trunk/HibernateExt/search/src/java/org/hibernate/search/engine/SearchFactoryImplementor.java 2007-07-29 15:10:49 UTC (rev 12845) @@ -5,16 +5,16 @@ import java.util.concurrent.locks.ReentrantLock; import org.hibernate.search.SearchFactory; +import org.hibernate.search.backend.BackendQueueProcessorFactory; +import org.hibernate.search.backend.LuceneIndexingParameters; +import org.hibernate.search.backend.Worker; import org.hibernate.search.filter.FilterCachingStrategy; import org.hibernate.search.store.DirectoryProvider; import org.hibernate.search.store.optimization.OptimizerStrategy; -import org.hibernate.search.backend.BackendQueueProcessorFactory; -import org.hibernate.search.backend.LuceneIndexingParameters; -import org.hibernate.search.backend.Worker; /** * Interface which gives access to the different directory providers and their configuration. - * + * * @author Emmanuel Bernard * @author Hardy Ferentschik */ @@ -36,8 +36,8 @@ FilterCachingStrategy getFilterCachingStrategy(); FilterDef getFilterDefinition(String name); - - public LuceneIndexingParameters getIndexingParameters(DirectoryProvider<?> provider ); - + + public LuceneIndexingParameters getIndexingParameters(DirectoryProvider<?> provider); + void addIndexingParmeters(DirectoryProvider<?> provider, LuceneIndexingParameters indexingParams); } Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/query/FullTextQueryImpl.java =================================================================== --- trunk/HibernateExt/search/src/java/org/hibernate/search/query/FullTextQueryImpl.java 2007-07-29 14:56:47 UTC (rev 12844) +++ trunk/HibernateExt/search/src/java/org/hibernate/search/query/FullTextQueryImpl.java 2007-07-29 15:10:49 UTC (rev 12845) @@ -408,7 +408,7 @@ //no class means all classes for (DocumentBuilder builder : builders.values()) { final DirectoryProvider[] directoryProviders = - builder.getDirectoryProviderSelectionStrategy().getDirectoryProvidersForQuery(); + builder.getDirectoryProviderSelectionStrategy().getDirectoryProvidersForAllShards(); for (DirectoryProvider provider : directoryProviders) { if ( !directories.contains( provider ) ) { directories.add( provider ); @@ -430,7 +430,7 @@ if ( builder == null ) throw new HibernateException( "Not a mapped entity (don't forget to add @Indexed): " + clazz ); final DirectoryProvider[] directoryProviders = - builder.getDirectoryProviderSelectionStrategy().getDirectoryProvidersForQuery(); + builder.getDirectoryProviderSelectionStrategy().getDirectoryProvidersForAllShards(); for (DirectoryProvider provider : directoryProviders) { if ( !directories.contains( provider ) ) { directories.add( provider ); Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/store/DirectoryProviderFactory.java =================================================================== --- trunk/HibernateExt/search/src/java/org/hibernate/search/store/DirectoryProviderFactory.java 2007-07-29 14:56:47 UTC (rev 12844) +++ trunk/HibernateExt/search/src/java/org/hibernate/search/store/DirectoryProviderFactory.java 2007-07-29 15:10:49 UTC (rev 12845) @@ -71,7 +71,10 @@ int nbrOfProviders = indexProps.length; DirectoryProvider[] providers = new DirectoryProvider[nbrOfProviders]; for (int index = 0 ; index < nbrOfProviders ; index++) { - providers[index] = createDirectoryProvider(directoryProviderName + "_" + index,indexProps[index], searchFactoryImplementor); + String providerName = nbrOfProviders > 1 ? + directoryProviderName + "_" + index : + directoryProviderName; + providers[index] = createDirectoryProvider( providerName,indexProps[index], searchFactoryImplementor); } //define sharding strategy Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/store/DirectoryProviderShardingStrategy.java =================================================================== --- trunk/HibernateExt/search/src/java/org/hibernate/search/store/DirectoryProviderShardingStrategy.java 2007-07-29 14:56:47 UTC (rev 12844) +++ trunk/HibernateExt/search/src/java/org/hibernate/search/store/DirectoryProviderShardingStrategy.java 2007-07-29 15:10:49 UTC (rev 12845) @@ -11,8 +11,7 @@ */ public interface DirectoryProviderShardingStrategy { void init(Properties properties, DirectoryProvider[] providers); - DirectoryProvider[] getDirectoryProvidersForQuery(); + DirectoryProvider[] getDirectoryProvidersForAllShards(); DirectoryProvider getDirectoryProviderForAddition(Class entity, Serializable id, String idInString, Document document); DirectoryProvider[] getDirectoryProvidersForDeletion(Class entity, Serializable id, String idInString); - DirectoryProvider[] getDirectoryProvidersForOptimization(Class entity); } Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/store/IdHashShardingStrategy.java =================================================================== --- trunk/HibernateExt/search/src/java/org/hibernate/search/store/IdHashShardingStrategy.java 2007-07-29 14:56:47 UTC (rev 12844) +++ trunk/HibernateExt/search/src/java/org/hibernate/search/store/IdHashShardingStrategy.java 2007-07-29 15:10:49 UTC (rev 12845) @@ -17,7 +17,7 @@ this.providers = providers; } - public DirectoryProvider[] getDirectoryProvidersForQuery() { + public DirectoryProvider[] getDirectoryProvidersForAllShards() { return providers; } @@ -29,10 +29,6 @@ return new DirectoryProvider[] { providers[ hashKey(idInString) ] }; } - public DirectoryProvider[] getDirectoryProvidersForOptimization(Class entity) { - return providers; - } - private int hashKey(String key) { //reproduce the hashCode implementaiton of String as documented in the javadoc // to be safe cross Java version (in case it changes some day) Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/store/NotShardedStrategy.java =================================================================== --- trunk/HibernateExt/search/src/java/org/hibernate/search/store/NotShardedStrategy.java 2007-07-29 14:56:47 UTC (rev 12844) +++ trunk/HibernateExt/search/src/java/org/hibernate/search/store/NotShardedStrategy.java 2007-07-29 15:10:49 UTC (rev 12845) @@ -19,7 +19,7 @@ } } - public DirectoryProvider[] getDirectoryProvidersForQuery() { + public DirectoryProvider[] getDirectoryProvidersForAllShards() { return directoryProvider; } @@ -31,7 +31,4 @@ return directoryProvider; } - public DirectoryProvider[] getDirectoryProvidersForOptimization(Class entity) { - return directoryProvider; - } }