[hibernate-commits] Hibernate SVN: r12845 - in trunk/HibernateExt/search/src/java/org/hibernate/search: backend/impl/lucene and 3 other directories.
hibernate-commits at lists.jboss.org
hibernate-commits at lists.jboss.org
Sun Jul 29 11:10:49 EDT 2007
Author: epbernard
Date: 2007-07-29 11:10:49 -0400 (Sun, 29 Jul 2007)
New Revision: 12845
Modified:
trunk/HibernateExt/search/src/java/org/hibernate/search/backend/Workspace.java
trunk/HibernateExt/search/src/java/org/hibernate/search/backend/impl/lucene/LuceneBackendQueueProcessor.java
trunk/HibernateExt/search/src/java/org/hibernate/search/backend/impl/lucene/LuceneWorker.java
trunk/HibernateExt/search/src/java/org/hibernate/search/engine/SearchFactoryImplementor.java
trunk/HibernateExt/search/src/java/org/hibernate/search/query/FullTextQueryImpl.java
trunk/HibernateExt/search/src/java/org/hibernate/search/store/DirectoryProviderFactory.java
trunk/HibernateExt/search/src/java/org/hibernate/search/store/DirectoryProviderShardingStrategy.java
trunk/HibernateExt/search/src/java/org/hibernate/search/store/IdHashShardingStrategy.java
trunk/HibernateExt/search/src/java/org/hibernate/search/store/NotShardedStrategy.java
Log:
HSEARCH-86 introduce Index Sharding, including sharding strategy
Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/backend/Workspace.java
===================================================================
--- trunk/HibernateExt/search/src/java/org/hibernate/search/backend/Workspace.java 2007-07-29 14:56:47 UTC (rev 12844)
+++ trunk/HibernateExt/search/src/java/org/hibernate/search/backend/Workspace.java 2007-07-29 15:10:49 UTC (rev 12845)
@@ -2,43 +2,39 @@
package org.hibernate.search.backend;
import java.io.IOException;
-import java.io.Serializable;
-import java.util.HashMap;
-import java.util.Map;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
-import java.util.Set;
-import java.util.HashSet;
+import java.util.Map;
import java.util.concurrent.locks.ReentrantLock;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.SimpleAnalyzer;
-import org.apache.lucene.document.Document;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.hibernate.annotations.common.AssertionFailure;
+import org.hibernate.search.SearchException;
import org.hibernate.search.engine.DocumentBuilder;
import org.hibernate.search.engine.SearchFactoryImplementor;
-import org.hibernate.search.SearchException;
import org.hibernate.search.store.DirectoryProvider;
import org.hibernate.search.store.optimization.OptimizerStrategy;
-import org.hibernate.annotations.common.AssertionFailure;
/**
* Lucene workspace.
- * <p>
+ * <p/>
* <b>This is not intended to be used in a multithreaded environment</b>.
* <p/>
* <ul>
- * <li>One cannot execute modification through an IndexReader when an IndexWriter has been acquired
- * on the same underlying directory
+ * <li>One cannot execute modification through an IndexReader when an IndexWriter has been acquired
+ * on the same underlying directory
* </li>
- * <li>One cannot get an IndexWriter when an IndexReader have been acquired and modificed on the same
- * underlying directory
+ * <li>One cannot get an IndexWriter when an IndexReader have been acquired and modificed on the same
+ * underlying directory
* </li>
* <li>The recommended approach is to execute all the modifications on the IndexReaders, {@link #clean()}, and acquire the
- * index writers
+ * index writers
* </li>
* </ul>
*
@@ -53,13 +49,13 @@
private List<DirectoryProvider> lockedProviders = new ArrayList<DirectoryProvider>();
private Map<DirectoryProvider, DPStatistics> dpStatistics = new HashMap<DirectoryProvider, DPStatistics>();
private SearchFactoryImplementor searchFactoryImplementor;
-
+
/**
* Flag indicating if the current work should be executed the Lucene parameters for batch indexing.
*/
private boolean isBatch = false;
-
+
public Workspace(SearchFactoryImplementor searchFactoryImplementor) {
this.searchFactoryImplementor = searchFactoryImplementor;
}
@@ -75,11 +71,11 @@
public IndexReader getIndexReader(DirectoryProvider provider, Class entity) {
//one cannot access a reader for update after a writer has been accessed
if ( writers.containsKey( provider ) )
- throw new AssertionFailure("Tries to read for update an index while a writer is accessed" + entity);
+ throw new AssertionFailure( "Tries to read for update an index while a writer is accessed" + entity );
IndexReader reader = readers.get( provider );
if ( reader != null ) return reader;
lockProvider( provider );
- dpStatistics.get(provider).operations++;
+ dpStatistics.get( provider ).operations++;
try {
reader = IndexReader.open( provider.getDirectory() );
readers.put( provider, reader );
@@ -114,32 +110,30 @@
IndexWriter writer = writers.get( provider );
if ( writer != null ) return writer;
lockProvider( provider );
- if (modificationOperation) dpStatistics.get(provider).operations++;
+ if ( modificationOperation ) dpStatistics.get( provider ).operations++;
try {
Analyzer analyzer = entity != null ?
searchFactoryImplementor.getDocumentBuilders().get( entity ).getAnalyzer() :
new SimpleAnalyzer(); //never used
writer = new IndexWriter( provider.getDirectory(), analyzer, false ); //has been created at init time
- LuceneIndexingParameters indexingParams = searchFactoryImplementor.getIndexingParameters(provider);
- if(isBatch)
- {
- writer.setMergeFactor(indexingParams.getBatchMergeFactor());
- writer.setMaxMergeDocs(indexingParams.getBatchMaxMergeDocs());
- writer.setMaxBufferedDocs(indexingParams.getBatchMaxBufferedDocs());
- }
- else
- {
- writer.setMergeFactor(indexingParams.getMergeFactor());
- writer.setMaxMergeDocs(indexingParams.getMaxMergeDocs());
- writer.setMaxBufferedDocs(indexingParams.getMaxBufferedDocs());
+ LuceneIndexingParameters indexingParams = searchFactoryImplementor.getIndexingParameters( provider );
+ if ( isBatch ) {
+ writer.setMergeFactor( indexingParams.getBatchMergeFactor() );
+ writer.setMaxMergeDocs( indexingParams.getBatchMaxMergeDocs() );
+ writer.setMaxBufferedDocs( indexingParams.getBatchMaxBufferedDocs() );
}
-
+ else {
+ writer.setMergeFactor( indexingParams.getMergeFactor() );
+ writer.setMaxMergeDocs( indexingParams.getMaxMergeDocs() );
+ writer.setMaxBufferedDocs( indexingParams.getMaxBufferedDocs() );
+ }
+
writers.put( provider, writer );
}
catch (IOException e) {
cleanUp(
- new SearchException( "Unable to open IndexWriter" + ( entity != null ? " for " + entity : "" ), e )
+ new SearchException( "Unable to open IndexWriter" + ( entity != null ? " for " + entity : "" ), e )
);
}
return writer;
@@ -159,7 +153,7 @@
private void cleanUp(SearchException originalException) {
//release all readers and writers, then release locks
SearchException raisedException = originalException;
- for ( IndexReader reader : readers.values() ) {
+ for (IndexReader reader : readers.values()) {
try {
reader.close();
}
@@ -175,16 +169,16 @@
readers.clear();
//TODO release lock of all indexes that do not need optimization early
//don't optimze if there is a failure
- if (raisedException == null) {
- for ( DirectoryProvider provider : lockedProviders ) {
- Workspace.DPStatistics stats = dpStatistics.get(provider);
- if (!stats.optimizationForced) {
+ if ( raisedException == null ) {
+ for (DirectoryProvider provider : lockedProviders) {
+ Workspace.DPStatistics stats = dpStatistics.get( provider );
+ if ( !stats.optimizationForced ) {
OptimizerStrategy optimizerStrategy = searchFactoryImplementor.getOptimizerStrategy( provider );
optimizerStrategy.addTransaction( stats.operations );
try {
optimizerStrategy.optimize( this );
}
- catch( SearchException e ) {
+ catch (SearchException e) {
raisedException = new SearchException( "Exception while optimizing directoryProvider: "
+ provider.getDirectory().toString(), e );
break; //no point in continuing
@@ -192,7 +186,7 @@
}
}
}
- for ( IndexWriter writer : writers.values() ) {
+ for (IndexWriter writer : writers.values()) {
try {
writer.close();
}
@@ -205,7 +199,7 @@
}
}
}
- for ( DirectoryProvider provider : lockedProviders ) {
+ for (DirectoryProvider provider : lockedProviders) {
searchFactoryImplementor.getLockableDirectoryProviders().get( provider ).unlock();
}
writers.clear();
@@ -221,18 +215,9 @@
cleanUp( null );
}
- public void optimize(Class entity) {
- DocumentBuilder documentBuilders = searchFactoryImplementor.getDocumentBuilders().get( entity );
- DirectoryProvider[] providers = documentBuilders.getDirectoryProviderSelectionStrategy()
- .getDirectoryProvidersForOptimization( entity );
- for (DirectoryProvider provider: providers) {
- optimize( provider );
- }
- }
-
- private void optimize(DirectoryProvider provider) {
+ public void optimize(DirectoryProvider provider) {
OptimizerStrategy optimizerStrategy = searchFactoryImplementor.getOptimizerStrategy( provider );
- dpStatistics.get(provider).optimizationForced = true;
+ dpStatistics.get( provider ).optimizationForced = true;
optimizerStrategy.optimizationForced();
}
Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/backend/impl/lucene/LuceneBackendQueueProcessor.java
===================================================================
--- trunk/HibernateExt/search/src/java/org/hibernate/search/backend/impl/lucene/LuceneBackendQueueProcessor.java 2007-07-29 14:56:47 UTC (rev 12844)
+++ trunk/HibernateExt/search/src/java/org/hibernate/search/backend/impl/lucene/LuceneBackendQueueProcessor.java 2007-07-29 15:10:49 UTC (rev 12845)
@@ -71,9 +71,7 @@
}
}
else if ( OptimizeLuceneWork.class.isAssignableFrom( work.getClass() ) ) {
- DirectoryProvider[] providers = shardingStrategy.getDirectoryProvidersForOptimization(
- work.getEntityClass()
- );
+ DirectoryProvider[] providers = shardingStrategy.getDirectoryProvidersForAllShards();
for (DirectoryProvider provider : providers) {
queueWithFlatDPs.add( new LuceneWorker.WorkWithPayload(work, provider) );
}
Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/backend/impl/lucene/LuceneWorker.java
===================================================================
--- trunk/HibernateExt/search/src/java/org/hibernate/search/backend/impl/lucene/LuceneWorker.java 2007-07-29 14:56:47 UTC (rev 12844)
+++ trunk/HibernateExt/search/src/java/org/hibernate/search/backend/impl/lucene/LuceneWorker.java 2007-07-29 15:10:49 UTC (rev 12845)
@@ -13,13 +13,13 @@
import org.apache.lucene.index.TermDocs;
import org.hibernate.annotations.common.AssertionFailure;
import org.hibernate.search.SearchException;
-import org.hibernate.search.store.DirectoryProvider;
import org.hibernate.search.backend.AddLuceneWork;
import org.hibernate.search.backend.DeleteLuceneWork;
import org.hibernate.search.backend.LuceneWork;
+import org.hibernate.search.backend.OptimizeLuceneWork;
import org.hibernate.search.backend.Workspace;
-import org.hibernate.search.backend.OptimizeLuceneWork;
import org.hibernate.search.engine.DocumentBuilder;
+import org.hibernate.search.store.DirectoryProvider;
/**
* Stateless implementation that performs a unit of work.
@@ -105,11 +105,11 @@
throw new SearchException( "Unable to remove from Lucene index: " + entity + "#" + id, e );
}
finally {
- if (termDocs != null) try {
+ if ( termDocs != null ) try {
termDocs.close();
}
catch (IOException e) {
- log.warn( "Unable to close termDocs properly", e);
+ log.warn( "Unable to close termDocs properly", e );
}
}
@@ -122,7 +122,7 @@
IndexWriter writer = workspace.getIndexWriter( provider, entity, false );
try {
writer.optimize();
- workspace.optimize(entity);
+ workspace.optimize( provider );
}
catch (IOException e) {
throw new SearchException( "Unable to optimize Lucene index: " + entity, e );
Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/engine/SearchFactoryImplementor.java
===================================================================
--- trunk/HibernateExt/search/src/java/org/hibernate/search/engine/SearchFactoryImplementor.java 2007-07-29 14:56:47 UTC (rev 12844)
+++ trunk/HibernateExt/search/src/java/org/hibernate/search/engine/SearchFactoryImplementor.java 2007-07-29 15:10:49 UTC (rev 12845)
@@ -5,16 +5,16 @@
import java.util.concurrent.locks.ReentrantLock;
import org.hibernate.search.SearchFactory;
+import org.hibernate.search.backend.BackendQueueProcessorFactory;
+import org.hibernate.search.backend.LuceneIndexingParameters;
+import org.hibernate.search.backend.Worker;
import org.hibernate.search.filter.FilterCachingStrategy;
import org.hibernate.search.store.DirectoryProvider;
import org.hibernate.search.store.optimization.OptimizerStrategy;
-import org.hibernate.search.backend.BackendQueueProcessorFactory;
-import org.hibernate.search.backend.LuceneIndexingParameters;
-import org.hibernate.search.backend.Worker;
/**
* Interface which gives access to the different directory providers and their configuration.
- *
+ *
* @author Emmanuel Bernard
* @author Hardy Ferentschik
*/
@@ -36,8 +36,8 @@
FilterCachingStrategy getFilterCachingStrategy();
FilterDef getFilterDefinition(String name);
-
- public LuceneIndexingParameters getIndexingParameters(DirectoryProvider<?> provider );
-
+
+ public LuceneIndexingParameters getIndexingParameters(DirectoryProvider<?> provider);
+
void addIndexingParmeters(DirectoryProvider<?> provider, LuceneIndexingParameters indexingParams);
}
Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/query/FullTextQueryImpl.java
===================================================================
--- trunk/HibernateExt/search/src/java/org/hibernate/search/query/FullTextQueryImpl.java 2007-07-29 14:56:47 UTC (rev 12844)
+++ trunk/HibernateExt/search/src/java/org/hibernate/search/query/FullTextQueryImpl.java 2007-07-29 15:10:49 UTC (rev 12845)
@@ -408,7 +408,7 @@
//no class means all classes
for (DocumentBuilder builder : builders.values()) {
final DirectoryProvider[] directoryProviders =
- builder.getDirectoryProviderSelectionStrategy().getDirectoryProvidersForQuery();
+ builder.getDirectoryProviderSelectionStrategy().getDirectoryProvidersForAllShards();
for (DirectoryProvider provider : directoryProviders) {
if ( !directories.contains( provider ) ) {
directories.add( provider );
@@ -430,7 +430,7 @@
if ( builder == null )
throw new HibernateException( "Not a mapped entity (don't forget to add @Indexed): " + clazz );
final DirectoryProvider[] directoryProviders =
- builder.getDirectoryProviderSelectionStrategy().getDirectoryProvidersForQuery();
+ builder.getDirectoryProviderSelectionStrategy().getDirectoryProvidersForAllShards();
for (DirectoryProvider provider : directoryProviders) {
if ( !directories.contains( provider ) ) {
directories.add( provider );
Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/store/DirectoryProviderFactory.java
===================================================================
--- trunk/HibernateExt/search/src/java/org/hibernate/search/store/DirectoryProviderFactory.java 2007-07-29 14:56:47 UTC (rev 12844)
+++ trunk/HibernateExt/search/src/java/org/hibernate/search/store/DirectoryProviderFactory.java 2007-07-29 15:10:49 UTC (rev 12845)
@@ -71,7 +71,10 @@
int nbrOfProviders = indexProps.length;
DirectoryProvider[] providers = new DirectoryProvider[nbrOfProviders];
for (int index = 0 ; index < nbrOfProviders ; index++) {
- providers[index] = createDirectoryProvider(directoryProviderName + "_" + index,indexProps[index], searchFactoryImplementor);
+ String providerName = nbrOfProviders > 1 ?
+ directoryProviderName + "_" + index :
+ directoryProviderName;
+ providers[index] = createDirectoryProvider( providerName,indexProps[index], searchFactoryImplementor);
}
//define sharding strategy
Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/store/DirectoryProviderShardingStrategy.java
===================================================================
--- trunk/HibernateExt/search/src/java/org/hibernate/search/store/DirectoryProviderShardingStrategy.java 2007-07-29 14:56:47 UTC (rev 12844)
+++ trunk/HibernateExt/search/src/java/org/hibernate/search/store/DirectoryProviderShardingStrategy.java 2007-07-29 15:10:49 UTC (rev 12845)
@@ -11,8 +11,7 @@
*/
public interface DirectoryProviderShardingStrategy {
void init(Properties properties, DirectoryProvider[] providers);
- DirectoryProvider[] getDirectoryProvidersForQuery();
+ DirectoryProvider[] getDirectoryProvidersForAllShards();
DirectoryProvider getDirectoryProviderForAddition(Class entity, Serializable id, String idInString, Document document);
DirectoryProvider[] getDirectoryProvidersForDeletion(Class entity, Serializable id, String idInString);
- DirectoryProvider[] getDirectoryProvidersForOptimization(Class entity);
}
Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/store/IdHashShardingStrategy.java
===================================================================
--- trunk/HibernateExt/search/src/java/org/hibernate/search/store/IdHashShardingStrategy.java 2007-07-29 14:56:47 UTC (rev 12844)
+++ trunk/HibernateExt/search/src/java/org/hibernate/search/store/IdHashShardingStrategy.java 2007-07-29 15:10:49 UTC (rev 12845)
@@ -17,7 +17,7 @@
this.providers = providers;
}
- public DirectoryProvider[] getDirectoryProvidersForQuery() {
+ public DirectoryProvider[] getDirectoryProvidersForAllShards() {
return providers;
}
@@ -29,10 +29,6 @@
return new DirectoryProvider[] { providers[ hashKey(idInString) ] };
}
- public DirectoryProvider[] getDirectoryProvidersForOptimization(Class entity) {
- return providers;
- }
-
private int hashKey(String key) {
//reproduce the hashCode implementaiton of String as documented in the javadoc
// to be safe cross Java version (in case it changes some day)
Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/store/NotShardedStrategy.java
===================================================================
--- trunk/HibernateExt/search/src/java/org/hibernate/search/store/NotShardedStrategy.java 2007-07-29 14:56:47 UTC (rev 12844)
+++ trunk/HibernateExt/search/src/java/org/hibernate/search/store/NotShardedStrategy.java 2007-07-29 15:10:49 UTC (rev 12845)
@@ -19,7 +19,7 @@
}
}
- public DirectoryProvider[] getDirectoryProvidersForQuery() {
+ public DirectoryProvider[] getDirectoryProvidersForAllShards() {
return directoryProvider;
}
@@ -31,7 +31,4 @@
return directoryProvider;
}
- public DirectoryProvider[] getDirectoryProvidersForOptimization(Class entity) {
- return directoryProvider;
- }
}
More information about the hibernate-commits
mailing list