[hibernate-commits] Hibernate SVN: r12806 - in trunk/HibernateExt/search/src/java/org/hibernate/search: backend/impl/lucene and 2 other directories.
hibernate-commits at lists.jboss.org
hibernate-commits at lists.jboss.org
Tue Jul 24 05:51:04 EDT 2007
Author: hardy.ferentschik
Date: 2007-07-24 05:51:04 -0400 (Tue, 24 Jul 2007)
New Revision: 12806
Modified:
trunk/HibernateExt/search/src/java/org/hibernate/search/backend/LuceneWork.java
trunk/HibernateExt/search/src/java/org/hibernate/search/backend/WorkType.java
trunk/HibernateExt/search/src/java/org/hibernate/search/backend/Workspace.java
trunk/HibernateExt/search/src/java/org/hibernate/search/backend/impl/lucene/LuceneBackendQueueProcessor.java
trunk/HibernateExt/search/src/java/org/hibernate/search/backend/impl/lucene/LuceneWorker.java
trunk/HibernateExt/search/src/java/org/hibernate/search/engine/DocumentBuilder.java
trunk/HibernateExt/search/src/java/org/hibernate/search/engine/SearchFactoryImplementor.java
trunk/HibernateExt/search/src/java/org/hibernate/search/impl/FullTextSessionImpl.java
trunk/HibernateExt/search/src/java/org/hibernate/search/impl/SearchFactoryImpl.java
Log:
HSEARCH-67.
Enabled the WorkType.INDEX and chaged all required files to use the lucene indexing parameters in case this new work type is used.
Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/backend/LuceneWork.java
===================================================================
--- trunk/HibernateExt/search/src/java/org/hibernate/search/backend/LuceneWork.java 2007-07-24 09:48:22 UTC (rev 12805)
+++ trunk/HibernateExt/search/src/java/org/hibernate/search/backend/LuceneWork.java 2007-07-24 09:51:04 UTC (rev 12806)
@@ -9,12 +9,18 @@
* Represent a Serializable Lucene unit work
*
* @author Emmanuel Bernard
+ * @author Hardy Ferentschik
*/
public abstract class LuceneWork implements Serializable {
//TODO set a serial id
private Document document;
private Class entityClass;
private Serializable id;
+
+ /**
+ * Flag indicating if this lucene work has to be indexed in batch mode.
+ */
+ private boolean batch = false;
public LuceneWork(Serializable id, Class entity) {
this( id, entity, null );
@@ -26,7 +32,14 @@
this.document = document;
}
+ public boolean isBatch() {
+ return batch;
+ }
+ public void setBatch(boolean batch) {
+ this.batch = batch;
+ }
+
public Document getDocument() {
return document;
}
Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/backend/WorkType.java
===================================================================
--- trunk/HibernateExt/search/src/java/org/hibernate/search/backend/WorkType.java 2007-07-24 09:48:22 UTC (rev 12805)
+++ trunk/HibernateExt/search/src/java/org/hibernate/search/backend/WorkType.java 2007-07-24 09:51:04 UTC (rev 12806)
@@ -2,11 +2,19 @@
package org.hibernate.search.backend;
/**
+ * Enumeration of the different types of Lucene work. This enumeration is used to specify the type
+ * of index operation to be executed.
+ *
* @author Emmanuel Bernard
+ * @author Hardy Ferentschik
*/
public enum WorkType {
ADD,
UPDATE,
- DELETE
- //add INDEX at some point to behave differently during the queue process?
+ DELETE,
+
+ /**
+ * This type is used for batch indexing.
+ */
+ INDEX
}
Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/backend/Workspace.java
===================================================================
--- trunk/HibernateExt/search/src/java/org/hibernate/search/backend/Workspace.java 2007-07-24 09:48:22 UTC (rev 12805)
+++ trunk/HibernateExt/search/src/java/org/hibernate/search/backend/Workspace.java 2007-07-24 09:51:04 UTC (rev 12806)
@@ -22,15 +22,24 @@
import org.hibernate.annotations.common.AssertionFailure;
/**
- * Lucene workspace
- * This is not intended to be used in a multithreaded environment
+ * Lucene workspace.
+ * <p>
+ * <b>This is not intended to be used in a multithreaded environment</b>.
* <p/>
- * One cannot execute modification through an IndexReader when an IndexWriter has been acquired on the same underlying directory
- * One cannot get an IndexWriter when an IndexReader have been acquired and modificed on the same underlying directory
- * The recommended approach is to execute all the modifications on the IndexReaders, {@link #clean()} }, and acquire the
- * index writers
+ * <ul>
+ * <li>One cannot execute modification through an IndexReader when an IndexWriter has been acquired
+ * on the same underlying directory
+ * </li>
+ * <li>One cannot get an IndexWriter when an IndexReader have been acquired and modificed on the same
+ * underlying directory
+ * </li>
+ * <li>The recommended approach is to execute all the modifications on the IndexReaders, {@link #clean()}, and acquire the
+ * index writers
+ * </li>
+ * </ul>
*
* @author Emmanuel Bernard
+ * @author Hardy Ferentschik
*/
//TODO introduce the notion of read only IndexReader? We cannot enforce it because Lucene use abstract classes, not interfaces
public class Workspace {
@@ -41,12 +50,16 @@
private Map<DirectoryProvider, DPStatistics> dpStatistics = new HashMap<DirectoryProvider, DPStatistics>();
private SearchFactoryImplementor searchFactoryImplementor;
+ /**
+ * Flag indicating if the current work should be executed the Lucene parameters for batch indexing.
+ */
+ private boolean isBatch = false;
+
public Workspace(SearchFactoryImplementor searchFactoryImplementor) {
this.searchFactoryImplementor = searchFactoryImplementor;
}
-
public DocumentBuilder getDocumentBuilder(Class entity) {
return searchFactoryImplementor.getDocumentBuilders().get( entity );
}
@@ -115,7 +128,22 @@
Analyzer analyzer = entity != null ?
searchFactoryImplementor.getDocumentBuilders().get( entity ).getAnalyzer() :
new SimpleAnalyzer(); //never used
- writer = new IndexWriter( provider.getDirectory(), analyzer, false ); //have been created at init time
+ writer = new IndexWriter( provider.getDirectory(), analyzer, false ); //has been created at init time
+
+ LuceneIndexingParameters indexingParams = searchFactoryImplementor.getIndexingParameters(provider);
+ if(isBatch)
+ {
+ writer.setMergeFactor(indexingParams.getBatchMergeFactor());
+ writer.setMaxMergeDocs(indexingParams.getBatchMaxMergeDocs());
+ writer.setMaxBufferedDocs(indexingParams.getBatchMaxBufferedDocs());
+ }
+ else
+ {
+ writer.setMergeFactor(indexingParams.getMergeFactor());
+ writer.setMaxMergeDocs(indexingParams.getMaxMergeDocs());
+ writer.setMaxBufferedDocs(indexingParams.getMaxBufferedDocs());
+ }
+
writers.put( provider, writer );
}
catch (IOException e) {
@@ -214,4 +242,11 @@
public long operations;
}
+ public boolean isBatch() {
+ return isBatch;
+ }
+
+ public void setBatch(boolean isBatch) {
+ this.isBatch = isBatch;
+ }
}
Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/backend/impl/lucene/LuceneBackendQueueProcessor.java
===================================================================
--- trunk/HibernateExt/search/src/java/org/hibernate/search/backend/impl/lucene/LuceneBackendQueueProcessor.java 2007-07-24 09:48:22 UTC (rev 12805)
+++ trunk/HibernateExt/search/src/java/org/hibernate/search/backend/impl/lucene/LuceneBackendQueueProcessor.java 2007-07-24 09:51:04 UTC (rev 12806)
@@ -5,6 +5,8 @@
import java.util.Comparator;
import java.util.List;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.hibernate.search.backend.AddLuceneWork;
import org.hibernate.search.backend.LuceneWork;
import org.hibernate.search.backend.Workspace;
@@ -13,12 +15,18 @@
import org.hibernate.search.store.DirectoryProvider;
/**
- * Apply the operations to Lucene directories
- * avoiding deadlocks
+ * Apply the operations to Lucene directories avoiding deadlocks.
*
* @author Emmanuel Bernard
+ * @author Hardy Ferentschik
*/
public class LuceneBackendQueueProcessor implements Runnable {
+
+ /**
+ * Class logger.
+ */
+ private static Log log = LogFactory.getLog( LuceneBackendQueueProcessor.class );
+
private List<LuceneWork> queue;
private SearchFactoryImplementor searchFactoryImplementor;
@@ -33,7 +41,8 @@
workspace = new Workspace( searchFactoryImplementor );
worker = new LuceneWorker( workspace );
try {
- deadlockFreeQueue(queue, workspace, searchFactoryImplementor);
+ deadlockFreeQueue(queue, workspace, searchFactoryImplementor);
+ checkForBatchIndexing(workspace);
for ( LuceneWork luceneWork : queue ) {
worker.performWork( luceneWork );
}
@@ -44,6 +53,16 @@
}
}
+ private void checkForBatchIndexing(Workspace workspace) {
+ for ( LuceneWork luceneWork : queue ) {
+ // if there is at least a single batch index job we put the work space into batch indexing mode.
+ if(luceneWork.isBatch()){
+ log.debug("Setting batch indexing mode.");
+ workspace.setBatch(true);
+ }
+ }
+ }
+
/**
* one must lock the directory providers in the exact same order to avoid
* dead lock between concurrent threads or processes
Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/backend/impl/lucene/LuceneWorker.java
===================================================================
--- trunk/HibernateExt/search/src/java/org/hibernate/search/backend/impl/lucene/LuceneWorker.java 2007-07-24 09:48:22 UTC (rev 12805)
+++ trunk/HibernateExt/search/src/java/org/hibernate/search/backend/impl/lucene/LuceneWorker.java 2007-07-24 09:51:04 UTC (rev 12806)
@@ -21,9 +21,10 @@
import org.hibernate.search.engine.DocumentBuilder;
/**
- * Stateless implementation that perform a work
+ * Stateless implementation that performs a unit of work.
*
* @author Emmanuel Bernard
+ * @author Hardy Ferentschik
*/
public class LuceneWorker {
private Workspace workspace;
Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/engine/DocumentBuilder.java
===================================================================
--- trunk/HibernateExt/search/src/java/org/hibernate/search/engine/DocumentBuilder.java 2007-07-24 09:48:22 UTC (rev 12805)
+++ trunk/HibernateExt/search/src/java/org/hibernate/search/engine/DocumentBuilder.java 2007-07-24 09:51:04 UTC (rev 12806)
@@ -54,6 +54,7 @@
* @author Emmanuel Bernard
* @author Sylvain Vieujot
* @author Richard Hallier
+ * @author Hardy Ferentschik
*/
public class DocumentBuilder<T> {
private static final Log log = LogFactory.getLog( DocumentBuilder.class );
@@ -393,6 +394,15 @@
queue.add( new AddLuceneWork( id, entityClass, doc ) );
searchForContainers = true;
}
+ else if ( workType == WorkType.INDEX ) {
+ Document doc = getDocument( entity, id );
+ queue.add(new DeleteLuceneWork(id, entityClass) );
+ LuceneWork work = new AddLuceneWork( id, entityClass, doc );
+ work.setBatch(true);
+ queue.add(work);
+ searchForContainers = true;
+ }
+
else {
throw new AssertionFailure("Unknown WorkType: " + workType);
}
Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/engine/SearchFactoryImplementor.java
===================================================================
--- trunk/HibernateExt/search/src/java/org/hibernate/search/engine/SearchFactoryImplementor.java 2007-07-24 09:48:22 UTC (rev 12805)
+++ trunk/HibernateExt/search/src/java/org/hibernate/search/engine/SearchFactoryImplementor.java 2007-07-24 09:51:04 UTC (rev 12806)
@@ -1,4 +1,4 @@
-//$Id: $
+// $Id$
package org.hibernate.search.engine;
import java.util.Map;
@@ -8,10 +8,14 @@
import org.hibernate.search.store.DirectoryProvider;
import org.hibernate.search.store.optimization.OptimizerStrategy;
import org.hibernate.search.backend.BackendQueueProcessorFactory;
+import org.hibernate.search.backend.LuceneIndexingParameters;
import org.hibernate.search.backend.Worker;
/**
+ * Interface which gives access to the different directory providers and their configuration.
+ *
* @author Emmanuel Bernard
+ * @author Hardy Ferentschik
*/
public interface SearchFactoryImplementor extends SearchFactory {
BackendQueueProcessorFactory getBackendQueueProcessorFactory();
@@ -27,4 +31,8 @@
void addOptimizerStrategy(DirectoryProvider<?> provider, OptimizerStrategy optimizerStrategy);
public OptimizerStrategy getOptimizerStrategy(DirectoryProvider<?> provider);
+
+ public LuceneIndexingParameters getIndexingParameters(DirectoryProvider<?> provider );
+
+ void addIndexingParmeters(DirectoryProvider<?> provider, LuceneIndexingParameters indexingParams);
}
Property changes on: trunk/HibernateExt/search/src/java/org/hibernate/search/engine/SearchFactoryImplementor.java
___________________________________________________________________
Name: svn:keywords
+ Id
Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/impl/FullTextSessionImpl.java
===================================================================
--- trunk/HibernateExt/search/src/java/org/hibernate/search/impl/FullTextSessionImpl.java 2007-07-24 09:48:22 UTC (rev 12805)
+++ trunk/HibernateExt/search/src/java/org/hibernate/search/impl/FullTextSessionImpl.java 2007-07-24 09:51:04 UTC (rev 12806)
@@ -52,7 +52,7 @@
import org.hibernate.type.Type;
/**
- * Lucene Full text search aware session
+ * Lucene full text search aware session.
*
* @author Emmanuel Bernard
*/
@@ -83,7 +83,7 @@
* Non indexable entities are ignored
* The entity must be associated with the session
*
- * @param entity must not be null
+ * @param entity The neity to index - must not be <code>null</code>.
*/
public void index(Object entity) {
if (entity == null) return;
@@ -94,7 +94,7 @@
DocumentBuilder<Object> builder = searchFactoryImplementor.getDocumentBuilders().get( clazz );
if ( builder != null ) {
Serializable id = session.getIdentifier( entity );
- searchFactoryImplementor.getWorker().performWork( entity, id, WorkType.UPDATE, eventSource );
+ searchFactoryImplementor.getWorker().performWork( entity, id, WorkType.INDEX, eventSource );
}
//TODO
//need to add elements in a queue kept at the Session level
Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/impl/SearchFactoryImpl.java
===================================================================
--- trunk/HibernateExt/search/src/java/org/hibernate/search/impl/SearchFactoryImpl.java 2007-07-24 09:48:22 UTC (rev 12805)
+++ trunk/HibernateExt/search/src/java/org/hibernate/search/impl/SearchFactoryImpl.java 2007-07-24 09:51:04 UTC (rev 12806)
@@ -1,13 +1,13 @@
//$Id$
package org.hibernate.search.impl;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.WeakHashMap;
-import java.util.List;
-import java.util.ArrayList;
import java.util.concurrent.locks.ReentrantLock;
import org.apache.lucene.analysis.Analyzer;
@@ -22,10 +22,11 @@
import org.hibernate.search.Version;
import org.hibernate.search.annotations.Indexed;
import org.hibernate.search.backend.BackendQueueProcessorFactory;
+import org.hibernate.search.backend.LuceneIndexingParameters;
+import org.hibernate.search.backend.LuceneWork;
+import org.hibernate.search.backend.OptimizeLuceneWork;
import org.hibernate.search.backend.Worker;
import org.hibernate.search.backend.WorkerFactory;
-import org.hibernate.search.backend.LuceneWork;
-import org.hibernate.search.backend.OptimizeLuceneWork;
import org.hibernate.search.engine.DocumentBuilder;
import org.hibernate.search.engine.SearchFactoryImplementor;
import org.hibernate.search.reader.ReaderProvider;
@@ -36,6 +37,7 @@
import org.hibernate.util.ReflectHelper;
/**
+ *
* @author Emmanuel Bernard
*/
public class SearchFactoryImpl implements SearchFactoryImplementor {
@@ -45,7 +47,7 @@
static {
Version.touch();
}
-
+
private Map<Class, DocumentBuilder<Object>> documentBuilders = new HashMap<Class, DocumentBuilder<Object>>();
//keep track of the index modifiers per DirectoryProvider since multiple entity can use the same directory provider
private Map<DirectoryProvider, ReentrantLock> lockableDirectoryProviders =
@@ -55,7 +57,15 @@
private Worker worker;
private ReaderProvider readerProvider;
private BackendQueueProcessorFactory backendQueueProcessorFactory;
+
+
+ /**
+ * Each directory provider (index) can have its own performance settings.
+ */
+ private Map<DirectoryProvider, LuceneIndexingParameters> dirProviderIndexingParams =
+ new HashMap<DirectoryProvider, LuceneIndexingParameters>();
+
public BackendQueueProcessorFactory getBackendQueueProcessorFactory() {
return backendQueueProcessorFactory;
}
@@ -64,63 +74,16 @@
this.backendQueueProcessorFactory = backendQueueProcessorFactory;
}
+ @SuppressWarnings( "unchecked" )
public SearchFactoryImpl(Configuration cfg) {
//yuk
ReflectionManager reflectionManager = getReflectionManager( cfg );
- Class analyzerClass;
- String analyzerClassName = cfg.getProperty( Environment.ANALYZER_CLASS );
- if ( analyzerClassName != null ) {
- try {
- analyzerClass = ReflectHelper.classForName( analyzerClassName );
- }
- catch (Exception e) {
- throw new SearchException(
- "Lucene analyzer class '" + analyzerClassName + "' defined in property '" + Environment.ANALYZER_CLASS + "' could not be found.",
- e
- );
- }
- }
- else {
- analyzerClass = StandardAnalyzer.class;
- }
- // Initialize analyzer
- Analyzer defaultAnalyzer;
- try {
- defaultAnalyzer = (Analyzer) analyzerClass.newInstance();
- }
- catch (ClassCastException e) {
- throw new SearchException(
- "Lucene analyzer does not implement " + Analyzer.class.getName() + ": " + analyzerClassName, e
- );
- }
- catch (Exception e) {
- throw new SearchException( "Failed to instantiate lucene analyzer with type " + analyzerClassName, e );
- }
-
- Iterator iter = cfg.getClassMappings();
- DirectoryProviderFactory factory = new DirectoryProviderFactory();
- while ( iter.hasNext() ) {
- PersistentClass clazz = (PersistentClass) iter.next();
- Class<?> mappedClass = clazz.getMappedClass();
- if ( mappedClass != null ) {
- XClass mappedXClass = reflectionManager.toXClass( mappedClass );
- if ( mappedXClass != null && mappedXClass.isAnnotationPresent( Indexed.class ) ) {
- DirectoryProvider provider = factory.createDirectoryProvider( mappedXClass, cfg, this );
- //TODO move that into DirectoryProviderFactory
- if ( !lockableDirectoryProviders.containsKey( provider ) ) {
- lockableDirectoryProviders.put( provider, new ReentrantLock() );
- }
- final DocumentBuilder<Object> documentBuilder = new DocumentBuilder<Object>(
- mappedXClass, defaultAnalyzer, provider, reflectionManager
- );
-
- documentBuilders.put( mappedClass, documentBuilder );
- }
- }
- }
+ Analyzer analyzer = initAnalyzer(cfg);
+ initDocumentBuilders(cfg, reflectionManager, analyzer);
+
Set<Class> indexedClasses = documentBuilders.keySet();
- for (DocumentBuilder builder : documentBuilders.values()) {
+ for (DocumentBuilder builder : documentBuilders.values()) {
builder.postInitialize( indexedClasses );
}
worker = WorkerFactory.createWorker( cfg, this );
@@ -161,10 +124,18 @@
public void addOptimizerStrategy(DirectoryProvider<?> provider, OptimizerStrategy optimizerStrategy) {
dirProviderOptimizerStrategies.put( provider, optimizerStrategy );
}
+
+ public void addIndexingParmeters(DirectoryProvider<?> provider, LuceneIndexingParameters indexingParams) {
+ dirProviderIndexingParams.put( provider, indexingParams );
+ }
public OptimizerStrategy getOptimizerStrategy(DirectoryProvider<?> provider) {
return dirProviderOptimizerStrategies.get( provider );
}
+
+ public LuceneIndexingParameters getIndexingParameters(DirectoryProvider<?> provider ) {
+ return dirProviderIndexingParams.get( provider );
+ }
public ReaderProvider getReaderProvider() {
return readerProvider;
@@ -207,4 +178,60 @@
queue.add( new OptimizeLuceneWork( entityType ) );
getBackendQueueProcessorFactory().getProcessor( queue ).run();
}
+
+ private void initDocumentBuilders(Configuration cfg, ReflectionManager reflectionManager, Analyzer analyzer) {
+ Iterator iter = cfg.getClassMappings();
+ DirectoryProviderFactory factory = new DirectoryProviderFactory();
+ while (iter.hasNext()) {
+ PersistentClass clazz = (PersistentClass) iter.next();
+ Class<?> mappedClass = clazz.getMappedClass();
+ if (mappedClass != null) {
+ XClass mappedXClass = reflectionManager.toXClass(mappedClass);
+ if (mappedXClass != null && mappedXClass.isAnnotationPresent(Indexed.class)) {
+ DirectoryProvider provider = factory.createDirectoryProvider(mappedXClass, cfg, this);
+ // TODO move that into DirectoryProviderFactory
+ if (!lockableDirectoryProviders.containsKey(provider)) {
+ lockableDirectoryProviders.put(provider, new ReentrantLock());
+ }
+ final DocumentBuilder<Object> documentBuilder = new DocumentBuilder<Object>(mappedXClass, analyzer,
+ provider, reflectionManager);
+
+ documentBuilders.put(mappedClass, documentBuilder);
+ }
+ }
+ }
+ }
+
+ /**
+ * Initilises the Lucene analyzer to use by reading the analyzer class from the configuration and instantiating it.
+ *
+ * @param cfg
+ * The current configuration.
+ * @return The Lucene analyzer to use for tokenisation.
+ */
+ private Analyzer initAnalyzer(Configuration cfg) {
+ Class analyzerClass;
+ String analyzerClassName = cfg.getProperty(Environment.ANALYZER_CLASS);
+ if (analyzerClassName != null) {
+ try {
+ analyzerClass = ReflectHelper.classForName(analyzerClassName);
+ } catch (Exception e) {
+ throw new SearchException("Lucene analyzer class '" + analyzerClassName + "' defined in property '"
+ + Environment.ANALYZER_CLASS + "' could not be found.", e);
+ }
+ } else {
+ analyzerClass = StandardAnalyzer.class;
+ }
+ // Initialize analyzer
+ Analyzer defaultAnalyzer;
+ try {
+ defaultAnalyzer = (Analyzer) analyzerClass.newInstance();
+ } catch (ClassCastException e) {
+ throw new SearchException("Lucene analyzer does not implement " + Analyzer.class.getName() + ": "
+ + analyzerClassName, e);
+ } catch (Exception e) {
+ throw new SearchException("Failed to instantiate lucene analyzer with type " + analyzerClassName, e);
+ }
+ return defaultAnalyzer;
+ }
}
More information about the hibernate-commits
mailing list