6:12 p.m.
Author: epbernard
Date: 2007-06-05 19:12:44 -0400 (Tue, 05 Jun 2007)
New Revision: 11634
Modified:
trunk/HibernateExt/search/src/java/org/hibernate/search/Environment.java
trunk/HibernateExt/search/src/java/org/hibernate/search/backend/WorkQueue.java
trunk/HibernateExt/search/src/java/org/hibernate/search/backend/impl/BatchedQueueingProcessor.java
trunk/HibernateExt/search/src/java/org/hibernate/search/impl/FullTextSessionImpl.java
trunk/HibernateExt/search/src/test/org/hibernate/search/test/session/MassIndexTest.java
Log:
HSEARCH-33 Support hibernate.search.worker.batch_size for massive indexation
Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/Environment.java
===================================================================
--- trunk/HibernateExt/search/src/java/org/hibernate/search/Environment.java 2007-06-05
20:58:24 UTC (rev 11633)
+++ trunk/HibernateExt/search/src/java/org/hibernate/search/Environment.java 2007-06-05
23:12:44 UTC (rev 11634)
@@ -25,6 +25,11 @@
public static final String WORKER_EXECUTION = WORKER_PREFIX + "execution";
/**
+ * Defines the maximum number of indexing operation batched per transaction
+ */
+ public static final String WORKER_BATCHSIZE = WORKER_PREFIX + "batch_size";
+
+ /**
* only used then execution is async
* Thread pool size
* default 1
Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/backend/WorkQueue.java
===================================================================
---
trunk/HibernateExt/search/src/java/org/hibernate/search/backend/WorkQueue.java 2007-06-05
20:58:24 UTC (rev 11633)
+++
trunk/HibernateExt/search/src/java/org/hibernate/search/backend/WorkQueue.java 2007-06-05
23:12:44 UTC (rev 11634)
@@ -19,6 +19,10 @@
queue = new ArrayList<Work>(size);
}
+ private WorkQueue(List<Work> queue) {
+ this.queue = queue;
+ }
+
public WorkQueue() {
this(10);
}
@@ -33,7 +37,13 @@
return queue;
}
+ public WorkQueue splitQueue() {
+ WorkQueue subQueue = new WorkQueue( queue );
+ this.queue = new ArrayList<Work>( queue.size() );
+ return subQueue;
+ }
+
public List<LuceneWork> getSealedQueue() {
if (sealedQueue == null) throw new AssertionFailure("Access a Sealed WorkQueue
whcih has not been sealed");
return sealedQueue;
@@ -49,4 +59,8 @@
queue.clear();
if (sealedQueue != null) sealedQueue.clear();
}
+
+ public int size() {
+ return queue.size();
+ }
}
Modified:
trunk/HibernateExt/search/src/java/org/hibernate/search/backend/impl/BatchedQueueingProcessor.java
===================================================================
---
trunk/HibernateExt/search/src/java/org/hibernate/search/backend/impl/BatchedQueueingProcessor.java 2007-06-05
20:58:24 UTC (rev 11633)
+++
trunk/HibernateExt/search/src/java/org/hibernate/search/backend/impl/BatchedQueueingProcessor.java 2007-06-05
23:12:44 UTC (rev 11634)
@@ -34,6 +34,7 @@
*/
public class BatchedQueueingProcessor implements QueueingProcessor {
private boolean sync;
+ private int batchSize;
private ExecutorService executorService;
private BackendQueueProcessorFactory backendQueueProcessorFactory;
private SearchFactoryImplementor searchFactoryImplementor;
@@ -52,6 +53,9 @@
int queueSize = Integer.parseInt(
properties.getProperty( Environment.WORKER_WORKQUEUE_SIZE, Integer.toString(
Integer.MAX_VALUE ) ).trim()
);
+ batchSize = Integer.parseInt(
+ properties.getProperty( Environment.WORKER_BATCHSIZE, "0" ).trim()
+ );
if ( !sync ) {
/**
* choose min = max with a sizable queue to be able to
@@ -96,6 +100,11 @@
//don't check for builder it's done in performWorks
Work work = new Work(entity, id, workType);
workQueue.add( work );
+ if ( batchSize > 0 && workQueue.size() >= batchSize ) {
+ WorkQueue subQueue = workQueue.splitQueue();
+ prepareWorks( subQueue );
+ performWorks( subQueue );
+ }
}
Modified:
trunk/HibernateExt/search/src/java/org/hibernate/search/impl/FullTextSessionImpl.java
===================================================================
---
trunk/HibernateExt/search/src/java/org/hibernate/search/impl/FullTextSessionImpl.java 2007-06-05
20:58:24 UTC (rev 11633)
+++
trunk/HibernateExt/search/src/java/org/hibernate/search/impl/FullTextSessionImpl.java 2007-06-05
23:12:44 UTC (rev 11634)
@@ -89,7 +89,7 @@
if (entity == null) return;
Class clazz = Hibernate.getClass( entity );
//TODO cache that at the FTSession level
- SearchFactoryImplementor searchFactoryImplementor = ContextHelper.getSearchFactory(
session );
+ SearchFactoryImplementor searchFactoryImplementor = getSearchFactoryImplementor();
//not strictly necessary but a small optimization
DocumentBuilder<Object> builder =
searchFactoryImplementor.getDocumentBuilders().get( clazz );
if ( builder != null ) {
@@ -112,6 +112,13 @@
return searchFactory;
}
+ private SearchFactoryImplementor getSearchFactoryImplementor() {
+ if (searchFactory == null) {
+ searchFactory = ContextHelper.getSearchFactory( session );
+ }
+ return searchFactory;
+ }
+
public Query createSQLQuery(String sql, String returnAlias, Class returnClass) {
return session.createSQLQuery( sql, returnAlias, returnClass );
}
Modified:
trunk/HibernateExt/search/src/test/org/hibernate/search/test/session/MassIndexTest.java
===================================================================
---
trunk/HibernateExt/search/src/test/org/hibernate/search/test/session/MassIndexTest.java 2007-06-05
20:58:24 UTC (rev 11633)
+++
trunk/HibernateExt/search/src/test/org/hibernate/search/test/session/MassIndexTest.java 2007-06-05
23:12:44 UTC (rev 11634)
@@ -3,14 +3,19 @@
import java.util.List;
import java.sql.ResultSet;
+import java.io.File;
import org.hibernate.search.test.SearchTestCase;
import org.hibernate.search.impl.FullTextSessionImpl;
import org.hibernate.search.FullTextSession;
import org.hibernate.search.Search;
+import org.hibernate.search.Environment;
+import org.hibernate.search.store.FSDirectoryProvider;
import org.hibernate.search.util.ContextHelper;
import org.hibernate.Transaction;
import org.hibernate.Session;
+import org.hibernate.ScrollMode;
+import org.hibernate.ScrollableResults;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.analysis.StopAnalyzer;
import org.apache.lucene.store.Directory;
@@ -26,6 +31,40 @@
*/
public class MassIndexTest extends SearchTestCase {
+ public void testBatchSize() throws Exception {
+ FullTextSession s = Search.createFullTextSession( openSession() );
+ Transaction tx = s.beginTransaction();
+ int loop = 14;
+ for (int i = 0 ; i < loop; i++) {
+ s.connection().createStatement().executeUpdate( "insert into Email(id, title,
body, header) values( + "
+ + (i+1) + ", 'Bob Sponge', 'Meet the guys who create the
software', 'nope')");
+ }
+ tx.commit();
+ s.close();
+
+ //check non created object does get found!!1
+ s = new FullTextSessionImpl( openSession() );
+ tx = s.beginTransaction();
+ ScrollableResults results = s.createCriteria( Email.class ).scroll(
ScrollMode.FORWARD_ONLY );
+ int index = 0;
+ while( results.next() ) {
+ index++;
+ s.index( results.get(0) );
+ if (index % 5 == 0) s.clear();
+ }
+ tx.commit();
+ s.clear();
+ tx = s.beginTransaction();
+ QueryParser parser = new QueryParser("id", new StopAnalyzer() );
+ List result = s.createFullTextQuery( parser.parse( "body:create" ) ).list();
+ assertEquals( 14, result.size() );
+ for (Object object : result) {
+ s.delete( object );
+ }
+ tx.commit();
+ s.close();
+ }
+
public void testTransactional() throws Exception {
FullTextSession s = Search.createFullTextSession( openSession() );
Transaction tx = s.beginTransaction();
@@ -127,6 +166,12 @@
s.close();
}
+ protected void configure(org.hibernate.cfg.Configuration cfg) {
+ super.configure( cfg );
+ cfg.setProperty( "hibernate.search.worker.batch_size", "5" );
+ cfg.setProperty( Environment.ANALYZER_CLASS, StopAnalyzer.class.getName() );
+ }
+
protected Class[] getMappings() {
return new Class[] {
Email.class,