Author: epbernard Date: 2006-11-01 23:22:27 -0500 (Wed, 01 Nov 2006) New Revision: 10682 Added: branches/Lucene_Integration/HibernateExt/metadata/src/java/org/hibernate/lucene/util/ContextHelper.java branches/Lucene_Integration/HibernateExt/metadata/src/test/org/hibernate/lucene/test/session/ branches/Lucene_Integration/HibernateExt/metadata/src/test/org/hibernate/lucene/test/session/Email.java branches/Lucene_Integration/HibernateExt/metadata/src/test/org/hibernate/lucene/test/session/MassIndexTest.java Modified: branches/Lucene_Integration/HibernateExt/metadata/src/java/org/hibernate/lucene/LuceneSession.java branches/Lucene_Integration/HibernateExt/metadata/src/java/org/hibernate/lucene/backend/impl/PostTransactionWorkQueueSynchronization.java branches/Lucene_Integration/HibernateExt/metadata/src/java/org/hibernate/lucene/event/LuceneEventListener.java branches/Lucene_Integration/HibernateExt/metadata/src/java/org/hibernate/lucene/query/LuceneQueryImpl.java Log: ANN-384 enable (re)indexing, batch per transaction Modified: branches/Lucene_Integration/HibernateExt/metadata/src/java/org/hibernate/lucene/LuceneSession.java =================================================================== --- branches/Lucene_Integration/HibernateExt/metadata/src/java/org/hibernate/lucene/LuceneSession.java 2006-11-01 23:06:05 UTC (rev 10681) +++ branches/Lucene_Integration/HibernateExt/metadata/src/java/org/hibernate/lucene/LuceneSession.java 2006-11-02 04:22:27 UTC (rev 10682) @@ -6,7 +6,11 @@ import java.util.Collection; import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.concurrent.locks.ReentrantLock; +import javax.transaction.Status; + import org.hibernate.CacheMode; import org.hibernate.Criteria; import org.hibernate.EntityMode; @@ -20,12 +24,22 @@ import org.hibernate.Session; import org.hibernate.SessionFactory; import org.hibernate.Transaction; -import org.hibernate.cfg.NotYetImplementedException; +import org.hibernate.event.PostInsertEventListener; import org.hibernate.engine.query.ParameterMetadata; import org.hibernate.impl.SessionImpl; import org.hibernate.lucene.query.LuceneQueryImpl; +import org.hibernate.lucene.event.LuceneEventListener; +import org.hibernate.lucene.util.ContextHelper; +import org.hibernate.lucene.engine.DocumentBuilder; +import org.hibernate.lucene.backend.UpdateWork; +import org.hibernate.lucene.backend.Work; +import org.hibernate.lucene.backend.WorkQueue; +import org.hibernate.lucene.backend.impl.BatchLuceneWorkQueue; +import org.hibernate.lucene.backend.impl.PostTransactionWorkQueueSynchronization; +import org.hibernate.lucene.store.DirectoryProvider; import org.hibernate.stat.SessionStatistics; import org.hibernate.type.Type; +import org.apache.lucene.document.Document; /** * Lucene aware session that allows lucene query creations @@ -34,6 +48,7 @@ */ public class LuceneSession implements Session { private final SessionImpl session; + private PostTransactionWorkQueueSynchronization postTransactionWorkQueueSynch; public LuceneSession(Session session) { this.session = (SessionImpl) session; @@ -49,8 +64,24 @@ return new LuceneQueryImpl( luceneQuery, entities, session, new ParameterMetadata(null, null) ); } - public void index(Object object) { - throw new NotYetImplementedException(""); + /** + * (re)index an entity. + * Non indexable entities are ignored + * The entity must be associated with the session + * + * @param entity must not be null + */ + public void index(Object entity) { + if (entity == null) return; + Class clazz = entity.getClass(); + LuceneEventListener listener = ContextHelper.getLuceneEventListener( session ); + DocumentBuilder<Object> builder = listener.getDocumentBuilders().get( clazz ); + if ( builder != null ) { + Serializable id = session.getIdentifier( entity ); + Document doc = builder.getDocument( entity, id ); + UpdateWork work = new UpdateWork( id, entity.getClass(), doc ); + processWork( work, listener.getDocumentBuilders(), listener.getLockableDirectoryProviders() ); + } //TODO //need to add elements in a queue kept at the Session level //the queue will be processed by a Lucene(Auto)FlushEventListener @@ -60,6 +91,31 @@ // this is an open discussion } + private void processWork(Work work, Map<Class, DocumentBuilder<Object>> documentBuilders, + Map<DirectoryProvider, ReentrantLock> lockableDirectoryProviders) { + if ( session.isTransactionInProgress() ) { + if ( postTransactionWorkQueueSynch == null || postTransactionWorkQueueSynch.isConsumed() ) { + postTransactionWorkQueueSynch = createWorkQueueSync( documentBuilders, lockableDirectoryProviders); + session.getTransaction().registerSynchronization( postTransactionWorkQueueSynch ); + } + postTransactionWorkQueueSynch.add( work ); + } + else { + //no transaction work right away + PostTransactionWorkQueueSynchronization sync = + createWorkQueueSync( documentBuilders, lockableDirectoryProviders ); + sync.add( work ); + sync.afterCompletion( Status.STATUS_COMMITTED ); + } + } + + private PostTransactionWorkQueueSynchronization createWorkQueueSync( + Map<Class, DocumentBuilder<Object>> documentBuilders, + Map<DirectoryProvider, ReentrantLock> lockableDirectoryProviders) { + WorkQueue workQueue = new BatchLuceneWorkQueue( documentBuilders, lockableDirectoryProviders ); + return new PostTransactionWorkQueueSynchronization( workQueue ); + } + public Query createSQLQuery(String sql, String returnAlias, Class returnClass) { return session.createSQLQuery( sql, returnAlias, returnClass ); } Modified: branches/Lucene_Integration/HibernateExt/metadata/src/java/org/hibernate/lucene/backend/impl/PostTransactionWorkQueueSynchronization.java =================================================================== --- branches/Lucene_Integration/HibernateExt/metadata/src/java/org/hibernate/lucene/backend/impl/PostTransactionWorkQueueSynchronization.java 2006-11-01 23:06:05 UTC (rev 10681) +++ branches/Lucene_Integration/HibernateExt/metadata/src/java/org/hibernate/lucene/backend/impl/PostTransactionWorkQueueSynchronization.java 2006-11-02 04:22:27 UTC (rev 10682) @@ -5,6 +5,7 @@ import javax.transaction.Synchronization; import org.hibernate.lucene.backend.WorkQueue; +import org.hibernate.lucene.backend.Work; /** * Execute some work inside a transaction sychronization @@ -13,24 +14,34 @@ */ public class PostTransactionWorkQueueSynchronization implements Synchronization { private WorkQueue workQueue; + private boolean consumed; public PostTransactionWorkQueueSynchronization(WorkQueue workQueue) { this.workQueue = workQueue; } + public void add(Work work) { + workQueue.add( work ); + } + + public boolean isConsumed() { + return consumed; + } + public void beforeCompletion() { } public void afterCompletion(int i) { - if ( Status.STATUS_COMMITTED == i ) { - workQueue.performWork(); + try { + if ( Status.STATUS_COMMITTED == i ) { + workQueue.performWork(); + } + else { + workQueue.cancelWork(); + } } - else { - workQueue.cancelWork(); + finally { + consumed = true; } } - - public static interface Work { - void perform(); - } } Modified: branches/Lucene_Integration/HibernateExt/metadata/src/java/org/hibernate/lucene/event/LuceneEventListener.java =================================================================== --- branches/Lucene_Integration/HibernateExt/metadata/src/java/org/hibernate/lucene/event/LuceneEventListener.java 2006-11-01 23:06:05 UTC (rev 10681) +++ branches/Lucene_Integration/HibernateExt/metadata/src/java/org/hibernate/lucene/event/LuceneEventListener.java 2006-11-02 04:22:27 UTC (rev 10682) @@ -173,4 +173,8 @@ sync.afterCompletion( Status.STATUS_COMMITTED ); } } + + public Map<DirectoryProvider, ReentrantLock> getLockableDirectoryProviders() { + return lockableDirectoryProviders; + } } Modified: branches/Lucene_Integration/HibernateExt/metadata/src/java/org/hibernate/lucene/query/LuceneQueryImpl.java =================================================================== --- branches/Lucene_Integration/HibernateExt/metadata/src/java/org/hibernate/lucene/query/LuceneQueryImpl.java 2006-11-01 23:06:05 UTC (rev 10681) +++ branches/Lucene_Integration/HibernateExt/metadata/src/java/org/hibernate/lucene/query/LuceneQueryImpl.java 2006-11-02 04:22:27 UTC (rev 10682) @@ -36,6 +36,7 @@ import org.hibernate.impl.AbstractQueryImpl; import org.hibernate.lucene.engine.DocumentBuilder; import org.hibernate.lucene.event.LuceneEventListener; +import org.hibernate.lucene.util.ContextHelper; /** * @author Emmanuel Bernard @@ -71,7 +72,7 @@ //user stop using it //scrollable is better in this area - LuceneEventListener listener = getLuceneEventListener(); + LuceneEventListener listener = ContextHelper.getLuceneEventListener( session ); //find the directories Searcher searcher = buildSearcher( listener ); Hits hits; @@ -106,7 +107,7 @@ public ScrollableResults scroll() throws HibernateException { //keep the searcher open until the resultset is closed - LuceneEventListener listener = getLuceneEventListener(); + LuceneEventListener listener = ContextHelper.getLuceneEventListener( session );; //find the directories Searcher searcher = buildSearcher( listener ); Hits hits; @@ -135,13 +136,12 @@ } public List list() throws HibernateException { - LuceneEventListener listener = getLuceneEventListener(); + LuceneEventListener listener = ContextHelper.getLuceneEventListener( session );; //find the directories Searcher searcher = buildSearcher( listener ); Hits hits; try { org.apache.lucene.search.Query query = filterQueryByClasses( luceneQuery ); - log.warn( query.toString() ); hits = searcher.search( query ); setResultSize( hits ); int first = first(); @@ -284,20 +284,6 @@ return this; } - private LuceneEventListener getLuceneEventListener() { - PostInsertEventListener[] listeners = session.getListeners().getPostInsertEventListeners(); - LuceneEventListener listener = null; - //FIXME this sucks since we mandante the event listener use - for ( PostInsertEventListener candidate : listeners ) { - if ( candidate instanceof LuceneEventListener ) { - listener = (LuceneEventListener) candidate; - break; - } - } - if ( listener == null ) throw new HibernateException( "Lucene event listener not initialized" ); - return listener; - } - public int executeUpdate() throws HibernateException { throw new HibernateException( "Not supported operation" ); } Added: branches/Lucene_Integration/HibernateExt/metadata/src/java/org/hibernate/lucene/util/ContextHelper.java =================================================================== --- branches/Lucene_Integration/HibernateExt/metadata/src/java/org/hibernate/lucene/util/ContextHelper.java 2006-11-01 23:06:05 UTC (rev 10681) +++ branches/Lucene_Integration/HibernateExt/metadata/src/java/org/hibernate/lucene/util/ContextHelper.java 2006-11-02 04:22:27 UTC (rev 10682) @@ -0,0 +1,27 @@ +//$Id: $ +package org.hibernate.lucene.util; + +import org.hibernate.HibernateException; +import org.hibernate.engine.SessionImplementor; +import org.hibernate.event.PostInsertEventListener; +import org.hibernate.lucene.event.LuceneEventListener; + +/** + * @author Emmanuel Bernard + */ +public abstract class ContextHelper { + + public static LuceneEventListener getLuceneEventListener(SessionImplementor session) { + PostInsertEventListener[] listeners = session.getListeners().getPostInsertEventListeners(); + LuceneEventListener listener = null; + //FIXME this sucks since we mandante the event listener use + for ( PostInsertEventListener candidate : listeners ) { + if ( candidate instanceof LuceneEventListener ) { + listener = (LuceneEventListener) candidate; + break; + } + } + if ( listener == null ) throw new HibernateException( "Lucene event listener not initialized" ); + return listener; + } +} Added: branches/Lucene_Integration/HibernateExt/metadata/src/test/org/hibernate/lucene/test/session/Email.java =================================================================== --- branches/Lucene_Integration/HibernateExt/metadata/src/test/org/hibernate/lucene/test/session/Email.java 2006-11-01 23:06:05 UTC (rev 10681) +++ branches/Lucene_Integration/HibernateExt/metadata/src/test/org/hibernate/lucene/test/session/Email.java 2006-11-02 04:22:27 UTC (rev 10682) @@ -0,0 +1,64 @@ +//$Id: $ +package org.hibernate.lucene.test.session; + +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.GeneratedValue; + +import org.hibernate.lucene.annotations.Indexed; +import org.hibernate.lucene.annotations.DocumentId; +import org.hibernate.lucene.annotations.Field; +import org.hibernate.lucene.annotations.Index; + +/** + * @author Emmanuel Bernard + */ +@Entity +@Indexed +public class Email { + @Id + @GeneratedValue + @DocumentId + private Long id; + + @Field(index = Index.TOKENIZED) + private String title; + + @Field(index = Index.TOKENIZED) + private String body; + + private String header; + + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public String getBody() { + return body; + } + + public void setBody(String body) { + this.body = body; + } + + public String getHeader() { + return header; + } + + public void setHeader(String header) { + this.header = header; + } +} Added: branches/Lucene_Integration/HibernateExt/metadata/src/test/org/hibernate/lucene/test/session/MassIndexTest.java =================================================================== --- branches/Lucene_Integration/HibernateExt/metadata/src/test/org/hibernate/lucene/test/session/MassIndexTest.java 2006-11-01 23:06:05 UTC (rev 10681) +++ branches/Lucene_Integration/HibernateExt/metadata/src/test/org/hibernate/lucene/test/session/MassIndexTest.java 2006-11-02 04:22:27 UTC (rev 10682) @@ -0,0 +1,68 @@ +//$Id: $ +package org.hibernate.lucene.test.session; + +import java.sql.SQLException; +import java.util.List; + +import org.hibernate.lucene.test.TestCase; +import org.hibernate.lucene.LuceneSession; +import org.hibernate.Session; +import org.hibernate.Transaction; +import org.apache.lucene.queryParser.QueryParser; +import org.apache.lucene.analysis.StopAnalyzer; + +/** + * @author Emmanuel Bernard + */ +public class MassIndexTest extends TestCase { + + public void testTransactional() throws Exception { + LuceneSession s = new LuceneSession( openSession() ); + Transaction tx = s.beginTransaction(); + int loop = 4; + for (int i = 0 ; i < loop; i++) { + Email email = new Email(); + email.setTitle( "JBoss World Berlin" ); + email.setBody( "Meet the guys who wrote the software"); + s.persist( email ); + } + tx.commit(); + s.close(); + + s = new LuceneSession( openSession() ); + s.getTransaction().begin(); + s.connection().createStatement().executeUpdate( "update Email set body='Meet the guys who write the software'"); + s.getTransaction().commit(); + s.close(); + + s = new LuceneSession( openSession() ); + tx = s.beginTransaction(); + QueryParser parser = new QueryParser("id", new StopAnalyzer() ); + List result = s.createLuceneQuery( parser.parse( "body:write" ) ).list(); + assertEquals( 0, result.size() ); + result = s.createCriteria( Email.class ).list(); + for (int i = 0 ; i < loop/2 ; i++) + s.index( result.get( i ) ); + tx.commit(); //do the process + s.index( result.get(loop/2) ); //do the process out of tx + tx = s.beginTransaction(); + for (int i = loop/2+1 ; i < loop; i++) + s.index( result.get( i ) ); + tx.commit(); //do the process + s.close(); + + s = new LuceneSession( openSession() ); + tx = s.beginTransaction(); + result = s.createLuceneQuery( parser.parse( "body:write" ) ).list(); + assertEquals( loop, result.size() ); + for (Object o : result) s.delete( o ); + tx.commit(); + s.close(); + } + + protected Class[] getMappings() { + return new Class[] { + Email.class + }; + } +}