Author: epbernard Date: 2007-07-29 10:56:47 -0400 (Sun, 29 Jul 2007) New Revision: 12844 Added: trunk/HibernateExt/search/src/java/org/hibernate/search/store/DirectoryProviderShardingStrategy.java trunk/HibernateExt/search/src/java/org/hibernate/search/store/IdHashShardingStrategy.java trunk/HibernateExt/search/src/java/org/hibernate/search/store/NotShardedStrategy.java trunk/HibernateExt/search/src/test/org/hibernate/search/test/shards/ trunk/HibernateExt/search/src/test/org/hibernate/search/test/shards/Animal.java trunk/HibernateExt/search/src/test/org/hibernate/search/test/shards/ShardsTest.java Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/SearchFactory.java trunk/HibernateExt/search/src/java/org/hibernate/search/backend/AddLuceneWork.java trunk/HibernateExt/search/src/java/org/hibernate/search/backend/DeleteLuceneWork.java trunk/HibernateExt/search/src/java/org/hibernate/search/backend/LuceneWork.java trunk/HibernateExt/search/src/java/org/hibernate/search/backend/OptimizeLuceneWork.java trunk/HibernateExt/search/src/java/org/hibernate/search/backend/Workspace.java trunk/HibernateExt/search/src/java/org/hibernate/search/backend/impl/lucene/LuceneBackendQueueProcessor.java trunk/HibernateExt/search/src/java/org/hibernate/search/backend/impl/lucene/LuceneWorker.java trunk/HibernateExt/search/src/java/org/hibernate/search/engine/DocumentBuilder.java trunk/HibernateExt/search/src/java/org/hibernate/search/impl/SearchFactoryImpl.java trunk/HibernateExt/search/src/java/org/hibernate/search/query/FullTextQueryImpl.java trunk/HibernateExt/search/src/java/org/hibernate/search/store/DirectoryProviderFactory.java trunk/HibernateExt/search/src/test/org/hibernate/search/test/SearchTestCase.java trunk/HibernateExt/search/src/test/org/hibernate/search/test/jms/master/JMSMasterTest.java Log: HSEARCH-86 introduce Index Sharding, including sharding strategy Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/SearchFactory.java =================================================================== --- trunk/HibernateExt/search/src/java/org/hibernate/search/SearchFactory.java 2007-07-29 14:12:13 UTC (rev 12843) +++ trunk/HibernateExt/search/src/java/org/hibernate/search/SearchFactory.java 2007-07-29 14:56:47 UTC (rev 12844) @@ -19,7 +19,7 @@ * Provide access to the DirectoryProvider (hence the Lucene Directory) * for a given entity */ - DirectoryProvider getDirectoryProvider(Class entity); + DirectoryProvider[] getDirectoryProviders(Class entity); /** * Optimize all indexes Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/backend/AddLuceneWork.java =================================================================== --- trunk/HibernateExt/search/src/java/org/hibernate/search/backend/AddLuceneWork.java 2007-07-29 14:12:13 UTC (rev 12843) +++ trunk/HibernateExt/search/src/java/org/hibernate/search/backend/AddLuceneWork.java 2007-07-29 14:56:47 UTC (rev 12844) @@ -9,7 +9,7 @@ * @author Emmanuel Bernard */ public class AddLuceneWork extends LuceneWork { - public AddLuceneWork(Serializable id, Class entity, Document document) { - super( id, entity, document ); + public AddLuceneWork(Serializable id, String idInString, Class entity, Document document) { + super( id, idInString, entity, document ); } } Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/backend/DeleteLuceneWork.java =================================================================== --- trunk/HibernateExt/search/src/java/org/hibernate/search/backend/DeleteLuceneWork.java 2007-07-29 14:12:13 UTC (rev 12843) +++ trunk/HibernateExt/search/src/java/org/hibernate/search/backend/DeleteLuceneWork.java 2007-07-29 14:56:47 UTC (rev 12844) @@ -7,7 +7,7 @@ * @author Emmanuel Bernard */ public class DeleteLuceneWork extends LuceneWork { - public DeleteLuceneWork(Serializable id, Class entity) { - super( id, entity ); + public DeleteLuceneWork(Serializable id, String idInString, Class entity) { + super( id, idInString, entity ); } } Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/backend/LuceneWork.java =================================================================== --- trunk/HibernateExt/search/src/java/org/hibernate/search/backend/LuceneWork.java 2007-07-29 14:12:13 UTC (rev 12843) +++ trunk/HibernateExt/search/src/java/org/hibernate/search/backend/LuceneWork.java 2007-07-29 14:56:47 UTC (rev 12844) @@ -21,13 +21,15 @@ * Flag indicating if this lucene work has to be indexed in batch mode. */ private boolean batch = false; + private String idInString; - public LuceneWork(Serializable id, Class entity) { - this( id, entity, null ); + public LuceneWork(Serializable id, String idInString, Class entity) { + this( id, idInString, entity, null ); } - public LuceneWork(Serializable id, Class entity, Document document) { + public LuceneWork(Serializable id, String idInString, Class entity, Document document) { this.id = id; + this.idInString = idInString; this.entityClass = entity; this.document = document; } @@ -51,4 +53,8 @@ public Serializable getId() { return id; } + + public String getIdInString() { + return idInString; + } } Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/backend/OptimizeLuceneWork.java =================================================================== --- trunk/HibernateExt/search/src/java/org/hibernate/search/backend/OptimizeLuceneWork.java 2007-07-29 14:12:13 UTC (rev 12843) +++ trunk/HibernateExt/search/src/java/org/hibernate/search/backend/OptimizeLuceneWork.java 2007-07-29 14:56:47 UTC (rev 12844) @@ -9,6 +9,6 @@ */ public class OptimizeLuceneWork extends LuceneWork { public OptimizeLuceneWork(Class entity) { - super( null, entity ); + super( null, null, entity ); } } Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/backend/Workspace.java =================================================================== --- trunk/HibernateExt/search/src/java/org/hibernate/search/backend/Workspace.java 2007-07-29 14:12:13 UTC (rev 12843) +++ trunk/HibernateExt/search/src/java/org/hibernate/search/backend/Workspace.java 2007-07-29 14:56:47 UTC (rev 12844) @@ -2,10 +2,13 @@ package org.hibernate.search.backend; import java.io.IOException; +import java.io.Serializable; import java.util.HashMap; import java.util.Map; import java.util.ArrayList; import java.util.List; +import java.util.Set; +import java.util.HashSet; import java.util.concurrent.locks.ReentrantLock; import org.apache.commons.logging.Log; @@ -14,6 +17,7 @@ import org.apache.lucene.index.IndexWriter; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.SimpleAnalyzer; +import org.apache.lucene.document.Document; import org.hibernate.search.engine.DocumentBuilder; import org.hibernate.search.engine.SearchFactoryImplementor; import org.hibernate.search.SearchException; @@ -65,18 +69,13 @@ } /** - * Retrieve an IndexWriter for a given entity - * - * @param entity entity processed - * @param modificationOperation true if the modification is expected to be an idnex state change - * @return appropriate indexWriter + * retrieve a read write IndexReader + * For a given DirectoryProvider, An IndexReader must be used before an IndexWriter */ - public IndexReader getIndexReader(Class entity, boolean modificationOperation) { - //TODO NPEs - DirectoryProvider provider = searchFactoryImplementor.getDirectoryProvider( entity ); + public IndexReader getIndexReader(DirectoryProvider provider, Class entity) { //one cannot access a reader for update after a writer has been accessed if ( writers.containsKey( provider ) ) - throw new AssertionFailure("Tries to read for update a index while a writer is accessed" + entity); + throw new AssertionFailure("Tries to read for update an index while a writer is accessed" + entity); IndexReader reader = readers.get( provider ); if ( reader != null ) return reader; lockProvider( provider ); @@ -91,24 +90,16 @@ return reader; } - /** - * Retrieve an IndexWriter for a given entity - * - * @param entity entity processed - * @param modificationOperation true if the modification is expected to be an idnex state change - * @return appropriate indexWriter - */ - public IndexWriter getIndexWriter(Class entity, boolean modificationOperation) { - DirectoryProvider provider = searchFactoryImplementor.getDirectoryProvider( entity ); - return getIndexWriter( provider, entity, modificationOperation ); - } - //for optimization public IndexWriter getIndexWriter(DirectoryProvider provider) { return getIndexWriter( provider, null, false ); } - private IndexWriter getIndexWriter(DirectoryProvider provider, Class entity, boolean modificationOperation) { + /** + * retrieve a read write IndexWriter + * For a given DirectoryProvider, An IndexReader must be used before an IndexWriter + */ + public IndexWriter getIndexWriter(DirectoryProvider provider, Class entity, boolean modificationOperation) { //one has to close a reader for update before a writer is accessed IndexReader reader = readers.get( provider ); if ( reader != null ) { @@ -123,7 +114,7 @@ IndexWriter writer = writers.get( provider ); if ( writer != null ) return writer; lockProvider( provider ); - dpStatistics.get(provider).operations++; + if (modificationOperation) dpStatistics.get(provider).operations++; try { Analyzer analyzer = entity != null ? searchFactoryImplementor.getDocumentBuilders().get( entity ).getAnalyzer() : @@ -231,7 +222,15 @@ } public void optimize(Class entity) { - DirectoryProvider provider = searchFactoryImplementor.getDirectoryProvider( entity ); + DocumentBuilder documentBuilders = searchFactoryImplementor.getDocumentBuilders().get( entity ); + DirectoryProvider[] providers = documentBuilders.getDirectoryProviderSelectionStrategy() + .getDirectoryProvidersForOptimization( entity ); + for (DirectoryProvider provider: providers) { + optimize( provider ); + } + } + + private void optimize(DirectoryProvider provider) { OptimizerStrategy optimizerStrategy = searchFactoryImplementor.getOptimizerStrategy( provider ); dpStatistics.get(provider).optimizationForced = true; optimizerStrategy.optimizationForced(); Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/backend/impl/lucene/LuceneBackendQueueProcessor.java =================================================================== --- trunk/HibernateExt/search/src/java/org/hibernate/search/backend/impl/lucene/LuceneBackendQueueProcessor.java 2007-07-29 14:12:13 UTC (rev 12843) +++ trunk/HibernateExt/search/src/java/org/hibernate/search/backend/impl/lucene/LuceneBackendQueueProcessor.java 2007-07-29 14:56:47 UTC (rev 12844) @@ -4,6 +4,7 @@ import java.util.Collections; import java.util.Comparator; import java.util.List; +import java.util.ArrayList; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -11,8 +12,12 @@ import org.hibernate.search.backend.LuceneWork; import org.hibernate.search.backend.Workspace; import org.hibernate.search.backend.OptimizeLuceneWork; +import org.hibernate.search.backend.DeleteLuceneWork; import org.hibernate.search.engine.SearchFactoryImplementor; +import org.hibernate.search.engine.DocumentBuilder; import org.hibernate.search.store.DirectoryProvider; +import org.hibernate.search.store.DirectoryProviderShardingStrategy; +import org.hibernate.annotations.common.AssertionFailure; /** * Apply the operations to Lucene directories avoiding deadlocks. @@ -41,9 +46,45 @@ workspace = new Workspace( searchFactoryImplementor ); worker = new LuceneWorker( workspace ); try { - deadlockFreeQueue(queue, workspace, searchFactoryImplementor); + List<LuceneWorker.WorkWithPayload> queueWithFlatDPs = new ArrayList<LuceneWorker.WorkWithPayload>( queue.size()*2 ); + for ( LuceneWork work : queue ) { + DocumentBuilder documentBuilder = searchFactoryImplementor.getDocumentBuilders().get( work.getEntityClass() ); + DirectoryProviderShardingStrategy shardingStrategy = documentBuilder.getDirectoryProviderSelectionStrategy(); + + if ( AddLuceneWork.class.isAssignableFrom( work.getClass() ) ) { + DirectoryProvider provider = shardingStrategy.getDirectoryProviderForAddition( + work.getEntityClass(), + work.getId(), + work.getIdInString(), + work.getDocument() + ); + queueWithFlatDPs.add( new LuceneWorker.WorkWithPayload(work, provider) ); + } + else if ( DeleteLuceneWork.class.isAssignableFrom( work.getClass() ) ) { + DirectoryProvider[] providers = shardingStrategy.getDirectoryProvidersForDeletion( + work.getEntityClass(), + work.getId(), + work.getIdInString() + ); + for (DirectoryProvider provider : providers) { + queueWithFlatDPs.add( new LuceneWorker.WorkWithPayload(work, provider) ); + } + } + else if ( OptimizeLuceneWork.class.isAssignableFrom( work.getClass() ) ) { + DirectoryProvider[] providers = shardingStrategy.getDirectoryProvidersForOptimization( + work.getEntityClass() + ); + for (DirectoryProvider provider : providers) { + queueWithFlatDPs.add( new LuceneWorker.WorkWithPayload(work, provider) ); + } + } + else { + throw new AssertionFailure( "Unknown work type: " + work.getClass() ); + } + } + deadlockFreeQueue(queueWithFlatDPs, searchFactoryImplementor); checkForBatchIndexing(workspace); - for ( LuceneWork luceneWork : queue ) { + for ( LuceneWorker.WorkWithPayload luceneWork : queueWithFlatDPs ) { worker.performWork( luceneWork ); } } @@ -57,8 +98,9 @@ for ( LuceneWork luceneWork : queue ) { // if there is at least a single batch index job we put the work space into batch indexing mode. if(luceneWork.isBatch()){ - log.debug("Setting batch indexing mode."); + log.trace("Setting batch indexing mode."); workspace.setBatch(true); + break; } } } @@ -68,9 +110,9 @@ * dead lock between concurrent threads or processes * To achieve that, the work will be done per directory provider */ - private void deadlockFreeQueue(List<LuceneWork> queue, final Workspace workspace, final SearchFactoryImplementor searchFactoryImplementor) { - Collections.sort( queue, new Comparator<LuceneWork>() { - public int compare(LuceneWork o1, LuceneWork o2) { + private void deadlockFreeQueue(List<LuceneWorker.WorkWithPayload> queue, final SearchFactoryImplementor searchFactoryImplementor) { + Collections.sort( queue, new Comparator<LuceneWorker.WorkWithPayload>() { + public int compare(LuceneWorker.WorkWithPayload o1, LuceneWorker.WorkWithPayload o2) { long h1 = getWorkHashCode( o1, searchFactoryImplementor ); long h2 = getWorkHashCode( o2, searchFactoryImplementor ); return h1 < h2 ? @@ -82,14 +124,13 @@ } ); } - private long getWorkHashCode(LuceneWork luceneWork, SearchFactoryImplementor searchFactoryImplementor) { - Class entity = luceneWork.getEntityClass(); - DirectoryProvider provider = searchFactoryImplementor.getDirectoryProvider( entity ); + private long getWorkHashCode(LuceneWorker.WorkWithPayload luceneWork, SearchFactoryImplementor searchFactoryImplementor) { + DirectoryProvider provider = luceneWork.getProvider(); int h = provider.getClass().hashCode(); h = 31 * h + provider.hashCode(); long extendedHash = h; //to be sure extendedHash + 1 < extendedHash + 2 is always true - if ( luceneWork instanceof AddLuceneWork ) extendedHash+=1; //addwork after deleteWork - if ( luceneWork instanceof OptimizeLuceneWork ) extendedHash+=2; //optimize after everything + if ( luceneWork.getWork() instanceof AddLuceneWork ) extendedHash+=1; //addwork after deleteWork + if ( luceneWork.getWork() instanceof OptimizeLuceneWork ) extendedHash+=2; //optimize after everything return extendedHash; } } Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/backend/impl/lucene/LuceneWorker.java =================================================================== --- trunk/HibernateExt/search/src/java/org/hibernate/search/backend/impl/lucene/LuceneWorker.java 2007-07-29 14:12:13 UTC (rev 12843) +++ trunk/HibernateExt/search/src/java/org/hibernate/search/backend/impl/lucene/LuceneWorker.java 2007-07-29 14:56:47 UTC (rev 12844) @@ -13,6 +13,7 @@ import org.apache.lucene.index.TermDocs; import org.hibernate.annotations.common.AssertionFailure; import org.hibernate.search.SearchException; +import org.hibernate.search.store.DirectoryProvider; import org.hibernate.search.backend.AddLuceneWork; import org.hibernate.search.backend.DeleteLuceneWork; import org.hibernate.search.backend.LuceneWork; @@ -34,32 +35,33 @@ this.workspace = workspace; } - public void performWork(LuceneWork luceneWork) { - if ( AddLuceneWork.class.isAssignableFrom( luceneWork.getClass() ) ) { - performWork( (AddLuceneWork) luceneWork ); + public void performWork(WorkWithPayload luceneWork) { + Class workClass = luceneWork.getWork().getClass(); + if ( AddLuceneWork.class.isAssignableFrom( workClass ) ) { + performWork( (AddLuceneWork) luceneWork.getWork(), luceneWork.getProvider() ); } - else if ( DeleteLuceneWork.class.isAssignableFrom( luceneWork.getClass() ) ) { - performWork( (DeleteLuceneWork) luceneWork ); + else if ( DeleteLuceneWork.class.isAssignableFrom( workClass ) ) { + performWork( (DeleteLuceneWork) luceneWork.getWork(), luceneWork.getProvider() ); } - else if ( OptimizeLuceneWork.class.isAssignableFrom( luceneWork.getClass() ) ) { - performWork( (OptimizeLuceneWork) luceneWork ); + else if ( OptimizeLuceneWork.class.isAssignableFrom( workClass ) ) { + performWork( (OptimizeLuceneWork) luceneWork.getWork(), luceneWork.getProvider() ); } else { - throw new AssertionFailure( "Unknown work type: " + luceneWork.getClass() ); + throw new AssertionFailure( "Unknown work type: " + workClass ); } } - public void performWork(AddLuceneWork work) { + public void performWork(AddLuceneWork work, DirectoryProvider provider) { Class entity = work.getEntityClass(); Serializable id = work.getId(); Document document = work.getDocument(); - add( entity, id, document ); + add( entity, id, document, provider ); } - private void add(Class entity, Serializable id, Document document) { + private void add(Class entity, Serializable id, Document document, DirectoryProvider provider) { if ( log.isTraceEnabled() ) log.trace( "add to Lucene index: " + entity + "#" + id + ": " + document ); - IndexWriter writer = workspace.getIndexWriter( entity, true ); + IndexWriter writer = workspace.getIndexWriter( provider, entity, true ); try { writer.addDocument( document ); } @@ -68,22 +70,22 @@ } } - public void performWork(DeleteLuceneWork work) { + public void performWork(DeleteLuceneWork work, DirectoryProvider provider) { Class entity = work.getEntityClass(); Serializable id = work.getId(); - remove( entity, id ); + remove( entity, id, provider ); } - private void remove(Class entity, Serializable id) { + private void remove(Class entity, Serializable id, DirectoryProvider provider) { /** - * even with Lucene 2.1, use of indexWriter to delte is not an option + * even with Lucene 2.1, use of indexWriter to delete is not an option * We can only delete by term, and the index doesn't have a termt that * uniquely identify the entry. See logic below */ log.trace( "remove from Lucene index: " + entity + "#" + id ); DocumentBuilder builder = workspace.getDocumentBuilder( entity ); Term term = builder.getTerm( id ); - IndexReader reader = workspace.getIndexReader( entity, true ); + IndexReader reader = workspace.getIndexReader( provider, entity ); TermDocs termDocs = null; try { //TODO is there a faster way? @@ -110,15 +112,14 @@ log.warn( "Unable to close termDocs properly", e); } } + } - public void performWork(OptimizeLuceneWork work) { + public void performWork(OptimizeLuceneWork work, DirectoryProvider provider) { Class entity = work.getEntityClass(); if ( log.isTraceEnabled() ) log.trace( "optimize Lucene index: " + entity ); - //TODO get a batchIndexWriter of some kind that deals with different merge factors and all - //TODO this one should not compete with a regular IndexWriter (ie exception from the workspace) - IndexWriter writer = workspace.getIndexWriter( entity, false ); + IndexWriter writer = workspace.getIndexWriter( provider, entity, false ); try { writer.optimize(); workspace.optimize(entity); @@ -127,4 +128,24 @@ throw new SearchException( "Unable to optimize Lucene index: " + entity, e ); } } + + public static class WorkWithPayload { + private LuceneWork work; + private DirectoryProvider provider; + + + public WorkWithPayload(LuceneWork work, DirectoryProvider provider) { + this.work = work; + this.provider = provider; + } + + + public LuceneWork getWork() { + return work; + } + + public DirectoryProvider getProvider() { + return provider; + } + } } Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/engine/DocumentBuilder.java =================================================================== --- trunk/HibernateExt/search/src/java/org/hibernate/search/engine/DocumentBuilder.java 2007-07-29 14:12:13 UTC (rev 12843) +++ trunk/HibernateExt/search/src/java/org/hibernate/search/engine/DocumentBuilder.java 2007-07-29 14:56:47 UTC (rev 12844) @@ -44,6 +44,7 @@ import org.hibernate.search.bridge.FieldBridge; import org.hibernate.search.bridge.TwoWayFieldBridge; import org.hibernate.search.store.DirectoryProvider; +import org.hibernate.search.store.DirectoryProviderShardingStrategy; import org.hibernate.search.util.BinderHelper; import org.hibernate.search.util.ScopedAnalyzer; @@ -61,7 +62,8 @@ private final PropertiesMetadata rootPropertiesMetadata; private final XClass beanClass; - private final DirectoryProvider directoryProvider; + private final DirectoryProvider[] directoryProviders; + private final DirectoryProviderShardingStrategy shardingStrategy; private String idKeywordName; private XMember idGetter; private Float idBoost; @@ -74,11 +76,12 @@ private ScopedAnalyzer analyzer; - public DocumentBuilder(XClass clazz, Analyzer defaultAnalyzer, DirectoryProvider directory, - ReflectionManager reflectionManager) { + public DocumentBuilder(XClass clazz, Analyzer defaultAnalyzer, DirectoryProvider[] directoryProviders, + DirectoryProviderShardingStrategy shardingStrategy, ReflectionManager reflectionManager) { this.analyzer = new ScopedAnalyzer(); this.beanClass = clazz; - this.directoryProvider = directory; + this.directoryProviders = directoryProviders; + this.shardingStrategy = shardingStrategy; //FIXME get rid of it when boost is stored? this.reflectionManager = reflectionManager; @@ -387,13 +390,14 @@ } boolean searchForContainers = false; + String idInString = idBridge.objectToString( id ); if ( workType == WorkType.ADD ) { Document doc = getDocument( entity, id ); - queue.add( new AddLuceneWork( id, entityClass, doc ) ); + queue.add( new AddLuceneWork( id, idInString, entityClass, doc ) ); searchForContainers = true; } else if ( workType == WorkType.DELETE ) { - queue.add( new DeleteLuceneWork( id, entityClass ) ); + queue.add( new DeleteLuceneWork( id, idInString, entityClass ) ); } else if ( workType == WorkType.UPDATE ) { Document doc = getDocument( entity, id ); @@ -404,14 +408,14 @@ * But essentially the optimization we are doing is the same Lucene is doing, the only extra cost is the * double file opening. */ - queue.add( new DeleteLuceneWork( id, entityClass ) ); - queue.add( new AddLuceneWork( id, entityClass, doc ) ); + queue.add( new DeleteLuceneWork( id, idInString, entityClass ) ); + queue.add( new AddLuceneWork( id, idInString, entityClass, doc ) ); searchForContainers = true; } else if ( workType == WorkType.INDEX ) { Document doc = getDocument( entity, id ); - queue.add( new DeleteLuceneWork( id, entityClass ) ); - LuceneWork work = new AddLuceneWork( id, entityClass, doc ); + queue.add( new DeleteLuceneWork( id, idInString, entityClass ) ); + LuceneWork work = new AddLuceneWork( id, idInString, entityClass, doc ); work.setBatch( true ); queue.add( work ); searchForContainers = true; @@ -578,10 +582,14 @@ return new Term( idKeywordName, idBridge.objectToString( id ) ); } - public DirectoryProvider getDirectoryProvider() { - return directoryProvider; + public DirectoryProvider[] getDirectoryProviders() { + return directoryProviders; } + public DirectoryProviderShardingStrategy getDirectoryProviderSelectionStrategy() { + return shardingStrategy; + } + public Analyzer getAnalyzer() { return analyzer; } Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/impl/SearchFactoryImpl.java =================================================================== --- trunk/HibernateExt/search/src/java/org/hibernate/search/impl/SearchFactoryImpl.java 2007-07-29 14:12:13 UTC (rev 12843) +++ trunk/HibernateExt/search/src/java/org/hibernate/search/impl/SearchFactoryImpl.java 2007-07-29 14:56:47 UTC (rev 12844) @@ -226,9 +226,9 @@ return reflectionManager; } - public DirectoryProvider getDirectoryProvider(Class entity) { + public DirectoryProvider[] getDirectoryProviders(Class entity) { DocumentBuilder<Object> documentBuilder = getDocumentBuilders().get( entity ); - return documentBuilder == null ? null : documentBuilder.getDirectoryProvider(); + return documentBuilder == null ? null : documentBuilder.getDirectoryProviders(); } public void optimize() { @@ -257,13 +257,11 @@ XClass mappedXClass = reflectionManager.toXClass(mappedClass); if ( mappedXClass != null) { if ( mappedXClass.isAnnotationPresent( Indexed.class ) ) { - DirectoryProvider provider = factory.createDirectoryProvider( mappedXClass, cfg, this ); - //TODO move that into DirectoryProviderFactory - if ( !lockableDirectoryProviders.containsKey( provider ) ) { - lockableDirectoryProviders.put( provider, new ReentrantLock() ); - } + DirectoryProviderFactory.DirectoryProviders providers = factory.createDirectoryProviders( mappedXClass, cfg, this ); + final DocumentBuilder<Object> documentBuilder = new DocumentBuilder<Object>( - mappedXClass, analyzer, provider, reflectionManager + mappedXClass, analyzer, providers.getProviders(), providers.getSelectionStrategy(), + reflectionManager ); documentBuilders.put( mappedClass, documentBuilder ); Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/query/FullTextQueryImpl.java =================================================================== --- trunk/HibernateExt/search/src/java/org/hibernate/search/query/FullTextQueryImpl.java 2007-07-29 14:12:13 UTC (rev 12843) +++ trunk/HibernateExt/search/src/java/org/hibernate/search/query/FullTextQueryImpl.java 2007-07-29 14:56:47 UTC (rev 12844) @@ -407,9 +407,12 @@ if ( classes == null || classes.length == 0 ) { //no class means all classes for (DocumentBuilder builder : builders.values()) { - final DirectoryProvider directoryProvider = builder.getDirectoryProvider(); - if ( !directories.contains( directoryProvider ) ) { - directories.add( directoryProvider ); + final DirectoryProvider[] directoryProviders = + builder.getDirectoryProviderSelectionStrategy().getDirectoryProvidersForQuery(); + for (DirectoryProvider provider : directoryProviders) { + if ( !directories.contains( provider ) ) { + directories.add( provider ); + } } } classesAndSubclasses = null; @@ -426,9 +429,12 @@ //TODO should we rather choose a polymorphic path and allow non mapped entities if ( builder == null ) throw new HibernateException( "Not a mapped entity (don't forget to add @Indexed): " + clazz ); - final DirectoryProvider directoryProvider = builder.getDirectoryProvider(); - if ( !directories.contains( directoryProvider ) ) { - directories.add( directoryProvider ); + final DirectoryProvider[] directoryProviders = + builder.getDirectoryProviderSelectionStrategy().getDirectoryProvidersForQuery(); + for (DirectoryProvider provider : directoryProviders) { + if ( !directories.contains( provider ) ) { + directories.add( provider ); + } } } classesAndSubclasses = involvedClasses; Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/store/DirectoryProviderFactory.java =================================================================== --- trunk/HibernateExt/search/src/java/org/hibernate/search/store/DirectoryProviderFactory.java 2007-07-29 14:12:13 UTC (rev 12843) +++ trunk/HibernateExt/search/src/java/org/hibernate/search/store/DirectoryProviderFactory.java 2007-07-29 14:56:47 UTC (rev 12844) @@ -5,6 +5,8 @@ import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.Set; +import java.util.concurrent.locks.ReentrantLock; import org.hibernate.HibernateException; import org.hibernate.cfg.Configuration; @@ -15,6 +17,7 @@ import org.hibernate.search.store.optimization.OptimizerStrategy; import org.hibernate.search.store.optimization.IncrementalOptimizerStrategy; import org.hibernate.search.store.optimization.NoOpOptimizerStrategy; +import org.hibernate.search.SearchException; import org.hibernate.mapping.PersistentClass; import org.hibernate.annotations.common.reflection.ReflectionManager; import org.hibernate.annotations.common.reflection.XClass; @@ -47,7 +50,7 @@ private static String LUCENE_DEFAULT = LUCENE_PREFIX + "default."; private static String DEFAULT_DIRECTORY_PROVIDER = FSDirectoryProvider.class.getName(); - // Lucene index performance paramters + // Lucene index performance parameters private static final String MERGE_FACTOR = "merge_factor"; private static final String MAX_MERGE_DOCS = "max_merge_docs"; private static final String MAX_BUFFERED_DOCS = "max_buffered_docs"; @@ -55,13 +58,66 @@ private static final String BATCH_MAX_MERGE_DOCS = "max_merge_docs.batch"; private static final String BATCH_MAX_BUFFERED_DOCS = "max_buffered_docs.batch"; - //TODO for the public? - public DirectoryProvider<?> createDirectoryProvider(XClass entity, Configuration cfg, SearchFactoryImplementor searchFactoryImplementor) { + private static final String SHARDING_STRATEGY = "sharding_strategy"; + private static final String NBR_OF_SHARDS = SHARDING_STRATEGY + ".nbr_of_shards"; + + + public DirectoryProviders createDirectoryProviders(XClass entity, Configuration cfg, SearchFactoryImplementor searchFactoryImplementor) { //get properties String directoryProviderName = getDirectoryProviderName( entity, cfg ); - Properties indexProps = getDirectoryProperties( cfg, directoryProviderName ); + Properties[] indexProps = getDirectoryProperties( cfg, directoryProviderName ); - //set up the directory + //set up the directories + int nbrOfProviders = indexProps.length; + DirectoryProvider[] providers = new DirectoryProvider[nbrOfProviders]; + for (int index = 0 ; index < nbrOfProviders ; index++) { + providers[index] = createDirectoryProvider(directoryProviderName + "_" + index,indexProps[index], searchFactoryImplementor); + } + + //define sharding strategy + DirectoryProviderShardingStrategy shardingStrategy; + Properties shardingProperties = new Properties(); + for (Map.Entry entry : indexProps[0].entrySet()) { + if ( ( (String) entry.getKey() ).startsWith( SHARDING_STRATEGY ) ) { + shardingProperties.put( entry.getKey(), entry.getValue() ); + } + } + String shardingStrategyName = shardingProperties.getProperty( SHARDING_STRATEGY ); + if ( shardingStrategyName == null) { + if ( indexProps.length == 1 ) { + shardingStrategy = new NotShardedStrategy(); + } + else { + shardingStrategy = new IdHashShardingStrategy(); + } + } + else { + try { + Class shardigStrategyClass = ReflectHelper.classForName( shardingStrategyName, this.getClass() ); + shardingStrategy = (DirectoryProviderShardingStrategy) shardigStrategyClass.newInstance(); + } + catch (ClassNotFoundException e) { + throw new SearchException("Unable to find ShardingStrategy class " + shardingStrategyName + " for " + directoryProviderName, e); + } + catch (IllegalAccessException e) { + throw new SearchException("Unable to create instance of ShardingStrategy class " + shardingStrategyName + + " Be sure to have a no-arg constructor", e); + } + catch (InstantiationException e) { + throw new SearchException("Unable to create instance of ShardingStrategy class " + shardingStrategyName + + " Be sure to have a no-arg constructor", e); + } + catch (ClassCastException e) { + throw new SearchException("ShardingStrategy class does not implements DirecotryProviderShardingStrategy: " + + shardingStrategyName, e); + } + } + shardingStrategy.init( shardingProperties, providers ); + + return new DirectoryProviders( shardingStrategy, providers ); + } + + private DirectoryProvider<?> createDirectoryProvider(String directoryProviderName, Properties indexProps, SearchFactoryImplementor searchFactoryImplementor) { String className = indexProps.getProperty( "directory_provider" ); if ( StringHelper.isEmpty( className ) ) { className = DEFAULT_DIRECTORY_PROVIDER; @@ -92,6 +148,9 @@ configureOptimizerStrategy(searchFactoryImplementor, indexProps, provider); configureIndexingParameters(searchFactoryImplementor, indexProps, provider); providers.add( provider ); + if ( !searchFactoryImplementor.getLockableDirectoryProviders().containsKey( provider ) ) { + searchFactoryImplementor.getLockableDirectoryProviders().put( provider, new ReentrantLock() ); + } return provider; } } @@ -186,26 +245,96 @@ } } searchFactoryImplementor.addIndexingParmeters(provider, indexingParams); - } + } - private static Properties getDirectoryProperties(Configuration cfg, String directoryProviderName) { + /** + * Returns an array of directory properties + * Properties are defaulted. For a given proeprty name, + * hibernate.search.indexname.n has priority over hibernate.search.indexname which has priority over hibernate.search + * If the Index is not sharded, a single Properties is returned + * If the index is sharded, the Properties index matches the shard index + */ + private static Properties[] getDirectoryProperties(Configuration cfg, String directoryProviderName) { Properties props = cfg.getProperties(); String indexName = LUCENE_PREFIX + directoryProviderName; - Properties indexProps = new Properties(); - Properties indexSpecificProps = new Properties(); + //indexSpecificProperties[i] >> indexSpecificDefaultproperties >> defaultProperties + Properties defaultProperties = new Properties(); + ArrayList<Properties> indexSpecificProps = new ArrayList<Properties>(); + Properties indexSpecificDefaultProps = new Properties(defaultProperties); for ( Map.Entry entry : props.entrySet() ) { String key = (String) entry.getKey(); if ( key.startsWith( LUCENE_DEFAULT ) ) { - indexProps.setProperty( key.substring( LUCENE_DEFAULT.length() ), (String) entry.getValue() ); + defaultProperties.setProperty( key.substring( LUCENE_DEFAULT.length() ), (String) entry.getValue() ); } else if ( key.startsWith( indexName ) ) { - indexSpecificProps.setProperty( key.substring( indexName.length() ), (String) entry.getValue() ); + String suffixedKey = key.substring( indexName.length() + 1 ); + int nextDoc = suffixedKey.indexOf( '.' ); + int index = -1; + if ( nextDoc != -1 ) { + String potentialNbr = suffixedKey.substring( 0, nextDoc ); + try { + index = Integer.parseInt( potentialNbr ); + } + catch ( Exception e ) { + //just not a number + index = -1; + } + } + if (index == -1) { + indexSpecificDefaultProps.setProperty( suffixedKey, (String) entry.getValue() ); + } + else { + String finalKeyName = suffixedKey.substring( nextDoc + 1 ); + //ignore sharding strategy properties + if ( ! finalKeyName.startsWith( SHARDING_STRATEGY ) ) { + ensureListSize( indexSpecificProps, index + 1 ); + Properties propertiesforIndex = indexSpecificProps.get( index ); + if ( propertiesforIndex == null ) { + propertiesforIndex = new Properties( indexSpecificDefaultProps ); + indexSpecificProps.set( index, propertiesforIndex ); + } + propertiesforIndex.setProperty( finalKeyName, (String) entry.getValue() ); + } + } } } - indexProps.putAll( indexSpecificProps ); - return indexProps; + String nbrOfShardsString = indexSpecificDefaultProps.getProperty( NBR_OF_SHARDS ); + int nbrOfShards = -1; + if ( nbrOfShardsString != null ) { + try { + nbrOfShards = Integer.parseInt( nbrOfShardsString ); + } + catch (NumberFormatException e) { + throw new SearchException(indexName + "." + NBR_OF_SHARDS + " is not a number", e); + } + } + if ( nbrOfShards <= 0 && indexSpecificProps.size() == 0 ) { + //no shard (a shareded subindex has to have at least one property + return new Properties[] { indexSpecificDefaultProps }; + } + else { + //sharded + nbrOfShards = nbrOfShards >= indexSpecificDefaultProps.size() ? + nbrOfShards : + indexSpecificDefaultProps.size(); + ensureListSize( indexSpecificProps, nbrOfShards ); + for ( int index = 0 ; index < nbrOfShards ; index++ ) { + if ( indexSpecificProps.get( index ) == null ) { + indexSpecificProps.set( index, new Properties( indexSpecificDefaultProps ) ); + } + } + return indexSpecificProps.toArray( new Properties[ indexSpecificProps.size() ] ); + } } + private static void ensureListSize(ArrayList<Properties> indexSpecificProps, int size) { + //ensure the index exists + indexSpecificProps.ensureCapacity( size ); + while ( indexSpecificProps.size() < size ) { + indexSpecificProps.add(null); + } + } + private static String getDirectoryProviderName(XClass clazz, Configuration cfg) { //yuk ReflectionManager reflectionManager = SearchFactoryImpl.getReflectionManager(cfg); @@ -237,4 +366,24 @@ "Trying to extract the index name from a non @Indexed class: " + clazz.getName() ); } } + + public class DirectoryProviders { + private DirectoryProviderShardingStrategy shardingStrategy; + private DirectoryProvider[] providers; + + + public DirectoryProviders(DirectoryProviderShardingStrategy shardingStrategy, DirectoryProvider[] providers) { + this.shardingStrategy = shardingStrategy; + this.providers = providers; + } + + + public DirectoryProviderShardingStrategy getSelectionStrategy() { + return shardingStrategy; + } + + public DirectoryProvider[] getProviders() { + return providers; + } + } } Added: trunk/HibernateExt/search/src/java/org/hibernate/search/store/DirectoryProviderShardingStrategy.java =================================================================== --- trunk/HibernateExt/search/src/java/org/hibernate/search/store/DirectoryProviderShardingStrategy.java (rev 0) +++ trunk/HibernateExt/search/src/java/org/hibernate/search/store/DirectoryProviderShardingStrategy.java 2007-07-29 14:56:47 UTC (rev 12844) @@ -0,0 +1,18 @@ +//$Id$ +package org.hibernate.search.store; + +import java.io.Serializable; +import java.util.Properties; + +import org.apache.lucene.document.Document; + +/** + * @author Emmanuel Bernard + */ +public interface DirectoryProviderShardingStrategy { + void init(Properties properties, DirectoryProvider[] providers); + DirectoryProvider[] getDirectoryProvidersForQuery(); + DirectoryProvider getDirectoryProviderForAddition(Class entity, Serializable id, String idInString, Document document); + DirectoryProvider[] getDirectoryProvidersForDeletion(Class entity, Serializable id, String idInString); + DirectoryProvider[] getDirectoryProvidersForOptimization(Class entity); +} Added: trunk/HibernateExt/search/src/java/org/hibernate/search/store/IdHashShardingStrategy.java =================================================================== --- trunk/HibernateExt/search/src/java/org/hibernate/search/store/IdHashShardingStrategy.java (rev 0) +++ trunk/HibernateExt/search/src/java/org/hibernate/search/store/IdHashShardingStrategy.java 2007-07-29 14:56:47 UTC (rev 12844) @@ -0,0 +1,46 @@ +//$Id$ +package org.hibernate.search.store; + +import java.util.Properties; +import java.io.Serializable; + +import org.apache.lucene.document.Document; + +/** + * This implementation use idInString as the hashKey. + * + * @author Emmanuel Bernard + */ +public class IdHashShardingStrategy implements DirectoryProviderShardingStrategy { + private DirectoryProvider[] providers; + public void init(Properties properties, DirectoryProvider[] providers) { + this.providers = providers; + } + + public DirectoryProvider[] getDirectoryProvidersForQuery() { + return providers; + } + + public DirectoryProvider getDirectoryProviderForAddition(Class entity, Serializable id, String idInString, Document document) { + return providers[ hashKey(idInString) ]; + } + + public DirectoryProvider[] getDirectoryProvidersForDeletion(Class entity, Serializable id, String idInString) { + return new DirectoryProvider[] { providers[ hashKey(idInString) ] }; + } + + public DirectoryProvider[] getDirectoryProvidersForOptimization(Class entity) { + return providers; + } + + private int hashKey(String key) { + //reproduce the hashCode implementaiton of String as documented in the javadoc + // to be safe cross Java version (in case it changes some day) + int hash = 0; + int length = key.length(); + for (int index = 0 ; index < length ; index++) { + hash = 31*hash + key.charAt( index ); + } + return hash % providers.length; + } +} Added: trunk/HibernateExt/search/src/java/org/hibernate/search/store/NotShardedStrategy.java =================================================================== --- trunk/HibernateExt/search/src/java/org/hibernate/search/store/NotShardedStrategy.java (rev 0) +++ trunk/HibernateExt/search/src/java/org/hibernate/search/store/NotShardedStrategy.java 2007-07-29 14:56:47 UTC (rev 12844) @@ -0,0 +1,37 @@ +//$Id$ +package org.hibernate.search.store; + +import java.util.Properties; +import java.io.Serializable; + +import org.apache.lucene.document.Document; +import org.hibernate.annotations.common.AssertionFailure; + +/** + * @author Emmanuel Bernard + */ +public class NotShardedStrategy implements DirectoryProviderShardingStrategy { + private DirectoryProvider[] directoryProvider; + public void init(Properties properties, DirectoryProvider[] providers) { + this.directoryProvider = providers; + if ( directoryProvider.length > 1) { + throw new AssertionFailure("Using SingleDirectoryProviderSelectionStrategy with multiple DirectryProviders"); + } + } + + public DirectoryProvider[] getDirectoryProvidersForQuery() { + return directoryProvider; + } + + public DirectoryProvider getDirectoryProviderForAddition(Class entity, Serializable id, String idInString, Document document) { + return directoryProvider[0]; + } + + public DirectoryProvider[] getDirectoryProvidersForDeletion(Class entity, Serializable id, String idInString) { + return directoryProvider; + } + + public DirectoryProvider[] getDirectoryProvidersForOptimization(Class entity) { + return directoryProvider; + } +} Modified: trunk/HibernateExt/search/src/test/org/hibernate/search/test/SearchTestCase.java =================================================================== --- trunk/HibernateExt/search/src/test/org/hibernate/search/test/SearchTestCase.java 2007-07-29 14:12:13 UTC (rev 12843) +++ trunk/HibernateExt/search/src/test/org/hibernate/search/test/SearchTestCase.java 2007-07-29 14:56:47 UTC (rev 12844) @@ -20,7 +20,7 @@ } protected Directory getDirectory(Class clazz) { - return getLuceneEventListener().getSearchFactoryImplementor().getDirectoryProvider( clazz ).getDirectory(); + return getLuceneEventListener().getSearchFactoryImplementor().getDirectoryProviders( clazz )[0].getDirectory(); } private FullTextIndexEventListener getLuceneEventListener() { Modified: trunk/HibernateExt/search/src/test/org/hibernate/search/test/jms/master/JMSMasterTest.java =================================================================== --- trunk/HibernateExt/search/src/test/org/hibernate/search/test/jms/master/JMSMasterTest.java 2007-07-29 14:12:13 UTC (rev 12843) +++ trunk/HibernateExt/search/src/test/org/hibernate/search/test/jms/master/JMSMasterTest.java 2007-07-29 14:56:47 UTC (rev 12844) @@ -57,7 +57,7 @@ doc.add( field ); field = new Field("logo", ts.getLogo(), Field.Store.NO, Field.Index.TOKENIZED ); doc.add( field ); - LuceneWork luceneWork = new AddLuceneWork(ts.getId(), ts.getClass(), doc ); + LuceneWork luceneWork = new AddLuceneWork(ts.getId(), String.valueOf( ts.getId() ), ts.getClass(), doc ); List<LuceneWork> queue = new ArrayList<LuceneWork>(); queue.add( luceneWork ); Added: trunk/HibernateExt/search/src/test/org/hibernate/search/test/shards/Animal.java =================================================================== --- trunk/HibernateExt/search/src/test/org/hibernate/search/test/shards/Animal.java (rev 0) +++ trunk/HibernateExt/search/src/test/org/hibernate/search/test/shards/Animal.java 2007-07-29 14:56:47 UTC (rev 12844) @@ -0,0 +1,40 @@ +//$Id$ +package org.hibernate.search.test.shards; + +import javax.persistence.Entity; +import javax.persistence.Id; + +import org.hibernate.search.annotations.Indexed; +import org.hibernate.search.annotations.DocumentId; +import org.hibernate.search.annotations.Field; +import org.hibernate.search.annotations.Index; + +/** + * @author Emmanuel Bernard + */ +@Entity +@Indexed(index = "Animal") +public class Animal { + @Id + @DocumentId + private Integer id; + @Field(index = Index.TOKENIZED) + private String name; + + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } +} Added: trunk/HibernateExt/search/src/test/org/hibernate/search/test/shards/ShardsTest.java =================================================================== --- trunk/HibernateExt/search/src/test/org/hibernate/search/test/shards/ShardsTest.java (rev 0) +++ trunk/HibernateExt/search/src/test/org/hibernate/search/test/shards/ShardsTest.java 2007-07-29 14:56:47 UTC (rev 12844) @@ -0,0 +1,190 @@ +//$Id$ +package org.hibernate.search.test.shards; + +import java.io.File; +import java.util.Properties; +import java.util.List; + +import org.hibernate.search.test.SearchTestCase; +import org.hibernate.search.store.RAMDirectoryProvider; +import org.hibernate.search.store.FSDirectoryProvider; +import org.hibernate.search.store.IdHashShardingStrategy; +import org.hibernate.search.store.DirectoryProvider; +import org.hibernate.search.Environment; +import org.hibernate.search.FullTextSession; +import org.hibernate.search.Search; +import org.hibernate.cfg.Configuration; +import org.hibernate.Session; +import org.hibernate.Transaction; +import org.apache.lucene.analysis.StopAnalyzer; +import org.apache.lucene.queryParser.QueryParser; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.TermDocs; +import org.apache.lucene.index.Term; + +/** + * @author Emmanuel Bernard + */ +public class ShardsTest extends SearchTestCase { + + + protected void configure(Configuration cfg) { + super.configure( cfg ); + cfg.setProperty( "hibernate.search.default.directory_provider", FSDirectoryProvider.class.getName() ); + File sub = getBaseIndexDir(); + cfg.setProperty( "hibernate.search.default.indexBase", sub.getAbsolutePath() ); + cfg.setProperty( Environment.ANALYZER_CLASS, StopAnalyzer.class.getName() ); + //is the default when multiple shards are set up + //cfg.setProperty( "hibernate.search.Animal.sharding_strategy", IdHashShardingStrategy.class ); + cfg.setProperty( "hibernate.search.Animal.sharding_strategy.nbr_of_shards", "2" ); + cfg.setProperty( "hibernate.search.Animal.0.indexName", "Animal00" ); + } + + public void testIdShardingStrategy() { + DirectoryProvider[] dps = new DirectoryProvider[] { new RAMDirectoryProvider(), new RAMDirectoryProvider() }; + IdHashShardingStrategy shardingStrategy = new IdHashShardingStrategy(); + shardingStrategy.init( null, dps); + assertTrue( dps[1] == shardingStrategy.getDirectoryProviderForAddition( Animal.class, 1, "1", null) ); + assertTrue( dps[0] == shardingStrategy.getDirectoryProviderForAddition( Animal.class, 2, "2", null) ); + } + + public void testBehavior() throws Exception { + Session s = openSession( ); + Transaction tx = s.beginTransaction(); + Animal a = new Animal(); + a.setId( 1 ); + a.setName( "Elephant" ); + s.persist( a ); + a = new Animal(); + a.setId( 2 ); + a.setName( "Bear" ); + s.persist( a ); + tx.commit(); + + s.clear(); + + tx = s.beginTransaction(); + a = (Animal) s.get(Animal.class, 1); + a.setName( "Mouse" ); + tx.commit(); + + s.clear(); + + tx = s.beginTransaction(); + FullTextSession fts = Search.createFullTextSession( s ); + QueryParser parser = new QueryParser("id", new StopAnalyzer() ); + + List results = fts.createFullTextQuery( parser.parse( "name:mouse OR name:bear" ) ).list(); + assertEquals( "Either double insert, single update, or query fails with shards", 2, results.size() ); + for (Object o : results) s.delete( o ); + tx.commit(); + s.close(); + } + + public void testInternalSharding() throws Exception { + Session s = openSession( ); + Transaction tx = s.beginTransaction(); + Animal a = new Animal(); + a.setId( 1 ); + a.setName( "Elephant" ); + s.persist( a ); + a = new Animal(); + a.setId( 2 ); + a.setName( "Bear" ); + s.persist( a ); + tx.commit(); + + s.clear(); + + IndexReader reader = IndexReader.open( new File( getBaseIndexDir(), "Animal00" ) ); + try { + int num = reader.numDocs(); + assertEquals( 1, num ); + } + finally { + reader.close(); + } + reader = IndexReader.open( new File( getBaseIndexDir(), "Animal_1" ) ); + try { + int num = reader.numDocs(); + assertEquals( 1, num ); + } + finally { + reader.close(); + } + + + tx = s.beginTransaction(); + a = (Animal) s.get(Animal.class, 1); + a.setName( "Mouse" ); + tx.commit(); + + s.clear(); + + reader = IndexReader.open( new File( getBaseIndexDir(), "Animal_1" ) ); + try { + int num = reader.numDocs(); + assertEquals( 1, num ); + TermDocs docs = reader.termDocs( new Term( "name", "mouse" ) ); + assertTrue( docs.next() ); + org.apache.lucene.document.Document doc = reader.document( docs.doc() ); + assertFalse( docs.next() ); + } + finally { + reader.close(); + } + + tx = s.beginTransaction(); + FullTextSession fts = Search.createFullTextSession( s ); + QueryParser parser = new QueryParser("id", new StopAnalyzer() ); + + List results = fts.createFullTextQuery( parser.parse( "name:mouse OR name:bear" ) ).list(); + assertEquals( "Either double insert, single update, or query fails with shards", 2, results.size() ); + for (Object o : results) s.delete( o ); + tx.commit(); + s.close(); + } + + protected void setUp() throws Exception { + File sub = getBaseIndexDir(); + sub.mkdir(); + File[] files = sub.listFiles(); + for (File file : files) { + if ( file.isDirectory() ) { + delete( file ); + } + } + //super.setUp(); //we need a fresh session factory each time for index set up + buildSessionFactory( getMappings(), getAnnotatedPackages(), getXmlFiles() ); + } + + private File getBaseIndexDir() { + File current = new File( "." ); + File sub = new File( current, "indextemp" ); + return sub; + } + + protected void tearDown() throws Exception { + super.tearDown(); + File sub = getBaseIndexDir(); + delete( sub ); + } + + private void delete(File sub) { + if ( sub.isDirectory() ) { + for ( File file : sub.listFiles() ) { + delete( file ); + } + sub.delete(); + } + else { + sub.delete(); + } + } + + protected Class[] getMappings() { + return new Class[] { + Animal.class + }; + } +}