Author: navssurtani Date: 2008-06-30 09:56:28 -0400 (Mon, 30 Jun 2008) New Revision: 14827 Added: search/branches/jboss_cache_integration/src/java/org/hibernate/search/backend/configuration/ConfigurationParseHelper.java search/branches/jboss_cache_integration/src/java/org/hibernate/search/engine/MultiClassesQueryLoader.java search/branches/jboss_cache_integration/src/java/org/hibernate/search/engine/ObjectLoaderHelper.java search/branches/jboss_cache_integration/src/java/org/hibernate/search/reader/SharingBufferReaderProvider.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/SerializationTestHelper.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/configuration/ConfigurationParseHelperTest.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/directoryProvider/DirectoryProviderHelperTest.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/embedded/State.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/query/MultiClassesQueryLoaderTest.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/functionality/ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/functionality/FilterOnDirectoryTest.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/functionality/SharingBufferIndexProviderTest.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/functionality/TestableSharingBufferReaderProvider.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/performance/ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/performance/AbstractActivity.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/performance/BufferSharingReaderPerfTest.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/performance/IndexFillRunnable.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/performance/InsertActivity.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/performance/NotSharedReaderPerfTest.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/performance/ReaderPerformance.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/performance/SearchActivity.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/performance/SharedReaderPerfTest.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/performance/UpdateActivity.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/session/Domain.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/session/MassIndexUsingManualFlushTest.java Modified: search/branches/jboss_cache_integration/build.xml search/branches/jboss_cache_integration/doc/reference/en/modules/configuration.xml search/branches/jboss_cache_integration/doc/reference/en/modules/mapping.xml search/branches/jboss_cache_integration/src/java/org/hibernate/search/FullTextSession.java search/branches/jboss_cache_integration/src/java/org/hibernate/search/backend/LuceneIndexingParameters.java search/branches/jboss_cache_integration/src/java/org/hibernate/search/backend/Workspace.java search/branches/jboss_cache_integration/src/java/org/hibernate/search/backend/configuration/IndexWriterSetting.java search/branches/jboss_cache_integration/src/java/org/hibernate/search/backend/configuration/MaskedProperty.java search/branches/jboss_cache_integration/src/java/org/hibernate/search/backend/impl/BatchedQueueingProcessor.java search/branches/jboss_cache_integration/src/java/org/hibernate/search/backend/impl/PostTransactionWorkQueueSynchronization.java search/branches/jboss_cache_integration/src/java/org/hibernate/search/bridge/BridgeFactory.java search/branches/jboss_cache_integration/src/java/org/hibernate/search/engine/DocumentBuilder.java search/branches/jboss_cache_integration/src/java/org/hibernate/search/engine/ObjectLoader.java search/branches/jboss_cache_integration/src/java/org/hibernate/search/engine/ProjectionLoader.java search/branches/jboss_cache_integration/src/java/org/hibernate/search/engine/QueryLoader.java search/branches/jboss_cache_integration/src/java/org/hibernate/search/engine/SearchFactoryImplementor.java search/branches/jboss_cache_integration/src/java/org/hibernate/search/filter/MRUFilterCachingStrategy.java search/branches/jboss_cache_integration/src/java/org/hibernate/search/impl/SearchFactoryImpl.java search/branches/jboss_cache_integration/src/java/org/hibernate/search/jpa/FullTextEntityManager.java search/branches/jboss_cache_integration/src/java/org/hibernate/search/jpa/impl/FullTextEntityManagerImpl.java search/branches/jboss_cache_integration/src/java/org/hibernate/search/query/FullTextQueryImpl.java search/branches/jboss_cache_integration/src/java/org/hibernate/search/reader/ReaderProviderHelper.java search/branches/jboss_cache_integration/src/java/org/hibernate/search/reader/SharedReaderProvider.java search/branches/jboss_cache_integration/src/java/org/hibernate/search/store/DirectoryProviderFactory.java search/branches/jboss_cache_integration/src/java/org/hibernate/search/store/DirectoryProviderHelper.java search/branches/jboss_cache_integration/src/java/org/hibernate/search/store/FSDirectoryProvider.java search/branches/jboss_cache_integration/src/java/org/hibernate/search/store/FSMasterDirectoryProvider.java search/branches/jboss_cache_integration/src/java/org/hibernate/search/store/FSSlaveDirectoryProvider.java search/branches/jboss_cache_integration/src/java/org/hibernate/search/store/optimization/IncrementalOptimizerStrategy.java search/branches/jboss_cache_integration/src/java/org/hibernate/search/util/FileHelper.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/FSDirectoryTest.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/analyzer/AnalyzerTest.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/configuration/LuceneIndexingParametersTest.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/configuration/MaskedPropertiesTest.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/directoryProvider/FSSlaveAndMasterDPTest.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/embedded/Author.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/embedded/Country.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/embedded/Person.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/embedded/Tower.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/embedded/doubleinsert/DoubleInsertEmbeddedTest.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/jms/master/JMSMasterTest.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/optimizer/OptimizerTestCase.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/perf/IndexTestDontRun.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/perf/SearcherThread.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/query/Employee.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/query/LuceneQuerySortTest.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/query/ProjectionQueryTest.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/NotSharedReaderPerfTest.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/ReaderPerfTestCase.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/SharedReaderPerfTest.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/session/Email.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/session/OptimizeTest.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/session/SessionTest.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/shards/ShardsTest.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/worker/ConcurrencyTest.java search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/worker/WorkerTestCase.java Log: Updated from trunk Modified: search/branches/jboss_cache_integration/build.xml =================================================================== --- search/branches/jboss_cache_integration/build.xml 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/build.xml 2008-06-30 13:56:28 UTC (rev 14827) @@ -20,6 +20,7 @@ <property name="version" value="3.1.0-SNAPSHOT"/> <property name="javadoc.packagenames" value="org.hibernate.search.*"/> <property name="copy.test" value="true"/> + <property name="copy.test" value="true"/> <property name="javac.source" value="1.5"/> <property name="javac.target" value="1.5"/> <property name="jdbc.dir" value="jdbc"/> Modified: search/branches/jboss_cache_integration/doc/reference/en/modules/configuration.xml =================================================================== --- search/branches/jboss_cache_integration/doc/reference/en/modules/configuration.xml 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/doc/reference/en/modules/configuration.xml 2008-06-30 13:56:28 UTC (rev 14827) @@ -33,12 +33,12 @@ <entry>org.hibernate.search.store.FSDirectoryProvider</entry> <entry>File system based directory. The directory used will be - &lt;indexBase&gt;/&lt; <literal&gt;(a)Indexed.name&lt;/literal&gt; + &lt;indexBase&gt;/&lt; <literal&gt;(a)Indexed.index&lt;/literal&gt; &gt;</entry> <entry><para><literal>indexBase</literal> : Base directory</para><para><literal>indexName</literal>: override - @Index.name (useful for sharded indexes)</para></entry> + @Indexed.index (useful for sharded indexes)</para></entry> </row> <row> @@ -52,19 +52,24 @@ 3600 seconds - 60 minutes).</para><para>Note that the copy is based on an incremental copy mechanism reducing the average copy time.</para><para>DirectoryProvider typically used on the master - node in a JMS back end cluster.</para>DirectoryProvider typically - used on slave nodes using a JMS back end.</entry> + node in a JMS back end cluster.</para><para>The <literal> + buffer_size_on_copy</literal> optimum depends + on your operating system and available RAM; most people reported + good results using values between 16 and 64MB.</para></entry> <entry><para><literal>indexBase</literal>: Base directory</para><para><literal>indexName</literal>: override - @Index.name (useful for sharded + @Indexed.index (useful for sharded indexes)</para><para><literal>sourceBase</literal>: Source (copy) base directory.</para><para><literal>source</literal>: Source - directory suffix (default to <literal&gt;(a)Indexed.name&lt;/literal>). + directory suffix (default to <literal&gt;(a)Indexed.index&lt;/literal>). The actual source directory name being <filename>&lt;sourceBase&gt;/&lt;source&gt;</filename> - </para><para>refresh: refresh period in second (the copy will take - place every refresh seconds).</para></entry> + </para><para><literal>refresh</literal>: refresh period in second + (the copy will take place every refresh seconds).</para><para> + <literal>buffer_size_on_copy</literal>: The amount of + MegaBytes to move in a single low level copy instruction; + defaults to 16MB.</para></entry> </row> <row> @@ -78,18 +83,24 @@ information (default 3600 seconds - 60 minutes).</para><para>Note that the copy is based on an incremental copy mechanism reducing the average copy time.</para><para>DirectoryProvider typically - used on slave nodes using a JMS back end.</para></entry> + used on slave nodes using a JMS back end.</para><para>The <literal> + buffer_size_on_copy</literal> optimum depends + on your operating system and available RAM; most people reported + good results using values between 16 and 64MB.</para></entry> <entry><para><literal>indexBase</literal>: Base directory</para><para><literal>indexName</literal>: override - @Index.name (useful for sharded + @Indexed.index (useful for sharded indexes)</para><para><literal>sourceBase</literal>: Source (copy) base directory.</para><para><literal>source</literal>: Source - directory suffix (default to <literal&gt;(a)Indexed.name&lt;/literal>). + directory suffix (default to <literal&gt;(a)Indexed.index&lt;/literal>). The actual source directory name being <filename>&lt;sourceBase&gt;/&lt;source&gt;</filename> - </para><para>refresh: refresh period in second (the copy will take - place every refresh seconds).</para></entry> + </para><para><literal>refresh</literal>: refresh period in second + (the copy will take place every refresh seconds).</para><para> + <literal>buffer_size_on_copy</literal>: The amount of + MegaBytes to move in a single low level copy instruction; + defaults to 16MB.</para></entry> </row> <row> @@ -97,7 +108,7 @@ <entry>Memory based directory, the directory will be uniquely identified (in the same deployment unit) by the - <literal&gt;(a)Indexed.name&lt;/literal&gt; element</entry> + <literal&gt;(a)Indexed.index&lt;/literal&gt; element</entry> <entry>none</entry> </row> @@ -108,7 +119,7 @@ <para>If the built-in directory providers does not fit your needs, you can write your own directory provider by implementing the <classname>org.hibernate.store.DirectoryProvider</classname> - interface</para> + interface.</para> <para>Each indexed entity is associated to a Lucene index (an index can be shared by several entities but this is not usually the case). You can @@ -123,15 +134,14 @@ <programlisting>hibernate.search.default.directory_provider org.hibernate.search.store.FSDirectoryProvider hibernate.search.default.indexBase=/usr/lucene/indexes +hibernate.search.Rules.directory_provider org.hibernate.search.store.RAMDirectoryProvider</programlisting> -hibernate.search.Rules.directory_provider org.hibernate.search.store.RAMDirectoryProvider </programlisting> - <para>applied on</para> - <programlisting>@Indexed(name="Status") + <programlisting>@Indexed(index="Status") public class Status { ... } -@Indexed(name="Rules") +@Indexed(index="Rules") public class Rule { ... }</programlisting> <para>will create a file system directory in @@ -140,7 +150,7 @@ <literal>Rules</literal> where Rule entities will be indexed.</para> <para>You can easily define common rules like the directory provider and - base directory, and overide those default later on on a per index + base directory, and override those default later on on a per index basis.</para> <para>Writing your own <classname>DirectoryProvider</classname>, you can @@ -155,7 +165,7 @@ several Lucene indexes. This solution is not recommended until you reach significant index sizes and index update time are slowing down. The main drawback of index sharding is that searches will end up being slower since - more files have to be opend for a single search. In other words don't do + more files have to be opened for a single search. In other words don't do it until you have problems :)</para> <para>Despite this strong warning, Hibernate Search allows you to index a @@ -451,7 +461,7 @@ <programlisting>hibernate.search.reader.strategy = my.corp.myapp.CustomReaderProvider</programlisting> <para>where <classname>my.corp.myapp.CustomReaderProvider</classname> is - the custom strategy implementation</para> + the custom strategy implementation.</para> </section> <section id="search-configuration-event" revision="2"> Modified: search/branches/jboss_cache_integration/doc/reference/en/modules/mapping.xml =================================================================== --- search/branches/jboss_cache_integration/doc/reference/en/modules/mapping.xml 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/doc/reference/en/modules/mapping.xml 2008-06-30 13:56:28 UTC (rev 14827) @@ -349,7 +349,7 @@ ... }</programlisting> - <para>Any <literal>@ManyToMany, @*ToOne</literal> and + <para>Any <literal>@*ToMany, @*ToOne</literal> and <literal>@Embedded</literal> attribute can be annotated with <literal>@IndexedEmbedded</literal>. The attributes of the associated class will then be added to the main entity index. In the previous @@ -900,4 +900,4 @@ </section> </section> </section> -</chapter> \ No newline at end of file +</chapter> Modified: search/branches/jboss_cache_integration/src/java/org/hibernate/search/FullTextSession.java =================================================================== --- search/branches/jboss_cache_integration/src/java/org/hibernate/search/FullTextSession.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/java/org/hibernate/search/FullTextSession.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -24,6 +24,7 @@ * Non indexable entities are ignored * * @param entity The entity to index - must not be <code>null</code>. + * @throws IllegalArgumentException if entity is null or not an @Indexed entity */ void index(Object entity); @@ -37,6 +38,8 @@ * * @param entityType * @param id + * + * @throws IllegalArgumentException if entityType is null or not an @Indexed entity type */ public void purge(Class entityType, Serializable id); @@ -44,6 +47,13 @@ * Remove all entities from a particular class of an index. * * @param entityType + * @throws IllegalArgumentException if entityType is null or not an @Indexed entity type */ public void purgeAll(Class entityType); + + /** + * flush full text changes to the index + * Force Hibernate Search to apply all changes to the index no waiting for the batch limit + */ + public void flushToIndexes(); } Modified: search/branches/jboss_cache_integration/src/java/org/hibernate/search/backend/LuceneIndexingParameters.java =================================================================== --- search/branches/jboss_cache_integration/src/java/org/hibernate/search/backend/LuceneIndexingParameters.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/java/org/hibernate/search/backend/LuceneIndexingParameters.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -2,7 +2,7 @@ package org.hibernate.search.backend; import java.io.Serializable; -import java.util.HashMap; +import java.util.EnumMap; import java.util.Map; import java.util.Properties; @@ -28,8 +28,6 @@ private static final long serialVersionUID = 5424606407623591663L; - private final Logger log = LoggerFactory.getLogger( LuceneIndexingParameters.class ); - // value keyword public static final String EXPLICIT_DEFAULT_VALUE = "default"; // property path keywords @@ -47,12 +45,14 @@ Properties transactionProps = new MaskedProperty( indexingParameters, TRANSACTION ); //get keys for "batch" (defaulting to transaction) Properties batchProps = new MaskedProperty( indexingParameters, BATCH, transactionProps ); //TODO to close HSEARCH-201 just remove 3° parameter - transactionIndexParameters = new ParameterSet( transactionProps, TRANSACTION ); - batchIndexParameters = new ParameterSet( batchProps, BATCH ); - doSanityChecks( transactionIndexParameters, batchIndexParameters ); + //logger only used during object construction: (logger not serializable). + Logger log = LoggerFactory.getLogger( LuceneIndexingParameters.class ); + transactionIndexParameters = new ParameterSet( transactionProps, TRANSACTION, log ); + batchIndexParameters = new ParameterSet( batchProps, BATCH, log ); + doSanityChecks( transactionIndexParameters, batchIndexParameters, log ); } - private void doSanityChecks(ParameterSet transParams, ParameterSet batchParams) { + private void doSanityChecks(ParameterSet transParams, ParameterSet batchParams, Logger log) { if ( log.isWarnEnabled() ) { Integer maxFieldLengthTransaction = transParams.parameters.get( MAX_FIELD_LENGTH ); Integer maxFieldLengthBatch = transParams.parameters.get( MAX_FIELD_LENGTH ); @@ -77,13 +77,13 @@ return batchIndexParameters; } - public class ParameterSet implements Serializable { + public static class ParameterSet implements Serializable { private static final long serialVersionUID = -6121723702279869524L; - final Map<IndexWriterSetting, Integer> parameters = new HashMap<IndexWriterSetting, Integer>(); + final Map<IndexWriterSetting, Integer> parameters = new EnumMap<IndexWriterSetting, Integer>(IndexWriterSetting.class); - public ParameterSet(Properties prop, String paramName) { + public ParameterSet(Properties prop, String paramName, Logger log) { //don't iterate on property entries as we know all the keys: for ( IndexWriterSetting t : IndexWriterSetting.values() ) { String key = t.getKey(); @@ -127,6 +127,32 @@ } } + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + + ((parameters == null) ? 0 : parameters.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + final ParameterSet other = (ParameterSet) obj; + if (parameters == null) { + if (other.parameters != null) + return false; + } else if (!parameters.equals(other.parameters)) + return false; + return true; + } + } public void applyToWriter(IndexWriter writer, boolean batch) { Modified: search/branches/jboss_cache_integration/src/java/org/hibernate/search/backend/Workspace.java =================================================================== --- search/branches/jboss_cache_integration/src/java/org/hibernate/search/backend/Workspace.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/java/org/hibernate/search/backend/Workspace.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -4,7 +4,7 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; -import java.util.concurrent.locks.ReentrantLock; +import java.util.concurrent.locks.Lock; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.SimpleAnalyzer; @@ -189,7 +189,7 @@ private class DPWorkspace { private final DirectoryProvider directoryProvider; - private final ReentrantLock lock; + private final Lock lock; private IndexReader reader; private IndexWriter writer; @@ -199,7 +199,7 @@ DPWorkspace(DirectoryProvider dp) { this.directoryProvider = dp; - this.lock = searchFactoryImplementor.getLockableDirectoryProviders().get( dp ); + this.lock = searchFactoryImplementor.getDirectoryProviderLock( dp ); } public boolean needsOptimization() { Added: search/branches/jboss_cache_integration/src/java/org/hibernate/search/backend/configuration/ConfigurationParseHelper.java =================================================================== --- search/branches/jboss_cache_integration/src/java/org/hibernate/search/backend/configuration/ConfigurationParseHelper.java (rev 0) +++ search/branches/jboss_cache_integration/src/java/org/hibernate/search/backend/configuration/ConfigurationParseHelper.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -0,0 +1,69 @@ +package org.hibernate.search.backend.configuration; + +import java.util.Properties; + +import org.hibernate.annotations.common.util.StringHelper; +import org.hibernate.search.SearchException; + +/** + * Helper class to avoid managing NumberFormatException and similar code + * and ensure consistent error messages across Configuration parsing problems. + * + * @author Sanne Grinovero + */ +public abstract class ConfigurationParseHelper { + + /** + * Parses a String to get an int value. + * @param value A string containing an int value to parse + * @param errorMsgOnParseFailure message being wrapped in a SearchException if value is null or not correct. + * @return the parsed value + * @throws SearchException both for null values and for Strings not containing a valid int. + */ + public static final int parseInt(String value, String errorMsgOnParseFailure) { + if ( value == null ) { + throw new SearchException( errorMsgOnParseFailure ); + } + else { + try { + return Integer.parseInt( value.trim() ); + } catch (NumberFormatException nfe) { + throw new SearchException( errorMsgOnParseFailure, nfe ); + } + } + } + + /** + * In case value is null or an empty string the defValue is returned + * @param value + * @param defValue + * @param errorMsgOnParseFailure + * @return the converted int. + * @throws SearchException if value can't be parsed. + */ + public static final int parseInt(String value, int defValue, String errorMsgOnParseFailure) { + if ( StringHelper.isEmpty( value ) ) { + return defValue; + } + else { + return parseInt( value, errorMsgOnParseFailure ); + } + } + + /** + * Looks for a numeric value in the Properties, returning + * defValue if not found or if an empty string is found. + * When the key the value is found but not in valid format + * a standard error message is generated. + * @param cfg + * @param key + * @param defValue + * @return the converted int. + * @throws SearchException for invalid format. + */ + public static final int getIntValue(Properties cfg, String key, int defValue) { + String propValue = cfg.getProperty( key ); + return parseInt( propValue, defValue, "Unable to parse " + key + ": " + propValue ); + } + +} Modified: search/branches/jboss_cache_integration/src/java/org/hibernate/search/backend/configuration/IndexWriterSetting.java =================================================================== --- search/branches/jboss_cache_integration/src/java/org/hibernate/search/backend/configuration/IndexWriterSetting.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/java/org/hibernate/search/backend/configuration/IndexWriterSetting.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -94,11 +94,8 @@ * @throws SearchException for unrecognized values */ public Integer parseVal(String value) { - try { - return Integer.valueOf( value ); - } catch (NumberFormatException ne) { - throw new SearchException( "Invalid value for " + cfgKey + ": " + value ); - } + return ConfigurationParseHelper.parseInt( value, + "Invalid value for " + cfgKey + ": " + value ); } } Modified: search/branches/jboss_cache_integration/src/java/org/hibernate/search/backend/configuration/MaskedProperty.java =================================================================== --- search/branches/jboss_cache_integration/src/java/org/hibernate/search/backend/configuration/MaskedProperty.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/java/org/hibernate/search/backend/configuration/MaskedProperty.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -2,6 +2,8 @@ import java.io.IOException; import java.io.InputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; import java.io.OutputStream; import java.io.PrintStream; import java.io.PrintWriter; @@ -29,7 +31,7 @@ private static final long serialVersionUID = -593307257383085113L; - private final Logger log = LoggerFactory.getLogger( MaskedProperty.class ); + private transient Logger log = LoggerFactory.getLogger( MaskedProperty.class ); private final Properties masked; private final Properties fallBack; private final String radix; @@ -344,4 +346,14 @@ return true; } + private void readObject(ObjectInputStream aInputStream) throws ClassNotFoundException, IOException { + //always perform the default de-serialization first + aInputStream.defaultReadObject(); + log = LoggerFactory.getLogger( MaskedProperty.class ); + } + + private void writeObject(ObjectOutputStream aOutputStream) throws IOException { + aOutputStream.defaultWriteObject(); + } + } Modified: search/branches/jboss_cache_integration/src/java/org/hibernate/search/backend/impl/BatchedQueueingProcessor.java =================================================================== --- search/branches/jboss_cache_integration/src/java/org/hibernate/search/backend/impl/BatchedQueueingProcessor.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/java/org/hibernate/search/backend/impl/BatchedQueueingProcessor.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -20,6 +20,7 @@ import org.hibernate.search.backend.Work; import org.hibernate.search.backend.WorkType; import org.hibernate.search.backend.WorkQueue; +import org.hibernate.search.backend.configuration.ConfigurationParseHelper; import org.hibernate.search.backend.impl.jms.JMSBackendQueueProcessorFactory; import org.hibernate.search.backend.impl.lucene.LuceneBackendQueueProcessorFactory; import org.hibernate.search.engine.DocumentBuilder; @@ -37,29 +38,24 @@ private static final Logger log = LoggerFactory.getLogger( BatchedQueueingProcessor.class ); - private boolean sync; - private int batchSize; - private ExecutorService executorService; - private BackendQueueProcessorFactory backendQueueProcessorFactory; - private SearchFactoryImplementor searchFactoryImplementor; + private final boolean sync; + private final int batchSize; + private final ExecutorService executorService; + private final BackendQueueProcessorFactory backendQueueProcessorFactory; + private final SearchFactoryImplementor searchFactoryImplementor; - public BatchedQueueingProcessor(SearchFactoryImplementor searchFactoryImplementor, - Properties properties) { + public BatchedQueueingProcessor(SearchFactoryImplementor searchFactoryImplementor, Properties properties) { this.searchFactoryImplementor = searchFactoryImplementor; //default to sync if none defined this.sync = !"async".equalsIgnoreCase( properties.getProperty( Environment.WORKER_EXECUTION ) ); //default to a simple asynchronous operation - int min = Integer.parseInt( - properties.getProperty( Environment.WORKER_THREADPOOL_SIZE, "1" ).trim() - ); + int min = ConfigurationParseHelper.getIntValue( properties, Environment.WORKER_THREADPOOL_SIZE, 1 ); //no queue limit - int queueSize = Integer.parseInt( - properties.getProperty( Environment.WORKER_WORKQUEUE_SIZE, Integer.toString( Integer.MAX_VALUE ) ).trim() - ); - batchSize = Integer.parseInt( - properties.getProperty( Environment.WORKER_BATCHSIZE, "0" ).trim() - ); + int queueSize = ConfigurationParseHelper.getIntValue( properties, Environment.WORKER_WORKQUEUE_SIZE, Integer.MAX_VALUE ); + + batchSize = ConfigurationParseHelper.getIntValue( properties, Environment.WORKER_BATCHSIZE, 0 ); + if ( !sync ) { /** * choose min = max with a sizable queue to be able to @@ -74,6 +70,9 @@ new ThreadPoolExecutor.CallerRunsPolicy() ); } + else { + executorService = null; + } String backend = properties.getProperty( Environment.WORKER_BACKEND ); if ( StringHelper.isEmpty( backend ) || "lucene".equalsIgnoreCase( backend ) ) { backendQueueProcessorFactory = new LuceneBackendQueueProcessorFactory(); Modified: search/branches/jboss_cache_integration/src/java/org/hibernate/search/backend/impl/PostTransactionWorkQueueSynchronization.java =================================================================== --- search/branches/jboss_cache_integration/src/java/org/hibernate/search/backend/impl/PostTransactionWorkQueueSynchronization.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/java/org/hibernate/search/backend/impl/PostTransactionWorkQueueSynchronization.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -56,4 +56,10 @@ if (queuePerTransaction != null) queuePerTransaction.removeValue( this ); } } + + public void flushWorks() { + WorkQueue subQueue = queue.splitQueue(); + queueingProcessor.prepareWorks( subQueue ); + queueingProcessor.performWorks( subQueue ); + } } Modified: search/branches/jboss_cache_integration/src/java/org/hibernate/search/bridge/BridgeFactory.java =================================================================== --- search/branches/jboss_cache_integration/src/java/org/hibernate/search/bridge/BridgeFactory.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/java/org/hibernate/search/bridge/BridgeFactory.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -69,12 +69,12 @@ public static final TwoWayFieldBridge Uri = new TwoWayString2FieldBridgeAdaptor( new UriBridge() ); - public static final FieldBridge DATE_YEAR = new String2FieldBridgeAdaptor( DateBridge.DATE_YEAR ); - public static final FieldBridge DATE_MONTH = new String2FieldBridgeAdaptor( DateBridge.DATE_MONTH ); - public static final FieldBridge DATE_DAY = new String2FieldBridgeAdaptor( DateBridge.DATE_DAY ); - public static final FieldBridge DATE_HOUR = new String2FieldBridgeAdaptor( DateBridge.DATE_HOUR ); - public static final FieldBridge DATE_MINUTE = new String2FieldBridgeAdaptor( DateBridge.DATE_MINUTE ); - public static final FieldBridge DATE_SECOND = new String2FieldBridgeAdaptor( DateBridge.DATE_SECOND ); + public static final FieldBridge DATE_YEAR = new TwoWayString2FieldBridgeAdaptor( DateBridge.DATE_YEAR ); + public static final FieldBridge DATE_MONTH = new TwoWayString2FieldBridgeAdaptor( DateBridge.DATE_MONTH ); + public static final FieldBridge DATE_DAY = new TwoWayString2FieldBridgeAdaptor( DateBridge.DATE_DAY ); + public static final FieldBridge DATE_HOUR = new TwoWayString2FieldBridgeAdaptor( DateBridge.DATE_HOUR ); + public static final FieldBridge DATE_MINUTE = new TwoWayString2FieldBridgeAdaptor( DateBridge.DATE_MINUTE ); + public static final FieldBridge DATE_SECOND = new TwoWayString2FieldBridgeAdaptor( DateBridge.DATE_SECOND ); public static final TwoWayFieldBridge DATE_MILLISECOND = new TwoWayString2FieldBridgeAdaptor( DateBridge.DATE_MILLISECOND ); Modified: search/branches/jboss_cache_integration/src/java/org/hibernate/search/engine/DocumentBuilder.java =================================================================== --- search/branches/jboss_cache_integration/src/java/org/hibernate/search/engine/DocumentBuilder.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/java/org/hibernate/search/engine/DocumentBuilder.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -47,6 +47,7 @@ import org.hibernate.search.bridge.BridgeFactory; import org.hibernate.search.bridge.FieldBridge; import org.hibernate.search.bridge.TwoWayFieldBridge; +import org.hibernate.search.bridge.TwoWayString2FieldBridgeAdaptor; import org.hibernate.search.store.DirectoryProvider; import org.hibernate.search.store.IndexShardingStrategy; import org.hibernate.search.util.BinderHelper; @@ -81,7 +82,13 @@ private int maxLevel = Integer.MAX_VALUE; private final ScopedAnalyzer analyzer = new ScopedAnalyzer(); private Similarity similarity; + private boolean isRoot; + //if composite id, use of (a, b) in ((1,2), (3,4)) fails on most database + private boolean safeFromTupleId; + public boolean isRoot() { + return isRoot; + } public DocumentBuilder(XClass clazz, InitContext context, DirectoryProvider[] directoryProviders, IndexShardingStrategy shardingStrategy, ReflectionManager reflectionManager) { @@ -103,6 +110,9 @@ if ( idKeywordName == null ) { throw new SearchException( "No document id in: " + clazz.getName() ); } + //if composite id, use of (a, b) in ((1,2)TwoWayString2FieldBridgeAdaptor, (3,4)) fails on most database + //a TwoWayString2FieldBridgeAdaptor is never a composite id + safeFromTupleId = TwoWayString2FieldBridgeAdaptor.class.isAssignableFrom( idBridge.getClass() ); } private Analyzer getAnalyzer(XAnnotatedElement annotatedElement, InitContext context) { @@ -788,7 +798,16 @@ for (Class currentClass : indexedClasses) { if ( plainClass.isAssignableFrom( currentClass ) ) tempMappedSubclasses.add( currentClass ); } - mappedSubclasses = Collections.unmodifiableSet( tempMappedSubclasses ); + this.mappedSubclasses = Collections.unmodifiableSet( tempMappedSubclasses ); + Class superClass = plainClass.getSuperclass(); + this.isRoot = true; + while ( superClass != null) { + if ( indexedClasses.contains( superClass ) ) { + this.isRoot = false; + break; + } + superClass = superClass.getSuperclass(); + } } @@ -796,6 +815,14 @@ return mappedSubclasses; } + /** + * Make sure to return false if there is a risk of composite id + * if composite id, use of (a, b) in ((1,2), (3,4)) fails on most database + */ + public boolean isSafeFromTupleId() { + return safeFromTupleId; + } + private static class PropertiesMetadata { public Float boost; public Analyzer analyzer; Added: search/branches/jboss_cache_integration/src/java/org/hibernate/search/engine/MultiClassesQueryLoader.java =================================================================== --- search/branches/jboss_cache_integration/src/java/org/hibernate/search/engine/MultiClassesQueryLoader.java (rev 0) +++ search/branches/jboss_cache_integration/src/java/org/hibernate/search/engine/MultiClassesQueryLoader.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -0,0 +1,124 @@ +package org.hibernate.search.engine; + +import java.util.List; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Map; +import java.util.Set; +import java.util.HashMap; +import java.util.Arrays; + +import org.hibernate.Session; +import org.hibernate.Criteria; +import org.hibernate.annotations.common.AssertionFailure; + +/** + * @author Emmanuel Bernard + */ +public class MultiClassesQueryLoader implements Loader { + private Session session; + private SearchFactoryImplementor searchFactoryImplementor; + private List<RootEntityMetadata> entityMatadata; + //useful if loading with a query is unsafe + private ObjectLoader objectLoader; + + public void init(Session session, SearchFactoryImplementor searchFactoryImplementor) { + this.session = session; + this.searchFactoryImplementor = searchFactoryImplementor; + this.objectLoader = new ObjectLoader(); + this.objectLoader.init( session, searchFactoryImplementor ); + } + + public void setEntityTypes(Class[] entityTypes) { + List<Class> safeEntityTypes; + //TODO should we go find the root entity for a given class rather than just checking for it's root status? + // root entity could lead to quite inefficient queries in Hibnernate when using table per class + if ( entityTypes.length == 0 ) { + //support all classes + safeEntityTypes = new ArrayList<Class>(); + for( Map.Entry<Class, DocumentBuilder<Object>> entry : searchFactoryImplementor.getDocumentBuilders().entrySet() ) { + //get only root entities to limit queries + if ( entry.getValue().isRoot() ) { + safeEntityTypes.add( entry.getKey() ); + } + } + } + else { + safeEntityTypes = Arrays.asList(entityTypes); + } + entityMatadata = new ArrayList<RootEntityMetadata>( safeEntityTypes.size() ); + for (Class clazz : safeEntityTypes) { + entityMatadata.add( new RootEntityMetadata( clazz, searchFactoryImplementor, session ) ); + } + } + + public Object load(EntityInfo entityInfo) { + return ObjectLoaderHelper.load( entityInfo, session ); + } + + public List load(EntityInfo... entityInfos) { + if ( entityInfos.length == 0 ) return Collections.EMPTY_LIST; + if ( entityInfos.length == 1 ) { + final Object entity = load( entityInfos[0] ); + if ( entity == null ) { + return Collections.EMPTY_LIST; + } + else { + final List<Object> list = new ArrayList<Object>( 1 ); + list.add( entity ); + return list; + } + } + + //split EntityInfo per root entity + Map<RootEntityMetadata, List<EntityInfo>> entityinfoBuckets = + new HashMap<RootEntityMetadata, List<EntityInfo>>( entityMatadata.size()); + for (EntityInfo entityInfo : entityInfos) { + boolean found = false; + for (RootEntityMetadata rootEntityInfo : entityMatadata) { + if ( rootEntityInfo.mappedSubclasses.contains( entityInfo.clazz ) ) { + List<EntityInfo> bucket = entityinfoBuckets.get( rootEntityInfo ); + if ( bucket == null ) { + bucket = new ArrayList<EntityInfo>(); + entityinfoBuckets.put( rootEntityInfo, bucket ); + } + bucket.add( entityInfo ); + found = true; + break; //we stop looping for the right bucket + } + } + if (!found) throw new AssertionFailure( "Could not find root entity for " + entityInfo.clazz ); + } + + //initialize objects by bucket + for ( Map.Entry<RootEntityMetadata, List<EntityInfo>> entry : entityinfoBuckets.entrySet() ) { + final RootEntityMetadata key = entry.getKey(); + final List<EntityInfo> value = entry.getValue(); + final EntityInfo[] bucketEntityInfos = value.toArray( new EntityInfo[value.size()] ); + if ( key.useObjectLoader ) { + objectLoader.load( bucketEntityInfos ); + } + else { + ObjectLoaderHelper.initializeObjects( bucketEntityInfos, + key.criteria, key.rootEntity, searchFactoryImplementor); + } + } + return ObjectLoaderHelper.returnAlreadyLoadedObjectsInCorrectOrder( entityInfos, session ); + } + + private static class RootEntityMetadata { + public final Class rootEntity; + public final Set<Class> mappedSubclasses; + private final Criteria criteria; + public final boolean useObjectLoader; + + RootEntityMetadata(Class rootEntity, SearchFactoryImplementor searchFactoryImplementor, Session session) { + this.rootEntity = rootEntity; + DocumentBuilder provider = searchFactoryImplementor.getDocumentBuilders().get( rootEntity ); + if ( provider == null) throw new AssertionFailure("Provider not found for class: " + rootEntity); + this.mappedSubclasses = provider.getMappedSubclasses(); + this.criteria = session.createCriteria( rootEntity ); + this.useObjectLoader = !provider.isSafeFromTupleId(); + } + } +} Modified: search/branches/jboss_cache_integration/src/java/org/hibernate/search/engine/ObjectLoader.java =================================================================== --- search/branches/jboss_cache_integration/src/java/org/hibernate/search/engine/ObjectLoader.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/java/org/hibernate/search/engine/ObjectLoader.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -3,6 +3,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Collections; import org.hibernate.Hibernate; import org.hibernate.Session; @@ -21,25 +22,22 @@ } public Object load(EntityInfo entityInfo) { - //be sure to get an initialized object - Object maybeProxy = session.get( entityInfo.clazz, entityInfo.id ); - try { - Hibernate.initialize( maybeProxy ); - } - catch (RuntimeException e) { - if ( LoaderHelper.isObjectNotFoundException( e ) ) { - log.debug( "Object found in Search index but not in database: {} with id {}", - entityInfo.clazz, entityInfo.id ); - maybeProxy = null; + return ObjectLoaderHelper.load( entityInfo, session ); + } + + public List load(EntityInfo... entityInfos) { + if ( entityInfos.length == 0 ) return Collections.EMPTY_LIST; + if ( entityInfos.length == 1 ) { + final Object entity = load( entityInfos[0] ); + if ( entity == null ) { + return Collections.EMPTY_LIST; } else { - throw e; + final List<Object> list = new ArrayList<Object>( 1 ); + list.add( entity ); + return list; } } - return maybeProxy; - } - - public List load(EntityInfo... entityInfos) { //use load to benefit from the batch-size //we don't face proxy casting issues since the exact class is extracted from the index for (EntityInfo entityInfo : entityInfos) { Added: search/branches/jboss_cache_integration/src/java/org/hibernate/search/engine/ObjectLoaderHelper.java =================================================================== --- search/branches/jboss_cache_integration/src/java/org/hibernate/search/engine/ObjectLoaderHelper.java (rev 0) +++ search/branches/jboss_cache_integration/src/java/org/hibernate/search/engine/ObjectLoaderHelper.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -0,0 +1,90 @@ +package org.hibernate.search.engine; + +import java.util.Collections; +import java.util.List; +import java.util.ArrayList; +import java.io.Serializable; + +import org.hibernate.Hibernate; +import org.hibernate.Session; +import org.hibernate.Criteria; +import org.hibernate.type.EntityType; +import org.hibernate.criterion.Disjunction; +import org.hibernate.criterion.Restrictions; +import org.hibernate.annotations.common.AssertionFailure; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author Emmanuel Bernard + */ +public class ObjectLoaderHelper { + + private static final int MAX_IN_CLAUSE = 500; + private static final Logger log = LoggerFactory.getLogger( ObjectLoader.class ); + + public static Object load(EntityInfo entityInfo, Session session) { + //be sure to get an initialized object but save from ONFE and ENFE + Object maybeProxy = session.load( entityInfo.clazz, entityInfo.id ); + try { + Hibernate.initialize( maybeProxy ); + } + catch (RuntimeException e) { + if ( LoaderHelper.isObjectNotFoundException( e ) ) { + log.debug( "Object found in Search index but not in database: {} with id {}", + entityInfo.clazz, entityInfo.id ); + maybeProxy = null; + } + else { + throw e; + } + } + return maybeProxy; + } + + public static void initializeObjects(EntityInfo[] entityInfos, Criteria criteria, Class entityType, + SearchFactoryImplementor searchFactoryImplementor) { + final int maxResults = entityInfos.length; + if ( maxResults == 0 ) return; + + DocumentBuilder builder = searchFactoryImplementor.getDocumentBuilders().get( entityType ); + String idName = builder.getIdentifierName(); + int loop = maxResults / MAX_IN_CLAUSE; + boolean exact = maxResults % MAX_IN_CLAUSE == 0; + if ( !exact ) loop++; + Disjunction disjunction = Restrictions.disjunction(); + for (int index = 0; index < loop; index++) { + int max = index * MAX_IN_CLAUSE + MAX_IN_CLAUSE <= maxResults ? + index * MAX_IN_CLAUSE + MAX_IN_CLAUSE : + maxResults; + List<Serializable> ids = new ArrayList<Serializable>( max - index * MAX_IN_CLAUSE ); + for (int entityInfoIndex = index * MAX_IN_CLAUSE; entityInfoIndex < max; entityInfoIndex++) { + ids.add( entityInfos[entityInfoIndex].id ); + } + disjunction.add( Restrictions.in( idName, ids ) ); + } + criteria.add( disjunction ); + criteria.list(); //load all objects + } + + + public static List returnAlreadyLoadedObjectsInCorrectOrder(EntityInfo[] entityInfos, Session session) { + //mandatory to keep the same ordering + List result = new ArrayList( entityInfos.length ); + for (EntityInfo entityInfo : entityInfos) { + Object element = session.load( entityInfo.clazz, entityInfo.id ); + if ( Hibernate.isInitialized( element ) ) { + //all existing elements should have been loaded by the query, + //the other ones are missing ones + result.add( element ); + } + else { + if ( log.isDebugEnabled() ) { + log.debug( "Object found in Search index but not in database: {} with {}", + entityInfo.clazz, entityInfo.id ); + } + } + } + return result; + } +} Modified: search/branches/jboss_cache_integration/src/java/org/hibernate/search/engine/ProjectionLoader.java =================================================================== --- search/branches/jboss_cache_integration/src/java/org/hibernate/search/engine/ProjectionLoader.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/java/org/hibernate/search/engine/ProjectionLoader.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -10,6 +10,7 @@ /** * @author Emmanuel Bernard */ +//TODO change the underlying ObjectLoader to a MutliClassesQueryLoader public class ProjectionLoader implements Loader { private SearchFactoryImplementor searchFactoryImplementor; private Session session; Modified: search/branches/jboss_cache_integration/src/java/org/hibernate/search/engine/QueryLoader.java =================================================================== --- search/branches/jboss_cache_integration/src/java/org/hibernate/search/engine/QueryLoader.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/java/org/hibernate/search/engine/QueryLoader.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -2,15 +2,13 @@ package org.hibernate.search.engine; import java.util.ArrayList; -import java.util.Collections; import java.util.List; +import java.util.Collections; import org.hibernate.Criteria; import org.hibernate.Hibernate; import org.hibernate.Session; import org.hibernate.annotations.common.AssertionFailure; -import org.hibernate.criterion.Disjunction; -import org.hibernate.criterion.Restrictions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -18,7 +16,6 @@ * @author Emmanuel Bernard */ public class QueryLoader implements Loader { - private static final int MAX_IN_CLAUSE = 500; private final Logger log = LoggerFactory.getLogger( QueryLoader.class ); private Session session; @@ -36,66 +33,16 @@ } public Object load(EntityInfo entityInfo) { - //be sure to get an initialized object - Object maybeProxy = session.get( entityInfo.clazz, entityInfo.id ); - try { - Hibernate.initialize( maybeProxy ); - } - catch (RuntimeException e) { - if ( LoaderHelper.isObjectNotFoundException( e ) ) { - log.debug( "Object found in Search index but not in database: {} with id {}", - entityInfo.clazz, entityInfo.id ); - maybeProxy = null; - } - else { - throw e; - } - } - return maybeProxy; + return ObjectLoaderHelper.load( entityInfo, session ); } public List load(EntityInfo... entityInfos) { - final int maxResults = entityInfos.length; - if ( maxResults == 0 ) return Collections.EMPTY_LIST; + if ( entityInfos.length == 0 ) return Collections.EMPTY_LIST; if ( entityType == null ) throw new AssertionFailure( "EntityType not defined" ); if ( criteria == null ) criteria = session.createCriteria( entityType ); - DocumentBuilder builder = searchFactoryImplementor.getDocumentBuilders().get( entityType ); - String idName = builder.getIdentifierName(); - int loop = maxResults / MAX_IN_CLAUSE; - boolean exact = maxResults % MAX_IN_CLAUSE == 0; - if ( !exact ) loop++; - Disjunction disjunction = Restrictions.disjunction(); - for (int index = 0; index < loop; index++) { - int max = index * MAX_IN_CLAUSE + MAX_IN_CLAUSE <= maxResults ? - index * MAX_IN_CLAUSE + MAX_IN_CLAUSE : - maxResults; - List ids = new ArrayList( max - index * MAX_IN_CLAUSE ); - for (int entityInfoIndex = index * MAX_IN_CLAUSE; entityInfoIndex < max; entityInfoIndex++) { - ids.add( entityInfos[entityInfoIndex].id ); - } - disjunction.add( Restrictions.in( idName, ids ) ); - } - criteria.add( disjunction ); - criteria.list(); //load all objects - - //mandatory to keep the same ordering - List result = new ArrayList( entityInfos.length ); - for (EntityInfo entityInfo : entityInfos) { - Object element = session.load( entityInfo.clazz, entityInfo.id ); - if ( Hibernate.isInitialized( element ) ) { - //all existing elements should have been loaded by the query, - //the other ones are missing ones - result.add( element ); - } - else { - if ( log.isDebugEnabled() ) { - log.debug( "Object found in Search index but not in database: {} with {}", - entityInfo.clazz, entityInfo.id ); - } - } - } - return result; + ObjectLoaderHelper.initializeObjects( entityInfos, criteria, entityType, searchFactoryImplementor ); + return ObjectLoaderHelper.returnAlreadyLoadedObjectsInCorrectOrder( entityInfos, session ); } public void setCriteria(Criteria criteria) { Modified: search/branches/jboss_cache_integration/src/java/org/hibernate/search/engine/SearchFactoryImplementor.java =================================================================== --- search/branches/jboss_cache_integration/src/java/org/hibernate/search/engine/SearchFactoryImplementor.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/java/org/hibernate/search/engine/SearchFactoryImplementor.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -3,7 +3,7 @@ import java.util.Map; import java.util.Set; -import java.util.concurrent.locks.ReentrantLock; +import java.util.concurrent.locks.Lock; import org.hibernate.search.SearchFactory; import org.hibernate.search.backend.BackendQueueProcessorFactory; @@ -26,8 +26,6 @@ Map<Class, DocumentBuilder<Object>> getDocumentBuilders(); - Map<DirectoryProvider, ReentrantLock> getLockableDirectoryProviders(); - Worker getWorker(); void addOptimizerStrategy(DirectoryProvider<?> provider, OptimizerStrategy optimizerStrategy); @@ -49,4 +47,11 @@ void addClassToDirectoryProvider(Class clazz, DirectoryProvider<?> directoryProvider); Set<Class> getClassesInDirectoryProvider(DirectoryProvider<?> directoryProvider); + + Set<DirectoryProvider> getDirectoryProviders(); + + Lock getDirectoryProviderLock(DirectoryProvider dp); + + void addDirectoryProvider(DirectoryProvider<?> provider); + } Modified: search/branches/jboss_cache_integration/src/java/org/hibernate/search/filter/MRUFilterCachingStrategy.java =================================================================== --- search/branches/jboss_cache_integration/src/java/org/hibernate/search/filter/MRUFilterCachingStrategy.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/java/org/hibernate/search/filter/MRUFilterCachingStrategy.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -5,7 +5,7 @@ import org.apache.lucene.search.Filter; import org.hibernate.search.Environment; -import org.hibernate.search.SearchException; +import org.hibernate.search.backend.configuration.ConfigurationParseHelper; import org.hibernate.util.SoftLimitMRUCache; /** @@ -16,23 +16,12 @@ * @author Emmanuel Bernard */ public class MRUFilterCachingStrategy implements FilterCachingStrategy { - private static final String DEFAULT_SIZE = "128"; + private static final int DEFAULT_SIZE = 128; private SoftLimitMRUCache cache; private static final String SIZE = Environment.FILTER_CACHING_STRATEGY + ".size"; public void initialize(Properties properties) { - int size; - try { - size = Integer.parseInt( - properties.getProperty( SIZE, DEFAULT_SIZE ) - ); - } - catch (NumberFormatException nfe) { - throw new SearchException( - "Unable to parse " + SIZE + ": " + properties.getProperty( SIZE, DEFAULT_SIZE ), nfe - ); - } - + int size = ConfigurationParseHelper.getIntValue( properties, SIZE, DEFAULT_SIZE ); cache = new SoftLimitMRUCache( size ); } Modified: search/branches/jboss_cache_integration/src/java/org/hibernate/search/impl/SearchFactoryImpl.java =================================================================== --- search/branches/jboss_cache_integration/src/java/org/hibernate/search/impl/SearchFactoryImpl.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/java/org/hibernate/search/impl/SearchFactoryImpl.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -13,6 +13,8 @@ import java.util.Properties; import java.util.Set; import java.util.WeakHashMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import org.apache.lucene.analysis.Analyzer; @@ -60,29 +62,25 @@ Version.touch(); } - private static final Logger log = LoggerFactory.getLogger( SearchFactoryImpl.class ); + private final Logger log = LoggerFactory.getLogger( SearchFactoryImpl.class ); private final Map<Class, DocumentBuilder<Object>> documentBuilders = new HashMap<Class, DocumentBuilder<Object>>(); //keep track of the index modifiers per DirectoryProvider since multiple entity can use the same directory provider - //TODO move the ReentrantLock into DirectoryProviderData.lock, add a getDPLock(DP) and add a Set<DP> getDirectoryProviders() method. - private final Map<DirectoryProvider, ReentrantLock> lockableDirectoryProviders = - new HashMap<DirectoryProvider, ReentrantLock>(); - private final Map<DirectoryProvider, DirectoryProviderData> dirProviderData = - new HashMap<DirectoryProvider, DirectoryProviderData>(); - private Worker worker; - private ReaderProvider readerProvider; + private final Map<DirectoryProvider, DirectoryProviderData> dirProviderData = new HashMap<DirectoryProvider, DirectoryProviderData>(); + private final Worker worker; + private final ReaderProvider readerProvider; private BackendQueueProcessorFactory backendQueueProcessorFactory; private final Map<String, FilterDef> filterDefinitions = new HashMap<String, FilterDef>(); - private FilterCachingStrategy filterCachingStrategy; + private final FilterCachingStrategy filterCachingStrategy; private Map<String, Analyzer> analyzers; - private boolean stopped = false; + private final AtomicBoolean stopped = new AtomicBoolean( false ); /** * Each directory provider (index) can have its own performance settings. */ private Map<DirectoryProvider, LuceneIndexingParameters> dirProviderIndexingParams = new HashMap<DirectoryProvider, LuceneIndexingParameters>(); - private String indexingStrategy; + private final String indexingStrategy; public BackendQueueProcessorFactory getBackendQueueProcessorFactory() { @@ -97,24 +95,24 @@ public SearchFactoryImpl(Configuration cfg) { //yuk ReflectionManager reflectionManager = getReflectionManager( cfg ); - setIndexingStrategy(cfg); //need to be done before the document builds - InitContext context = new InitContext(cfg); - initDocumentBuilders(cfg, reflectionManager, context ); + this.indexingStrategy = defineIndexingStrategy( cfg ); //need to be done before the document builds + initDocumentBuilders( cfg, reflectionManager ); Set<Class> indexedClasses = documentBuilders.keySet(); for (DocumentBuilder builder : documentBuilders.values()) { builder.postInitialize( indexedClasses ); } - worker = WorkerFactory.createWorker( cfg, this ); - readerProvider = ReaderProviderFactory.createReaderProvider( cfg, this ); - buildFilterCachingStrategy( cfg.getProperties() ); + this.worker = WorkerFactory.createWorker( cfg, this ); + this.readerProvider = ReaderProviderFactory.createReaderProvider( cfg, this ); + this.filterCachingStrategy = buildFilterCachingStrategy( cfg.getProperties() ); } - private void setIndexingStrategy(Configuration cfg) { - indexingStrategy = cfg.getProperties().getProperty( Environment.INDEXING_STRATEGY, "event" ); + private static String defineIndexingStrategy(Configuration cfg) { + String indexingStrategy = cfg.getProperties().getProperty( Environment.INDEXING_STRATEGY, "event" ); if ( ! ("event".equals( indexingStrategy ) || "manual".equals( indexingStrategy ) ) ) { - throw new SearchException(Environment.INDEXING_STRATEGY + " unknown: " + indexingStrategy); + throw new SearchException( Environment.INDEXING_STRATEGY + " unknown: " + indexingStrategy ); } + return indexingStrategy; } public String getIndexingStrategy() { @@ -122,8 +120,7 @@ } public void close() { - if (!stopped) { - stopped = true; + if ( stopped.compareAndSet( false, true) ) { try { worker.close(); } @@ -131,7 +128,7 @@ log.error( "Worker raises an exception on close()", e ); } //TODO move to DirectoryProviderFactory for cleaner - for (DirectoryProvider dp : lockableDirectoryProviders.keySet() ) { + for (DirectoryProvider dp : getDirectoryProviders() ) { try { dp.stop(); } @@ -223,7 +220,6 @@ SearchFactoryImpl searchFactory = contextMap.get( cfg ); if ( searchFactory == null ) { searchFactory = new SearchFactoryImpl( cfg ); - contextMap.put( cfg, searchFactory ); } return searchFactory; @@ -234,8 +230,8 @@ return documentBuilders; } - public Map<DirectoryProvider, ReentrantLock> getLockableDirectoryProviders() { - return lockableDirectoryProviders; + public Set<DirectoryProvider> getDirectoryProviders() { + return this.dirProviderData.keySet(); } public Worker getWorker() { @@ -272,7 +268,7 @@ ReflectionManager reflectionManager; try { //TODO introduce a ReflectionManagerHolder interface to avoid reflection - //I want to avoid hard link between HAN and Validator for usch a simple need + //I want to avoid hard link between HAN and Validator for such a simple need //reuse the existing reflectionManager one when possible reflectionManager = (ReflectionManager) cfg.getClass().getMethod( "getReflectionManager" ).invoke( cfg ); @@ -311,11 +307,12 @@ return analyzer; } - private void initDocumentBuilders(Configuration cfg, ReflectionManager reflectionManager, InitContext context) { + private void initDocumentBuilders(Configuration cfg, ReflectionManager reflectionManager) { + InitContext context = new InitContext( cfg ); Iterator iter = cfg.getClassMappings(); DirectoryProviderFactory factory = new DirectoryProviderFactory(); - while (iter.hasNext()) { + while ( iter.hasNext() ) { PersistentClass clazz = (PersistentClass) iter.next(); Class<?> mappedClass = clazz.getMappedClass(); if (mappedClass != null) { @@ -340,7 +337,8 @@ factory.startDirectoryProviders(); } - private void buildFilterCachingStrategy(Properties properties) { + private static FilterCachingStrategy buildFilterCachingStrategy(Properties properties) { + FilterCachingStrategy filterCachingStrategy; String impl = properties.getProperty( Environment.FILTER_CACHING_STRATEGY ); if ( StringHelper.isEmpty( impl ) || "mru".equalsIgnoreCase( impl ) ) { filterCachingStrategy = new MRUFilterCachingStrategy(); @@ -361,6 +359,7 @@ } } filterCachingStrategy.initialize( properties ); + return filterCachingStrategy; } public FilterCachingStrategy getFilterCachingStrategy() { @@ -372,7 +371,17 @@ } private static class DirectoryProviderData { + public final Lock dirLock = new ReentrantLock(); public OptimizerStrategy optimizerStrategy; public Set<Class> classes = new HashSet<Class>(2); } + + public Lock getDirectoryProviderLock(DirectoryProvider dp) { + return this.dirProviderData.get( dp ).dirLock; + } + + public void addDirectoryProvider(DirectoryProvider<?> provider) { + this.dirProviderData.put( provider, new DirectoryProviderData() ); + } + } Modified: search/branches/jboss_cache_integration/src/java/org/hibernate/search/jpa/FullTextEntityManager.java =================================================================== --- search/branches/jboss_cache_integration/src/java/org/hibernate/search/jpa/FullTextEntityManager.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/java/org/hibernate/search/jpa/FullTextEntityManager.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -22,6 +22,8 @@ /** * Force the (re)indexing of a given <b>managed</b> object. * Indexation is batched per transaction + * + * @throws IllegalArgumentException if entity is null or not an @Indexed entity */ void index(Object entity); @@ -34,6 +36,8 @@ * * @param entityType * @param id + * + * @throws IllegalArgumentException if entityType is null or not an @Indexed entity type */ public void purge(Class entityType, Serializable id); @@ -41,7 +45,15 @@ * Remove all entities from a particular class of an index. * * @param entityType + * + * @throws IllegalArgumentException if entityType is null or not an @Indexed entity type */ public void purgeAll(Class entityType); + /** + * flush index change + * Force Hibernate Search to apply all changes to the index no waiting for the batch limit + */ + public void flushToIndexes(); + } Modified: search/branches/jboss_cache_integration/src/java/org/hibernate/search/jpa/impl/FullTextEntityManagerImpl.java =================================================================== --- search/branches/jboss_cache_integration/src/java/org/hibernate/search/jpa/impl/FullTextEntityManagerImpl.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/java/org/hibernate/search/jpa/impl/FullTextEntityManagerImpl.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -77,7 +77,11 @@ getFullTextSession().purgeAll( entityType ); } + public void flushToIndexes() { + getFullTextSession().flushToIndexes(); + } + public void persist(Object entity) { em.persist( entity ); } Modified: search/branches/jboss_cache_integration/src/java/org/hibernate/search/query/FullTextQueryImpl.java =================================================================== --- search/branches/jboss_cache_integration/src/java/org/hibernate/search/query/FullTextQueryImpl.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/java/org/hibernate/search/query/FullTextQueryImpl.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -43,10 +43,10 @@ import org.hibernate.search.engine.EntityInfo; import org.hibernate.search.engine.FilterDef; import org.hibernate.search.engine.Loader; -import org.hibernate.search.engine.ObjectLoader; import org.hibernate.search.engine.ProjectionLoader; import org.hibernate.search.engine.QueryLoader; import org.hibernate.search.engine.SearchFactoryImplementor; +import org.hibernate.search.engine.MultiClassesQueryLoader; import org.hibernate.search.filter.ChainedFilter; import org.hibernate.search.filter.FilterKey; import org.hibernate.search.reader.ReaderProvider; @@ -66,7 +66,7 @@ //TODO implements setParameter() public class FullTextQueryImpl extends AbstractQueryImpl implements FullTextQuery { private final Logger log = LoggerFactory.getLogger( FullTextQueryImpl.class ); - private org.apache.lucene.search.Query luceneQuery; + private final org.apache.lucene.search.Query luceneQuery; private Class[] classes; private Set<Class> classesAndSubclasses; //optimization: if we can avoid the filter clause (we can most of the time) do it as it has a significant perf impact @@ -124,7 +124,7 @@ //find the directories IndexSearcher searcher = buildSearcher( searchFactoryImplementor ); if ( searcher == null ) { - return new IteratorImpl( new ArrayList<EntityInfo>( 0 ), noLoader ); + return new IteratorImpl( Collections.EMPTY_LIST, noLoader ); } try { Hits hits = getHits( searcher ); @@ -185,15 +185,16 @@ return loader; } else if ( classes.length == 1 ) { - QueryLoader loader = new QueryLoader(); + final QueryLoader loader = new QueryLoader(); loader.init( session, searchFactoryImplementor ); loader.setEntityType( classes[0] ); return loader; } else { - final ObjectLoader objectLoader = new ObjectLoader(); - objectLoader.init( session, searchFactoryImplementor ); - return objectLoader; + final MultiClassesQueryLoader loader = new MultiClassesQueryLoader(); + loader.init( session, searchFactoryImplementor ); + loader.setEntityTypes( classes ); + return loader; } } @@ -234,7 +235,7 @@ SearchFactoryImplementor searchFactoryImplementor = ContextHelper.getSearchFactoryBySFI( session ); //find the directories IndexSearcher searcher = buildSearcher( searchFactoryImplementor ); - if ( searcher == null ) return new ArrayList( 0 ); + if ( searcher == null ) return Collections.EMPTY_LIST; Hits hits; try { hits = getHits( searcher ); Modified: search/branches/jboss_cache_integration/src/java/org/hibernate/search/reader/ReaderProviderHelper.java =================================================================== --- search/branches/jboss_cache_integration/src/java/org/hibernate/search/reader/ReaderProviderHelper.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/java/org/hibernate/search/reader/ReaderProviderHelper.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -2,10 +2,12 @@ package org.hibernate.search.reader; import java.io.IOException; +import java.lang.reflect.Field; import java.util.HashSet; import java.util.Set; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.MultiReader; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MultiSearcher; import org.apache.lucene.search.Searchable; @@ -15,6 +17,28 @@ * @author Emmanuel Bernard */ public abstract class ReaderProviderHelper { + + private static final Field subReadersField = getSubReadersField(); + + private static Field getSubReadersField() { + try { + Field field = MultiReader.class.getDeclaredField( "subReaders" ); + if ( ! field.isAccessible() ) field.setAccessible( true ); + return field; + } + catch (NoSuchFieldException e) { + throw new SearchException( "Incompatible version of Lucene: MultiReader.subReaders not available", e ); + } + } + + public static IndexReader[] getSubReadersFromMultiReader(MultiReader parentReader) { + try { + return (IndexReader[]) subReadersField.get( parentReader ); + } catch (IllegalAccessException e) { + throw new SearchException( "Incompatible version of Lucene: MultiReader.subReaders not accessible", e ); + } + } + @SuppressWarnings( { "ThrowableInstanceNeverThrown" } ) public static IndexReader buildMultiReader(int length, IndexReader[] readers) { if ( length == 0 ) { Modified: search/branches/jboss_cache_integration/src/java/org/hibernate/search/reader/SharedReaderProvider.java =================================================================== --- search/branches/jboss_cache_integration/src/java/org/hibernate/search/reader/SharedReaderProvider.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/java/org/hibernate/search/reader/SharedReaderProvider.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -2,7 +2,6 @@ package org.hibernate.search.reader; import java.io.IOException; -import java.lang.reflect.Field; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -28,7 +27,6 @@ * @author Emmanuel Bernard */ public class SharedReaderProvider implements ReaderProvider { - private static Field subReadersField; private final Logger log = LoggerFactory.getLogger ( SharedReaderProvider.class ); /** * nonfair lock. Need to be acquired on indexReader acquisition or release (semaphore) @@ -151,7 +149,7 @@ if ( outOfDateReader != null ) { ReaderData readerData = searchIndexReaderSemaphores.get( outOfDateReader ); if ( readerData == null ) { - closeOutOfDateReader = false; //already removed by another prevous thread + closeOutOfDateReader = false; //already removed by another previous thread } else if ( readerData.semaphore == 0 ) { searchIndexReaderSemaphores.remove( outOfDateReader ); @@ -211,12 +209,7 @@ IndexReader[] readers; //TODO should it be CacheableMultiReader? Probably no if ( reader instanceof MultiReader ) { - try { - readers = (IndexReader[]) subReadersField.get( reader ); - } - catch (IllegalAccessException e) { - throw new SearchException( "Incompatible version of Lucene: MultiReader.subReaders not accessible", e ); - } + readers = ReaderProviderHelper.getSubReadersFromMultiReader( (MultiReader) reader ); if ( trace ) log.trace( "Closing MultiReader: {}", reader ); } else { @@ -289,16 +282,7 @@ } public void initialize(Properties props, SearchFactoryImplementor searchFactoryImplementor) { - if ( subReadersField == null ) { - try { - subReadersField = MultiReader.class.getDeclaredField( "subReaders" ); - if ( !subReadersField.isAccessible() ) subReadersField.setAccessible( true ); - } - catch (NoSuchFieldException e) { - throw new SearchException( "Incompatible version of Lucene: MultiReader.subReaders not accessible", e ); - } - } - Set<DirectoryProvider> providers = searchFactoryImplementor.getLockableDirectoryProviders().keySet(); + Set<DirectoryProvider> providers = searchFactoryImplementor.getDirectoryProviders(); perDirectoryProviderManipulationLocks = new HashMap<DirectoryProvider, Lock>( providers.size() ); for (DirectoryProvider dp : providers) { perDirectoryProviderManipulationLocks.put( dp, new ReentrantLock() ); @@ -306,7 +290,7 @@ perDirectoryProviderManipulationLocks = Collections.unmodifiableMap( perDirectoryProviderManipulationLocks ); } - private class ReaderData { + private static class ReaderData { public ReaderData(int semaphore, DirectoryProvider provider) { this.semaphore = semaphore; Added: search/branches/jboss_cache_integration/src/java/org/hibernate/search/reader/SharingBufferReaderProvider.java =================================================================== --- search/branches/jboss_cache_integration/src/java/org/hibernate/search/reader/SharingBufferReaderProvider.java (rev 0) +++ search/branches/jboss_cache_integration/src/java/org/hibernate/search/reader/SharingBufferReaderProvider.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -0,0 +1,230 @@ +package org.hibernate.search.reader; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.MultiReader; +import org.hibernate.annotations.common.AssertionFailure; +import org.hibernate.search.SearchException; +import org.hibernate.search.engine.SearchFactoryImplementor; +import org.hibernate.search.store.DirectoryProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * As does SharedReaderProvider this also shares IndexReaders as long as they are "current"; + * main difference with SharedReaderProvider is the way to update the Readers when needed: + * this uses IndexReader.reopen() which should improve performance on larger indexes + * as it shares buffers with previous IndexReader generation for the segments which didn't change. + * Current drawbacks are: need of Lucene > 2.3.0 and less mature (experimental). + * + * @author Sanne Grinovero + */ +public class SharingBufferReaderProvider implements ReaderProvider { + + /** + * contains all Readers (most current per DP and all unclosed old) + */ + //TODO ConcurrentHashMap's constructor could benefit from some hints as arguments. + protected final Map<IndexReader,ReaderUsagePair> allReaders = new ConcurrentHashMap<IndexReader,ReaderUsagePair>(); + + /** + * contains last updated Reader; protected by lockOnOpenLatest (in the values) + */ + protected Map<DirectoryProvider,PerDirectoryLatestReader> currentReaders; + + private final Logger log = LoggerFactory.getLogger( SharingBufferReaderProvider.class ); + + public void closeReader(IndexReader multiReader) { + if ( multiReader == null ) return; + IndexReader[] readers; + if ( multiReader instanceof MultiReader ) { + readers = ReaderProviderHelper.getSubReadersFromMultiReader( (MultiReader) multiReader ); + } + else { + throw new AssertionFailure( "Everything should be wrapped in a MultiReader" ); + } + log.debug( "Closing MultiReader: {}", multiReader ); + for ( IndexReader reader : readers ) { + ReaderUsagePair container = allReaders.get( reader ); + container.close();//virtual + } + log.trace( "IndexReader closed." ); + } + + public void initialize(Properties props, SearchFactoryImplementor searchFactoryImplementor) { + Map<DirectoryProvider,PerDirectoryLatestReader> map = new HashMap<DirectoryProvider,PerDirectoryLatestReader>(); + Set<DirectoryProvider> providers = searchFactoryImplementor.getDirectoryProviders(); + for ( DirectoryProvider provider : providers ) { + try { + map.put( provider, new PerDirectoryLatestReader( provider ) ); + } catch (IOException e) { + throw new SearchException( "Unable to open Lucene IndexReader", e ); + } + } + //FIXME I'm not convinced this non-final fields are safe without locks, but I may be wrong. + currentReaders = Collections.unmodifiableMap( map ); + } + + public IndexReader openReader(DirectoryProvider... directoryProviders) { + int length = directoryProviders.length; + IndexReader[] readers = new IndexReader[length]; + log.debug( "Opening IndexReader for directoryProviders: {}", length ); + for ( int index = 0; index < length; index++ ) { + DirectoryProvider directoryProvider = directoryProviders[index]; + if ( log.isTraceEnabled() ) { + log.trace( "Opening IndexReader from {}", directoryProvider.getDirectory() ); + } + PerDirectoryLatestReader directoryLatestReader = currentReaders.get( directoryProvider ); + readers[index] = directoryLatestReader.refreshAndGet(); + } + // don't use ReaderProviderHelper.buildMultiReader as we need our own cleanup. + if ( length == 0 ) { + return null; + } + else { + try { + return new CacheableMultiReader( readers ); + } + catch (Exception e) { + //Lucene 2.2 used to throw IOExceptions here + for ( IndexReader ir : readers ) { + ReaderUsagePair readerUsagePair = allReaders.get( ir ); + readerUsagePair.close(); + } + throw new SearchException( "Unable to open a MultiReader", e ); + } + } + } + + //overridable method for testability: + protected IndexReader readerFactory(DirectoryProvider provider) throws IOException { + return IndexReader.open( provider.getDirectory() ); + } + + /** + * Container for the couple IndexReader,UsageCounter. + */ + protected final class ReaderUsagePair { + + public final IndexReader reader; + /** + * When reaching 0 (always test on change) the reader should be really + * closed and then discarded. + * Starts at 2 because: + * first usage token is artificial: means "current" is not to be closed (+1) + * additionally when creating it will be used (+1) + */ + protected final AtomicInteger usageCounter = new AtomicInteger( 2 ); + + ReaderUsagePair(IndexReader r) { + reader = r; + } + + /** + * closes the IndexReader if no other resource is using it; + * in this case the reference to this container will also be removed. + */ + public void close() { + int refCount = usageCounter.decrementAndGet(); + if ( refCount==0 ) { + //TODO I've been experimenting with the idea of an async-close: didn't appear to have an interesting benefit, + //so discarded the code. should try with bigger indexes to see if the effect gets more impressive. + ReaderUsagePair removed = allReaders.remove( reader );//remove ourself + try { + reader.close(); + } catch (IOException e) { + log.warn( "Unable to close Lucene IndexReader", e ); + } + assert removed != null; + } + else if ( refCount<0 ) { + //doesn't happen with current code, could help spotting future bugs? + throw new AssertionFailure( "Closing an IndexReader for which you didn't own a lock-token, or somebody else which didn't own closed already." ); + } + } + + public String toString(){ + return "Reader:" + this.hashCode() + " ref.count=" + usageCounter.get(); + } + + } + + /** + * An instance for each DirectoryProvider, + * establishing the association between "current" ReaderUsagePair + * for a DirectoryProvider and it's lock. + */ + protected final class PerDirectoryLatestReader { + + /** + * Reference to the most current IndexReader for a DirectoryProvider; + * guarded by lockOnReplaceCurrent; + */ + public ReaderUsagePair current; //guarded by lockOnReplaceCurrent + private final Lock lockOnReplaceCurrent = new ReentrantLock(); + + /** + * @param provider The DirectoryProvider for which we manage the IndexReader. + * @throws IOException when the index initialization fails. + */ + public PerDirectoryLatestReader(DirectoryProvider provider) throws IOException { + IndexReader reader = readerFactory( provider ); + ReaderUsagePair initialPair = new ReaderUsagePair( reader ); + initialPair.usageCounter.set( 1 );//a token to mark as active (preventing real close). + lockOnReplaceCurrent.lock();//no harm, just ensuring safe publishing. + current = initialPair; + lockOnReplaceCurrent.unlock(); + allReaders.put( reader, initialPair ); + } + + /** + * Gets an updated IndexReader for the current DirectoryProvider; + * the index status will be checked. + * @return the current IndexReader if it's in sync with underlying index, a new one otherwise. + */ + public IndexReader refreshAndGet() { + ReaderUsagePair previousCurrent; + IndexReader updatedReader; + lockOnReplaceCurrent.lock(); + try { + IndexReader beforeUpdateReader = current.reader; + try { + updatedReader = beforeUpdateReader.reopen(); + } catch (IOException e) { + throw new SearchException( "Unable to reopen IndexReader", e ); + } + if ( beforeUpdateReader == updatedReader ) { + previousCurrent = null; + current.usageCounter.incrementAndGet(); + } + else { + ReaderUsagePair newPair = new ReaderUsagePair( updatedReader ); + //no need to increment usageCounter in newPair, as it is constructed with correct number 2. + assert newPair.usageCounter.get() == 2; + previousCurrent = current; + current = newPair; + allReaders.put( updatedReader, newPair );//unfortunately still needs lock + } + } finally { + lockOnReplaceCurrent.unlock(); + } + // doesn't need lock: + if ( previousCurrent != null ) { + previousCurrent.close();// release a token as it's not the current any more. + } + return updatedReader; + } + + } + +} Modified: search/branches/jboss_cache_integration/src/java/org/hibernate/search/store/DirectoryProviderFactory.java =================================================================== --- search/branches/jboss_cache_integration/src/java/org/hibernate/search/store/DirectoryProviderFactory.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/java/org/hibernate/search/store/DirectoryProviderFactory.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -13,6 +13,7 @@ import org.hibernate.search.SearchException; import org.hibernate.search.annotations.Indexed; import org.hibernate.search.backend.LuceneIndexingParameters; +import org.hibernate.search.backend.configuration.ConfigurationParseHelper; import org.hibernate.search.backend.configuration.MaskedProperty; import org.hibernate.search.engine.SearchFactoryImplementor; import org.hibernate.search.impl.SearchFactoryImpl; @@ -46,7 +47,7 @@ public class DirectoryProviderFactory { private final List<DirectoryProvider<?>> providers = new ArrayList<DirectoryProvider<?>>(); - private static String DEFAULT_DIRECTORY_PROVIDER = FSDirectoryProvider.class.getName(); + private static final String DEFAULT_DIRECTORY_PROVIDER = FSDirectoryProvider.class.getName(); private static final String SHARDING_STRATEGY = "sharding_strategy"; private static final String NBR_OF_SHARDS = SHARDING_STRATEGY + ".nbr_of_shards"; @@ -148,8 +149,8 @@ configureIndexingParameters( searchFactoryImplementor, indexProps, provider ); providers.add( provider ); searchFactoryImplementor.addClassToDirectoryProvider(entity, provider); - if ( !searchFactoryImplementor.getLockableDirectoryProviders().containsKey( provider ) ) { - searchFactoryImplementor.getLockableDirectoryProviders().put( provider, new ReentrantLock() ); + if ( ! searchFactoryImplementor.getDirectoryProviders().contains( provider ) ) { + searchFactoryImplementor.addDirectoryProvider( provider ); } return provider; } @@ -208,14 +209,7 @@ return new Properties[] { directoryLocalProperties }; } else { // count shards - int shardsCount; - { - try { - shardsCount = Integer.parseInt( shardsCountValue ); - } catch (NumberFormatException e) { - throw new SearchException( shardsCountValue + " is not a number", e); - } - } + int shardsCount = ConfigurationParseHelper.parseInt( shardsCountValue, shardsCountValue + " is not a number" ); // create shard-specific Props Properties[] shardLocalProperties = new Properties[shardsCount]; for ( int i = 0; i < shardsCount; i++ ) { @@ -258,7 +252,7 @@ } } - public class DirectoryProviders { + public static class DirectoryProviders { private final IndexShardingStrategy shardingStrategy; private final DirectoryProvider[] providers; Modified: search/branches/jboss_cache_integration/src/java/org/hibernate/search/store/DirectoryProviderHelper.java =================================================================== --- search/branches/jboss_cache_integration/src/java/org/hibernate/search/store/DirectoryProviderHelper.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/java/org/hibernate/search/store/DirectoryProviderHelper.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -6,6 +6,7 @@ import java.io.IOException; import org.hibernate.search.SearchException; +import org.hibernate.search.util.FileHelper; import org.hibernate.annotations.common.util.StringHelper; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.index.IndexReader; @@ -23,6 +24,7 @@ private static final Logger log = LoggerFactory.getLogger( DirectoryProviderHelper.class ); private static final String ROOTINDEX_PROP_NAME = "sourceBase"; private static final String RELATIVEINDEX_PROP_NAME = "source"; + public static final String COPYBUFFERSIZE_PROP_NAME = "buffer_size_on_copy"; /** * Build a directory name out of a root and relative path, guessing the significant part @@ -142,5 +144,31 @@ log.debug( "Refresh period: {} seconds", period ); return period * 1000; //per second } + + /** + * Users may configure the number of MB to use as + * "chunk size" for large file copy operations performed + * by DirectoryProviders. + * @param directoryProviderName + * @param properties + * @return the number of Bytes to use as "chunk size" in file copy operations. + */ + public static long getCopyBufferSize(String directoryProviderName, Properties properties) { + String value = properties.getProperty( COPYBUFFERSIZE_PROP_NAME ); + long size = FileHelper.DEFAULT_COPY_BUFFER_SIZE; + if ( value != null ) { + try { + size = Long.parseLong( value ) * 1024 * 1024; //from MB to B. + } catch (NumberFormatException nfe) { + throw new SearchException( "Unable to initialize index " + + directoryProviderName +"; "+ COPYBUFFERSIZE_PROP_NAME + " is not numeric.", nfe ); + } + if ( size <= 0 ) { + throw new SearchException( "Unable to initialize index " + + directoryProviderName +"; "+ COPYBUFFERSIZE_PROP_NAME + " needs to be greater than zero."); + } + } + return size; + } } Modified: search/branches/jboss_cache_integration/src/java/org/hibernate/search/store/FSDirectoryProvider.java =================================================================== --- search/branches/jboss_cache_integration/src/java/org/hibernate/search/store/FSDirectoryProvider.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/java/org/hibernate/search/store/FSDirectoryProvider.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -6,7 +6,6 @@ import java.util.Properties; import org.apache.lucene.store.FSDirectory; -import org.hibernate.search.Environment; import org.hibernate.search.SearchException; import org.hibernate.search.engine.SearchFactoryImplementor; import org.slf4j.Logger; @@ -51,7 +50,7 @@ directory.close(); } catch (Exception e) { - log.error( "Unable to property close Lucene directory {}" + directory.getFile(), e ); + log.error( "Unable to properly close Lucene directory {}" + directory.getFile(), e ); } } Modified: search/branches/jboss_cache_integration/src/java/org/hibernate/search/store/FSMasterDirectoryProvider.java =================================================================== --- search/branches/jboss_cache_integration/src/java/org/hibernate/search/store/FSMasterDirectoryProvider.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/java/org/hibernate/search/store/FSMasterDirectoryProvider.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -4,7 +4,7 @@ import java.util.Timer; import java.util.Properties; import java.util.TimerTask; -import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executor; import java.util.concurrent.Executors; import java.util.concurrent.locks.Lock; import java.io.File; @@ -38,6 +38,7 @@ private String indexName; private Timer timer; private SearchFactoryImplementor searchFactory; + private long copyChunkSize; //variables needed between initialize and start private File sourceDir; @@ -60,6 +61,7 @@ catch (IOException e) { throw new SearchException( "Unable to initialize index: " + directoryProviderName, e ); } + copyChunkSize = DirectoryProviderHelper.getCopyBufferSize( directoryProviderName, properties ); this.searchFactory = searchFactoryImplementor; } @@ -79,7 +81,7 @@ } String currentString = Integer.valueOf( current ).toString(); File subDir = new File( sourceDir, currentString ); - FileHelper.synchronize( indexDir, subDir, true ); + FileHelper.synchronize( indexDir, subDir, true, copyChunkSize ); new File( sourceDir, "current1 ").delete(); new File( sourceDir, "current2" ).delete(); //TODO small hole, no file can be found here @@ -125,13 +127,13 @@ directory.close(); } catch (Exception e) { - log.error( "Unable to property close Lucene directory {}" + directory.getFile(), e ); + log.error( "Unable to properly close Lucene directory {}" + directory.getFile(), e ); } } class TriggerTask extends TimerTask { - private final ExecutorService executor; + private final Executor executor; private final FSMasterDirectoryProvider.CopyDirectory copyTask; public TriggerTask(File source, File destination, DirectoryProvider directoryProvider) { @@ -140,7 +142,7 @@ } public void run() { - if (!copyTask.inProgress) { + if ( ! copyTask.inProgress ) { executor.execute( copyTask ); } else { @@ -167,7 +169,7 @@ long start = System.currentTimeMillis(); inProgress = true; if ( directoryProviderLock == null ) { - directoryProviderLock = searchFactory.getLockableDirectoryProviders().get( directoryProvider ); + directoryProviderLock = searchFactory.getDirectoryProviderLock( directoryProvider ); directoryProvider = null; searchFactory = null; //get rid of any useless link (help hot redeployment?) } @@ -176,11 +178,10 @@ int oldIndex = current; int index = current == 1 ? 2 : 1; - File destinationFile = new File(destination, Integer.valueOf(index).toString() ); - //TODO make smart a parameter + File destinationFile = new File( destination, Integer.valueOf(index).toString() ); try { log.trace( "Copying {} into {}", source, destinationFile ); - FileHelper.synchronize( source, destinationFile, true ); + FileHelper.synchronize( source, destinationFile, true, copyChunkSize ); current = index; } catch (IOException e) { Modified: search/branches/jboss_cache_integration/src/java/org/hibernate/search/store/FSSlaveDirectoryProvider.java =================================================================== --- search/branches/jboss_cache_integration/src/java/org/hibernate/search/store/FSSlaveDirectoryProvider.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/java/org/hibernate/search/store/FSSlaveDirectoryProvider.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -4,7 +4,7 @@ import java.util.Properties; import java.util.Timer; import java.util.TimerTask; -import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executor; import java.util.concurrent.Executors; import java.io.File; import java.io.IOException; @@ -19,7 +19,7 @@ /** * File based directory provider that takes care of getting a version of the index - * from a given source + * from a given source. * The base directory is represented by hibernate.search.<index>.indexBase * The index is created in <base directory>/<index name> * The source (aka copy) directory is built from <sourceBase>/<index name> @@ -38,7 +38,8 @@ private int current; private String indexName; private Timer timer; - + private long copyChunkSize; + //variables needed between initialize and start private File sourceIndexDir; private File indexDir; @@ -62,6 +63,7 @@ catch (IOException e) { throw new SearchException( "Unable to initialize index: " + directoryProviderName, e ); } + copyChunkSize = DirectoryProviderHelper.getCopyBufferSize( directoryProviderName, properties ); } public void start() { @@ -96,12 +98,13 @@ throw new AssertionFailure( "No current file marker found in source directory: " + sourceIndexDir.getPath() ); } try { - FileHelper.synchronize( new File( sourceIndexDir, String.valueOf(sourceCurrent) ), destinationFile, true); + FileHelper.synchronize( new File( sourceIndexDir, String.valueOf(sourceCurrent) ), + destinationFile, true, copyChunkSize ); } catch (IOException e) { throw new SearchException( "Unable to synchronize directory: " + indexName, e ); } - if (! currentMarker.createNewFile() ) { + if ( ! currentMarker.createNewFile() ) { throw new SearchException( "Unable to create the directory marker file: " + indexName ); } } @@ -149,7 +152,7 @@ class TriggerTask extends TimerTask { - private final ExecutorService executor; + private final Executor executor; private final CopyDirectory copyTask; public TriggerTask(File sourceIndexDir, File destination) { @@ -158,7 +161,7 @@ } public void run() { - if (!copyTask.inProgress) { + if ( ! copyTask.inProgress ) { executor.execute( copyTask ); } else { @@ -197,10 +200,9 @@ } File destinationFile = new File( destination, Integer.valueOf( index ).toString() ); - //TODO make smart a parameter try { log.trace( "Copying {} into {}", sourceFile, destinationFile ); - FileHelper.synchronize( sourceFile, destinationFile, true ); + FileHelper.synchronize( sourceFile, destinationFile, true, copyChunkSize ); current = index; } catch (IOException e) { @@ -232,13 +234,13 @@ directory1.close(); } catch (Exception e) { - log.error( "Unable to property close Lucene directory {}" + directory1.getFile(), e ); + log.error( "Unable to properly close Lucene directory {}" + directory1.getFile(), e ); } try { directory2.close(); } catch (Exception e) { - log.error( "Unable to property close Lucene directory {}" + directory2.getFile(), e ); + log.error( "Unable to properly close Lucene directory {}" + directory2.getFile(), e ); } } } Modified: search/branches/jboss_cache_integration/src/java/org/hibernate/search/store/optimization/IncrementalOptimizerStrategy.java =================================================================== --- search/branches/jboss_cache_integration/src/java/org/hibernate/search/store/optimization/IncrementalOptimizerStrategy.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/java/org/hibernate/search/store/optimization/IncrementalOptimizerStrategy.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -6,9 +6,9 @@ import org.hibernate.search.engine.SearchFactoryImplementor; import org.hibernate.search.backend.Workspace; +import org.hibernate.search.backend.configuration.ConfigurationParseHelper; import org.hibernate.search.store.DirectoryProvider; import org.hibernate.search.SearchException; -import org.hibernate.annotations.common.util.StringHelper; import org.apache.lucene.index.IndexWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,15 +28,8 @@ public void initialize(DirectoryProvider directoryProvider, Properties indexProperties, SearchFactoryImplementor searchFactoryImplementor) { this.directoryProvider = directoryProvider; - String maxString = indexProperties.getProperty( "optimizer.operation_limit.max" ); - if ( StringHelper.isNotEmpty( maxString ) ) { - operationMax = Integer.parseInt( maxString ); - } - - maxString = indexProperties.getProperty( "optimizer.transaction_limit.max" ); - if ( StringHelper.isNotEmpty( maxString ) ) { - transactionMax = Integer.parseInt( maxString ); - } + operationMax = ConfigurationParseHelper.getIntValue( indexProperties, "optimizer.operation_limit.max", -1 ); + transactionMax = ConfigurationParseHelper.getIntValue( indexProperties, "optimizer.transaction_limit.max", -1 ); } public void optimizationForced() { Modified: search/branches/jboss_cache_integration/src/java/org/hibernate/search/util/FileHelper.java =================================================================== --- search/branches/jboss_cache_integration/src/java/org/hibernate/search/util/FileHelper.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/java/org/hibernate/search/util/FileHelper.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -10,16 +10,33 @@ import java.util.Arrays; import java.nio.channels.FileChannel; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * @author Emmanuel Bernard + * @author Sanne Grinovero */ public abstract class FileHelper { + private static final int FAT_PRECISION = 2000; + public static final long DEFAULT_COPY_BUFFER_SIZE = 16 * 1024 * 1024; // 16 MB + private static final Logger log = LoggerFactory.getLogger( FileHelper.class ); + + public static void synchronize(File source, File destination, boolean smart) throws IOException { + synchronize( source, destination, smart, DEFAULT_COPY_BUFFER_SIZE ); + } - public static void synchronize(File source, File destination, boolean smart) throws IOException { + public static void synchronize(File source, File destination, boolean smart, long chunkSize) throws IOException { + if ( chunkSize <= 0 ) { + log.warn( "Chunk size must be positive: using default value." ); + chunkSize = DEFAULT_COPY_BUFFER_SIZE; + } if ( source.isDirectory() ) { if ( ! destination.exists() ) { - destination.mkdirs(); + if ( ! destination.mkdirs() ){ + throw new IOException("Could not create path " + destination); + } } else if ( ! destination.isDirectory() ) { throw new IOException("Source and Destination not of the same type:" @@ -39,7 +56,7 @@ for (String fileName : sources) { File srcFile = new File(source, fileName); File destFile = new File(destination, fileName); - synchronize( srcFile, destFile, smart ); + synchronize( srcFile, destFile, smart, chunkSize ); } } else { @@ -51,16 +68,16 @@ long dts = destination.lastModified() / FAT_PRECISION; //do not copy if smart and same timestamp and same length if ( !smart || sts == 0 || sts != dts || source.length() != destination.length() ) { - copyFile(source, destination); + copyFile(source, destination, chunkSize); } } else { - copyFile(source, destination); + copyFile(source, destination, chunkSize); } } } - private static void copyFile(File srcFile, File destFile) throws IOException { + private static void copyFile(File srcFile, File destFile, long chunkSize) throws IOException { FileInputStream is = null; FileOutputStream os = null; try { @@ -68,19 +85,41 @@ FileChannel iChannel = is.getChannel(); os = new FileOutputStream( destFile, false ); FileChannel oChannel = os.getChannel(); - oChannel.transferFrom( iChannel, 0, srcFile.length() ); + long doneBytes = 0L; + long todoBytes = srcFile.length(); + while ( todoBytes != 0L ) { + long iterationBytes = Math.min( todoBytes, chunkSize ); + long transferredLength = oChannel.transferFrom( iChannel, doneBytes, iterationBytes ); + if ( iterationBytes != transferredLength ) { + throw new IOException( "Error during file transfer: expected " + + iterationBytes + " bytes, only "+ transferredLength + " bytes copied." ); + } + doneBytes += transferredLength; + todoBytes -= transferredLength; + } } finally { if (is != null) is.close(); if (os != null) os.close(); } - destFile.setLastModified( srcFile.lastModified() ); + boolean successTimestampOp = destFile.setLastModified( srcFile.lastModified() ); + if ( ! successTimestampOp ) { + log.warn( "Could not change timestamp for " + destFile + + ". Index synchronization may be slow." ); + } } public static void delete(File file) { if ( file.isDirectory() ) { - for ( File subFile : file.listFiles() ) delete( subFile ); + for ( File subFile : file.listFiles() ) { + delete( subFile ); + } } - if ( file.exists() ) file.delete(); + if ( file.exists() ) { + if ( ! file.delete() ) { + log.error( "Could not delete " + file ); + } + } } + } Modified: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/FSDirectoryTest.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/FSDirectoryTest.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/FSDirectoryTest.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -3,7 +3,6 @@ import java.io.File; import java.util.List; -import java.util.Properties; import org.apache.lucene.analysis.StopAnalyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer; @@ -16,12 +15,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.hibernate.Session; -import org.hibernate.event.PostDeleteEventListener; -import org.hibernate.event.PostInsertEventListener; -import org.hibernate.event.PostUpdateEventListener; import org.hibernate.search.Environment; -import org.hibernate.search.event.FullTextIndexEventListener; -import org.hibernate.search.store.DirectoryProviderHelper; import org.hibernate.search.store.FSDirectoryProvider; import org.hibernate.search.util.FileHelper; @@ -54,36 +48,6 @@ FileHelper.delete( sub ); } - public void testDirectoryProviderHelperMkdirsGetSource() throws Exception { - String root = "./testDir"; - String relative = "dir1/dir2/dir3"; - - Properties properties = new Properties(); - properties.put( "sourceBase", root ); - properties.put( "source", relative ); - - File rel = DirectoryProviderHelper.getSourceDirectory( "name", properties, true ); - - assertTrue( rel.exists() ); - - FileHelper.delete( new File( root ) ); - } - - public void testDirectoryProviderHelperMkdirsDetermineIndex() throws Exception { - String root = "./testDir/dir1/dir2"; - String relative = "dir3"; - - Properties properties = new Properties(); - properties.put( "indexBase", root ); - properties.put( "indexName", relative ); - - File f = DirectoryProviderHelper.getVerifiedIndexDir( "name", properties, true ); - - assertTrue( new File( root ).exists() ); - - FileHelper.delete( new File( "./testDir" ) ); - } - public void testEventIntegration() throws Exception { Session s = getSessions().openSession(); @@ -111,7 +75,7 @@ s = getSessions().openSession(); s.getTransaction().begin(); - Document entity = (Document) s.get( Document.class, new Long( 1 ) ); + Document entity = (Document) s.get( Document.class, Long.valueOf( 1 ) ); entity.setSummary( "Object/relational mapping with EJB3" ); s.persist( new Document( "Seam in Action", "", "blah blah blah blah" ) ); s.getTransaction().commit(); @@ -223,10 +187,6 @@ cfg.setProperty( "hibernate.search.default.indexBase", sub.getAbsolutePath() ); cfg.setProperty( "hibernate.search.default.directory_provider", FSDirectoryProvider.class.getName() ); cfg.setProperty( Environment.ANALYZER_CLASS, StopAnalyzer.class.getName() ); - FullTextIndexEventListener del = new FullTextIndexEventListener(); - cfg.getEventListeners().setPostDeleteEventListeners( new PostDeleteEventListener[] { del } ); - cfg.getEventListeners().setPostUpdateEventListeners( new PostUpdateEventListener[] { del } ); - cfg.getEventListeners().setPostInsertEventListeners( new PostInsertEventListener[] { del } ); } } Added: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/SerializationTestHelper.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/SerializationTestHelper.java (rev 0) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/SerializationTestHelper.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -0,0 +1,107 @@ +package org.hibernate.search.test; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +import org.hibernate.search.test.SerializationTestHelper.Foo.TestInnerClass; + +import junit.framework.TestCase; + +/** + * @author Sanne Grinovero + */ +public class SerializationTestHelper extends TestCase { + + /** + * Duplicates an object using Serialization, it moves + * state to and from a buffer. Should be used to test + * correct serializability. + * @param o The object to "clone" + * @return the clone. + * @throws IOException + * @throws ClassNotFoundException + */ + public static Object duplicateBySerialization(Object o) throws IOException, ClassNotFoundException { + //Serialize to buffer: + java.io.ByteArrayOutputStream outStream = new java.io.ByteArrayOutputStream(); + ObjectOutputStream objectOutStream = new ObjectOutputStream( outStream ); + objectOutStream.writeObject( o ); + objectOutStream.flush(); + objectOutStream.close(); + //buffer version of Object: + byte[] objectBuffer = outStream.toByteArray(); + //deserialize to new instance: + java.io.ByteArrayInputStream inStream = new ByteArrayInputStream( objectBuffer ); + ObjectInputStream objectInStream = new ObjectInputStream( inStream ); + Object copy = objectInStream.readObject(); + return copy; + } + + public void testSelf() throws IOException, ClassNotFoundException { + Foo a = new Foo(); + a.list.add( new TestInnerClass(30) ); + Foo b = (Foo) duplicateBySerialization( a ); + assertEquals( Integer.valueOf(6), a.integer); + assertEquals( Integer.valueOf(7), b.integer); + assertEquals( a.list, b.list ); + } + + static class Foo implements Serializable { + + List<TestInnerClass> list = new ArrayList<TestInnerClass>(); + transient Integer integer = Integer.valueOf( 6 ); + + static class TestInnerClass implements Serializable { + private final int v; + + public TestInnerClass(int i) { + v = i; + } + + public void print() { + System.out.println(v); + } + + public String toString(){ + return ""+v; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + v; + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + final TestInnerClass other = (TestInnerClass) obj; + if (v != other.v) + return false; + return true; + } + } + + private void readObject(ObjectInputStream aInputStream) throws ClassNotFoundException, IOException { + aInputStream.defaultReadObject(); + integer = Integer.valueOf( 7 ); + } + + private void writeObject(ObjectOutputStream aOutputStream) throws IOException { + aOutputStream.defaultWriteObject(); + } + } + +} Modified: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/analyzer/AnalyzerTest.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/analyzer/AnalyzerTest.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/analyzer/AnalyzerTest.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -5,9 +5,7 @@ import org.hibernate.search.FullTextSession; import org.hibernate.search.Search; import org.hibernate.search.FullTextQuery; -import org.hibernate.Session; import org.hibernate.Transaction; -import org.hibernate.Query; import org.apache.lucene.queryParser.QueryParser; import org.apache.lucene.analysis.standard.StandardAnalyzer; Added: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/configuration/ConfigurationParseHelperTest.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/configuration/ConfigurationParseHelperTest.java (rev 0) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/configuration/ConfigurationParseHelperTest.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -0,0 +1,34 @@ +package org.hibernate.search.test.configuration; + +import java.util.Properties; + +import junit.framework.TestCase; + +import org.hibernate.search.SearchException; +import org.hibernate.search.backend.configuration.ConfigurationParseHelper; + +/** + * @author Sanne Grinovero + */ +public class ConfigurationParseHelperTest extends TestCase { + + public void testIntegerParsers(){ + assertEquals( 0, ConfigurationParseHelper.parseInt( " 0 ", "not important") ); + assertEquals( 8, ConfigurationParseHelper.parseInt( null, 8, null ) ); + assertEquals( 56, ConfigurationParseHelper.parseInt( "56", 8, null ) ); + Properties props = new Properties(); + props.setProperty( "value1", "58" ); + assertEquals( 58, ConfigurationParseHelper.getIntValue( props, "value1", 8 ) ); + assertEquals( 8, ConfigurationParseHelper.getIntValue( props, "value2", 8 ) ); + props.setProperty( "value2", "nand" ); + boolean exceptionLaunched; + try { + ConfigurationParseHelper.getIntValue( props, "value2", 8 ); + exceptionLaunched = false; + } catch (SearchException e) { + exceptionLaunched = true; + } + assertTrue( exceptionLaunched ); + } + +} Modified: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/configuration/LuceneIndexingParametersTest.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/configuration/LuceneIndexingParametersTest.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/configuration/LuceneIndexingParametersTest.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -1,6 +1,11 @@ package org.hibernate.search.test.configuration; +import java.io.IOException; +import java.util.Properties; + +import org.hibernate.search.backend.LuceneIndexingParameters; import org.hibernate.search.test.Document; +import org.hibernate.search.test.SerializationTestHelper; import org.hibernate.search.test.query.Author; import org.hibernate.search.test.query.Book; import static org.hibernate.search.backend.configuration.IndexWriterSetting.MAX_BUFFERED_DOCS; @@ -85,6 +90,14 @@ assertValueIsDefault( Document.class, TRANSACTION, RAM_BUFFER_SIZE ); } + public void testSerializability() throws IOException, ClassNotFoundException { + LuceneIndexingParameters param = new LuceneIndexingParameters( new Properties() ); + LuceneIndexingParameters paramCopy = (LuceneIndexingParameters) + SerializationTestHelper.duplicateBySerialization( param ); + assertEquals(param.getBatchIndexParameters(), paramCopy.getBatchIndexParameters()); + assertEquals(param.getTransactionIndexParameters(), paramCopy.getTransactionIndexParameters()); + } + protected Class[] getMappings() { return new Class[] { Book.class, Modified: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/configuration/MaskedPropertiesTest.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/configuration/MaskedPropertiesTest.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/configuration/MaskedPropertiesTest.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -1,8 +1,10 @@ package org.hibernate.search.test.configuration; +import java.io.IOException; import java.util.Properties; import org.hibernate.search.backend.configuration.MaskedProperty; +import org.hibernate.search.test.SerializationTestHelper; /** * @author Sanne Grinovero @@ -37,4 +39,14 @@ assertEquals( "5" , transaction.getProperty( "max_merge_docs" ) ); } + public void testSerializability() throws IOException, ClassNotFoundException { + Properties cfg = new Properties(); + cfg.setProperty( "base.key", "value" ); + MaskedProperty originalProps = new MaskedProperty( cfg, "base" ); + MaskedProperty theCopy = (MaskedProperty) + SerializationTestHelper.duplicateBySerialization( originalProps ); + //this is also testing the logger (transient) has been restored: + assertEquals( "value", theCopy.getProperty( "key" ) ); + } + } Added: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/directoryProvider/DirectoryProviderHelperTest.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/directoryProvider/DirectoryProviderHelperTest.java (rev 0) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/directoryProvider/DirectoryProviderHelperTest.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -0,0 +1,78 @@ +package org.hibernate.search.test.directoryProvider; + +import java.io.File; +import java.util.Properties; +import junit.framework.TestCase; +import org.hibernate.search.SearchException; +import org.hibernate.search.store.DirectoryProviderHelper; +import org.hibernate.search.util.FileHelper; + +/** + * @author Gavin King + * @author Sanne Grinovero + */ +public class DirectoryProviderHelperTest extends TestCase { + + public void testMkdirsDetermineIndex() throws Exception { + String root = "./testDir/dir1/dir2"; + String relative = "dir3"; + + Properties properties = new Properties(); + properties.put( "indexBase", root ); + properties.put( "indexName", relative ); + + File f = DirectoryProviderHelper.getVerifiedIndexDir( "name", properties, true ); + + assertTrue( new File( root ).exists() ); + + FileHelper.delete( new File( "./testDir" ) ); + } + + public void testMkdirsGetSource() throws Exception { + String root = "./testDir"; + String relative = "dir1/dir2/dir3"; + + Properties properties = new Properties(); + properties.put( "sourceBase", root ); + properties.put( "source", relative ); + + File rel = DirectoryProviderHelper.getSourceDirectory( "name", properties, true ); + + assertTrue( rel.exists() ); + + FileHelper.delete( new File( root ) ); + } + + public void testConfiguringCopyBufferSize() { + Properties prop = new Properties(); + long mB = 1024 * 1024; + + //default to FileHelper default: + assertEquals( FileHelper.DEFAULT_COPY_BUFFER_SIZE, DirectoryProviderHelper.getCopyBufferSize( "testIdx", prop ) ); + + //any value from MegaBytes: + prop.setProperty( DirectoryProviderHelper.COPYBUFFERSIZE_PROP_NAME, "4" ); + assertEquals( 4*mB, DirectoryProviderHelper.getCopyBufferSize( "testIdx", prop ) ); + prop.setProperty( DirectoryProviderHelper.COPYBUFFERSIZE_PROP_NAME, "1000" ); + assertEquals( 1000*mB, DirectoryProviderHelper.getCopyBufferSize( "testIdx", prop ) ); + + //invalid values + prop.setProperty( DirectoryProviderHelper.COPYBUFFERSIZE_PROP_NAME, "0" ); + boolean testOk = false; + try { + DirectoryProviderHelper.getCopyBufferSize( "testIdx", prop ); + } catch (SearchException e){ + testOk = true; + } + assertTrue( testOk ); + prop.setProperty( DirectoryProviderHelper.COPYBUFFERSIZE_PROP_NAME, "-100" ); + testOk = false; + try { + DirectoryProviderHelper.getCopyBufferSize( "testIdx", prop ); + } catch (SearchException e){ + testOk = true; + } + assertTrue( testOk ); + } + +} Modified: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/directoryProvider/FSSlaveAndMasterDPTest.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/directoryProvider/FSSlaveAndMasterDPTest.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/directoryProvider/FSSlaveAndMasterDPTest.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -10,12 +10,8 @@ import org.hibernate.Session; import org.hibernate.SessionFactory; import org.hibernate.cfg.Configuration; -import org.hibernate.event.PostDeleteEventListener; -import org.hibernate.event.PostInsertEventListener; -import org.hibernate.event.PostUpdateEventListener; import org.hibernate.search.FullTextSession; import org.hibernate.search.Search; -import org.hibernate.search.event.FullTextIndexEventListener; import org.hibernate.search.util.FileHelper; /** @@ -94,7 +90,6 @@ } } - protected void setUp() throws Exception { File base = new File("."); File root = new File(base, "lucenedirs"); Modified: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/embedded/Author.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/embedded/Author.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/embedded/Author.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -5,7 +5,6 @@ import javax.persistence.Id; import javax.persistence.GeneratedValue; -import org.hibernate.search.annotations.Indexed; import org.hibernate.search.annotations.Field; import org.hibernate.search.annotations.Index; import org.hibernate.search.annotations.DocumentId; Modified: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/embedded/Country.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/embedded/Country.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/embedded/Country.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -1,13 +1,21 @@ -//$ +// $Id:$ package org.hibernate.search.test.embedded; +import java.util.ArrayList; +import java.util.List; + import javax.persistence.Entity; +import javax.persistence.FetchType; import javax.persistence.GeneratedValue; import javax.persistence.Id; +import javax.persistence.OneToMany; +import org.hibernate.annotations.Cascade; +import org.hibernate.annotations.IndexColumn; import org.hibernate.search.annotations.DocumentId; import org.hibernate.search.annotations.Field; import org.hibernate.search.annotations.Indexed; +import org.hibernate.search.annotations.IndexedEmbedded; /** * @author Emmanuel Bernard @@ -19,9 +27,16 @@ @GeneratedValue @DocumentId private Integer id; + @Field private String name; + @OneToMany(fetch = FetchType.LAZY) + @IndexColumn(name = "list_position") + @Cascade(org.hibernate.annotations.CascadeType.ALL) + @IndexedEmbedded + private List<State> states = new ArrayList<State>(); + public Integer getId() { return id; } @@ -37,4 +52,12 @@ public void setName(String name) { this.name = name; } + + public List<State> getStates() { + return states; + } + + public void setStates(List<State> states) { + this.states = states; + } } Modified: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/embedded/Person.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/embedded/Person.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/embedded/Person.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -1,4 +1,4 @@ -//$ +// $Id:$ package org.hibernate.search.test.embedded; /** Added: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/embedded/State.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/embedded/State.java (rev 0) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/embedded/State.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -0,0 +1,37 @@ +// $Id:$ +package org.hibernate.search.test.embedded; + +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.Id; + +import org.hibernate.search.annotations.Field; + +/** + * @author Hardy Ferentschik + */ +@Entity +public class State { + @Id + @GeneratedValue + private Integer id; + + @Field + private String name; + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } +} Modified: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/embedded/Tower.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/embedded/Tower.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/embedded/Tower.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -5,7 +5,6 @@ import javax.persistence.Entity; import javax.persistence.GeneratedValue; import javax.persistence.Id; -import javax.persistence.OneToOne; import javax.persistence.ManyToOne; import org.hibernate.search.annotations.DocumentId; Modified: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/embedded/doubleinsert/DoubleInsertEmbeddedTest.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/embedded/doubleinsert/DoubleInsertEmbeddedTest.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/embedded/doubleinsert/DoubleInsertEmbeddedTest.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -2,20 +2,12 @@ package org.hibernate.search.test.embedded.doubleinsert; import java.util.Date; -import java.io.File; import org.apache.lucene.index.Term; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.analysis.StopAnalyzer; import org.hibernate.Query; -import org.hibernate.event.PostDeleteEventListener; -import org.hibernate.event.PostUpdateEventListener; -import org.hibernate.event.PostInsertEventListener; import org.hibernate.search.FullTextSession; import org.hibernate.search.Search; -import org.hibernate.search.Environment; -import org.hibernate.search.event.FullTextIndexEventListener; -import org.hibernate.search.store.FSDirectoryProvider; import org.hibernate.search.test.SearchTestCase; /** Modified: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/jms/master/JMSMasterTest.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/jms/master/JMSMasterTest.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/jms/master/JMSMasterTest.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -2,6 +2,7 @@ package org.hibernate.search.test.jms.master; import java.io.Serializable; +import java.sql.Statement; import java.util.ArrayList; import java.util.List; import javax.jms.ObjectMessage; @@ -43,9 +44,11 @@ //create an object wo trigggering indexing Session s = openSession( ); s.getTransaction().begin(); - s.connection().createStatement().executeUpdate( + Statement statement = s.connection().createStatement(); + statement.executeUpdate( "insert into TShirt_Master(id, logo, size) values( '1', 'JBoss balls', 'large')" ); + statement.close(); TShirt ts = (TShirt) s.get(TShirt.class, 1); s.getTransaction().commit(); s.close(); Modified: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/optimizer/OptimizerTestCase.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/optimizer/OptimizerTestCase.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/optimizer/OptimizerTestCase.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -4,7 +4,6 @@ import java.io.File; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.Date; import org.apache.lucene.analysis.StopAnalyzer; import org.apache.lucene.queryParser.ParseException; @@ -18,6 +17,7 @@ import org.hibernate.search.impl.FullTextSessionImpl; import org.hibernate.search.store.FSDirectoryProvider; import org.hibernate.search.test.SearchTestCase; +import org.hibernate.search.util.FileHelper; /** * @author Emmanuel Bernard @@ -25,12 +25,12 @@ public class OptimizerTestCase extends SearchTestCase { protected void setUp() throws Exception { File sub = getBaseIndexDir(); - delete( sub ); + FileHelper.delete( sub ); sub.mkdir(); File[] files = sub.listFiles(); for (File file : files) { if ( file.isDirectory() ) { - delete( file ); + FileHelper.delete( file ); } } //super.setUp(); //we need a fresh session factory each time for index set up @@ -47,21 +47,9 @@ protected void tearDown() throws Exception { super.tearDown(); File sub = getBaseIndexDir(); - delete( sub ); + FileHelper.delete( sub ); } - private void delete(File sub) { - if ( sub.isDirectory() ) { - for (File file : sub.listFiles()) { - delete( file ); - } - sub.delete(); - } - else { - sub.delete(); - } - } - public void testConcurrency() throws Exception { int nThreads = 15; ExecutorService es = Executors.newFixedThreadPool( nThreads ); @@ -80,8 +68,8 @@ .currentTimeMillis() - start ) ); } - protected class Work implements Runnable { - private SessionFactory sf; + protected static class Work implements Runnable { + private final SessionFactory sf; public volatile int count = 0; public Work(SessionFactory sf) { @@ -147,7 +135,7 @@ } } - protected class ReverseWork implements Runnable { + protected static class ReverseWork implements Runnable { private SessionFactory sf; public ReverseWork(SessionFactory sf) { Modified: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/perf/IndexTestDontRun.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/perf/IndexTestDontRun.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/perf/IndexTestDontRun.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -3,6 +3,10 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import junit.textui.TestRunner; import org.apache.lucene.search.IndexSearcher; @@ -18,15 +22,16 @@ * @author Emmanuel Bernard */ public class IndexTestDontRun extends SearchTestCase { - private static boolean isLucene; + + private static final int TOTAL_SEARCHES = 800; + private static final int SEARCH_THREADS = 100; public static void main(String[] args) { //isLucene = Boolean.parseBoolean( args[0] ); TestRunner.run( IndexTestDontRun.class ); - } - public void NonestInit() throws Exception { + public void notestInit() throws Exception { long time = System.currentTimeMillis(); Session s = openSession(); Transaction tx = s.beginTransaction(); @@ -37,26 +42,39 @@ s.close(); System.out.println( " init time = " + ( System.currentTimeMillis() - time ) ); } + + public void testPerformance() throws Exception { + measure(true);//JVM warmup + measure(false);//JVM warmup + long measureLucene = measure( true ); + long measureSearch = measure( false ); + System.out.println( "Totaltime Lucene = " + measureLucene ); + System.out.println( "Totaltime Search = " + measureSearch ); + } - public void testPerf() throws Exception { - boolean useLucene = false; - - List<SearcherThread> threads = new ArrayList<SearcherThread>( 100 ); - IndexSearcher indexsearcher = getNewSearcher(); - SearcherThread searcherThrea = new SearcherThread( 0, "name:maria OR description:long" + 0, getSessions(), indexsearcher, useLucene ); - searcherThrea.run(); - for (int i = 1; i <= 100; i++) { + public long measure(boolean plainLucene) throws Exception { + ThreadPoolExecutor threadPool = (ThreadPoolExecutor) Executors.newFixedThreadPool( SEARCH_THREADS ); + threadPool.prestartAllCoreThreads(); + CountDownLatch startSignal = new CountDownLatch(1); + List<SearcherThread> threadsList = new ArrayList<SearcherThread>( TOTAL_SEARCHES ); + IndexSearcher indexSearcher = getNewSearcher(); + for (int i = 0; i < TOTAL_SEARCHES; i++) { // Create a thread and invoke it - //if ( i % 100 == 0) indexsearcher = getNewSearcher(); - SearcherThread searcherThread = new SearcherThread( i, "name:maria OR description:long" + i, getSessions(), indexsearcher, useLucene ); - searcherThread.setDaemon( false ); - threads.add( searcherThread ); - searcherThread.start(); + //if ( i % 100 == 0) indexSearcher = getNewSearcher(); + SearcherThread searcherThread = new SearcherThread( i, "name:maria OR description:long" + i, getSessions(), indexSearcher, plainLucene, startSignal ); + threadsList.add( searcherThread ); + threadPool.execute( searcherThread ); } - Thread.sleep( 5000 ); + threadPool.shutdown();//required to enable awaitTermination functionality + startSignal.countDown();//start all created threads + boolean terminationOk = threadPool.awaitTermination( 60, TimeUnit.SECONDS ); + if ( terminationOk==false ) { + System.out.println( "No enough time to complete the tests!" ); + return 0; + } long totalTime = 0; - for (SearcherThread t : threads) totalTime += t.time; - System.out.println( "Totaltime=" + totalTime ); + for (SearcherThread t : threadsList) totalTime += t.getTime(); + return totalTime; } private IndexSearcher getNewSearcher() throws IOException { @@ -75,7 +93,6 @@ protected void configure(Configuration cfg) { super.configure( cfg ); cfg.setProperty( "hibernate.search.default.directory_provider", FSDirectoryProvider.class.getName() ); - //cfg.setProperty( "hibernate.search.reader.strategy", DumbSharedReaderProvider.class.getName() ); - +// cfg.setProperty( "hibernate.search.reader.strategy", DumbSharedReaderProvider.class.getName() ); } } Modified: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/perf/SearcherThread.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/perf/SearcherThread.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/perf/SearcherThread.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -3,6 +3,7 @@ import java.io.IOException; import java.util.List; import java.util.ArrayList; +import java.util.concurrent.CountDownLatch; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.queryParser.ParseException; @@ -11,9 +12,7 @@ import org.apache.lucene.search.Hits; import org.apache.lucene.search.Query; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.store.Directory; import org.hibernate.SessionFactory; -import org.hibernate.classic.Session; import org.hibernate.search.Search; import org.hibernate.search.FullTextSession; import org.hibernate.search.FullTextQuery; @@ -23,27 +22,39 @@ /** * @author Emmanuel Bernard */ -public class SearcherThread extends Thread { - private static Logger log = LoggerFactory.getLogger( SearcherThread.class ); - private int threadId; - private String queryString; - private SessionFactory sf; - private IndexSearcher indexsearcher; - private boolean isLucene; - public long time; +public class SearcherThread implements Runnable { + private static final Logger log = LoggerFactory.getLogger( SearcherThread.class ); + private final int threadId; + private final String queryString; + private final SessionFactory sf; + private final IndexSearcher indexsearcher; + private final boolean isLucene; + private final CountDownLatch startSignal; + private long time; /** - * Initialize with thread-id, querystring, indexsearcher + * Initialize with thread-id, queryString, indexSearcher + * @param startSignal */ - public SearcherThread(int threadId, String queryString, SessionFactory sf, IndexSearcher indexSearcher, boolean isLucene) { + public SearcherThread(int threadId, String queryString, SessionFactory sf, IndexSearcher indexSearcher, boolean isLucene, CountDownLatch startSignal) { this.isLucene = isLucene; this.threadId = threadId; this.queryString = queryString; this.sf = sf; this.indexsearcher = indexSearcher; + this.startSignal = startSignal; } + /** + * @see java.lang.Runnable#run() + */ public void run() { + try { + startSignal.await(); + } catch (InterruptedException e) { + log.error( "tests canceled", e ); + return; + } if ( isLucene ) { runLucene(); } @@ -52,34 +63,22 @@ } } - /** - * @see java.lang.Runnable#run() - */ public void runLucene() { - try { - QueryParser qp = new QueryParser( "t", - new StandardAnalyzer() ); - qp.setLowercaseExpandedTerms( true ); - // Parse the query - Query q = qp.parse( queryString ); - if ( q instanceof BooleanQuery ) { - BooleanQuery - .setMaxClauseCount( Integer.MAX_VALUE ); - } + Query q = getQuery(); long start = System.currentTimeMillis(); // Search Hits hits = indexsearcher.search( q ); List<String> names = new ArrayList<String>(100); - for (int i = 1 ; i <= 100 ; i++) { + for (int i = 0 ; i < 100 ; i++) { names.add( hits.doc( i ).get( "name" ) ); } + int resultSize = hits.length(); long totalTime = System.currentTimeMillis() - start; - log.error( "Lucene [ Thread-id : " + threadId + " ] Total time taken for search is : " + totalTime + "ms with total no. of matching records : " + hits.length() ); - time = totalTime; +// log.error( "Lucene [ Thread-id : " + threadId + " ] Total time taken for search is : " + totalTime + "ms with total no. of matching records : " + hits.length() ); + setTime( totalTime ); } catch (ParseException e) { - // TODO Auto-generated catch block System.out.println( "[ Thread-id : " + threadId + " ] Parse Exception for queryString : " + queryString ); e.printStackTrace(); } @@ -91,32 +90,33 @@ } } + private Query getQuery() throws ParseException { + QueryParser qp = new QueryParser( "t", new StandardAnalyzer() ); + qp.setLowercaseExpandedTerms( true ); + // Parse the query + Query q = qp.parse( queryString ); + if ( q instanceof BooleanQuery ) { + BooleanQuery.setMaxClauseCount( Integer.MAX_VALUE ); + } + return q; + } + public void runHSearch() { - try { - QueryParser qp = new QueryParser( "t", - new StandardAnalyzer() ); - qp.setLowercaseExpandedTerms( true ); - - // Parse the query - Query q = qp.parse( queryString ); - - + Query q = getQuery(); // Search FullTextSession ftSession = Search.createFullTextSession( sf.openSession( ) ); - final FullTextQuery textQuery = ftSession.createFullTextQuery( q, Boat.class ) .setMaxResults( 100 ).setProjection( "name" ); long start = System.currentTimeMillis(); List results = textQuery.list(); + int resultSize = textQuery.getResultSize(); long totalTime = System.currentTimeMillis() - start; ftSession.close(); - - log.error( "HSearch [ Thread-id : " + threadId + " ] Total time taken for search is : " + totalTime + "ms with total no. of matching records : " + textQuery.getResultSize() ); - time = totalTime; +// log.error( "HSearch [ Thread-id : " + threadId + " ] Total time taken for search is : " + totalTime + "ms with total no. of matching records : " + resultSize ); + setTime( totalTime ); } catch (ParseException e) { - // TODO Auto-generated catch block log.error( "[ Thread-id : " + threadId + " ] Parse Exception for queryString : " + queryString ); e.printStackTrace(); } @@ -125,4 +125,13 @@ e.printStackTrace( ); } } + + public synchronized long getTime() { + return time; + } + + public synchronized void setTime(long time) { + this.time = time; + } + } Modified: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/query/Employee.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/query/Employee.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/query/Employee.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -1,6 +1,7 @@ //$Id$ package org.hibernate.search.test.query; +import java.util.Date; import javax.persistence.Entity; import javax.persistence.Id; @@ -9,6 +10,8 @@ import org.hibernate.search.annotations.Index; import org.hibernate.search.annotations.Indexed; import org.hibernate.search.annotations.Store; +import org.hibernate.search.annotations.DateBridge; +import org.hibernate.search.annotations.Resolution; /** * @author John Grffin @@ -20,6 +23,18 @@ private String lastname; private String dept; + @Field(store=Store.YES, index = Index.UN_TOKENIZED) + @DateBridge(resolution = Resolution.DAY) + public Date getHireDate() { + return hireDate; + } + + public void setHireDate(Date hireDate) { + this.hireDate = hireDate; + } + + private Date hireDate; + public Employee() { } Modified: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/query/LuceneQuerySortTest.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/query/LuceneQuerySortTest.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/query/LuceneQuerySortTest.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -41,7 +41,7 @@ // into the index. int id = 1; for(Book b : result) { - assertEquals("Expected another id", new Integer(id), b.getId()); + assertEquals( "Expected another id", Integer.valueOf( id ), b.getId() ); id++; } @@ -55,7 +55,7 @@ assertEquals( "Wrong number of test results.", 3, result.size() ); id = 3; for(Book b : result) { - assertEquals("Expected another id", new Integer(id), b.getId()); + assertEquals("Expected another id", Integer.valueOf( id ), b.getId()); id--; } Added: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/query/MultiClassesQueryLoaderTest.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/query/MultiClassesQueryLoaderTest.java (rev 0) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/query/MultiClassesQueryLoaderTest.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -0,0 +1,56 @@ +//$ +package org.hibernate.search.test.query; + +import java.sql.Statement; +import java.util.List; + +import org.hibernate.search.test.SearchTestCase; +import org.hibernate.search.FullTextSession; +import org.hibernate.search.Search; +import org.hibernate.search.FullTextQuery; +import org.hibernate.Session; +import org.hibernate.Transaction; +import org.apache.lucene.queryParser.QueryParser; +import org.apache.lucene.analysis.KeywordAnalyzer; +import org.apache.lucene.search.Query; + +/** + * @author Emmanuel Bernard + */ +public class MultiClassesQueryLoaderTest extends SearchTestCase { + + public void testObjectNotFound() throws Exception { + Session sess = openSession(); + Transaction tx = sess.beginTransaction(); + Author author = new Author(); + author.setName( "Moo Cow" ); + sess.persist( author ); + + tx.commit(); + sess.clear(); + Statement statement = sess.connection().createStatement(); + statement.executeUpdate( "DELETE FROM Author" ); + statement.close(); + FullTextSession s = Search.createFullTextSession( sess ); + tx = s.beginTransaction(); + QueryParser parser = new QueryParser( "title", new KeywordAnalyzer() ); + Query query = parser.parse( "name:moo" ); + FullTextQuery hibQuery = s.createFullTextQuery( query, Author.class, Music.class ); + List result = hibQuery.list(); + assertEquals( "Should have returned no author", 0, result.size() ); + + for (Object o : s.createCriteria( Object.class ).list()) { + s.delete( o ); + } + + tx.commit(); + s.close(); + } + + protected Class[] getMappings() { + return new Class[] { + Author.class, + Music.class + }; + } +} Modified: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/query/ProjectionQueryTest.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/query/ProjectionQueryTest.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/query/ProjectionQueryTest.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -5,6 +5,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Date; import org.apache.lucene.analysis.StopAnalyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer; @@ -236,13 +237,13 @@ assertEquals( "SCORE incorrect", 1.0F, projection[4] ); assertEquals( "BOOST incorrect", 1.0F, projection[5] ); assertTrue( "DOCUMENT incorrect", projection[6] instanceof Document ); - assertEquals( "DOCUMENT size incorrect", 4, ( (Document) projection[6] ).getFields().size() ); + assertEquals( "DOCUMENT size incorrect", 5, ( (Document) projection[6] ).getFields().size() ); assertEquals( "ID incorrect", 1001, projection[7] ); assertNotNull( "Lucene internal doc id", projection[8] ); // Change the projection order and null one hibQuery.setProjection( FullTextQuery.DOCUMENT, FullTextQuery.THIS, FullTextQuery.SCORE, null, FullTextQuery.ID, - "id", "lastname", "dept", FullTextQuery.DOCUMENT_ID ); + "id", "lastname", "dept", "hireDate", FullTextQuery.DOCUMENT_ID ); result = hibQuery.list(); assertNotNull( result ); @@ -251,7 +252,7 @@ assertNotNull( projection ); assertTrue( "DOCUMENT incorrect", projection[0] instanceof Document ); - assertEquals( "DOCUMENT size incorrect", 4, ( (Document) projection[0] ).getFields().size() ); + assertEquals( "DOCUMENT size incorrect", 5, ( (Document) projection[0] ).getFields().size() ); assertEquals( "THIS incorrect", projection[1], s.get( Employee.class, (Serializable) projection[4] ) ); assertEquals( "SCORE incorrect", 1.0F, projection[2] ); assertNull( "BOOST not removed", projection[3] ); @@ -259,7 +260,8 @@ assertEquals( "id incorrect", 1001, projection[5] ); assertEquals( "last name incorrect", "Jackson", projection[6] ); assertEquals( "dept incorrect", "Accounting", projection[7] ); - assertNotNull( "Lucene internal doc id", projection[8] ); + assertNotNull( "Date", projection[8] ); + assertNotNull( "Lucene internal doc id", projection[9] ); //cleanup for (Object element : s.createQuery( "from " + Employee.class.getName() ).list()) s.delete( element ); @@ -272,6 +274,7 @@ Employee e1 = new Employee( 1000, "Griffin", "ITech" ); s.save( e1 ); Employee e2 = new Employee( 1001, "Jackson", "Accounting" ); + e2.setHireDate( new Date() ); s.save( e2 ); Employee e3 = new Employee( 1002, "Jimenez", "ITech" ); s.save( e3 ); Modified: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/NotSharedReaderPerfTest.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/NotSharedReaderPerfTest.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/NotSharedReaderPerfTest.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -2,9 +2,7 @@ package org.hibernate.search.test.reader; import org.hibernate.cfg.Configuration; -import org.hibernate.search.store.FSDirectoryProvider; import org.hibernate.search.Environment; -import org.apache.lucene.analysis.StopAnalyzer; /** * @author Emmanuel Bernard @@ -12,9 +10,6 @@ public class NotSharedReaderPerfTest extends ReaderPerfTestCase { protected void configure(Configuration cfg) { super.configure( cfg ); - cfg.setProperty( "hibernate.search.default.directory_provider", FSDirectoryProvider.class.getName() ); - cfg.setProperty( "hibernate.search.default.indexBase", "./indextemp" ); - cfg.setProperty( Environment.ANALYZER_CLASS, StopAnalyzer.class.getName() ); cfg.setProperty( Environment.READER_STRATEGY, "not-shared" ); } } Modified: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/ReaderPerfTestCase.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/ReaderPerfTestCase.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/ReaderPerfTestCase.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -2,41 +2,38 @@ package org.hibernate.search.test.reader; import java.io.File; +import java.util.List; +import java.util.Random; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.Random; -import java.util.List; -import org.hibernate.search.test.SearchTestCase; -import org.hibernate.search.Environment; -import org.hibernate.search.Search; -import org.hibernate.search.FullTextQuery; -import org.hibernate.search.event.FullTextIndexEventListener; -import org.hibernate.search.store.FSDirectoryProvider; -import org.hibernate.SessionFactory; -import org.hibernate.Session; -import org.hibernate.Transaction; -import org.hibernate.event.PostDeleteEventListener; -import org.hibernate.event.PostUpdateEventListener; -import org.hibernate.event.PostInsertEventListener; -import org.apache.lucene.queryParser.QueryParser; -import org.apache.lucene.queryParser.ParseException; -import org.apache.lucene.queryParser.MultiFieldQueryParser; import org.apache.lucene.analysis.StopAnalyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.queryParser.MultiFieldQueryParser; +import org.apache.lucene.queryParser.ParseException; +import org.apache.lucene.queryParser.QueryParser; import org.apache.lucene.search.Query; +import org.hibernate.Session; +import org.hibernate.SessionFactory; +import org.hibernate.Transaction; +import org.hibernate.search.Environment; +import org.hibernate.search.FullTextQuery; +import org.hibernate.search.Search; +import org.hibernate.search.store.FSDirectoryProvider; +import org.hibernate.search.test.SearchTestCase; +import org.hibernate.search.util.FileHelper; /** * @author Emmanuel Bernard */ -public class ReaderPerfTestCase extends SearchTestCase { +public abstract class ReaderPerfTestCase extends SearchTestCase { protected void setUp() throws Exception { File sub = getBaseIndexDir(); sub.mkdir(); File[] files = sub.listFiles(); for ( File file : files ) { if ( file.isDirectory() ) { - delete( file ); + FileHelper.delete( file ); } } //super.setUp(); //we need a fresh session factory each time for index set up @@ -58,21 +55,9 @@ protected void tearDown() throws Exception { super.tearDown(); File sub = getBaseIndexDir(); - delete( sub ); + FileHelper.delete( sub ); } - private void delete(File sub) { - if ( sub.isDirectory() ) { - for ( File file : sub.listFiles() ) { - delete( file ); - } - sub.delete(); - } - else { - sub.delete(); - } - } - public boolean insert = true; public void testConcurrency() throws Exception { @@ -180,7 +165,7 @@ } } - protected class ReverseWork implements Runnable { + protected static class ReverseWork implements Runnable { private SessionFactory sf; private Random random = new Random(); @@ -236,12 +221,8 @@ super.configure( cfg ); File sub = getBaseIndexDir(); cfg.setProperty( "hibernate.search.default.indexBase", sub.getAbsolutePath() ); - cfg.setProperty( "hibernate.search.Clock.directory_provider", FSDirectoryProvider.class.getName() ); + cfg.setProperty( "hibernate.search.default.directory_provider", FSDirectoryProvider.class.getName() ); cfg.setProperty( Environment.ANALYZER_CLASS, StopAnalyzer.class.getName() ); - FullTextIndexEventListener del = new FullTextIndexEventListener(); - cfg.getEventListeners().setPostDeleteEventListeners( new PostDeleteEventListener[]{del} ); - cfg.getEventListeners().setPostUpdateEventListeners( new PostUpdateEventListener[]{del} ); - cfg.getEventListeners().setPostInsertEventListeners( new PostInsertEventListener[]{del} ); } } Modified: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/SharedReaderPerfTest.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/SharedReaderPerfTest.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/SharedReaderPerfTest.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -2,10 +2,7 @@ package org.hibernate.search.test.reader; import org.hibernate.cfg.Configuration; -import org.hibernate.search.store.RAMDirectoryProvider; -import org.hibernate.search.store.FSDirectoryProvider; import org.hibernate.search.Environment; -import org.apache.lucene.analysis.StopAnalyzer; /** * @author Emmanuel Bernard @@ -13,9 +10,6 @@ public class SharedReaderPerfTest extends ReaderPerfTestCase { protected void configure(Configuration cfg) { super.configure( cfg ); - cfg.setProperty( "hibernate.search.default.directory_provider", FSDirectoryProvider.class.getName() ); - cfg.setProperty( "hibernate.search.default.indexBase", "./indextemp" ); - cfg.setProperty( Environment.ANALYZER_CLASS, StopAnalyzer.class.getName() ); cfg.setProperty( Environment.READER_STRATEGY, "shared" ); } } Added: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/functionality/FilterOnDirectoryTest.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/functionality/FilterOnDirectoryTest.java (rev 0) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/functionality/FilterOnDirectoryTest.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -0,0 +1,70 @@ +package org.hibernate.search.test.reader.functionality; + +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.hibernate.Session; +import org.hibernate.Transaction; +import org.hibernate.search.Environment; +import org.hibernate.search.FullTextQuery; +import org.hibernate.search.FullTextSession; +import org.hibernate.search.Search; +import org.hibernate.search.reader.SharingBufferReaderProvider; +import org.hibernate.search.test.SearchTestCase; +import org.hibernate.search.test.reader.Detective; +import org.hibernate.search.test.reader.Suspect; + +public class FilterOnDirectoryTest extends SearchTestCase { + + public void testFilteredClasses() throws Exception { + createDoeFamily(); + FullTextSession fts = Search.createFullTextSession( openSession() ); + Transaction tx = fts.beginTransaction(); + Query q = new TermQuery( new Term( "name", "doe" ) ); + + assertEquals( 2, fts.createFullTextQuery( q ).getResultSize() ); + assertEquals( 2, fts.createFullTextQuery( q, Detective.class, Suspect.class ).getResultSize() ); + + FullTextQuery detectiveQuery = fts.createFullTextQuery( q, Detective.class ); + assertEquals( 1, detectiveQuery.getResultSize() ); + assertTrue( detectiveQuery.list().get(0) instanceof Detective ); + + FullTextQuery suspectQuery = fts.createFullTextQuery( q, Suspect.class ); + assertEquals( 1, suspectQuery.getResultSize() ); + assertTrue( suspectQuery.list().get(0) instanceof Suspect ); + + assertEquals( 2, fts.createFullTextQuery( q ).getResultSize() ); + assertEquals( 2, fts.createFullTextQuery( q, Detective.class, Suspect.class ).getResultSize() ); + + tx.commit(); + fts.close(); + } + + private void createDoeFamily() { + Session s = openSession( ); + Transaction tx = s.beginTransaction(); + Detective detective = new Detective(); + detective.setName( "John Doe" ); + s.persist( detective ); + Suspect suspect = new Suspect(); + suspect.setName( "Jane Doe" ); + s.persist( suspect ); + tx.commit(); + s.close(); + } + + protected void configure(org.hibernate.cfg.Configuration cfg) { + super.configure( cfg ); + cfg.setProperty( Environment.ANALYZER_CLASS, StandardAnalyzer.class.getName() ); + cfg.setProperty( Environment.READER_STRATEGY, SharingBufferReaderProvider.class.getName() ); + } + + protected Class[] getMappings() { + return new Class[] { + Detective.class, + Suspect.class + }; + } + +} Added: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/functionality/SharingBufferIndexProviderTest.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/functionality/SharingBufferIndexProviderTest.java (rev 0) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/functionality/SharingBufferIndexProviderTest.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -0,0 +1,104 @@ +package org.hibernate.search.test.reader.functionality; + +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.lucene.index.IndexReader; +import org.hibernate.search.store.DirectoryProvider; +import org.hibernate.search.test.reader.functionality.TestableSharingBufferReaderProvider.MockIndexReader; +import org.hibernate.search.test.reader.functionality.TestableSharingBufferReaderProvider.TestManipulatorPerDP; + +import junit.framework.TestCase; + +/** + * @author Sanne Grinovero + */ +public class SharingBufferIndexProviderTest extends TestCase { + + private final TestableSharingBufferReaderProvider readerProvider = new TestableSharingBufferReaderProvider(); + private final CountDownLatch startSignal = new CountDownLatch(1); + private final Runnable searchTask = new SearchTask(); + private final Runnable changeTask = new ChangeTask(); + private final AtomicInteger countDoneSearches = new AtomicInteger(); + private final AtomicInteger countDoneIndexmods = new AtomicInteger(); + private static final int SEARCHES_NUM = 50000; + private static final Random random = new Random(); + + public void testStressingMock() throws InterruptedException { + readerProvider.initialize(null, null); + ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool( 200 );//much chaos + for ( int i = 0; i < SEARCHES_NUM; i++ ) { + executor.execute( makeTask( i ) ); + } + executor.shutdown(); + startSignal.countDown(); + executor.awaitTermination( 500, TimeUnit.SECONDS ); + assertTrue( "memory leak: holding a reference to some unused IndexReader", readerProvider.areAllOldReferencesGone() ); + for ( MockIndexReader reader : readerProvider.getCreatedIndexReaders() ) { + if ( readerProvider.isReaderCurrent( reader ) ) { + assertTrue( "the most current reader should be open", ! reader.isClosed() ); + } + else { + assertTrue( "an IndexReader is still open", reader.isClosed() ); + } + } + assertEquals( SEARCHES_NUM, countDoneSearches.get() ); + assertEquals( SEARCHES_NUM/10, countDoneIndexmods.get() ); + } + + private Runnable makeTask(int i) { + if ( i % 10 == 0) { + return changeTask; + } + else { + return searchTask; + } + } + + private DirectoryProvider[] getRandomEvailableDPs() { + int arraySize = random.nextInt( readerProvider.manipulators.size() - 1 ) + 1; + DirectoryProvider[] array = new DirectoryProvider[arraySize]; + List<DirectoryProvider> availableDPs = new ArrayList<DirectoryProvider>( readerProvider.manipulators.keySet() ); + for (int i=0; i<arraySize; i++){ + int chosenDpIndex = random.nextInt( availableDPs.size() ); + array[i] = availableDPs.get( chosenDpIndex ); + availableDPs.remove( array[i] ); + } + return array; + } + + private class SearchTask implements Runnable { + public void run() { + try { + startSignal.await(); + } catch (InterruptedException e) { + //manage termination: + return; + } + IndexReader fakeOpenReader = readerProvider.openReader( getRandomEvailableDPs() ); + Thread.yield(); + readerProvider.closeReader( fakeOpenReader ); + countDoneSearches.incrementAndGet(); + } + } + + private class ChangeTask extends SearchTask { + public void run() { + super.run(); + Thread.yield(); + DirectoryProvider[] randomEvailableDPs = getRandomEvailableDPs(); + for ( DirectoryProvider dp : randomEvailableDPs ) { + TestManipulatorPerDP testManipulatorPerDP = readerProvider.manipulators.get( dp ); + testManipulatorPerDP.setIndexChanged(); + } + countDoneIndexmods.incrementAndGet(); + } + } + +} Added: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/functionality/TestableSharingBufferReaderProvider.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/functionality/TestableSharingBufferReaderProvider.java (rev 0) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/functionality/TestableSharingBufferReaderProvider.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -0,0 +1,266 @@ +package org.hibernate.search.test.reader.functionality; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Vector; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.FieldSelector; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.MultiReader; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermDocs; +import org.apache.lucene.index.TermEnum; +import org.apache.lucene.index.TermFreqVector; +import org.apache.lucene.index.TermPositions; +import org.apache.lucene.index.TermVectorMapper; +import org.hibernate.search.SearchException; +import org.hibernate.search.engine.SearchFactoryImplementor; +import org.hibernate.search.reader.ReaderProviderHelper; +import org.hibernate.search.reader.SharingBufferReaderProvider; +import org.hibernate.search.store.DirectoryProvider; +import org.hibernate.search.store.RAMDirectoryProvider; + +/** + * @author Sanne Grinovero + */ +public class TestableSharingBufferReaderProvider extends SharingBufferReaderProvider { + + private static final int NUM_DIRECTORY_PROVIDERS = 4; + private final Vector<MockIndexReader> createdReadersHistory = new Vector<MockIndexReader>( 500 ); + final Map<DirectoryProvider,TestManipulatorPerDP> manipulators = new ConcurrentHashMap<DirectoryProvider,TestManipulatorPerDP>(); + + public TestableSharingBufferReaderProvider() { + for (int i=0; i<NUM_DIRECTORY_PROVIDERS; i++) { + TestManipulatorPerDP tm = new TestManipulatorPerDP( i ); + manipulators.put( tm.dp, tm ); + } + } + + public static class TestManipulatorPerDP { + private final AtomicBoolean isIndexReaderCurrent = new AtomicBoolean( false );//starts at true, see MockIndexReader contructor + private final AtomicBoolean isReaderCreated = new AtomicBoolean( false ); + private final DirectoryProvider dp = new RAMDirectoryProvider(); + + public TestManipulatorPerDP( int seed ) { + dp.initialize( "dp" + seed, null, null ); + dp.start(); + } + + public void setIndexChanged() { + isIndexReaderCurrent.set( false ); + } + + } + + public boolean isReaderCurrent(MockIndexReader reader) { + //avoid usage of allReaders or test would be useless + for (PerDirectoryLatestReader latest : super.currentReaders.values() ) { + IndexReader latestReader = latest.current.reader; + if ( latestReader == reader) { + return true; + } + } + return false; + } + + @Override + protected IndexReader readerFactory(DirectoryProvider provider) { + TestManipulatorPerDP manipulatorPerDP = manipulators.get( provider ); + if ( ! manipulatorPerDP.isReaderCreated.compareAndSet( false, true ) ) { + throw new IllegalStateException( "IndexReader1 created twice" ); + } + else { + return new MockIndexReader( manipulatorPerDP.isIndexReaderCurrent ); + } + } + + @Override + public void initialize(Properties props, SearchFactoryImplementor searchFactoryImplementor) { + Map<DirectoryProvider,PerDirectoryLatestReader> map = new HashMap<DirectoryProvider,PerDirectoryLatestReader>(); + try { + for ( DirectoryProvider dp : manipulators.keySet() ) { + map.put( dp, new PerDirectoryLatestReader( dp ) ); + } + } catch (IOException e) { + throw new SearchException( "Unable to open Lucene IndexReader", e ); + } + currentReaders = Collections.unmodifiableMap( map ); + } + + public boolean areAllOldReferencesGone() { + int numReferencesReaders = super.allReaders.size(); + int numExpectedActiveReaders = manipulators.size(); + return numReferencesReaders == numExpectedActiveReaders; + } + + public List<MockIndexReader> getCreatedIndexReaders(){ + return createdReadersHistory; + } + + public MockIndexReader getCurrentMockReaderPerDP(DirectoryProvider dp) { + IndexReader[] indexReaders = ReaderProviderHelper.getSubReadersFromMultiReader( (MultiReader) super.openReader( new DirectoryProvider[]{ dp } ) ); + if ( indexReaders.length != 1 ){ + throw new IllegalStateException( "Expecting one reader" ); + } + return (MockIndexReader) indexReaders[0]; + } + + public class MockIndexReader extends IndexReader { + + private final AtomicBoolean closed = new AtomicBoolean( false ); + private final AtomicBoolean hasAlreadyBeenReOpened = new AtomicBoolean( false ); + private final AtomicBoolean isIndexReaderCurrent; + + MockIndexReader(AtomicBoolean isIndexReaderCurrent) { + this.isIndexReaderCurrent = isIndexReaderCurrent; + if ( ! isIndexReaderCurrent.compareAndSet(false, true) ) { + throw new IllegalStateException( "Unnecessarily reopened" ); + } + createdReadersHistory.add( this ); + } + + public final boolean isClosed() { + return closed.get(); + } + + @Override + protected void doClose() throws IOException { + boolean okToClose = closed.compareAndSet( false, true ); + if ( ! okToClose ) { + throw new IllegalStateException( "Attempt to close a closed IndexReader" ); + } + if ( ! hasAlreadyBeenReOpened.get() ) { + throw new IllegalStateException( "Attempt to close the most current IndexReader" ); + } + } + + @Override + public synchronized IndexReader reopen(){ + if ( isIndexReaderCurrent.get() ) { + return this; + } + else { + if ( hasAlreadyBeenReOpened.compareAndSet( false, true) ) { + return new MockIndexReader( isIndexReaderCurrent ); + } + else + throw new IllegalStateException( "Attempt to reopen an old IndexReader more than once" ); + } + } + + @Override + protected void doCommit() { + throw new UnsupportedOperationException(); + } + + @Override + protected void doDelete(int docNum) { + throw new UnsupportedOperationException(); + } + + @Override + protected void doSetNorm(int doc, String field, byte value) { + throw new UnsupportedOperationException(); + } + + @Override + protected void doUndeleteAll() { + throw new UnsupportedOperationException(); + } + + @Override + public int docFreq(Term t) { + throw new UnsupportedOperationException(); + } + + @Override + public Document document(int n, FieldSelector fieldSelector) { + throw new UnsupportedOperationException(); + } + + @Override + public Collection getFieldNames(FieldOption fldOption) { + throw new UnsupportedOperationException(); + } + + @Override + public TermFreqVector getTermFreqVector(int docNumber, String field) { + throw new UnsupportedOperationException(); + } + + @Override + public void getTermFreqVector(int docNumber, String field, TermVectorMapper mapper) { + throw new UnsupportedOperationException(); + } + + @Override + public void getTermFreqVector(int docNumber, TermVectorMapper mapper) { + throw new UnsupportedOperationException(); + } + + @Override + public TermFreqVector[] getTermFreqVectors(int docNumber) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean hasDeletions() { + return false;//just something to make MultiReader constructor happy + } + + @Override + public boolean isDeleted(int n) { + throw new UnsupportedOperationException(); + } + + @Override + public int maxDoc() { + return 10;//just something to make MultiReader constructor happy + } + + @Override + public byte[] norms(String field) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public void norms(String field, byte[] bytes, int offset) { + throw new UnsupportedOperationException(); + } + + @Override + public int numDocs() { + throw new UnsupportedOperationException(); + } + + @Override + public TermDocs termDocs() { + throw new UnsupportedOperationException(); + } + + @Override + public TermPositions termPositions() { + throw new UnsupportedOperationException(); + } + + @Override + public TermEnum terms() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public TermEnum terms(Term t) throws IOException { + throw new UnsupportedOperationException(); + } + + } + +} Added: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/performance/AbstractActivity.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/performance/AbstractActivity.java (rev 0) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/performance/AbstractActivity.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -0,0 +1,80 @@ +package org.hibernate.search.test.reader.performance; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.queryParser.MultiFieldQueryParser; +import org.apache.lucene.queryParser.ParseException; +import org.apache.lucene.queryParser.QueryParser; +import org.apache.lucene.search.Query; +import org.hibernate.Session; +import org.hibernate.SessionFactory; +import org.hibernate.Transaction; +import org.hibernate.search.FullTextQuery; +import org.hibernate.search.FullTextSession; +import org.hibernate.search.Search; + +/** + * @author Sanne Grinovero + */ +public abstract class AbstractActivity implements Runnable { + + private final ThreadLocal<QueryParser> parsers = new ThreadLocal<QueryParser>(){ + @Override + protected QueryParser initialValue(){ + return new MultiFieldQueryParser( + new String[] {"name", "physicalDescription", "suspectCharge"}, + new StandardAnalyzer() ); + } + }; + + private final SessionFactory sf; + private final AtomicInteger jobSeed = new AtomicInteger(); + private final CountDownLatch startSignal; + + AbstractActivity(SessionFactory sf, CountDownLatch startSignal) { + this.startSignal = startSignal; + this.sf = sf; + } + + public final void run() { + try { + startSignal.await(); + } catch (InterruptedException e) { + e.printStackTrace(); + return; + } + Session s = sf.openSession(); + try { + FullTextSession fts = Search.createFullTextSession( s ); + Transaction tx = s.beginTransaction(); + boolean ok = false; + try { + doAction( fts, jobSeed.getAndIncrement() ); + ok = true; + } finally { + if (ok) + tx.commit(); + else + tx.rollback(); + } + } finally { + s.close(); + } + } + + protected FullTextQuery getQuery(String queryString, FullTextSession s, Class... classes) { + Query luceneQuery = null; + try { + luceneQuery = parsers.get().parse(queryString); + } + catch (ParseException e) { + e.printStackTrace(); + } + return s.createFullTextQuery( luceneQuery, classes ); + } + + protected abstract void doAction(FullTextSession s, int jobSeed); + +} Added: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/performance/BufferSharingReaderPerfTest.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/performance/BufferSharingReaderPerfTest.java (rev 0) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/performance/BufferSharingReaderPerfTest.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -0,0 +1,15 @@ +package org.hibernate.search.test.reader.performance; + +import org.hibernate.search.reader.SharingBufferReaderProvider; + +/** + * @author Sanne Grinovero + */ +public class BufferSharingReaderPerfTest extends ReaderPerformance { + + @Override + protected String getReaderStrategyName() { + return SharingBufferReaderProvider.class.getName(); + } + +} Added: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/performance/IndexFillRunnable.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/performance/IndexFillRunnable.java (rev 0) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/performance/IndexFillRunnable.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -0,0 +1,37 @@ +package org.hibernate.search.test.reader.performance; + +import java.io.IOException; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.Field.Index; +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.index.IndexWriter; + +/** + * @author Sanne Grinovero + */ +public class IndexFillRunnable implements Runnable { + + private volatile int jobSeed = 0; + private final IndexWriter iw; + + public IndexFillRunnable(IndexWriter iw) { + super(); + this.iw = iw; + } + + public void run() { + Field f1 = new Field("name", "Some One " + jobSeed++, Store.NO, Index.TOKENIZED ); + Field f2 = new Field("physicalDescription", " just more people sitting around and filling my index... ", Store.NO, Index.TOKENIZED ); + Document d = new Document(); + d.add( f1 ); + d.add( f2 ); + try { + iw.addDocument( d ); + } catch (IOException e) { + e.printStackTrace(); + } + } + +} Added: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/performance/InsertActivity.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/performance/InsertActivity.java (rev 0) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/performance/InsertActivity.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -0,0 +1,38 @@ +package org.hibernate.search.test.reader.performance; + +import java.util.concurrent.CountDownLatch; + +import org.hibernate.SessionFactory; +import org.hibernate.search.FullTextSession; +import org.hibernate.search.test.reader.Detective; +import org.hibernate.search.test.reader.Suspect; + +/** + * @author Emmanuel Bernard + * @author Sanne Grinovero + */ +public class InsertActivity extends AbstractActivity { + + InsertActivity(SessionFactory sf, CountDownLatch startSignal) { + super(sf, startSignal); + } + + @Override + protected void doAction(FullTextSession s, int jobSeed) { + Detective detective = new Detective(); + detective.setName("John Doe " + jobSeed); + detective.setBadge("123455" + jobSeed); + detective.setPhysicalDescription("Blond green eye etc etc"); + s.persist(detective); + Suspect suspect = new Suspect(); + suspect.setName("Jane Doe " + jobSeed); + suspect.setPhysicalDescription("brunette, short, 30-ish"); + if (jobSeed % 20 == 0) { + suspect.setSuspectCharge("thief liar "); + } else { + suspect.setSuspectCharge(" It's 1875 in London. The police have captured career criminal Montmorency. In the process he has been grievously wounded and it is up to a young surgeon to treat his wounds. During his recovery Montmorency learns of the city's new sewer system and sees in it the perfect underground highway for his thievery. Washington Post columnist John Kelly recommends this title for middle schoolers, especially to be read aloud."); + } + s.persist(suspect); + } + +} Added: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/performance/NotSharedReaderPerfTest.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/performance/NotSharedReaderPerfTest.java (rev 0) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/performance/NotSharedReaderPerfTest.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -0,0 +1,13 @@ +package org.hibernate.search.test.reader.performance; + +/** + * @author Sanne Grinovero + */ +public class NotSharedReaderPerfTest extends ReaderPerformance { + + @Override + protected String getReaderStrategyName() { + return "not-shared"; + } + +} Added: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/performance/ReaderPerformance.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/performance/ReaderPerformance.java (rev 0) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/performance/ReaderPerformance.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -0,0 +1,128 @@ +package org.hibernate.search.test.reader.performance; + +import java.io.File; +import java.io.IOException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import org.apache.lucene.analysis.SimpleAnalyzer; +import org.apache.lucene.analysis.StopAnalyzer; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.LockObtainFailedException; +import org.hibernate.search.Environment; +import org.hibernate.search.store.FSDirectoryProvider; +import org.hibernate.search.test.SearchTestCase; +import org.hibernate.search.test.reader.Detective; +import org.hibernate.search.test.reader.Suspect; +import org.hibernate.search.util.FileHelper; + +/** + * To enable performance tests: de-comment buildBigIndex(); in setUp() and rename no_testPerformance + * @author Sanne Grinovero + */ +public abstract class ReaderPerformance extends SearchTestCase { + + private static final File baseIndexDir = new File( new File( "." ), "indextemp" ); + + //more iterations for more reliable measures: + private static final int TOTAL_WORK_BATCHES = 10; + //the next 3 define the kind of workload mix to test on: + private static final int SEARCHERS_PER_BATCH = 10; + private static final int UPDATES_PER_BATCH = 2; + private static final int INSERTIONS_PER_BATCH = 1; + + private static final int WORKER_THREADS = 20; + + private static final int WARMUP_CYCLES = 6; + + protected void setUp() throws Exception { + baseIndexDir.mkdir(); + File[] files = baseIndexDir.listFiles(); + for ( File file : files ) { + FileHelper.delete( file ); + } + super.setUp(); + } + + public void testFakeTest(){ + //to make JUnit happy when disabling performance test + } + + private void buildBigIndex() throws InterruptedException, CorruptIndexException, LockObtainFailedException, IOException { + System.out.println( "Going to create fake index..." ); + FSDirectory directory = FSDirectory.getDirectory(new File(baseIndexDir, Detective.class.getCanonicalName())); + IndexWriter iw = new IndexWriter( directory, new SimpleAnalyzer(), true ); + IndexFillRunnable filler = new IndexFillRunnable( iw ); + ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool( WORKER_THREADS ); + for (int batch=0; batch<=5000000; batch++){ + executor.execute( filler ); + } + executor.shutdown(); + executor.awaitTermination( 600, TimeUnit.SECONDS ); + iw.optimize(); + iw.close(); + System.out.println( "Index created." ); + } + + protected Class[] getMappings() { + return new Class[] { + Detective.class, + Suspect.class + }; + } + + protected void tearDown() throws Exception { + super.tearDown(); + FileHelper.delete( baseIndexDir ); + } + + protected void configure(org.hibernate.cfg.Configuration cfg) { + super.configure( cfg ); + cfg.setProperty( "hibernate.search.default.directory_provider", FSDirectoryProvider.class.getName() ); + cfg.setProperty( "hibernate.search.default.indexBase", baseIndexDir.getAbsolutePath() ); + cfg.setProperty( "hibernate.search.default.optimizer.transaction_limit.max", "10" ); // workaround too many open files + cfg.setProperty( Environment.ANALYZER_CLASS, StopAnalyzer.class.getName() ); + cfg.setProperty( Environment.READER_STRATEGY, getReaderStrategyName() ); + } + + protected abstract String getReaderStrategyName(); + + //this test is disabled as it is very slow (and someone should read the output) + public final void disabled_testPerformance() throws InterruptedException, CorruptIndexException, LockObtainFailedException, IOException { + buildBigIndex(); + for (int i=0; i<WARMUP_CYCLES; i++) { + timeMs(); + } + } + + private final void timeMs() throws InterruptedException { + ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool( WORKER_THREADS ); + CountDownLatch startSignal = new CountDownLatch(1); + InsertActivity insertionTask = new InsertActivity( getSessions(), startSignal ); + SearchActivity searchTask = new SearchActivity( getSessions(), startSignal ); + UpdateActivity updateTask = new UpdateActivity( getSessions(), startSignal ); + //we declare needed activities in order, scheduler will "mix": + for (int batch=0; batch<=TOTAL_WORK_BATCHES; batch++){ + for ( int inserters=0; inserters<INSERTIONS_PER_BATCH; inserters++) + executor.execute( insertionTask ); + for ( int searchers=0; searchers<SEARCHERS_PER_BATCH; searchers++) + executor.execute( searchTask ); + for ( int updaters=0; updaters<UPDATES_PER_BATCH; updaters++) + executor.execute( updateTask ); + } + executor.shutdown(); + long startTime = System.currentTimeMillis(); + startSignal.countDown();//start! + executor.awaitTermination( 600, TimeUnit.SECONDS ); + long endTime = System.currentTimeMillis(); + System.out.println( "Performance test for " + getReaderStrategyName() + ": " + (endTime - startTime) +"ms. (" + + (TOTAL_WORK_BATCHES*SEARCHERS_PER_BATCH) + " searches, " + + (TOTAL_WORK_BATCHES*INSERTIONS_PER_BATCH) + " insertions, " + + (TOTAL_WORK_BATCHES*UPDATES_PER_BATCH) + " updates)" ); + } + +} Added: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/performance/SearchActivity.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/performance/SearchActivity.java (rev 0) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/performance/SearchActivity.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -0,0 +1,27 @@ +package org.hibernate.search.test.reader.performance; + +import java.util.concurrent.CountDownLatch; + +import org.hibernate.SessionFactory; +import org.hibernate.search.FullTextQuery; +import org.hibernate.search.FullTextSession; +import org.hibernate.search.test.reader.Detective; + +/** + * @author Emmanuel Bernard + * @author Sanne Grinovero + */ +public class SearchActivity extends AbstractActivity { + + SearchActivity(SessionFactory sf, CountDownLatch startSignal) { + super(sf, startSignal); + } + + @Override + protected void doAction(FullTextSession s, int jobSeed) { + FullTextQuery q = getQuery( "John Doe", s, Detective.class); + q.setMaxResults( 10 ); + q.getResultSize(); + } + +} Added: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/performance/SharedReaderPerfTest.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/performance/SharedReaderPerfTest.java (rev 0) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/performance/SharedReaderPerfTest.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -0,0 +1,13 @@ +package org.hibernate.search.test.reader.performance; + +/** + * @author Sanne Grinovero + */ +public class SharedReaderPerfTest extends ReaderPerformance { + + @Override + protected String getReaderStrategyName() { + return "shared"; + } + +} Added: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/performance/UpdateActivity.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/performance/UpdateActivity.java (rev 0) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/reader/performance/UpdateActivity.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -0,0 +1,30 @@ +package org.hibernate.search.test.reader.performance; + +import java.util.List; +import java.util.concurrent.CountDownLatch; + +import org.hibernate.SessionFactory; +import org.hibernate.search.FullTextQuery; +import org.hibernate.search.FullTextSession; +import org.hibernate.search.test.reader.Detective; + +/** + * @author Sanne Grinovero + */ +public class UpdateActivity extends AbstractActivity { + + UpdateActivity(SessionFactory sf, CountDownLatch startSignal) { + super(sf, startSignal); + } + + @Override + protected void doAction(FullTextSession s, int jobSeed) { + FullTextQuery q = getQuery( "John", s, Detective.class ); + List list = q.setMaxResults( 1 ).list(); + for ( Object o : list){ + Detective detective = (Detective) o; + detective.setPhysicalDescription( "old" ); + } + } + +} Added: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/session/Domain.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/session/Domain.java (rev 0) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/session/Domain.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -0,0 +1,37 @@ +package org.hibernate.search.test.session; + +import javax.persistence.Entity; +import javax.persistence.Id; + +import org.hibernate.search.annotations.Indexed; +import org.hibernate.search.annotations.DocumentId; +import org.hibernate.search.annotations.Field; + +/** + * @author Emmanuel Bernard + */ +@Entity +@Indexed +public class Domain { + @Id + @DocumentId + private Integer id; + @Field + private String name; + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } +} Modified: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/session/Email.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/session/Email.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/session/Email.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -4,11 +4,14 @@ import javax.persistence.Entity; import javax.persistence.Id; import javax.persistence.GeneratedValue; +import javax.persistence.ManyToOne; +import javax.persistence.FetchType; import org.hibernate.search.annotations.Indexed; import org.hibernate.search.annotations.DocumentId; import org.hibernate.search.annotations.Field; import org.hibernate.search.annotations.Index; +import org.hibernate.search.annotations.IndexedEmbedded; /** * @author Emmanuel Bernard @@ -28,7 +31,17 @@ private String header; + @IndexedEmbedded @ManyToOne(fetch = FetchType.LAZY) + private Domain domain; + public Domain getDomain() { + return domain; + } + + public void setDomain(Domain domain) { + this.domain = domain; + } + public Long getId() { return id; } Added: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/session/MassIndexUsingManualFlushTest.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/session/MassIndexUsingManualFlushTest.java (rev 0) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/session/MassIndexUsingManualFlushTest.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -0,0 +1,74 @@ +package org.hibernate.search.test.session; + +import java.sql.Statement; +import java.util.List; + +import org.hibernate.search.FullTextSession; +import org.hibernate.search.Search; +import org.hibernate.search.Environment; +import org.hibernate.search.test.SearchTestCase; +import org.hibernate.search.impl.FullTextSessionImpl; +import org.hibernate.Transaction; +import org.hibernate.ScrollableResults; +import org.hibernate.ScrollMode; +import org.apache.lucene.queryParser.QueryParser; +import org.apache.lucene.analysis.StopAnalyzer; + +/** + * @author Emmanuel Bernard + */ +public class MassIndexUsingManualFlushTest extends SearchTestCase { + public void testManualIndexFlush() throws Exception { + FullTextSession s = Search.createFullTextSession( openSession() ); + Transaction tx = s.beginTransaction(); + int loop = 14; + for (int i = 0; i < loop; i++) { + Statement statmt = s.connection().createStatement(); + statmt.executeUpdate( "insert into Domain(id, name) values( + " + + ( i + 1 ) + ", 'sponge" + i + "')" ); + statmt.executeUpdate( "insert into Email(id, title, body, header, domain_id) values( + " + + ( i + 1 ) + ", 'Bob Sponge', 'Meet the guys who create the software', 'nope', " + ( i + 1 ) +")" ); + statmt.close(); + } + tx.commit(); + s.close(); + + //check non created object does get found!!1 + s = new FullTextSessionImpl( openSession() ); + tx = s.beginTransaction(); + ScrollableResults results = s.createCriteria( Email.class ).scroll( ScrollMode.FORWARD_ONLY ); + int index = 0; + while ( results.next() ) { + index++; + final Email o = (Email) results.get( 0 ); + s.index( o ); + if ( index % 5 == 0 ) { + s.flushToIndexes(); + s.clear(); + } + } + tx.commit(); + s.clear(); + tx = s.beginTransaction(); + QueryParser parser = new QueryParser( "id", new StopAnalyzer() ); + List result = s.createFullTextQuery( parser.parse( "body:create" ) ).list(); + assertEquals( 14, result.size() ); + for (Object object : result) { + s.delete( object ); + } + tx.commit(); + s.close(); + } + + protected void configure(org.hibernate.cfg.Configuration cfg) { + super.configure( cfg ); + cfg.setProperty( Environment.ANALYZER_CLASS, StopAnalyzer.class.getName() ); + } + + protected Class[] getMappings() { + return new Class[] { + Email.class, + Domain.class + }; + } +} Modified: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/session/OptimizeTest.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/session/OptimizeTest.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/session/OptimizeTest.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -6,16 +6,13 @@ import org.apache.lucene.analysis.StopAnalyzer; import org.apache.lucene.queryParser.QueryParser; import org.hibernate.Transaction; -import org.hibernate.event.PostDeleteEventListener; -import org.hibernate.event.PostInsertEventListener; -import org.hibernate.event.PostUpdateEventListener; import org.hibernate.search.Environment; import org.hibernate.search.FullTextSession; import org.hibernate.search.Search; -import org.hibernate.search.event.FullTextIndexEventListener; import org.hibernate.search.impl.FullTextSessionImpl; import org.hibernate.search.store.FSDirectoryProvider; import org.hibernate.search.test.SearchTestCase; +import org.hibernate.search.util.FileHelper; /** * @author Emmanuel Bernard @@ -67,7 +64,7 @@ File[] files = sub.listFiles(); for (File file : files) { if ( file.isDirectory() ) { - delete( file ); + FileHelper.delete( file ); } } //super.setUp(); //we need a fresh session factory each time for index set up @@ -82,24 +79,13 @@ protected void tearDown() throws Exception { super.tearDown(); File sub = getBaseIndexDir(); - delete( sub ); + FileHelper.delete( sub ); } - private void delete(File sub) { - if ( sub.isDirectory() ) { - for (File file : sub.listFiles()) { - delete( file ); - } - sub.delete(); - } - else { - sub.delete(); - } - } - protected Class[] getMappings() { return new Class[] { - Email.class + Email.class, + Domain.class }; } } Modified: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/session/SessionTest.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/session/SessionTest.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/session/SessionTest.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -32,7 +32,7 @@ wrapper ); try { - FullTextSession fts = Search.createFullTextSession( wrapped ); + Search.createFullTextSession( wrapped ); } catch( ClassCastException e ) { e.printStackTrace( ); @@ -57,7 +57,8 @@ protected Class[] getMappings() { return new Class[] { - Email.class + Email.class, + Domain.class }; } } Modified: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/shards/ShardsTest.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/shards/ShardsTest.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/shards/ShardsTest.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -5,6 +5,7 @@ import java.util.List; import org.hibernate.search.test.SearchTestCase; +import org.hibernate.search.util.FileHelper; import org.hibernate.search.store.RAMDirectoryProvider; import org.hibernate.search.store.FSDirectoryProvider; import org.hibernate.search.store.IdHashShardingStrategy; @@ -158,7 +159,7 @@ File[] files = sub.listFiles(); for (File file : files) { if ( file.isDirectory() ) { - delete( file ); + FileHelper.delete( file ); } } //super.setUp(); //we need a fresh session factory each time for index set up @@ -173,21 +174,9 @@ protected void tearDown() throws Exception { super.tearDown(); File sub = getBaseIndexDir(); - delete( sub ); + FileHelper.delete( sub ); } - private void delete(File sub) { - if ( sub.isDirectory() ) { - for ( File file : sub.listFiles() ) { - delete( file ); - } - sub.delete(); - } - else { - sub.delete(); - } - } - protected Class[] getMappings() { return new Class[] { Animal.class, Modified: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/worker/ConcurrencyTest.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/worker/ConcurrencyTest.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/worker/ConcurrencyTest.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -3,7 +3,6 @@ import org.hibernate.search.test.SearchTestCase; import org.hibernate.Session; -import org.hibernate.annotations.common.AssertionFailure; /** * @author Emmanuel Bernard Modified: search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/worker/WorkerTestCase.java =================================================================== --- search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/worker/WorkerTestCase.java 2008-06-30 12:55:26 UTC (rev 14826) +++ search/branches/jboss_cache_integration/src/test/org/hibernate/search/test/worker/WorkerTestCase.java 2008-06-30 13:56:28 UTC (rev 14827) @@ -12,15 +12,12 @@ import org.hibernate.Session; import org.hibernate.SessionFactory; import org.hibernate.Transaction; -import org.hibernate.event.PostDeleteEventListener; -import org.hibernate.event.PostInsertEventListener; -import org.hibernate.event.PostUpdateEventListener; import org.hibernate.search.Environment; import org.hibernate.search.FullTextSession; -import org.hibernate.search.event.FullTextIndexEventListener; import org.hibernate.search.impl.FullTextSessionImpl; import org.hibernate.search.store.FSDirectoryProvider; import org.hibernate.search.test.SearchTestCase; +import org.hibernate.search.util.FileHelper; /** * @author Emmanuel Bernard @@ -33,7 +30,7 @@ File[] files = sub.listFiles(); for ( File file : files ) { if ( file.isDirectory() ) { - delete( file ); + FileHelper.delete( file ); } } //super.setUp(); //we need a fresh session factory each time for index set up @@ -48,21 +45,9 @@ protected void tearDown() throws Exception { super.tearDown(); File sub = getBaseIndexDir(); - delete( sub ); + FileHelper.delete( sub ); } - private void delete(File sub) { - if ( sub.isDirectory() ) { - for ( File file : sub.listFiles() ) { - delete( file ); - } - sub.delete(); - } - else { - sub.delete(); - } - } - public void testConcurrency() throws Exception { int nThreads = 15; ExecutorService es = Executors.newFixedThreadPool( nThreads ); @@ -82,7 +67,7 @@ .currentTimeMillis() - start ) ); } - protected class Work implements Runnable { + protected static class Work implements Runnable { private SessionFactory sf; public volatile int count = 0; @@ -147,7 +132,7 @@ } } - protected class ReverseWork implements Runnable { + protected static class ReverseWork implements Runnable { private SessionFactory sf; public ReverseWork(SessionFactory sf) { @@ -192,10 +177,6 @@ cfg.setProperty( "hibernate.search.default.indexBase", sub.getAbsolutePath() ); cfg.setProperty( "hibernate.search.Clock.directory_provider", FSDirectoryProvider.class.getName() ); cfg.setProperty( Environment.ANALYZER_CLASS, StopAnalyzer.class.getName() ); - FullTextIndexEventListener del = new FullTextIndexEventListener(); - cfg.getEventListeners().setPostDeleteEventListeners( new PostDeleteEventListener[]{del} ); - cfg.getEventListeners().setPostUpdateEventListeners( new PostUpdateEventListener[]{del} ); - cfg.getEventListeners().setPostInsertEventListeners( new PostInsertEventListener[]{del} ); } protected Class[] getMappings() {