Author: epbernard Date: 2007-07-31 14:48:40 -0400 (Tue, 31 Jul 2007) New Revision: 12870 Modified: trunk/HibernateExt/search/build.xml trunk/HibernateExt/search/changelog.txt trunk/HibernateExt/search/doc/reference/en/master.xml trunk/HibernateExt/search/doc/reference/en/modules/architecture.xml trunk/HibernateExt/search/doc/reference/en/modules/batchindex.xml trunk/HibernateExt/search/doc/reference/en/modules/configuration.xml trunk/HibernateExt/search/doc/reference/en/modules/lucene-native.xml trunk/HibernateExt/search/doc/reference/en/modules/mapping.xml trunk/HibernateExt/search/doc/reference/en/modules/optimize.xml trunk/HibernateExt/search/doc/reference/en/modules/query.xml trunk/HibernateExt/search/readme.txt trunk/HibernateExt/search/src/java/org/hibernate/search/Version.java Log: Doc plus version for 3.0.0.Beta4 Modified: trunk/HibernateExt/search/build.xml =================================================================== --- trunk/HibernateExt/search/build.xml 2007-07-31 17:54:39 UTC (rev 12869) +++ trunk/HibernateExt/search/build.xml 2007-07-31 18:48:40 UTC (rev 12870) @@ -16,7 +16,7 @@ <!-- Name of project and version, used to create filenames --> <property name="Name" value="Hibernate Search"/> <property name="name" value="hibernate-search"/> - <property name="version" value="3.0.0.Beta3"/> + <property name="version" value="3.0.0.Beta4"/> <property name="javadoc.packagenames" value="org.hibernate.search.*"/> <property name="copy.test" value="true"/> <property name="javac.source" value="1.5"/> @@ -180,7 +180,7 @@ <target name="junit" depends="compiletest, prepare-test-resources"> <mkdir dir="test_output"/> - <junit forkmode="perTest" printsummary="yes" haltonfailure="yes"> + <junit forkmode="once" printsummary="yes" haltonfailure="yes"> <classpath> <path path="${build.testresources.dir}"/> <!-- dirset dir="${build.testresources.dir}"> Modified: trunk/HibernateExt/search/changelog.txt =================================================================== --- trunk/HibernateExt/search/changelog.txt 2007-07-31 17:54:39 UTC (rev 12869) +++ trunk/HibernateExt/search/changelog.txt 2007-07-31 18:48:40 UTC (rev 12870) @@ -1,8 +1,45 @@ Hibernate Search Changelog ========================== +3.0.0.Beta4 (1-08-2007) +----------------------- + +** Bug + * [HSEARCH-88] - Unable to update 2 entity types in the same transaction if they share the same index + * [HSEARCH-90] - Use of setFirstResult / setMaxResults can lead to a list with negative capacity (John Griffin) + * [HSEARCH-92] - NPE for null fields on projection + * [HSEARCH-99] - Avoid returning non initialized proxies in scroll() and iterate() (loader.load(EntityInfo)) + + +** Improvement + * [HSEARCH-79] - Recommend to use FlushMode.APPLICATION on massive indexing + * [HSEARCH-84] - Migrate to Lucene 2.2 + * [HSEARCH-91] - Avoid wrapping a Session object if the Session is already FullTextSession + * [HSEARCH-100] - Rename fullTextSession.setIndexProjection() to fullTextSession.setProjection() + * [HSEARCH-102] - Default index operation in @Field to TOKENIZED + * [HSEARCH-106] - Use the shared reader strategy as the default strategy + +** New Feature + * [HSEARCH-6] - Provide access to the Hit.getScore() and potentially the Document on a query + * [HSEARCH-15] - Notion of Filtered Lucene queries (Hardy Ferentschik) + * [HSEARCH-41] - Allow fine grained analyzers (Entity, attribute, @Field) + * [HSEARCH-45] - Support @Fields() for multiple indexing per property (useful for sorting) + * [HSEARCH-58] - Support named Filters (and caching) + * [HSEARCH-67] - Expose mergeFactor, maxMergeDocs and minMergeDocs (Hardy Ferentschik) + * [HSEARCH-73] - IncrementalOptimizerStrategy triggered on transactions or operations limits + * [HSEARCH-74] - Ability to project Lucene meta information (Score, Boost, Document, Id, This) (John Griffin) + * [HSEARCH-83] - Introduce OptimizerStrategy + * [HSEARCH-86] - Index sharding: multiple Lucene indexes per entity type + * [HSEARCH-89] - FullText wrapper for JPA APIs + * [HSEARCH-103] - Ability to override the indexName in the FSDirectoryProviders family + + +** Task + * [HSEARCH-94] - Deprecate ContextHelper + + 3.0.0.Beta3 (6-06-2007) ------------------------- +----------------------- ** Bug * [HSEARCH-64] - Exception Thrown If Index Directory Does Not Exist Modified: trunk/HibernateExt/search/doc/reference/en/master.xml =================================================================== --- trunk/HibernateExt/search/doc/reference/en/master.xml 2007-07-31 17:54:39 UTC (rev 12869) +++ trunk/HibernateExt/search/doc/reference/en/master.xml 2007-07-31 18:48:40 UTC (rev 12870) @@ -18,7 +18,7 @@ <subtitle>Reference Guide</subtitle> - <releaseinfo>3.0.0.Beta3</releaseinfo> + <releaseinfo>3.0.0.Beta4</releaseinfo> <mediaobject> <imageobject> Modified: trunk/HibernateExt/search/doc/reference/en/modules/architecture.xml =================================================================== --- trunk/HibernateExt/search/doc/reference/en/modules/architecture.xml 2007-07-31 17:54:39 UTC (rev 12869) +++ trunk/HibernateExt/search/doc/reference/en/modules/architecture.xml 2007-07-31 18:48:40 UTC (rev 12870) @@ -22,7 +22,8 @@ context is shared between Hibernate and Hibernate Search ; as a matter of fact, the Search Session is built on top of the Hibernate Session. The application code use the unified <classname>org.hibernate.Query</classname> - API exactly the way a HQL or native query would be done.</para> + or <classname>javax.persistence.Query</classname> APIs exactly the way a + HQL, JPA-QL or native queries would be done.</para> <section> <title>Batching Scope</title> @@ -198,8 +199,7 @@ <para>Every time a query is executed, a Lucene IndexReader is opened. This strategy is not the most efficient: opening and warming up an - IndexReader can be a relatively expensive operation. This strategy is - the default, but will be changed in the future.</para> + IndexReader can be a relatively expensive operation.</para> </section> <section> @@ -211,7 +211,8 @@ IndexReader is not up-to-date, an new one is opened and provided. Generally speaking, this strategy provides much better performances than the <literal>not-shared</literal> strategy. It is especially true if the - number of updates is much lower than the reads.</para> + number of updates is much lower than the reads. This strategy is the + default.</para> </section> <section> Modified: trunk/HibernateExt/search/doc/reference/en/modules/batchindex.xml =================================================================== --- trunk/HibernateExt/search/doc/reference/en/modules/batchindex.xml 2007-07-31 17:54:39 UTC (rev 12869) +++ trunk/HibernateExt/search/doc/reference/en/modules/batchindex.xml 2007-07-31 18:48:40 UTC (rev 12870) @@ -1,6 +1,7 @@ <?xml version="1.0" encoding="UTF-8"?> <!-- $Id$ --> <chapter id="search-batchindex"> + <title>Indexing</title> <para>It is sometimes useful to index an object event if this object is not @@ -35,17 +36,32 @@ also applied on regular transparent indexing (and not only when <literal>session.index()</literal> is used). That's why a sensitive <literal>batch_size</literal> value is expected.</para> - - Other parameters which also can effect indexing time and memory consumption are - <literal>hibernate.search.[default|&lt;indexname&gt;].merge_factor</literal>, <literal>hibernate.search.[default|&lt;indexname&gt;].max_merge_docs</literal> - and <literal>hibernate.search.[default|&lt;indexname&gt;].max_buffered_docs</literal>. These parameters are Lucene - specific and Hibernate Search is just passing these paramters through - see <xref linkend="lucene-indexing-performance"/> - for more details. + <para>Other parameters which also can effect indexing time and memory consumption are + + <literal>hibernate.search.[default|&lt;indexname&gt;].batch.merge_factor</literal> + + , + + <literal>hibernate.search.[default|&lt;indexname&gt;].batch.max_merge_docs</literal> + + and + + <literal>hibernate.search.[default|&lt;indexname&gt;].batch.max_buffered_docs</literal> + + . These parameters are Lucene specific and Hibernate Search is just passing these paramters through - see + + <xref linkend="lucene-indexing-performance" /> + + for more details. +</para> <para>Here is an especially efficient way to index a given class (useful for index (re)initialization):</para> - <programlisting>transaction = fullTextSession.beginTransaction(); + + + <programlisting>fullTextSession.setFlushMode(FlushMode.APPLICATION); +transaction = fullTextSession.beginTransaction(); //Scrollable results will avoid loading too many objects in memory ScrollableResults results = fullTextSession.createCriteria( Email.class ).scroll( ScrollMode.FORWARD_ONLY ); int index = 0; @@ -56,7 +72,11 @@ } transaction.commit();</programlisting> + + <para>It is critical that <literal>batchSize</literal> in the previous example matches the <literal>batch_size</literal> value described previously.</para> + + </chapter> \ No newline at end of file Modified: trunk/HibernateExt/search/doc/reference/en/modules/configuration.xml =================================================================== --- trunk/HibernateExt/search/doc/reference/en/modules/configuration.xml 2007-07-31 17:54:39 UTC (rev 12869) +++ trunk/HibernateExt/search/doc/reference/en/modules/configuration.xml 2007-07-31 18:48:40 UTC (rev 12870) @@ -8,9 +8,9 @@ <para>Apache Lucene has a notion of Directory to store the index files. The Directory implementation can be customized, but Lucene comes bundled - with a file system (<literal>FSDirectoryProvider</literal>) and a full memory - (RAMDirectoryProvider) implementation. Hibernate Search has - the notion of <literal>DirectoryProvider</literal> that handles the + with a file system (<literal>FSDirectoryProvider</literal>) and a full + memory (RAMDirectoryProvider) implementation. Hibernate Search has the + notion of <literal>DirectoryProvider</literal> that handles the configuration and the initialization of the Lucene Directory.</para> <table> @@ -20,7 +20,7 @@ <thead> <row> <entry align="center">Class</entry> - + <entry align="center">Description</entry> <entry align="center">Properties</entry> @@ -35,20 +35,12 @@ &lt;indexBase&gt;/&lt; <literal&gt;(a)Indexed.name&lt;/literal&gt; &gt;</entry> - <entry><literal>indexBase</literal> : Base directory</entry> + <entry><para><literal>indexBase</literal> : Base + directory</para><para><literal>indexName</literal>: override + @Index.name (useful for sharded indexes)</para></entry> </row> <row> - <entry>org.hibernate.search.store.RAMDirectoryProvider</entry> - - <entry>Memory based directory, the directory will be uniquely - indentified by the <literal&gt;(a)Indexed.name&lt;/literal&gt; - element</entry> - - <entry>none</entry> - </row> - - <row> <entry>org.hibernate.search.store.FSMasterDirectoryProvider</entry> <entry><para>File system based directory. Like @@ -63,7 +55,9 @@ used on slave nodes using a JMS back end.</entry> <entry><para><literal>indexBase</literal>: Base - directory</para><para><literal>sourceBase</literal>: Source (copy) + directory</para><para><literal>indexName</literal>: override + @Index.name (useful for sharded + indexes)</para><para><literal>sourceBase</literal>: Source (copy) base directory.</para><para><literal>source</literal>: Source directory suffix (default to <literal&gt;(a)Indexed.name&lt;/literal>). The actual source directory name being @@ -86,7 +80,9 @@ used on slave nodes using a JMS back end.</para></entry> <entry><para><literal>indexBase</literal>: Base - directory</para><para><literal>sourceBase</literal>: Source (copy) + directory</para><para><literal>indexName</literal>: override + @Index.name (useful for sharded + indexes)</para><para><literal>sourceBase</literal>: Source (copy) base directory.</para><para><literal>source</literal>: Source directory suffix (default to <literal&gt;(a)Indexed.name&lt;/literal>). The actual source directory name being @@ -147,8 +143,87 @@ and base directory, and overide those default later on on a per index basis.</para> - <para>Writing your own <classname>DirectoryProvider</classname> , you can + <para>Writing your own <classname>DirectoryProvider</classname>, you can benefit this configuration mechanism too.</para> + + <section id="search-configuration-directory-sharding" revision="1"> + <title>Index sharding</title> + + <para>In some extreme cases involving huge indexes (in size), it is + necessary to split (shard) the indexing data of a given entity type into + several Lucene indexes. This solution is not recommended until you reach + significant index sizes and important indexing slows down, the main + drawback being that search will end up being slower (more files to + open). In other words don't do it until you have problems :)</para> + + <para>Despite this strong warning, Hibernate Search allows you to index + a given entity type into several sub indexes. Data is sharded into the + different sub indexes thanks to an + <classname>IndexShardingStrategy</classname>. By default, no sharding + strategy is enabled, unless the number of shards is configured. To + configure the number of shards use the following property</para> + + <programlisting>hibernate.search.&lt;indexName&gt;.sharding_strategy.nbr_of_shards 5</programlisting> + + <para>This will use 5 different shards.</para> + + <para>The default sharding strategy, when shards are set up, splits the + data according to the hash value of the id string representation + (generated by the Field Bridge). This ensures a fairly balanced + sharding. You can replace the strategy by implementing + <literal>IndexShardingStrategy</literal> and by setting the following + property</para> + + <programlisting>hibernate.search.&lt;indexName&gt;.sharding_strategy my.shardingstrategy.Implementation</programlisting> + + <para>Each shard has an independent directory provider configuration as + described in <xref linkend="search-configuration-directory" />. The + DirectoryProvider default name for the previous example are + <literal>&lt;indexName&gt;.0</literal> to + <literal>&lt;indexName&gt;.4</literal>. In other words, each shard has + the name of it's owning index followed by <constant>.</constant> (dot) + and its index number.</para> + + <programlisting>hibernate.search.default.indexBase /usr/lucene/indexes + +hibernate.search.Animal.sharding_strategy.nbr_of_shards 5 +hibernate.search.Animal.directory_provider org.hibernate.search.store.FSDirectoryProvider +hibernate.search.Animal.0.indexName Animal00 +hibernate.search.Animal.3.indexBase /usr/lucene/sharded +hibernate.search.Animal.3.indexName Animal03</programlisting> + + <para>This configuration uses the default id string hashing strategy and + shards the Animal index intp 5 subindexes. All subindexes are + FSDirectoryProvider instances and the directory where each subindex is + stored is as followed:</para> + + <itemizedlist> + <listitem> + <para>for subindex 0: /usr/lucene/indexes/Animal00 (shared indexBase + but overridden indexName)</para> + </listitem> + + <listitem> + <para>for subindex 1: /usr/lucene/indexes/Animal.1 (shared + indexBase, default indexName)</para> + </listitem> + + <listitem> + <para>for subindex 2: /usr/lucene/indexes/Animal.2 (shared + indexBase, default indexName)</para> + </listitem> + + <listitem> + <para>for subindex 3: /usr/lucene/shared/Animal03 (overridden + indexBase, overridden indexName)</para> + </listitem> + + <listitem> + <para>for subindex 4: /usr/lucene/indexes/Animal.4 (shared + indexBase, default indexName)</para> + </listitem> + </itemizedlist> + </section> </section> <section> @@ -368,11 +443,11 @@ <para>The different reader strategies are described in <xref linkend="search-architecture-readerstrategy" /> .The default reader - strategy is <literal>not-shared</literal>. This can be adjusted:</para> + strategy is <literal>shared</literal>. This can be adjusted:</para> - <programlisting>hibernate.search.reader.strategy = shared</programlisting> + <programlisting>hibernate.search.reader.strategy = not-shared</programlisting> - <para>Adding this property switch to the <literal>shared</literal> + <para>Adding this property switch to the <literal>non shared</literal> strategy.</para> <para>Or if you have a custom reader strategy:</para> @@ -430,87 +505,166 @@ </listitem> </itemizedlist> </section> - + <section id="lucene-indexing-performance" revision="1"> - <title>Tuning Lucene Indexing Performance</title> + <title>Tuning Lucene indexing performance</title> - <para>Hibernate Search allows you to tune the Lucene indexing performance by specifying - a set of parameters which are passed through to underlying Lucene - <literal>IndexWriter</literal> as <literal>mergeFactor</literal>, <literal>maxMergeDocs</literal> - and <literal>maxBufferedDocs</literal>. You can specify these paramters either as default values - applying for all indeces or on a per index base. - </para> - <para>There are two sets of parameters - allowing for different performance settings depending on the use case. During indexing operations - triggered by database modifications - <literal>hibernate.search.[default|&lt;indexname&gt;].merge_factor</literal>, <literal>hibernate.search.[default|&lt;indexname&gt;].max_merge_docs</literal> - and <literal>hibernate.search.[default|&lt;indexname&gt;].max_buffered_docs</literal> are used. The corresponding - <literal>.batch</literal> properties are applied when indexing occurs via - <literal>FullTextSession.index()</literal> (see <xref linkend="search-batchindex" />). - </para> - <para> - Unless the corresponding <literal>.batch</literal> property is set explicitly it will have the - same value as the non batch parameter. - </para> - <para> - For more information about Lucene indexing performace please refer to the Lucene documentation. - </para> - - <tgroup cols="4"> - <thead> - <row> - <entry align="center">Property</entry> - <entry align="center">Description</entry> - <entry align="center">Default Value</entry> - </row> - </thead> + <para>Hibernate Search allows you to tune the Lucene indexing performance + by specifying a set of parameters which are passed through to underlying + Lucene <literal>IndexWriter</literal> as <literal>mergeFactor</literal>, + <literal>maxMergeDocs</literal> and <literal>maxBufferedDocs</literal>. + You can specify these parameters either as default values applying for all + indexes or on a per index basis.</para> - <tbody> - <row> - <entry><literal>hibernate.search.[default|&lt;indexname&gt;].merge_factor</literal></entry> - <entry>Controls segement merge frequency ans size. Used by Hibernate Search during - index update operations as part of database modifications. - </entry> - <entry>10</entry> - </row> - - <row> - <entry><literal>hibernate.search.[default|&lt;indexname&gt;].max_merge_docs</literal></entry> - <entry>Limits the number of documents per segment. Used by Hibernate Search during - index update operations as part of database modifications.</entry> - <entry>Integer.MAX_VALUE</entry> - </row> - - <row> - <entry><literal>hibernate.search.[default|&lt;indexname&gt;].max_buffered_docs</literal></entry> - <entry>Controls the amount of documents buffered during indexing. Used by Hibernate - Search during index update operations as part of database modifications. - </entry> - <entry>10</entry> - </row> - - <row> - <entry><literal>hibernate.search.[default|&lt;indexname&gt;].merge_factor.batch</literal></entry> - <entry>Controls segement merge frequency ans size.Used during indexing via - <literal>FullTextSession.index()</literal></entry> - <entry>10</entry> - </row> - - <row> - <entry><literal>hibernate.search.[default|&lt;indexname&gt;].max_merge_docs.batch</literal></entry> - <entry>Limits the number of documents per segment. Used during indexing via - <literal>FullTextSession.index()</literal></entry> - <entry>Integer.MAX_VALUE</entry> - </row> - - <row> - <entry><literal>hibernate.search.[default|&lt;indexname&gt;].max_buffered_docs.batch</literal></entry> - <entry>Controls the amount of documents buffered during indexing.Used during indexing via - <literal>FullTextSession.index()</literal></entry> - <entry>10</entry> - </row> - </tbody> + <para>There are two sets of parameters allowing for different performance + settings depending on the use case. During indexing operations triggered + by database modifications + <literal>hibernate.search.[default|&lt;indexname&gt;].transaction.merge_factor</literal>, + <literal>hibernate.search.[default|&lt;indexname&gt;].transaction.max_merge_docs</literal> + and + <literal>hibernate.search.[default|&lt;indexname&gt;].transaction.max_buffered_docs</literal> + are used. The corresponding + <literal>hibernate.search.[default|&lt;indexname&gt;].batch.merge_factor</literal>, + <literal>hibernate.search.[default|&lt;indexname&gt;].batch.max_merge_docs</literal> + and + <literal>hibernate.search.[default|&lt;indexname&gt;].batch.max_buffered_docs</literal> + properties are applied when indexing occurs via + <literal>FullTextSession.index()</literal> (see <xref + linkend="search-batchindex" />).</para> + + <para>Unless the corresponding <literal>.batch</literal> property is + explicitly set, the value will default to the + <literal>.transaction</literal> property.</para> + + <para>For more information about Lucene indexing performances, please + refer to the Lucene documentation.</para> + + <tgroup cols="3"> + <thead> + <row> + <entry align="center">Property</entry> + + <entry align="center">Description</entry> + + <entry align="center">Default Value</entry> + </row> + </thead> + + <tbody> + <row> + <entry> + <literal>hibernate.search.[default|&lt;indexname&gt;].transaction.merge_factor</literal> + </entry> + + <entry> + <para>Controls segment merge frequency and size. </para> + + <para>Determines how often segment indices are merged when + insertion occurs. With smaller values, less RAM is used while + indexing, and searches on unoptimized indices are faster, but + indexing speed is slower. With larger values, more RAM is used + during indexing, and while searches on unoptimized indices are + slower, indexing is faster. Thus larger values (&gt; 10) are best + for batch index creation, and smaller values (&lt; 10) for indices + that are interactively maintained. The value must no be lower than + 2.</para> + + <para>Used by Hibernate Search during index update operations as + part of database modifications.</para> + </entry> + + <entry>10</entry> + </row> + + <row> + <entry> + <literal>hibernate.search.[default|&lt;indexname&gt;].transaction.max_merge_docs</literal> + </entry> + + <entry> + <para>Defines the largest number of documents allowed in a + segment.</para> + + <para>Used by Hibernate Search during index update operations as + part of database modifications.</para> + </entry> + + <entry>Unlimited (Integer.MAX_VALUE)</entry> + </row> + + <row> + <entry> + <literal>hibernate.search.[default|&lt;indexname&gt;].transaction.max_buffered_docs</literal> + </entry> + + <entry> + <para>Controls the amount of documents buffered in memory during + indexing. The bigger the more RAM is consumed.</para> + + <para>Used by Hibernate Search during index update operations as + part of database modifications.</para> + </entry> + + <entry>10</entry> + </row> + + <row> + <entry> + <literal>hibernate.search.[default|&lt;indexname&gt;].batch.merge_factor</literal> + </entry> + + <entry> + <para>Controls segment merge frequency and size.</para> + + <para>Determines how often segment indices are merged when + insertion occurs. With smaller values, less RAM is used while + indexing, and searches on unoptimized indices are faster, but + indexing speed is slower. With larger values, more RAM is used + during indexing, and while searches on unoptimized indices are + slower, indexing is faster. Thus larger values (&gt; 10) are best + for batch index creation, and smaller values (&lt; 10) for indices + that are interactively maintained. The value must no be lower than + 2.</para> + + <para>Used during indexing via + <literal>FullTextSession.index()</literal></para> + </entry> + + <entry>10</entry> + </row> + + <row> + <entry> + <literal>hibernate.search.[default|&lt;indexname&gt;].batch.max_merge_docs</literal> + </entry> + + <entry> + <para>Defines the largest number of documents allowed in a + segment.</para> + + <para>Used during indexing via + <literal>FullTextSession.index()</literal></para> + </entry> + + <entry>Unlimited (Integer.MAX_VALUE)</entry> + </row> + + <row> + <entry> + <literal>hibernate.search.[default|&lt;indexname&gt;].batch.max_buffered_docs</literal> + </entry> + + <entry> + <para>Controls the amount of documents buffered in memory during + indexing. The bigger the more RAM is consumed.</para> + + <para>Used during indexing via + <literal>FullTextSession.index()</literal></para> + </entry> + + <entry>10</entry> + </row> + </tbody> </tgroup> </section> - </chapter> \ No newline at end of file Modified: trunk/HibernateExt/search/doc/reference/en/modules/lucene-native.xml =================================================================== --- trunk/HibernateExt/search/doc/reference/en/modules/lucene-native.xml 2007-07-31 17:54:39 UTC (rev 12869) +++ trunk/HibernateExt/search/doc/reference/en/modules/lucene-native.xml 2007-07-31 18:48:40 UTC (rev 12870) @@ -1,4 +1,4 @@ -<?xml version="1.0" encoding="UTF-8"?> +<?xml version="1.0" encoding="UTF-8"?> <!-- $Id$ --> <chapter id="search-lucene-native"> <title>Accessing Lucene natively</title> @@ -21,13 +21,16 @@ <para>You can always access the Lucene directories through plain Lucene, the Directory structure is in no way different with or without Hibernate Search. However there are some more convenient ways to access a given - Directory. The <classname>SearchFactory</classname> keeps track of a - <classname>DirectoryProvider</classname> per indexed class (one directory + Directory. The <classname>SearchFactory</classname> keeps track of the + <classname>DirectoryProvider</classname>s per indexed class. One directory provider can be shared amongst several indexed classes if the classes - share the same underlying index directory).</para> + share the same underlying index directory. While usually not the case, a + given entity can have several <classname>DirectoryProvider</classname>s is + the index is sharded (see <xref + linkend="search-configuration-directory-sharding" />).</para> - <programlisting>DirectoryProvider provider = searchFactory.getDirectoryProvider(Order.class); -org.apache.lucene.store.Directory directory = provider.getDirectory();</programlisting> + <programlisting>DirectoryProvider[] provider = searchFactory.getDirectoryProviders(Order.class); +org.apache.lucene.store.Directory directory = provider[0].getDirectory();</programlisting> <para>In this example, directory points to the lucene index storing <classname>Order</classname>s information. Note that the obtained Lucene @@ -43,8 +46,8 @@ code can access such cached / shared resources. You will just have to follow some "good citizen" rules.</para> - <programlisting>DirectoryProvider orderProvider = searchFactory.getDirectoryProvider(Order.class); -DirectoryProvider clientProvider = searchFactory.getDirectoryProvider(Client.class); + <programlisting>DirectoryProvider orderProvider = searchFactory.getDirectoryProviders(Order.class)[0]; +DirectoryProvider clientProvider = searchFactory.getDirectoryProviders(Client.class)[0]; ReaderProvider readerProvider = searchFactory.getReaderProvider(); IndexReader reader = readerProvider.openReader(orderProvider, clientProvider); Modified: trunk/HibernateExt/search/doc/reference/en/modules/mapping.xml =================================================================== --- trunk/HibernateExt/search/doc/reference/en/modules/mapping.xml 2007-07-31 17:54:39 UTC (rev 12869) +++ trunk/HibernateExt/search/doc/reference/en/modules/mapping.xml 2007-07-31 18:48:40 UTC (rev 12870) @@ -123,6 +123,51 @@ </section> <section> + <title>mapping a property multiple time</title> + + <para>It is sometimes needed to map a proeprty multiple times per index, + with slightly different indexing strategies. Especially, sorting a query + by field requires the field to be <literal>UN_TOKENIZED</literal>. If + one want to search by words in this proeprty and still sort it, one need + to index it twice, once tokenized, once untokenized. @Fields allows to + achieve this goal.</para> + + <programlisting>@Entity +@Indexed(index = "Book" ) +public class Book { + @Fields( { + @Field(index = Index.TOKENIZED), + @Field(name = "summary_forSort", index = Index.UN_TOKENIZED, store = Store.YES) + } ) + public String getSummary() { + return summary; + } + + ... +}</programlisting> + + <para>The field summary is indexed twice, once as + <literal>summary</literal> in a tokenized way, and once as + <literal>summary_forSort</literal> in an untokenized way. @Field + supports 2 attributes useful when @Fields is used:</para> + + <itemizedlist> + <listitem> + <para>analyzer: defines a @Analyzer annotation per field rather than + per property</para> + </listitem> + + <listitem> + <para>bridge" defines a @FieldBridge annotation per field rather + than per property</para> + </listitem> + </itemizedlist> + + <para>See below for more information about analyzers and field + bridges.</para> + </section> + + <section> <title>Embedded and associated objects</title> <para>Associated objects as well as embedded objects can be indexed as @@ -358,11 +403,52 @@ <section> <title>Analyser</title> - <para>The analyzer class used to index the elements is configurable - through the <literal>hibernate.search.analyzer</literal> property. If - none defined, + <para>The default analyzer class used to index the elements is + configurable through the <literal>hibernate.search.analyzer</literal> + property. If none defined, <classname>org.apache.lucene.analysis.standard.StandardAnalyzer</classname> is used as the default.</para> + + <para>You can also define the analyzer class per entity, per property + and even per @Field (useful when multiple fields are indexed from a + single proeprty).</para> + + <programlisting>@Entity +@Indexed +@Analyzer(impl = EntityAnalyzer.class) +public class MyEntity { + @Id + @GeneratedValue + @DocumentId + private Integer id; + + @Field(index = Index.TOKENIZED) + private String name; + + @Field(index = Index.TOKENIZED) + @Analyzer(impl = PropertyAnalyzer.class) + private String summary; + + @Field(index = Index.TOKENIZED, analyzer = @Analyzer(impl = FieldAnalyzer.class) + private String body; + + ... +}</programlisting> + + <para>In this example, <classname>EntityAnalyzer</classname> is used + index all tokenized properties (eg. <literal>name</literal>), except for + <literal>summary</literal> and <literal>body</literal> which are indexed + with <classname>PropertyAnalyzer</classname> and + <classname>FieldAnalyzer</classname> respectively.</para> + + <caution> + <para>Mixing different analyzers in the same entity is most of the + time a bad practice. It makes query building more complex and results + less predictable (for the novice), especially if you are using a + QueryParser (which uses the same analyzer for the whole query). As a + thumb rule, the same analyzer should be used for both the indexing and + the query for a given field.</para> + </caution> </section> </section> Modified: trunk/HibernateExt/search/doc/reference/en/modules/optimize.xml =================================================================== --- trunk/HibernateExt/search/doc/reference/en/modules/optimize.xml 2007-07-31 17:54:39 UTC (rev 12869) +++ trunk/HibernateExt/search/doc/reference/en/modules/optimize.xml 2007-07-31 18:48:40 UTC (rev 12870) @@ -1,25 +1,14 @@ -<?xml version="1.0" encoding="UTF-8"?> +<?xml version="1.0" encoding="UTF-8"?> <!-- $Id$ --> <chapter id="search-optimize"> <title>Index Optimization</title> - <para>From time to time, the Lucene index needs to be optimize. The process + <para>From time to time, the Lucene index needs to be optimized. The process is essentially a defragmentation: until the optimization occurs, deleted documents are just marked as such, no physical deletion is applied, the optimization can also adjust the number of files in the Lucene Directory.</para> - <para>You can programmatically optimize (defragment) a Lucene index from - Hibernate Search through the <classname>SearchFactory</classname></para> - - <programlisting>searchFactory.optimize(Order.class); - -searchFactory.optimize(); -</programlisting> - - <para>The first example reindex the Lucene index holding - <classname>Order</classname>s, the second, optimize all indexes.</para> - <para>The optimization speeds up searches but in no way speeds up indexation (update). During an optimization, searches can be performed (but will most likely be slowed down), and all index updates will be stopped. Prefer @@ -36,18 +25,86 @@ </listitem> </itemizedlist> - <para>The <classname>SearchFactory</classname> can be accessed from a - <classname>FullTextSession</classname>:</para> + <section> + <title>Automatic optimization</title> - <programlisting>FullTextSession fullTextSession = Search.createFullTextSession(regularSession); + <para>Hibernate Search can optimize automatically an index after:</para> + + <itemizedlist> + <listitem> + <para>a certain amount of operations have been applied (insertion, + deletion)</para> + </listitem> + + <listitem> + <para>or a certain amout of transactions have been applied</para> + </listitem> + </itemizedlist> + + <para>The configuration can be global or defined at the index + level:</para> + + <programlisting>hibernate.search.default.optimizer.operation_limit.max = 1000 +hibernate.search.default.optimizer.transaction_limit.max = 100 + +hibernate.search.Animal.optimizer.transaction_limit.max = 50</programlisting> + + <para>An optimization will be triggered to the <literal>Animal</literal> + index as soon as either:</para> + + <itemizedlist> + <listitem> + <para>the number of addition and deletion reaches 1000</para> + </listitem> + + <listitem> + <para>the number of transactions reaches 50 + (<constant>hibernate.search.Animal.optimizer.transaction_limit.max</constant> + having priority over + <constant>hibernate.search.default.optimizer.transaction_limit.max</constant>)</para> + </listitem> + </itemizedlist> + + <para>If none of these parameters are defined, not optimization is + processed automatically.</para> + </section> + + <section> + <title>Manual optimization</title> + + <para>You can programmatically optimize (defragment) a Lucene index from + Hibernate Search through the <classname>SearchFactory</classname></para> + + <programlisting>searchFactory.optimize(Order.class); + +searchFactory.optimize();</programlisting> + + <para>The first example reindex the Lucene index holding + <classname>Order</classname>s, the second, optimize all indexes.</para> + + <para>The <classname>SearchFactory</classname> can be accessed from a + <classname>FullTextSession</classname>:</para> + + <programlisting>FullTextSession fullTextSession = Search.createFullTextSession(regularSession); SearchFactory searchFactory = fullTextSession.getSearchFactory();</programlisting> - <para>Note that <literal>searchFactory.optimize()</literal> has no effect on - a JMS backend. You must apply the optimize operation on the Master - node.</para> + <para>Note that <literal>searchFactory.optimize()</literal> has no effect + on a JMS backend. You must apply the optimize operation on the Master + node.</para> + </section> - <para>Further index optimisation parameters include <literal>hibernate.search.[default|&lt;indexname&gt;].merge_factor</literal>, - <literal>hibernate.search.[default|&lt;indexname&gt;].max_merge_docs</literal> and <literal>hibernate.search.[default|&lt;indexname&gt;].max_buffered_docs</literal> - - see <xref linkend="lucene-indexing-performance"/> for more details. - </para> + <section> + <title>Adjusting optimization</title> + + <para>Apache Lucene has a few parameters to influence how optimization is + performed. Hibernate Search expose those parameters.</para> + + <para>Further index optimisation parameters include + <literal>hibernate.search.[default|&lt;indexname&gt;].merge_factor</literal>, + <literal>hibernate.search.[default|&lt;indexname&gt;].max_merge_docs</literal> + and + <literal>hibernate.search.[default|&lt;indexname&gt;].max_buffered_docs</literal> + - see <xref linkend="lucene-indexing-performance" /> for more + details.</para> + </section> </chapter> \ No newline at end of file Modified: trunk/HibernateExt/search/doc/reference/en/modules/query.xml =================================================================== --- trunk/HibernateExt/search/doc/reference/en/modules/query.xml 2007-07-31 17:54:39 UTC (rev 12869) +++ trunk/HibernateExt/search/doc/reference/en/modules/query.xml 2007-07-31 18:48:40 UTC (rev 12870) @@ -36,6 +36,26 @@ <literal>iterate()</literal> and <literal>scroll()</literal> can be used.</para> + <para>For people using Java Persistence (aka EJB 3.0 Persistence) APIs of + Hibernate, the same extensions exist:</para> + + <programlisting>EntityManager em = entityManagerFactory.createEntityManager(); + +FullTextEntityManager fullTextEntityManager = + org.hibernate.hibernate.search.jpa.Search.createFullTextEntityManager(em); + +... +org.apache.lucene.queryParser.QueryParser parser = new QueryParser("title", new StopAnalyzer() ); + +org.apache.lucene.search.Query luceneQuery = parser.parse( "summary:Festina Or brand:Seiko" ); +<emphasis role="bold">javax.persistence.Query fullTextQuery = fullTextEntityManager.createFullTextQuery( luceneQuery );</emphasis> + +List result = fullTextQuery.getResultList(); //return a list of managed objects </programlisting> + + <para>The following examples show the Hibernate APIs but the same example + can be easily rewritten with the Java Persistence API by just adjusting the + way the FullTextQuery is retrieved.</para> + <section> <title>Building queries</title> @@ -188,7 +208,52 @@ </listitem> </itemizedlist> - <para></para> + <para>Projection is useful for another kind of usecases. Lucene + provides some metadata informations to the user about the results. By + using some special placeholders, the projection mechanism can retrieve + them:</para> + + <programlisting>org.hibernate.search.FullTextQuery query = s.createFullTextQuery( luceneQuery, Book.class ); +query.<emphasis role="bold">setProjection( FullTextQuery.SCORE, FullTextQuery.BOOST, FullTextQuery.THIS, "mainAuthor.name" )</emphasis>; +List results = query.list(); +Object[] firstResult = (Object[]) results.get(0); +float score = firstResult[0]; +float boost = firstResult[1]; +Book book = firstResult[2]; +String authorName = firstResult[3];</programlisting> + + <para>You can mix and match regular fields and special placeholders. + Here is the list of available placeholders:</para> + + <itemizedlist> + <listitem> + <para>FullTextQuery.THIS: returns the intialized and managed + entity (as a non projected query would have done)</para> + </listitem> + + <listitem> + <para>FullTextQuery.DOCUMENT: returns the Lucene Document related + to the object projected</para> + </listitem> + + <listitem> + <para>FullTextQuery.SCORE: returns the document score in the + query. The score is guatanteed to be between 0 and 1 but the + highest score is not necessarily equals to 1. Scores are handy to + compare one result against an other for a given query but are + useless when comparing the result of different queries.</para> + </listitem> + + <listitem> + <para>FullTextQuery.BOOST: the boost value of the Lucene + Document</para> + </listitem> + + <listitem> + <para>FullTextQuery.ID: the id property value of the projected + object</para> + </listitem> + </itemizedlist> </section> </section> </section> @@ -268,6 +333,199 @@ </section> <section> + <title>Filters</title> + + <para>Apache Lucene has a powerful feature that allows to filters results + from a query according to a custom filtering process. This is a very + powerful way to apply some data restrictions after a query, especially + since filters can be cached and reused. Some interesting usecase + are:</para> + + <itemizedlist> + <listitem> + <para>security</para> + </listitem> + + <listitem> + <para>temporal data (eg. view only last month's data)</para> + </listitem> + + <listitem> + <para>population filter (eg. search limited to a given + category)</para> + </listitem> + + <listitem> + <para>and many more</para> + </listitem> + </itemizedlist> + + <para>Hibernate Search pushes the concept further by introducing the + notion of parameterizable named filters which are transparantly cached. + For people familiar with the notion of Hibernate Core filters, the API is + very similar.</para> + + <programlisting>fullTextQuery = s.createFullTextQuery( query, Driver.class ); +fullTextQuery.enableFullTextFilter("bestDriver"); +fullTextQuery.enableFullTextFilter("security").setParameter( "login", "andre" ); +fullTextQuery.list(); //returns only best drivers where andre has credentials</programlisting> + + <para>In this example we enabled 2 filters on top of this query. You can + enable (or disable) as many filters as you want.</para> + + <para>Declaring filters is done through the + <classname>@FullTextFilterDef</classname> annotation. This annotation can + be on any <literal>@Indexed</literal> entity regardless of the filter + operation.</para> + + <programlisting>@Entity +@Indexed +@FullTextFilterDefs( { + <emphasis role="bold">@FullTextFilterDef(name = "bestDriver", impl = BestDriversFilter.class, cache=false)</emphasis>, //actual Filter implementation + <emphasis role="bold">@FullTextFilterDef(name = "security", impl = SecurityFilterFactory.class)</emphasis> //Filter factory with parameters +}) +public class Driver { ... }</programlisting> + + <para>Each named filter points to an actual filter implementation.</para> + + <programlisting>public class BestDriversFilter extends <emphasis + role="bold">org.apache.lucene.search.Filter</emphasis> { + + public BitSet bits(IndexReader reader) throws IOException { + BitSet bitSet = new BitSet( reader.maxDoc() ); + TermDocs termDocs = reader.termDocs( new Term("score", "5") ); + while ( termDocs.next() ) { + bitSet.set( termDocs.doc() ); + } + return bitSet; + } +}</programlisting> + + <para><classname>BestDriversFilter</classname> is an example of a simple + Lucene filter that will filter all results to only return drivers whose + score is 5. The filters must have a no-arg constructor when referenced in + a <literal>FulltextFilterDef.impl</literal>.</para> + + <para>The cache flag defaulted to true, tells Hibernate Search to search + the filter in its internal cache and reuse it if found.</para> + + <para>Note that, usually, filter using the + <classname>IndexReader</classname> are usually wrapped in a Lucene + <classname>CachingWrapperFilter</classname> to benefit from some caching + speed improvement. If your Filter creation requires additional steps or if + the filter you are willing to use does not have a no-arg constructor, you + can use the factory pattern:</para> + + <programlisting>@Entity +@Indexed +@FullTextFilterDef(name = "bestDriver", impl = BestDriversFilterFactory.class) //Filter factory +public class Driver { ... } + +public class BestDriversFilterFactory { + + <emphasis role="bold">@Factory</emphasis> + public Filter getFilter() { + //some additional steps to cache the filter results per IndexReader + Filter bestDriversFilter = new BestDriversFilter(); + return new CachingWrapperFilter(bestDriversFilter); + } +}</programlisting> + + <para>Hibernate Search will look for a <literal>@Factory</literal> + annotated method and use it to build the filter instance. The factory must + have a no-arg constructor. For people familiar with JBoss Seam, this is + similar to the component factory pattern, but the annotation is + different!</para> + + <para>Named filters comes in handy where the filters have parameters. For + example a security filter needs to know which credentials you are willing + to filter by:</para> + + <programlisting>fullTextQuery = s.createFullTextQuery( query, Driver.class ); +fullTextQuery.enableFullTextFilter("security")<emphasis role="bold">.setParameter( "level", 5 )</emphasis>;</programlisting> + + <para>Each parameter name should have an associated setter on either the + filter or filter factory of the targeted named filter definition. </para> + + <programlisting>public class SecurityFilterFactory { + private Integer level; + + /** + * injected parameter + */ + <emphasis role="bold">public void setLevel(Integer level)</emphasis> { + this.level = level; + } + + <emphasis role="bold">@Key + public FilterKey getKey()</emphasis> { + StandardFilterKey key = new StandardFilterKey(); + key.addParameter( level ); + return key; + } + + @Factory + public Filter getFilter() { + Query query = new TermQuery( new Term("level", level.toString() ) ); + return new CachingWrapperFilter( new QueryWrapperFilter(query) ); + } +}</programlisting> + + <para>Note the method annotated <classname>@Key</classname> and returning + a <classname>FilterKey</classname> object. The returned object has a + special contract: the key object must implement equals / hashcode so that + 2 keys are equals if and only if the given Filter types are the same and + the set of parameters are the same. In other words, 2 filter keys are + equal if and only if the filters from which the keys are generated can be + interchanged. The key object is used as a key in the cache + mechanism.</para> + + <para><classname>@Key</classname> methods are needed only if:</para> + + <itemizedlist> + <listitem> + <para>you enabled the filter caching system (enabled by + default)</para> + </listitem> + + <listitem> + <para>your filter has parameters</para> + </listitem> + </itemizedlist> + + <para>In most cases, using the <literal>StandardFilterKey</literal> + implementation will be good enough. It delegates the equals/hashcode + implementation to each of the parameters equals and hashcode + methods.</para> + + <para>Should filters be cached? There are 2 area where filter caching + shines:</para> + + <itemizedlist> + <listitem> + <para>the system does not update the targeted entity index often (in + other words, the IndexReader is reused a lot)</para> + </listitem> + + <listitem> + <para>the Filter BitSet is expenseve to consume (compared to the time + spent to execute the query)</para> + </listitem> + </itemizedlist> + + <para>Cache is enabled by default and use the notion of SoftReferences to + dispose memory when needed. To adjust the size of the hard reference + cache, use <literal>hibernate.search.filter.cache_strategy.size</literal> + (defaults to 128). Don't forget to use a + <classname>CachingWrapperFilter</classname> when the filter is cacheable + and the Filter's bits methods makes use of IndexReader.</para> + + <para>For advance use of filter caching, you can implement your own + <classname>FilterCachingStrategy</classname>. The classname is defined by + <literal>hibernate.search.filter.cache_strategy</literal>.</para> + </section> + + <section> <title>Optimizing the query process</title> <para>Query performance depends on several criteria:</para> Modified: trunk/HibernateExt/search/readme.txt =================================================================== --- trunk/HibernateExt/search/readme.txt 2007-07-31 17:54:39 UTC (rev 12869) +++ trunk/HibernateExt/search/readme.txt 2007-07-31 18:48:40 UTC (rev 12870) @@ -1,6 +1,6 @@ Hibernate Search ================================================== -Version: 3.0.0.Beta3, 6.06.2007 +Version: 3.0.0.Beta4, 1.08.2007 Description ----------- Modified: trunk/HibernateExt/search/src/java/org/hibernate/search/Version.java =================================================================== --- trunk/HibernateExt/search/src/java/org/hibernate/search/Version.java 2007-07-31 17:54:39 UTC (rev 12869) +++ trunk/HibernateExt/search/src/java/org/hibernate/search/Version.java 2007-07-31 18:48:40 UTC (rev 12870) @@ -10,7 +10,7 @@ * @author Emmanuel Bernard */ public class Version { - public static final String VERSION = "3.0.0.Beta4SNAPSHOT" + new Date(); + public static final String VERSION = "3.0.0.Beta4"; // + new Date(); private static Log log = LogFactory.getLog( Version.class ); static {