[hibernate-commits] Hibernate SVN: r19187 - in search/trunk/hibernate-search/src: main/java/org/hibernate/search/bridge and 6 other directories.

Author: sannegrinovero Date: 2010-04-08 05:50:52 -0400 (Thu, 08 Apr 2010) New Revision: 19187 Added: search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/compression/ search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/compression/CompressionTest.java search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/compression/HTMLBoldFieldBridge.java search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/compression/LargeDocument.java Modified: search/trunk/hibernate-search/src/main/docbook/en-US/modules/mapping.xml search/trunk/hibernate-search/src/main/java/org/hibernate/search/bridge/LuceneOptions.java search/trunk/hibernate-search/src/main/java/org/hibernate/search/bridge/String2FieldBridgeAdaptor.java search/trunk/hibernate-search/src/main/java/org/hibernate/search/bridge/TwoWayString2FieldBridgeAdaptor.java search/trunk/hibernate-search/src/main/java/org/hibernate/search/engine/DocumentBuilderContainedEntity.java search/trunk/hibernate-search/src/main/java/org/hibernate/search/engine/DocumentBuilderIndexedEntity.java search/trunk/hibernate-search/src/main/java/org/hibernate/search/engine/LuceneOptionsImpl.java search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/bridge/CatDeptsFieldsClassBridge.java search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/bridge/CatFieldsClassBridge.java search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/bridge/DateSplitBridge.java search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/bridge/EquipmentType.java search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/bridge/TruncateFieldBridge.java search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/configuration/CatDeptsFieldsClassBridge.java search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/configuration/EquipmentType.java search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/id/PersonPKBridge.java Log: HSEARCH-425 Reimplement support for compressed fields (support removed in Lucene3) Modified: search/trunk/hibernate-search/src/main/docbook/en-US/modules/mapping.xml =================================================================== --- search/trunk/hibernate-search/src/main/docbook/en-US/modules/mapping.xml 2010-04-08 09:09:22 UTC (rev 19186) +++ search/trunk/hibernate-search/src/main/docbook/en-US/modules/mapping.xml 2010-04-08 09:50:52 UTC (rev 19187) @@ -1340,7 +1340,7 @@ to use a slightly extended version of <literal>StringBridge</literal> named <classname>TwoWayStringBridge</classname>. Hibernate Search needs to read the string representation of the identifier and generate - the object out of it. There is not difference in the way the + the object out of it. There is no difference in the way the <literal>@FieldBridge</literal> annotation is used.</para> <example> @@ -1395,8 +1395,8 @@ greatest possible flexibility you can also implement a bridge as a <classname>FieldBridge</classname>. This interface gives you a property value and let you map it the way you want in your Lucene - <classname>Document</classname>.The interface is very similar in its - concept to the Hibernate<classname> UserType</classname>s.</para> + <classname>Document</classname>. The interface is very similar in its + concept to the Hibernate <classname>UserType</classname>s.</para> <para>You can for example store a given property in two different document fields:</para> @@ -1408,7 +1408,6 @@ <programlisting>/** * Store the date in 3 different fields - year, month, day - to ease Range Query per * year, month or day (eg get all the elements of December for the last 5 years). - * * @author Emmanuel Bernard */ public class DateSplitBridge implements FieldBridge { @@ -1424,25 +1423,22 @@ int day = cal.get(Calendar.DAY_OF_MONTH); // set year - Field field = new Field(name + ".year", String.valueOf(year), - luceneOptions.getStore(), luceneOptions.getIndex(), - luceneOptions.getTermVector()); - field.setBoost(luceneOptions.getBoost()); - document.add(field); + luceneOptions.addFieldToDocument( + name + ".year", + String.valueOf( year ), + document ); // set month and pad it if needed - field = new Field(name + ".month", month &lt; 10 ? "0" : "" - + String.valueOf(month), luceneOptions.getStore(), - luceneOptions.getIndex(), luceneOptions.getTermVector()); - field.setBoost(luceneOptions.getBoost()); - document.add(field); + luceneOptions.addFieldToDocument( + name + ".month", + month &lt; 10 ? "0" : "" + String.valueOf( month ), + document ); // set day and pad it if needed - field = new Field(name + ".day", day &lt; 10 ? "0" : "" - + String.valueOf(day), luceneOptions.getStore(), - luceneOptions.getIndex(), luceneOptions.getTermVector()); - field.setBoost(luceneOptions.getBoost()); - document.add(field); + luceneOptions.addFieldToDocument( + name + ".day", + day &lt; 10 ? "0" : "" + String.valueOf( day ), + document ); } } @@ -1450,6 +1446,20 @@ <emphasis role="bold">@FieldBridge(impl = DateSplitBridge.class)</emphasis> private Date date; </programlisting> </example> + + <para>In the previous example the fields where not added directly to Document + but we where delegating this task to the <classname>LuceneOptions</classname> helper; this will apply the + options you have selected on <literal>@Field</literal>, like <literal>Store</literal> + or <literal>TermVector</literal> options, or apply the choosen <classname>@Boost</classname> + value. It is especially useful to encapsulate the complexity of <literal>COMPRESS</literal> + implementations so it's recommended to delegate to <classname>LuceneOptions</classname> to add fields to the + <classname>Document</classname>, but nothing stops you from editing + the <classname>Document</classname> directly and ignore the <classname>LuceneOptions</classname> in case you need to. + </para> + <tip><para>Classes like <classname>LuceneOptions</classname> are created to shield your application from + changes in Lucene API and simplify your code. Use them if you can, but if you need more flexibility + you're not required to.</para></tip> + </section> <section> @@ -1485,7 +1495,6 @@ ... } - public class CatFieldsClassBridge implements FieldBridge, ParameterizedBridge { private String sepChar; @@ -2199,7 +2208,7 @@ @Fields({ @Field, @Field(name="street1_abridged", - bridge= @FieldBridge(impl = ConcatStringBridge.class, + bridge = @FieldBridge( impl = ConcatStringBridge.class, params = @Parameter( name="size", value="4" )) }) private String address1; Modified: search/trunk/hibernate-search/src/main/java/org/hibernate/search/bridge/LuceneOptions.java =================================================================== --- search/trunk/hibernate-search/src/main/java/org/hibernate/search/bridge/LuceneOptions.java 2010-04-08 09:09:22 UTC (rev 19186) +++ search/trunk/hibernate-search/src/main/java/org/hibernate/search/bridge/LuceneOptions.java 2010-04-08 09:50:52 UTC (rev 19187) @@ -24,23 +24,42 @@ */ package org.hibernate.search.bridge; +import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; /** * A wrapper class for Lucene parameters needed for indexing. * * @author Emmanuel Bernard + * @author Sanne Grinovero */ public interface LuceneOptions { + + void addFieldToDocument(String name, String indexedString, Document document); + + /** + * Might be removed in version 3.3 to better support Lucene 3 + * which is missing COMPRESS Store Type. + * To use compression either use #addFieldToDocument or refer + * to Lucene documentation to implement your own compression + * strategy. + * @deprecated use addToDocument to add fields to the Document if possible + */ Field.Store getStore(); + /** + * @deprecated likely to be removed in version 3.3, use #addFieldToDocument + */ Field.Index getIndex(); + /** + * @deprecated likely to be removed in version 3.3, use #addFieldToDocument + */ Field.TermVector getTermVector(); /** - * @return the boost value. If <code>boost == null</code>, the default boost value - * 1.0 is returned. + * @deprecated likely to be removed in version 3.3, use #addFieldToDocument */ Float getBoost(); + } Modified: search/trunk/hibernate-search/src/main/java/org/hibernate/search/bridge/String2FieldBridgeAdaptor.java =================================================================== --- search/trunk/hibernate-search/src/main/java/org/hibernate/search/bridge/String2FieldBridgeAdaptor.java 2010-04-08 09:09:22 UTC (rev 19186) +++ search/trunk/hibernate-search/src/main/java/org/hibernate/search/bridge/String2FieldBridgeAdaptor.java 2010-04-08 09:50:52 UTC (rev 19187) @@ -25,8 +25,6 @@ package org.hibernate.search.bridge; import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.hibernate.util.StringHelper; /** * Bridge to use a StringBridge as a FieldBridge. @@ -42,13 +40,7 @@ public void set(String name, Object value, Document document, LuceneOptions luceneOptions) { String indexedString = stringBridge.objectToString( value ); - //Do not add fields on empty strings, seems a sensible default in most situations - //TODO if Store, probably also save empty ones - if ( StringHelper.isNotEmpty( indexedString ) ) { - Field field = new Field( name, indexedString, luceneOptions.getStore(), luceneOptions.getIndex(), luceneOptions.getTermVector() ); - field.setBoost( luceneOptions.getBoost() ); - document.add( field ); - } + luceneOptions.addFieldToDocument( name, indexedString, document ); } } Modified: search/trunk/hibernate-search/src/main/java/org/hibernate/search/bridge/TwoWayString2FieldBridgeAdaptor.java =================================================================== --- search/trunk/hibernate-search/src/main/java/org/hibernate/search/bridge/TwoWayString2FieldBridgeAdaptor.java 2010-04-08 09:09:22 UTC (rev 19186) +++ search/trunk/hibernate-search/src/main/java/org/hibernate/search/bridge/TwoWayString2FieldBridgeAdaptor.java 2010-04-08 09:50:52 UTC (rev 19187) @@ -24,8 +24,12 @@ */ package org.hibernate.search.bridge; +import java.util.zip.DataFormatException; + +import org.apache.lucene.document.CompressionTools; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; +import org.hibernate.search.SearchException; /** * Bridge to use a TwoWayStringBridge as a TwoWayFieldBridge @@ -52,7 +56,19 @@ return stringBridge.stringToObject( null ); } else { - return stringBridge.stringToObject( field.stringValue() ); + String stringValue; + if ( field.isBinary() ) { + try { + stringValue = CompressionTools.decompressString( field.getBinaryValue() ); + } + catch (DataFormatException e) { + throw new SearchException( "Field " + name + " looks like binary but couldn't be decompressed" ); + } + } + else { + stringValue = field.stringValue(); + } + return stringBridge.stringToObject( stringValue ); } } } Modified: search/trunk/hibernate-search/src/main/java/org/hibernate/search/engine/DocumentBuilderContainedEntity.java =================================================================== --- search/trunk/hibernate-search/src/main/java/org/hibernate/search/engine/DocumentBuilderContainedEntity.java 2010-04-08 09:09:22 UTC (rev 19186) +++ search/trunk/hibernate-search/src/main/java/org/hibernate/search/engine/DocumentBuilderContainedEntity.java 2010-04-08 09:50:52 UTC (rev 19187) @@ -459,7 +459,7 @@ private void bindClassBridgeAnnotation(String prefix, PropertiesMetadata propertiesMetadata, ClassBridge ann, InitContext context) { String fieldName = prefix + ann.name(); propertiesMetadata.classNames.add( fieldName ); - propertiesMetadata.classStores.add( getStore( ann.store() ) ); + propertiesMetadata.classStores.add( ann.store() ); propertiesMetadata.classIndexes.add( getIndex( ann.index() ) ); propertiesMetadata.classTermVectors.add( getTermVector( ann.termVector() ) ); propertiesMetadata.classBridges.add( BridgeFactory.extractType( ann ) ); @@ -480,7 +480,7 @@ propertiesMetadata.fieldGetters.add( member ); String fieldName = prefix + ReflectionHelper.getAttributeName( member, fieldAnn.name() ); propertiesMetadata.fieldNames.add( fieldName ); - propertiesMetadata.fieldStore.add( getStore( fieldAnn.store() ) ); + propertiesMetadata.fieldStore.add( fieldAnn.store() ); propertiesMetadata.fieldIndex.add( getIndex( fieldAnn.index() ) ); propertiesMetadata.fieldBoosts.add( getBoost( member, fieldAnn ) ); propertiesMetadata.dynamicFieldBoosts.add( getDynamicBoost( member ) ); @@ -540,19 +540,6 @@ return localPrefix; } - protected Field.Store getStore(Store store) { - switch ( store ) { - case NO: - return Field.Store.NO; - case YES: - return Field.Store.YES; - case COMPRESS: - return Field.Store.COMPRESS; - default: - throw new AssertionFailure( "Unexpected Store: " + store ); - } - } - protected Field.TermVector getTermVector(TermVector vector) { switch ( vector ) { case NO: @@ -785,7 +772,7 @@ public final List<String> fieldNames = new ArrayList<String>(); public final List<XMember> fieldGetters = new ArrayList<XMember>(); public final List<FieldBridge> fieldBridges = new ArrayList<FieldBridge>(); - public final List<Field.Store> fieldStore = new ArrayList<Field.Store>(); + public final List<Store> fieldStore = new ArrayList<Store>(); public final List<Field.Index> fieldIndex = new ArrayList<Field.Index>(); public final List<Float> fieldBoosts = new ArrayList<Float>(); public final List<BoostStrategy> dynamicFieldBoosts = new ArrayList<BoostStrategy>(); @@ -797,7 +784,7 @@ public final List<XMember> containedInGetters = new ArrayList<XMember>(); public final List<String> classNames = new ArrayList<String>(); - public final List<Field.Store> classStores = new ArrayList<Field.Store>(); + public final List<Store> classStores = new ArrayList<Store>(); public final List<Field.Index> classIndexes = new ArrayList<Field.Index>(); public final List<FieldBridge> classBridges = new ArrayList<FieldBridge>(); public final List<Field.TermVector> classTermVectors = new ArrayList<Field.TermVector>(); Modified: search/trunk/hibernate-search/src/main/java/org/hibernate/search/engine/DocumentBuilderIndexedEntity.java =================================================================== --- search/trunk/hibernate-search/src/main/java/org/hibernate/search/engine/DocumentBuilderIndexedEntity.java 2010-04-08 09:09:22 UTC (rev 19186) +++ search/trunk/hibernate-search/src/main/java/org/hibernate/search/engine/DocumentBuilderIndexedEntity.java 2010-04-08 09:50:52 UTC (rev 19187) @@ -206,7 +206,7 @@ propertiesMetadata.fieldGetters.add( member ); String fieldName = prefix + attributeName; propertiesMetadata.fieldNames.add( fieldName ); - propertiesMetadata.fieldStore.add( getStore( Store.YES ) ); + propertiesMetadata.fieldStore.add( Store.YES ); propertiesMetadata.fieldIndex.add( getIndex( Index.UN_TOKENIZED ) ); propertiesMetadata.fieldTermVectors.add( getTermVector( TermVector.NO ) ); propertiesMetadata.fieldBridges.add( BridgeFactory.guessType( null, member, reflectionManager ) ); @@ -403,7 +403,7 @@ // now add the entity id to the document LuceneOptions luceneOptions = new LuceneOptionsImpl( - Field.Store.YES, + Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS, Field.TermVector.NO, idBoost ); idBridge.set( idKeywordName, id, doc, luceneOptions ); @@ -623,7 +623,7 @@ populateResult( builderIndexedEntity.idKeywordName, builderIndexedEntity.idBridge, - Field.Store.YES, + Store.YES, fields, result, document @@ -635,19 +635,19 @@ return result; } - private static void populateResult(String fieldName, FieldBridge fieldBridge, Field.Store store, + private static void populateResult(String fieldName, FieldBridge fieldBridge, Store store, String[] fields, Object[] result, Document document) { int matchingPosition = getFieldPosition( fields, fieldName ); if ( matchingPosition != -1 ) { //TODO make use of an isTwoWay() method - if ( store != Field.Store.NO && TwoWayFieldBridge.class.isAssignableFrom( fieldBridge.getClass() ) ) { + if ( store != Store.NO && TwoWayFieldBridge.class.isAssignableFrom( fieldBridge.getClass() ) ) { result[matchingPosition] = ( ( TwoWayFieldBridge ) fieldBridge ).get( fieldName, document ); if ( log.isTraceEnabled() ) { log.trace( "Field {} projected as {}", fieldName, result[matchingPosition] ); } } else { - if ( store == Field.Store.NO ) { + if ( store == Store.NO ) { throw new SearchException( "Projecting an unstored field: " + fieldName ); } else { Modified: search/trunk/hibernate-search/src/main/java/org/hibernate/search/engine/LuceneOptionsImpl.java =================================================================== --- search/trunk/hibernate-search/src/main/java/org/hibernate/search/engine/LuceneOptionsImpl.java 2010-04-08 09:09:22 UTC (rev 19186) +++ search/trunk/hibernate-search/src/main/java/org/hibernate/search/engine/LuceneOptionsImpl.java 2010-04-08 09:50:52 UTC (rev 19187) @@ -24,10 +24,14 @@ */ package org.hibernate.search.engine; +import org.apache.lucene.document.CompressionTools; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; import org.apache.lucene.document.Field.Index; -import org.apache.lucene.document.Field.Store; import org.apache.lucene.document.Field.TermVector; +import org.hibernate.annotations.common.util.StringHelper; +import org.hibernate.search.annotations.Store; import org.hibernate.search.bridge.LuceneOptions; /** @@ -35,36 +39,53 @@ * This is a package level class * * @author Hardy Ferentschik + * @author Sanne Grinovero */ class LuceneOptionsImpl implements LuceneOptions { - private final Store store; - private final Index index; + + private final boolean storeCompressed; + private final boolean storeUncompressed; + private final Index indexMode; private final TermVector termVector; private final Float boost; + private final Store storeType; - public LuceneOptionsImpl(Store store, Index index, TermVector termVector, Float boost) { - this.store = store; - this.index = index; + public LuceneOptionsImpl(Store store, Index indexMode, TermVector termVector, Float boost) { + this.indexMode = indexMode; this.termVector = termVector; this.boost = boost; + this.storeType = store; + this.storeCompressed = store.equals( Store.COMPRESS ); + this.storeUncompressed = store.equals( Store.YES ); } - public Store getStore() { - return store; + public void addFieldToDocument(String name, String indexedString, Document document) { + //Do not add fields on empty strings, seems a sensible default in most situations + //TODO if Store, probably also save empty ones + if ( StringHelper.isNotEmpty( indexedString ) ) { + if ( ! ( indexMode.equals( Index.NO ) && storeCompressed ) ) { + standardFieldAdd( name, indexedString, document ); + } + if ( storeCompressed ) { + compressedFieldAdd( name, indexedString, document ); + } + } } - public Index getIndex() { - return index; + private void standardFieldAdd(String name, String indexedString, Document document) { + Field field = new Field( name, false, indexedString, storeUncompressed ? Field.Store.YES : Field.Store.NO , indexMode, termVector ); + if ( boost != null ) + field.setBoost( boost ); + document.add( field ); } - - public TermVector getTermVector() { - return termVector; + + private void compressedFieldAdd(String name, String indexedString, Document document) { + byte[] compressedString = CompressionTools.compressString( indexedString ); + // indexed is implicitly set to false when using byte[] + Field field = new Field( name, compressedString, Field.Store.YES ); + document.add( field ); } - /** - * @return the boost value. If <code>boost == null</code>, the default boost value - * 1.0 is returned. - */ public Float getBoost() { if ( boost != null ) { return boost; @@ -72,4 +93,33 @@ return 1.0f; } } + + public Index getIndex() { + return this.indexMode; + } + + /** + * @deprecated likely to be removed in 3.3 + */ + public org.apache.lucene.document.Field.Store getStore() { + if (storeCompressed) + return org.apache.lucene.document.Field.Store.COMPRESS; + else if (storeUncompressed) + return org.apache.lucene.document.Field.Store.YES; + else + return org.apache.lucene.document.Field.Store.NO; + } + + public TermVector getTermVector() { + return this.termVector; + } + + /** + * Might be useful for a bridge implementation, but not currently part + * of LuceneOptions API as we are considering to remove the getters. + */ + public Store getStoreStrategy() { + return storeType; + } + } Modified: search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/bridge/CatDeptsFieldsClassBridge.java =================================================================== --- search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/bridge/CatDeptsFieldsClassBridge.java 2010-04-08 09:09:22 UTC (rev 19186) +++ search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/bridge/CatDeptsFieldsClassBridge.java 2010-04-08 09:50:52 UTC (rev 19187) @@ -27,7 +27,6 @@ import java.util.Map; import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; import org.hibernate.search.bridge.FieldBridge; import org.hibernate.search.bridge.LuceneOptions; import org.hibernate.search.bridge.ParameterizedBridge; @@ -58,9 +57,7 @@ if ( fieldValue2 == null ) { fieldValue2 = ""; } - String fieldValue = fieldValue1 + sepChar + fieldValue2; - Field field = new Field( name, fieldValue, luceneOptions.getStore(), luceneOptions.getIndex(), luceneOptions.getTermVector() ); - field.setBoost( luceneOptions.getBoost() ); - document.add( field ); + String indexedString = fieldValue1 + sepChar + fieldValue2; + luceneOptions.addFieldToDocument( name, indexedString, document ); } } Modified: search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/bridge/CatFieldsClassBridge.java =================================================================== --- search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/bridge/CatFieldsClassBridge.java 2010-04-08 09:09:22 UTC (rev 19186) +++ search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/bridge/CatFieldsClassBridge.java 2010-04-08 09:50:52 UTC (rev 19187) @@ -27,7 +27,6 @@ import java.util.Map; import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; import org.hibernate.search.bridge.FieldBridge; import org.hibernate.search.bridge.LuceneOptions; import org.hibernate.search.bridge.ParameterizedBridge; @@ -58,9 +57,7 @@ if ( fieldValue2 == null ) { fieldValue2 = ""; } - String fieldValue = fieldValue1 + sepChar + fieldValue2; - Field field = new Field( name, fieldValue, luceneOptions.getStore(), luceneOptions.getIndex(), luceneOptions.getTermVector() ); - field.setBoost( luceneOptions.getBoost() ); - document.add( field ); + String indexedString = fieldValue1 + sepChar + fieldValue2; + luceneOptions.addFieldToDocument( name, indexedString, document ); } } Modified: search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/bridge/DateSplitBridge.java =================================================================== --- search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/bridge/DateSplitBridge.java 2010-04-08 09:09:22 UTC (rev 19186) +++ search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/bridge/DateSplitBridge.java 2010-04-08 09:50:52 UTC (rev 19187) @@ -30,7 +30,6 @@ import java.util.TimeZone; import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; import org.hibernate.search.bridge.FieldBridge; import org.hibernate.search.bridge.LuceneOptions; @@ -52,24 +51,14 @@ int day = cal.get(Calendar.DAY_OF_MONTH); // set year - Field field = new Field(name + ".year", String.valueOf(year), - luceneOptions.getStore(), luceneOptions.getIndex(), - luceneOptions.getTermVector()); - field.setBoost(luceneOptions.getBoost()); - document.add(field); + luceneOptions.addFieldToDocument( name + ".year", String.valueOf( year ), document ); // set month and pad it if needed - field = new Field(name + ".month", month < 10 ? "0" : "" - + String.valueOf(month), luceneOptions.getStore(), - luceneOptions.getIndex(), luceneOptions.getTermVector()); - field.setBoost(luceneOptions.getBoost()); - document.add(field); + luceneOptions.addFieldToDocument( name + ".month", + month < 10 ? "0" : "" + String.valueOf( month ), document ); // set day and pad it if needed - field = new Field(name + ".day", day < 10 ? "0" : "" - + String.valueOf(day), luceneOptions.getStore(), - luceneOptions.getIndex(), luceneOptions.getTermVector()); - field.setBoost(luceneOptions.getBoost()); - document.add(field); + luceneOptions.addFieldToDocument( name + ".day", + day < 10 ? "0" : "" + String.valueOf( day ), document ); } } Modified: search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/bridge/EquipmentType.java =================================================================== --- search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/bridge/EquipmentType.java 2010-04-08 09:09:22 UTC (rev 19186) +++ search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/bridge/EquipmentType.java 2010-04-08 09:50:52 UTC (rev 19187) @@ -27,7 +27,6 @@ import java.util.Map; import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; import org.hibernate.search.bridge.FieldBridge; import org.hibernate.search.bridge.LuceneOptions; @@ -51,16 +50,11 @@ // a requirement. It just works that way in this instance. The // actual name could be supplied by hard coding it below. Departments deps = ( Departments ) value; - Field field; String fieldValue1 = deps.getManufacturer(); if ( fieldValue1 != null ) { - String fieldValue = ( String ) equips.get( fieldValue1 ); - field = new Field( - name, fieldValue, luceneOptions.getStore(), luceneOptions.getIndex(), luceneOptions.getTermVector() - ); - field.setBoost( luceneOptions.getBoost() ); - document.add( field ); + String indexedString = ( String ) equips.get( fieldValue1 ); + luceneOptions.addFieldToDocument( name, indexedString, document ); } } } Modified: search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/bridge/TruncateFieldBridge.java =================================================================== --- search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/bridge/TruncateFieldBridge.java 2010-04-08 09:09:22 UTC (rev 19186) +++ search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/bridge/TruncateFieldBridge.java 2010-04-08 09:50:52 UTC (rev 19187) @@ -28,26 +28,23 @@ import org.apache.lucene.document.Field; import org.hibernate.search.bridge.FieldBridge; import org.hibernate.search.bridge.LuceneOptions; -import org.hibernate.util.StringHelper; /** * @author Emmanuel Bernard */ public class TruncateFieldBridge implements FieldBridge { + public Object get(String name, Document document) { Field field = document.getField( name ); return field.stringValue(); } public void set(String name, Object value, Document document, LuceneOptions luceneOptions) { - String indexedString = (String) value; - //Do not add fields on empty strings, seems a sensible default in most situations - if ( StringHelper.isNotEmpty( indexedString ) ) { - Field field = new Field(name, indexedString.substring(0, - indexedString.length() / 2), luceneOptions.getStore(), - luceneOptions.getIndex(), luceneOptions.getTermVector()); - field.setBoost( luceneOptions.getBoost() ); - document.add( field ); + String stringValue = (String) value; + if ( stringValue != null ) { + String indexedString = stringValue.substring( 0, stringValue.length() / 2 ); + luceneOptions.addFieldToDocument( name, indexedString, document ); } } + } Added: search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/compression/CompressionTest.java =================================================================== --- search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/compression/CompressionTest.java (rev 0) +++ search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/compression/CompressionTest.java 2010-04-08 09:50:52 UTC (rev 19187) @@ -0,0 +1,193 @@ +/* $Id$ + * + * Hibernate, Relational Persistence for Idiomatic Java + * + * Copyright (c) 2009, Red Hat, Inc. and/or its affiliates or third-party contributors as + * indicated by the @author tags or express copyright attribution + * statements applied by the authors. All third-party contributions are + * distributed under license by Red Hat, Inc. + * + * This copyrighted material is made available to anyone wishing to use, modify, + * copy, or redistribute it subject to the terms and conditions of the GNU + * Lesser General Public License, as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this distribution; if not, write to: + * Free Software Foundation, Inc. + * 51 Franklin Street, Fifth Floor + * Boston, MA 02110-1301 USA + */ +package org.hibernate.search.test.compression; + +import java.util.List; +import java.util.zip.DataFormatException; + +import junit.framework.Assert; + +import org.apache.lucene.analysis.SimpleAnalyzer; +import org.apache.lucene.document.CompressionTools; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.queryParser.ParseException; +import org.apache.lucene.queryParser.QueryParser; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TopDocs; +import org.hibernate.Session; +import org.hibernate.search.FullTextQuery; +import org.hibernate.search.FullTextSession; +import org.hibernate.search.Search; +import org.hibernate.search.store.DirectoryProvider; +import org.hibernate.search.test.SearchTestCase; + +public class CompressionTest extends SearchTestCase { + + /** + * verifies the fields are really stored in compressed format + */ + public void testFieldWasCompressed() throws Exception { + DirectoryProvider[] directoryProviders = getSearchFactory().getDirectoryProviders( LargeDocument.class ); + IndexReader indexReader = getSearchFactory().getReaderProvider().openReader( directoryProviders ); + try { + IndexSearcher searcher = new IndexSearcher( indexReader ); + TopDocs topDocs = searcher.search( new MatchAllDocsQuery(), null, 10 ); + Assert.assertEquals( 1, topDocs.totalHits ); + + ScoreDoc doc = topDocs.scoreDocs[0]; + Document document = indexReader.document( doc.doc ); + { + Field[] fields = document.getFields( "title" ); + Assert.assertEquals( 1, fields.length ); + Assert.assertFalse( fields[0].isCompressed() ); + Assert.assertTrue( fields[0].isIndexed() ); + Assert.assertTrue( fields[0].isStored() ); + Assert.assertEquals( + "Hibernate in Action, third edition", + fields[0].stringValue() + ); + } + { + Field[] fields = document.getFields( "abstract" ); + Assert.assertEquals( 1, fields.length ); + Assert.assertTrue( isCompressed( fields[0] ) ); + Assert.assertTrue( fields[0].isCompressed() ); + Assert.assertEquals( + "<b>JPA2 with Hibernate</b>", + restoreValue( fields[0] ) + ); + } + { + Field[] fields = document.getFields( "text" ); + Assert.assertEquals( 1, fields.length ); + Assert.assertTrue( isCompressed( fields[0] ) ); + Assert.assertEquals( + "This is a placeholder for the new text that you should write", + restoreValue( fields[0] ) + ); + } + } + finally { + getSearchFactory().getReaderProvider().closeReader( indexReader ); + } + } + + /** + * Verifies the compressed fields are also searchable + */ + public void testCompressedFieldSearch() throws ParseException { + assertFindsN( 1, "title:third" ); + assertFindsN( 1, "abstract:jpa2" ); + assertFindsN( 1, "text:write" ); + assertFindsN( 0, "text:jpa2" ); + } + + private void assertFindsN(int expectedToFind, String queryString) throws ParseException { + openSession().beginTransaction(); + try { + FullTextSession fullTextSession = Search.getFullTextSession( session ); + QueryParser qparser = new QueryParser( getTargetLuceneVersion(), "", new SimpleAnalyzer() ); + Query query = qparser.parse( queryString ); + FullTextQuery fullTextQuery = fullTextSession.createFullTextQuery( + query, + LargeDocument.class ); + List<LargeDocument> list = fullTextQuery.list(); + Assert.assertEquals( expectedToFind, list.size() ); + if ( expectedToFind == 1 ) + Assert.assertEquals( "Hibernate in Action, third edition", list.get( 0 ).getTitle() ); + } + finally { + session.getTransaction().commit(); + session.close(); + } + } + + /** + * Verify that projection is able to inflate stored data + */ + public void testProjectionOnCompressedFields() { + openSession().beginTransaction(); + try { + FullTextSession fullTextSession = Search.getFullTextSession( session ); + FullTextQuery fullTextQuery = fullTextSession.createFullTextQuery( + new MatchAllDocsQuery(), + LargeDocument.class ); + List list = fullTextQuery.setProjection( "title", "abstract", "text" ).list(); + Assert.assertEquals( 1, list.size() ); + Object[] results = (Object[]) list.get( 0 ); + Assert.assertEquals( "Hibernate in Action, third edition", results[0] ); + Assert.assertEquals( "JPA2 with Hibernate", results[1] ); + Assert.assertEquals( "This is a placeholder for the new text that you should write", results[2] ); + } + finally { + session.getTransaction().commit(); + session.close(); + } + } + + // test helpers: + + private String restoreValue(Field field) throws DataFormatException { + if ( field.isBinary() ) { + Assert.assertNull( "we rely on this in the Projection implementation", field.stringValue() ); + return CompressionTools.decompressString( field.getBinaryValue() ); + } + else { + return field.stringValue(); + } + } + + private boolean isCompressed(Field field) { + return ( field.isBinary() || field.isCompressed() ); + } + + // test setup: + + protected Class<?>[] getMappings() { + return new Class[] { + LargeDocument.class + }; + } + + protected void setUp() throws Exception { + super.setUp(); + Session s = openSession(); + s.getTransaction().begin(); + s.persist( + new LargeDocument( "Hibernate in Action, third edition", + "JPA2 with Hibernate", + "This is a placeholder for the new text that you should write" ) + ); + s.getTransaction().commit(); + s.close(); + } + +} + Added: search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/compression/HTMLBoldFieldBridge.java =================================================================== --- search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/compression/HTMLBoldFieldBridge.java (rev 0) +++ search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/compression/HTMLBoldFieldBridge.java 2010-04-08 09:50:52 UTC (rev 19187) @@ -0,0 +1,64 @@ +/* $Id$ + * + * Hibernate, Relational Persistence for Idiomatic Java + * + * Copyright (c) 2009, Red Hat, Inc. and/or its affiliates or third-party contributors as + * indicated by the @author tags or express copyright attribution + * statements applied by the authors. All third-party contributions are + * distributed under license by Red Hat, Inc. + * + * This copyrighted material is made available to anyone wishing to use, modify, + * copy, or redistribute it subject to the terms and conditions of the GNU + * Lesser General Public License, as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this distribution; if not, write to: + * Free Software Foundation, Inc. + * 51 Franklin Street, Fifth Floor + * Boston, MA 02110-1301 USA + */ +package org.hibernate.search.test.compression; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.hibernate.search.bridge.FieldBridge; +import org.hibernate.search.bridge.LuceneOptions; +import org.hibernate.search.bridge.TwoWayFieldBridge; + +/** + * This FieldBridge is storing strings in the index wrapping the + * entity value in html bold tags. + * It's using deprecated method "getStore" to verify backwards compatibility + * Almost useless, needed for CompressionTest + * @see LuceneOptions + * @author Sanne Grinovero + */ +public class HTMLBoldFieldBridge implements FieldBridge, TwoWayFieldBridge { + + public void set(String name, Object value, Document document, LuceneOptions luceneOptions) { + String fieldValue = objectToString( value ); + Field field = new Field( name, fieldValue, luceneOptions.getStore(), + luceneOptions.getIndex(), luceneOptions.getTermVector() ); + field.setBoost( luceneOptions.getBoost() ); + document.add( field ); + } + + public Object get(String name, Document document) { + Field field = document.getField( name ); + String stringValue = field.stringValue(); + return stringValue.substring( 3, stringValue.length()-4 ); + } + + public String objectToString(Object value) { + String originalValue = value.toString(); + String fieldValue = "<b>" + originalValue + "</b>"; + return fieldValue; + } + +} + Added: search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/compression/LargeDocument.java =================================================================== --- search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/compression/LargeDocument.java (rev 0) +++ search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/compression/LargeDocument.java 2010-04-08 09:50:52 UTC (rev 19187) @@ -0,0 +1,97 @@ +/* $Id$ + * + * Hibernate, Relational Persistence for Idiomatic Java + * + * Copyright (c) 2009, Red Hat, Inc. and/or its affiliates or third-party contributors as + * indicated by the @author tags or express copyright attribution + * statements applied by the authors. All third-party contributions are + * distributed under license by Red Hat, Inc. + * + * This copyrighted material is made available to anyone wishing to use, modify, + * copy, or redistribute it subject to the terms and conditions of the GNU + * Lesser General Public License, as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this distribution; if not, write to: + * Free Software Foundation, Inc. + * 51 Franklin Street, Fifth Floor + * Boston, MA 02110-1301 USA + */ +package org.hibernate.search.test.compression; + +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.Id; + +import org.hibernate.search.annotations.DocumentId; +import org.hibernate.search.annotations.Field; +import org.hibernate.search.annotations.FieldBridge; +import org.hibernate.search.annotations.Index; +import org.hibernate.search.annotations.Indexed; +import org.hibernate.search.annotations.Store; + +@Entity +@Indexed +public class LargeDocument { + + private Long id; + private String title; + private String summary; + private String text; + + LargeDocument() { + } + + public LargeDocument(String title, String summary, String text) { + super(); + this.summary = summary; + this.text = text; + this.title = title; + } + + @Id + @GeneratedValue + @DocumentId + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + @Field( store = Store.YES, index = Index.TOKENIZED ) + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + @Field( name="abstract", store = Store.COMPRESS, index = Index.TOKENIZED, + bridge = @FieldBridge( impl = HTMLBoldFieldBridge.class ) ) + public String getSummary() { + return summary; + } + + public void setSummary(String summary) { + this.summary = summary; + } + + @Field( store = Store.COMPRESS, index = Index.TOKENIZED ) + public String getText() { + return text; + } + + public void setText(String text) { + this.text = text; + } + +} + Modified: search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/configuration/CatDeptsFieldsClassBridge.java =================================================================== --- search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/configuration/CatDeptsFieldsClassBridge.java 2010-04-08 09:09:22 UTC (rev 19186) +++ search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/configuration/CatDeptsFieldsClassBridge.java 2010-04-08 09:50:52 UTC (rev 19187) @@ -27,7 +27,6 @@ import java.util.Map; import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; import org.hibernate.search.bridge.FieldBridge; import org.hibernate.search.bridge.LuceneOptions; import org.hibernate.search.bridge.ParameterizedBridge; @@ -58,9 +57,7 @@ if ( fieldValue2 == null ) { fieldValue2 = ""; } - String fieldValue = fieldValue1 + sepChar + fieldValue2; - Field field = new Field( name, fieldValue, luceneOptions.getStore(), luceneOptions.getIndex(), luceneOptions.getTermVector() ); - field.setBoost( luceneOptions.getBoost() ); - document.add( field ); + String indexedString = fieldValue1 + sepChar + fieldValue2; + luceneOptions.addFieldToDocument( name, indexedString, document ); } } Modified: search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/configuration/EquipmentType.java =================================================================== --- search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/configuration/EquipmentType.java 2010-04-08 09:09:22 UTC (rev 19186) +++ search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/configuration/EquipmentType.java 2010-04-08 09:50:52 UTC (rev 19187) @@ -27,7 +27,6 @@ import java.util.Map; import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; import org.hibernate.search.bridge.FieldBridge; import org.hibernate.search.bridge.LuceneOptions; @@ -38,6 +37,7 @@ */ @SuppressWarnings("unchecked") public class EquipmentType implements FieldBridge, ParameterizedBridge { + private Map equips; public void setParameterValues(Map parameters) { @@ -51,16 +51,12 @@ // a requirement. It just works that way in this instance. The // actual name could be supplied by hard coding it below. Departments deps = ( Departments ) value; - Field field; - String fieldValue1 = deps.getManufacturer(); + String fieldValue = deps.getManufacturer(); - if ( fieldValue1 != null ) { - String fieldValue = ( String ) equips.get( fieldValue1 ); - field = new Field( - name, fieldValue, luceneOptions.getStore(), luceneOptions.getIndex(), luceneOptions.getTermVector() - ); - field.setBoost( luceneOptions.getBoost() ); - document.add( field ); + if ( fieldValue != null ) { + String indexedString = ( String ) equips.get( fieldValue ); + luceneOptions.addFieldToDocument( name, indexedString, document ); } } + } Modified: search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/id/PersonPKBridge.java =================================================================== --- search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/id/PersonPKBridge.java 2010-04-08 09:09:22 UTC (rev 19186) +++ search/trunk/hibernate-search/src/test/java/org/hibernate/search/test/id/PersonPKBridge.java 2010-04-08 09:50:52 UTC (rev 19187) @@ -55,35 +55,13 @@ PersonPK id = ( PersonPK ) value; //store each property in a unique field - Field field = new Field( - name + ".firstName", - id.getFirstName(), - luceneOptions.getStore(), - luceneOptions.getIndex(), - luceneOptions.getTermVector() - ); - field.setBoost( luceneOptions.getBoost() ); - document.add( field ); + luceneOptions.addFieldToDocument( name + ".firstName", id.getFirstName(), document ); - field = new Field( - name + ".lastName", - id.getLastName(), - luceneOptions.getStore(), - luceneOptions.getIndex(), - luceneOptions.getTermVector() - ); - field.setBoost( luceneOptions.getBoost() ); - document.add( field ); - + luceneOptions.addFieldToDocument( name + ".lastName", id.getLastName(), document ); + //store the unique string representation in the named field - field = new Field( - name, - objectToString( id ), - luceneOptions.getStore(), - luceneOptions.getIndex(), - luceneOptions.getTermVector() - ); - field.setBoost( luceneOptions.getBoost() ); - document.add( field ); + luceneOptions.addFieldToDocument( name, objectToString( id ), document ); + } + }