[hibernate-commits] Hibernate SVN: r15549 - in search/trunk: src/test/org/hibernate/search/test/analyzer/solr and 1 other directories.
hibernate-commits at lists.jboss.org
hibernate-commits at lists.jboss.org
Tue Nov 11 15:16:11 EST 2008
Author: hardy.ferentschik
Date: 2008-11-11 15:16:10 -0500 (Tue, 11 Nov 2008)
New Revision: 15549
Added:
search/trunk/src/test-resources/org/hibernate/search/test/analyzer/solr/synonyms.properties
search/trunk/src/test/org/hibernate/search/test/analyzer/solr/InsertWhitespaceFilter.java
search/trunk/src/test/org/hibernate/search/test/analyzer/solr/InsertWhitespaceFilterFactory.java
Modified:
search/trunk/ivy.xml
search/trunk/src/test/org/hibernate/search/test/analyzer/solr/SolrAnalyzerTest.java
search/trunk/src/test/org/hibernate/search/test/analyzer/solr/Team.java
Log:
HSEARCH-255 Added a whole bunch of tests for different tokenizers and filters
Modified: search/trunk/ivy.xml
===================================================================
--- search/trunk/ivy.xml 2008-11-11 13:32:00 UTC (rev 15548)
+++ search/trunk/ivy.xml 2008-11-11 20:16:10 UTC (rev 15549)
@@ -21,10 +21,12 @@
<dependency org="javax.transaction" name="jta" rev="1.1" conf="default->default"/>
<dependency org="org.apache.lucene" name="lucene-core" rev="2.4.0" conf="default->default"/>
<dependency org="org.apache.lucene" name="lucene-snowball" rev="2.4.0" conf="default->default"/>
+ <dependency org="org.apache.lucene" name="lucene-analyzers" rev="2.4.0" conf="default->default"/> <!-- optional -->
+ <dependency org="org.apache.commons" name="commons-codec" rev="1.3" conf="default->default"/> <!-- optional -->
<dependency org="javax.jms" name="jms" rev="1.1" conf="default->default"/> <!-- optional -->
<dependency org="javax.annotation" name="jsr250-api" rev="1.0" conf="default->default"/> <!-- optional -->
- <dependency org="org.apache.solr" name="solr-core" rev="1.3.0" conf="default->default"/>
- <dependency org="org.apache.solr" name="solr-common" rev="1.3.0" conf="default->default"/>
+ <dependency org="org.apache.solr" name="solr-core" rev="1.3.0" conf="default->default"/> <!-- optional -->
+ <dependency org="org.apache.solr" name="solr-common" rev="1.3.0" conf="default->default"/> <!-- optional -->
<!-- transitive dependencies -->
<dependency org="antlr" name="antlr" rev="2.7.6" conf="test->default"/>
@@ -46,4 +48,4 @@
<dependency org="org.apache.derby" name="derby" rev="10.2.1.6" conf="test->default"/>
</dependencies>
-</ivy-module>
\ No newline at end of file
+</ivy-module>
Added: search/trunk/src/test/org/hibernate/search/test/analyzer/solr/InsertWhitespaceFilter.java
===================================================================
--- search/trunk/src/test/org/hibernate/search/test/analyzer/solr/InsertWhitespaceFilter.java (rev 0)
+++ search/trunk/src/test/org/hibernate/search/test/analyzer/solr/InsertWhitespaceFilter.java 2008-11-11 20:16:10 UTC (rev 15549)
@@ -0,0 +1,36 @@
+// $Id: AbstractTestAnalyzer.java 15547 2008-11-11 12:57:47Z hardy.ferentschik $
+package org.hibernate.search.test.analyzer.solr;
+
+import java.io.Reader;
+import java.io.IOException;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.search.Filter;
+
+
+/**
+ * A filter which will actually insert spaces. Most filters/tokenizers remove them, but for testing it is
+ * sometimes better to insert them again ;-)
+ *
+ * @author Hardy Ferentschik
+ */
+public class InsertWhitespaceFilter extends TokenFilter {
+ public InsertWhitespaceFilter(TokenStream in) {
+ super( in );
+ }
+
+ public Token next(final Token reusableToken) throws IOException {
+ Token nextToken = input.next( reusableToken );
+ if ( nextToken != null ) {
+ nextToken.setTermBuffer( " " + nextToken.term() + " " );
+ return nextToken;
+ }
+ else {
+ return null;
+ }
+ }
+}
\ No newline at end of file
Added: search/trunk/src/test/org/hibernate/search/test/analyzer/solr/InsertWhitespaceFilterFactory.java
===================================================================
--- search/trunk/src/test/org/hibernate/search/test/analyzer/solr/InsertWhitespaceFilterFactory.java (rev 0)
+++ search/trunk/src/test/org/hibernate/search/test/analyzer/solr/InsertWhitespaceFilterFactory.java 2008-11-11 20:16:10 UTC (rev 15549)
@@ -0,0 +1,30 @@
+// $Id$
+/*
+* JBoss, Home of Professional Open Source
+* Copyright 2008, Red Hat Middleware LLC, and individual contributors
+* by the @authors tag. See the copyright.txt in the distribution for a
+* full listing of individual contributors.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+* http://www.apache.org/licenses/LICENSE-2.0
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.hibernate.search.test.analyzer.solr;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.solr.analysis.BaseTokenFilterFactory;
+
+/**
+ * @author Hardy Ferentschik
+ */
+public class InsertWhitespaceFilterFactory extends BaseTokenFilterFactory {
+ public InsertWhitespaceFilter create(TokenStream input) {
+ return new InsertWhitespaceFilter( input );
+ }
+}
Property changes on: search/trunk/src/test/org/hibernate/search/test/analyzer/solr/InsertWhitespaceFilterFactory.java
___________________________________________________________________
Name: svn:keywords
+ Id
Modified: search/trunk/src/test/org/hibernate/search/test/analyzer/solr/SolrAnalyzerTest.java
===================================================================
--- search/trunk/src/test/org/hibernate/search/test/analyzer/solr/SolrAnalyzerTest.java 2008-11-11 13:32:00 UTC (rev 15548)
+++ search/trunk/src/test/org/hibernate/search/test/analyzer/solr/SolrAnalyzerTest.java 2008-11-11 20:16:10 UTC (rev 15549)
@@ -3,11 +3,14 @@
import org.apache.lucene.index.Term;
import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.Token;
import org.hibernate.Transaction;
import org.hibernate.search.FullTextSession;
import org.hibernate.search.Search;
import org.hibernate.search.test.SearchTestCase;
+import org.hibernate.search.test.util.AnalyzerUtils;
/**
* Tests the Solr analyzer creation framework.
@@ -62,6 +65,90 @@
fts.close();
}
+ /**
+ * Tests the analyzers defined on {@link Team}.
+ *
+ * @throws Exception in case the test fails.
+ */
+ public void testAnalyzers() throws Exception {
+ FullTextSession fts = Search.getFullTextSession( openSession() );
+
+ Analyzer analyzer = fts.getSearchFactory().getAnalyzer( "standard_analyzer" );
+ String text = "This is just FOOBAR's";
+ Token[] tokens = AnalyzerUtils.tokensFromAnalysis( analyzer, "name", text );
+ AnalyzerUtils.assertTokensEqual( tokens, new String[] { "This", "is", "just", "FOOBAR" } );
+
+ analyzer = fts.getSearchFactory().getAnalyzer( "html_standard_analyzer" );
+ text = "This is <b>foo</b><i>bar's</i>";
+ tokens = AnalyzerUtils.tokensFromAnalysis( analyzer, "name", text );
+ AnalyzerUtils.assertTokensEqual( tokens, new String[] { "This", "is", "foo", "bar" } );
+
+ analyzer = fts.getSearchFactory().getAnalyzer( "html_whitespace_analyzer" );
+ text = "This is <b>foo</b><i>bar's</i>";
+ tokens = AnalyzerUtils.tokensFromAnalysis( analyzer, "name", text );
+ AnalyzerUtils.assertTokensEqual( tokens, new String[] { "This", "is", "foo", "bar's" } );
+
+ analyzer = fts.getSearchFactory().getAnalyzer( "trim_analyzer" );
+ text = " Kittens! ";
+ tokens = AnalyzerUtils.tokensFromAnalysis( analyzer, "name", text );
+ AnalyzerUtils.assertTokensEqual( tokens, new String[] { "kittens" } );
+
+ analyzer = fts.getSearchFactory().getAnalyzer( "length_analyzer" );
+ text = "ab abc abcd abcde abcdef";
+ tokens = AnalyzerUtils.tokensFromAnalysis( analyzer, "name", text );
+ AnalyzerUtils.assertTokensEqual( tokens, new String[] { "abc", "abcd", "abcde" } );
+
+ analyzer = fts.getSearchFactory().getAnalyzer( "length_analyzer" );
+ text = "ab abc abcd abcde abcdef";
+ tokens = AnalyzerUtils.tokensFromAnalysis( analyzer, "name", text );
+ AnalyzerUtils.assertTokensEqual( tokens, new String[] { "abc", "abcd", "abcde" } );
+
+ analyzer = fts.getSearchFactory().getAnalyzer( "porter_analyzer" );
+ text = "bikes bikes biking";
+ tokens = AnalyzerUtils.tokensFromAnalysis( analyzer, "name", text );
+ AnalyzerUtils.assertTokensEqual( tokens, new String[] { "bike", "bike", "bike" } );
+
+ analyzer = fts.getSearchFactory().getAnalyzer( "word_analyzer" );
+ text = "CamelCase";
+ tokens = AnalyzerUtils.tokensFromAnalysis( analyzer, "name", text );
+ AnalyzerUtils.assertTokensEqual( tokens, new String[] { "Camel", "Case" } );
+
+ analyzer = fts.getSearchFactory().getAnalyzer( "synonym_analyzer" );
+ text = "ipod cosmos";
+ tokens = AnalyzerUtils.tokensFromAnalysis( analyzer, "name", text );
+ AnalyzerUtils.assertTokensEqual( tokens, new String[] { "ipod", "i-pod", "universe", "cosmos" } );
+
+ analyzer = fts.getSearchFactory().getAnalyzer( "shingle_analyzer" );
+ text = "please divide this sentence into shingles";
+ tokens = AnalyzerUtils.tokensFromAnalysis( analyzer, "name", text );
+ AnalyzerUtils.assertTokensEqual(
+ tokens,
+ new String[] {
+ "please",
+ "please divide",
+ "divide",
+ "divide this",
+ "this",
+ "this sentence",
+ "sentence",
+ "sentence into",
+ "into",
+ "into shingles",
+ "shingles"
+ }
+ );
+
+ analyzer = fts.getSearchFactory().getAnalyzer( "phonetic_analyzer" );
+ text = "The quick brown fox jumped over the lazy dogs";
+ tokens = AnalyzerUtils.tokensFromAnalysis( analyzer, "name", text );
+ AnalyzerUtils.displayTokens( analyzer, "name", text );
+ AnalyzerUtils.assertTokensEqual(
+ tokens, new String[] { "0", "KK", "BRN", "FKS", "JMPT", "OFR", "0", "LS", "TKS" }
+ );
+
+ fts.close();
+ }
+
protected Class[] getMappings() {
return new Class[] {
Team.class
Modified: search/trunk/src/test/org/hibernate/search/test/analyzer/solr/Team.java
===================================================================
--- search/trunk/src/test/org/hibernate/search/test/analyzer/solr/Team.java 2008-11-11 13:32:00 UTC (rev 15548)
+++ search/trunk/src/test/org/hibernate/search/test/analyzer/solr/Team.java 2008-11-11 20:16:10 UTC (rev 15549)
@@ -5,14 +5,26 @@
import javax.persistence.GeneratedValue;
import javax.persistence.Id;
+import org.apache.solr.analysis.HTMLStripStandardTokenizerFactory;
+import org.apache.solr.analysis.HTMLStripWhitespaceTokenizerFactory;
import org.apache.solr.analysis.ISOLatin1AccentFilterFactory;
+import org.apache.solr.analysis.LengthFilterFactory;
import org.apache.solr.analysis.LowerCaseFilterFactory;
+import org.apache.solr.analysis.LowerCaseTokenizerFactory;
+import org.apache.solr.analysis.PorterStemFilterFactory;
+import org.apache.solr.analysis.ShingleFilterFactory;
import org.apache.solr.analysis.SnowballPorterFilterFactory;
+import org.apache.solr.analysis.StandardFilterFactory;
import org.apache.solr.analysis.StandardTokenizerFactory;
import org.apache.solr.analysis.StopFilterFactory;
+import org.apache.solr.analysis.SynonymFilterFactory;
+import org.apache.solr.analysis.TrimFilterFactory;
+import org.apache.solr.analysis.WordDelimiterFilterFactory;
+import org.apache.solr.analysis.PhoneticFilterFactory;
import org.hibernate.search.annotations.Analyzer;
import org.hibernate.search.annotations.AnalyzerDef;
+import org.hibernate.search.annotations.AnalyzerDefs;
import org.hibernate.search.annotations.DocumentId;
import org.hibernate.search.annotations.Field;
import org.hibernate.search.annotations.Indexed;
@@ -25,20 +37,94 @@
*/
@Entity
@Indexed
- at AnalyzerDef(name = "customanalyzer",
- tokenizer = @TokenizerDef(factory = StandardTokenizerFactory.class),
- filters = {
- @TokenFilterDef(factory = ISOLatin1AccentFilterFactory.class),
- @TokenFilterDef(factory = LowerCaseFilterFactory.class),
- @TokenFilterDef(factory = StopFilterFactory.class, params = {
- @Parameter(name = "words",
- value = "org/hibernate/search/test/analyzer/solr/stoplist.properties"),
- @Parameter(name = "ignoreCase", value = "true")
+ at AnalyzerDefs({
+ @AnalyzerDef(name = "customanalyzer",
+ tokenizer = @TokenizerDef(factory = StandardTokenizerFactory.class),
+ filters = {
+ @TokenFilterDef(factory = ISOLatin1AccentFilterFactory.class),
+ @TokenFilterDef(factory = LowerCaseFilterFactory.class),
+ @TokenFilterDef(factory = StopFilterFactory.class, params = {
+ @Parameter(name = "words",
+ value = "org/hibernate/search/test/analyzer/solr/stoplist.properties"),
+ @Parameter(name = "ignoreCase", value = "true")
+ }),
+ @TokenFilterDef(factory = SnowballPorterFilterFactory.class, params = {
+ @Parameter(name = "language", value = "English")
+ })
}),
- @TokenFilterDef(factory = SnowballPorterFilterFactory.class, params = {
- @Parameter(name = "language", value = "English")
+
+ @AnalyzerDef(name = "standard_analyzer",
+ tokenizer = @TokenizerDef(factory = StandardTokenizerFactory.class),
+ filters = {
+ @TokenFilterDef(factory = StandardFilterFactory.class)
+ }),
+
+ @AnalyzerDef(name = "html_standard_analyzer",
+ tokenizer = @TokenizerDef(factory = HTMLStripStandardTokenizerFactory.class),
+ filters = {
+ @TokenFilterDef(factory = StandardFilterFactory.class)
+ }),
+
+ @AnalyzerDef(name = "html_whitespace_analyzer",
+ tokenizer = @TokenizerDef(factory = HTMLStripWhitespaceTokenizerFactory.class),
+ filters = {
+ @TokenFilterDef(factory = StandardFilterFactory.class)
+ }),
+
+ @AnalyzerDef(name = "trim_analyzer",
+ tokenizer = @TokenizerDef(factory = LowerCaseTokenizerFactory.class),
+ filters = {
+ @TokenFilterDef(factory = InsertWhitespaceFilterFactory.class),
+ @TokenFilterDef(factory = TrimFilterFactory.class)
+ }),
+
+ @AnalyzerDef(name = "length_analyzer",
+ tokenizer = @TokenizerDef(factory = StandardTokenizerFactory.class),
+ filters = {
+ @TokenFilterDef(factory = LengthFilterFactory.class, params = {
+ @Parameter(name = "min", value = "3"),
+ @Parameter(name = "max", value = "5")
+ })
+ }),
+
+ @AnalyzerDef(name = "porter_analyzer",
+ tokenizer = @TokenizerDef(factory = StandardTokenizerFactory.class),
+ filters = {
+ @TokenFilterDef(factory = PorterStemFilterFactory.class)
+ }),
+
+ @AnalyzerDef(name = "word_analyzer",
+ tokenizer = @TokenizerDef(factory = HTMLStripStandardTokenizerFactory.class),
+ filters = {
+ @TokenFilterDef(factory = WordDelimiterFilterFactory.class, params = {
+ @Parameter(name = "splitOnCaseChange", value = "1")
+ })
+ }),
+
+ @AnalyzerDef(name = "synonym_analyzer",
+ tokenizer = @TokenizerDef(factory = HTMLStripStandardTokenizerFactory.class),
+ filters = {
+ @TokenFilterDef(factory = SynonymFilterFactory.class, params = {
+ @Parameter(name = "synonyms",
+ value = "org/hibernate/search/test/analyzer/solr/synonyms.properties")
+ })
+ }),
+
+ @AnalyzerDef(name = "shingle_analyzer",
+ tokenizer = @TokenizerDef(factory = HTMLStripStandardTokenizerFactory.class),
+ filters = {
+ @TokenFilterDef(factory = ShingleFilterFactory.class)
+ }),
+
+ @AnalyzerDef(name = "phonetic_analyzer",
+ tokenizer = @TokenizerDef(factory = StandardTokenizerFactory.class),
+ filters = {
+ @TokenFilterDef(factory = PhoneticFilterFactory.class, params = {
+ @Parameter(name = "encoder", value = "Metaphone"),
+ @Parameter(name = "inject", value = "true")
+ })
})
- })
+})
public class Team {
@Id
@DocumentId
Added: search/trunk/src/test-resources/org/hibernate/search/test/analyzer/solr/synonyms.properties
===================================================================
--- search/trunk/src/test-resources/org/hibernate/search/test/analyzer/solr/synonyms.properties (rev 0)
+++ search/trunk/src/test-resources/org/hibernate/search/test/analyzer/solr/synonyms.properties 2008-11-11 20:16:10 UTC (rev 15549)
@@ -0,0 +1,2 @@
+ipod, i-pod
+universe , cosmos
More information about the hibernate-commits
mailing list