Author: hardy.ferentschik Date: 2008-11-11 07:57:47 -0500 (Tue, 11 Nov 2008) New Revision: 15547 Modified: search/trunk/src/java/org/hibernate/search/backend/Workspace.java search/trunk/src/java/org/hibernate/search/backend/impl/lucene/PerDPQueueProcessor.java search/trunk/src/java/org/hibernate/search/engine/DocumentBuilder.java search/trunk/src/java/org/hibernate/search/store/DirectoryProviderHelper.java search/trunk/src/java/org/hibernate/search/store/RAMDirectoryProvider.java search/trunk/src/test/org/hibernate/search/test/analyzer/AbstractTestAnalyzer.java search/trunk/src/test/org/hibernate/search/test/filter/BestDriversFilter.java search/trunk/src/test/org/hibernate/search/test/filter/ExcludeAllFilter.java search/trunk/src/test/org/hibernate/search/test/filter/InstanceBasedExcludeAllFilter.java search/trunk/src/test/org/hibernate/search/test/jms/master/JMSMasterTest.java search/trunk/src/test/org/hibernate/search/test/reader/performance/IndexFillRunnable.java search/trunk/src/test/org/hibernate/search/test/reader/performance/ReaderPerformance.java search/trunk/src/test/org/hibernate/search/test/util/AnalyzerUtils.java Log: HSEARCH-283 * Updated deprecated field constants * changed IndexWriter constructor Modified: search/trunk/src/java/org/hibernate/search/backend/Workspace.java =================================================================== --- search/trunk/src/java/org/hibernate/search/backend/Workspace.java 2008-11-11 10:14:22 UTC (rev 15546) +++ search/trunk/src/java/org/hibernate/search/backend/Workspace.java 2008-11-11 12:57:47 UTC (rev 15547) @@ -25,11 +25,11 @@ /** * Lucene workspace for a DirectoryProvider.<p/> * <ul> - * <li>Before using getIndexWriter or getIndexReader the lock must be acquired, and resources must be closed - * before releasing the lock.</li> + * <li>Before using {@link #getIndexWriter} or {@link #getIndexReader} the lock must be acquired, + * and resources must be closed before releasing the lock.</li> * <li>One cannot get an IndexWriter when an IndexReader has been acquired and not closed, and vice-versa.</li> - * <li>The recommended approach is to execute all the modifications on the IndexReader, and after that on - * the IndexWriter</li> + * <li>The recommended approach is to execute all the modifications on the <code>IndexReader</code>, and after that on + * the <code>IndexWriter</code></li>. * </ul> * * @author Emmanuel Bernard @@ -173,8 +173,8 @@ return writer; try { // don't care about the Analyzer as it will be selected during usage of IndexWriter. - //FIXME use the non deprecated constructor => requires to call #Commit() - writer = new IndexWriter( directoryProvider.getDirectory(), SIMPLE_ANALYZER, false ); // has been created at init time + IndexWriter.MaxFieldLength fieldLength = new IndexWriter.MaxFieldLength( IndexWriter.DEFAULT_MAX_FIELD_LENGTH ); + writer = new IndexWriter( directoryProvider.getDirectory(), SIMPLE_ANALYZER, false, fieldLength ); // has been created at init time indexingParams.applyToWriter( writer, batchmode ); log.trace( "IndexWriter opened" ); } @@ -186,6 +186,30 @@ } /** + * Commits changes to a previously opened index writer. + * + * @throws SearchException on IOException during Lucene close operation. + * @throws AssertionFailure if there is no IndexWriter to close, or if the lock is not owned. + */ + public synchronized void commitIndexWriter() { + assertOwnLock(); + if ( writer != null ) { + try { + writer.commit(); + log.trace( "Index changes commited." ); + } + catch ( IOException e ) { + throw new SearchException( "Exception while commiting index changes", e ); + } + } + else { + throw new AssertionFailure( "No open IndexWriter to commit changes." ); + } + } + + + + /** * Closes a previously opened IndexWriter. * @throws SearchException on IOException during Lucene close operation. * @throws AssertionFailure if there is no IndexWriter to close, or if the lock is not owned. Modified: search/trunk/src/java/org/hibernate/search/backend/impl/lucene/PerDPQueueProcessor.java =================================================================== --- search/trunk/src/java/org/hibernate/search/backend/impl/lucene/PerDPQueueProcessor.java 2008-11-11 10:14:22 UTC (rev 15546) +++ search/trunk/src/java/org/hibernate/search/backend/impl/lucene/PerDPQueueProcessor.java 2008-11-11 12:57:47 UTC (rev 15547) @@ -103,6 +103,7 @@ for (LuceneWork lw : workOnWriter) { lw.getWorkDelegate( worker ).performWork( lw, indexWriter ); } + workspace.commitIndexWriter(); //TODO next line is assuming the OptimizerStrategy will need an IndexWriter; // would be nicer to have the strategy put an OptimizeWork on the queue, // or just return "yes please" (true) to some method? Modified: search/trunk/src/java/org/hibernate/search/engine/DocumentBuilder.java =================================================================== --- search/trunk/src/java/org/hibernate/search/engine/DocumentBuilder.java 2008-11-11 10:14:22 UTC (rev 15546) +++ search/trunk/src/java/org/hibernate/search/engine/DocumentBuilder.java 2008-11-11 12:57:47 UTC (rev 15547) @@ -583,11 +583,11 @@ case NO: return Field.Index.NO; case NO_NORMS: - return Field.Index.NO_NORMS; + return Field.Index.NOT_ANALYZED_NO_NORMS; case TOKENIZED: - return Field.Index.TOKENIZED; + return Field.Index.ANALYZED; case UN_TOKENIZED: - return Field.Index.UN_TOKENIZED; + return Field.Index.NOT_ANALYZED; default: throw new AssertionFailure( "Unexpected Index: " + index ); } @@ -750,10 +750,10 @@ } { Field classField = - new Field( CLASS_FIELDNAME, entityType.getName(), Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.NO ); + new Field( CLASS_FIELDNAME, entityType.getName(), Field.Store.YES, Field.Index.NOT_ANALYZED, Field.TermVector.NO ); doc.add( classField ); LuceneOptions luceneOptions = new LuceneOptionsImpl( Field.Store.YES, - Field.Index.UN_TOKENIZED, Field.TermVector.NO, idBoost ); + Field.Index.NOT_ANALYZED, Field.TermVector.NO, idBoost ); idBridge.set( idKeywordName, id, doc, luceneOptions ); } buildDocumentFields( instance, doc, rootPropertiesMetadata ); Modified: search/trunk/src/java/org/hibernate/search/store/DirectoryProviderHelper.java =================================================================== --- search/trunk/src/java/org/hibernate/search/store/DirectoryProviderHelper.java 2008-11-11 10:14:22 UTC (rev 15546) +++ search/trunk/src/java/org/hibernate/search/store/DirectoryProviderHelper.java 2008-11-11 12:57:47 UTC (rev 15547) @@ -77,7 +77,8 @@ FSDirectory fsDirectory = FSDirectory.getDirectory( indexDir ); if ( ! IndexReader.indexExists( fsDirectory ) ) { log.debug( "Initialize index: '{}'", indexDir.getAbsolutePath() ); - IndexWriter iw = new IndexWriter( fsDirectory, new StandardAnalyzer(), true ); + IndexWriter.MaxFieldLength fieldLength = new IndexWriter.MaxFieldLength( IndexWriter.DEFAULT_MAX_FIELD_LENGTH ); + IndexWriter iw = new IndexWriter( fsDirectory, new StandardAnalyzer(), true, fieldLength ); iw.close(); } return fsDirectory; Modified: search/trunk/src/java/org/hibernate/search/store/RAMDirectoryProvider.java =================================================================== --- search/trunk/src/java/org/hibernate/search/store/RAMDirectoryProvider.java 2008-11-11 10:14:22 UTC (rev 15546) +++ search/trunk/src/java/org/hibernate/search/store/RAMDirectoryProvider.java 2008-11-11 12:57:47 UTC (rev 15547) @@ -28,7 +28,8 @@ public void start() { directory = new RAMDirectory(); try { - IndexWriter iw = new IndexWriter( directory, new StandardAnalyzer(), true ); + IndexWriter.MaxFieldLength fieldLength = new IndexWriter.MaxFieldLength( IndexWriter.DEFAULT_MAX_FIELD_LENGTH ); + IndexWriter iw = new IndexWriter( directory, new StandardAnalyzer(), true, fieldLength ); iw.close(); } catch (IOException e) { Modified: search/trunk/src/test/org/hibernate/search/test/analyzer/AbstractTestAnalyzer.java =================================================================== --- search/trunk/src/test/org/hibernate/search/test/analyzer/AbstractTestAnalyzer.java 2008-11-11 10:14:22 UTC (rev 15546) +++ search/trunk/src/test/org/hibernate/search/test/analyzer/AbstractTestAnalyzer.java 2008-11-11 12:57:47 UTC (rev 15547) @@ -23,12 +23,17 @@ private class InternalTokenStream extends TokenStream { private int position; - public Token next() throws IOException { - if ( position >= getTokens().length) { + public Token next(final Token reusableToken) throws IOException { + assert reusableToken != null; + if ( position >= getTokens().length ) { return null; } else { - return new Token( getTokens()[position++], 0, 0 ); + reusableToken.clear(); + reusableToken.setTermBuffer( getTokens()[position++] ); + reusableToken.setStartOffset( 0 ); + reusableToken.setEndOffset( 0 ); + return reusableToken; } } } Modified: search/trunk/src/test/org/hibernate/search/test/filter/BestDriversFilter.java =================================================================== --- search/trunk/src/test/org/hibernate/search/test/filter/BestDriversFilter.java 2008-11-11 10:14:22 UTC (rev 15546) +++ search/trunk/src/test/org/hibernate/search/test/filter/BestDriversFilter.java 2008-11-11 12:57:47 UTC (rev 15547) @@ -1,22 +1,23 @@ //$Id$ package org.hibernate.search.test.filter; -import java.util.BitSet; import java.io.IOException; -import org.apache.lucene.search.Filter; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Term; import org.apache.lucene.index.TermDocs; -import org.apache.lucene.index.Term; +import org.apache.lucene.search.DocIdSet; +import org.apache.lucene.search.Filter; +import org.apache.lucene.util.OpenBitSet; /** * @author Emmanuel Bernard */ public class BestDriversFilter extends Filter { - public BitSet bits(IndexReader reader) throws IOException { - BitSet bitSet = new BitSet( reader.maxDoc() ); - TermDocs termDocs = reader.termDocs( new Term("score", "5") ); + public DocIdSet getDocIdSet(IndexReader reader) throws IOException { + OpenBitSet bitSet = new OpenBitSet( reader.maxDoc() ); + TermDocs termDocs = reader.termDocs( new Term( "score", "5" ) ); while ( termDocs.next() ) { bitSet.set( termDocs.doc() ); } Modified: search/trunk/src/test/org/hibernate/search/test/filter/ExcludeAllFilter.java =================================================================== --- search/trunk/src/test/org/hibernate/search/test/filter/ExcludeAllFilter.java 2008-11-11 10:14:22 UTC (rev 15546) +++ search/trunk/src/test/org/hibernate/search/test/filter/ExcludeAllFilter.java 2008-11-11 12:57:47 UTC (rev 15547) @@ -7,6 +7,7 @@ import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.Filter; import org.apache.lucene.index.IndexReader; + import org.hibernate.search.filter.EmptyDocIdBitSet; /** @@ -14,13 +15,15 @@ */ @SuppressWarnings("serial") public class ExcludeAllFilter extends Filter { - - //ugly but useful for test purposes + + // ugly but useful for test purposes private static volatile boolean done = false; @Override public BitSet bits(IndexReader reader) throws IOException { - if ( done ) throw new IllegalStateException( "Called twice" ); + if ( done ) { + throw new IllegalStateException( "Called twice" ); + } BitSet bitSet = new BitSet( reader.maxDoc() ); done = true; return bitSet; @@ -28,9 +31,10 @@ @Override public DocIdSet getDocIdSet(IndexReader reader) throws IOException { - if ( done ) throw new IllegalStateException( "Called twice" ); + if ( done ) { + throw new IllegalStateException( "Called twice" ); + } done = true; return EmptyDocIdBitSet.instance; } - } Modified: search/trunk/src/test/org/hibernate/search/test/filter/InstanceBasedExcludeAllFilter.java =================================================================== --- search/trunk/src/test/org/hibernate/search/test/filter/InstanceBasedExcludeAllFilter.java 2008-11-11 10:14:22 UTC (rev 15546) +++ search/trunk/src/test/org/hibernate/search/test/filter/InstanceBasedExcludeAllFilter.java 2008-11-11 12:57:47 UTC (rev 15547) @@ -5,17 +5,32 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.Filter; +import org.apache.lucene.search.DocIdSet; +import org.hibernate.search.filter.EmptyDocIdBitSet; + /** * @author Emmanuel Bernard */ public class InstanceBasedExcludeAllFilter extends Filter { private volatile boolean done = false; + @Override public BitSet bits(IndexReader reader) throws IOException { - if (done) throw new IllegalStateException("Called twice"); + if ( done ) { + throw new IllegalStateException( "Called twice" ); + } BitSet bitSet = new BitSet( reader.maxDoc() ); done = true; return bitSet; } + + @Override + public DocIdSet getDocIdSet(IndexReader reader) throws IOException { + if ( done ) { + throw new IllegalStateException( "Called twice" ); + } + done = true; + return EmptyDocIdBitSet.instance; + } } Modified: search/trunk/src/test/org/hibernate/search/test/jms/master/JMSMasterTest.java =================================================================== --- search/trunk/src/test/org/hibernate/search/test/jms/master/JMSMasterTest.java 2008-11-11 10:14:22 UTC (rev 15546) +++ search/trunk/src/test/org/hibernate/search/test/jms/master/JMSMasterTest.java 2008-11-11 12:57:47 UTC (rev 15547) @@ -54,11 +54,11 @@ s.close(); //create the work queue to send Document doc = new Document(); - Field field = new Field( DocumentBuilder.CLASS_FIELDNAME, ts.getClass().getName(), Field.Store.YES, Field.Index.UN_TOKENIZED ); + Field field = new Field( DocumentBuilder.CLASS_FIELDNAME, ts.getClass().getName(), Field.Store.YES, Field.Index.NOT_ANALYZED ); doc.add( field ); - field = new Field("id", "1", Field.Store.YES, Field.Index.UN_TOKENIZED ); + field = new Field("id", "1", Field.Store.YES, Field.Index.NOT_ANALYZED ); doc.add( field ); - field = new Field("logo", ts.getLogo(), Field.Store.NO, Field.Index.TOKENIZED ); + field = new Field("logo", ts.getLogo(), Field.Store.NO, Field.Index.ANALYZED ); doc.add( field ); LuceneWork luceneWork = new AddLuceneWork(ts.getId(), String.valueOf( ts.getId() ), ts.getClass(), doc ); List<LuceneWork> queue = new ArrayList<LuceneWork>(); Modified: search/trunk/src/test/org/hibernate/search/test/reader/performance/IndexFillRunnable.java =================================================================== --- search/trunk/src/test/org/hibernate/search/test/reader/performance/IndexFillRunnable.java 2008-11-11 10:14:22 UTC (rev 15546) +++ search/trunk/src/test/org/hibernate/search/test/reader/performance/IndexFillRunnable.java 2008-11-11 12:57:47 UTC (rev 15547) @@ -16,23 +16,29 @@ private volatile int jobSeed = 0; private final IndexWriter iw; - + public IndexFillRunnable(IndexWriter iw) { super(); this.iw = iw; } public void run() { - Field f1 = new Field("name", "Some One " + jobSeed++, Store.NO, Index.TOKENIZED ); - Field f2 = new Field("physicalDescription", " just more people sitting around and filling my index... ", Store.NO, Index.TOKENIZED ); + Field f1 = new Field( "name", "Some One " + jobSeed++, Store.NO, Index.ANALYZED ); + Field f2 = new Field( + "physicalDescription", + " just more people sitting around and filling my index... ", + Store.NO, + Index.ANALYZED + ); Document d = new Document(); d.add( f1 ); d.add( f2 ); try { iw.addDocument( d ); - } catch (IOException e) { + } + catch ( IOException e ) { e.printStackTrace(); } } - + } Modified: search/trunk/src/test/org/hibernate/search/test/reader/performance/ReaderPerformance.java =================================================================== --- search/trunk/src/test/org/hibernate/search/test/reader/performance/ReaderPerformance.java 2008-11-11 10:14:22 UTC (rev 15546) +++ search/trunk/src/test/org/hibernate/search/test/reader/performance/ReaderPerformance.java 2008-11-11 12:57:47 UTC (rev 15547) @@ -55,7 +55,8 @@ private void buildBigIndex() throws InterruptedException, CorruptIndexException, LockObtainFailedException, IOException { System.out.println( "Going to create fake index..." ); FSDirectory directory = FSDirectory.getDirectory(new File(getBaseIndexDir(), Detective.class.getCanonicalName())); - IndexWriter iw = new IndexWriter( directory, new SimpleAnalyzer(), true ); + IndexWriter.MaxFieldLength fieldLength = new IndexWriter.MaxFieldLength( IndexWriter.DEFAULT_MAX_FIELD_LENGTH ); + IndexWriter iw = new IndexWriter( directory, new SimpleAnalyzer(), true, fieldLength ); IndexFillRunnable filler = new IndexFillRunnable( iw ); ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool( WORKER_THREADS ); for (int batch=0; batch<=5000000; batch++){ @@ -63,6 +64,7 @@ } executor.shutdown(); executor.awaitTermination( 600, TimeUnit.SECONDS ); + iw.commit(); iw.optimize(); iw.close(); System.out.println( "Index created." ); Modified: search/trunk/src/test/org/hibernate/search/test/util/AnalyzerUtils.java =================================================================== --- search/trunk/src/test/org/hibernate/search/test/util/AnalyzerUtils.java 2008-11-11 10:14:22 UTC (rev 15546) +++ search/trunk/src/test/org/hibernate/search/test/util/AnalyzerUtils.java 2008-11-11 12:57:47 UTC (rev 15547) @@ -26,23 +26,24 @@ public static Token[] tokensFromAnalysis(Analyzer analyzer, String field, String text) throws IOException { TokenStream stream = analyzer.tokenStream( field, new StringReader( text ) ); List<Token> tokenList = new ArrayList<Token>(); + Token reusableToken = new Token(); while ( true ) { - Token token = stream.next(); + + Token token = stream.next( reusableToken ); if ( token == null ) { break; } - tokenList.add( token ); + tokenList.add( ( Token ) token.clone() ); } - return ( Token[] ) tokenList.toArray( new Token[0] ); + return tokenList.toArray( new Token[tokenList.size()] ); } public static void displayTokens(Analyzer analyzer, String field, String text) throws IOException { Token[] tokens = tokensFromAnalysis( analyzer, field, text ); - for ( int i = 0; i < tokens.length; i++ ) { - Token token = tokens[i]; + for ( Token token : tokens ) { log.debug( "[" + getTermText( token ) + "] " ); } } @@ -52,9 +53,7 @@ int position = 0; - for ( int i = 0; i < tokens.length; i++ ) { - Token token = tokens[i]; - + for ( Token token : tokens ) { int increment = token.getPositionIncrement(); if ( increment > 0 ) { @@ -72,9 +71,7 @@ StringBuilder builder = new StringBuilder(); int position = 0; - for ( int i = 0; i < tokens.length; i++ ) { - Token token = tokens[i]; - + for ( Token token : tokens ) { int increment = token.getPositionIncrement(); if ( increment > 0 ) {