[exo-jcr-commits] exo-jcr SVN: r2380 - in jcr/branches/1.12-LIC-709/exo.jcr.component.core/src: test/java/org/exoplatform/services/jcr/api/core/query/lucene and 1 other directory.

Author: nzamosenchuk Date: 2010-05-14 11:07:42 -0400 (Fri, 14 May 2010) New Revision: 2380 Added: jcr/branches/1.12-LIC-709/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/SingleIndex.java Modified: jcr/branches/1.12-LIC-709/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/SearchIndex.java jcr/branches/1.12-LIC-709/exo.jcr.component.core/src/test/java/org/exoplatform/services/jcr/api/core/query/lucene/IndexingAggregateTest.java Log: EXOJCR-709: first approaches of making POC Modified: jcr/branches/1.12-LIC-709/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/SearchIndex.java =================================================================== --- jcr/branches/1.12-LIC-709/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/SearchIndex.java 2010-05-13 13:47:26 UTC (rev 2379) +++ jcr/branches/1.12-LIC-709/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/SearchIndex.java 2010-05-14 15:07:42 UTC (rev 2380) @@ -129,6 +129,7 @@ * calculated as follows: 2 * * Runtime.getRuntime().availableProcessors(). */ + @Deprecated public static final int DEFAULT_EXTRACTOR_POOL_SIZE = 0; /** @@ -173,6 +174,11 @@ private MultiIndex index; /** + * Single actual index + */ + private SingleIndex singleIndex; + + /** * The analyzer we use for indexing. */ private JcrStandartAnalyzer analyzer; @@ -473,6 +479,7 @@ * if an error occurs while initializing this handler. * @throws RepositoryException */ + @Override public void doInit() throws IOException, RepositoryException { QueryHandlerContext context = getContext(); @@ -542,6 +549,9 @@ analyzer.setIndexingConfig(indexingConfig); index = new MultiIndex(this, context.getIndexingTree(), modeHandler, getIndexInfos(), getIndexUpdateMonitor()); + + singleIndex = new SingleIndex(this, indexDirectory, analyzer); + // if RW mode, create initial index and start check if (modeHandler.getMode() == IndexerIoMode.READ_WRITE) { @@ -670,7 +680,42 @@ final Set<String> removedNodeIds = new HashSet<String>(); final Set<String> addedNodeIds = new HashSet<String>(); - index.update(IteratorUtils.toList(new TransformIterator(remove, new Transformer() +// index.update(IteratorUtils.toList(new TransformIterator(remove, new Transformer() +// { +// public Object transform(Object input) +// { +// String uuid = ((String)input); +// removedNodeIds.add(uuid); +// return uuid; +// } +// })), IteratorUtils.toList(new TransformIterator(add, new Transformer() +// { +// public Object transform(Object input) +// { +// NodeData state = (NodeData)input; +// if (state == null) +// { +// return null; +// } +// String uuid = state.getIdentifier(); +// addedNodeIds.add(uuid); +// removedNodeIds.remove(uuid); +// Document doc = null; +// try +// { +// doc = createDocument(state, getNamespaceMappings(), index.getIndexFormatVersion()); +// retrieveAggregateRoot(state, aggregateRoots); +// } +// catch (RepositoryException e) +// { +// log +// .warn("Exception while creating document for node: " + state.getIdentifier() + ": " + e.toString(), e); +// } +// return doc; +// } +// }))); + + singleIndex.update(IteratorUtils.toList(new TransformIterator(remove, new Transformer() { public Object transform(Object input) { @@ -693,7 +738,7 @@ Document doc = null; try { - doc = createDocument(state, getNamespaceMappings(), index.getIndexFormatVersion()); + doc = createDocument(state, getNamespaceMappings(), singleIndex.getIndexFormatVersion()); retrieveAggregateRoot(state, aggregateRoots); } catch (RepositoryException e) @@ -703,7 +748,7 @@ } return doc; } - }))); + })));; // remove any aggregateRoot nodes that are new // and therefore already up-to-date @@ -722,7 +767,7 @@ NodeData state = (NodeData)input; try { - return createDocument(state, getNamespaceMappings(), index.getIndexFormatVersion()); + return createDocument(state, getNamespaceMappings(), singleIndex.getIndexFormatVersion()); } catch (RepositoryException e) { @@ -733,7 +778,7 @@ } }); modified.addAll(aggregateRoots.values()); - index.update(aggregateRoots.keySet(), modified); + singleIndex.update(aggregateRoots.keySet(), modified); } } @@ -824,7 +869,8 @@ { spellChecker.close(); } - index.close(); + //index.close(); + singleIndex.close(); getContext().destroy(); closed = true; log.info("Index closed: " + path); @@ -865,6 +911,7 @@ return new FilterMultiColumnQueryHits(searcher.execute(query, sort, resultFetchHint, QueryImpl.DEFAULT_SELECTOR_NAME)) { + @Override public void close() throws IOException { try @@ -912,6 +959,7 @@ searcher.setSimilarity(getSimilarity()); return new FilterMultiColumnQueryHits(query.execute(searcher, sort, resultFetchHint)) { + @Override public void close() throws IOException { try @@ -1057,18 +1105,18 @@ if (getContext().getParentHandler() instanceof SearchIndex) { SearchIndex parent = (SearchIndex)getContext().getParentHandler(); - if (parent.getIndexFormatVersion().getVersion() < index.getIndexFormatVersion().getVersion()) + if (parent.getIndexFormatVersion().getVersion() < singleIndex.getIndexFormatVersion().getVersion()) { indexFormatVersion = parent.getIndexFormatVersion(); } else { - indexFormatVersion = index.getIndexFormatVersion(); + indexFormatVersion = singleIndex.getIndexFormatVersion(); } } else { - indexFormatVersion = index.getIndexFormatVersion(); + indexFormatVersion = singleIndex.getIndexFormatVersion(); } } return indexFormatVersion; @@ -1097,23 +1145,25 @@ protected IndexReader getIndexReader(boolean includeSystemIndex) throws IOException { QueryHandler parentHandler = getContext().getParentHandler(); - CachingMultiIndexReader parentReader = null; - if (parentHandler instanceof SearchIndex && includeSystemIndex) - { - parentReader = ((SearchIndex)parentHandler).index.getIndexReader(); - } + IndexReader parentReader = null; +// if (parentHandler instanceof SearchIndex && includeSystemIndex) +// { +// parentReader = ((SearchIndex)parentHandler).singleIndex.getIndexReader(); +// } - IndexReader reader; - if (parentReader != null) +// IndexReader reader; +// if (parentReader != null) +// { +// //CachingMultiIndexReader[] readers = {index.getIndexReader(), parentReader}; +// return singleIndex.createJcrSingleIndexReader(new MultiReader(new IndexReader[] {singleIndex.getIndexReader(), parentReader})); +// // reader = new CombinedIndexReader(readers); +// } +// else { - CachingMultiIndexReader[] readers = {index.getIndexReader(), parentReader}; - reader = new CombinedIndexReader(readers); + return singleIndex.createJcrSingleIndexReader(singleIndex.getIndexReader()); + //reader = index.getIndexReader(); } - else - { - reader = index.getIndexReader(); - } - return new JcrIndexReader(reader); + //return new JcrIndexReader(reader); } /** @@ -1181,9 +1231,9 @@ * * @return the actual index. */ - public MultiIndex getIndex() + public SingleIndex getIndex() { - return index; + return singleIndex; } /** @@ -1457,7 +1507,7 @@ for (int j = 0; j < aggregates.length; j++) { Document aDoc = - createDocument(aggregates[j], getNamespaceMappings(), index.getIndexFormatVersion()); + createDocument(aggregates[j], getNamespaceMappings(), singleIndex.getIndexFormatVersion()); // transfer fields to doc if there are any Fieldable[] fulltextFields = aDoc.getFieldables(FieldNames.FULLTEXT); if (fulltextFields != null) @@ -1638,7 +1688,7 @@ long time = System.currentTimeMillis(); try { - CachingMultiIndexReader reader = index.getIndexReader(); + IndexReader reader = singleIndex.getIndexReader(); try { Term aggregateUUIDs = new Term(FieldNames.AGGREGATED_NODE_UUID, ""); @@ -1676,7 +1726,7 @@ } finally { - reader.release(); + reader.decRef(); } } catch (Exception e) @@ -1798,6 +1848,7 @@ return hi; } + @Override public boolean equals(Object obj) { if (obj instanceof CombinedIndexReader) @@ -1808,6 +1859,7 @@ return false; } + @Override public int hashCode() { int hash = 0; Added: jcr/branches/1.12-LIC-709/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/SingleIndex.java =================================================================== --- jcr/branches/1.12-LIC-709/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/SingleIndex.java (rev 0) +++ jcr/branches/1.12-LIC-709/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/SingleIndex.java 2010-05-14 15:07:42 UTC (rev 2380) @@ -0,0 +1,200 @@ +/* + * Copyright (C) 2010 eXo Platform SAS. + * + * This is free software; you can redistribute it and/or modify it + * under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation; either version 2.1 of + * the License, or (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this software; if not, write to the Free + * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA + * 02110-1301 USA, or see the FSF site: http://www.fsf.org. + */ +package org.exoplatform.services.jcr.impl.core.query.lucene; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.FilterIndexReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermDocs; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.Collection; +import java.util.Iterator; + +/** + * This class implements basic functionality of single-index JCR search index. + * + * @author <a href="mailto:nikolazius@gmail.com">Nikolay Zamosenchuk</a> + * @version $Id: SingleIndex.java 34360 2009-07-22 23:58:59Z nzamosenchuk $ + * + */ +public class SingleIndex +{ + private static final Logger log = LoggerFactory.getLogger("exo.jcr.component.core.SingleIndex"); + + /** + * Index writer instance + */ + private IndexWriter writer; + + /** + * Index directory instance + */ + private Directory directory; + + /** + * Index reader instance + */ + private IndexReader reader; + + private final DocNumberCache cache; + + /** + * Creates instance of SingleIndex class + * @throws IOException + */ + public SingleIndex(SearchIndex handler, File indexDirectory, Analyzer analyzer) throws IOException + { + String path = indexDirectory.getAbsolutePath() + "_new"; + File indexDirectory2 = new File(path); + if (!indexDirectory2.exists()) + { + if (!indexDirectory2.mkdirs()) + { + log.error("DIRECTORY NOT CREATED!!!!!!!"); + } + } + this.directory = FSDirectory.getDirectory(indexDirectory2); + this.writer = new IndexWriter(directory, analyzer, IndexWriter.MaxFieldLength.UNLIMITED); + this.reader = IndexReader.open(directory, true); + this.cache = new DocNumberCache(handler.getCacheSize()); + + log.info("/!\\ \t\t Initialized on path:" + indexDirectory.getPath()); + } + + /** + * Closes index + */ + public void close() + { + try + { + writer.close(); + reader.close(); + } + catch (IOException e) + { + // TODO Auto-generated catch block + log.error(e.getMessage()); + } + } + + /** + * Updates index by removing and adding documents + * + * @param remove + * @param add + * @throws IOException + */ + public synchronized void update(Collection remove, Collection add) throws IOException + { + for (Iterator it = remove.iterator(); it.hasNext();) + { + Term idTerm = new Term(FieldNames.UUID, (String)it.next()); + writer.deleteDocuments(idTerm); + } + for (Iterator it = add.iterator(); it.hasNext();) + { + Document doc = (Document)it.next(); + if (doc != null) + { + writer.addDocument(doc); + } + } + writer.commit(); + } + + /** + * Returns up2date index reader + * + * @return + * @throws IOException + */ + public IndexReader getIndexReader() throws IOException + { + if (!reader.isCurrent()) + { + IndexReader oldReader = reader; + reader = reader.reopen(); + if (reader != oldReader) + { + oldReader.close(); + } + } + reader.incRef(); + return reader; + } + + public IndexReader createJcrSingleIndexReader(IndexReader reader) throws IOException + { + return new JcrSingleIndexReader(reader); + } + + private class JcrSingleIndexReader extends FilterIndexReader implements HierarchyResolver + { + /** + * @param reader + */ + public JcrSingleIndexReader(IndexReader reader) + { + super(reader); + } + + /** + * @see org.exoplatform.services.jcr.impl.core.query.lucene.HierarchyResolver#getParents(int, int[]) + */ + public int[] getParents(int n, int[] docNumbers) throws IOException + { + Document doc = document(n, FieldSelectors.UUID_AND_PARENT); + String[] parentUUIDs = doc.getValues(FieldNames.PARENT); + + Term id = new Term(FieldNames.UUID, parentUUIDs[0]); + TermDocs docs = termDocs(id); + int[] parents; + if (docs.next()) + { + parents = new int[]{docs.doc()}; + if (docs.next()) + { + log.warn("/!\\ \t\t MULTIPLE PARENTS DETECTED!!! "); + } + } + else + { + parents = new int[0]; + } + docs.close(); + return parents; + } + } + + + public IndexFormatVersion getIndexFormatVersion() + { + return IndexFormatVersion.getVersion(reader); + } +} Property changes on: jcr/branches/1.12-LIC-709/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/SingleIndex.java ___________________________________________________________________ Name: svn:mime-type + text/plain Modified: jcr/branches/1.12-LIC-709/exo.jcr.component.core/src/test/java/org/exoplatform/services/jcr/api/core/query/lucene/IndexingAggregateTest.java =================================================================== --- jcr/branches/1.12-LIC-709/exo.jcr.component.core/src/test/java/org/exoplatform/services/jcr/api/core/query/lucene/IndexingAggregateTest.java 2010-05-13 13:47:26 UTC (rev 2379) +++ jcr/branches/1.12-LIC-709/exo.jcr.component.core/src/test/java/org/exoplatform/services/jcr/api/core/query/lucene/IndexingAggregateTest.java 2010-05-14 15:07:42 UTC (rev 2380) @@ -17,23 +17,21 @@ package org.exoplatform.services.jcr.api.core.query.lucene; import org.exoplatform.services.jcr.api.core.query.AbstractIndexingTest; -import org.exoplatform.services.jcr.impl.core.query.lucene.IndexingQueue; -import org.exoplatform.services.jcr.impl.core.query.lucene.SearchIndex; -import javax.jcr.RepositoryException; -import javax.jcr.Node; -import javax.jcr.query.Query; - +import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; -import java.io.Writer; import java.io.OutputStreamWriter; -import java.io.ByteArrayInputStream; -import java.util.Calendar; -import java.util.List; +import java.io.Writer; import java.util.ArrayList; +import java.util.Calendar; import java.util.Collections; import java.util.Iterator; +import java.util.List; +import javax.jcr.Node; +import javax.jcr.RepositoryException; +import javax.jcr.query.Query; + /** * <code>IndexingAggregateTest</code> checks if the nt:file nt:resource * aggregate defined in workspace indexing-test works properly. @@ -104,16 +102,16 @@ protected void waitUntilQueueEmpty() throws Exception { - SearchIndex index = (SearchIndex)getQueryHandler(); - IndexingQueue queue = index.getIndex().getIndexingQueue(); - index.getIndex().flush(); - synchronized (index.getIndex()) - { - while (queue.getNumPendingDocuments() > 0) - { - index.getIndex().wait(50); - } - } +// SearchIndex index = (SearchIndex)getQueryHandler(); +// IndexingQueue queue = index.getIndex().getIndexingQueue(); +// index.getIndex().flush(); +// synchronized (index.getIndex()) +// { +// while (queue.getNumPendingDocuments() > 0) +// { +// index.getIndex().wait(50); +// } +// } } public void testContentLastModified() throws RepositoryException