Author: skabashnyuk Date: 2010-05-16 09:49:27 -0400 (Sun, 16 May 2010) New Revision: 2381 Modified: jcr/branches/1.12-LIC-709/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/SearchIndex.java jcr/branches/1.12-LIC-709/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/SingleIndex.java Log: EXOJCR-709 : index initialization Modified: jcr/branches/1.12-LIC-709/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/SearchIndex.java =================================================================== --- jcr/branches/1.12-LIC-709/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/SearchIndex.java 2010-05-14 15:07:42 UTC (rev 2380) +++ jcr/branches/1.12-LIC-709/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/SearchIndex.java 2010-05-16 13:49:27 UTC (rev 2381) @@ -548,12 +548,13 @@ indexingConfig = createIndexingConfiguration(nsMappings); analyzer.setIndexingConfig(indexingConfig); - index = new MultiIndex(this, context.getIndexingTree(), modeHandler, getIndexInfos(), getIndexUpdateMonitor()); + //index = new MultiIndex(this, context.getIndexingTree(), modeHandler, getIndexInfos(), getIndexUpdateMonitor()); - singleIndex = new SingleIndex(this, indexDirectory, analyzer); + singleIndex = new SingleIndex(this, indexDirectory, analyzer , context.getIndexingTree()); + singleIndex.createInitialIndex(context.getItemStateManager(), this); // if RW mode, create initial index and start check - if (modeHandler.getMode() == IndexerIoMode.READ_WRITE) + if (index != null && modeHandler.getMode() == IndexerIoMode.READ_WRITE) { if (index.numDocs() == 0 && context.isCreateInitialIndex()) { @@ -592,19 +593,19 @@ // initialize spell checker spellChecker = createSpellChecker(); - log.info("Index initialized: {} Version: {}", new Object[]{path, index.getIndexFormatVersion()}); - if (!index.getIndexFormatVersion().equals(getIndexFormatVersion())) - { - log.warn("Using Version {} for reading. Please re-index version " + "storage for optimal performance.", - new Integer(getIndexFormatVersion().getVersion())); - } + // log.info("Index initialized: {} Version: {}", new Object[]{path, index.getIndexFormatVersion()}); + // if (!index.getIndexFormatVersion().equals(getIndexFormatVersion())) + // { + // log.warn("Using Version {} for reading. Please re-index version " + "storage for optimal performance.", + // new Integer(getIndexFormatVersion().getVersion())); + // } File file = new File(indexDirectory, ERROR_LOG); errorLog = new ErrorLog(file, errorLogfileSize); // reprocess any notfinished notifies; if (modeHandler.getMode() == IndexerIoMode.READ_WRITE) { - recoverErrorLog(errorLog); + // recoverErrorLog(errorLog); } modeHandler.addIndexerIoModeListener(this); @@ -680,40 +681,40 @@ final Set<String> removedNodeIds = new HashSet<String>(); final Set<String> addedNodeIds = new HashSet<String>(); -// index.update(IteratorUtils.toList(new TransformIterator(remove, new Transformer() -// { -// public Object transform(Object input) -// { -// String uuid = ((String)input); -// removedNodeIds.add(uuid); -// return uuid; -// } -// })), IteratorUtils.toList(new TransformIterator(add, new Transformer() -// { -// public Object transform(Object input) -// { -// NodeData state = (NodeData)input; -// if (state == null) -// { -// return null; -// } -// String uuid = state.getIdentifier(); -// addedNodeIds.add(uuid); -// removedNodeIds.remove(uuid); -// Document doc = null; -// try -// { -// doc = createDocument(state, getNamespaceMappings(), index.getIndexFormatVersion()); -// retrieveAggregateRoot(state, aggregateRoots); -// } -// catch (RepositoryException e) -// { -// log -// .warn("Exception while creating document for node: " + state.getIdentifier() + ": " + e.toString(), e); -// } -// return doc; -// } -// }))); + // index.update(IteratorUtils.toList(new TransformIterator(remove, new Transformer() + // { + // public Object transform(Object input) + // { + // String uuid = ((String)input); + // removedNodeIds.add(uuid); + // return uuid; + // } + // })), IteratorUtils.toList(new TransformIterator(add, new Transformer() + // { + // public Object transform(Object input) + // { + // NodeData state = (NodeData)input; + // if (state == null) + // { + // return null; + // } + // String uuid = state.getIdentifier(); + // addedNodeIds.add(uuid); + // removedNodeIds.remove(uuid); + // Document doc = null; + // try + // { + // doc = createDocument(state, getNamespaceMappings(), index.getIndexFormatVersion()); + // retrieveAggregateRoot(state, aggregateRoots); + // } + // catch (RepositoryException e) + // { + // log + // .warn("Exception while creating document for node: " + state.getIdentifier() + ": " + e.toString(), e); + // } + // return doc; + // } + // }))); singleIndex.update(IteratorUtils.toList(new TransformIterator(remove, new Transformer() { @@ -1146,19 +1147,19 @@ { QueryHandler parentHandler = getContext().getParentHandler(); IndexReader parentReader = null; -// if (parentHandler instanceof SearchIndex && includeSystemIndex) -// { -// parentReader = ((SearchIndex)parentHandler).singleIndex.getIndexReader(); -// } + // if (parentHandler instanceof SearchIndex && includeSystemIndex) + // { + // parentReader = ((SearchIndex)parentHandler).singleIndex.getIndexReader(); + // } -// IndexReader reader; -// if (parentReader != null) -// { -// //CachingMultiIndexReader[] readers = {index.getIndexReader(), parentReader}; -// return singleIndex.createJcrSingleIndexReader(new MultiReader(new IndexReader[] {singleIndex.getIndexReader(), parentReader})); -// // reader = new CombinedIndexReader(readers); -// } -// else + // IndexReader reader; + // if (parentReader != null) + // { + // //CachingMultiIndexReader[] readers = {index.getIndexReader(), parentReader}; + // return singleIndex.createJcrSingleIndexReader(new MultiReader(new IndexReader[] {singleIndex.getIndexReader(), parentReader})); + // // reader = new CombinedIndexReader(readers); + // } + // else { return singleIndex.createJcrSingleIndexReader(singleIndex.getIndexReader()); //reader = index.getIndexReader(); @@ -1226,6 +1227,11 @@ return doc; } + Document createDocument(NodeData node) throws RepositoryException + { + return createDocument(node, nsMappings, IndexFormatVersion.V3); + } + /** * Returns the actual index. * Modified: jcr/branches/1.12-LIC-709/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/SingleIndex.java =================================================================== --- jcr/branches/1.12-LIC-709/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/SingleIndex.java 2010-05-14 15:07:42 UTC (rev 2380) +++ jcr/branches/1.12-LIC-709/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/SingleIndex.java 2010-05-16 13:49:27 UTC (rev 2381) @@ -27,6 +27,12 @@ import org.apache.lucene.index.TermDocs; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.RAMDirectory; +import org.exoplatform.services.jcr.dataflow.ItemDataConsumer; +import org.exoplatform.services.jcr.datamodel.NodeData; +import org.exoplatform.services.jcr.impl.core.query.IndexingTree; + +import org.exoplatform.services.jcr.util.IdGenerator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,7 +40,12 @@ import java.io.IOException; import java.util.Collection; import java.util.Iterator; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import javax.jcr.ItemNotFoundException; +import javax.jcr.RepositoryException; + /** * This class implements basic functionality of single-index JCR search index. * @@ -63,12 +74,16 @@ private final DocNumberCache cache; + private final IndexingTree indexingTree; + /** * Creates instance of SingleIndex class * @throws IOException */ - public SingleIndex(SearchIndex handler, File indexDirectory, Analyzer analyzer) throws IOException + public SingleIndex(SearchIndex handler, File indexDirectory, Analyzer analyzer, IndexingTree indexingTree) + throws IOException { + this.indexingTree = indexingTree; String path = indexDirectory.getAbsolutePath() + "_new"; File indexDirectory2 = new File(path); if (!indexDirectory2.exists()) @@ -78,7 +93,11 @@ log.error("DIRECTORY NOT CREATED!!!!!!!"); } } - this.directory = FSDirectory.getDirectory(indexDirectory2); + //this.directory = new RAMDirectory(); + String dir = indexDirectory2 + IdGenerator.generate().toString(); + this.directory = FSDirectory.getDirectory(dir); + log.info(dir); + //FSDirectory.getDirectory(indexDirectory2); this.writer = new IndexWriter(directory, analyzer, IndexWriter.MaxFieldLength.UNLIMITED); this.reader = IndexReader.open(directory, true); this.cache = new DocNumberCache(handler.getCacheSize()); @@ -191,10 +210,94 @@ return parents; } } - - + public IndexFormatVersion getIndexFormatVersion() { return IndexFormatVersion.getVersion(reader); } + + /** + * Creates an initial index by traversing the node hierarchy starting at the + * node with <code>rootId</code>. + * @param searchIndex + * @throws RepositoryException + * @throws IOException + */ + public void createInitialIndex(ItemDataConsumer itemStateManager, SearchIndex searchIndex) throws IOException, + RepositoryException + { + long count = 0; + // traverse and index workspace + //executeAndLog(new Start(Action.INTERNAL_TRANSACTION)); + // NodeData rootState = (NodeData) stateMgr.getItemData(rootId); + count = createIndex(indexingTree.getIndexingRoot(), itemStateManager, count, searchIndex); + //executeAndLog(new Commit(getTransactionId())); + writer.commit(); + log.info("Created initial index for {} nodes", new Long(count)); + + } + + /** + * Recursively creates an index starting with the NodeState + * <code>node</code>. + * + * @param node + * the current NodeState. + * @param path + * the path of the current node. + * @param stateMgr + * the shared item state manager. + * @param count + * the number of nodes already indexed. + * @param searchIndex + * @return the number of nodes indexed so far. + * @throws IOException + * if an error occurs while writing to the index. + * @throws ItemStateException + * if an node state cannot be found. + * @throws RepositoryException + * if any other error occurs + */ + private long createIndex(NodeData node, ItemDataConsumer stateMgr, long count, SearchIndex searchIndex) + throws IOException, RepositoryException + { + // NodeId id = node.getNodeId(); + + if (indexingTree.isExcluded(node)) + { + return count; + } + //executeAndLog(new AddNode(getTransactionId(), node.getIdentifier())); + writer.addDocument(searchIndex.createDocument(node)); + if (++count % 100 == 0) + { + + log.info("indexing... {} ({})", node.getQPath().getAsString(), new Long(count)); + } + if (count % 1000 == 0) + { + writer.commit(); + } + //checkVolatileCommit(); + List<NodeData> children = stateMgr.getChildNodesData(node); + for (NodeData nodeData : children) + { + + NodeData childState = (NodeData)stateMgr.getItemData(nodeData.getIdentifier()); + if (childState == null) + { + //TODO + // handler.getOnWorkspaceInconsistencyHandler().handleMissingChildNode( + // new ItemNotFoundException("Child not found "), handler, nodeData.getQPath(), node, nodeData); + } + + if (nodeData != null) + { + count = createIndex(nodeData, stateMgr, count, searchIndex); + } + } + + return count; + } + }