[exo-jcr-commits] exo-jcr SVN: r3292 - in jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query: jbosscache and 1 other directories.
do-not-reply at jboss.org
do-not-reply at jboss.org
Tue Oct 12 05:47:34 EDT 2010
Author: nzamosenchuk
Date: 2010-10-12 05:47:34 -0400 (Tue, 12 Oct 2010)
New Revision: 3292
Added:
jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/ChunkIndex.java
Removed:
jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/MultiIndex.java
Modified:
jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/AbstractQueryHandler.java
jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/QueryHandler.java
jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/jbosscache/JBossCacheIndexInfos.java
jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/ConsistencyCheck.java
jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/IndexInfos.java
jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/IndexingQueue.java
jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/SearchIndex.java
Log:
EXOJCR-987 : Cleaning up the code, simplifying. Refactoring.
Modified: jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/AbstractQueryHandler.java
===================================================================
--- jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/AbstractQueryHandler.java 2010-10-11 16:33:49 UTC (rev 3291)
+++ jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/AbstractQueryHandler.java 2010-10-12 09:47:34 UTC (rev 3292)
@@ -21,7 +21,7 @@
import org.exoplatform.services.jcr.impl.core.query.lucene.DefaultIndexUpdateMonitor;
import org.exoplatform.services.jcr.impl.core.query.lucene.IndexInfos;
import org.exoplatform.services.jcr.impl.core.query.lucene.IndexUpdateMonitor;
-import org.exoplatform.services.jcr.impl.core.query.lucene.MultiIndex;
+import org.exoplatform.services.jcr.impl.core.query.lucene.ChunkIndex;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -70,7 +70,7 @@
protected IndexerIoModeHandler modeHandler;
/**
- * {@link IndexInfos} instance that is passed to {@link MultiIndex}
+ * {@link IndexInfos} instance that is passed to {@link ChunkIndex}
*/
protected IndexInfos indexInfos;
Modified: jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/QueryHandler.java
===================================================================
--- jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/QueryHandler.java 2010-10-11 16:33:49 UTC (rev 3291)
+++ jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/QueryHandler.java 2010-10-12 09:47:34 UTC (rev 3292)
@@ -23,7 +23,7 @@
import org.exoplatform.services.jcr.impl.core.SessionImpl;
import org.exoplatform.services.jcr.impl.core.query.lucene.IndexInfos;
import org.exoplatform.services.jcr.impl.core.query.lucene.IndexUpdateMonitor;
-import org.exoplatform.services.jcr.impl.core.query.lucene.MultiIndex;
+import org.exoplatform.services.jcr.impl.core.query.lucene.ChunkIndex;
import org.exoplatform.services.jcr.impl.core.query.lucene.QueryHits;
import java.io.IOException;
@@ -156,7 +156,7 @@
QueryHits executeQuery(Query query) throws IOException;
/**
- * Sets {@link IndexInfos} instance into QueryHandler, which is later passed to {@link MultiIndex}.
+ * Sets {@link IndexInfos} instance into QueryHandler, which is later passed to {@link ChunkIndex}.
*
* @param indexInfos
*/
Modified: jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/jbosscache/JBossCacheIndexInfos.java
===================================================================
--- jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/jbosscache/JBossCacheIndexInfos.java 2010-10-11 16:33:49 UTC (rev 3291)
+++ jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/jbosscache/JBossCacheIndexInfos.java 2010-10-12 09:47:34 UTC (rev 3292)
@@ -22,7 +22,7 @@
import org.exoplatform.services.jcr.impl.core.query.IndexerIoModeHandler;
import org.exoplatform.services.jcr.impl.core.query.IndexerIoModeListener;
import org.exoplatform.services.jcr.impl.core.query.lucene.IndexInfos;
-import org.exoplatform.services.jcr.impl.core.query.lucene.MultiIndex;
+import org.exoplatform.services.jcr.impl.core.query.lucene.ChunkIndex;
import org.exoplatform.services.jcr.impl.util.io.PrivilegedCacheHelper;
import org.exoplatform.services.log.ExoLogger;
import org.exoplatform.services.log.Log;
Copied: jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/ChunkIndex.java (from rev 3288, jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/MultiIndex.java)
===================================================================
--- jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/ChunkIndex.java (rev 0)
+++ jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/ChunkIndex.java 2010-10-12 09:47:34 UTC (rev 3292)
@@ -0,0 +1,795 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.exoplatform.services.jcr.impl.core.query.lucene;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.store.Directory;
+import org.exoplatform.services.jcr.dataflow.ItemDataConsumer;
+import org.exoplatform.services.jcr.datamodel.ItemData;
+import org.exoplatform.services.jcr.datamodel.NodeData;
+import org.exoplatform.services.jcr.impl.core.query.IndexerIoMode;
+import org.exoplatform.services.jcr.impl.core.query.IndexerIoModeHandler;
+import org.exoplatform.services.jcr.impl.core.query.IndexingTree;
+import org.exoplatform.services.jcr.impl.core.query.lucene.directory.DirectoryManager;
+import org.exoplatform.services.jcr.impl.util.SecurityHelper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.Map.Entry;
+
+import javax.jcr.ItemNotFoundException;
+import javax.jcr.RepositoryException;
+
+/**
+ * TODO REWRITE JAVADOC!!!
+ */
+public class ChunkIndex
+{
+
+ /**
+ * The logger instance for this class
+ */
+ private static final Logger log = LoggerFactory.getLogger("exo.jcr.component.core.MultiIndex");
+
+ /**
+ * Names of active persistent index directories.
+ */
+ private IndexInfos indexNames;
+
+ /**
+ * List of open persistent indexes. This list may also contain an open
+ * PersistentIndex owned by the IndexMerger daemon. Such an index is not
+ * registered with indexNames and <b>must not</b> be used in regular index
+ * operations (delete node, etc.)!
+ */
+ private final List<PersistentIndex> indexes = new ArrayList<PersistentIndex>();
+
+ /**
+ * The internal namespace mappings of the query manager.
+ */
+ private final NamespaceMappings nsMappings;
+
+ /**
+ * The directory manager.
+ */
+ private final DirectoryManager directoryManager;
+
+ /**
+ * The base directory to store the index.
+ */
+ private final Directory indexDir;
+
+ /**
+ * The query handler
+ */
+ private final SearchIndex handler;
+
+ /**
+ * If not <code>null</code> points to a valid <code>IndexReader</code> that
+ * reads from all indexes, including volatile and persistent indexes.
+ */
+ private CachingMultiIndexReader multiReader;
+
+ /**
+ * Shared document number cache across all persistent indexes.
+ */
+ private final DocNumberCache cache;
+
+ /**
+ * The indexing queue with pending text extraction jobs.
+ */
+ private IndexingQueue indexingQueue;
+
+ /**
+ * Set<NodeId> of uuids that should not be indexed.
+ */
+ private final IndexingTree indexingTree;
+
+ /**
+ * Flag indicating whether re-indexing is running.
+ */
+ private boolean reindexing = false;
+
+ /**
+ * The index format version of this multi index.
+ */
+ private final IndexFormatVersion version;
+
+ /**
+ * The handler of the Indexer io mode
+ */
+ private final IndexerIoModeHandler modeHandler;
+
+ /**
+ * Creates a new MultiIndex.
+ *
+ * @param handler
+ * the search handler
+ * @param excludedIDs
+ * Set<NodeId> that contains uuids that should not be indexed
+ * nor further traversed.
+ * @throws IOException
+ * if an error occurs
+ */
+ ChunkIndex(SearchIndex handler, IndexingTree indexingTree, IndexerIoModeHandler modeHandler, IndexInfos indexInfos,
+ IndexUpdateMonitor indexUpdateMonitor) throws IOException
+ {
+ this.modeHandler = modeHandler;
+ this.directoryManager = handler.getDirectoryManager();
+ // this method is run in privileged mode internally
+ this.indexDir = directoryManager.getDirectory(".");
+ this.handler = handler;
+ this.cache = new DocNumberCache(handler.getCacheSize());
+ this.indexingTree = indexingTree;
+ this.nsMappings = handler.getNamespaceMappings();
+ this.indexNames = indexInfos;
+ this.indexNames.setDirectory(indexDir);
+ // this method is run in privileged mode internally
+ this.indexNames.read();
+
+ // TODO remove hard-coded piece
+ this.indexNames.addName("1");
+ this.indexNames.addName("2");
+ this.indexNames.addName("3");
+ this.indexNames.addName("4");
+
+ // this method is run in privileged mode internally
+ IndexingQueueStore store = new IndexingQueueStore(indexDir);
+
+ // initialize indexing queue
+ this.indexingQueue = new IndexingQueue(store);
+ // copy current index names
+ Set<String> currentNames = new HashSet<String>(indexNames.getNames());
+
+ // open persistent indexes
+ for (String name : currentNames)
+ {
+ // only open if it still exists
+ // it is possible that indexNames still contains a name for
+ // an index that has been deleted, but indexNames has not been
+ // written to disk.
+
+ // TODO THIS CHECK WAS SKIPPED
+ // if (!directoryManager.hasDirectory(name))
+ // {
+ // log.debug("index does not exist anymore: " + name);
+ // // move on to next index
+ // continue;
+ // }
+ PersistentIndex index =
+ new PersistentIndex(name, handler.getTextAnalyzer(), handler.getSimilarity(), cache, indexingQueue,
+ directoryManager);
+ index.setMaxFieldLength(handler.getMaxFieldLength());
+ index.setUseCompoundFile(handler.getUseCompoundFile());
+ index.setTermInfosIndexDivisor(handler.getTermInfosIndexDivisor());
+ indexes.add(index);
+ }
+
+ // set index format version and at the same time
+ // initialize hierarchy cache if requested.
+ final CachingMultiIndexReader reader = getIndexReader(handler.isInitializeHierarchyCache());
+ try
+ {
+ version = IndexFormatVersion.getVersion(reader);
+ }
+ finally
+ {
+ SecurityHelper.doPriviledgedIOExceptionAction(new PrivilegedExceptionAction<Object>()
+ {
+ public Object run() throws Exception
+ {
+ reader.release();
+ return null;
+ }
+ });
+ }
+ indexingQueue.initialize(this);
+ this.indexNames.setMultiIndex(this);
+ }
+
+ /**
+ * Returns the number of documents in this index.
+ *
+ * @return the number of documents in this index.
+ * @throws IOException
+ * if an error occurs while reading from the index.
+ */
+ int numDocs() throws IOException
+ {
+ if (indexNames.size() == 0)
+ {
+ return 0;
+ }
+ else
+ {
+ final CachingMultiIndexReader reader = getIndexReader();
+ try
+ {
+ return reader.numDocs();
+ }
+ finally
+ {
+ SecurityHelper.doPriviledgedIOExceptionAction(new PrivilegedExceptionAction<Object>()
+ {
+ public Object run() throws Exception
+ {
+ reader.release();
+ return null;
+ }
+ });
+ }
+ }
+ }
+
+ /**
+ * @return the index format version for this multi index.
+ */
+ IndexFormatVersion getIndexFormatVersion()
+ {
+ return version;
+ }
+
+ /**
+ * Creates an initial index by traversing the node hierarchy starting at the
+ * node with <code>rootId</code>.
+ *
+ * @param stateMgr
+ * the item state manager.
+ * @param rootId
+ * the id of the node from where to start.
+ * @param rootPath
+ * the path of the node from where to start.
+ * @throws IOException
+ * if an error occurs while indexing the workspace.
+ * @throws IllegalStateException
+ * if this index is not empty.
+ */
+ void createInitialIndex(ItemDataConsumer stateMgr) throws IOException
+ {
+ // TODO: re-study check!!!
+ CachingMultiIndexReader reader = getIndexReader();
+ int numDocs = reader.numDocs();
+ reader.release();
+ if (numDocs == 0)
+ {
+ reindexing = true;
+ try
+ {
+ long count = 0;
+ // traverse and index workspace
+
+ // NodeData rootState = (NodeData) stateMgr.getItemData(rootId);
+ count = createIndex(indexingTree.getIndexingRoot(), stateMgr, count);
+
+ log.info("Created initial index for {} nodes", new Long(count));
+ releaseMultiReader();
+ }
+ catch (Exception e)
+ {
+ String msg = "Error indexing workspace";
+ IOException ex = new IOException(msg);
+ ex.initCause(e);
+ throw ex;
+ }
+ finally
+ {
+ reindexing = false;
+ }
+ }
+ else
+ {
+ throw new IllegalStateException("Index already present");
+ }
+ }
+
+ /**
+ * Atomically updates the index by removing some documents and adding
+ * others.
+ *
+ * @param remove
+ * collection of <code>UUID</code>s that identify documents to
+ * remove
+ * @param add
+ * collection of <code>Document</code>s to add. Some of the
+ * elements in this collection may be <code>null</code>, to
+ * indicate that a node could not be indexed successfully.
+ * @throws IOException
+ * if an error occurs while updating the index.
+ */
+ synchronized void update(final Collection<String> remove, final Collection<Document> add) throws IOException
+ {
+ SecurityHelper.doPriviledgedIOExceptionAction(new PrivilegedExceptionAction<Object>()
+ {
+ public Object run() throws Exception
+ {
+ // make sure a reader is available during long updates
+ if (add.size() > handler.getBufferSize())
+ {
+ try
+ {
+ getIndexReader().release();
+ }
+ catch (IOException e)
+ {
+ // do not fail if an exception is thrown here
+ log.warn("unable to prepare index reader " + "for queries during update", e);
+ }
+ }
+ try
+ {
+ for (Iterator<String> it = remove.iterator(); it.hasNext();)
+ {
+ String uuidString = it.next();
+ // check if indexing queue is still working on
+ // this node from a previous update
+ Document doc = indexingQueue.removeDocument(uuidString);
+ if (doc != null)
+ {
+ Util.disposeDocument(doc);
+ }
+ Term idTerm = new Term(FieldNames.UUID, uuidString);
+ getChunk(uuidString).removeDocument(idTerm);
+ }
+ for (Iterator<Document> it = add.iterator(); it.hasNext();)
+ {
+ Document doc = it.next();
+ if (doc != null)
+ {
+ if (doc != null)
+ {
+ String uuid = doc.get(FieldNames.UUID);
+ getChunk(uuid).addDocuments(new Document[]{doc});
+ }
+ }
+ }
+ // TODO for owning indexes only
+ for (PersistentIndex idx : indexes)
+ {
+ idx.commit();
+ }
+ }
+ finally
+ {
+ releaseMultiReader();
+ }
+ return null;
+ }
+ });
+ }
+
+ /**
+ * Adds a document to the index.
+ *
+ * @param doc
+ * the document to add.
+ * @throws IOException
+ * if an error occurs while adding the document to the index.
+ */
+ void addDocument(Document doc) throws IOException
+ {
+ update(Collections.EMPTY_LIST, Arrays.asList(new Document[]{doc}));
+ }
+
+ /**
+ * Deletes the first document that matches the <code>uuid</code>.
+ *
+ * @param uuid
+ * document that match this <code>uuid</code> will be deleted.
+ * @throws IOException
+ * if an error occurs while deleting the document.
+ */
+ void removeDocument(String uuid) throws IOException
+ {
+ update(Arrays.asList(new String[]{uuid}), Collections.EMPTY_LIST);
+ }
+
+ /**
+ * Deletes all documents that match the <code>uuid</code>.
+ *
+ * @param uuid
+ * documents that match this <code>uuid</code> will be deleted.
+ * @return the number of deleted documents.
+ * @throws IOException
+ * if an error occurs while deleting documents.
+ */
+ synchronized int removeAllDocuments(String uuid) throws IOException
+ {
+ int num;
+ try
+ {
+ Term idTerm = new Term(FieldNames.UUID, uuid.toString());
+ num = getChunk(uuid).removeDocument(idTerm);
+ for (int i = 0; i < indexes.size(); i++)
+ {
+ PersistentIndex index = indexes.get(i);
+ // TODO only remove documents from owning indexes
+ int removed = index.removeDocument(idTerm);
+ num += removed;
+ }
+ }
+ finally
+ {
+ releaseMultiReader();
+ }
+ return num;
+ }
+
+ /**
+ * Returns an read-only <code>IndexReader</code> that spans alls indexes of
+ * this <code>MultiIndex</code>.
+ *
+ * @return an <code>IndexReader</code>.
+ * @throws IOException
+ * if an error occurs constructing the <code>IndexReader</code>.
+ */
+ public CachingMultiIndexReader getIndexReader() throws IOException
+ {
+ return getIndexReader(false);
+ }
+
+ /**
+ * Returns an read-only <code>IndexReader</code> that spans alls indexes of
+ * this <code>MultiIndex</code>.
+ *
+ * @param initCache
+ * when set <code>true</code> the hierarchy cache is completely
+ * initialized before this call returns.
+ * @return an <code>IndexReader</code>.
+ * @throws IOException
+ * if an error occurs constructing the <code>IndexReader</code>.
+ */
+ public synchronized CachingMultiIndexReader getIndexReader(final boolean initCache) throws IOException
+ {
+ return SecurityHelper.doPriviledgedIOExceptionAction(new PrivilegedExceptionAction<CachingMultiIndexReader>()
+ {
+ public CachingMultiIndexReader run() throws Exception
+ {
+ if (multiReader != null)
+ {
+ multiReader.acquire();
+ return multiReader;
+ }
+ // no reader available
+ // some other read thread might have created the reader in the
+ // meantime -> check again
+ if (multiReader == null)
+ {
+ List<ReadOnlyIndexReader> readerList = new ArrayList<ReadOnlyIndexReader>();
+ for (PersistentIndex idx : indexes)
+ {
+ readerList.add(idx.getReadOnlyIndexReader(initCache));
+ }
+ ReadOnlyIndexReader[] readers = readerList.toArray(new ReadOnlyIndexReader[readerList.size()]);
+ multiReader = new CachingMultiIndexReader(readers, cache);
+ }
+ multiReader.acquire();
+ return multiReader;
+ }
+ });
+
+ }
+
+ /**
+ * Closes this <code>MultiIndex</code>.
+ */
+ void close()
+ {
+ if (modeHandler.getMode().equals(IndexerIoMode.READ_WRITE))
+ {
+
+ // stop index merger
+ // when calling this method we must not lock this MultiIndex, otherwise
+ // a deadlock might occur
+
+ synchronized (this)
+ {
+
+ // commit / close indexes
+ try
+ {
+ releaseMultiReader();
+ }
+ catch (IOException e)
+ {
+ log.error("Exception while closing search index.", e);
+ }
+
+ // TODO Should they be commited before close?!
+ for (int i = 0; i < indexes.size(); i++)
+ {
+ (indexes.get(i)).close();
+ }
+
+ // close indexing queue
+ indexingQueue.close();
+
+ // finally close directory
+ try
+ {
+ indexDir.close();
+ }
+ catch (IOException e)
+ {
+ log.error("Exception while closing directory.", e);
+ }
+ }
+ }
+ }
+
+ /**
+ * Returns the namespace mappings of this search index.
+ *
+ * @return the namespace mappings of this search index.
+ */
+ NamespaceMappings getNamespaceMappings()
+ {
+ return nsMappings;
+ }
+
+ /**
+ * Returns the indexing queue for this multi index.
+ *
+ * @return the indexing queue for this multi index.
+ */
+ public IndexingQueue getIndexingQueue()
+ {
+ return indexingQueue;
+ }
+
+ /**
+ * Returns a lucene Document for the <code>node</code>.
+ *
+ * @param node
+ * the node to index.
+ * @return the index document.
+ * @throws RepositoryException
+ * if an error occurs while reading from the workspace.
+ */
+ Document createDocument(NodeData node) throws RepositoryException
+ {
+ return handler.createDocument(node, nsMappings, version);
+ }
+
+ /**
+ * Returns a lucene Document for the Node with <code>id</code>.
+ *
+ * @param id
+ * the id of the node to index.
+ * @return the index document.
+ * @throws RepositoryException
+ * if an error occurs while reading from the workspace or if
+ * there is no node with <code>id</code>.
+ */
+ Document createDocument(String id) throws RepositoryException
+ {
+ ItemData data = handler.getContext().getItemStateManager().getItemData(id);
+ if (data == null)
+ {
+ throw new ItemNotFoundException("Item id=" + id + " not found");
+ }
+ if (!data.isNode())
+ {
+ throw new RepositoryException("Item with id " + id + " is not a node");
+ }
+ return createDocument((NodeData)data);
+
+ }
+
+ /**
+ * Releases the {@link #multiReader} and sets it <code>null</code>. If the
+ * reader is already <code>null</code> this method does nothing. When this
+ * method returns {@link #multiReader} is guaranteed to be <code>null</code>
+ * even if an exception is thrown.
+ * <p/>
+ * Please note that this method does not take care of any synchronization. A
+ * caller must ensure that it is the only thread operating on this multi
+ * index, or that it holds the {@link #updateMonitor}.
+ *
+ * @throws IOException
+ * if an error occurs while releasing the reader.
+ */
+ void releaseMultiReader() throws IOException
+ {
+ SecurityHelper.doPriviledgedIOExceptionAction(new PrivilegedExceptionAction<Object>()
+ {
+ public Object run() throws Exception
+ {
+ if (multiReader != null)
+ {
+ try
+ {
+ multiReader.release();
+ }
+ finally
+ {
+ multiReader = null;
+ }
+ }
+ return null;
+ }
+ });
+ }
+
+ // -------------------------< internal
+ // >-------------------------------------
+
+ /**
+ * Recursively creates an index starting with the NodeState
+ * <code>node</code>.
+ *
+ * @param node
+ * the current NodeState.
+ * @param path
+ * the path of the current node.
+ * @param stateMgr
+ * the shared item state manager.
+ * @param count
+ * the number of nodes already indexed.
+ * @return the number of nodes indexed so far.
+ * @throws IOException
+ * if an error occurs while writing to the index.
+ * @throws ItemStateException
+ * if an node state cannot be found.
+ * @throws RepositoryException
+ * if any other error occurs
+ */
+ private long createIndex(NodeData node, ItemDataConsumer stateMgr, long count) throws IOException,
+ RepositoryException
+ {
+ if (indexingTree.isExcluded(node))
+ {
+ return count;
+ }
+ getChunk(node.getIdentifier()).addDocuments(new Document[]{createDocument(node)});
+
+ if (++count % 100 == 0)
+ {
+
+ log.info("indexing... {} ({})", node.getQPath().getAsString(), new Long(count));
+ }
+ if (count % 10 == 0)
+ {
+ checkIndexingQueue(true);
+ }
+ List<NodeData> children = stateMgr.getChildNodesData(node);
+ for (NodeData nodeData : children)
+ {
+
+ NodeData childState = (NodeData)stateMgr.getItemData(nodeData.getIdentifier());
+ if (childState == null)
+ {
+ handler.getOnWorkspaceInconsistencyHandler().handleMissingChildNode(
+ new ItemNotFoundException("Child not found "), handler, nodeData.getQPath(), node, nodeData);
+ }
+
+ if (nodeData != null)
+ {
+ count = createIndex(nodeData, stateMgr, count);
+ }
+ }
+
+ return count;
+ }
+
+ /**
+ * Checks the indexing queue for finished text extrator jobs and updates the
+ * index accordingly if there are any new ones. This method is synchronized
+ * and should only be called by the timer task that periodically checks if
+ * there are documents ready in the indexing queue. A new transaction is
+ * used when documents are transfered from the indexing queue to the index.
+ */
+ private synchronized void checkIndexingQueue()
+ {
+ checkIndexingQueue(false);
+ }
+
+ /**
+ * Checks the indexing queue for finished text extrator jobs and updates the
+ * index accordingly if there are any new ones.
+ *
+ * @param transactionPresent
+ * whether a transaction is in progress and the current
+ * {@link #getTransactionId()} should be used. If
+ * <code>false</code> a new transaction is created when documents
+ * are transfered from the indexing queue to the index.
+ */
+ private void checkIndexingQueue(boolean transactionPresent)
+ {
+ Document[] docs = indexingQueue.getFinishedDocuments();
+ Map<String, Document> finished = new HashMap<String, Document>();
+ for (int i = 0; i < docs.length; i++)
+ {
+ String uuid = docs[i].get(FieldNames.UUID);
+ finished.put(uuid, docs[i]);
+ }
+
+ // now update index with the remaining ones if there are any
+ if (!finished.isEmpty())
+ {
+ log.info("updating index with {} nodes from indexing queue.", new Long(finished.size()));
+
+ // remove documents from the queue
+ for (Iterator<String> it = finished.keySet().iterator(); it.hasNext();)
+ {
+ indexingQueue.removeDocument(it.next());
+ }
+
+ try
+ {
+ if (transactionPresent)
+ {
+ for (Iterator<String> it = finished.keySet().iterator(); it.hasNext();)
+ {
+ String uuidString = it.next();
+ // check if indexing queue is still working on
+ // this node from a previous update
+ Document doc = indexingQueue.removeDocument(uuidString);
+ if (doc != null)
+ {
+ Util.disposeDocument(doc);
+ }
+ Term idTerm = new Term(FieldNames.UUID, uuidString);
+ // if the document cannot be deleted from the volatile index
+ // delete it from one of the persistent indexes.
+ getChunk(uuidString).removeDocument(idTerm);
+ }
+ for (Iterator<Document> it = finished.values().iterator(); it.hasNext();)
+ {
+ Document doc = it.next();
+ String uuid = doc.get(FieldNames.UUID);
+ getChunk(uuid).addDocuments(new Document[]{doc});
+ }
+ }
+ else
+ {
+ update(finished.keySet(), finished.values());
+ }
+ }
+ catch (IOException e)
+ {
+ // update failed
+ log.warn("Failed to update index with deferred text extraction", e);
+ }
+ }
+ }
+
+ @Deprecated
+ public boolean getRedoLogApplied()
+ {
+ return false;
+ }
+
+ private PersistentIndex getChunk(String uuid)
+ {
+ return indexes.get(0);
+ }
+}
Modified: jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/ConsistencyCheck.java
===================================================================
--- jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/ConsistencyCheck.java 2010-10-11 16:33:49 UTC (rev 3291)
+++ jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/ConsistencyCheck.java 2010-10-12 09:47:34 UTC (rev 3292)
@@ -61,7 +61,7 @@
/**
* The index to check.
*/
- private final MultiIndex index;
+ private final ChunkIndex index;
/**
* All the document UUIDs within the index.
@@ -76,7 +76,7 @@
/**
* Private constructor.
*/
- private ConsistencyCheck(MultiIndex index, ItemDataConsumer mgr)
+ private ConsistencyCheck(ChunkIndex index, ItemDataConsumer mgr)
{
this.index = index;
this.stateMgr = mgr;
@@ -91,7 +91,7 @@
* @throws IOException if an error occurs while checking.
* @throws RepositoryException
*/
- static ConsistencyCheck run(MultiIndex index, ItemDataConsumer mgr) throws IOException, RepositoryException
+ static ConsistencyCheck run(ChunkIndex index, ItemDataConsumer mgr) throws IOException, RepositoryException
{
ConsistencyCheck check = new ConsistencyCheck(index, mgr);
check.run();
Modified: jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/IndexInfos.java
===================================================================
--- jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/IndexInfos.java 2010-10-11 16:33:49 UTC (rev 3291)
+++ jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/IndexInfos.java 2010-10-12 09:47:34 UTC (rev 3292)
@@ -74,9 +74,9 @@
private Directory dir;
/**
- * {@link MultiIndex} instance for callbacking when list of indexes changed
+ * {@link ChunkIndex} instance for callbacking when list of indexes changed
*/
- private MultiIndex multiIndex;
+ private ChunkIndex multiIndex;
/**
* Creates a new IndexInfos using <code>"indexes"</code> as a filename.
@@ -304,10 +304,10 @@
}
/**
- * Sets {@link MultiIndex} instance.
+ * Sets {@link ChunkIndex} instance.
* @param multiIndex
*/
- public void setMultiIndex(MultiIndex multiIndex)
+ public void setMultiIndex(ChunkIndex multiIndex)
{
this.multiIndex = multiIndex;
}
@@ -316,7 +316,7 @@
* Returns multiIndex instance
* @return
*/
- public MultiIndex getMultiIndex()
+ public ChunkIndex getMultiIndex()
{
return multiIndex;
}
Modified: jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/IndexingQueue.java
===================================================================
--- jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/IndexingQueue.java 2010-10-11 16:33:49 UTC (rev 3291)
+++ jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/IndexingQueue.java 2010-10-12 09:47:34 UTC (rev 3292)
@@ -57,7 +57,7 @@
/**
* Flag that indicates whether this indexing queue had been
- * {@link #initialize(MultiIndex) initialized}.
+ * {@link #initialize(ChunkIndex) initialized}.
*/
private volatile boolean initialized = false;
@@ -77,7 +77,7 @@
* @param index the multi index this indexing queue belongs to.
* @throws IOException if an error occurs while reading from the index.
*/
- void initialize(final MultiIndex index) throws IOException
+ void initialize(final ChunkIndex index) throws IOException
{
SecurityHelper.doPriviledgedIOExceptionAction(new PrivilegedExceptionAction<Object>()
{
Deleted: jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/MultiIndex.java
===================================================================
--- jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/MultiIndex.java 2010-10-11 16:33:49 UTC (rev 3291)
+++ jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/MultiIndex.java 2010-10-12 09:47:34 UTC (rev 3292)
@@ -1,1069 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.exoplatform.services.jcr.impl.core.query.lucene;
-
-import org.apache.lucene.document.Document;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.store.Directory;
-import org.exoplatform.services.jcr.dataflow.ItemDataConsumer;
-import org.exoplatform.services.jcr.datamodel.ItemData;
-import org.exoplatform.services.jcr.datamodel.NodeData;
-import org.exoplatform.services.jcr.impl.core.query.IndexerIoMode;
-import org.exoplatform.services.jcr.impl.core.query.IndexerIoModeHandler;
-import org.exoplatform.services.jcr.impl.core.query.IndexingTree;
-import org.exoplatform.services.jcr.impl.core.query.lucene.directory.DirectoryManager;
-import org.exoplatform.services.jcr.impl.util.SecurityHelper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.security.PrivilegedAction;
-import java.security.PrivilegedExceptionAction;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.Map.Entry;
-
-import javax.jcr.ItemNotFoundException;
-import javax.jcr.RepositoryException;
-
-/**
- * A <code>MultiIndex</code> consists of a {@link VolatileIndex} and multiple
- * {@link PersistentIndex}es. The goal is to keep most parts of the index open
- * with index readers and write new index data to the volatile index. When the
- * volatile index reaches a certain size (see
- * {@link SearchIndex#setMinMergeDocs(int)}) a new persistent index is created
- * with the index data from the volatile index, the same happens when the
- * volatile index has been idle for some time (see
- * {@link SearchIndex#setVolatileIdleTime(int)}). The new persistent index is
- * then added to the list of already existing persistent indexes. Further
- * operations on the new persistent index will however only require an
- * <code>IndexReader</code> which serves for queries but also for delete
- * operations on the index.
- * <p/>
- * The persistent indexes are merged from time to time. The merge behaviour is
- * configurable using the methods: {@link SearchIndex#setMaxMergeDocs(int)},
- * {@link SearchIndex#setMergeFactor(int)} and
- * {@link SearchIndex#setMinMergeDocs(int)}. For detailed description of the
- * configuration parameters see also the lucene <code>IndexWriter</code> class.
- * <p/>
- * This class is thread-safe.
- * <p/>
- * Note on implementation: Multiple modifying threads are synchronized on a
- * <code>MultiIndex</code> instance itself. Sychronization between a modifying
- * thread and reader threads is done using {@link #updateMonitor} and
- * {@link #updateInProgress}.
- */
-public class MultiIndex
-{
-
- /**
- * The logger instance for this class
- */
- private static final Logger log = LoggerFactory.getLogger("exo.jcr.component.core.MultiIndex");
-
- /**
- * Names of active persistent index directories.
- */
- private IndexInfos indexNames;
-
- /**
- * List of open persistent indexes. This list may also contain an open
- * PersistentIndex owned by the IndexMerger daemon. Such an index is not
- * registered with indexNames and <b>must not</b> be used in regular index
- * operations (delete node, etc.)!
- */
- private final List<PersistentIndex> indexes = new ArrayList<PersistentIndex>();
-
- /**
- * The internal namespace mappings of the query manager.
- */
- private final NamespaceMappings nsMappings;
-
- /**
- * The directory manager.
- */
- private final DirectoryManager directoryManager;
-
- /**
- * The base directory to store the index.
- */
- private final Directory indexDir;
-
- /**
- * The query handler
- */
- private final SearchIndex handler;
-
- /**
- * If not <code>null</code> points to a valid <code>IndexReader</code> that
- * reads from all indexes, including volatile and persistent indexes.
- */
- private CachingMultiIndexReader multiReader;
-
- /**
- * Shared document number cache across all persistent indexes.
- */
- private final DocNumberCache cache;
-
- /**
- * The indexing queue with pending text extraction jobs.
- */
- private IndexingQueue indexingQueue;
-
- /**
- * Set<NodeId> of uuids that should not be indexed.
- */
- private final IndexingTree indexingTree;
-
- /**
- * Flag indicating whether re-indexing is running.
- */
- private boolean reindexing = false;
-
- /**
- * The index format version of this multi index.
- */
- private final IndexFormatVersion version;
-
- /**
- * The handler of the Indexer io mode
- */
- private final IndexerIoModeHandler modeHandler;
-
- /**
- * Creates a new MultiIndex.
- *
- * @param handler
- * the search handler
- * @param excludedIDs
- * Set<NodeId> that contains uuids that should not be indexed
- * nor further traversed.
- * @throws IOException
- * if an error occurs
- */
- MultiIndex(SearchIndex handler, IndexingTree indexingTree, IndexerIoModeHandler modeHandler, IndexInfos indexInfos,
- IndexUpdateMonitor indexUpdateMonitor) throws IOException
- {
- this.modeHandler = modeHandler;
- this.directoryManager = handler.getDirectoryManager();
- // this method is run in privileged mode internally
- this.indexDir = directoryManager.getDirectory(".");
- this.handler = handler;
- this.cache = new DocNumberCache(handler.getCacheSize());
- this.indexingTree = indexingTree;
- this.nsMappings = handler.getNamespaceMappings();
- this.indexNames = indexInfos;
- this.indexNames.setDirectory(indexDir);
- // this method is run in privileged mode internally
- this.indexNames.read();
- this.indexNames.addName("1");
- this.indexNames.addName("2");
- this.indexNames.addName("3");
- this.indexNames.addName("4");
-
- // this method is run in privileged mode internally
- // as of 1.5 deletable file is not used anymore
- removeDeletable();
-
- // this method is run in privileged mode internally
- IndexingQueueStore store = new IndexingQueueStore(indexDir);
-
- // initialize indexing queue
- this.indexingQueue = new IndexingQueue(store);
- // copy current index names
- Set<String> currentNames = new HashSet<String>(indexNames.getNames());
- //Set<String> currentNames = new HashSet<String>();
-
- // open persistent indexes
- for (String name : currentNames)
- {
- // only open if it still exists
- // it is possible that indexNames still contains a name for
- // an index that has been deleted, but indexNames has not been
- // written to disk.
-
- // TODO THIS CHECK WAS SKIPPED
- // if (!directoryManager.hasDirectory(name))
- // {
- // log.debug("index does not exist anymore: " + name);
- // // move on to next index
- // continue;
- // }
- PersistentIndex index =
- new PersistentIndex(name, handler.getTextAnalyzer(), handler.getSimilarity(), cache, indexingQueue,
- directoryManager);
- index.setMaxFieldLength(handler.getMaxFieldLength());
- index.setUseCompoundFile(handler.getUseCompoundFile());
- index.setTermInfosIndexDivisor(handler.getTermInfosIndexDivisor());
- indexes.add(index);
- }
-
- // init volatile index
- // TODO commented line below:
- //resetVolatileIndex();
-
- // set index format version and at the same time
- // initialize hierarchy cache if requested.
- final CachingMultiIndexReader reader = getIndexReader(handler.isInitializeHierarchyCache());
- try
- {
- version = IndexFormatVersion.getVersion(reader);
- }
- finally
- {
- SecurityHelper.doPriviledgedIOExceptionAction(new PrivilegedExceptionAction<Object>()
- {
- public Object run() throws Exception
- {
- reader.release();
- return null;
- }
- });
- }
- indexingQueue.initialize(this);
- this.indexNames.setMultiIndex(this);
- }
-
- /**
- * Returns the number of documents in this index.
- *
- * @return the number of documents in this index.
- * @throws IOException
- * if an error occurs while reading from the index.
- */
- int numDocs() throws IOException
- {
- if (indexNames.size() == 0)
- {
- // TODO commented line below:
- //return volatileIndex.getNumDocuments();
- return 0;
- }
- else
- {
- final CachingMultiIndexReader reader = getIndexReader();
- try
- {
- return reader.numDocs();
- }
- finally
- {
- SecurityHelper.doPriviledgedIOExceptionAction(new PrivilegedExceptionAction<Object>()
- {
- public Object run() throws Exception
- {
- reader.release();
- return null;
- }
- });
- }
- }
- }
-
- /**
- * @return the index format version for this multi index.
- */
- IndexFormatVersion getIndexFormatVersion()
- {
- return version;
- }
-
- /**
- * Creates an initial index by traversing the node hierarchy starting at the
- * node with <code>rootId</code>.
- *
- * @param stateMgr
- * the item state manager.
- * @param rootId
- * the id of the node from where to start.
- * @param rootPath
- * the path of the node from where to start.
- * @throws IOException
- * if an error occurs while indexing the workspace.
- * @throws IllegalStateException
- * if this index is not empty.
- */
- void createInitialIndex(ItemDataConsumer stateMgr) throws IOException
- {
-
- // TODO: re-study check!!!
- // only do an initial index if there are no indexes at all
- // if (indexNames.size() == 0)
- {
- reindexing = true;
- try
- {
- long count = 0;
- // traverse and index workspace
- // TODO: this was removed
- //executeAndLog(new Start(Action.INTERNAL_TRANSACTION));
-
- // NodeData rootState = (NodeData) stateMgr.getItemData(rootId);
- count = createIndex(indexingTree.getIndexingRoot(), stateMgr, count);
-
- // TODO : this was replaced
- //executeAndLog(new Commit(getTransactionId()));
-
- log.info("Created initial index for {} nodes", new Long(count));
- releaseMultiReader();
- // TODO commented line below:
- //scheduleFlushTask();
- }
- catch (Exception e)
- {
- String msg = "Error indexing workspace";
- IOException ex = new IOException(msg);
- ex.initCause(e);
- throw ex;
- }
- finally
- {
- reindexing = false;
- }
- }
- // else
- // {
- // throw new IllegalStateException("Index already present");
- // }
- }
-
- /**
- * Atomically updates the index by removing some documents and adding
- * others.
- *
- * @param remove
- * collection of <code>UUID</code>s that identify documents to
- * remove
- * @param add
- * collection of <code>Document</code>s to add. Some of the
- * elements in this collection may be <code>null</code>, to
- * indicate that a node could not be indexed successfully.
- * @throws IOException
- * if an error occurs while updating the index.
- */
- synchronized void update(final Collection<String> remove, final Collection<Document> add) throws IOException
- {
- SecurityHelper.doPriviledgedIOExceptionAction(new PrivilegedExceptionAction<Object>()
- {
- public Object run() throws Exception
- {
- // make sure a reader is available during long updates
- if (add.size() > handler.getBufferSize())
- {
- try
- {
- getIndexReader().release();
- }
- catch (IOException e)
- {
- // do not fail if an exception is thrown here
- log.warn("unable to prepare index reader " + "for queries during update", e);
- }
- }
-// boolean flush = false;
- try
- {
- // TODO: this was removed
- //executeAndLog(new Start(transactionId));
-
- for (Iterator<String> it = remove.iterator(); it.hasNext();)
- {
- // TODO this was replaced
- //executeAndLog(new DeleteNode(transactionId, (String)it.next()));
-
- String uuidString = it.next();
- // check if indexing queue is still working on
- // this node from a previous update
- Document doc = indexingQueue.removeDocument(uuidString);
- if (doc != null)
- {
- Util.disposeDocument(doc);
- }
- Term idTerm = new Term(FieldNames.UUID, uuidString);
- // if the document cannot be deleted from the volatile index
- // delete it from one of the persistent indexes.
-
- // TODO watch here
- int num = getChunk(uuidString).removeDocument(idTerm);
- if (num == 0)
- {
- for (int i = indexes.size() - 1; i >= 0; i--)
- {
- // only look in registered indexes
- PersistentIndex idx = indexes.get(i);
- if (indexNames.contains(idx.getName()))
- {
- num = idx.removeDocument(idTerm);
- if (num > 0)
- {
- break;
- }
- }
- }
- }
-
- }
- for (Iterator<Document> it = add.iterator(); it.hasNext();)
- {
- Document doc = it.next();
- if (doc != null)
- {
- // TODO: ths is replaced
- //executeAndLog(new AddNode(transactionId, doc));
-
- if (doc != null)
- {
- // TODO watch this
- String uuid = doc.get(FieldNames.UUID);
- getChunk(uuid).addDocuments(new Document[]{doc});
- }
-
- // commit volatile index if needed
- // TODO commented line below:
- //flush |= checkVolatileCommit();
- }
- }
-
- // TODO : this was replaced
- //executeAndLog(new Commit(transactionId));
-
- // TODO commented line below:
- //flush();
- getChunk(null).commit();
- }
- finally
- {
- releaseMultiReader();
- }
- return null;
- }
- });
- }
-
- /**
- * Adds a document to the index.
- *
- * @param doc
- * the document to add.
- * @throws IOException
- * if an error occurs while adding the document to the index.
- */
- void addDocument(Document doc) throws IOException
- {
- update(Collections.EMPTY_LIST, Arrays.asList(new Document[]{doc}));
- }
-
- /**
- * Deletes the first document that matches the <code>uuid</code>.
- *
- * @param uuid
- * document that match this <code>uuid</code> will be deleted.
- * @throws IOException
- * if an error occurs while deleting the document.
- */
- void removeDocument(String uuid) throws IOException
- {
- update(Arrays.asList(new String[]{uuid}), Collections.EMPTY_LIST);
- }
-
- /**
- * Deletes all documents that match the <code>uuid</code>.
- *
- * @param uuid
- * documents that match this <code>uuid</code> will be deleted.
- * @return the number of deleted documents.
- * @throws IOException
- * if an error occurs while deleting documents.
- */
- synchronized int removeAllDocuments(String uuid) throws IOException
- {
- int num;
- try
- {
- Term idTerm = new Term(FieldNames.UUID, uuid.toString());
-
- // TODO: this was removed
- //executeAndLog(new Start(Action.INTERNAL_TRANSACTION));
-
- // TODO REMOVE WAS HERE
- num = getChunk(uuid).removeDocument(idTerm);
- if (num > 0)
- {
- // TODO: removed line
- //redoLog.append(new DeleteNode(getTransactionId(), uuid));
- }
- for (int i = 0; i < indexes.size(); i++)
- {
- PersistentIndex index = indexes.get(i);
- // only remove documents from registered indexes
- if (indexNames.contains(index.getName()))
- {
- int removed = index.removeDocument(idTerm);
- if (removed > 0)
- {
- // TODO: removed line
- //redoLog.append(new DeleteNode(getTransactionId(), uuid));
- }
- num += removed;
- }
- }
- // TODO : this was replaced
- //executeAndLog(new Commit(getTransactionId()));
- }
- finally
- {
- releaseMultiReader();
- }
- return num;
- }
-
- /**
- * Returns <code>IndexReader</code>s for the indexes named
- * <code>indexNames</code>. An <code>IndexListener</code> is registered and
- * notified when documents are deleted from one of the indexes in
- * <code>indexNames</code>.
- * <p/>
- * Note: the number of <code>IndexReaders</code> returned by this method is
- * not necessarily the same as the number of index names passed. An index
- * might have been deleted and is not reachable anymore.
- *
- * @param indexNames
- * the names of the indexes for which to obtain readers.
- * @param listener
- * the listener to notify when documents are deleted.
- * @return the <code>IndexReaders</code>.
- * @throws IOException
- * if an error occurs acquiring the index readers.
- */
- synchronized IndexReader[] getIndexReaders(String[] indexNames, IndexListener listener) throws IOException
- {
- Set<String> names = new HashSet<String>(Arrays.asList(indexNames));
- Map<ReadOnlyIndexReader, PersistentIndex> indexReaders = new HashMap<ReadOnlyIndexReader, PersistentIndex>();
-
- try
- {
- for (Iterator<PersistentIndex> it = indexes.iterator(); it.hasNext();)
- {
- PersistentIndex index = it.next();
- if (names.contains(index.getName()))
- {
- indexReaders.put(index.getReadOnlyIndexReader(listener), index);
- }
- }
- }
- catch (IOException e)
- {
- // release readers obtained so far
- for (Iterator<Entry<ReadOnlyIndexReader, PersistentIndex>> it = indexReaders.entrySet().iterator(); it.hasNext();)
- {
- Map.Entry<ReadOnlyIndexReader, PersistentIndex> entry = it.next();
- final ReadOnlyIndexReader reader = entry.getKey();
- try
- {
- SecurityHelper.doPriviledgedIOExceptionAction(new PrivilegedExceptionAction<Object>()
- {
- public Object run() throws Exception
- {
- reader.release();
- return null;
- }
- });
- }
- catch (IOException ex)
- {
- log.warn("Exception releasing index reader: " + ex);
- }
- (entry.getValue()).resetListener();
- }
- throw e;
- }
-
- return indexReaders.keySet().toArray(new IndexReader[indexReaders.size()]);
- }
-
- /**
- * Returns <code>true</code> if this multi index has an index segment with
- * the given name. This method even returns <code>true</code> if an index
- * segments has not yet been loaded / initialized but exists on disk.
- *
- * @param indexName
- * the name of the index segment.
- * @return <code>true</code> if it exists; otherwise <code>false</code>.
- * @throws IOException
- * if an error occurs while checking existence of directory.
- */
- synchronized boolean hasIndex(String indexName) throws IOException
- {
- // check existing
- for (Iterator<PersistentIndex> it = indexes.iterator(); it.hasNext();)
- {
- PersistentIndex idx = it.next();
- if (idx.getName().equals(indexName))
- {
- return true;
- }
- }
- // check if it exists on disk
- return directoryManager.hasDirectory(indexName);
- }
-
- /**
- * Returns an read-only <code>IndexReader</code> that spans alls indexes of
- * this <code>MultiIndex</code>.
- *
- * @return an <code>IndexReader</code>.
- * @throws IOException
- * if an error occurs constructing the <code>IndexReader</code>.
- */
- public CachingMultiIndexReader getIndexReader() throws IOException
- {
- return getIndexReader(false);
- }
-
- /**
- * Returns an read-only <code>IndexReader</code> that spans alls indexes of
- * this <code>MultiIndex</code>.
- *
- * @param initCache
- * when set <code>true</code> the hierarchy cache is completely
- * initialized before this call returns.
- * @return an <code>IndexReader</code>.
- * @throws IOException
- * if an error occurs constructing the <code>IndexReader</code>.
- */
- public synchronized CachingMultiIndexReader getIndexReader(final boolean initCache) throws IOException
- {
- return SecurityHelper.doPriviledgedIOExceptionAction(new PrivilegedExceptionAction<CachingMultiIndexReader>()
- {
- public CachingMultiIndexReader run() throws Exception
- {
- if (multiReader != null)
- {
- multiReader.acquire();
- return multiReader;
- }
- // no reader available
- // some other read thread might have created the reader in the
- // meantime -> check again
- if (multiReader == null)
- {
- List<ReadOnlyIndexReader> readerList = new ArrayList<ReadOnlyIndexReader>();
- for (int i = 0; i < indexes.size(); i++)
- {
- PersistentIndex pIdx = indexes.get(i);
-
- if (indexNames.contains(pIdx.getName()))
- {
- try
- {
- readerList.add(pIdx.getReadOnlyIndexReader(initCache));
- }
- catch (FileNotFoundException e)
- {
- if (directoryManager.hasDirectory(pIdx.getName()))
- {
- throw e;
- }
- }
- }
- }
- // TODO commented line below:
- //readerList.add(volatileIndex.getReadOnlyIndexReader());
- ReadOnlyIndexReader[] readers = readerList.toArray(new ReadOnlyIndexReader[readerList.size()]);
- multiReader = new CachingMultiIndexReader(readers, cache);
- }
- multiReader.acquire();
- return multiReader;
- }
- });
-
- }
-
- /**
- * Closes this <code>MultiIndex</code>.
- */
- void close()
- {
- if (modeHandler.getMode().equals(IndexerIoMode.READ_WRITE))
- {
-
- // stop index merger
- // when calling this method we must not lock this MultiIndex, otherwise
- // a deadlock might occur
-
- synchronized (this)
- {
-
- // commit / close indexes
- try
- {
- releaseMultiReader();
- }
- catch (IOException e)
- {
- log.error("Exception while closing search index.", e);
- }
- // try
- // {
- // // TODO commented line below:
- // //flush();
- // }
- // catch (IOException e)
- // {
- // log.error("Exception while closing search index.", e);
- // }
- // TODO commented line below:
- //volatileIndex.close();
- // indexes will be closed now:
- for (int i = 0; i < indexes.size(); i++)
- {
- (indexes.get(i)).close();
- }
-
- // close indexing queue
- indexingQueue.close();
-
- // finally close directory
- try
- {
- indexDir.close();
- }
- catch (IOException e)
- {
- log.error("Exception while closing directory.", e);
- }
- }
- }
- }
-
- /**
- * Returns the namespace mappings of this search index.
- *
- * @return the namespace mappings of this search index.
- */
- NamespaceMappings getNamespaceMappings()
- {
- return nsMappings;
- }
-
- /**
- * Returns the indexing queue for this multi index.
- *
- * @return the indexing queue for this multi index.
- */
- public IndexingQueue getIndexingQueue()
- {
- return indexingQueue;
- }
-
- /**
- * Returns a lucene Document for the <code>node</code>.
- *
- * @param node
- * the node to index.
- * @return the index document.
- * @throws RepositoryException
- * if an error occurs while reading from the workspace.
- */
- Document createDocument(NodeData node) throws RepositoryException
- {
- return handler.createDocument(node, nsMappings, version);
- }
-
- /**
- * Returns a lucene Document for the Node with <code>id</code>.
- *
- * @param id
- * the id of the node to index.
- * @return the index document.
- * @throws RepositoryException
- * if an error occurs while reading from the workspace or if
- * there is no node with <code>id</code>.
- */
- Document createDocument(String id) throws RepositoryException
- {
- ItemData data = handler.getContext().getItemStateManager().getItemData(id);
- if (data == null)
- {
- throw new ItemNotFoundException("Item id=" + id + " not found");
- }
- if (!data.isNode())
- {
- throw new RepositoryException("Item with id " + id + " is not a node");
- }
- return createDocument((NodeData)data);
-
- }
-
- /**
- * Releases the {@link #multiReader} and sets it <code>null</code>. If the
- * reader is already <code>null</code> this method does nothing. When this
- * method returns {@link #multiReader} is guaranteed to be <code>null</code>
- * even if an exception is thrown.
- * <p/>
- * Please note that this method does not take care of any synchronization. A
- * caller must ensure that it is the only thread operating on this multi
- * index, or that it holds the {@link #updateMonitor}.
- *
- * @throws IOException
- * if an error occurs while releasing the reader.
- */
- void releaseMultiReader() throws IOException
- {
- SecurityHelper.doPriviledgedIOExceptionAction(new PrivilegedExceptionAction<Object>()
- {
- public Object run() throws Exception
- {
- if (multiReader != null)
- {
- try
- {
- multiReader.release();
- }
- finally
- {
- multiReader = null;
- }
- }
- return null;
- }
- });
- }
-
- // -------------------------< internal
- // >-------------------------------------
-
- /**
- * Recursively creates an index starting with the NodeState
- * <code>node</code>.
- *
- * @param node
- * the current NodeState.
- * @param path
- * the path of the current node.
- * @param stateMgr
- * the shared item state manager.
- * @param count
- * the number of nodes already indexed.
- * @return the number of nodes indexed so far.
- * @throws IOException
- * if an error occurs while writing to the index.
- * @throws ItemStateException
- * if an node state cannot be found.
- * @throws RepositoryException
- * if any other error occurs
- */
- private long createIndex(NodeData node, ItemDataConsumer stateMgr, long count) throws IOException,
- RepositoryException
- {
- // NodeId id = node.getNodeId();
-
- if (indexingTree.isExcluded(node))
- {
- return count;
- }
-
- // TODO: this is replaced
- //executeAndLog(new AddNode(getTransactionId(), node.getIdentifier()));
- getChunk(node.getIdentifier()).addDocuments(new Document[]{createDocument(node)});
-
- if (++count % 100 == 0)
- {
-
- log.info("indexing... {} ({})", node.getQPath().getAsString(), new Long(count));
- }
- if (count % 10 == 0)
- {
- checkIndexingQueue(true);
- }
- // TODO commented line below:
- //checkVolatileCommit();
- List<NodeData> children = stateMgr.getChildNodesData(node);
- for (NodeData nodeData : children)
- {
-
- NodeData childState = (NodeData)stateMgr.getItemData(nodeData.getIdentifier());
- if (childState == null)
- {
- handler.getOnWorkspaceInconsistencyHandler().handleMissingChildNode(
- new ItemNotFoundException("Child not found "), handler, nodeData.getQPath(), node, nodeData);
- }
-
- if (nodeData != null)
- {
- count = createIndex(nodeData, stateMgr, count);
- }
- }
-
- return count;
- }
-
- /**
- * Removes the deletable file if it exists. The file is not used anymore in
- * Jackrabbit versions >= 1.5.
- */
- private void removeDeletable()
- {
- SecurityHelper.doPriviledgedAction(new PrivilegedAction<Object>()
- {
- public Object run()
- {
- String fileName = "deletable";
- try
- {
- if (indexDir.fileExists(fileName))
- {
- indexDir.deleteFile(fileName);
- }
- }
- catch (IOException e)
- {
- log.warn("Unable to remove file 'deletable'.", e);
- }
- return null;
- }
- });
- }
-
- /**
- * Checks the indexing queue for finished text extrator jobs and updates the
- * index accordingly if there are any new ones. This method is synchronized
- * and should only be called by the timer task that periodically checks if
- * there are documents ready in the indexing queue. A new transaction is
- * used when documents are transfered from the indexing queue to the index.
- */
- private synchronized void checkIndexingQueue()
- {
- checkIndexingQueue(false);
- }
-
- /**
- * Checks the indexing queue for finished text extrator jobs and updates the
- * index accordingly if there are any new ones.
- *
- * @param transactionPresent
- * whether a transaction is in progress and the current
- * {@link #getTransactionId()} should be used. If
- * <code>false</code> a new transaction is created when documents
- * are transfered from the indexing queue to the index.
- */
- private void checkIndexingQueue(boolean transactionPresent)
- {
- Document[] docs = indexingQueue.getFinishedDocuments();
- Map<String, Document> finished = new HashMap<String, Document>();
- for (int i = 0; i < docs.length; i++)
- {
- String uuid = docs[i].get(FieldNames.UUID);
- finished.put(uuid, docs[i]);
- }
-
- // now update index with the remaining ones if there are any
- if (!finished.isEmpty())
- {
- log.info("updating index with {} nodes from indexing queue.", new Long(finished.size()));
-
- // remove documents from the queue
- for (Iterator<String> it = finished.keySet().iterator(); it.hasNext();)
- {
- indexingQueue.removeDocument(it.next());
- }
-
- try
- {
- if (transactionPresent)
- {
- for (Iterator<String> it = finished.keySet().iterator(); it.hasNext();)
- {
- // TODO this was replaced
- //executeAndLog(new DeleteNode(getTransactionId(), (String)it.next()));
-
- String uuidString = it.next();
- // check if indexing queue is still working on
- // this node from a previous update
- Document doc = indexingQueue.removeDocument(uuidString);
- if (doc != null)
- {
- Util.disposeDocument(doc);
- }
- Term idTerm = new Term(FieldNames.UUID, uuidString);
- // if the document cannot be deleted from the volatile index
- // delete it from one of the persistent indexes.
- int num = getChunk(uuidString).removeDocument(idTerm);
- if (num == 0)
- {
- for (int i = indexes.size() - 1; i >= 0; i--)
- {
- // only look in registered indexes
- PersistentIndex idx = indexes.get(i);
- if (indexNames.contains(idx.getName()))
- {
- num = idx.removeDocument(idTerm);
- if (num > 0)
- {
- return;
- }
- }
- }
- }
-
- }
- for (Iterator<Document> it = finished.values().iterator(); it.hasNext();)
- {
- // TODO this was replaced
- //executeAndLog(new AddNode(getTransactionId(), (Document)it.next()));
- Document doc = it.next();
- String uuid = doc.get(FieldNames.UUID);
- getChunk(uuid).addDocuments(new Document[]{doc});
- }
- }
- else
- {
- update(finished.keySet(), finished.values());
- }
- }
- catch (IOException e)
- {
- // update failed
- log.warn("Failed to update index with deferred text extraction", e);
- }
- }
- }
-
- // ------------------------< Actions
- // >---------------------------------------
-
- @Deprecated
- public boolean getRedoLogApplied()
- {
- return false;
- }
-
- private PersistentIndex getChunk(String uuid)
- {
- return indexes.get(0);
- }
-}
Modified: jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/SearchIndex.java
===================================================================
--- jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/SearchIndex.java 2010-10-11 16:33:49 UTC (rev 3291)
+++ jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/SearchIndex.java 2010-10-12 09:47:34 UTC (rev 3292)
@@ -175,7 +175,7 @@
/**
* The actual index
*/
- private MultiIndex index;
+ private ChunkIndex index;
/**
* The analyzer we use for indexing.
@@ -370,7 +370,7 @@
/**
* Indicates the index format version which is relevant to a <b>query</b>.
* This value may be different from what
- * {@link MultiIndex#getIndexFormatVersion()} returns because queries may be
+ * {@link ChunkIndex#getIndexFormatVersion()} returns because queries may be
* executed on two physical indexes with different formats. Index format
* versions are considered backward compatible. That is, the lower version
* of the two physical indexes is used for querying.
@@ -570,7 +570,7 @@
indexingConfig = createIndexingConfiguration(nsMappings);
analyzer.setIndexingConfig(indexingConfig);
- index = new MultiIndex(this, context.getIndexingTree(), modeHandler, getIndexInfos(), getIndexUpdateMonitor());
+ index = new ChunkIndex(this, context.getIndexingTree(), modeHandler, getIndexInfos(), getIndexUpdateMonitor());
// if RW mode, create initial index and start check
if (modeHandler.getMode() == IndexerIoMode.READ_WRITE)
{
@@ -676,7 +676,7 @@
/**
* This implementation forwards the call to
- * {@link MultiIndex#update(Collection, Collection)} and transforms the two
+ * {@link ChunkIndex#update(Collection, Collection)} and transforms the two
* iterators to the required types.
*
* @param remove
@@ -1212,7 +1212,7 @@
*
* @return the actual index.
*/
- public MultiIndex getIndex()
+ public ChunkIndex getIndex()
{
return index;
}
More information about the exo-jcr-commits
mailing list