exo-jcr SVN: r3300 - in jcr/trunk/exo.jcr.component.webdav/src: main/java/org/exoplatform/services/jcr/webdav/command and 2 other directories.
by do-not-reply＠jboss.org
Author: dkuleshov
Date: 2010-10-19 08:50:39 -0400 (Tue, 19 Oct 2010)
New Revision: 3300
Modified:
jcr/trunk/exo.jcr.component.webdav/src/main/java/org/exoplatform/services/jcr/webdav/WebDavConst.java
jcr/trunk/exo.jcr.component.webdav/src/main/java/org/exoplatform/services/jcr/webdav/command/GetCommand.java
jcr/trunk/exo.jcr.component.webdav/src/main/java/org/exoplatform/services/jcr/webdav/resource/VersionResource.java
jcr/trunk/exo.jcr.component.webdav/src/test/java/org/exoplatform/services/jcr/webdav/command/TestGet.java
Log:
EXOJCR-1001: fixed date parsing during if-modified-since header usage
Modified: jcr/trunk/exo.jcr.component.webdav/src/main/java/org/exoplatform/services/jcr/webdav/WebDavConst.java
===================================================================
--- jcr/trunk/exo.jcr.component.webdav/src/main/java/org/exoplatform/services/jcr/webdav/WebDavConst.java 2010-10-19 11:35:09 UTC (rev 3299)
+++ jcr/trunk/exo.jcr.component.webdav/src/main/java/org/exoplatform/services/jcr/webdav/WebDavConst.java 2010-10-19 12:50:39 UTC (rev 3300)
@@ -18,10 +18,10 @@
*/
package org.exoplatform.services.jcr.webdav;
+import java.util.Hashtable;
+
import org.exoplatform.common.http.HTTPStatus;
-import java.util.Hashtable;
-
/**
* Constants used for webdav service implemetation.
*
@@ -468,7 +468,7 @@
/**
* If-Modified-Since date psttern.
*/
- public static final String IF_MODIFIED_SINCE_PATTERN = "EEE, d MMM yyyy HH:mm:ss z";
+ public static final String IF_MODIFIED_SINCE_PATTERN = "EEE, dd MMM yyyy HH:mm:ss z";
}
/**
Modified: jcr/trunk/exo.jcr.component.webdav/src/main/java/org/exoplatform/services/jcr/webdav/command/GetCommand.java
===================================================================
--- jcr/trunk/exo.jcr.component.webdav/src/main/java/org/exoplatform/services/jcr/webdav/command/GetCommand.java 2010-10-19 11:35:09 UTC (rev 3299)
+++ jcr/trunk/exo.jcr.component.webdav/src/main/java/org/exoplatform/services/jcr/webdav/command/GetCommand.java 2010-10-19 12:50:39 UTC (rev 3300)
@@ -18,6 +18,26 @@
*/
package org.exoplatform.services.jcr.webdav.command;
+import java.io.InputStream;
+import java.net.URI;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+
+import javax.jcr.Node;
+import javax.jcr.PathNotFoundException;
+import javax.jcr.RepositoryException;
+import javax.jcr.Session;
+import javax.ws.rs.core.HttpHeaders;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.xml.transform.stream.StreamSource;
+
import org.exoplatform.common.http.HTTPStatus;
import org.exoplatform.common.util.HierarchicalProperty;
import org.exoplatform.services.jcr.webdav.Range;
@@ -39,26 +59,6 @@
import org.exoplatform.services.rest.ext.provider.XSLTStreamingOutput;
import org.exoplatform.services.rest.impl.header.MediaTypeHelper;
-import java.io.InputStream;
-import java.net.URI;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-
-import javax.jcr.Node;
-import javax.jcr.PathNotFoundException;
-import javax.jcr.RepositoryException;
-import javax.jcr.Session;
-import javax.ws.rs.core.HttpHeaders;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-import javax.xml.transform.stream.StreamSource;
-
/**
* Created by The eXo Platform SAS Author : <a
* href="gavrikvetal(a)gmail.com">Vitaly Guly</a>.
@@ -118,7 +118,7 @@
VersionedResource versionedFile = new VersionedFileResource(uri, node, nsContext);
resource = versionedFile.getVersionHistory().getVersion(version);
- lastModifiedProperty = resource.getProperty(FileResource.CREATIONDATE);
+ lastModifiedProperty = resource.getProperty(FileResource.GETLASTMODIFIED);
istream = ((VersionResource)resource).getContentAsStream();
}
else
@@ -133,10 +133,10 @@
if (ifModifiedSince != null)
{
- DateFormat dateFormat = new SimpleDateFormat(WebDavConst.DateFormat.IF_MODIFIED_SINCE_PATTERN, Locale.US);
+ DateFormat dateFormat = new SimpleDateFormat(WebDavConst.DateFormat.MODIFICATION, Locale.US);
Date lastModifiedDate = dateFormat.parse(lastModifiedProperty.getValue());
- dateFormat = new SimpleDateFormat(WebDavConst.DateFormat.MODIFICATION, Locale.US);
+ dateFormat = new SimpleDateFormat(WebDavConst.DateFormat.IF_MODIFIED_SINCE_PATTERN, Locale.US);
Date ifModifiedSinceDate = dateFormat.parse(ifModifiedSince);
if(ifModifiedSinceDate.getTime() >= lastModifiedDate.getTime()){
Modified: jcr/trunk/exo.jcr.component.webdav/src/main/java/org/exoplatform/services/jcr/webdav/resource/VersionResource.java
===================================================================
--- jcr/trunk/exo.jcr.component.webdav/src/main/java/org/exoplatform/services/jcr/webdav/resource/VersionResource.java 2010-10-19 11:35:09 UTC (rev 3299)
+++ jcr/trunk/exo.jcr.component.webdav/src/main/java/org/exoplatform/services/jcr/webdav/resource/VersionResource.java 2010-10-19 12:50:39 UTC (rev 3300)
@@ -18,10 +18,6 @@
*/
package org.exoplatform.services.jcr.webdav.resource;
-import org.exoplatform.common.util.HierarchicalProperty;
-import org.exoplatform.services.jcr.webdav.util.DeltaVConstants;
-import org.exoplatform.services.jcr.webdav.xml.WebDavNamespaceContext;
-
import java.io.InputStream;
import java.net.URI;
import java.util.Calendar;
@@ -34,6 +30,10 @@
import javax.jcr.version.Version;
import javax.xml.namespace.QName;
+import org.exoplatform.common.util.HierarchicalProperty;
+import org.exoplatform.services.jcr.webdav.util.DeltaVConstants;
+import org.exoplatform.services.jcr.webdav.xml.WebDavNamespaceContext;
+
/**
* Created by The eXo Platform SARL .<br/>
*
@@ -174,6 +174,14 @@
return creationDate;
}
+ else if (DeltaVConstants.GETLASTMODIFIED.equals(name))
+ {
+ Calendar created = version.getNode("jcr:frozenNode").getProperty("jcr:created").getDate();
+ HierarchicalProperty creationDate = new HierarchicalProperty(name, created, MODIFICATION_PATTERN);
+ creationDate.setAttribute("b:dt", "dateTime.1123");
+ return creationDate;
+
+ }
else
{
throw new PathNotFoundException();
Modified: jcr/trunk/exo.jcr.component.webdav/src/test/java/org/exoplatform/services/jcr/webdav/command/TestGet.java
===================================================================
--- jcr/trunk/exo.jcr.component.webdav/src/test/java/org/exoplatform/services/jcr/webdav/command/TestGet.java 2010-10-19 11:35:09 UTC (rev 3299)
+++ jcr/trunk/exo.jcr.component.webdav/src/test/java/org/exoplatform/services/jcr/webdav/command/TestGet.java 2010-10-19 12:50:39 UTC (rev 3300)
@@ -18,29 +18,32 @@
*/
package org.exoplatform.services.jcr.webdav.command;
-import org.exoplatform.common.http.HTTPStatus;
-import org.exoplatform.services.jcr.core.nodetype.ExtendedNodeTypeManager;
-import org.exoplatform.services.jcr.core.nodetype.NodeTypeDataManager;
-import org.exoplatform.services.jcr.impl.core.version.VersionImpl;
-import org.exoplatform.services.jcr.webdav.BaseStandaloneTest;
-import org.exoplatform.services.jcr.webdav.WebDavConstants.WebDAVMethods;
-import org.exoplatform.services.jcr.webdav.utils.TestUtils;
-import org.exoplatform.services.rest.ext.provider.XSLTStreamingOutput;
-import org.exoplatform.services.rest.impl.ContainerResponse;
-
import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
-import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.StringWriter;
+import java.text.SimpleDateFormat;
import java.util.Calendar;
+import java.util.Locale;
import javax.jcr.Node;
+import javax.ws.rs.core.MultivaluedMap;
+import org.exoplatform.common.http.HTTPStatus;
+import org.exoplatform.services.jcr.core.nodetype.ExtendedNodeTypeManager;
+import org.exoplatform.services.jcr.core.nodetype.NodeTypeDataManager;
+import org.exoplatform.services.jcr.impl.core.version.VersionImpl;
+import org.exoplatform.services.jcr.webdav.BaseStandaloneTest;
+import org.exoplatform.services.jcr.webdav.WebDavConst;
+import org.exoplatform.services.jcr.webdav.WebDavConstants.WebDAVMethods;
+import org.exoplatform.services.jcr.webdav.utils.TestUtils;
+import org.exoplatform.services.rest.ExtHttpHeaders;
+import org.exoplatform.services.rest.impl.ContainerResponse;
+import org.exoplatform.services.rest.impl.MultivaluedMapImpl;
+
/**
* Created by The eXo Platform SAS Author : Dmytro Katayev
* work.visor.ck(a)gmail.com Aug 13, 2008
@@ -124,6 +127,51 @@
}
+ /**
+ * Tests if date passed through header If-modified-since of GET method parsed correctly.
+ * Details can be found here: http://jira.exoplatform.org/browse/JCR-1470
+ * @throws Exception
+ */
+ public void testIfModifiedSinceDateParsing() throws Exception
+ {
+ Node fileNode = session.getRootNode().addNode("node", "nt:file");
+ fileNode.addMixin("mix:versionable");
+
+ Node contentNode = fileNode.addNode("jcr:content", "nt:resource");
+ contentNode.setProperty("jcr:mimeType", "text/plain");
+
+ Calendar creationDate = Calendar.getInstance();
+ Calendar IfModifiedSince = Calendar.getInstance();
+ IfModifiedSince.add(Calendar.HOUR, -2);
+ creationDate.add(Calendar.HOUR, -1);
+ contentNode.setProperty("jcr:data", creationDate);
+ contentNode.setProperty("jcr:lastModified", creationDate);
+
+ session.save();
+
+ fileNode.checkin();
+ fileNode.checkout();
+
+ String path =
+ getPathWS() + "/" + fileNode.getName() + "?time=" + IfModifiedSince.getTimeInMillis() + "&version=1";
+
+ SimpleDateFormat dateFormat = new SimpleDateFormat(WebDavConst.DateFormat.IF_MODIFIED_SINCE_PATTERN, Locale.US);
+ String ifModifiedSinceDate = dateFormat.format(IfModifiedSince.getTime());
+
+ MultivaluedMap<String, String> headers = new MultivaluedMapImpl();
+ headers.add(ExtHttpHeaders.IF_MODIFIED_SINCE, ifModifiedSinceDate);
+ ContainerResponse response = service(WebDAVMethods.GET, path, "", headers, null);
+ assertEquals(HTTPStatus.OK, response.getStatus());
+
+ headers.clear();
+
+ IfModifiedSince.add(Calendar.HOUR, +4);
+ ifModifiedSinceDate = dateFormat.format(IfModifiedSince.getTime());
+ headers.add(ExtHttpHeaders.IF_MODIFIED_SINCE, ifModifiedSinceDate);
+ response = service(WebDAVMethods.GET, path, "", headers, null);
+ assertEquals(HTTPStatus.NOT_MODIFIED, response.getStatus());
+ }
+
@Override
protected String getRepositoryName()
{
15 years, 9 months
- 1
- 0
exo-jcr SVN: r3299 - jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/lock/jbosscache.
by do-not-reply＠jboss.org
Author: tolusha
Date: 2010-10-19 07:35:09 -0400 (Tue, 19 Oct 2010)
New Revision: 3299
Modified:
jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/lock/jbosscache/CacheableLockManagerImpl.java
jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/lock/jbosscache/ControllerCacheLoader.java
Log:
JCR-1482: Fix issue when after restarting JCR doesn't retrieve all lock data from DB. After restarting server node will remain locked as expected.
Modified: jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/lock/jbosscache/CacheableLockManagerImpl.java
===================================================================
--- jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/lock/jbosscache/CacheableLockManagerImpl.java 2010-10-19 08:11:37 UTC (rev 3298)
+++ jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/lock/jbosscache/CacheableLockManagerImpl.java 2010-10-19 11:35:09 UTC (rev 3299)
@@ -1065,7 +1065,7 @@
{
public List<LockData> execute(Object arg) throws LockException
{
- Set<Object> nodesId = ((CacheSPI<Serializable, Object>)cache).getNode(lockRoot).getChildrenNamesDirect();
+ Set<Object> nodesId = cache.getChildrenNames(lockRoot);
List<LockData> locksData = new ArrayList<LockData>();
for (Object nodeId : nodesId)
Modified: jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/lock/jbosscache/ControllerCacheLoader.java
===================================================================
--- jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/lock/jbosscache/ControllerCacheLoader.java 2010-10-19 08:11:37 UTC (rev 3298)
+++ jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/lock/jbosscache/ControllerCacheLoader.java 2010-10-19 11:35:09 UTC (rev 3299)
@@ -169,14 +169,7 @@
*/
public Set<?> getChildrenNames(Fqn fqn) throws Exception
{
- if (cache.getCacheStatus() == CacheStatus.STARTING)
- {
- // Try to get the list of children name from the nested cache loader
- return cl.getChildrenNames(fqn);
- }
- // All the data is loaded at startup, so no need to call the nested cache loader for another
- // cache status other than CacheStatus.STARTING
- return null;
+ return cl.getChildrenNames(fqn);
}
/**
15 years, 9 months
- 1
- 0
exo-jcr SVN: r3298 - in jcr/trunk/exo.jcr.component.core/src: test/java/org/exoplatform/services/jcr/api/lock and 1 other directory.
by do-not-reply＠jboss.org
Author: tolusha
Date: 2010-10-19 04:11:37 -0400 (Tue, 19 Oct 2010)
New Revision: 3298
Modified:
jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/lock/jbosscache/CacheableLockManagerImpl.java
jcr/trunk/exo.jcr.component.core/src/test/java/org/exoplatform/services/jcr/api/lock/TestLock.java
Log:
JCR-1482: Fix issue when after restarting JCR doesn't retrieve all lock data from DB. After restarting server node will remain locked as expected.
Modified: jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/lock/jbosscache/CacheableLockManagerImpl.java
===================================================================
--- jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/lock/jbosscache/CacheableLockManagerImpl.java 2010-10-15 14:52:04 UTC (rev 3297)
+++ jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/lock/jbosscache/CacheableLockManagerImpl.java 2010-10-19 08:11:37 UTC (rev 3298)
@@ -1065,7 +1065,7 @@
{
public List<LockData> execute(Object arg) throws LockException
{
- Set<Object> nodesId = cache.getChildrenNames(lockRoot);
+ Set<Object> nodesId = ((CacheSPI<Serializable, Object>)cache).getNode(lockRoot).getChildrenNamesDirect();
List<LockData> locksData = new ArrayList<LockData>();
for (Object nodeId : nodesId)
Modified: jcr/trunk/exo.jcr.component.core/src/test/java/org/exoplatform/services/jcr/api/lock/TestLock.java
===================================================================
--- jcr/trunk/exo.jcr.component.core/src/test/java/org/exoplatform/services/jcr/api/lock/TestLock.java 2010-10-15 14:52:04 UTC (rev 3297)
+++ jcr/trunk/exo.jcr.component.core/src/test/java/org/exoplatform/services/jcr/api/lock/TestLock.java 2010-10-19 08:11:37 UTC (rev 3298)
@@ -42,6 +42,7 @@
private Node lockedNode = null;
+ @Override
public void setUp() throws Exception
{
@@ -428,4 +429,26 @@
assertFalse(childLockNode.isLocked());
}
+
+ /**
+ * Test indicate if locked node after restarting still locked.
+ */
+ public void testLockWithoutDBClean() throws Exception
+ {
+ String locNodeName = "TestLockNode";
+ if (session.getRootNode().hasNode(locNodeName))
+ {
+ Node node = session.getRootNode().getNode(locNodeName);
+ assertTrue(node.isLocked());
+ }
+ else
+ {
+ Node node = session.getRootNode().addNode(locNodeName);
+ node.addMixin("mix:lockable");
+ session.save();
+
+ node.lock(false, true);
+ }
+ }
+
}
15 years, 9 months
- 1
- 0
exo-jcr SVN: r3297 - in jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query: jbosscache and 1 other directories.
by do-not-reply＠jboss.org
Author: nzamosenchuk
Date: 2010-10-15 10:52:04 -0400 (Fri, 15 Oct 2010)
New Revision: 3297
Modified:
jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/AbstractQueryHandler.java
jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/QueryHandler.java
jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/jbosscache/JBossCacheIndexInfos.java
jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/ChunkIndex.java
jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/SearchIndex.java
Log:
EXOJCR-987 : refactoring.
Modified: jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/AbstractQueryHandler.java
===================================================================
--- jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/AbstractQueryHandler.java 2010-10-14 14:11:54 UTC (rev 3296)
+++ jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/AbstractQueryHandler.java 2010-10-15 14:52:04 UTC (rev 3297)
@@ -18,10 +18,10 @@
import org.exoplatform.services.jcr.config.RepositoryConfigurationException;
import org.exoplatform.services.jcr.datamodel.NodeData;
+import org.exoplatform.services.jcr.impl.core.query.lucene.ChunkIndex;
import org.exoplatform.services.jcr.impl.core.query.lucene.DefaultIndexUpdateMonitor;
import org.exoplatform.services.jcr.impl.core.query.lucene.IndexInfos;
import org.exoplatform.services.jcr.impl.core.query.lucene.IndexUpdateMonitor;
-import org.exoplatform.services.jcr.impl.core.query.lucene.ChunkIndex;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Modified: jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/QueryHandler.java
===================================================================
--- jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/QueryHandler.java 2010-10-14 14:11:54 UTC (rev 3296)
+++ jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/QueryHandler.java 2010-10-15 14:52:04 UTC (rev 3297)
@@ -21,9 +21,9 @@
import org.exoplatform.services.jcr.datamodel.NodeData;
import org.exoplatform.services.jcr.impl.core.SessionDataManager;
import org.exoplatform.services.jcr.impl.core.SessionImpl;
+import org.exoplatform.services.jcr.impl.core.query.lucene.ChunkIndex;
import org.exoplatform.services.jcr.impl.core.query.lucene.IndexInfos;
import org.exoplatform.services.jcr.impl.core.query.lucene.IndexUpdateMonitor;
-import org.exoplatform.services.jcr.impl.core.query.lucene.ChunkIndex;
import org.exoplatform.services.jcr.impl.core.query.lucene.QueryHits;
import java.io.IOException;
Modified: jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/jbosscache/JBossCacheIndexInfos.java
===================================================================
--- jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/jbosscache/JBossCacheIndexInfos.java 2010-10-14 14:11:54 UTC (rev 3296)
+++ jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/jbosscache/JBossCacheIndexInfos.java 2010-10-15 14:52:04 UTC (rev 3297)
@@ -21,8 +21,8 @@
import org.exoplatform.services.jcr.impl.core.query.IndexerIoMode;
import org.exoplatform.services.jcr.impl.core.query.IndexerIoModeHandler;
import org.exoplatform.services.jcr.impl.core.query.IndexerIoModeListener;
+import org.exoplatform.services.jcr.impl.core.query.lucene.ChunkIndex;
import org.exoplatform.services.jcr.impl.core.query.lucene.IndexInfos;
-import org.exoplatform.services.jcr.impl.core.query.lucene.ChunkIndex;
import org.exoplatform.services.jcr.impl.util.io.PrivilegedCacheHelper;
import org.exoplatform.services.log.ExoLogger;
import org.exoplatform.services.log.Log;
Modified: jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/ChunkIndex.java
===================================================================
--- jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/ChunkIndex.java 2010-10-14 14:11:54 UTC (rev 3296)
+++ jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/ChunkIndex.java 2010-10-15 14:52:04 UTC (rev 3297)
@@ -17,21 +17,18 @@
package org.exoplatform.services.jcr.impl.core.query.lucene;
import org.apache.lucene.document.Document;
-import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
import org.exoplatform.services.jcr.dataflow.ItemDataConsumer;
import org.exoplatform.services.jcr.datamodel.ItemData;
import org.exoplatform.services.jcr.datamodel.NodeData;
-import org.exoplatform.services.jcr.impl.core.query.IndexerIoMode;
-import org.exoplatform.services.jcr.impl.core.query.IndexerIoModeHandler;
+import org.exoplatform.services.jcr.impl.core.query.ChunkService;
import org.exoplatform.services.jcr.impl.core.query.IndexingTree;
import org.exoplatform.services.jcr.impl.core.query.lucene.directory.DirectoryManager;
import org.exoplatform.services.jcr.impl.util.SecurityHelper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.FileNotFoundException;
import java.io.IOException;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
@@ -39,12 +36,10 @@
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.Map.Entry;
import javax.jcr.ItemNotFoundException;
import javax.jcr.RepositoryException;
@@ -61,17 +56,12 @@
private static final Logger log = LoggerFactory.getLogger("exo.jcr.component.core.MultiIndex");
/**
- * Names of active persistent index directories.
- */
- private IndexInfos indexNames;
-
- /**
* List of open persistent indexes. This list may also contain an open
- * PersistentIndex owned by the IndexMerger daemon. Such an index is not
+ * PersistentIndexChunk owned by the IndexMerger daemon. Such an index is not
* registered with indexNames and <b>must not</b> be used in regular index
* operations (delete node, etc.)!
*/
- private final List<PersistentIndex> indexes = new ArrayList<PersistentIndex>();
+ private final List<PersistentIndexChunk> indexes = new ArrayList<PersistentIndexChunk>();
/**
* The internal namespace mappings of the query manager.
@@ -124,10 +114,7 @@
*/
private final IndexFormatVersion version;
- /**
- * The handler of the Indexer io mode
- */
- private final IndexerIoModeHandler modeHandler;
+ private final ChunkService chunkService;
/**
* Creates a new MultiIndex.
@@ -140,10 +127,8 @@
* @throws IOException
* if an error occurs
*/
- ChunkIndex(SearchIndex handler, IndexingTree indexingTree, IndexerIoModeHandler modeHandler, IndexInfos indexInfos,
- IndexUpdateMonitor indexUpdateMonitor) throws IOException
+ ChunkIndex(SearchIndex handler, IndexingTree indexingTree) throws IOException
{
- this.modeHandler = modeHandler;
this.directoryManager = handler.getDirectoryManager();
// this method is run in privileged mode internally
this.indexDir = directoryManager.getDirectory(".");
@@ -151,27 +136,20 @@
this.cache = new DocNumberCache(handler.getCacheSize());
this.indexingTree = indexingTree;
this.nsMappings = handler.getNamespaceMappings();
- this.indexNames = indexInfos;
- this.indexNames.setDirectory(indexDir);
- // this method is run in privileged mode internally
- this.indexNames.read();
+ // TODO remove this stub
+ this.chunkService = new ChunkServiceImpl();
- // TODO remove hard-coded piece
- this.indexNames.addName("1");
- this.indexNames.addName("2");
- this.indexNames.addName("3");
- this.indexNames.addName("4");
-
// this method is run in privileged mode internally
IndexingQueueStore store = new IndexingQueueStore(indexDir);
// initialize indexing queue
this.indexingQueue = new IndexingQueue(store);
- // copy current index names
- Set<String> currentNames = new HashSet<String>(indexNames.getNames());
// open persistent indexes
- for (String name : currentNames)
+
+ Set<Integer> assignedChunks = chunkService.getAssignedChunks();
+
+ for (int i = 0; i < chunkService.getChunkCount(); i++)
{
// only open if it still exists
// it is possible that indexNames still contains a name for
@@ -185,9 +163,10 @@
// // move on to next index
// continue;
// }
- PersistentIndex index =
- new PersistentIndex(name, handler.getTextAnalyzer(), handler.getSimilarity(), cache, indexingQueue,
- directoryManager);
+ PersistentIndexChunk index =
+ new PersistentIndexChunk(i, handler.getTextAnalyzer(), handler.getSimilarity(), cache, indexingQueue,
+ directoryManager, !assignedChunks.contains(Integer.valueOf(i)));
+
index.setMaxFieldLength(handler.getMaxFieldLength());
index.setUseCompoundFile(handler.getUseCompoundFile());
index.setTermInfosIndexDivisor(handler.getTermInfosIndexDivisor());
@@ -213,52 +192,105 @@
});
}
indexingQueue.initialize(this);
- this.indexNames.setMultiIndex(this);
}
/**
- * Returns the number of documents in this index.
+ * Adds a document to the index.
*
- * @return the number of documents in this index.
+ * @param doc
+ * the document to add.
* @throws IOException
- * if an error occurs while reading from the index.
+ * if an error occurs while adding the document to the index.
*/
- int numDocs() throws IOException
+ public void addDocument(Document doc) throws IOException
{
- if (indexNames.size() == 0)
+ update(Collections.EMPTY_LIST, Arrays.asList(new Document[]{doc}));
+ }
+
+ /**
+ * Closes this <code>MultiIndex</code>.
+ */
+ public void close()
+ {
+
+ // stop index merger
+ // when calling this method we must not lock this MultiIndex, otherwise
+ // a deadlock might occur
+
+ synchronized (this)
{
- return 0;
- }
- else
- {
- final CachingMultiIndexReader reader = getIndexReader();
+
+ // commit / close indexes
try
{
- return reader.numDocs();
+ releaseMultiReader();
}
- finally
+ catch (IOException e)
{
- SecurityHelper.doPriviledgedIOExceptionAction(new PrivilegedExceptionAction<Object>()
- {
- public Object run() throws Exception
- {
- reader.release();
- return null;
- }
- });
+ log.error("Exception while closing search index.", e);
}
+
+ // TODO Should they be commited before close?!
+ for (int i = 0; i < indexes.size(); i++)
+ {
+ (indexes.get(i)).close();
+ }
+
+ // close indexing queue
+ indexingQueue.close();
+
+ // finally close directory
+ try
+ {
+ indexDir.close();
+ }
+ catch (IOException e)
+ {
+ log.error("Exception while closing directory.", e);
+ }
}
}
/**
- * @return the index format version for this multi index.
+ * Returns a lucene Document for the <code>node</code>.
+ *
+ * @param node
+ * the node to index.
+ * @return the index document.
+ * @throws RepositoryException
+ * if an error occurs while reading from the workspace.
*/
- IndexFormatVersion getIndexFormatVersion()
+ public Document createDocument(NodeData node) throws RepositoryException
{
- return version;
+ return handler.createDocument(node, nsMappings, version);
}
/**
+ * Returns a lucene Document for the Node with <code>id</code>.
+ *
+ * @param id
+ * the id of the node to index.
+ * @return the index document.
+ * @throws RepositoryException
+ * if an error occurs while reading from the workspace or if
+ * there is no node with <code>id</code>.
+ */
+ public Document createDocument(String id) throws RepositoryException
+ {
+ ItemData data = handler.getContext().getItemStateManager().getItemData(id);
+ if (data == null)
+ {
+ throw new ItemNotFoundException("Item id=" + id + " not found");
+ }
+ if (!data.isNode())
+ {
+ throw new RepositoryException("Item with id " + id + " is not a node");
+ }
+ return createDocument((NodeData)data);
+
+ }
+
+ /**
* Creates an initial index by traversing the node hierarchy starting at the
* node with <code>rootId</code>.
*
@@ -273,9 +305,8 @@
* @throws IllegalStateException
* if this index is not empty.
*/
- void createInitialIndex(ItemDataConsumer stateMgr) throws IOException
+ public void createInitialIndex(ItemDataConsumer stateMgr) throws IOException
{
- // TODO: re-study check!!!
CachingMultiIndexReader reader = getIndexReader();
int numDocs = reader.numDocs();
reader.release();
@@ -286,8 +317,6 @@
{
long count = 0;
// traverse and index workspace
-
- // NodeData rootState = (NodeData) stateMgr.getItemData(rootId);
count = createIndex(indexingTree.getIndexingRoot(), stateMgr, count);
log.info("Created initial index for {} nodes", new Long(count));
@@ -312,138 +341,24 @@
}
/**
- * Atomically updates the index by removing some documents and adding
- * others.
- *
- * @param remove
- * collection of <code>UUID</code>s that identify documents to
- * remove
- * @param add
- * collection of <code>Document</code>s to add. Some of the
- * elements in this collection may be <code>null</code>, to
- * indicate that a node could not be indexed successfully.
- * @throws IOException
- * if an error occurs while updating the index.
+ * @return the index format version for this multi index.
*/
- synchronized void update(final Collection<String> remove, final Collection<Document> add) throws IOException
+ public IndexFormatVersion getIndexFormatVersion()
{
- SecurityHelper.doPriviledgedIOExceptionAction(new PrivilegedExceptionAction<Object>()
- {
- public Object run() throws Exception
- {
- // make sure a reader is available during long updates
- if (add.size() > handler.getBufferSize())
- {
- try
- {
- getIndexReader().release();
- }
- catch (IOException e)
- {
- // do not fail if an exception is thrown here
- log.warn("unable to prepare index reader " + "for queries during update", e);
- }
- }
- try
- {
- for (Iterator<String> it = remove.iterator(); it.hasNext();)
- {
- String uuidString = it.next();
- // check if indexing queue is still working on
- // this node from a previous update
- Document doc = indexingQueue.removeDocument(uuidString);
- if (doc != null)
- {
- Util.disposeDocument(doc);
- }
- Term idTerm = new Term(FieldNames.UUID, uuidString);
- getChunk(uuidString).removeDocument(idTerm);
- }
- for (Iterator<Document> it = add.iterator(); it.hasNext();)
- {
- Document doc = it.next();
- if (doc != null)
- {
- if (doc != null)
- {
- String uuid = doc.get(FieldNames.UUID);
- getChunk(uuid).addDocuments(new Document[]{doc});
- }
- }
- }
- // TODO for owning indexes only
- for (PersistentIndex idx : indexes)
- {
- idx.commit();
- }
- }
- finally
- {
- releaseMultiReader();
- }
- return null;
- }
- });
+ return version;
}
/**
- * Adds a document to the index.
+ * Returns the indexing queue for this multi index.
*
- * @param doc
- * the document to add.
- * @throws IOException
- * if an error occurs while adding the document to the index.
+ * @return the indexing queue for this multi index.
*/
- void addDocument(Document doc) throws IOException
+ public IndexingQueue getIndexingQueue()
{
- update(Collections.EMPTY_LIST, Arrays.asList(new Document[]{doc}));
+ return indexingQueue;
}
/**
- * Deletes the first document that matches the <code>uuid</code>.
- *
- * @param uuid
- * document that match this <code>uuid</code> will be deleted.
- * @throws IOException
- * if an error occurs while deleting the document.
- */
- void removeDocument(String uuid) throws IOException
- {
- update(Arrays.asList(new String[]{uuid}), Collections.EMPTY_LIST);
- }
-
- /**
- * Deletes all documents that match the <code>uuid</code>.
- *
- * @param uuid
- * documents that match this <code>uuid</code> will be deleted.
- * @return the number of deleted documents.
- * @throws IOException
- * if an error occurs while deleting documents.
- */
- synchronized int removeAllDocuments(String uuid) throws IOException
- {
- int num;
- try
- {
- Term idTerm = new Term(FieldNames.UUID, uuid.toString());
- num = getChunk(uuid).removeDocument(idTerm);
- for (int i = 0; i < indexes.size(); i++)
- {
- PersistentIndex index = indexes.get(i);
- // TODO only remove documents from owning indexes
- int removed = index.removeDocument(idTerm);
- num += removed;
- }
- }
- finally
- {
- releaseMultiReader();
- }
- return num;
- }
-
- /**
* Returns an read-only <code>IndexReader</code> that spans alls indexes of
* this <code>MultiIndex</code>.
*
@@ -484,7 +399,7 @@
if (multiReader == null)
{
List<ReadOnlyIndexReader> readerList = new ArrayList<ReadOnlyIndexReader>();
- for (PersistentIndex idx : indexes)
+ for (PersistentIndexChunk idx : indexes)
{
readerList.add(idx.getReadOnlyIndexReader(initCache));
}
@@ -495,153 +410,244 @@
return multiReader;
}
});
+ }
+ /**
+ * TODO solve this architecture issue
+ * This belongs only to MultiIndex with volatile present
+ *
+ * @return
+ */
+ @Deprecated
+ public boolean getRedoLogApplied()
+ {
+ return false;
}
/**
- * Closes this <code>MultiIndex</code>.
+ * Returns the number of documents in this index.
+ *
+ * @return the number of documents in this index.
+ * @throws IOException
+ * if an error occurs while reading from the index.
*/
- void close()
+ public int numDocs() throws IOException
{
- if (modeHandler.getMode().equals(IndexerIoMode.READ_WRITE))
+ final CachingMultiIndexReader reader = getIndexReader();
+ try
{
-
- // stop index merger
- // when calling this method we must not lock this MultiIndex, otherwise
- // a deadlock might occur
-
- synchronized (this)
+ return reader.numDocs();
+ }
+ finally
+ {
+ SecurityHelper.doPriviledgedIOExceptionAction(new PrivilegedExceptionAction<Object>()
{
-
- // commit / close indexes
- try
+ public Object run() throws Exception
{
- releaseMultiReader();
+ reader.release();
+ return null;
}
- catch (IOException e)
- {
- log.error("Exception while closing search index.", e);
- }
-
- // TODO Should they be commited before close?!
- for (int i = 0; i < indexes.size(); i++)
- {
- (indexes.get(i)).close();
- }
-
- // close indexing queue
- indexingQueue.close();
-
- // finally close directory
- try
- {
- indexDir.close();
- }
- catch (IOException e)
- {
- log.error("Exception while closing directory.", e);
- }
- }
+ });
}
}
/**
- * Returns the namespace mappings of this search index.
- *
- * @return the namespace mappings of this search index.
+ * TODO
*/
- NamespaceMappings getNamespaceMappings()
+ public void reassignChunks()
{
- return nsMappings;
- }
- /**
- * Returns the indexing queue for this multi index.
- *
- * @return the indexing queue for this multi index.
- */
- public IndexingQueue getIndexingQueue()
- {
- return indexingQueue;
}
/**
- * Returns a lucene Document for the <code>node</code>.
+ * Deletes all documents that match the <code>uuid</code>.
*
- * @param node
- * the node to index.
- * @return the index document.
- * @throws RepositoryException
- * if an error occurs while reading from the workspace.
+ * @param uuid
+ * documents that match this <code>uuid</code> will be deleted.
+ * @return the number of deleted documents.
+ * @throws IOException
+ * if an error occurs while deleting documents.
*/
- Document createDocument(NodeData node) throws RepositoryException
+ public synchronized int removeAllDocuments(String uuid) throws IOException
{
- return handler.createDocument(node, nsMappings, version);
+ int num;
+ try
+ {
+ Term idTerm = new Term(FieldNames.UUID, uuid.toString());
+ num = getChunk(uuid).removeDocument(idTerm);
+ for (int i = 0; i < indexes.size(); i++)
+ {
+ PersistentIndexChunk index = indexes.get(i);
+ // TODO only remove documents from owning indexes
+ int removed = index.removeDocument(idTerm);
+ num += removed;
+ }
+ }
+ finally
+ {
+ releaseMultiReader();
+ }
+ return num;
}
/**
- * Returns a lucene Document for the Node with <code>id</code>.
+ * Deletes the first document that matches the <code>uuid</code>.
*
- * @param id
- * the id of the node to index.
- * @return the index document.
- * @throws RepositoryException
- * if an error occurs while reading from the workspace or if
- * there is no node with <code>id</code>.
+ * @param uuid
+ * document that match this <code>uuid</code> will be deleted.
+ * @throws IOException
+ * if an error occurs while deleting the document.
*/
- Document createDocument(String id) throws RepositoryException
+ public void removeDocument(String uuid) throws IOException
{
- ItemData data = handler.getContext().getItemStateManager().getItemData(id);
- if (data == null)
- {
- throw new ItemNotFoundException("Item id=" + id + " not found");
- }
- if (!data.isNode())
- {
- throw new RepositoryException("Item with id " + id + " is not a node");
- }
- return createDocument((NodeData)data);
-
+ update(Arrays.asList(new String[]{uuid}), Collections.EMPTY_LIST);
}
/**
- * Releases the {@link #multiReader} and sets it <code>null</code>. If the
- * reader is already <code>null</code> this method does nothing. When this
- * method returns {@link #multiReader} is guaranteed to be <code>null</code>
- * even if an exception is thrown.
- * <p/>
- * Please note that this method does not take care of any synchronization. A
- * caller must ensure that it is the only thread operating on this multi
- * index, or that it holds the {@link #updateMonitor}.
+ * Atomically updates the index by removing some documents and adding
+ * others.
*
+ * @param remove
+ * collection of <code>UUID</code>s that identify documents to
+ * remove
+ * @param add
+ * collection of <code>Document</code>s to add. Some of the
+ * elements in this collection may be <code>null</code>, to
+ * indicate that a node could not be indexed successfully.
* @throws IOException
- * if an error occurs while releasing the reader.
+ * if an error occurs while updating the index.
*/
- void releaseMultiReader() throws IOException
+ public synchronized void update(final Collection<String> remove, final Collection<Document> add) throws IOException
{
SecurityHelper.doPriviledgedIOExceptionAction(new PrivilegedExceptionAction<Object>()
{
public Object run() throws Exception
{
- if (multiReader != null)
+ // make sure a reader is available during long updates
+ if (add.size() > handler.getBufferSize())
{
try
{
- multiReader.release();
+ getIndexReader().release();
}
- finally
+ catch (IOException e)
{
- multiReader = null;
+ // do not fail if an exception is thrown here
+ log.warn("unable to prepare index reader " + "for queries during update", e);
}
}
+ try
+ {
+ for (Iterator<String> it = remove.iterator(); it.hasNext();)
+ {
+ String uuidString = it.next();
+ // check if indexing queue is still working on
+ // this node from a previous update
+ Document doc = indexingQueue.removeDocument(uuidString);
+ if (doc != null)
+ {
+ Util.disposeDocument(doc);
+ }
+ Term idTerm = new Term(FieldNames.UUID, uuidString);
+ getChunk(uuidString).removeDocument(idTerm);
+ }
+ for (Iterator<Document> it = add.iterator(); it.hasNext();)
+ {
+ Document doc = it.next();
+ if (doc != null)
+ {
+ if (doc != null)
+ {
+ String uuid = doc.get(FieldNames.UUID);
+ getChunk(uuid).addDocuments(new Document[]{doc});
+ }
+ }
+ }
+ // TODO for owning indexes only
+ for (PersistentIndexChunk idx : indexes)
+ {
+ idx.commit();
+ }
+ }
+ finally
+ {
+ releaseMultiReader();
+ }
return null;
}
});
}
- // -------------------------< internal
- // >-------------------------------------
+ /**
+ * Checks the indexing queue for finished text extrator jobs and updates the
+ * index accordingly if there are any new ones.
+ *
+ * @param transactionPresent
+ * whether a transaction is in progress and the current
+ * {@link #getTransactionId()} should be used. If
+ * <code>false</code> a new transaction is created when documents
+ * are transfered from the indexing queue to the index.
+ */
+ private void checkIndexingQueue(boolean transactionPresent)
+ {
+ Document[] docs = indexingQueue.getFinishedDocuments();
+ Map<String, Document> finished = new HashMap<String, Document>();
+ for (int i = 0; i < docs.length; i++)
+ {
+ String uuid = docs[i].get(FieldNames.UUID);
+ finished.put(uuid, docs[i]);
+ }
+ // now update index with the remaining ones if there are any
+ if (!finished.isEmpty())
+ {
+ log.info("updating index with {} nodes from indexing queue.", new Long(finished.size()));
+
+ // remove documents from the queue
+ for (Iterator<String> it = finished.keySet().iterator(); it.hasNext();)
+ {
+ indexingQueue.removeDocument(it.next());
+ }
+
+ try
+ {
+ if (transactionPresent)
+ {
+ for (Iterator<String> it = finished.keySet().iterator(); it.hasNext();)
+ {
+ String uuidString = it.next();
+ // check if indexing queue is still working on
+ // this node from a previous update
+ Document doc = indexingQueue.removeDocument(uuidString);
+ if (doc != null)
+ {
+ Util.disposeDocument(doc);
+ }
+ Term idTerm = new Term(FieldNames.UUID, uuidString);
+ // if the document cannot be deleted from the volatile index
+ // delete it from one of the persistent indexes.
+ getChunk(uuidString).removeDocument(idTerm);
+ }
+ for (Iterator<Document> it = finished.values().iterator(); it.hasNext();)
+ {
+ Document doc = it.next();
+ String uuid = doc.get(FieldNames.UUID);
+ getChunk(uuid).addDocuments(new Document[]{doc});
+ }
+ }
+ else
+ {
+ update(finished.keySet(), finished.values());
+ }
+ }
+ catch (IOException e)
+ {
+ // update failed
+ log.warn("Failed to update index with deferred text extraction", e);
+ }
+ }
+ }
+
/**
* Recursively creates an index starting with the NodeState
* <code>node</code>.
@@ -700,96 +706,43 @@
return count;
}
- /**
- * Checks the indexing queue for finished text extrator jobs and updates the
- * index accordingly if there are any new ones. This method is synchronized
- * and should only be called by the timer task that periodically checks if
- * there are documents ready in the indexing queue. A new transaction is
- * used when documents are transfered from the indexing queue to the index.
- */
- private synchronized void checkIndexingQueue()
+ private PersistentIndexChunk getChunk(String uuid)
{
- checkIndexingQueue(false);
+ return indexes.get(chunkService.getChunkId(uuid));
}
/**
- * Checks the indexing queue for finished text extrator jobs and updates the
- * index accordingly if there are any new ones.
+ * Releases the {@link #multiReader} and sets it <code>null</code>. If the
+ * reader is already <code>null</code> this method does nothing. When this
+ * method returns {@link #multiReader} is guaranteed to be <code>null</code>
+ * even if an exception is thrown.
+ * <p/>
+ * Please note that this method does not take care of any synchronization. A
+ * caller must ensure that it is the only thread operating on this multi
+ * index, or that it holds the {@link #updateMonitor}.
*
- * @param transactionPresent
- * whether a transaction is in progress and the current
- * {@link #getTransactionId()} should be used. If
- * <code>false</code> a new transaction is created when documents
- * are transfered from the indexing queue to the index.
+ * @throws IOException
+ * if an error occurs while releasing the reader.
*/
- private void checkIndexingQueue(boolean transactionPresent)
+ private void releaseMultiReader() throws IOException
{
- Document[] docs = indexingQueue.getFinishedDocuments();
- Map<String, Document> finished = new HashMap<String, Document>();
- for (int i = 0; i < docs.length; i++)
+ SecurityHelper.doPriviledgedIOExceptionAction(new PrivilegedExceptionAction<Object>()
{
- String uuid = docs[i].get(FieldNames.UUID);
- finished.put(uuid, docs[i]);
- }
-
- // now update index with the remaining ones if there are any
- if (!finished.isEmpty())
- {
- log.info("updating index with {} nodes from indexing queue.", new Long(finished.size()));
-
- // remove documents from the queue
- for (Iterator<String> it = finished.keySet().iterator(); it.hasNext();)
+ public Object run() throws Exception
{
- indexingQueue.removeDocument(it.next());
- }
-
- try
- {
- if (transactionPresent)
+ if (multiReader != null)
{
- for (Iterator<String> it = finished.keySet().iterator(); it.hasNext();)
+ try
{
- String uuidString = it.next();
- // check if indexing queue is still working on
- // this node from a previous update
- Document doc = indexingQueue.removeDocument(uuidString);
- if (doc != null)
- {
- Util.disposeDocument(doc);
- }
- Term idTerm = new Term(FieldNames.UUID, uuidString);
- // if the document cannot be deleted from the volatile index
- // delete it from one of the persistent indexes.
- getChunk(uuidString).removeDocument(idTerm);
+ multiReader.release();
}
- for (Iterator<Document> it = finished.values().iterator(); it.hasNext();)
+ finally
{
- Document doc = it.next();
- String uuid = doc.get(FieldNames.UUID);
- getChunk(uuid).addDocuments(new Document[]{doc});
+ multiReader = null;
}
}
- else
- {
- update(finished.keySet(), finished.values());
- }
+ return null;
}
- catch (IOException e)
- {
- // update failed
- log.warn("Failed to update index with deferred text extraction", e);
- }
- }
+ });
}
-
- @Deprecated
- public boolean getRedoLogApplied()
- {
- return false;
- }
-
- private PersistentIndex getChunk(String uuid)
- {
- return indexes.get(0);
- }
-}
+}
\ No newline at end of file
Modified: jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/SearchIndex.java
===================================================================
--- jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/SearchIndex.java 2010-10-14 14:11:54 UTC (rev 3296)
+++ jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/SearchIndex.java 2010-10-15 14:52:04 UTC (rev 3297)
@@ -570,7 +570,7 @@
indexingConfig = createIndexingConfiguration(nsMappings);
analyzer.setIndexingConfig(indexingConfig);
- index = new ChunkIndex(this, context.getIndexingTree(), modeHandler, getIndexInfos(), getIndexUpdateMonitor());
+ index = new ChunkIndex(this, context.getIndexingTree());
// if RW mode, create initial index and start check
if (modeHandler.getMode() == IndexerIoMode.READ_WRITE)
{
15 years, 9 months
- 1
- 0
exo-jcr SVN: r3296 - jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/dataflow/persistent/jbosscache.
by do-not-reply＠jboss.org
Author: tolusha
Date: 2010-10-14 10:11:54 -0400 (Thu, 14 Oct 2010)
New Revision: 3296
Modified:
jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/dataflow/persistent/jbosscache/JBossCacheWorkspaceStorageCache.java
Log:
EXOJCR-998: support shareable cache
Modified: jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/dataflow/persistent/jbosscache/JBossCacheWorkspaceStorageCache.java
===================================================================
--- jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/dataflow/persistent/jbosscache/JBossCacheWorkspaceStorageCache.java 2010-10-14 13:35:50 UTC (rev 3295)
+++ jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/dataflow/persistent/jbosscache/JBossCacheWorkspaceStorageCache.java 2010-10-14 14:11:54 UTC (rev 3296)
@@ -332,7 +332,7 @@
JBOSSCACHE_EXPIRATION_DEFAULT));
this.itemsRoot = Fqn.fromRelativeElements(rootFqn, ITEMS);
- this.nullItemsRoot = Fqn.fromElements(NULL_ITEMS);
+ this.nullItemsRoot = Fqn.fromRelativeElements(rootFqn, NULL_ITEMS);
this.childNodes = Fqn.fromRelativeElements(rootFqn, CHILD_NODES);
this.childProps = Fqn.fromRelativeElements(rootFqn, CHILD_PROPS);
this.childNodesList = Fqn.fromRelativeElements(rootFqn, CHILD_NODES_LIST);
15 years, 9 months
- 1
- 0
exo-jcr SVN: r3295 - jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/dataflow/persistent.
by do-not-reply＠jboss.org
Author: tolusha
Date: 2010-10-14 09:35:50 -0400 (Thu, 14 Oct 2010)
New Revision: 3295
Modified:
jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/dataflow/persistent/VersionableWorkspaceDataManager.java
Log:
EXOJCR-993: Check that CacheableWorkspaceDataManager.getCachedItemData() method can return NullNodeData
Modified: jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/dataflow/persistent/VersionableWorkspaceDataManager.java
===================================================================
--- jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/dataflow/persistent/VersionableWorkspaceDataManager.java 2010-10-14 13:01:15 UTC (rev 3294)
+++ jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/dataflow/persistent/VersionableWorkspaceDataManager.java 2010-10-14 13:35:50 UTC (rev 3295)
@@ -28,6 +28,7 @@
import org.exoplatform.services.jcr.datamodel.ItemData;
import org.exoplatform.services.jcr.datamodel.ItemType;
import org.exoplatform.services.jcr.datamodel.NodeData;
+import org.exoplatform.services.jcr.datamodel.NullNodeData;
import org.exoplatform.services.jcr.datamodel.PropertyData;
import org.exoplatform.services.jcr.datamodel.QPath;
import org.exoplatform.services.jcr.datamodel.QPathEntry;
@@ -158,30 +159,43 @@
{
// from cache at first
ItemData cdata = persistentManager.getCachedItemData(identifier);
- if (cdata != null)
+ if (cdata != null && !(cdata instanceof NullNodeData))
+ {
return super.getItemData(identifier);
+ }
if (!this.equals(versionDataManager) && !identifier.equals(Constants.ROOT_UUID))
{
// search in System cache for /jcr:system nodes only
cdata = versionDataManager.persistentManager.getCachedItemData(identifier);
- if (cdata != null)
+ if (cdata != null && !(cdata instanceof NullNodeData))
+ {
if (isSystemDescendant(cdata.getQPath()))
+ {
return versionDataManager.getItemData(identifier);
+ }
else
+ {
return null;
+ }
+ }
}
// then from persistence
ItemData data = super.getItemData(identifier);
if (data != null)
+ {
return data;
+ }
+
else if (!this.equals(versionDataManager))
{
// try from version storage if not the same
data = versionDataManager.getItemData(identifier);
if (data != null && isSystemDescendant(data.getQPath()))
+ {
return data;
+ }
}
return null;
}
15 years, 9 months
- 1
- 0
exo-jcr SVN: r3294 - jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/dataflow/persistent/jbosscache.
by do-not-reply＠jboss.org
Author: tolusha
Date: 2010-10-14 09:01:15 -0400 (Thu, 14 Oct 2010)
New Revision: 3294
Modified:
jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/dataflow/persistent/jbosscache/JBossCacheWorkspaceStorageCache.java
Log:
EXOJCR-998: Null values managed locally
Modified: jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/dataflow/persistent/jbosscache/JBossCacheWorkspaceStorageCache.java
===================================================================
--- jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/dataflow/persistent/jbosscache/JBossCacheWorkspaceStorageCache.java 2010-10-13 11:53:35 UTC (rev 3293)
+++ jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/dataflow/persistent/jbosscache/JBossCacheWorkspaceStorageCache.java 2010-10-14 13:01:15 UTC (rev 3294)
@@ -906,7 +906,7 @@
// remove possible NullNodeData from cache
boolean local = cache.isLocal();
- cache.setLocal(false);
+ cache.setLocal(true);
removeNullNode(node);
@@ -999,7 +999,7 @@
// remove possible NullNodeData from cache
boolean local = cache.isLocal();
- cache.setLocal(false);
+ cache.setLocal(true);
removeNullNode(prop);
@@ -1016,8 +1016,7 @@
cache.addToList(makeChildListFqn(childPropsList, prop.getParentIdentifier()), ITEM_LIST, prop.getIdentifier());
}
- ItemData result =
- get(prop.getParentIdentifier(), prop.getQPath().getEntries()[prop.getQPath().getEntries().length - 1]);
+ get(prop.getParentIdentifier(), prop.getQPath().getEntries()[prop.getQPath().getEntries().length - 1]);
// add in ITEMS
return (PropertyData)cache.put(makeItemFqn(prop.getIdentifier()), ITEM_DATA, prop);
15 years, 9 months
- 1
- 0
exo-jcr SVN: r3293 - kernel/trunk/exo.kernel.component.common/src/main/java/org/exoplatform/services/rpc/impl and 1 other directories.
by do-not-reply＠jboss.org
Author: nfilotto
Date: 2010-10-13 07:53:35 -0400 (Wed, 13 Oct 2010)
New Revision: 3293
Modified:
jcr/trunk/exo.jcr.docs/exo.jcr.docs.developer/en/src/main/docbook/en-US/modules/kernel/rpc-service.xml
kernel/trunk/exo.kernel.component.common/src/main/java/org/exoplatform/services/rpc/impl/RPCServiceImpl.java
kernel/trunk/exo.kernel.component.common/src/test/java/org/exoplatform/services/rpc/impl/TestRPCServiceImpl.java
Log:
EXOJCR-967: Parameters allow-failover and retry-timeout have been added
Modified: jcr/trunk/exo.jcr.docs/exo.jcr.docs.developer/en/src/main/docbook/en-US/modules/kernel/rpc-service.xml
===================================================================
--- jcr/trunk/exo.jcr.docs/exo.jcr.docs.developer/en/src/main/docbook/en-US/modules/kernel/rpc-service.xml 2010-10-12 09:47:34 UTC (rev 3292)
+++ jcr/trunk/exo.jcr.docs/exo.jcr.docs.developer/en/src/main/docbook/en-US/modules/kernel/rpc-service.xml 2010-10-13 11:53:35 UTC (rev 3293)
@@ -149,6 +149,14 @@
<name>jgroups-default-timeout</name>
<value>0</value>
</value-param>
+ <value-param>
+ <name>allow-failover</name>
+ <value>true</value>
+ </value-param>
+ <value-param>
+ <name>retry-timeout</name>
+ <value>20000</value>
+ </value-param>
</init-params>
</component>
...
@@ -182,8 +190,31 @@
<entry>This is the default timeout to use if the timeout is not
given, if no response could be get after this timeout an exception
will be thrown. This parameter is optional and its default value
- is 0 which means that we don't use any timeout by default.</entry>
+ is 0 which means that we don't use any timeout by default. This
+ parameter is expressed in milliseconds.</entry>
</row>
+
+ <row>
+ <entry><emphasis>allow-failover</emphasis></entry>
+
+ <entry>This is parameter indicates whether a command on the
+ coordinator needs to be relaunched or not if the coordintator
+ seems to have left the cluster. This parameter only affects the
+ behavior of the methods
+ <emphasis>executeCommandOnCoordinator</emphasis>. This parameter
+ is optional and its default value is true.</entry>
+ </row>
+
+ <row>
+ <entry><emphasis>retry-timeout</emphasis></entry>
+
+ <entry>This parameter is the maximum amount of time to wait until
+ the new coordinator is elected. This parameter is linked to the
+ parameter <emphasis>allow-failover</emphasis>, and thus used in
+ the exact same conditions. This parameter is optional and its
+ default value is 20000. This parameter is expressed in
+ milliseconds.</entry>
+ </row>
</tbody>
</tgroup>
</table>
@@ -213,8 +244,8 @@
<listitem>
<para>Register a <emphasis>SingleMethodCallCommand</emphasis> that
will call <emphasis>getName()</emphasis> on the Object
- <emphasis>myService</emphasis> anytime the command will be executed.
- </para>
+ <emphasis>myService</emphasis> anytime the command will be
+ executed.</para>
</listitem>
<listitem>
Modified: kernel/trunk/exo.kernel.component.common/src/main/java/org/exoplatform/services/rpc/impl/RPCServiceImpl.java
===================================================================
--- kernel/trunk/exo.kernel.component.common/src/main/java/org/exoplatform/services/rpc/impl/RPCServiceImpl.java 2010-10-12 09:47:34 UTC (rev 3292)
+++ kernel/trunk/exo.kernel.component.common/src/main/java/org/exoplatform/services/rpc/impl/RPCServiceImpl.java 2010-10-13 11:53:35 UTC (rev 3293)
@@ -85,27 +85,42 @@
/**
* The name of the parameter for the location of the JGroups configuration.
*/
- public static final String PARAM_JGROUPS_CONFIG = "jgroups-configuration";
+ protected static final String PARAM_JGROUPS_CONFIG = "jgroups-configuration";
/**
* The name of the parameter for the name of the cluster.
*/
- public static final String PARAM_CLUSTER_NAME = "jgroups-cluster-name";
+ protected static final String PARAM_CLUSTER_NAME = "jgroups-cluster-name";
/**
* The name of the parameter for the default timeout
*/
- public static final String PARAM_DEFAULT_TIMEOUT = "jgroups-default-timeout";
+ protected static final String PARAM_DEFAULT_TIMEOUT = "jgroups-default-timeout";
/**
+ * The name of the parameter to allow the failover
+ */
+ protected static final String PARAM_ALLOW_FAILOVER = "allow-failover";
+
+ /**
+ * The name of the parameter for the retry timeout
+ */
+ protected static final String PARAM_RETRY_TIMEOUT = "retry-timeout";
+
+ /**
* The value of the default timeout
*/
- public static final int DEFAULT_TIMEOUT = 0;
+ protected static final int DEFAULT_TIMEOUT = 0;
+
+ /**
+ * The value of the default retry timeout
+ */
+ protected static final int DEFAULT_RETRY_TIMEOUT = 20000;
/**
* The default value of the cluster name
*/
- public static final String CLUSTER_NAME = "RPCService-Cluster";
+ protected static final String CLUSTER_NAME = "RPCService-Cluster";
/**
* The configurator used to create the JGroups Channel
@@ -113,6 +128,11 @@
private final ProtocolStackConfigurator configurator;
/**
+ * The lock used to synchronize all the threads waiting for a topology change.
+ */
+ private final Object topologyChangeLock = new Object();
+
+ /**
* The name of the cluster
*/
private final String clusterName;
@@ -143,6 +163,16 @@
private long defaultTimeout = DEFAULT_TIMEOUT;
/**
+ * The value of the retry timeout
+ */
+ private long retryTimeout = DEFAULT_RETRY_TIMEOUT;
+
+ /**
+ * Indicates whether the failover capabilities are enabled
+ */
+ private boolean allowFailover = true;
+
+ /**
* The dispatcher used to launch the command of the cluster nodes
*/
private MessageDispatcher dispatcher;
@@ -212,6 +242,24 @@
LOG.debug("The default timeout of the RPCServiceImpl has been set to " + defaultTimeout);
}
}
+ String sAllowFailover = getValueParam(params, PARAM_ALLOW_FAILOVER);
+ if (sAllowFailover != null)
+ {
+ allowFailover = Boolean.valueOf(sAllowFailover);
+ if (LOG.isDebugEnabled())
+ {
+ LOG.debug("The parameter '" + PARAM_ALLOW_FAILOVER + "' of the RPCServiceImpl has been set to " + allowFailover);
+ }
+ }
+ sTimeout = getValueParam(params, PARAM_RETRY_TIMEOUT);
+ if (sTimeout != null)
+ {
+ retryTimeout = Integer.parseInt(sTimeout);
+ if (LOG.isDebugEnabled())
+ {
+ LOG.debug("The retry timeout of the RPCServiceImpl has been set to " + retryTimeout);
+ }
+ }
this.state = State.INITIALIZED;
}
@@ -294,18 +342,31 @@
v.add(coordinator);
List<Object> lResults = excecuteCommand(v, command, synchronous, timeout, args);
Object result = lResults == null || lResults.size() == 0 ? null : lResults.get(0);
- if (result instanceof MemberHasLeftException)
+ if (allowFailover && result instanceof MemberHasLeftException)
{
+ // The failover capabilities have been enabled and the coordinator seems to have left
if (coordinator.equals(this.coordinator))
{
- throw new RPCException("The coordinator did not change, we faced an unexpected situation",
- (MemberHasLeftException)result);
+ synchronized(topologyChangeLock)
+ {
+ if (coordinator.equals(this.coordinator))
+ {
+ if (LOG.isTraceEnabled())
+ LOG.trace("The coordinator did not change yet, we will relaunch the command after " + retryTimeout + " ms or once a topology change has been detected");
+ try
+ {
+ topologyChangeLock.wait(retryTimeout);
+ }
+ catch (InterruptedException e)
+ {
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
}
- else
- {
- // The coordinator has changed, we will automatically retry with the new coordinator
- return executeCommandOnCoordinator(command, synchronous, timeout, args);
- }
+ if (LOG.isTraceEnabled())
+ LOG.trace("The coordinator has changed, we will automatically retry with the new coordinator");
+ return executeCommandOnCoordinator(command, synchronous, timeout, args);
}
else if (result instanceof RPCException)
{
@@ -441,11 +502,18 @@
*/
public void viewAccepted(View view)
{
- this.members = view.getMembers();
- Address currentCoordinator = coordinator;
- this.coordinator = members != null && members.size() > 0 ? members.get(0) : null;
- this.isCoordinator = coordinator != null && coordinator.equals(channel.getLocalAddress());
- onTopologyChange(currentCoordinator != null && !currentCoordinator.equals(coordinator));
+ boolean coordinatorHasChanged;
+ synchronized (topologyChangeLock)
+ {
+ this.members = view.getMembers();
+ Address currentCoordinator = coordinator;
+ this.coordinator = members != null && members.size() > 0 ? members.get(0) : null;
+ this.isCoordinator = coordinator != null && coordinator.equals(channel.getLocalAddress());
+ coordinatorHasChanged = currentCoordinator != null && !currentCoordinator.equals(coordinator);
+ // Release all the nodes
+ topologyChangeLock.notifyAll();
+ }
+ onTopologyChange(coordinatorHasChanged);
}
/**
@@ -665,7 +733,7 @@
* Gives the value of the default timeout
* @return the default timeout
*/
- public long getDefaultTimeout()
+ protected long getDefaultTimeout()
{
return defaultTimeout;
}
@@ -674,12 +742,31 @@
* Gives the name of the cluster
* @return the name of the cluster
*/
- public String getClusterName()
+ protected String getClusterName()
{
return clusterName;
}
+
+ /**
+ * Gives the value of the retry timeout
+ * @return the value of the retry timeout
+ */
+ protected long getRetryTimeout()
+ {
+ return retryTimeout;
+ }
/**
+ * Indicates whether the failover capabilities are enabled or not
+ * @return <code>true</code> if the failover capabilities are allowed, <code>false</code>
+ * otherwise
+ */
+ protected boolean isAllowFailover()
+ {
+ return allowFailover;
+ }
+
+ /**
* Gives the value of the {@link ValueParam} corresponding to the given key
* @param params the list of initial parameters from which we want to extract the {@link ValueParam}
* @param parameterKey the name of the {@link ValueParam} that we are looking for
Modified: kernel/trunk/exo.kernel.component.common/src/test/java/org/exoplatform/services/rpc/impl/TestRPCServiceImpl.java
===================================================================
--- kernel/trunk/exo.kernel.component.common/src/test/java/org/exoplatform/services/rpc/impl/TestRPCServiceImpl.java 2010-10-12 09:47:34 UTC (rev 3292)
+++ kernel/trunk/exo.kernel.component.common/src/test/java/org/exoplatform/services/rpc/impl/TestRPCServiceImpl.java 2010-10-13 11:53:35 UTC (rev 3293)
@@ -106,6 +106,8 @@
{
service = new RPCServiceImpl(container.getContext(), params, configManager);
assertEquals(RPCServiceImpl.DEFAULT_TIMEOUT, service.getDefaultTimeout());
+ assertEquals(RPCServiceImpl.DEFAULT_RETRY_TIMEOUT, service.getRetryTimeout());
+ assertEquals(true, service.isAllowFailover());
assertEquals(RPCServiceImpl.CLUSTER_NAME + "-" + container.getContext().getName(), service.getClusterName());
}
finally
@@ -133,6 +135,8 @@
{
service = new RPCServiceImpl(container.getContext(), params, configManager);
assertEquals(60, service.getDefaultTimeout());
+ assertEquals(RPCServiceImpl.DEFAULT_RETRY_TIMEOUT, service.getRetryTimeout());
+ assertEquals(true, service.isAllowFailover());
assertEquals(RPCServiceImpl.CLUSTER_NAME + "-" + container.getContext().getName(), service.getClusterName());
}
finally
@@ -142,7 +146,72 @@
service.stop();
}
}
- ValueParam paramClusterName = new ValueParam();
+ ValueParam paramRetryTimeout = new ValueParam();
+ paramRetryTimeout.setName(RPCServiceImpl.PARAM_RETRY_TIMEOUT);
+ paramRetryTimeout.setValue("fakeValue");
+ params.addParameter(paramRetryTimeout);
+ try
+ {
+ new RPCServiceImpl(container.getContext(), params, configManager);
+ fail("We expect a NumberFormatException since the retry timeout is not properly set");
+ }
+ catch (NumberFormatException e)
+ {
+ // OK
+ }
+ paramRetryTimeout.setValue("60");
+ try
+ {
+ service = new RPCServiceImpl(container.getContext(), params, configManager);
+ assertEquals(60, service.getDefaultTimeout());
+ assertEquals(60, service.getRetryTimeout());
+ assertEquals(true, service.isAllowFailover());
+ assertEquals(RPCServiceImpl.CLUSTER_NAME + "-" + container.getContext().getName(), service.getClusterName());
+ }
+ finally
+ {
+ if (service != null)
+ {
+ service.stop();
+ }
+ }
+ ValueParam paramAllowFailover = new ValueParam();
+ paramAllowFailover.setName(RPCServiceImpl.PARAM_ALLOW_FAILOVER);
+ paramAllowFailover.setValue("fakeValue");
+ params.addParameter(paramAllowFailover);
+ try
+ {
+ service = new RPCServiceImpl(container.getContext(), params, configManager);
+ assertEquals(60, service.getDefaultTimeout());
+ assertEquals(60, service.getRetryTimeout());
+ assertEquals(false, service.isAllowFailover());
+ assertEquals(RPCServiceImpl.CLUSTER_NAME + "-" + container.getContext().getName(), service.getClusterName());
+ }
+ finally
+ {
+ if (service != null)
+ {
+ service.stop();
+ }
+ }
+ paramAllowFailover.setValue("TRUE");
+ try
+ {
+ service = new RPCServiceImpl(container.getContext(), params, configManager);
+ assertEquals(60, service.getDefaultTimeout());
+ assertEquals(60, service.getRetryTimeout());
+ assertEquals(true, service.isAllowFailover());
+ assertEquals(RPCServiceImpl.CLUSTER_NAME + "-" + container.getContext().getName(), service.getClusterName());
+ }
+ finally
+ {
+ if (service != null)
+ {
+ service.stop();
+ }
+ }
+
+ ValueParam paramClusterName = new ValueParam();
paramClusterName.setName(RPCServiceImpl.PARAM_CLUSTER_NAME);
paramClusterName.setValue("MyName");
params.addParameter(paramClusterName);
15 years, 9 months
- 1
- 0
exo-jcr SVN: r3292 - in jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query: jbosscache and 1 other directories.
by do-not-reply＠jboss.org
Author: nzamosenchuk
Date: 2010-10-12 05:47:34 -0400 (Tue, 12 Oct 2010)
New Revision: 3292
Added:
jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/ChunkIndex.java
Removed:
jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/MultiIndex.java
Modified:
jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/AbstractQueryHandler.java
jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/QueryHandler.java
jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/jbosscache/JBossCacheIndexInfos.java
jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/ConsistencyCheck.java
jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/IndexInfos.java
jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/IndexingQueue.java
jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/SearchIndex.java
Log:
EXOJCR-987 : Cleaning up the code, simplifying. Refactoring.
Modified: jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/AbstractQueryHandler.java
===================================================================
--- jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/AbstractQueryHandler.java 2010-10-11 16:33:49 UTC (rev 3291)
+++ jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/AbstractQueryHandler.java 2010-10-12 09:47:34 UTC (rev 3292)
@@ -21,7 +21,7 @@
import org.exoplatform.services.jcr.impl.core.query.lucene.DefaultIndexUpdateMonitor;
import org.exoplatform.services.jcr.impl.core.query.lucene.IndexInfos;
import org.exoplatform.services.jcr.impl.core.query.lucene.IndexUpdateMonitor;
-import org.exoplatform.services.jcr.impl.core.query.lucene.MultiIndex;
+import org.exoplatform.services.jcr.impl.core.query.lucene.ChunkIndex;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -70,7 +70,7 @@
protected IndexerIoModeHandler modeHandler;
/**
- * {@link IndexInfos} instance that is passed to {@link MultiIndex}
+ * {@link IndexInfos} instance that is passed to {@link ChunkIndex}
*/
protected IndexInfos indexInfos;
Modified: jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/QueryHandler.java
===================================================================
--- jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/QueryHandler.java 2010-10-11 16:33:49 UTC (rev 3291)
+++ jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/QueryHandler.java 2010-10-12 09:47:34 UTC (rev 3292)
@@ -23,7 +23,7 @@
import org.exoplatform.services.jcr.impl.core.SessionImpl;
import org.exoplatform.services.jcr.impl.core.query.lucene.IndexInfos;
import org.exoplatform.services.jcr.impl.core.query.lucene.IndexUpdateMonitor;
-import org.exoplatform.services.jcr.impl.core.query.lucene.MultiIndex;
+import org.exoplatform.services.jcr.impl.core.query.lucene.ChunkIndex;
import org.exoplatform.services.jcr.impl.core.query.lucene.QueryHits;
import java.io.IOException;
@@ -156,7 +156,7 @@
QueryHits executeQuery(Query query) throws IOException;
/**
- * Sets {@link IndexInfos} instance into QueryHandler, which is later passed to {@link MultiIndex}.
+ * Sets {@link IndexInfos} instance into QueryHandler, which is later passed to {@link ChunkIndex}.
*
* @param indexInfos
*/
Modified: jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/jbosscache/JBossCacheIndexInfos.java
===================================================================
--- jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/jbosscache/JBossCacheIndexInfos.java 2010-10-11 16:33:49 UTC (rev 3291)
+++ jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/jbosscache/JBossCacheIndexInfos.java 2010-10-12 09:47:34 UTC (rev 3292)
@@ -22,7 +22,7 @@
import org.exoplatform.services.jcr.impl.core.query.IndexerIoModeHandler;
import org.exoplatform.services.jcr.impl.core.query.IndexerIoModeListener;
import org.exoplatform.services.jcr.impl.core.query.lucene.IndexInfos;
-import org.exoplatform.services.jcr.impl.core.query.lucene.MultiIndex;
+import org.exoplatform.services.jcr.impl.core.query.lucene.ChunkIndex;
import org.exoplatform.services.jcr.impl.util.io.PrivilegedCacheHelper;
import org.exoplatform.services.log.ExoLogger;
import org.exoplatform.services.log.Log;
Copied: jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/ChunkIndex.java (from rev 3288, jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/MultiIndex.java)
===================================================================
--- jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/ChunkIndex.java (rev 0)
+++ jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/ChunkIndex.java 2010-10-12 09:47:34 UTC (rev 3292)
@@ -0,0 +1,795 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.exoplatform.services.jcr.impl.core.query.lucene;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.store.Directory;
+import org.exoplatform.services.jcr.dataflow.ItemDataConsumer;
+import org.exoplatform.services.jcr.datamodel.ItemData;
+import org.exoplatform.services.jcr.datamodel.NodeData;
+import org.exoplatform.services.jcr.impl.core.query.IndexerIoMode;
+import org.exoplatform.services.jcr.impl.core.query.IndexerIoModeHandler;
+import org.exoplatform.services.jcr.impl.core.query.IndexingTree;
+import org.exoplatform.services.jcr.impl.core.query.lucene.directory.DirectoryManager;
+import org.exoplatform.services.jcr.impl.util.SecurityHelper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.Map.Entry;
+
+import javax.jcr.ItemNotFoundException;
+import javax.jcr.RepositoryException;
+
+/**
+ * TODO REWRITE JAVADOC!!!
+ */
+public class ChunkIndex
+{
+
+ /**
+ * The logger instance for this class
+ */
+ private static final Logger log = LoggerFactory.getLogger("exo.jcr.component.core.MultiIndex");
+
+ /**
+ * Names of active persistent index directories.
+ */
+ private IndexInfos indexNames;
+
+ /**
+ * List of open persistent indexes. This list may also contain an open
+ * PersistentIndex owned by the IndexMerger daemon. Such an index is not
+ * registered with indexNames and <b>must not</b> be used in regular index
+ * operations (delete node, etc.)!
+ */
+ private final List<PersistentIndex> indexes = new ArrayList<PersistentIndex>();
+
+ /**
+ * The internal namespace mappings of the query manager.
+ */
+ private final NamespaceMappings nsMappings;
+
+ /**
+ * The directory manager.
+ */
+ private final DirectoryManager directoryManager;
+
+ /**
+ * The base directory to store the index.
+ */
+ private final Directory indexDir;
+
+ /**
+ * The query handler
+ */
+ private final SearchIndex handler;
+
+ /**
+ * If not <code>null</code> points to a valid <code>IndexReader</code> that
+ * reads from all indexes, including volatile and persistent indexes.
+ */
+ private CachingMultiIndexReader multiReader;
+
+ /**
+ * Shared document number cache across all persistent indexes.
+ */
+ private final DocNumberCache cache;
+
+ /**
+ * The indexing queue with pending text extraction jobs.
+ */
+ private IndexingQueue indexingQueue;
+
+ /**
+ * Set<NodeId> of uuids that should not be indexed.
+ */
+ private final IndexingTree indexingTree;
+
+ /**
+ * Flag indicating whether re-indexing is running.
+ */
+ private boolean reindexing = false;
+
+ /**
+ * The index format version of this multi index.
+ */
+ private final IndexFormatVersion version;
+
+ /**
+ * The handler of the Indexer io mode
+ */
+ private final IndexerIoModeHandler modeHandler;
+
+ /**
+ * Creates a new MultiIndex.
+ *
+ * @param handler
+ * the search handler
+ * @param excludedIDs
+ * Set<NodeId> that contains uuids that should not be indexed
+ * nor further traversed.
+ * @throws IOException
+ * if an error occurs
+ */
+ ChunkIndex(SearchIndex handler, IndexingTree indexingTree, IndexerIoModeHandler modeHandler, IndexInfos indexInfos,
+ IndexUpdateMonitor indexUpdateMonitor) throws IOException
+ {
+ this.modeHandler = modeHandler;
+ this.directoryManager = handler.getDirectoryManager();
+ // this method is run in privileged mode internally
+ this.indexDir = directoryManager.getDirectory(".");
+ this.handler = handler;
+ this.cache = new DocNumberCache(handler.getCacheSize());
+ this.indexingTree = indexingTree;
+ this.nsMappings = handler.getNamespaceMappings();
+ this.indexNames = indexInfos;
+ this.indexNames.setDirectory(indexDir);
+ // this method is run in privileged mode internally
+ this.indexNames.read();
+
+ // TODO remove hard-coded piece
+ this.indexNames.addName("1");
+ this.indexNames.addName("2");
+ this.indexNames.addName("3");
+ this.indexNames.addName("4");
+
+ // this method is run in privileged mode internally
+ IndexingQueueStore store = new IndexingQueueStore(indexDir);
+
+ // initialize indexing queue
+ this.indexingQueue = new IndexingQueue(store);
+ // copy current index names
+ Set<String> currentNames = new HashSet<String>(indexNames.getNames());
+
+ // open persistent indexes
+ for (String name : currentNames)
+ {
+ // only open if it still exists
+ // it is possible that indexNames still contains a name for
+ // an index that has been deleted, but indexNames has not been
+ // written to disk.
+
+ // TODO THIS CHECK WAS SKIPPED
+ // if (!directoryManager.hasDirectory(name))
+ // {
+ // log.debug("index does not exist anymore: " + name);
+ // // move on to next index
+ // continue;
+ // }
+ PersistentIndex index =
+ new PersistentIndex(name, handler.getTextAnalyzer(), handler.getSimilarity(), cache, indexingQueue,
+ directoryManager);
+ index.setMaxFieldLength(handler.getMaxFieldLength());
+ index.setUseCompoundFile(handler.getUseCompoundFile());
+ index.setTermInfosIndexDivisor(handler.getTermInfosIndexDivisor());
+ indexes.add(index);
+ }
+
+ // set index format version and at the same time
+ // initialize hierarchy cache if requested.
+ final CachingMultiIndexReader reader = getIndexReader(handler.isInitializeHierarchyCache());
+ try
+ {
+ version = IndexFormatVersion.getVersion(reader);
+ }
+ finally
+ {
+ SecurityHelper.doPriviledgedIOExceptionAction(new PrivilegedExceptionAction<Object>()
+ {
+ public Object run() throws Exception
+ {
+ reader.release();
+ return null;
+ }
+ });
+ }
+ indexingQueue.initialize(this);
+ this.indexNames.setMultiIndex(this);
+ }
+
+ /**
+ * Returns the number of documents in this index.
+ *
+ * @return the number of documents in this index.
+ * @throws IOException
+ * if an error occurs while reading from the index.
+ */
+ int numDocs() throws IOException
+ {
+ if (indexNames.size() == 0)
+ {
+ return 0;
+ }
+ else
+ {
+ final CachingMultiIndexReader reader = getIndexReader();
+ try
+ {
+ return reader.numDocs();
+ }
+ finally
+ {
+ SecurityHelper.doPriviledgedIOExceptionAction(new PrivilegedExceptionAction<Object>()
+ {
+ public Object run() throws Exception
+ {
+ reader.release();
+ return null;
+ }
+ });
+ }
+ }
+ }
+
+ /**
+ * @return the index format version for this multi index.
+ */
+ IndexFormatVersion getIndexFormatVersion()
+ {
+ return version;
+ }
+
+ /**
+ * Creates an initial index by traversing the node hierarchy starting at the
+ * node with <code>rootId</code>.
+ *
+ * @param stateMgr
+ * the item state manager.
+ * @param rootId
+ * the id of the node from where to start.
+ * @param rootPath
+ * the path of the node from where to start.
+ * @throws IOException
+ * if an error occurs while indexing the workspace.
+ * @throws IllegalStateException
+ * if this index is not empty.
+ */
+ void createInitialIndex(ItemDataConsumer stateMgr) throws IOException
+ {
+ // TODO: re-study check!!!
+ CachingMultiIndexReader reader = getIndexReader();
+ int numDocs = reader.numDocs();
+ reader.release();
+ if (numDocs == 0)
+ {
+ reindexing = true;
+ try
+ {
+ long count = 0;
+ // traverse and index workspace
+
+ // NodeData rootState = (NodeData) stateMgr.getItemData(rootId);
+ count = createIndex(indexingTree.getIndexingRoot(), stateMgr, count);
+
+ log.info("Created initial index for {} nodes", new Long(count));
+ releaseMultiReader();
+ }
+ catch (Exception e)
+ {
+ String msg = "Error indexing workspace";
+ IOException ex = new IOException(msg);
+ ex.initCause(e);
+ throw ex;
+ }
+ finally
+ {
+ reindexing = false;
+ }
+ }
+ else
+ {
+ throw new IllegalStateException("Index already present");
+ }
+ }
+
+ /**
+ * Atomically updates the index by removing some documents and adding
+ * others.
+ *
+ * @param remove
+ * collection of <code>UUID</code>s that identify documents to
+ * remove
+ * @param add
+ * collection of <code>Document</code>s to add. Some of the
+ * elements in this collection may be <code>null</code>, to
+ * indicate that a node could not be indexed successfully.
+ * @throws IOException
+ * if an error occurs while updating the index.
+ */
+ synchronized void update(final Collection<String> remove, final Collection<Document> add) throws IOException
+ {
+ SecurityHelper.doPriviledgedIOExceptionAction(new PrivilegedExceptionAction<Object>()
+ {
+ public Object run() throws Exception
+ {
+ // make sure a reader is available during long updates
+ if (add.size() > handler.getBufferSize())
+ {
+ try
+ {
+ getIndexReader().release();
+ }
+ catch (IOException e)
+ {
+ // do not fail if an exception is thrown here
+ log.warn("unable to prepare index reader " + "for queries during update", e);
+ }
+ }
+ try
+ {
+ for (Iterator<String> it = remove.iterator(); it.hasNext();)
+ {
+ String uuidString = it.next();
+ // check if indexing queue is still working on
+ // this node from a previous update
+ Document doc = indexingQueue.removeDocument(uuidString);
+ if (doc != null)
+ {
+ Util.disposeDocument(doc);
+ }
+ Term idTerm = new Term(FieldNames.UUID, uuidString);
+ getChunk(uuidString).removeDocument(idTerm);
+ }
+ for (Iterator<Document> it = add.iterator(); it.hasNext();)
+ {
+ Document doc = it.next();
+ if (doc != null)
+ {
+ if (doc != null)
+ {
+ String uuid = doc.get(FieldNames.UUID);
+ getChunk(uuid).addDocuments(new Document[]{doc});
+ }
+ }
+ }
+ // TODO for owning indexes only
+ for (PersistentIndex idx : indexes)
+ {
+ idx.commit();
+ }
+ }
+ finally
+ {
+ releaseMultiReader();
+ }
+ return null;
+ }
+ });
+ }
+
+ /**
+ * Adds a document to the index.
+ *
+ * @param doc
+ * the document to add.
+ * @throws IOException
+ * if an error occurs while adding the document to the index.
+ */
+ void addDocument(Document doc) throws IOException
+ {
+ update(Collections.EMPTY_LIST, Arrays.asList(new Document[]{doc}));
+ }
+
+ /**
+ * Deletes the first document that matches the <code>uuid</code>.
+ *
+ * @param uuid
+ * document that match this <code>uuid</code> will be deleted.
+ * @throws IOException
+ * if an error occurs while deleting the document.
+ */
+ void removeDocument(String uuid) throws IOException
+ {
+ update(Arrays.asList(new String[]{uuid}), Collections.EMPTY_LIST);
+ }
+
+ /**
+ * Deletes all documents that match the <code>uuid</code>.
+ *
+ * @param uuid
+ * documents that match this <code>uuid</code> will be deleted.
+ * @return the number of deleted documents.
+ * @throws IOException
+ * if an error occurs while deleting documents.
+ */
+ synchronized int removeAllDocuments(String uuid) throws IOException
+ {
+ int num;
+ try
+ {
+ Term idTerm = new Term(FieldNames.UUID, uuid.toString());
+ num = getChunk(uuid).removeDocument(idTerm);
+ for (int i = 0; i < indexes.size(); i++)
+ {
+ PersistentIndex index = indexes.get(i);
+ // TODO only remove documents from owning indexes
+ int removed = index.removeDocument(idTerm);
+ num += removed;
+ }
+ }
+ finally
+ {
+ releaseMultiReader();
+ }
+ return num;
+ }
+
+ /**
+ * Returns an read-only <code>IndexReader</code> that spans alls indexes of
+ * this <code>MultiIndex</code>.
+ *
+ * @return an <code>IndexReader</code>.
+ * @throws IOException
+ * if an error occurs constructing the <code>IndexReader</code>.
+ */
+ public CachingMultiIndexReader getIndexReader() throws IOException
+ {
+ return getIndexReader(false);
+ }
+
+ /**
+ * Returns an read-only <code>IndexReader</code> that spans alls indexes of
+ * this <code>MultiIndex</code>.
+ *
+ * @param initCache
+ * when set <code>true</code> the hierarchy cache is completely
+ * initialized before this call returns.
+ * @return an <code>IndexReader</code>.
+ * @throws IOException
+ * if an error occurs constructing the <code>IndexReader</code>.
+ */
+ public synchronized CachingMultiIndexReader getIndexReader(final boolean initCache) throws IOException
+ {
+ return SecurityHelper.doPriviledgedIOExceptionAction(new PrivilegedExceptionAction<CachingMultiIndexReader>()
+ {
+ public CachingMultiIndexReader run() throws Exception
+ {
+ if (multiReader != null)
+ {
+ multiReader.acquire();
+ return multiReader;
+ }
+ // no reader available
+ // some other read thread might have created the reader in the
+ // meantime -> check again
+ if (multiReader == null)
+ {
+ List<ReadOnlyIndexReader> readerList = new ArrayList<ReadOnlyIndexReader>();
+ for (PersistentIndex idx : indexes)
+ {
+ readerList.add(idx.getReadOnlyIndexReader(initCache));
+ }
+ ReadOnlyIndexReader[] readers = readerList.toArray(new ReadOnlyIndexReader[readerList.size()]);
+ multiReader = new CachingMultiIndexReader(readers, cache);
+ }
+ multiReader.acquire();
+ return multiReader;
+ }
+ });
+
+ }
+
+ /**
+ * Closes this <code>MultiIndex</code>.
+ */
+ void close()
+ {
+ if (modeHandler.getMode().equals(IndexerIoMode.READ_WRITE))
+ {
+
+ // stop index merger
+ // when calling this method we must not lock this MultiIndex, otherwise
+ // a deadlock might occur
+
+ synchronized (this)
+ {
+
+ // commit / close indexes
+ try
+ {
+ releaseMultiReader();
+ }
+ catch (IOException e)
+ {
+ log.error("Exception while closing search index.", e);
+ }
+
+ // TODO Should they be commited before close?!
+ for (int i = 0; i < indexes.size(); i++)
+ {
+ (indexes.get(i)).close();
+ }
+
+ // close indexing queue
+ indexingQueue.close();
+
+ // finally close directory
+ try
+ {
+ indexDir.close();
+ }
+ catch (IOException e)
+ {
+ log.error("Exception while closing directory.", e);
+ }
+ }
+ }
+ }
+
+ /**
+ * Returns the namespace mappings of this search index.
+ *
+ * @return the namespace mappings of this search index.
+ */
+ NamespaceMappings getNamespaceMappings()
+ {
+ return nsMappings;
+ }
+
+ /**
+ * Returns the indexing queue for this multi index.
+ *
+ * @return the indexing queue for this multi index.
+ */
+ public IndexingQueue getIndexingQueue()
+ {
+ return indexingQueue;
+ }
+
+ /**
+ * Returns a lucene Document for the <code>node</code>.
+ *
+ * @param node
+ * the node to index.
+ * @return the index document.
+ * @throws RepositoryException
+ * if an error occurs while reading from the workspace.
+ */
+ Document createDocument(NodeData node) throws RepositoryException
+ {
+ return handler.createDocument(node, nsMappings, version);
+ }
+
+ /**
+ * Returns a lucene Document for the Node with <code>id</code>.
+ *
+ * @param id
+ * the id of the node to index.
+ * @return the index document.
+ * @throws RepositoryException
+ * if an error occurs while reading from the workspace or if
+ * there is no node with <code>id</code>.
+ */
+ Document createDocument(String id) throws RepositoryException
+ {
+ ItemData data = handler.getContext().getItemStateManager().getItemData(id);
+ if (data == null)
+ {
+ throw new ItemNotFoundException("Item id=" + id + " not found");
+ }
+ if (!data.isNode())
+ {
+ throw new RepositoryException("Item with id " + id + " is not a node");
+ }
+ return createDocument((NodeData)data);
+
+ }
+
+ /**
+ * Releases the {@link #multiReader} and sets it <code>null</code>. If the
+ * reader is already <code>null</code> this method does nothing. When this
+ * method returns {@link #multiReader} is guaranteed to be <code>null</code>
+ * even if an exception is thrown.
+ * <p/>
+ * Please note that this method does not take care of any synchronization. A
+ * caller must ensure that it is the only thread operating on this multi
+ * index, or that it holds the {@link #updateMonitor}.
+ *
+ * @throws IOException
+ * if an error occurs while releasing the reader.
+ */
+ void releaseMultiReader() throws IOException
+ {
+ SecurityHelper.doPriviledgedIOExceptionAction(new PrivilegedExceptionAction<Object>()
+ {
+ public Object run() throws Exception
+ {
+ if (multiReader != null)
+ {
+ try
+ {
+ multiReader.release();
+ }
+ finally
+ {
+ multiReader = null;
+ }
+ }
+ return null;
+ }
+ });
+ }
+
+ // -------------------------< internal
+ // >-------------------------------------
+
+ /**
+ * Recursively creates an index starting with the NodeState
+ * <code>node</code>.
+ *
+ * @param node
+ * the current NodeState.
+ * @param path
+ * the path of the current node.
+ * @param stateMgr
+ * the shared item state manager.
+ * @param count
+ * the number of nodes already indexed.
+ * @return the number of nodes indexed so far.
+ * @throws IOException
+ * if an error occurs while writing to the index.
+ * @throws ItemStateException
+ * if an node state cannot be found.
+ * @throws RepositoryException
+ * if any other error occurs
+ */
+ private long createIndex(NodeData node, ItemDataConsumer stateMgr, long count) throws IOException,
+ RepositoryException
+ {
+ if (indexingTree.isExcluded(node))
+ {
+ return count;
+ }
+ getChunk(node.getIdentifier()).addDocuments(new Document[]{createDocument(node)});
+
+ if (++count % 100 == 0)
+ {
+
+ log.info("indexing... {} ({})", node.getQPath().getAsString(), new Long(count));
+ }
+ if (count % 10 == 0)
+ {
+ checkIndexingQueue(true);
+ }
+ List<NodeData> children = stateMgr.getChildNodesData(node);
+ for (NodeData nodeData : children)
+ {
+
+ NodeData childState = (NodeData)stateMgr.getItemData(nodeData.getIdentifier());
+ if (childState == null)
+ {
+ handler.getOnWorkspaceInconsistencyHandler().handleMissingChildNode(
+ new ItemNotFoundException("Child not found "), handler, nodeData.getQPath(), node, nodeData);
+ }
+
+ if (nodeData != null)
+ {
+ count = createIndex(nodeData, stateMgr, count);
+ }
+ }
+
+ return count;
+ }
+
+ /**
+ * Checks the indexing queue for finished text extrator jobs and updates the
+ * index accordingly if there are any new ones. This method is synchronized
+ * and should only be called by the timer task that periodically checks if
+ * there are documents ready in the indexing queue. A new transaction is
+ * used when documents are transfered from the indexing queue to the index.
+ */
+ private synchronized void checkIndexingQueue()
+ {
+ checkIndexingQueue(false);
+ }
+
+ /**
+ * Checks the indexing queue for finished text extrator jobs and updates the
+ * index accordingly if there are any new ones.
+ *
+ * @param transactionPresent
+ * whether a transaction is in progress and the current
+ * {@link #getTransactionId()} should be used. If
+ * <code>false</code> a new transaction is created when documents
+ * are transfered from the indexing queue to the index.
+ */
+ private void checkIndexingQueue(boolean transactionPresent)
+ {
+ Document[] docs = indexingQueue.getFinishedDocuments();
+ Map<String, Document> finished = new HashMap<String, Document>();
+ for (int i = 0; i < docs.length; i++)
+ {
+ String uuid = docs[i].get(FieldNames.UUID);
+ finished.put(uuid, docs[i]);
+ }
+
+ // now update index with the remaining ones if there are any
+ if (!finished.isEmpty())
+ {
+ log.info("updating index with {} nodes from indexing queue.", new Long(finished.size()));
+
+ // remove documents from the queue
+ for (Iterator<String> it = finished.keySet().iterator(); it.hasNext();)
+ {
+ indexingQueue.removeDocument(it.next());
+ }
+
+ try
+ {
+ if (transactionPresent)
+ {
+ for (Iterator<String> it = finished.keySet().iterator(); it.hasNext();)
+ {
+ String uuidString = it.next();
+ // check if indexing queue is still working on
+ // this node from a previous update
+ Document doc = indexingQueue.removeDocument(uuidString);
+ if (doc != null)
+ {
+ Util.disposeDocument(doc);
+ }
+ Term idTerm = new Term(FieldNames.UUID, uuidString);
+ // if the document cannot be deleted from the volatile index
+ // delete it from one of the persistent indexes.
+ getChunk(uuidString).removeDocument(idTerm);
+ }
+ for (Iterator<Document> it = finished.values().iterator(); it.hasNext();)
+ {
+ Document doc = it.next();
+ String uuid = doc.get(FieldNames.UUID);
+ getChunk(uuid).addDocuments(new Document[]{doc});
+ }
+ }
+ else
+ {
+ update(finished.keySet(), finished.values());
+ }
+ }
+ catch (IOException e)
+ {
+ // update failed
+ log.warn("Failed to update index with deferred text extraction", e);
+ }
+ }
+ }
+
+ @Deprecated
+ public boolean getRedoLogApplied()
+ {
+ return false;
+ }
+
+ private PersistentIndex getChunk(String uuid)
+ {
+ return indexes.get(0);
+ }
+}
Modified: jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/ConsistencyCheck.java
===================================================================
--- jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/ConsistencyCheck.java 2010-10-11 16:33:49 UTC (rev 3291)
+++ jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/ConsistencyCheck.java 2010-10-12 09:47:34 UTC (rev 3292)
@@ -61,7 +61,7 @@
/**
* The index to check.
*/
- private final MultiIndex index;
+ private final ChunkIndex index;
/**
* All the document UUIDs within the index.
@@ -76,7 +76,7 @@
/**
* Private constructor.
*/
- private ConsistencyCheck(MultiIndex index, ItemDataConsumer mgr)
+ private ConsistencyCheck(ChunkIndex index, ItemDataConsumer mgr)
{
this.index = index;
this.stateMgr = mgr;
@@ -91,7 +91,7 @@
* @throws IOException if an error occurs while checking.
* @throws RepositoryException
*/
- static ConsistencyCheck run(MultiIndex index, ItemDataConsumer mgr) throws IOException, RepositoryException
+ static ConsistencyCheck run(ChunkIndex index, ItemDataConsumer mgr) throws IOException, RepositoryException
{
ConsistencyCheck check = new ConsistencyCheck(index, mgr);
check.run();
Modified: jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/IndexInfos.java
===================================================================
--- jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/IndexInfos.java 2010-10-11 16:33:49 UTC (rev 3291)
+++ jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/IndexInfos.java 2010-10-12 09:47:34 UTC (rev 3292)
@@ -74,9 +74,9 @@
private Directory dir;
/**
- * {@link MultiIndex} instance for callbacking when list of indexes changed
+ * {@link ChunkIndex} instance for callbacking when list of indexes changed
*/
- private MultiIndex multiIndex;
+ private ChunkIndex multiIndex;
/**
* Creates a new IndexInfos using <code>"indexes"</code> as a filename.
@@ -304,10 +304,10 @@
}
/**
- * Sets {@link MultiIndex} instance.
+ * Sets {@link ChunkIndex} instance.
* @param multiIndex
*/
- public void setMultiIndex(MultiIndex multiIndex)
+ public void setMultiIndex(ChunkIndex multiIndex)
{
this.multiIndex = multiIndex;
}
@@ -316,7 +316,7 @@
* Returns multiIndex instance
* @return
*/
- public MultiIndex getMultiIndex()
+ public ChunkIndex getMultiIndex()
{
return multiIndex;
}
Modified: jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/IndexingQueue.java
===================================================================
--- jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/IndexingQueue.java 2010-10-11 16:33:49 UTC (rev 3291)
+++ jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/IndexingQueue.java 2010-10-12 09:47:34 UTC (rev 3292)
@@ -57,7 +57,7 @@
/**
* Flag that indicates whether this indexing queue had been
- * {@link #initialize(MultiIndex) initialized}.
+ * {@link #initialize(ChunkIndex) initialized}.
*/
private volatile boolean initialized = false;
@@ -77,7 +77,7 @@
* @param index the multi index this indexing queue belongs to.
* @throws IOException if an error occurs while reading from the index.
*/
- void initialize(final MultiIndex index) throws IOException
+ void initialize(final ChunkIndex index) throws IOException
{
SecurityHelper.doPriviledgedIOExceptionAction(new PrivilegedExceptionAction<Object>()
{
Deleted: jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/MultiIndex.java
===================================================================
--- jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/MultiIndex.java 2010-10-11 16:33:49 UTC (rev 3291)
+++ jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/MultiIndex.java 2010-10-12 09:47:34 UTC (rev 3292)
@@ -1,1069 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.exoplatform.services.jcr.impl.core.query.lucene;
-
-import org.apache.lucene.document.Document;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.store.Directory;
-import org.exoplatform.services.jcr.dataflow.ItemDataConsumer;
-import org.exoplatform.services.jcr.datamodel.ItemData;
-import org.exoplatform.services.jcr.datamodel.NodeData;
-import org.exoplatform.services.jcr.impl.core.query.IndexerIoMode;
-import org.exoplatform.services.jcr.impl.core.query.IndexerIoModeHandler;
-import org.exoplatform.services.jcr.impl.core.query.IndexingTree;
-import org.exoplatform.services.jcr.impl.core.query.lucene.directory.DirectoryManager;
-import org.exoplatform.services.jcr.impl.util.SecurityHelper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.security.PrivilegedAction;
-import java.security.PrivilegedExceptionAction;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.Map.Entry;
-
-import javax.jcr.ItemNotFoundException;
-import javax.jcr.RepositoryException;
-
-/**
- * A <code>MultiIndex</code> consists of a {@link VolatileIndex} and multiple
- * {@link PersistentIndex}es. The goal is to keep most parts of the index open
- * with index readers and write new index data to the volatile index. When the
- * volatile index reaches a certain size (see
- * {@link SearchIndex#setMinMergeDocs(int)}) a new persistent index is created
- * with the index data from the volatile index, the same happens when the
- * volatile index has been idle for some time (see
- * {@link SearchIndex#setVolatileIdleTime(int)}). The new persistent index is
- * then added to the list of already existing persistent indexes. Further
- * operations on the new persistent index will however only require an
- * <code>IndexReader</code> which serves for queries but also for delete
- * operations on the index.
- * <p/>
- * The persistent indexes are merged from time to time. The merge behaviour is
- * configurable using the methods: {@link SearchIndex#setMaxMergeDocs(int)},
- * {@link SearchIndex#setMergeFactor(int)} and
- * {@link SearchIndex#setMinMergeDocs(int)}. For detailed description of the
- * configuration parameters see also the lucene <code>IndexWriter</code> class.
- * <p/>
- * This class is thread-safe.
- * <p/>
- * Note on implementation: Multiple modifying threads are synchronized on a
- * <code>MultiIndex</code> instance itself. Sychronization between a modifying
- * thread and reader threads is done using {@link #updateMonitor} and
- * {@link #updateInProgress}.
- */
-public class MultiIndex
-{
-
- /**
- * The logger instance for this class
- */
- private static final Logger log = LoggerFactory.getLogger("exo.jcr.component.core.MultiIndex");
-
- /**
- * Names of active persistent index directories.
- */
- private IndexInfos indexNames;
-
- /**
- * List of open persistent indexes. This list may also contain an open
- * PersistentIndex owned by the IndexMerger daemon. Such an index is not
- * registered with indexNames and <b>must not</b> be used in regular index
- * operations (delete node, etc.)!
- */
- private final List<PersistentIndex> indexes = new ArrayList<PersistentIndex>();
-
- /**
- * The internal namespace mappings of the query manager.
- */
- private final NamespaceMappings nsMappings;
-
- /**
- * The directory manager.
- */
- private final DirectoryManager directoryManager;
-
- /**
- * The base directory to store the index.
- */
- private final Directory indexDir;
-
- /**
- * The query handler
- */
- private final SearchIndex handler;
-
- /**
- * If not <code>null</code> points to a valid <code>IndexReader</code> that
- * reads from all indexes, including volatile and persistent indexes.
- */
- private CachingMultiIndexReader multiReader;
-
- /**
- * Shared document number cache across all persistent indexes.
- */
- private final DocNumberCache cache;
-
- /**
- * The indexing queue with pending text extraction jobs.
- */
- private IndexingQueue indexingQueue;
-
- /**
- * Set<NodeId> of uuids that should not be indexed.
- */
- private final IndexingTree indexingTree;
-
- /**
- * Flag indicating whether re-indexing is running.
- */
- private boolean reindexing = false;
-
- /**
- * The index format version of this multi index.
- */
- private final IndexFormatVersion version;
-
- /**
- * The handler of the Indexer io mode
- */
- private final IndexerIoModeHandler modeHandler;
-
- /**
- * Creates a new MultiIndex.
- *
- * @param handler
- * the search handler
- * @param excludedIDs
- * Set<NodeId> that contains uuids that should not be indexed
- * nor further traversed.
- * @throws IOException
- * if an error occurs
- */
- MultiIndex(SearchIndex handler, IndexingTree indexingTree, IndexerIoModeHandler modeHandler, IndexInfos indexInfos,
- IndexUpdateMonitor indexUpdateMonitor) throws IOException
- {
- this.modeHandler = modeHandler;
- this.directoryManager = handler.getDirectoryManager();
- // this method is run in privileged mode internally
- this.indexDir = directoryManager.getDirectory(".");
- this.handler = handler;
- this.cache = new DocNumberCache(handler.getCacheSize());
- this.indexingTree = indexingTree;
- this.nsMappings = handler.getNamespaceMappings();
- this.indexNames = indexInfos;
- this.indexNames.setDirectory(indexDir);
- // this method is run in privileged mode internally
- this.indexNames.read();
- this.indexNames.addName("1");
- this.indexNames.addName("2");
- this.indexNames.addName("3");
- this.indexNames.addName("4");
-
- // this method is run in privileged mode internally
- // as of 1.5 deletable file is not used anymore
- removeDeletable();
-
- // this method is run in privileged mode internally
- IndexingQueueStore store = new IndexingQueueStore(indexDir);
-
- // initialize indexing queue
- this.indexingQueue = new IndexingQueue(store);
- // copy current index names
- Set<String> currentNames = new HashSet<String>(indexNames.getNames());
- //Set<String> currentNames = new HashSet<String>();
-
- // open persistent indexes
- for (String name : currentNames)
- {
- // only open if it still exists
- // it is possible that indexNames still contains a name for
- // an index that has been deleted, but indexNames has not been
- // written to disk.
-
- // TODO THIS CHECK WAS SKIPPED
- // if (!directoryManager.hasDirectory(name))
- // {
- // log.debug("index does not exist anymore: " + name);
- // // move on to next index
- // continue;
- // }
- PersistentIndex index =
- new PersistentIndex(name, handler.getTextAnalyzer(), handler.getSimilarity(), cache, indexingQueue,
- directoryManager);
- index.setMaxFieldLength(handler.getMaxFieldLength());
- index.setUseCompoundFile(handler.getUseCompoundFile());
- index.setTermInfosIndexDivisor(handler.getTermInfosIndexDivisor());
- indexes.add(index);
- }
-
- // init volatile index
- // TODO commented line below:
- //resetVolatileIndex();
-
- // set index format version and at the same time
- // initialize hierarchy cache if requested.
- final CachingMultiIndexReader reader = getIndexReader(handler.isInitializeHierarchyCache());
- try
- {
- version = IndexFormatVersion.getVersion(reader);
- }
- finally
- {
- SecurityHelper.doPriviledgedIOExceptionAction(new PrivilegedExceptionAction<Object>()
- {
- public Object run() throws Exception
- {
- reader.release();
- return null;
- }
- });
- }
- indexingQueue.initialize(this);
- this.indexNames.setMultiIndex(this);
- }
-
- /**
- * Returns the number of documents in this index.
- *
- * @return the number of documents in this index.
- * @throws IOException
- * if an error occurs while reading from the index.
- */
- int numDocs() throws IOException
- {
- if (indexNames.size() == 0)
- {
- // TODO commented line below:
- //return volatileIndex.getNumDocuments();
- return 0;
- }
- else
- {
- final CachingMultiIndexReader reader = getIndexReader();
- try
- {
- return reader.numDocs();
- }
- finally
- {
- SecurityHelper.doPriviledgedIOExceptionAction(new PrivilegedExceptionAction<Object>()
- {
- public Object run() throws Exception
- {
- reader.release();
- return null;
- }
- });
- }
- }
- }
-
- /**
- * @return the index format version for this multi index.
- */
- IndexFormatVersion getIndexFormatVersion()
- {
- return version;
- }
-
- /**
- * Creates an initial index by traversing the node hierarchy starting at the
- * node with <code>rootId</code>.
- *
- * @param stateMgr
- * the item state manager.
- * @param rootId
- * the id of the node from where to start.
- * @param rootPath
- * the path of the node from where to start.
- * @throws IOException
- * if an error occurs while indexing the workspace.
- * @throws IllegalStateException
- * if this index is not empty.
- */
- void createInitialIndex(ItemDataConsumer stateMgr) throws IOException
- {
-
- // TODO: re-study check!!!
- // only do an initial index if there are no indexes at all
- // if (indexNames.size() == 0)
- {
- reindexing = true;
- try
- {
- long count = 0;
- // traverse and index workspace
- // TODO: this was removed
- //executeAndLog(new Start(Action.INTERNAL_TRANSACTION));
-
- // NodeData rootState = (NodeData) stateMgr.getItemData(rootId);
- count = createIndex(indexingTree.getIndexingRoot(), stateMgr, count);
-
- // TODO : this was replaced
- //executeAndLog(new Commit(getTransactionId()));
-
- log.info("Created initial index for {} nodes", new Long(count));
- releaseMultiReader();
- // TODO commented line below:
- //scheduleFlushTask();
- }
- catch (Exception e)
- {
- String msg = "Error indexing workspace";
- IOException ex = new IOException(msg);
- ex.initCause(e);
- throw ex;
- }
- finally
- {
- reindexing = false;
- }
- }
- // else
- // {
- // throw new IllegalStateException("Index already present");
- // }
- }
-
- /**
- * Atomically updates the index by removing some documents and adding
- * others.
- *
- * @param remove
- * collection of <code>UUID</code>s that identify documents to
- * remove
- * @param add
- * collection of <code>Document</code>s to add. Some of the
- * elements in this collection may be <code>null</code>, to
- * indicate that a node could not be indexed successfully.
- * @throws IOException
- * if an error occurs while updating the index.
- */
- synchronized void update(final Collection<String> remove, final Collection<Document> add) throws IOException
- {
- SecurityHelper.doPriviledgedIOExceptionAction(new PrivilegedExceptionAction<Object>()
- {
- public Object run() throws Exception
- {
- // make sure a reader is available during long updates
- if (add.size() > handler.getBufferSize())
- {
- try
- {
- getIndexReader().release();
- }
- catch (IOException e)
- {
- // do not fail if an exception is thrown here
- log.warn("unable to prepare index reader " + "for queries during update", e);
- }
- }
-// boolean flush = false;
- try
- {
- // TODO: this was removed
- //executeAndLog(new Start(transactionId));
-
- for (Iterator<String> it = remove.iterator(); it.hasNext();)
- {
- // TODO this was replaced
- //executeAndLog(new DeleteNode(transactionId, (String)it.next()));
-
- String uuidString = it.next();
- // check if indexing queue is still working on
- // this node from a previous update
- Document doc = indexingQueue.removeDocument(uuidString);
- if (doc != null)
- {
- Util.disposeDocument(doc);
- }
- Term idTerm = new Term(FieldNames.UUID, uuidString);
- // if the document cannot be deleted from the volatile index
- // delete it from one of the persistent indexes.
-
- // TODO watch here
- int num = getChunk(uuidString).removeDocument(idTerm);
- if (num == 0)
- {
- for (int i = indexes.size() - 1; i >= 0; i--)
- {
- // only look in registered indexes
- PersistentIndex idx = indexes.get(i);
- if (indexNames.contains(idx.getName()))
- {
- num = idx.removeDocument(idTerm);
- if (num > 0)
- {
- break;
- }
- }
- }
- }
-
- }
- for (Iterator<Document> it = add.iterator(); it.hasNext();)
- {
- Document doc = it.next();
- if (doc != null)
- {
- // TODO: ths is replaced
- //executeAndLog(new AddNode(transactionId, doc));
-
- if (doc != null)
- {
- // TODO watch this
- String uuid = doc.get(FieldNames.UUID);
- getChunk(uuid).addDocuments(new Document[]{doc});
- }
-
- // commit volatile index if needed
- // TODO commented line below:
- //flush |= checkVolatileCommit();
- }
- }
-
- // TODO : this was replaced
- //executeAndLog(new Commit(transactionId));
-
- // TODO commented line below:
- //flush();
- getChunk(null).commit();
- }
- finally
- {
- releaseMultiReader();
- }
- return null;
- }
- });
- }
-
- /**
- * Adds a document to the index.
- *
- * @param doc
- * the document to add.
- * @throws IOException
- * if an error occurs while adding the document to the index.
- */
- void addDocument(Document doc) throws IOException
- {
- update(Collections.EMPTY_LIST, Arrays.asList(new Document[]{doc}));
- }
-
- /**
- * Deletes the first document that matches the <code>uuid</code>.
- *
- * @param uuid
- * document that match this <code>uuid</code> will be deleted.
- * @throws IOException
- * if an error occurs while deleting the document.
- */
- void removeDocument(String uuid) throws IOException
- {
- update(Arrays.asList(new String[]{uuid}), Collections.EMPTY_LIST);
- }
-
- /**
- * Deletes all documents that match the <code>uuid</code>.
- *
- * @param uuid
- * documents that match this <code>uuid</code> will be deleted.
- * @return the number of deleted documents.
- * @throws IOException
- * if an error occurs while deleting documents.
- */
- synchronized int removeAllDocuments(String uuid) throws IOException
- {
- int num;
- try
- {
- Term idTerm = new Term(FieldNames.UUID, uuid.toString());
-
- // TODO: this was removed
- //executeAndLog(new Start(Action.INTERNAL_TRANSACTION));
-
- // TODO REMOVE WAS HERE
- num = getChunk(uuid).removeDocument(idTerm);
- if (num > 0)
- {
- // TODO: removed line
- //redoLog.append(new DeleteNode(getTransactionId(), uuid));
- }
- for (int i = 0; i < indexes.size(); i++)
- {
- PersistentIndex index = indexes.get(i);
- // only remove documents from registered indexes
- if (indexNames.contains(index.getName()))
- {
- int removed = index.removeDocument(idTerm);
- if (removed > 0)
- {
- // TODO: removed line
- //redoLog.append(new DeleteNode(getTransactionId(), uuid));
- }
- num += removed;
- }
- }
- // TODO : this was replaced
- //executeAndLog(new Commit(getTransactionId()));
- }
- finally
- {
- releaseMultiReader();
- }
- return num;
- }
-
- /**
- * Returns <code>IndexReader</code>s for the indexes named
- * <code>indexNames</code>. An <code>IndexListener</code> is registered and
- * notified when documents are deleted from one of the indexes in
- * <code>indexNames</code>.
- * <p/>
- * Note: the number of <code>IndexReaders</code> returned by this method is
- * not necessarily the same as the number of index names passed. An index
- * might have been deleted and is not reachable anymore.
- *
- * @param indexNames
- * the names of the indexes for which to obtain readers.
- * @param listener
- * the listener to notify when documents are deleted.
- * @return the <code>IndexReaders</code>.
- * @throws IOException
- * if an error occurs acquiring the index readers.
- */
- synchronized IndexReader[] getIndexReaders(String[] indexNames, IndexListener listener) throws IOException
- {
- Set<String> names = new HashSet<String>(Arrays.asList(indexNames));
- Map<ReadOnlyIndexReader, PersistentIndex> indexReaders = new HashMap<ReadOnlyIndexReader, PersistentIndex>();
-
- try
- {
- for (Iterator<PersistentIndex> it = indexes.iterator(); it.hasNext();)
- {
- PersistentIndex index = it.next();
- if (names.contains(index.getName()))
- {
- indexReaders.put(index.getReadOnlyIndexReader(listener), index);
- }
- }
- }
- catch (IOException e)
- {
- // release readers obtained so far
- for (Iterator<Entry<ReadOnlyIndexReader, PersistentIndex>> it = indexReaders.entrySet().iterator(); it.hasNext();)
- {
- Map.Entry<ReadOnlyIndexReader, PersistentIndex> entry = it.next();
- final ReadOnlyIndexReader reader = entry.getKey();
- try
- {
- SecurityHelper.doPriviledgedIOExceptionAction(new PrivilegedExceptionAction<Object>()
- {
- public Object run() throws Exception
- {
- reader.release();
- return null;
- }
- });
- }
- catch (IOException ex)
- {
- log.warn("Exception releasing index reader: " + ex);
- }
- (entry.getValue()).resetListener();
- }
- throw e;
- }
-
- return indexReaders.keySet().toArray(new IndexReader[indexReaders.size()]);
- }
-
- /**
- * Returns <code>true</code> if this multi index has an index segment with
- * the given name. This method even returns <code>true</code> if an index
- * segments has not yet been loaded / initialized but exists on disk.
- *
- * @param indexName
- * the name of the index segment.
- * @return <code>true</code> if it exists; otherwise <code>false</code>.
- * @throws IOException
- * if an error occurs while checking existence of directory.
- */
- synchronized boolean hasIndex(String indexName) throws IOException
- {
- // check existing
- for (Iterator<PersistentIndex> it = indexes.iterator(); it.hasNext();)
- {
- PersistentIndex idx = it.next();
- if (idx.getName().equals(indexName))
- {
- return true;
- }
- }
- // check if it exists on disk
- return directoryManager.hasDirectory(indexName);
- }
-
- /**
- * Returns an read-only <code>IndexReader</code> that spans alls indexes of
- * this <code>MultiIndex</code>.
- *
- * @return an <code>IndexReader</code>.
- * @throws IOException
- * if an error occurs constructing the <code>IndexReader</code>.
- */
- public CachingMultiIndexReader getIndexReader() throws IOException
- {
- return getIndexReader(false);
- }
-
- /**
- * Returns an read-only <code>IndexReader</code> that spans alls indexes of
- * this <code>MultiIndex</code>.
- *
- * @param initCache
- * when set <code>true</code> the hierarchy cache is completely
- * initialized before this call returns.
- * @return an <code>IndexReader</code>.
- * @throws IOException
- * if an error occurs constructing the <code>IndexReader</code>.
- */
- public synchronized CachingMultiIndexReader getIndexReader(final boolean initCache) throws IOException
- {
- return SecurityHelper.doPriviledgedIOExceptionAction(new PrivilegedExceptionAction<CachingMultiIndexReader>()
- {
- public CachingMultiIndexReader run() throws Exception
- {
- if (multiReader != null)
- {
- multiReader.acquire();
- return multiReader;
- }
- // no reader available
- // some other read thread might have created the reader in the
- // meantime -> check again
- if (multiReader == null)
- {
- List<ReadOnlyIndexReader> readerList = new ArrayList<ReadOnlyIndexReader>();
- for (int i = 0; i < indexes.size(); i++)
- {
- PersistentIndex pIdx = indexes.get(i);
-
- if (indexNames.contains(pIdx.getName()))
- {
- try
- {
- readerList.add(pIdx.getReadOnlyIndexReader(initCache));
- }
- catch (FileNotFoundException e)
- {
- if (directoryManager.hasDirectory(pIdx.getName()))
- {
- throw e;
- }
- }
- }
- }
- // TODO commented line below:
- //readerList.add(volatileIndex.getReadOnlyIndexReader());
- ReadOnlyIndexReader[] readers = readerList.toArray(new ReadOnlyIndexReader[readerList.size()]);
- multiReader = new CachingMultiIndexReader(readers, cache);
- }
- multiReader.acquire();
- return multiReader;
- }
- });
-
- }
-
- /**
- * Closes this <code>MultiIndex</code>.
- */
- void close()
- {
- if (modeHandler.getMode().equals(IndexerIoMode.READ_WRITE))
- {
-
- // stop index merger
- // when calling this method we must not lock this MultiIndex, otherwise
- // a deadlock might occur
-
- synchronized (this)
- {
-
- // commit / close indexes
- try
- {
- releaseMultiReader();
- }
- catch (IOException e)
- {
- log.error("Exception while closing search index.", e);
- }
- // try
- // {
- // // TODO commented line below:
- // //flush();
- // }
- // catch (IOException e)
- // {
- // log.error("Exception while closing search index.", e);
- // }
- // TODO commented line below:
- //volatileIndex.close();
- // indexes will be closed now:
- for (int i = 0; i < indexes.size(); i++)
- {
- (indexes.get(i)).close();
- }
-
- // close indexing queue
- indexingQueue.close();
-
- // finally close directory
- try
- {
- indexDir.close();
- }
- catch (IOException e)
- {
- log.error("Exception while closing directory.", e);
- }
- }
- }
- }
-
- /**
- * Returns the namespace mappings of this search index.
- *
- * @return the namespace mappings of this search index.
- */
- NamespaceMappings getNamespaceMappings()
- {
- return nsMappings;
- }
-
- /**
- * Returns the indexing queue for this multi index.
- *
- * @return the indexing queue for this multi index.
- */
- public IndexingQueue getIndexingQueue()
- {
- return indexingQueue;
- }
-
- /**
- * Returns a lucene Document for the <code>node</code>.
- *
- * @param node
- * the node to index.
- * @return the index document.
- * @throws RepositoryException
- * if an error occurs while reading from the workspace.
- */
- Document createDocument(NodeData node) throws RepositoryException
- {
- return handler.createDocument(node, nsMappings, version);
- }
-
- /**
- * Returns a lucene Document for the Node with <code>id</code>.
- *
- * @param id
- * the id of the node to index.
- * @return the index document.
- * @throws RepositoryException
- * if an error occurs while reading from the workspace or if
- * there is no node with <code>id</code>.
- */
- Document createDocument(String id) throws RepositoryException
- {
- ItemData data = handler.getContext().getItemStateManager().getItemData(id);
- if (data == null)
- {
- throw new ItemNotFoundException("Item id=" + id + " not found");
- }
- if (!data.isNode())
- {
- throw new RepositoryException("Item with id " + id + " is not a node");
- }
- return createDocument((NodeData)data);
-
- }
-
- /**
- * Releases the {@link #multiReader} and sets it <code>null</code>. If the
- * reader is already <code>null</code> this method does nothing. When this
- * method returns {@link #multiReader} is guaranteed to be <code>null</code>
- * even if an exception is thrown.
- * <p/>
- * Please note that this method does not take care of any synchronization. A
- * caller must ensure that it is the only thread operating on this multi
- * index, or that it holds the {@link #updateMonitor}.
- *
- * @throws IOException
- * if an error occurs while releasing the reader.
- */
- void releaseMultiReader() throws IOException
- {
- SecurityHelper.doPriviledgedIOExceptionAction(new PrivilegedExceptionAction<Object>()
- {
- public Object run() throws Exception
- {
- if (multiReader != null)
- {
- try
- {
- multiReader.release();
- }
- finally
- {
- multiReader = null;
- }
- }
- return null;
- }
- });
- }
-
- // -------------------------< internal
- // >-------------------------------------
-
- /**
- * Recursively creates an index starting with the NodeState
- * <code>node</code>.
- *
- * @param node
- * the current NodeState.
- * @param path
- * the path of the current node.
- * @param stateMgr
- * the shared item state manager.
- * @param count
- * the number of nodes already indexed.
- * @return the number of nodes indexed so far.
- * @throws IOException
- * if an error occurs while writing to the index.
- * @throws ItemStateException
- * if an node state cannot be found.
- * @throws RepositoryException
- * if any other error occurs
- */
- private long createIndex(NodeData node, ItemDataConsumer stateMgr, long count) throws IOException,
- RepositoryException
- {
- // NodeId id = node.getNodeId();
-
- if (indexingTree.isExcluded(node))
- {
- return count;
- }
-
- // TODO: this is replaced
- //executeAndLog(new AddNode(getTransactionId(), node.getIdentifier()));
- getChunk(node.getIdentifier()).addDocuments(new Document[]{createDocument(node)});
-
- if (++count % 100 == 0)
- {
-
- log.info("indexing... {} ({})", node.getQPath().getAsString(), new Long(count));
- }
- if (count % 10 == 0)
- {
- checkIndexingQueue(true);
- }
- // TODO commented line below:
- //checkVolatileCommit();
- List<NodeData> children = stateMgr.getChildNodesData(node);
- for (NodeData nodeData : children)
- {
-
- NodeData childState = (NodeData)stateMgr.getItemData(nodeData.getIdentifier());
- if (childState == null)
- {
- handler.getOnWorkspaceInconsistencyHandler().handleMissingChildNode(
- new ItemNotFoundException("Child not found "), handler, nodeData.getQPath(), node, nodeData);
- }
-
- if (nodeData != null)
- {
- count = createIndex(nodeData, stateMgr, count);
- }
- }
-
- return count;
- }
-
- /**
- * Removes the deletable file if it exists. The file is not used anymore in
- * Jackrabbit versions >= 1.5.
- */
- private void removeDeletable()
- {
- SecurityHelper.doPriviledgedAction(new PrivilegedAction<Object>()
- {
- public Object run()
- {
- String fileName = "deletable";
- try
- {
- if (indexDir.fileExists(fileName))
- {
- indexDir.deleteFile(fileName);
- }
- }
- catch (IOException e)
- {
- log.warn("Unable to remove file 'deletable'.", e);
- }
- return null;
- }
- });
- }
-
- /**
- * Checks the indexing queue for finished text extrator jobs and updates the
- * index accordingly if there are any new ones. This method is synchronized
- * and should only be called by the timer task that periodically checks if
- * there are documents ready in the indexing queue. A new transaction is
- * used when documents are transfered from the indexing queue to the index.
- */
- private synchronized void checkIndexingQueue()
- {
- checkIndexingQueue(false);
- }
-
- /**
- * Checks the indexing queue for finished text extrator jobs and updates the
- * index accordingly if there are any new ones.
- *
- * @param transactionPresent
- * whether a transaction is in progress and the current
- * {@link #getTransactionId()} should be used. If
- * <code>false</code> a new transaction is created when documents
- * are transfered from the indexing queue to the index.
- */
- private void checkIndexingQueue(boolean transactionPresent)
- {
- Document[] docs = indexingQueue.getFinishedDocuments();
- Map<String, Document> finished = new HashMap<String, Document>();
- for (int i = 0; i < docs.length; i++)
- {
- String uuid = docs[i].get(FieldNames.UUID);
- finished.put(uuid, docs[i]);
- }
-
- // now update index with the remaining ones if there are any
- if (!finished.isEmpty())
- {
- log.info("updating index with {} nodes from indexing queue.", new Long(finished.size()));
-
- // remove documents from the queue
- for (Iterator<String> it = finished.keySet().iterator(); it.hasNext();)
- {
- indexingQueue.removeDocument(it.next());
- }
-
- try
- {
- if (transactionPresent)
- {
- for (Iterator<String> it = finished.keySet().iterator(); it.hasNext();)
- {
- // TODO this was replaced
- //executeAndLog(new DeleteNode(getTransactionId(), (String)it.next()));
-
- String uuidString = it.next();
- // check if indexing queue is still working on
- // this node from a previous update
- Document doc = indexingQueue.removeDocument(uuidString);
- if (doc != null)
- {
- Util.disposeDocument(doc);
- }
- Term idTerm = new Term(FieldNames.UUID, uuidString);
- // if the document cannot be deleted from the volatile index
- // delete it from one of the persistent indexes.
- int num = getChunk(uuidString).removeDocument(idTerm);
- if (num == 0)
- {
- for (int i = indexes.size() - 1; i >= 0; i--)
- {
- // only look in registered indexes
- PersistentIndex idx = indexes.get(i);
- if (indexNames.contains(idx.getName()))
- {
- num = idx.removeDocument(idTerm);
- if (num > 0)
- {
- return;
- }
- }
- }
- }
-
- }
- for (Iterator<Document> it = finished.values().iterator(); it.hasNext();)
- {
- // TODO this was replaced
- //executeAndLog(new AddNode(getTransactionId(), (Document)it.next()));
- Document doc = it.next();
- String uuid = doc.get(FieldNames.UUID);
- getChunk(uuid).addDocuments(new Document[]{doc});
- }
- }
- else
- {
- update(finished.keySet(), finished.values());
- }
- }
- catch (IOException e)
- {
- // update failed
- log.warn("Failed to update index with deferred text extraction", e);
- }
- }
- }
-
- // ------------------------< Actions
- // >---------------------------------------
-
- @Deprecated
- public boolean getRedoLogApplied()
- {
- return false;
- }
-
- private PersistentIndex getChunk(String uuid)
- {
- return indexes.get(0);
- }
-}
Modified: jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/SearchIndex.java
===================================================================
--- jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/SearchIndex.java 2010-10-11 16:33:49 UTC (rev 3291)
+++ jcr/branches/1.14-CNK/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/SearchIndex.java 2010-10-12 09:47:34 UTC (rev 3292)
@@ -175,7 +175,7 @@
/**
* The actual index
*/
- private MultiIndex index;
+ private ChunkIndex index;
/**
* The analyzer we use for indexing.
@@ -370,7 +370,7 @@
/**
* Indicates the index format version which is relevant to a <b>query</b>.
* This value may be different from what
- * {@link MultiIndex#getIndexFormatVersion()} returns because queries may be
+ * {@link ChunkIndex#getIndexFormatVersion()} returns because queries may be
* executed on two physical indexes with different formats. Index format
* versions are considered backward compatible. That is, the lower version
* of the two physical indexes is used for querying.
@@ -570,7 +570,7 @@
indexingConfig = createIndexingConfiguration(nsMappings);
analyzer.setIndexingConfig(indexingConfig);
- index = new MultiIndex(this, context.getIndexingTree(), modeHandler, getIndexInfos(), getIndexUpdateMonitor());
+ index = new ChunkIndex(this, context.getIndexingTree(), modeHandler, getIndexInfos(), getIndexUpdateMonitor());
// if RW mode, create initial index and start check
if (modeHandler.getMode() == IndexerIoMode.READ_WRITE)
{
@@ -676,7 +676,7 @@
/**
* This implementation forwards the call to
- * {@link MultiIndex#update(Collection, Collection)} and transforms the two
+ * {@link ChunkIndex#update(Collection, Collection)} and transforms the two
* iterators to the required types.
*
* @param remove
@@ -1212,7 +1212,7 @@
*
* @return the actual index.
*/
- public MultiIndex getIndex()
+ public ChunkIndex getIndex()
{
return index;
}
15 years, 9 months
- 1
- 0
exo-jcr SVN: r3291 - jcr/trunk/exo.jcr.docs/exo.jcr.docs.developer/en/src/main/docbook/en-US/modules/kernel.
by do-not-reply＠jboss.org
Author: nfilotto
Date: 2010-10-11 12:33:49 -0400 (Mon, 11 Oct 2010)
New Revision: 3291
Modified:
jcr/trunk/exo.jcr.docs/exo.jcr.docs.developer/en/src/main/docbook/en-US/modules/kernel/rpc-service.xml
Log:
EXOJCR-967: Doc improved
Modified: jcr/trunk/exo.jcr.docs/exo.jcr.docs.developer/en/src/main/docbook/en-US/modules/kernel/rpc-service.xml
===================================================================
--- jcr/trunk/exo.jcr.docs/exo.jcr.docs.developer/en/src/main/docbook/en-US/modules/kernel/rpc-service.xml 2010-10-11 15:59:50 UTC (rev 3290)
+++ jcr/trunk/exo.jcr.docs/exo.jcr.docs.developer/en/src/main/docbook/en-US/modules/kernel/rpc-service.xml 2010-10-11 16:33:49 UTC (rev 3291)
@@ -199,9 +199,10 @@
proposed by default. This command will dynamically execute a method on a
given object.</para>
- <programlisting>// Register the command first
+ <programlisting>// Register the command first (to be done before that the RPCService has been started)
RemoteCommand commandGetName = rpcService.registerCommand(new SingleMethodCallCommand(myService, "getName"));
-// Execute the command on the coordinator
+...
+// Execute the command on the coordinator (can be done only after having started the RPCService)
String name = rpcService.executeCommandOnCoordinator(commandGetName, true);
// Print the name
System.out.println("Name : " + name);</programlisting>
@@ -212,8 +213,8 @@
<listitem>
<para>Register a <emphasis>SingleMethodCallCommand</emphasis> that
will call <emphasis>getName()</emphasis> on the Object
- <emphasis>myService</emphasis> anytime the command will be
- executed.</para>
+ <emphasis>myService</emphasis> anytime the command will be executed.
+ </para>
</listitem>
<listitem>
@@ -226,5 +227,24 @@
<para>Print the name got from the coordinator</para>
</listitem>
</orderedlist>
+
+ <note>
+ <para>As any <emphasis>RemoteCommand</emphasis>, it has to be registered
+ before being executed and before the <emphasis>RPCService</emphasis> is
+ launched.</para>
+ </note>
+
+ <note>
+ <para>As any <emphasis>RemoteCommand</emphasis>, the command can be
+ executed only once the <emphasis>RPCService</emphasis> is
+ launched.</para>
+ </note>
+
+ <note>
+ <para>The <emphasis>SingleMethodCallCommand</emphasis> only allow public
+ methods, if you try to register a non public method an
+ <emphasis>RPCException</emphasis> will be thrown at creation
+ level.</para>
+ </note>
</section>
</chapter>
15 years, 9 months
- 1
- 0