[exo-jcr-commits] exo-jcr SVN: r3761 - in jcr/branches/1.14-IMPR/exo.jcr.component.core: src/main/java/org/exoplatform/services/jcr/impl and 6 other directories.

Author: nfilotto Date: 2010-12-31 12:35:02 -0500 (Fri, 31 Dec 2010) New Revision: 3761 Modified: jcr/branches/1.14-IMPR/exo.jcr.component.core/pom.xml jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/RepositoryServiceImpl.java jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/IndexerChangesFilter.java jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/QueryHandler.java jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/SearchManager.java jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/SystemSearchManager.java jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/jbosscache/ChangesFilterListsWrapper.java jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/jbosscache/IndexerCacheLoader.java jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/jbosscache/IndexerSingletonStoreCacheLoader.java jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/jbosscache/JBossCacheIndexChangesFilter.java jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/ChangesHolder.java jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/SearchIndex.java jcr/branches/1.14-IMPR/exo.jcr.component.core/src/test/java/org/exoplatform/services/jcr/api/core/query/lucene/SlowQueryHandler.java jcr/branches/1.14-IMPR/exo.jcr.component.core/src/test/resources/conf/standalone/cluster/test-configuration.xml jcr/branches/1.14-IMPR/exo.jcr.component.core/src/test/resources/conf/standalone/test-configuration.xml Log: EXOJCR-1080: Implementation without the RPCService, any classes that use the RPCService have been reverted. NB: With the default parameters the build fails due to an OutOffMemory because it consumes a little bit more memory, so we need to allocate to the JVM a little bit more memory Modified: jcr/branches/1.14-IMPR/exo.jcr.component.core/pom.xml =================================================================== --- jcr/branches/1.14-IMPR/exo.jcr.component.core/pom.xml 2010-12-30 18:07:06 UTC (rev 3760) +++ jcr/branches/1.14-IMPR/exo.jcr.component.core/pom.xml 2010-12-31 17:35:02 UTC (rev 3761) @@ -656,6 +656,7 @@ <artifactId>maven-surefire-plugin</artifactId> <configuration> <!-- TAKE CARE TO UPDATE ALSO run-tck PROFILE --> + <argLine>${env.MAVEN_OPTS}</argLine> <systemProperties> <property> <name>jcr.test.configuration.file</name> Modified: jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/RepositoryServiceImpl.java =================================================================== --- jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/RepositoryServiceImpl.java 2010-12-30 18:07:06 UTC (rev 3760) +++ jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/RepositoryServiceImpl.java 2010-12-31 17:35:02 UTC (rev 3761) @@ -35,7 +35,6 @@ import org.exoplatform.services.jcr.impl.core.SessionRegistry; import org.exoplatform.services.log.ExoLogger; import org.exoplatform.services.log.Log; -import org.exoplatform.services.rpc.RPCService; import org.picocontainer.Startable; import java.io.InputStream; @@ -91,22 +90,13 @@ currentRepositoryName.set(config.getDefaultRepositoryName()); managerStartChanges = new ManagerStartChanges(); } - - public RepositoryServiceImpl(RepositoryServiceConfiguration configuration, ExoContainerContext context, RPCService rpcService) - { - this(configuration, context); - } public void addPlugin(ComponentPlugin plugin) { if (plugin instanceof AddNodeTypePlugin) - { addNodeTypePlugins.add(plugin); - } else if (plugin instanceof AddNamespacesPlugin) - { addNamespacesPlugins.add(plugin); - } else if (plugin instanceof RepositoryChangesListenerRegisterPlugin) { managerStartChanges.addPlugin((RepositoryChangesListenerRegisterPlugin)plugin); @@ -177,9 +167,7 @@ public ManageableRepository getCurrentRepository() throws RepositoryException { if (currentRepositoryName.get() == null) - { return getDefaultRepository(); - } return getRepository(currentRepositoryName.get()); } @@ -204,9 +192,7 @@ RepositoryContainer repositoryContainer = repositoryContainers.get(name); log.debug("RepositoryServiceimpl() getRepository " + name); if (repositoryContainer == null) - { throw new RepositoryException("Repository '" + name + "' not found."); - } return (ManageableRepository)repositoryContainer.getComponentInstanceOfType(ManageableRepository.class); } @@ -222,9 +208,7 @@ public void setCurrentRepositoryName(String repositoryName) throws RepositoryConfigurationException { if (!repositoryContainers.containsKey(repositoryName)) - { throw new RepositoryConfigurationException("Repository is not configured. Name " + repositoryName); - } currentRepositoryName.set(repositoryName); } @@ -368,10 +352,8 @@ private void removeRepository(String name, boolean allowRemoveDefaultRepository) throws RepositoryException { if (!canRemoveRepository(name, allowRemoveDefaultRepository)) - { throw new RepositoryException("Repository " + name + " in use. If you want to " + " remove repository close all open sessions"); - } try { @@ -425,18 +407,14 @@ { // Check non system workspaces if (!repo.getSystemWorkspaceName().equals(wsEntry.getName()) && !repo.canRemoveWorkspace(wsEntry.getName())) - { return false; - } } // check system workspace RepositoryContainer repositoryContainer = repositoryContainers.get(name); SessionRegistry sessionRegistry = (SessionRegistry)repositoryContainer.getComponentInstance(SessionRegistry.class); if (sessionRegistry == null || sessionRegistry.isInUse(repo.getSystemWorkspaceName())) - { return false; - } } catch (RepositoryConfigurationException e) Modified: jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/IndexerChangesFilter.java =================================================================== --- jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/IndexerChangesFilter.java 2010-12-30 18:07:06 UTC (rev 3760) +++ jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/IndexerChangesFilter.java 2010-12-31 17:35:02 UTC (rev 3761) @@ -109,9 +109,6 @@ */ public void onSaveItems(ItemStateChangesLog itemStates) { - - long time = System.currentTimeMillis(); - // nodes that need to be removed from the index. final Set<String> removedNodes = new HashSet<String>(); // nodes that need to be added to the index. Modified: jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/QueryHandler.java =================================================================== --- jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/QueryHandler.java 2010-12-30 18:07:06 UTC (rev 3760) +++ jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/QueryHandler.java 2010-12-31 17:35:02 UTC (rev 3761) @@ -21,6 +21,7 @@ import org.exoplatform.services.jcr.datamodel.NodeData; import org.exoplatform.services.jcr.impl.core.SessionDataManager; import org.exoplatform.services.jcr.impl.core.SessionImpl; +import org.exoplatform.services.jcr.impl.core.query.lucene.ChangesHolder; import org.exoplatform.services.jcr.impl.core.query.lucene.IndexInfos; import org.exoplatform.services.jcr.impl.core.query.lucene.IndexUpdateMonitor; import org.exoplatform.services.jcr.impl.core.query.lucene.MultiIndex; @@ -79,6 +80,23 @@ void updateNodes(Iterator<String> remove, Iterator<NodeData> add) throws RepositoryException, IOException; /** + * Extracts all the changes and returns them as a {@link ChangesHolder} instance + * @param remove Iterator of <code>NodeIds</code> of nodes to delete + * @param add Iterator of <code>NodeState</code> instance to add to the + * index. + * @return a {@link ChangesHolder} instance that contains all the changes + */ + ChangesHolder getChanges(Iterator<String> remove, Iterator<NodeData> add); + + /** + * Applies the given changes to the indes in an atomic operation + * @param changes the changes to apply + * @throws RepositoryException if an error occurs while indexing a node. + * @throws IOException if an error occurs while updating the index. + */ + void apply(ChangesHolder changes) throws RepositoryException, IOException; + + /** * Closes this <code>QueryHandler</code> and frees resources attached * to this handler. */ Modified: jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/SearchManager.java =================================================================== --- jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/SearchManager.java 2010-12-30 18:07:06 UTC (rev 3760) +++ jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/SearchManager.java 2010-12-31 17:35:02 UTC (rev 3761) @@ -43,6 +43,7 @@ import org.exoplatform.services.jcr.impl.core.NamespaceRegistryImpl; import org.exoplatform.services.jcr.impl.core.SessionDataManager; import org.exoplatform.services.jcr.impl.core.SessionImpl; +import org.exoplatform.services.jcr.impl.core.query.lucene.ChangesHolder; import org.exoplatform.services.jcr.impl.core.query.lucene.FieldNames; import org.exoplatform.services.jcr.impl.core.query.lucene.LuceneVirtualTableResolver; import org.exoplatform.services.jcr.impl.core.query.lucene.QueryHits; @@ -54,7 +55,6 @@ import org.exoplatform.services.jcr.impl.dataflow.persistent.WorkspacePersistentDataManager; import org.exoplatform.services.log.ExoLogger; import org.exoplatform.services.log.Log; -import org.exoplatform.services.rpc.RPCService; import org.jboss.cache.factories.annotations.NonVolatile; import org.picocontainer.Startable; @@ -141,9 +141,7 @@ * The unique name of the related workspace */ protected final String wsId; - - protected final RPCService rpcService; - + /** * Creates a new <code>SearchManager</code>. * @@ -168,11 +166,10 @@ * @throws RepositoryConfigurationException */ - public SearchManager(WorkspaceEntry wsConfig, QueryHandlerEntry config, NamespaceRegistryImpl nsReg, - NodeTypeDataManager ntReg, WorkspacePersistentDataManager itemMgr, SystemSearchManagerHolder parentSearchManager, - DocumentReaderService extractor, ConfigurationManager cfm, - final RepositoryIndexSearcherHolder indexSearcherHolder, RPCService rpcService) throws RepositoryException, - RepositoryConfigurationException + public SearchManager(WorkspaceEntry wsConfig, QueryHandlerEntry config, NamespaceRegistryImpl nsReg, NodeTypeDataManager ntReg, + WorkspacePersistentDataManager itemMgr, SystemSearchManagerHolder parentSearchManager, + DocumentReaderService extractor, ConfigurationManager cfm, final RepositoryIndexSearcherHolder indexSearcherHolder) + throws RepositoryException, RepositoryConfigurationException { this.wsId = wsConfig.getUniqueName(); this.extractor = extractor; @@ -184,7 +181,6 @@ this.cfm = cfm; this.virtualTableResolver = new LuceneVirtualTableResolver(nodeTypeDataManager, nsReg); this.parentSearchManager = parentSearchManager != null ? parentSearchManager.get() : null; - this.rpcService = rpcService; if (parentSearchManager != null) { ((WorkspacePersistentDataManager)this.itemMgr).addItemPersistenceListener(this); @@ -282,9 +278,7 @@ try { if (reader != null) - { reader.close(); - } } catch (IOException e) { @@ -397,9 +391,7 @@ { if (log.isDebugEnabled()) - { log.debug("start"); - } try { if (indexingTree == null) @@ -421,9 +413,7 @@ { ItemData excludeData = itemMgr.getItemData(stringTokenizer.nextToken()); if (excludeData != null) - { excludedPath.add(excludeData.getQPath()); - } } catch (RepositoryException e) { @@ -440,9 +430,7 @@ { ItemData indexingRootDataItem = itemMgr.getItemData(rootNodeIdentifer); if (indexingRootDataItem != null && indexingRootDataItem.isNode()) - { indexingRootData = (NodeData)indexingRootDataItem; - } } catch (RepositoryException e) { @@ -497,6 +485,23 @@ public void updateIndex(final Set<String> removedNodes, final Set<String> addedNodes) throws RepositoryException, IOException { + final ChangesHolder changes = getChanges(removedNodes, addedNodes); + apply(changes); + } + + public void apply(ChangesHolder changes) throws RepositoryException, IOException + { + if (handler != null && changes != null && (!changes.getAdd().isEmpty() || !changes.getRemove().isEmpty())) + { + handler.apply(changes); + } + } + + /** + * Extracts all the changes and returns them as a {@link ChangesHolder} instance + */ + public ChangesHolder getChanges(final Set<String> removedNodes, final Set<String> addedNodes) + { if (handler != null) { Iterator<NodeData> addedStates = new Iterator<NodeData>() @@ -523,20 +528,14 @@ if (item.isNode()) { if (!indexingTree.isExcluded(item)) - { return (NodeData)item; - } } else - { log.warn("Node not found, but property " + id + ", " + item.getQPath().getAsString() + " found. "); - } } else - { log.warn("Unable to index node with id " + id + ", node does not exist."); - } } catch (RepositoryException e) @@ -583,12 +582,12 @@ if (removedNodes.size() > 0 || addedNodes.size() > 0) { - handler.updateNodes(removedIds, addedStates); + return handler.getChanges(removedIds, addedStates); } } - + return null; } - + protected QueryHandlerContext createQueryHandlerContext(QueryHandler parentHandler) throws RepositoryConfigurationException { @@ -727,19 +726,14 @@ // initialize query handler String className = config.getType(); if (className == null) - { throw new RepositoryConfigurationException("Content hanler configuration fail"); - } try { Class qHandlerClass = Class.forName(className, true, this.getClass().getClassLoader()); - Constructor constuctor = - qHandlerClass.getConstructor(QueryHandlerEntry.class, ConfigurationManager.class, RPCService.class, - String.class); + Constructor constuctor = qHandlerClass.getConstructor(QueryHandlerEntry.class, ConfigurationManager.class); + handler = (QueryHandler)constuctor.newInstance(config, cfm); QueryHandler parentHandler = (this.parentSearchManager != null) ? parentSearchManager.getHandler() : null; - handler = - (QueryHandler)constuctor.newInstance(config, cfm, rpcService, wsId + (parentHandler != null ? "" : "-sys")); QueryHandlerContext context = createQueryHandlerContext(parentHandler); handler.setContext(context); Modified: jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/SystemSearchManager.java =================================================================== --- jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/SystemSearchManager.java 2010-12-30 18:07:06 UTC (rev 3760) +++ jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/SystemSearchManager.java 2010-12-31 17:35:02 UTC (rev 3761) @@ -29,7 +29,6 @@ import org.exoplatform.services.jcr.impl.dataflow.persistent.WorkspacePersistentDataManager; import org.exoplatform.services.log.ExoLogger; import org.exoplatform.services.log.Log; -import org.exoplatform.services.rpc.RPCService; import java.util.ArrayList; import java.util.List; @@ -58,12 +57,11 @@ public static final String INDEX_DIR_SUFFIX = "system"; - public SystemSearchManager(WorkspaceEntry wsConfig, QueryHandlerEntry config, NamespaceRegistryImpl nsReg, - NodeTypeDataManager ntReg, WorkspacePersistentDataManager itemMgr, DocumentReaderService service, - ConfigurationManager cfm, RepositoryIndexSearcherHolder indexSearcherHolder, RPCService rpcService) - throws RepositoryException, RepositoryConfigurationException + public SystemSearchManager(WorkspaceEntry wsConfig, QueryHandlerEntry config, NamespaceRegistryImpl nsReg, NodeTypeDataManager ntReg, + WorkspacePersistentDataManager itemMgr, DocumentReaderService service, ConfigurationManager cfm, + RepositoryIndexSearcherHolder indexSearcherHolder) throws RepositoryException, RepositoryConfigurationException { - super(wsConfig, config, nsReg, ntReg, itemMgr, null, service, cfm, indexSearcherHolder, rpcService); + super(wsConfig, config, nsReg, ntReg, itemMgr, null, service, cfm, indexSearcherHolder); } @Override Modified: jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/jbosscache/ChangesFilterListsWrapper.java =================================================================== --- jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/jbosscache/ChangesFilterListsWrapper.java 2010-12-30 18:07:06 UTC (rev 3760) +++ jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/jbosscache/ChangesFilterListsWrapper.java 2010-12-31 17:35:02 UTC (rev 3761) @@ -18,6 +18,8 @@ */ package org.exoplatform.services.jcr.impl.core.query.jbosscache; +import org.exoplatform.services.jcr.impl.core.query.lucene.ChangesHolder; + import java.io.Serializable; import java.util.Set; @@ -40,6 +42,10 @@ private Set<String> parentRemovedNodes; + private ChangesHolder changes; + + private ChangesHolder parentChanges; + /** * Creates ChangesFilterListsWrapper data class, containing given lists. * @@ -56,7 +62,31 @@ this.parentAddedNodes = parentAddedNodes; this.parentRemovedNodes = parentRemovedNodes; } + + /** + * Creates ChangesFilterListsWrapper data class, containing given lists. + */ + public ChangesFilterListsWrapper(ChangesHolder changes, ChangesHolder parentChanges) + { + this.changes = changes; + this.parentChanges = parentChanges; + } + public boolean withChanges() + { + return changes != null || parentChanges != null; + } + + public ChangesHolder getChanges() + { + return changes; + } + + public ChangesHolder getParentChanges() + { + return parentChanges; + } + public Set<String> getAddedNodes() { return addedNodes; @@ -76,15 +106,4 @@ { return parentRemovedNodes; } - - public String dump() - { - StringBuffer buffer = new StringBuffer(); - buffer.append("\n"); - buffer.append("Added=").append(addedNodes.toString()).append("\n"); - buffer.append("Removed=").append(removedNodes.toString()).append("\n"); - buffer.append("ParentAdded=").append(parentAddedNodes.toString()).append("\n"); - buffer.append("ParentRemoved=").append(parentRemovedNodes.toString()); - return buffer.toString(); - } } Modified: jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/jbosscache/IndexerCacheLoader.java =================================================================== --- jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/jbosscache/IndexerCacheLoader.java 2010-12-30 18:07:06 UTC (rev 3760) +++ jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/jbosscache/IndexerCacheLoader.java 2010-12-31 17:35:02 UTC (rev 3761) @@ -21,6 +21,7 @@ import org.exoplatform.services.jcr.impl.core.query.IndexerIoModeHandler; import org.exoplatform.services.jcr.impl.core.query.QueryHandler; import org.exoplatform.services.jcr.impl.core.query.SearchManager; +import org.exoplatform.services.jcr.impl.core.query.lucene.ChangesHolder; import org.exoplatform.services.jcr.impl.storage.jbosscache.AbstractWriteOnlyCacheLoader; import org.exoplatform.services.log.ExoLogger; import org.exoplatform.services.log.Log; @@ -30,6 +31,7 @@ import java.io.IOException; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -99,10 +101,14 @@ log.warn("No indexer could be found for the fqn " + name.getParent()); if (log.isDebugEnabled()) log.debug("The current content of the map of indexers is " + indexers); } + else if (wrapper.withChanges()) + { + indexer.updateIndex(wrapper.getChanges(), wrapper.getParentChanges()); + } else { indexer.updateIndex(wrapper.getAddedNodes(), wrapper.getRemovedNodes(), wrapper.getParentAddedNodes(), wrapper - .getParentRemovedNodes()); + .getParentRemovedNodes()); } } finally @@ -250,7 +256,7 @@ log.error("Error indexing changes " + e, e); try { - parentHandler.logErrorChanges(removedNodes, addedNodes); + parentHandler.logErrorChanges(parentRemovedNodes, parentAddedNodes); } catch (IOException ioe) { @@ -259,5 +265,60 @@ } } } + + /** + * Flushes lists of added/removed nodes to SearchManagers, starting indexing. + */ + protected void updateIndex(ChangesHolder changes, ChangesHolder parentChanges) + { + // pass lists to search manager + if (searchManager != null && changes != null) + { + try + { + searchManager.apply(changes); + } + catch (RepositoryException e) + { + log.error("Error indexing changes " + e, e); + } + catch (IOException e) + { + log.error("Error indexing changes " + e, e); + try + { + handler.logErrorChanges(new HashSet<String>(changes.getRemove()), new HashSet<String>(changes.getAddIds())); + } + catch (IOException ioe) + { + log.warn("Exception occure when errorLog writed. Error log is not complete. " + ioe, ioe); + } + } + } + // pass lists to parent search manager + if (parentSearchManager != null && parentChanges!= null) + { + try + { + parentSearchManager.apply(parentChanges); + } + catch (RepositoryException e) + { + log.error("Error indexing changes " + e, e); + } + catch (IOException e) + { + log.error("Error indexing changes " + e, e); + try + { + parentHandler.logErrorChanges(new HashSet<String>(parentChanges.getRemove()), new HashSet<String>(parentChanges.getAddIds())); + } + catch (IOException ioe) + { + log.warn("Exception occure when errorLog writed. Error log is not complete. " + ioe, ioe); + } + } + } + } } } Modified: jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/jbosscache/IndexerSingletonStoreCacheLoader.java =================================================================== --- jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/jbosscache/IndexerSingletonStoreCacheLoader.java 2010-12-30 18:07:06 UTC (rev 3760) +++ jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/jbosscache/IndexerSingletonStoreCacheLoader.java 2010-12-31 17:35:02 UTC (rev 3761) @@ -77,22 +77,47 @@ { Fqn<?> fqn = aChildren.getFqn(); Object value = cache.get(fqn, JBossCacheIndexChangesFilter.LISTWRAPPER); - if (value != null && value instanceof ChangesFilterListsWrapper) + if (value instanceof ChangesFilterListsWrapper) { // get wrapper object ChangesFilterListsWrapper listsWrapper = (ChangesFilterListsWrapper)value; - // get search manager lists - addedNodes.addAll(listsWrapper.getAddedNodes()); - removedNodes.addAll(listsWrapper.getRemovedNodes()); - // parent search manager lists - parentAddedNodes.addAll(listsWrapper.getParentAddedNodes()); - parentRemovedNodes.addAll(listsWrapper.getParentAddedNodes()); - } + if (listsWrapper.withChanges()) + { + if (listsWrapper.getChanges() != null) + { + // get search manager lists + addedNodes.addAll(listsWrapper.getChanges().getAddIds()); + removedNodes.addAll(listsWrapper.getChanges().getRemove()); + } + if (listsWrapper.getParentChanges() != null) + { + // parent search manager lists + parentAddedNodes.addAll(listsWrapper.getParentChanges().getAddIds()); + parentRemovedNodes.addAll(listsWrapper.getParentChanges().getRemove()); + } + } + else + { + // get search manager lists + addedNodes.addAll(listsWrapper.getAddedNodes()); + removedNodes.addAll(listsWrapper.getRemovedNodes()); + // parent search manager lists + parentAddedNodes.addAll(listsWrapper.getParentAddedNodes()); + parentRemovedNodes.addAll(listsWrapper.getParentAddedNodes()); + } + } } //TODO: recover logic is here, lists are: removedNodes and addedNodes String id = IdGenerator.generate(); String id = IdGenerator.generate(); cache.put(Fqn.fromRelativeElements(wsChildren.getFqn(), id), JBossCacheIndexChangesFilter.LISTWRAPPER, new ChangesFilterListsWrapper(addedNodes, removedNodes, parentAddedNodes, parentRemovedNodes)); + // Once we put the merged changes into the cache we can remove other changes from the cache + for (NodeSPI aChildren : changes) + { + // Remove the node from the cache and do it asynchronously + cache.getInvocationContext().getOptionOverrides().setForceAsynchronous(true); + cache.removeNode(aChildren.getFqn()); + } } if (debugEnabled) log.debug("in-memory state passed to cache cacheLoader successfully"); Modified: jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/jbosscache/JBossCacheIndexChangesFilter.java =================================================================== --- jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/jbosscache/JBossCacheIndexChangesFilter.java 2010-12-30 18:07:06 UTC (rev 3760) +++ jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/jbosscache/JBossCacheIndexChangesFilter.java 2010-12-31 17:35:02 UTC (rev 3761) @@ -27,6 +27,7 @@ import org.exoplatform.services.jcr.impl.core.query.IndexingTree; import org.exoplatform.services.jcr.impl.core.query.QueryHandler; import org.exoplatform.services.jcr.impl.core.query.SearchManager; +import org.exoplatform.services.jcr.impl.core.query.lucene.ChangesHolder; import org.exoplatform.services.jcr.impl.util.io.PrivilegedCacheHelper; import org.exoplatform.services.jcr.jbosscache.ExoJBossCacheFactory; import org.exoplatform.services.jcr.jbosscache.ExoJBossCacheFactory.CacheType; @@ -189,66 +190,26 @@ protected void doUpdateIndex(Set<String> removedNodes, Set<String> addedNodes, Set<String> parentRemovedNodes, Set<String> parentAddedNodes) { - - - try + + ChangesHolder changes = searchManager.getChanges(removedNodes, addedNodes); + ChangesHolder parentChanges = parentSearchManager.getChanges(parentRemovedNodes, parentAddedNodes); + + if (changes == null && parentChanges == null) { - searchManager.updateIndex(removedNodes, addedNodes); + return; } - catch (RepositoryException e) - { - log.error("Error indexing changes " + e, e); - } - catch (IOException e) - { - log.error("Error indexing changes " + e, e); - try - { - handler.logErrorChanges(removedNodes, addedNodes); - } - catch (IOException ioe) - { - log.warn("Exception occure when errorLog writed. Error log is not complete. " + ioe, ioe); - } - } - + String id = IdGenerator.generate(); try { - parentSearchManager.updateIndex(parentRemovedNodes, parentAddedNodes); + PrivilegedCacheHelper.put(cache, Fqn.fromRelativeElements(rootFqn, id), LISTWRAPPER, + new ChangesFilterListsWrapper(changes, parentChanges)); } - catch (RepositoryException e) + catch (CacheException e) { - log.error("Error indexing changes " + e, e); + log.error(e.getLocalizedMessage(), e); + logErrorChanges(handler, removedNodes, addedNodes); + logErrorChanges(parentHandler, parentRemovedNodes, parentAddedNodes); } - catch (IOException e) - { - log.error("Error indexing changes " + e, e); - try - { - parentHandler.logErrorChanges(removedNodes, addedNodes); - } - catch (IOException ioe) - { - log.warn("Exception occure when errorLog writed. Error log is not complete. " + ioe, ioe); - } - } - - - - - -// String id = IdGenerator.generate(); -// try -// { -// PrivilegedCacheHelper.put(cache, Fqn.fromRelativeElements(rootFqn, id), LISTWRAPPER, -// new ChangesFilterListsWrapper(addedNodes, removedNodes, parentAddedNodes, parentRemovedNodes)); -// } -// catch (CacheException e) -// { -// log.error(e.getLocalizedMessage(), e); -// logErrorChanges(handler, removedNodes, addedNodes); -// logErrorChanges(parentHandler, parentRemovedNodes, parentAddedNodes); -// } } /** Modified: jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/ChangesHolder.java =================================================================== --- jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/ChangesHolder.java 2010-12-30 18:07:06 UTC (rev 3760) +++ jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/ChangesHolder.java 2010-12-31 17:35:02 UTC (rev 3761) @@ -109,6 +109,27 @@ } /** + * @return the collection of id of lucene document to add + */ + public Collection<String> getAddIds() + { + Collection<String> ids = new LinkedList<String>(); + for (Document doc : add) + { + ids.add(getDocId(doc)); + } + return ids; + } + + /** + * @return the id of the given lucene doc + */ + public String getDocId(Document doc) + { + return doc.get(FieldNames.UUID); + } + + /** * {@inheritDoc} */ public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException Modified: jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/SearchIndex.java =================================================================== --- jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/SearchIndex.java 2010-12-30 18:07:06 UTC (rev 3760) +++ jcr/branches/1.14-IMPR/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/SearchIndex.java 2010-12-31 17:35:02 UTC (rev 3761) @@ -62,9 +62,6 @@ import org.exoplatform.services.jcr.impl.util.SecurityHelper; import org.exoplatform.services.jcr.impl.util.io.PrivilegedFileHelper; import org.exoplatform.services.jcr.impl.util.io.PrivilegedSystemHelper; -import org.exoplatform.services.rpc.RPCException; -import org.exoplatform.services.rpc.RPCService; -import org.exoplatform.services.rpc.RemoteCommand; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.Element; @@ -74,7 +71,6 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; -import java.io.Serializable; import java.security.PrivilegedAction; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; @@ -456,27 +452,19 @@ private final ConfigurationManager cfm; - private final RPCService rpcService; - - private final String wsId; - - private RemoteCommand sendDocs; - /** * Working constructor. * * @throws RepositoryConfigurationException * @throws IOException */ - public SearchIndex(QueryHandlerEntry queryHandlerConfig, ConfigurationManager cfm, RPCService rpcService, String wsId) - throws IOException, RepositoryConfigurationException + public SearchIndex(QueryHandlerEntry queryHandlerConfig, ConfigurationManager cfm) throws IOException, + RepositoryConfigurationException { this.analyzer = new JcrStandartAnalyzer(); // this.queryHandlerConfig = new QueryHandlerEntryWrapper( // queryHandlerConfig); this.cfm = cfm; - this.rpcService = rpcService; - this.wsId = wsId; SearchIndexConfigurationHelper searchIndexConfigurationHelper = new SearchIndexConfigurationHelper(this); searchIndexConfigurationHelper.init(queryHandlerConfig); } @@ -489,8 +477,6 @@ this.analyzer = new JcrStandartAnalyzer(); // this.queryHandlerConfig = null; this.cfm = null; - this.rpcService = null; - this.wsId = null; } /** @@ -641,30 +627,6 @@ } modeHandler.addIndexerIoModeListener(this); - - // register the command - sendDocs = rpcService.registerCommand(new RemoteCommand() - { - public String getId() - { - return "sendDocs-" + wsId; - } - - public Serializable execute(Serializable[] args) throws Throwable - { - // additional check! if to JCR nodes are launched on same machine, - // RPCService will call this task on both instances. - //if (rpcService.isCoordinator()) - if (modeHandler.getMode() == IndexerIoMode.READ_WRITE) - { - ChangesHolder changes = (ChangesHolder)args[0]; - index.update(changes.getRemove(), changes.getAdd()); - } - return null; - } - }); - log.info("COMMAND '" + sendDocs.getId() + "' REGISTERED!"); - } /** @@ -733,11 +695,28 @@ IOException { checkOpen(); + apply(getChanges(remove, add)); + } + + /** + * {@inheritDoc} + */ + public void apply(ChangesHolder changes) throws RepositoryException, IOException + { + checkOpen(); + index.update(changes.getRemove(), changes.getAdd()); + } + + /** + * {@inheritDoc} + */ + public ChangesHolder getChanges(Iterator<String> remove, Iterator<NodeData> add) + { final Map<String, NodeData> aggregateRoots = new HashMap<String, NodeData>(); final Set<String> removedNodeIds = new HashSet<String>(); final Set<String> addedNodeIds = new HashSet<String>(); - updateInternal(IteratorUtils.toList(new TransformIterator(remove, new Transformer() + Collection<String> docIdsToRemove = IteratorUtils.toList(new TransformIterator(remove, new Transformer() { public Object transform(Object input) { @@ -745,7 +724,8 @@ removedNodeIds.add(uuid); return uuid; } - })), IteratorUtils.toList(new TransformIterator(add, new Transformer() + })); + Collection<Document> docsToAdd = IteratorUtils.toList(new TransformIterator(add, new Transformer() { public Object transform(Object input) { @@ -770,7 +750,7 @@ } return doc; } - }))); + })); // remove any aggregateRoot nodes that are new // and therefore already up-to-date @@ -800,35 +780,14 @@ } }); modified.addAll(aggregateRoots.values()); - updateInternal(aggregateRoots.keySet(), modified); + docIdsToRemove.addAll(aggregateRoots.keySet()); + docsToAdd.addAll(modified); } - } - - private void updateInternal(final Collection<String> remove, final Collection<Document> add) throws IOException - { - // if RPC not configured or current node is in READ_WRITE mode - if (rpcService == null || modeHandler.getMode() == IndexerIoMode.READ_WRITE) + if (docIdsToRemove.isEmpty() && docsToAdd.isEmpty()) { - index.update(remove, add); + return null; } - else - { - try - { - rpcService.executeCommandOnCoordinator(sendDocs, true, new ChangesHolder(remove, add)); - } - catch (SecurityException e) - { - e.printStackTrace(); - throw new IOException(e.getMessage()); - } - catch (RPCException e) - { - e.printStackTrace(); - throw new IOException(e.getMessage()); - } - } - + return new ChangesHolder(docIdsToRemove, docsToAdd); } /** Modified: jcr/branches/1.14-IMPR/exo.jcr.component.core/src/test/java/org/exoplatform/services/jcr/api/core/query/lucene/SlowQueryHandler.java =================================================================== --- jcr/branches/1.14-IMPR/exo.jcr.component.core/src/test/java/org/exoplatform/services/jcr/api/core/query/lucene/SlowQueryHandler.java 2010-12-30 18:07:06 UTC (rev 3760) +++ jcr/branches/1.14-IMPR/exo.jcr.component.core/src/test/java/org/exoplatform/services/jcr/api/core/query/lucene/SlowQueryHandler.java 2010-12-31 17:35:02 UTC (rev 3761) @@ -23,9 +23,11 @@ import org.exoplatform.services.jcr.impl.core.query.AbstractQueryHandler; import org.exoplatform.services.jcr.impl.core.query.ExecutableQuery; import org.exoplatform.services.jcr.impl.core.query.QueryHandlerContext; +import org.exoplatform.services.jcr.impl.core.query.lucene.ChangesHolder; import org.exoplatform.services.jcr.impl.core.query.lucene.QueryHits; import java.io.IOException; +import java.util.Iterator; import java.util.Set; import javax.jcr.RepositoryException; @@ -92,4 +94,16 @@ // TODO Auto-generated method stub return null; } + + public void apply(ChangesHolder changes) throws RepositoryException, IOException + { + // TODO Auto-generated method stub + + } + + public ChangesHolder getChanges(Iterator<String> remove, Iterator<NodeData> add) + { + // TODO Auto-generated method stub + return null; + } } Modified: jcr/branches/1.14-IMPR/exo.jcr.component.core/src/test/resources/conf/standalone/cluster/test-configuration.xml =================================================================== --- jcr/branches/1.14-IMPR/exo.jcr.component.core/src/test/resources/conf/standalone/cluster/test-configuration.xml 2010-12-30 18:07:06 UTC (rev 3760) +++ jcr/branches/1.14-IMPR/exo.jcr.component.core/src/test/resources/conf/standalone/cluster/test-configuration.xml 2010-12-31 17:35:02 UTC (rev 3761) @@ -108,7 +108,7 @@ <init-params> <value-param> <name>jgroups-configuration</name> - <value>classpath:/udp.xml</value> + <value>udp-mux.xml</value> </value-param> <value-param> <name>jgroups-cluster-name</name> Modified: jcr/branches/1.14-IMPR/exo.jcr.component.core/src/test/resources/conf/standalone/test-configuration.xml =================================================================== --- jcr/branches/1.14-IMPR/exo.jcr.component.core/src/test/resources/conf/standalone/test-configuration.xml 2010-12-30 18:07:06 UTC (rev 3760) +++ jcr/branches/1.14-IMPR/exo.jcr.component.core/src/test/resources/conf/standalone/test-configuration.xml 2010-12-31 17:35:02 UTC (rev 3761) @@ -102,34 +102,6 @@ </init-params> </component> - - <component> - <key>org.exoplatform.services.rpc.RPCService</key> - <type>org.exoplatform.services.rpc.impl.RPCServiceImpl</type> - <init-params> - <value-param> - <name>jgroups-configuration</name> - <value>classpath:/udp.xml</value> - </value-param> - <value-param> - <name>jgroups-cluster-name</name> - <value>RPCService-Cluster</value> - </value-param> - <value-param> - <name>jgroups-default-timeout</name> - <value>0</value> - </value-param> - <value-param> - <name>allow-failover</name> - <value>true</value> - </value-param> - <value-param> - <name>retry-timeout</name> - <value>20000</value> - </value-param> - </init-params> - </component> - <component> <key>org.exoplatform.services.jcr.RepositoryService</key> <type>org.exoplatform.services.jcr.impl.RepositoryServiceImpl</type>