Author: tolusha Date: 2011-02-03 15:40:43 -0500 (Thu, 03 Feb 2011) New Revision: 3916 Modified: jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/Indexable.java jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/MultiIndex.java jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/JDBCStorageConnection.java jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/JDBCWorkspaceDataContainer.java jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/db/HSQLDBMultiDbJDBCConnection.java jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/db/HSQLDBSingleDbJDBCConnection.java jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/db/MultiDbJDBCConnection.java jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/db/SingleDbJDBCConnection.java jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/indexing/JdbcNodeDataIndexingIterator.java jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/optimisation/db/HSQLDBMultiDbJDBCConnection.java jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/optimisation/db/HSQLDBSingleDbJDBCConnection.java jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/optimisation/db/MultiDbJDBCConnection.java jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/optimisation/db/SingleDbJDBCConnection.java Log: EXOJCR-1104: using paging Modified: jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/Indexable.java =================================================================== --- jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/Indexable.java 2011-02-03 07:32:17 UTC (rev 3915) +++ jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/Indexable.java 2011-02-03 20:40:43 UTC (rev 3916) @@ -30,5 +30,14 @@ */ public interface Indexable { - NodeDataIndexingIterator getNodeDataIndexingIterator(int pageSize) throws RepositoryException; + /** + * Returns NodeDataIndexingIterator. + * + * @param pageSize + * the maximum amount of the rows which can be retrieved from database per once + * @return NodeDataIndexingIterator + * @throws RepositoryException + */ + NodeDataIndexingIterator getNodeDataIndexingIterator(int pageSize) + throws RepositoryException; } Modified: jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/MultiIndex.java =================================================================== --- jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/MultiIndex.java 2011-02-03 07:32:17 UTC (rev 3915) +++ jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/core/query/lucene/MultiIndex.java 2011-02-03 20:40:43 UTC (rev 3916) @@ -1479,7 +1479,8 @@ private long createIndex(Indexable indexableComponent, NodeData rootNode, ItemDataConsumer stateMgr, long count) throws IOException, RepositoryException { - NodeDataIndexingIterator iterator = indexableComponent.getNodeDataIndexingIterator(1000); + NodeDataIndexingIterator iterator = indexableComponent.getNodeDataIndexingIterator(100000); + while (iterator.hasNext()) { for (NodeDataIndexing node : iterator.next()) Modified: jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/JDBCStorageConnection.java =================================================================== --- jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/JDBCStorageConnection.java 2011-02-03 07:32:17 UTC (rev 3915) +++ jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/JDBCStorageConnection.java 2011-02-03 20:40:43 UTC (rev 3916) @@ -60,7 +60,6 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; -import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.List; @@ -1012,20 +1011,29 @@ } /** - * {@inheritDoc} + * GetNodesAndProperties. + * + * @param offset + * @param limit + * @param result + * @return + * @throws RepositoryException + * @throws IllegalStateException */ - public List<NodeDataIndexing> getNodesAndProperties(int offset, int limit) throws RepositoryException, + public int getNodesAndProperties(int offset, int limit, List<NodeDataIndexing> result) throws RepositoryException, IllegalStateException { - List<NodeDataIndexing> result = new ArrayList<NodeDataIndexing>(); + int read = 0; checkIfOpened(); try { ResultSet resultSet = findNodesAndProperties(offset, limit); + try { TempNodeData tempNodeData = null; + while (resultSet.next()) { if (tempNodeData == null) @@ -1034,8 +1042,7 @@ } else if (!resultSet.getString(COLUMN_ID).equals(tempNodeData.cid)) { - result.add(createNodeData(tempNodeData)); - + result.add(createNodeDataIndexing(tempNodeData)); tempNodeData = new TempNodeData(resultSet); } @@ -1049,11 +1056,13 @@ } values.add(new ExtendedTempPropertyData(resultSet)); + + ++read; } if (tempNodeData != null) { - result.add(createNodeData(tempNodeData)); + result.add(createNodeDataIndexing(tempNodeData)); } } finally @@ -1081,7 +1090,7 @@ throw new RepositoryException(e); } - return result; + return read; } /** @@ -2540,84 +2549,92 @@ * @throws SQLException * @throws IllegalNameException */ - protected NodeDataIndexing createNodeData(TempNodeData tempNode) throws RepositoryException, SQLException, + protected NodeDataIndexing createNodeDataIndexing(TempNodeData tempNode) throws RepositoryException, SQLException, IOException, IllegalNameException { + String parentCid; QPath parentPath; - String parentCid; if (tempNode.cpid.equals(Constants.ROOT_PARENT_UUID)) { // root node + parentCid = null; parentPath = Constants.ROOT_PATH; - parentCid = null; } else { + parentCid = tempNode.cpid; parentPath = QPath.makeChildPath(traverseQPath(tempNode.cpid), InternalQName.parse(tempNode.cname), tempNode.cindex); - parentCid = tempNode.cpid; } - // primary type - SortedSet<TempPropertyData> primaryTypeTempProp = - tempNode.properties.get(Constants.JCR_PRIMARYTYPE.getAsString()); - if (primaryTypeTempProp == null) + // primary type if exists in the list of properties + InternalQName ptName = null; + ValueData ptValue = null; + + SortedSet<TempPropertyData> ptTempProp = tempNode.properties.get(Constants.JCR_PRIMARYTYPE.getAsString()); + if (ptTempProp != null) { - throw new PrimaryTypeNotFoundException("FATAL ERROR primary type record not found. Node " - + parentPath.getAsString() + ", id " + tempNode.cid + ", container " + this.containerName, null); + byte[] data = ptTempProp.first().getAsByteArray(); + + ptValue = new ByteArrayPersistedValueData(ptTempProp.first().orderNum, data); + ptName = InternalQName.parse(new String((data != null ? data : new byte[]{}), Constants.DEFAULT_ENCODING)); } - byte[] data = primaryTypeTempProp.first().getAsByteArray(); - primaryTypeTempProp.first().data = new ByteArrayInputStream(data); + // mixins if exist in the list of properties + List<ValueData> mixinsData = new ArrayList<ValueData>(); + List<InternalQName> mixins = new ArrayList<InternalQName>(); - InternalQName ptName = - InternalQName.parse(new String((data != null ? data : new byte[]{}), Constants.DEFAULT_ENCODING)); - - // mixins - List<InternalQName> mixins = new ArrayList<InternalQName>(); Set<TempPropertyData> mixinsTempProps = tempNode.properties.get(Constants.JCR_MIXINTYPES.getAsString()); if (mixinsTempProps != null) { - for (TempPropertyData mxnb : mixinsTempProps) { - data = mxnb.getAsByteArray(); - mxnb.data = new ByteArrayInputStream(data); + byte[] data = mxnb.getAsByteArray(); + mixinsData.add(new ByteArrayPersistedValueData(mxnb.orderNum, data)); mixins.add(InternalQName.parse(new String(data, Constants.DEFAULT_ENCODING))); } } // build node data NodeData nodeData = - new PersistedNodeData(getIdentifier(tempNode.cid), parentPath, getIdentifier(parentCid), tempNode.cversion, - tempNode.cnordernumb, ptName, mixins.toArray(new InternalQName[mixins.size()]), null); + new PersistedNodeData(getIdentifier(tempNode.cid), parentPath, getIdentifier(parentCid), + tempNode.cversion, tempNode.cnordernumb, ptName, mixins.toArray(new InternalQName[mixins.size()]), null); List<PropertyData> childProps = new ArrayList<PropertyData>(); - for (String propName : tempNode.properties.keySet()) { ExtendedTempPropertyData prop = (ExtendedTempPropertyData)tempNode.properties.get(propName).first(); + String identifier = getIdentifier(prop.id); + QPath qpath = QPath.makeChildPath(parentPath, InternalQName.parse(prop.name)); - // read values List<ValueData> valueData = new ArrayList<ValueData>(); - for (TempPropertyData tempProp : tempNode.properties.get(propName)) + + if (propName.equals(Constants.JCR_PRIMARYTYPE.getAsString())) { - ExtendedTempPropertyData extTempProp = (ExtendedTempPropertyData)tempProp; + valueData.add(ptValue); + } + else if (propName.equals(Constants.JCR_MIXINTYPES.getAsString())) + { + valueData = mixinsData; + } + else + { + for (TempPropertyData tempProp : tempNode.properties.get(propName)) + { + ExtendedTempPropertyData extTempProp = (ExtendedTempPropertyData)tempProp; - ValueData vdata = - extTempProp.storage_desc == null ? readValueData(extTempProp.id, extTempProp.orderNum, - extTempProp.version, extTempProp.data) : readValueData(identifier, extTempProp.orderNum, - extTempProp.storage_desc); + ValueData vdata = + extTempProp.storage_desc == null ? readValueData(extTempProp.id, extTempProp.orderNum, + extTempProp.version, extTempProp.data) : readValueData(identifier, extTempProp.orderNum, + extTempProp.storage_desc); - valueData.add(vdata); + valueData.add(vdata); + } } - Collections.sort(valueData, COMPARATOR_VALUE_DATA); - QPath qpath = QPath.makeChildPath(parentPath, InternalQName.parse(prop.name)); - // build property data PropertyData pdata = new PersistedPropertyData(identifier, qpath, tempNode.cid, prop.version, prop.type, prop.multi, Modified: jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/JDBCWorkspaceDataContainer.java =================================================================== --- jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/JDBCWorkspaceDataContainer.java 2011-02-03 07:32:17 UTC (rev 3915) +++ jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/JDBCWorkspaceDataContainer.java 2011-02-03 20:40:43 UTC (rev 3916) @@ -1425,37 +1425,9 @@ /** * {@inheritDoc} */ - public NodeDataIndexingIterator getNodeDataIndexingIterator(int pageSize) throws RepositoryException + public NodeDataIndexingIterator getNodeDataIndexingIterator(int pageSize) + throws RepositoryException { - try - { - final DataSource ds = (DataSource)new InitialContext().lookup(dbSourceName); - - if (ds != null) - { - Connection jdbcConn = - SecurityHelper.doPrivilegedSQLExceptionAction(new PrivilegedExceptionAction<Connection>() - { - public Connection run() throws Exception - { - return ds.getConnection(); - } - }); - - return new JdbcNodeDataIndexingIterator(connFactory, pageSize); - } - else - { - throw new NameNotFoundException("Data source " + dbSourceName + " not found"); - } - } - catch (SQLException e) - { - throw new RepositoryException(e); - } - catch (NamingException e) - { - throw new RepositoryException(e); - } + return new JdbcNodeDataIndexingIterator(connFactory, pageSize); } } Modified: jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/db/HSQLDBMultiDbJDBCConnection.java =================================================================== --- jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/db/HSQLDBMultiDbJDBCConnection.java 2011-02-03 07:32:17 UTC (rev 3915) +++ jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/db/HSQLDBMultiDbJDBCConnection.java 2011-02-03 20:40:43 UTC (rev 3916) @@ -25,7 +25,6 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; -import java.security.PrivilegedAction; import java.security.PrivilegedExceptionAction; import java.sql.Connection; import java.sql.ResultSet; @@ -42,7 +41,51 @@ public class HSQLDBMultiDbJDBCConnection extends MultiDbJDBCConnection { + /** + * HSQLDB Multidatabase JDBC Connection constructor. + * + * @param dbConnection + * JDBC connection, shoudl be opened before + * @param readOnly + * boolean if true the dbConnection was marked as READ-ONLY. + * @param containerName + * Workspace Storage Container name (see configuration) + * @param valueStorageProvider + * External Value Storages provider + * @param maxBufferSize + * Maximum buffer size (see configuration) + * @param swapDirectory + * Swap directory File (see configuration) + * @param swapCleaner + * Swap cleaner (internal FileCleaner). + * @throws SQLException + * + * @see org.exoplatform.services.jcr.impl.util.io.FileCleaner + */ + public HSQLDBMultiDbJDBCConnection(Connection dbConnection, boolean readOnly, String containerName, + ValueStoragePluginProvider valueStorageProvider, int maxBufferSize, File swapDirectory, FileCleaner swapCleaner) + throws SQLException + { + super(dbConnection, readOnly, containerName, valueStorageProvider, maxBufferSize, swapDirectory, swapCleaner); + } + + /** + * {@inheritDoc} + */ @Override + protected void prepareQueries() throws SQLException + { + + super.prepareQueries(); + FIND_PROPERTY_BY_NAME = + "select V.DATA" + " from JCR_MITEM I, JCR_MVALUE V" + + " where I.PARENT_ID=? and I.I_CLASS=2 and I.NAME=? and I.ID=V.PROPERTY_ID order by V.ORDER_NUM"; + FIND_NODES_BY_PARENTID = "select * from JCR_MITEM" + " where PARENT_ID=? and I_CLASS=1" + " order by N_ORDER_NUM"; + FIND_NODES_COUNT_BY_PARENTID = "select count(ID) from JCR_MITEM" + " where PARENT_ID=? and I_CLASS=1"; + FIND_PROPERTIES_BY_PARENTID = "select * from JCR_MITEM" + " where PARENT_ID=? and I_CLASS=2" + " order by ID"; + } + + @Override protected int addNodeRecord(final NodeData data) throws SQLException { return SecurityHelper.doPrivilegedSQLExceptionAction(new PrivilegedExceptionAction<Integer>() @@ -290,46 +333,18 @@ } /** - * HSQLDB Multidatabase JDBC Connection constructor. - * - * @param dbConnection - * JDBC connection, shoudl be opened before - * @param readOnly - * boolean if true the dbConnection was marked as READ-ONLY. - * @param containerName - * Workspace Storage Container name (see configuration) - * @param valueStorageProvider - * External Value Storages provider - * @param maxBufferSize - * Maximum buffer size (see configuration) - * @param swapDirectory - * Swap directory File (see configuration) - * @param swapCleaner - * Swap cleaner (internal FileCleaner). - * @throws SQLException - * - * @see org.exoplatform.services.jcr.impl.util.io.FileCleaner - */ - public HSQLDBMultiDbJDBCConnection(Connection dbConnection, boolean readOnly, String containerName, - ValueStoragePluginProvider valueStorageProvider, int maxBufferSize, File swapDirectory, FileCleaner swapCleaner) - throws SQLException - { - super(dbConnection, readOnly, containerName, valueStorageProvider, maxBufferSize, swapDirectory, swapCleaner); - } - - /** * {@inheritDoc} */ @Override - protected void prepareQueries() throws SQLException + protected ResultSet findNodesAndProperties(final int offset, final int limit) throws SQLException { - - super.prepareQueries(); - FIND_PROPERTY_BY_NAME = - "select V.DATA" + " from JCR_MITEM I, JCR_MVALUE V" - + " where I.PARENT_ID=? and I.I_CLASS=2 and I.NAME=? and I.ID=V.PROPERTY_ID order by V.ORDER_NUM"; - FIND_NODES_BY_PARENTID = "select * from JCR_MITEM" + " where PARENT_ID=? and I_CLASS=1" + " order by N_ORDER_NUM"; - FIND_NODES_COUNT_BY_PARENTID = "select count(ID) from JCR_MITEM" + " where PARENT_ID=? and I_CLASS=1"; - FIND_PROPERTIES_BY_PARENTID = "select * from JCR_MITEM" + " where PARENT_ID=? and I_CLASS=2" + " order by ID"; + PrivilegedExceptionAction<ResultSet> action = new PrivilegedExceptionAction<ResultSet>() + { + public ResultSet run() throws Exception + { + return HSQLDBMultiDbJDBCConnection.super.findNodesAndProperties(offset, limit); + } + }; + return SecurityHelper.doPrivilegedSQLExceptionAction(action); } } Modified: jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/db/HSQLDBSingleDbJDBCConnection.java =================================================================== --- jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/db/HSQLDBSingleDbJDBCConnection.java 2011-02-03 07:32:17 UTC (rev 3915) +++ jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/db/HSQLDBSingleDbJDBCConnection.java 2011-02-03 20:40:43 UTC (rev 3916) @@ -459,4 +459,20 @@ }; return SecurityHelper.doPrivilegedSQLExceptionAction(action); } + + /** + * {@inheritDoc} + */ + @Override + protected ResultSet findNodesAndProperties(final int offset, final int limit) throws SQLException + { + PrivilegedExceptionAction<ResultSet> action = new PrivilegedExceptionAction<ResultSet>() + { + public ResultSet run() throws Exception + { + return HSQLDBSingleDbJDBCConnection.super.findNodesAndProperties(offset, limit); + } + }; + return SecurityHelper.doPrivilegedSQLExceptionAction(action); + } } Modified: jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/db/MultiDbJDBCConnection.java =================================================================== --- jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/db/MultiDbJDBCConnection.java 2011-02-03 07:32:17 UTC (rev 3915) +++ jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/db/MultiDbJDBCConnection.java 2011-02-03 20:40:43 UTC (rev 3916) @@ -157,8 +157,8 @@ "select I.ID, I.PARENT_ID, I.NAME, I.VERSION, I.I_INDEX, I.N_ORDER_NUM," + " P.ID AS P_ID, P.NAME AS P_NAME, P.VERSION AS P_VERSION, P.P_TYPE, P.P_MULTIVALUED," + " V.DATA, V.ORDER_NUM, V.STORAGE_DESC" - + " from JCR_MITEM I, JCR_MITEM P, JCR_MVALUE V where I.I_CLASS=1 and" - + " P.I_CLASS=2 and V.PROPERTY_ID=P.ID order by ID"; + + " from JCR_MITEM I, JCR_MITEM P, JCR_MVALUE V" + + " where I.I_CLASS=1 and P.I_CLASS=2 and P.PARENT_ID=I.ID and V.PROPERTY_ID=P.ID order by ID LIMIT ? OFFSET ?"; } /** @@ -537,6 +537,9 @@ findNodesAndProperties.clearParameters(); } + findNodesAndProperties.setInt(1, limit); + findNodesAndProperties.setInt(2, offset); + return findNodesAndProperties.executeQuery(); } } Modified: jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/db/SingleDbJDBCConnection.java =================================================================== --- jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/db/SingleDbJDBCConnection.java 2011-02-03 07:32:17 UTC (rev 3915) +++ jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/db/SingleDbJDBCConnection.java 2011-02-03 20:40:43 UTC (rev 3916) @@ -165,8 +165,9 @@ "select I.ID, I.PARENT_ID, I.NAME, I.VERSION, I.I_INDEX, I.N_ORDER_NUM," + " P.ID AS P_ID, P.NAME AS P_NAME, P.VERSION AS P_VERSION, P.P_TYPE, P.P_MULTIVALUED," + " V.DATA, V.ORDER_NUM, V.STORAGE_DESC" - + " from JCR_SITEM I, JCR_SITEM P, JCR_SVALUE V where I.I_CLASS=1 and I.CONTAINER_NAME=? and" - + " P.I_CLASS=2 and P.CONTAINER_NAME=? and P.PARENT_ID=I.ID" + " and V.PROPERTY_ID=P.ID order by ID"; + + " from JCR_SITEM I, JCR_SITEM P, JCR_SVALUE V" + + " where I.I_CLASS=1 and I.CONTAINER_NAME=? and P.I_CLASS=2 and P.CONTAINER_NAME=? and P.PARENT_ID=I.ID" + + " and V.PROPERTY_ID=P.ID order by ID LIMIT ? OFFSET ?"; } /** @@ -560,6 +561,8 @@ findNodesAndProperties.setString(1, containerName); findNodesAndProperties.setString(2, containerName); + findNodesAndProperties.setInt(3, limit); + findNodesAndProperties.setInt(4, offset); return findNodesAndProperties.executeQuery(); } Modified: jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/indexing/JdbcNodeDataIndexingIterator.java =================================================================== --- jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/indexing/JdbcNodeDataIndexingIterator.java 2011-02-03 07:32:17 UTC (rev 3915) +++ jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/indexing/JdbcNodeDataIndexingIterator.java 2011-02-03 20:40:43 UTC (rev 3916) @@ -18,7 +18,13 @@ */ package org.exoplatform.services.jcr.impl.storage.jdbc.indexing; +import org.exoplatform.services.jcr.dataflow.persistent.PersistedNodeData; +import org.exoplatform.services.jcr.dataflow.persistent.PersistedPropertyData; +import org.exoplatform.services.jcr.datamodel.InternalQName; +import org.exoplatform.services.jcr.datamodel.NodeData; import org.exoplatform.services.jcr.datamodel.NodeDataIndexing; +import org.exoplatform.services.jcr.datamodel.PropertyData; +import org.exoplatform.services.jcr.datamodel.ValueData; import org.exoplatform.services.jcr.impl.core.query.NodeDataIndexingIterator; import org.exoplatform.services.jcr.impl.storage.jdbc.JDBCStorageConnection; import org.exoplatform.services.jcr.impl.storage.jdbc.db.GenericConnectionFactory; @@ -26,7 +32,9 @@ import org.exoplatform.services.log.Log; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import javax.jcr.RepositoryException; @@ -69,6 +77,18 @@ private List<NodeDataIndexing> current; /** + * Node data which may not contains all its properties. It is the last + * data in the resulted list in readNext() method. Should be merged with + * first one during next invoking readNext() method. + */ + private NodeDataIndexing unCompletedNode; + + /** + * Indicates if all rows has been read and no need more access to database. + */ + private boolean isReadAll = false; + + /** * Constructor JdbcIndexingDataIterator. * */ @@ -93,29 +113,143 @@ public List<NodeDataIndexing> next() throws RepositoryException { List<NodeDataIndexing> next = this.current; - this.current = new ArrayList<NodeDataIndexing>();//readNext(); + this.current = readNext(); return next; } /** - * Read next nodes from database. + * Read nodes from database. * * @return List<NodeDataIndexing> * @throws RepositoryException */ private List<NodeDataIndexing> readNext() throws RepositoryException { + if (isReadAll) + { + return new ArrayList<NodeDataIndexing>(); + } + + List<NodeDataIndexing> result = getNodesAndProperties(); + + if (result.isEmpty()) + { + if (unCompletedNode != null) + { + result.add(unCompletedNode); + unCompletedNode = null; + } + } + else + { + if (unCompletedNode != null) + { + NodeDataIndexing node = result.get(0); + if (unCompletedNode.getIdentifier().equals(node.getIdentifier())) + { + result.set(0, mergeWithUnCompletedNode(node)); + unCompletedNode = null; + } + else + { + result.add(0, unCompletedNode); + } + } + + if (!isReadAll) + { + unCompletedNode = result.remove(result.size() - 1); + } + + if (result.isEmpty()) + { + return readNext(); + } + } + + return result; + } + + /** + * Read next nodes from database. + * + * @return List + * @throws RepositoryException + */ + private List<NodeDataIndexing> getNodesAndProperties() throws RepositoryException + { JDBCStorageConnection conn = (JDBCStorageConnection)connFactory.openConnection(); try { - return conn.getNodesAndProperties(offset, pageSize); - // TODO offset + ? + List<NodeDataIndexing> result = new ArrayList<NodeDataIndexing>(); + + int read = conn.getNodesAndProperties(offset, pageSize, result); + offset += pageSize; + + isReadAll = read != pageSize; + + return result; } finally { conn.close(); } } + + /** + * Merge two nodes. + */ + private NodeDataIndexing mergeWithUnCompletedNode(NodeDataIndexing node) + { + InternalQName primaryType = unCompletedNode.getPrimaryTypeName(); + if (primaryType == null) + { + primaryType = node.getPrimaryTypeName(); + } + + List<InternalQName> mixins = new ArrayList<InternalQName>(); + for (InternalQName mixin : unCompletedNode.getMixinTypeNames()) + { + mixins.add(mixin); + } + + for (InternalQName mixin : node.getMixinTypeNames()) + { + mixins.add(mixin); + } + + NodeData nodeData = + new PersistedNodeData(unCompletedNode.getIdentifier(), unCompletedNode.getQPath(), + unCompletedNode.getParentIdentifier(), unCompletedNode.getPersistedVersion(), + unCompletedNode.getOrderNumber(), primaryType, mixins.toArray(new InternalQName[mixins.size()]), null); + + Map<String, PropertyData> props = new HashMap<String, PropertyData>(); + for (PropertyData prop : unCompletedNode.getChildPropertiesData()) + { + props.put(prop.getIdentifier(), prop); + } + + for (PropertyData prop : node.getChildPropertiesData()) + { + if (props.containsKey(prop.getIdentifier())) + { + List<ValueData> values = new ArrayList<ValueData>(props.get(prop.getIdentifier()).getValues()); + values.addAll(prop.getValues()); + + PropertyData propertyData = + new PersistedPropertyData(prop.getIdentifier(), prop.getQPath(), prop.getParentIdentifier(), + prop.getPersistedVersion(), prop.getType(), prop.isMultiValued(), values); + + props.put(prop.getIdentifier(), propertyData); + } + else + { + props.put(prop.getIdentifier(), prop); + } + } + + return new NodeDataIndexing(nodeData, new ArrayList<PropertyData>(props.values())); + } } Modified: jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/optimisation/db/HSQLDBMultiDbJDBCConnection.java =================================================================== --- jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/optimisation/db/HSQLDBMultiDbJDBCConnection.java 2011-02-03 07:32:17 UTC (rev 3915) +++ jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/optimisation/db/HSQLDBMultiDbJDBCConnection.java 2011-02-03 20:40:43 UTC (rev 3916) @@ -44,8 +44,68 @@ */ public class HSQLDBMultiDbJDBCConnection extends MultiDbJDBCConnection { + /** + * HSQLDB Multidatabase JDBC Connection constructor. + * + * @param dbConnection + * JDBC connection, shoudl be opened before + * @param readOnly + * boolean if true the dbConnection was marked as READ-ONLY. + * @param containerName + * Workspace Storage Container name (see configuration) + * @param valueStorageProvider + * External Value Storages provider + * @param maxBufferSize + * Maximum buffer size (see configuration) + * @param swapDirectory + * Swap directory File (see configuration) + * @param swapCleaner + * Swap cleaner (internal FileCleaner). + * @throws SQLException + * + * @see org.exoplatform.services.jcr.impl.util.io.FileCleaner + */ + public HSQLDBMultiDbJDBCConnection(Connection dbConnection, boolean readOnly, String containerName, + ValueStoragePluginProvider valueStorageProvider, int maxBufferSize, File swapDirectory, FileCleaner swapCleaner) + throws SQLException + { + super(dbConnection, readOnly, containerName, valueStorageProvider, maxBufferSize, swapDirectory, swapCleaner); + } + /** + * {@inheritDoc} + */ @Override + protected void prepareQueries() throws SQLException + { + super.prepareQueries(); + FIND_PROPERTY_BY_NAME = + "select V.DATA" + " from JCR_MITEM I, JCR_MVALUE V" + + " where I.PARENT_ID=? and I.I_CLASS=2 and I.NAME=? and I.ID=V.PROPERTY_ID order by V.ORDER_NUM"; + FIND_NODES_BY_PARENTID = "select * from JCR_MITEM" + " where PARENT_ID=? and I_CLASS=1" + " order by N_ORDER_NUM"; + FIND_NODES_COUNT_BY_PARENTID = "select count(ID) from JCR_MITEM" + " where PARENT_ID=? and I_CLASS=1"; + FIND_PROPERTIES_BY_PARENTID = "select * from JCR_MITEM" + " where PARENT_ID=? and I_CLASS=2" + " order by ID"; + FIND_NODES_BY_PARENTID_CQ = + "select I.*, P.NAME AS PROP_NAME, V.ORDER_NUM, V.DATA" + + " from JCR_MITEM I, JCR_MITEM P, JCR_MVALUE V" + + " where I.PARENT_ID=? and I.I_CLASS=1 and (P.PARENT_ID=I.ID and P.I_CLASS=2 and (P.NAME='[http://www.jcp.org/jcr/1.0]primaryType' or P.NAME='[http://www.jcp.org/jcr/1.0]mixinTypes' or P.NAME='[http://www.exoplatform.com/jcr/exo/1.0]owner' or P.NAME='[http://www.exoplatform.com/jcr/exo/1.0]permissions') and V.PROPERTY_ID=P.ID)" + + " order by I.N_ORDER_NUM, I.ID"; + FIND_PROPERTIES_BY_PARENTID_CQ = + "select I.ID, I.PARENT_ID, I.NAME, I.VERSION, I.I_CLASS, I.I_INDEX, I.N_ORDER_NUM, I.P_TYPE, I.P_MULTIVALUED," + + " V.ORDER_NUM, V.DATA, V.STORAGE_DESC from JCR_MITEM I LEFT OUTER JOIN JCR_MVALUE V ON (V.PROPERTY_ID=I.ID)" + + " where I.PARENT_ID=? and I.I_CLASS=2 order by I.NAME"; + } + + /** + * Use simple queries since it is much faster + */ + @Override + protected QPath traverseQPath(String cpid) throws SQLException, InvalidItemStateException, IllegalNameException + { + return traverseQPathSQ(cpid); + } + + @Override protected int addNodeRecord(final NodeData data) throws SQLException { return SecurityHelper.doPrivilegedSQLExceptionAction(new PrivilegedExceptionAction<Integer>() @@ -337,64 +397,18 @@ } /** - * HSQLDB Multidatabase JDBC Connection constructor. - * - * @param dbConnection - * JDBC connection, shoudl be opened before - * @param readOnly - * boolean if true the dbConnection was marked as READ-ONLY. - * @param containerName - * Workspace Storage Container name (see configuration) - * @param valueStorageProvider - * External Value Storages provider - * @param maxBufferSize - * Maximum buffer size (see configuration) - * @param swapDirectory - * Swap directory File (see configuration) - * @param swapCleaner - * Swap cleaner (internal FileCleaner). - * @throws SQLException - * - * @see org.exoplatform.services.jcr.impl.util.io.FileCleaner - */ - public HSQLDBMultiDbJDBCConnection(Connection dbConnection, boolean readOnly, String containerName, - ValueStoragePluginProvider valueStorageProvider, int maxBufferSize, File swapDirectory, FileCleaner swapCleaner) - throws SQLException - { - super(dbConnection, readOnly, containerName, valueStorageProvider, maxBufferSize, swapDirectory, swapCleaner); - } - - /** * {@inheritDoc} */ @Override - protected void prepareQueries() throws SQLException + protected ResultSet findNodesAndProperties(final int offset, final int limit) throws SQLException { - - super.prepareQueries(); - FIND_PROPERTY_BY_NAME = - "select V.DATA" + " from JCR_MITEM I, JCR_MVALUE V" - + " where I.PARENT_ID=? and I.I_CLASS=2 and I.NAME=? and I.ID=V.PROPERTY_ID order by V.ORDER_NUM"; - FIND_NODES_BY_PARENTID = "select * from JCR_MITEM" + " where PARENT_ID=? and I_CLASS=1" + " order by N_ORDER_NUM"; - FIND_NODES_COUNT_BY_PARENTID = "select count(ID) from JCR_MITEM" + " where PARENT_ID=? and I_CLASS=1"; - FIND_PROPERTIES_BY_PARENTID = "select * from JCR_MITEM" + " where PARENT_ID=? and I_CLASS=2" + " order by ID"; - FIND_NODES_BY_PARENTID_CQ = - "select I.*, P.NAME AS PROP_NAME, V.ORDER_NUM, V.DATA" - + " from JCR_MITEM I, JCR_MITEM P, JCR_MVALUE V" - + " where I.PARENT_ID=? and I.I_CLASS=1 and (P.PARENT_ID=I.ID and P.I_CLASS=2 and (P.NAME='[http://www.jcp.org/jcr/1.0]primaryType' or P.NAME='[http://www.jcp.org/jcr/1.0]mixinTypes' or P.NAME='[http://www.exoplatform.com/jcr/exo/1.0]owner' or P.NAME='[http://www.exoplatform.com/jcr/exo/1.0]permissions') and V.PROPERTY_ID=P.ID)" - + " order by I.N_ORDER_NUM, I.ID"; - FIND_PROPERTIES_BY_PARENTID_CQ = - "select I.ID, I.PARENT_ID, I.NAME, I.VERSION, I.I_CLASS, I.I_INDEX, I.N_ORDER_NUM, I.P_TYPE, I.P_MULTIVALUED," - + " V.ORDER_NUM, V.DATA, V.STORAGE_DESC from JCR_MITEM I LEFT OUTER JOIN JCR_MVALUE V ON (V.PROPERTY_ID=I.ID)" - + " where I.PARENT_ID=? and I.I_CLASS=2 order by I.NAME"; + PrivilegedExceptionAction<ResultSet> action = new PrivilegedExceptionAction<ResultSet>() + { + public ResultSet run() throws Exception + { + return HSQLDBMultiDbJDBCConnection.super.findNodesAndProperties(offset, limit); + } + }; + return SecurityHelper.doPrivilegedSQLExceptionAction(action); } - - /** - * Use simple queries since it is much faster - */ - @Override - protected QPath traverseQPath(String cpid) throws SQLException, InvalidItemStateException, IllegalNameException - { - return traverseQPathSQ(cpid); - } } Modified: jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/optimisation/db/HSQLDBSingleDbJDBCConnection.java =================================================================== --- jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/optimisation/db/HSQLDBSingleDbJDBCConnection.java 2011-02-03 07:32:17 UTC (rev 3915) +++ jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/optimisation/db/HSQLDBSingleDbJDBCConnection.java 2011-02-03 20:40:43 UTC (rev 3916) @@ -563,4 +563,20 @@ }; return SecurityHelper.doPrivilegedSQLExceptionAction(action); } + + /** + * {@inheritDoc} + */ + @Override + protected ResultSet findNodesAndProperties(final int offset, final int limit) throws SQLException + { + PrivilegedExceptionAction<ResultSet> action = new PrivilegedExceptionAction<ResultSet>() + { + public ResultSet run() throws Exception + { + return HSQLDBSingleDbJDBCConnection.super.findNodesAndProperties(offset, limit); + } + }; + return SecurityHelper.doPrivilegedSQLExceptionAction(action); + } } Modified: jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/optimisation/db/MultiDbJDBCConnection.java =================================================================== --- jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/optimisation/db/MultiDbJDBCConnection.java 2011-02-03 07:32:17 UTC (rev 3915) +++ jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/optimisation/db/MultiDbJDBCConnection.java 2011-02-03 20:40:43 UTC (rev 3916) @@ -184,8 +184,8 @@ "select I.ID, I.PARENT_ID, I.NAME, I.VERSION, I.I_INDEX, I.N_ORDER_NUM," + " P.ID AS P_ID, P.NAME AS P_NAME, P.VERSION AS P_VERSION, P.P_TYPE, P.P_MULTIVALUED," + " V.DATA, V.ORDER_NUM, V.STORAGE_DESC" - + " from JCR_MITEM I, JCR_MITEM P, JCR_MVALUE V where I.I_CLASS=1 and" - + " P.I_CLASS=2 and V.PROPERTY_ID=P.ID order by ID"; + + " from JCR_MITEM I, JCR_MITEM P, JCR_MVALUE V" + + " where I.I_CLASS=1 and P.I_CLASS=2 and P.PARENT_ID=I.ID and V.PROPERTY_ID=P.ID order by ID LIMIT ? OFFSET ?"; } /** @@ -625,6 +625,9 @@ findNodesAndProperties.clearParameters(); } + findNodesAndProperties.setInt(1, limit); + findNodesAndProperties.setInt(2, offset); + return findNodesAndProperties.executeQuery(); } } Modified: jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/optimisation/db/SingleDbJDBCConnection.java =================================================================== --- jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/optimisation/db/SingleDbJDBCConnection.java 2011-02-03 07:32:17 UTC (rev 3915) +++ jcr/trunk/exo.jcr.component.core/src/main/java/org/exoplatform/services/jcr/impl/storage/jdbc/optimisation/db/SingleDbJDBCConnection.java 2011-02-03 20:40:43 UTC (rev 3916) @@ -191,8 +191,9 @@ "select I.ID, I.PARENT_ID, I.NAME, I.VERSION, I.I_INDEX, I.N_ORDER_NUM," + " P.ID AS P_ID, P.NAME AS P_NAME, P.VERSION AS P_VERSION, P.P_TYPE, P.P_MULTIVALUED," + " V.DATA, V.ORDER_NUM, V.STORAGE_DESC" - + " from JCR_SITEM I, JCR_SITEM P, JCR_SVALUE V where I.I_CLASS=1 and I.CONTAINER_NAME=? and" - + " P.I_CLASS=2 and P.CONTAINER_NAME=? and P.PARENT_ID=I.ID" + " and V.PROPERTY_ID=P.ID order by ID"; + + " from JCR_SITEM I, JCR_SITEM P, JCR_SVALUE V" + + " where I.I_CLASS=1 and I.CONTAINER_NAME=? and P.I_CLASS=2 and P.CONTAINER_NAME=? and P.PARENT_ID=I.ID" + + " and V.PROPERTY_ID=P.ID order by ID LIMIT ? OFFSET ?"; } /** @@ -648,6 +649,8 @@ findNodesAndProperties.setString(1, containerName); findNodesAndProperties.setString(2, containerName); + findNodesAndProperties.setInt(3, limit); + findNodesAndProperties.setInt(4, offset); return findNodesAndProperties.executeQuery(); }