[hibernate-commits] Hibernate SVN: r11403 - in branches/Branch_3_2/Hibernate3: src/org/hibernate/cfg and 4 other directories.

Author: steve.ebersole(a)jboss.com Date: 2007-04-11 10:25:13 -0400 (Wed, 11 Apr 2007) New Revision: 11403 Added: branches/Branch_3_2/Hibernate3/test/org/hibernate/test/insertordering/ branches/Branch_3_2/Hibernate3/test/org/hibernate/test/insertordering/Group.java branches/Branch_3_2/Hibernate3/test/org/hibernate/test/insertordering/InsertOrderingTest.java branches/Branch_3_2/Hibernate3/test/org/hibernate/test/insertordering/Mapping.hbm.xml branches/Branch_3_2/Hibernate3/test/org/hibernate/test/insertordering/Membership.java branches/Branch_3_2/Hibernate3/test/org/hibernate/test/insertordering/User.java Modified: branches/Branch_3_2/Hibernate3/src/org/hibernate/action/EntityAction.java branches/Branch_3_2/Hibernate3/src/org/hibernate/action/EntityInsertAction.java branches/Branch_3_2/Hibernate3/src/org/hibernate/cfg/Environment.java branches/Branch_3_2/Hibernate3/src/org/hibernate/cfg/Settings.java branches/Branch_3_2/Hibernate3/src/org/hibernate/cfg/SettingsFactory.java branches/Branch_3_2/Hibernate3/src/org/hibernate/engine/ActionQueue.java branches/Branch_3_2/Hibernate3/src/org/hibernate/event/def/AbstractFlushingEventListener.java branches/Branch_3_2/Hibernate3/test/org/hibernate/test/AllTests.java Log: HHH-1 : batching insertions (partial) Modified: branches/Branch_3_2/Hibernate3/src/org/hibernate/action/EntityAction.java =================================================================== --- branches/Branch_3_2/Hibernate3/src/org/hibernate/action/EntityAction.java 2007-04-11 14:24:35 UTC (rev 11402) +++ branches/Branch_3_2/Hibernate3/src/org/hibernate/action/EntityAction.java 2007-04-11 14:25:13 UTC (rev 11403) @@ -12,53 +12,88 @@ import java.io.Serializable; /** - * Any action relating to insert/update/delete of an entity instance + * Base class for actions relating to insert/update/delete of an entity + * instance. + * * @author Gavin King */ public abstract class EntityAction implements Executable, Serializable, Comparable { - private final SessionImplementor session; + private final String entityName; private final Serializable id; private final Object instance; - private final String entityName; + private final SessionImplementor session; private transient EntityPersister persister; + /** + * Instantiate an action. + * + * @param session The session from which this action is coming. + * @param id The id of the entity + * @param instance The entiyt instance + * @param persister The entity persister + */ protected EntityAction(SessionImplementor session, Serializable id, Object instance, EntityPersister persister) { + this.entityName = persister.getEntityName(); + this.id = id; + this.instance = instance; this.session = session; - this.id = id; this.persister = persister; - this.instance = instance; - this.entityName = persister.getEntityName(); } - private void readObject(ObjectInputStream ois) throws IOException, ClassNotFoundException { - ois.defaultReadObject(); - persister = session.getFactory() - .getEntityPersister( entityName ); - } + protected abstract boolean hasPostCommitEventListeners(); - public final Serializable[] getPropertySpaces() { - return persister.getPropertySpaces(); + /** + * entity name accessor + * + * @return The entity name + */ + public String getEntityName() { + return entityName; } - protected final SessionImplementor getSession() { - return session; - } - - protected final Serializable getId() { + /** + * entity id accessor + * + * @return The entity id + */ + public final Serializable getId() { if ( id instanceof DelayedPostInsertIdentifier ) { return session.getPersistenceContext().getEntry( instance ).getId(); } return id; } - protected final EntityPersister getPersister() { + /** + * entity instance accessor + * + * @return The entity instance + */ + public final Object getInstance() { + return instance; + } + + /** + * originating session accessor + * + * @return The session from which this action originated. + */ + public final SessionImplementor getSession() { + return session; + } + + /** + * entity persister accessor + * + * @return The entity persister + */ + public final EntityPersister getPersister() { return persister; } - protected final Object getInstance() { - return instance; + public final Serializable[] getPropertySpaces() { + return persister.getPropertySpaces(); } public void beforeExecutions() { @@ -68,8 +103,6 @@ public boolean hasAfterTransactionCompletion() { return persister.hasCache() || hasPostCommitEventListeners(); } - - protected abstract boolean hasPostCommitEventListeners(); public String toString() { return StringHelper.unqualify( getClass().getName() ) + MessageHelper.infoString( entityName, id ); @@ -84,14 +117,20 @@ } else { //then by id - return persister.getIdentifierType() - .compare( id, action.id, session.getEntityMode() ); + return persister.getIdentifierType().compare( id, action.id, session.getEntityMode() ); } } + + /** + * Serialization... + * + * @param ois Thed object stream + * @throws IOException Problem performing the default stream reading + * @throws ClassNotFoundException Problem performing the default stream reading + */ + private void readObject(ObjectInputStream ois) throws IOException, ClassNotFoundException { + ois.defaultReadObject(); + persister = session.getFactory().getEntityPersister( entityName ); + } } - - - - - Modified: branches/Branch_3_2/Hibernate3/src/org/hibernate/action/EntityInsertAction.java =================================================================== --- branches/Branch_3_2/Hibernate3/src/org/hibernate/action/EntityInsertAction.java 2007-04-11 14:24:35 UTC (rev 11402) +++ branches/Branch_3_2/Hibernate3/src/org/hibernate/action/EntityInsertAction.java 2007-04-11 14:25:13 UTC (rev 11403) @@ -36,6 +36,10 @@ this.version = version; } + public Object[] getState() { + return state; + } + public void execute() throws HibernateException { EntityPersister persister = getPersister(); SessionImplementor session = getSession(); Modified: branches/Branch_3_2/Hibernate3/src/org/hibernate/cfg/Environment.java =================================================================== --- branches/Branch_3_2/Hibernate3/src/org/hibernate/cfg/Environment.java 2007-04-11 14:24:35 UTC (rev 11402) +++ branches/Branch_3_2/Hibernate3/src/org/hibernate/cfg/Environment.java 2007-04-11 14:25:13 UTC (rev 11403) @@ -453,6 +453,11 @@ public static final String ORDER_UPDATES = "hibernate.order_updates"; /** + * Enable ordering of insert statements for the purpose of more effecient JDBC batching. + */ + public static final String ORDER_INSERTS = "hibernate.order_inserts"; + + /** * The EntityMode in which set the Session opened from the SessionFactory. */ public static final String DEFAULT_ENTITY_MODE = "hibernate.default_entity_mode"; Modified: branches/Branch_3_2/Hibernate3/src/org/hibernate/cfg/Settings.java =================================================================== --- branches/Branch_3_2/Hibernate3/src/org/hibernate/cfg/Settings.java 2007-04-11 14:24:35 UTC (rev 11402) +++ branches/Branch_3_2/Hibernate3/src/org/hibernate/cfg/Settings.java 2007-04-11 14:25:13 UTC (rev 11403) @@ -61,6 +61,7 @@ private SQLExceptionConverter sqlExceptionConverter; private boolean wrapResultSetsEnabled; private boolean orderUpdatesEnabled; + private boolean orderInsertsEnabled; private EntityMode defaultEntityMode; private boolean dataDefinitionImplicitCommit; private boolean dataDefinitionInTransactionSupported; @@ -224,6 +225,10 @@ return orderUpdatesEnabled; } + public boolean isOrderInsertsEnabled() { + return orderInsertsEnabled; + } + public boolean isStructuredCacheEntriesEnabled() { return structuredCacheEntriesEnabled; } @@ -403,6 +408,10 @@ this.orderUpdatesEnabled = orderUpdatesEnabled; } + void setOrderInsertsEnabled(boolean orderInsertsEnabled) { + this.orderInsertsEnabled = orderInsertsEnabled; + } + void setStructuredCacheEntriesEnabled(boolean structuredCacheEntriesEnabled) { this.structuredCacheEntriesEnabled = structuredCacheEntriesEnabled; } Modified: branches/Branch_3_2/Hibernate3/src/org/hibernate/cfg/SettingsFactory.java =================================================================== --- branches/Branch_3_2/Hibernate3/src/org/hibernate/cfg/SettingsFactory.java 2007-04-11 14:24:35 UTC (rev 11402) +++ branches/Branch_3_2/Hibernate3/src/org/hibernate/cfg/SettingsFactory.java 2007-04-11 14:25:13 UTC (rev 11403) @@ -208,11 +208,15 @@ boolean comments = PropertiesHelper.getBoolean(Environment.USE_SQL_COMMENTS, properties); log.info( "Generate SQL with comments: " + enabledDisabled(comments) ); settings.setCommentsEnabled(comments); + + boolean orderUpdates = PropertiesHelper.getBoolean( Environment.ORDER_UPDATES, properties ); + log.info( "Order SQL updates by primary key: " + enabledDisabled( orderUpdates ) ); + settings.setOrderUpdatesEnabled( orderUpdates ); + + boolean orderInserts = PropertiesHelper.getBoolean(Environment.ORDER_INSERTS, properties); + log.info( "Order SQL inserts for batching: " + enabledDisabled( orderInserts ) ); + settings.setOrderInsertsEnabled( orderInserts ); - boolean orderUpdates = PropertiesHelper.getBoolean(Environment.ORDER_UPDATES, properties); - log.info( "Order SQL updates by primary key: " + enabledDisabled(orderUpdates) ); - settings.setOrderUpdatesEnabled(orderUpdates); - //Query parser settings: settings.setQueryTranslatorFactory( createQueryTranslatorFactory(properties) ); Modified: branches/Branch_3_2/Hibernate3/src/org/hibernate/engine/ActionQueue.java =================================================================== --- branches/Branch_3_2/Hibernate3/src/org/hibernate/engine/ActionQueue.java 2007-04-11 14:24:35 UTC (rev 11402) +++ branches/Branch_3_2/Hibernate3/src/org/hibernate/engine/ActionQueue.java 2007-04-11 14:25:13 UTC (rev 11403) @@ -19,6 +19,8 @@ import java.util.ArrayList; import java.util.List; import java.util.Set; +import java.util.HashMap; +import java.util.Iterator; import java.io.ObjectInputStream; import java.io.IOException; import java.io.Serializable; @@ -306,13 +308,91 @@ } } - public void sortUpdateActions() { + public void sortActions() { if ( session.getFactory().getSettings().isOrderUpdatesEnabled() ) { //sort the updates by pk java.util.Collections.sort( updates ); } + if ( session.getFactory().getSettings().isOrderInsertsEnabled() ) { + sortInsertActions(); + } } + /** + * Order the {@link #insertions} queue such that we group inserts + * against the same entity together (without violating constraints). The + * original order is generated by cascade order, which in turn is based on + * the directionality of foreign-keys. So even though we will be changing + * the ordering here, we need to make absolutely certain that we do not + * circumvent this FK ordering to the extent of causing constraint + * violations + */ + private void sortInsertActions() { + // IMPLEMENTATION NOTES: + // + // The main data structure in this ordering algorithm is the 'positionToAction' + // map. Essentially this can be thought of as an put-ordered map (the problem with + // actually implementing it that way and doing away with the 'nameList' is that + // we'd end up having potential duplicate key values). 'positionToAction' maitains + // a mapping from a position within the 'nameList' structure to a "partial queue" + // of actions. + + HashMap positionToAction = new HashMap(); + List nameList = new ArrayList(); + + loopInsertion: while( !insertions.isEmpty() ) { + EntityInsertAction action = ( EntityInsertAction ) insertions.remove( 0 ); + String thisEntityName = action.getEntityName(); + + // see if we have already encountered this entity-name... + if ( ! nameList.contains( thisEntityName ) ) { + // we have not, so create the proper entries in nameList and positionToAction + ArrayList segmentedActionQueue = new ArrayList(); + segmentedActionQueue.add( action ); + nameList.add( thisEntityName ); + positionToAction.put( new Integer( nameList.indexOf( thisEntityName ) ), segmentedActionQueue ); + } + else { + // we have seen it before, so we need to determine if this insert action is + // is depenedent upon a previously processed action in terms of FK + // relationships (this FK checking is done against the entity's property-state + // associated with the action...) + int lastPos = nameList.lastIndexOf( thisEntityName ); + Object[] states = action.getState(); + for ( int i = 0; i < states.length; i++ ) { + for ( int j = 0; j < nameList.size(); j++ ) { + ArrayList tmpList = ( ArrayList ) positionToAction.get( new Integer( j ) ); + for ( int k = 0; k < tmpList.size(); k++ ) { + final EntityInsertAction checkAction = ( EntityInsertAction ) tmpList.get( k ); + if ( checkAction.getInstance() == states[i] && j > lastPos ) { + // 'checkAction' is inserting an entity upon which 'action' + // depends... + // note: this is an assumption and may not be correct in the case of one-to-one + ArrayList segmentedActionQueue = new ArrayList(); + segmentedActionQueue.add( action ); + nameList.add( thisEntityName ); + positionToAction.put(new Integer( nameList.lastIndexOf( thisEntityName ) ), segmentedActionQueue ); + continue loopInsertion; + } + } + } + } + + ArrayList actionQueue = ( ArrayList ) positionToAction.get( new Integer( lastPos ) ); + actionQueue.add( action ); + } + } + + // now iterate back through positionToAction map and move entityInsertAction back to insertion list + for ( int p = 0; p < nameList.size(); p++ ) { + ArrayList actionQueue = ( ArrayList ) positionToAction.get( new Integer( p ) ); + Iterator itr = actionQueue.iterator(); + while ( itr.hasNext() ) { + insertions.add( itr.next() ); + } + } + } + public ArrayList cloneDeletions() { return (ArrayList) deletions.clone(); } Modified: branches/Branch_3_2/Hibernate3/src/org/hibernate/event/def/AbstractFlushingEventListener.java =================================================================== --- branches/Branch_3_2/Hibernate3/src/org/hibernate/event/def/AbstractFlushingEventListener.java 2007-04-11 14:24:35 UTC (rev 11402) +++ branches/Branch_3_2/Hibernate3/src/org/hibernate/event/def/AbstractFlushingEventListener.java 2007-04-11 14:25:13 UTC (rev 11403) @@ -198,7 +198,7 @@ } } - source.getActionQueue().sortUpdateActions(); + source.getActionQueue().sortActions(); } /** Modified: branches/Branch_3_2/Hibernate3/test/org/hibernate/test/AllTests.java =================================================================== --- branches/Branch_3_2/Hibernate3/test/org/hibernate/test/AllTests.java 2007-04-11 14:24:35 UTC (rev 11402) +++ branches/Branch_3_2/Hibernate3/test/org/hibernate/test/AllTests.java 2007-04-11 14:25:13 UTC (rev 11403) @@ -129,6 +129,7 @@ import org.hibernate.test.version.db.DbVersionTest; import org.hibernate.test.version.sybase.SybaseTimestampVersioningTest; import org.hibernate.test.where.WhereTest; +import org.hibernate.test.insertordering.InsertOrderingTest; /** * @author Gavin King @@ -301,6 +302,7 @@ suite.addTest( KeyManyToOneSuite.suite() ); suite.addTest( DialectFunctionalTestsSuite.suite() ); suite.addTest( DialectUnitTestsSuite.suite() ); + suite.addTest( InsertOrderingTest.suite() ); return suite; } Added: branches/Branch_3_2/Hibernate3/test/org/hibernate/test/insertordering/Group.java =================================================================== --- branches/Branch_3_2/Hibernate3/test/org/hibernate/test/insertordering/Group.java (rev 0) +++ branches/Branch_3_2/Hibernate3/test/org/hibernate/test/insertordering/Group.java 2007-04-11 14:25:13 UTC (rev 11403) @@ -0,0 +1,29 @@ +package org.hibernate.test.insertordering; + +/** + * {@inheritDoc} + * + * @author Steve Ebersole + */ +public class Group { + private Long id; + private String name; + + /** + * for persistence + */ + Group() { + } + + public Group(String name) { + this.name = name; + } + + public Long getId() { + return id; + } + + public String getName() { + return name; + } +} Added: branches/Branch_3_2/Hibernate3/test/org/hibernate/test/insertordering/InsertOrderingTest.java =================================================================== --- branches/Branch_3_2/Hibernate3/test/org/hibernate/test/insertordering/InsertOrderingTest.java (rev 0) +++ branches/Branch_3_2/Hibernate3/test/org/hibernate/test/insertordering/InsertOrderingTest.java 2007-04-11 14:25:13 UTC (rev 11403) @@ -0,0 +1,127 @@ +package org.hibernate.test.insertordering; + +import java.util.List; +import java.util.ArrayList; +import java.util.Iterator; +import java.sql.SQLException; +import java.sql.PreparedStatement; + +import junit.framework.Test; + +import org.hibernate.junit.functional.FunctionalTestCase; +import org.hibernate.junit.functional.FunctionalTestClassTestSuite; +import org.hibernate.cfg.Configuration; +import org.hibernate.cfg.Environment; +import org.hibernate.Session; +import org.hibernate.Interceptor; +import org.hibernate.HibernateException; +import org.hibernate.jdbc.BatchingBatcher; +import org.hibernate.jdbc.ConnectionManager; +import org.hibernate.jdbc.Expectation; +import org.hibernate.jdbc.BatcherFactory; +import org.hibernate.jdbc.Batcher; + +/** + * {@inheritDoc} + * + * @author Steve Ebersole + */ +public class InsertOrderingTest extends FunctionalTestCase { + public InsertOrderingTest(String string) { + super( string ); + } + + public static Test suite() { + return new FunctionalTestClassTestSuite( InsertOrderingTest.class ); + } + + public String[] getMappings() { + return new String[] { "insertordering/Mapping.hbm.xml" }; + } + + public void configure(Configuration cfg) { + super.configure( cfg ); + cfg.setProperty( Environment.ORDER_INSERTS, "true" ); + cfg.setProperty( Environment.STATEMENT_BATCH_SIZE, "10" ); + cfg.setProperty( Environment.BATCH_STRATEGY, StatsBatcherFactory.class.getName() ); + } + + public void testBatchOrdering() { + Session s = openSession(); + s.beginTransaction(); + int iterations = 12; + for ( int i = 0; i < iterations; i++ ) { + User user = new User( "user-" + i ); + Group group = new Group( "group-" + i ); + s.save( user ); + s.save( group ); + user.addMembership( group ); + } + StatsBatcher.reset(); + s.getTransaction().commit(); + s.close(); + + assertEquals( 6, StatsBatcher.batchSizes.size() ); // 2 batches of each insert statement + + s = openSession(); + s.beginTransaction(); + Iterator users = s.createQuery( "from User u left join fetch u.memberships m left join fetch m.group" ).list().iterator(); + while ( users.hasNext() ) { + s.delete( users.next() ); + } + s.getTransaction().commit(); + s.close(); + } + + public static class Counter { + public int count = 0; + } + + public static class StatsBatcher extends BatchingBatcher { + private static String batchSQL; + private static List batchSizes = new ArrayList(); + private static int currentBatch = -1; + + public StatsBatcher(ConnectionManager connectionManager, Interceptor interceptor) { + super( connectionManager, interceptor ); + } + + static void reset() { + batchSizes = new ArrayList(); + currentBatch = -1; + batchSQL = null; + } + + public PreparedStatement prepareBatchStatement(String sql) throws SQLException { + PreparedStatement rtn = super.prepareBatchStatement( sql ); + if ( batchSQL == null || !batchSQL.equals( sql ) ) { + currentBatch++; + batchSQL = sql; + batchSizes.add( currentBatch, new Counter() ); + System.out.println( "--------------------------------------------------------" ); + System.out.println( "Preparing statement [" + sql + "]" ); + } + return rtn; + } + + public void addToBatch(Expectation expectation) throws SQLException, HibernateException { + Counter counter = ( Counter ) batchSizes.get( currentBatch ); + counter.count++; + System.out.println( "Adding to batch [" + batchSQL + "]" ); + super.addToBatch( expectation ); + } + + protected void doExecuteBatch(PreparedStatement ps) throws SQLException, HibernateException { + System.out.println( "executing batch [" + batchSQL + "]" ); + System.out.println( "--------------------------------------------------------" ); + batchSQL = null; + super.doExecuteBatch( ps ); + } + } + + public static class StatsBatcherFactory implements BatcherFactory { + public Batcher createBatcher(ConnectionManager connectionManager, Interceptor interceptor) { + return new StatsBatcher( connectionManager, interceptor ); + } + } +} Added: branches/Branch_3_2/Hibernate3/test/org/hibernate/test/insertordering/Mapping.hbm.xml =================================================================== --- branches/Branch_3_2/Hibernate3/test/org/hibernate/test/insertordering/Mapping.hbm.xml (rev 0) +++ branches/Branch_3_2/Hibernate3/test/org/hibernate/test/insertordering/Mapping.hbm.xml 2007-04-11 14:25:13 UTC (rev 11403) @@ -0,0 +1,35 @@ +<?xml version="1.0"?> +<!DOCTYPE hibernate-mapping PUBLIC + "-//Hibernate/Hibernate Mapping DTD 3.0//EN" + "http://hibernate.sourceforge.net/hibernate-mapping-3.0.dtd"> + +<hibernate-mapping package="org.hibernate.test.insertordering" default-access="field"> + + <class name="User" table="INS_ORD_USR"> + <id name="id"> + <generator class="increment"/> + </id> + <property name="username" column="USR_NM" /> + <set name="memberships" fetch="select" lazy="true" inverse="true" cascade="all"> + <key column="USR_ID"/> + <one-to-many class="Membership"/> + </set> + </class> + + <class name="Group" table="INS_ORD_GRP"> + <id name="id"> + <generator class="increment"/> + </id> + <property name="name"/> + </class> + + <class name="Membership" table="INS_ORD_MEM"> + <id name="id"> + <generator class="increment" /> + </id> + <many-to-one name="user" class="User" column="USR_ID" cascade="all"/> + <many-to-one name="group" class="Group" column="GRP_ID" cascade="all"/> + <property name="activationDate" type="timestamp" column="JN_DT"/> + </class> +</hibernate-mapping> + Added: branches/Branch_3_2/Hibernate3/test/org/hibernate/test/insertordering/Membership.java =================================================================== --- branches/Branch_3_2/Hibernate3/test/org/hibernate/test/insertordering/Membership.java (rev 0) +++ branches/Branch_3_2/Hibernate3/test/org/hibernate/test/insertordering/Membership.java 2007-04-11 14:25:13 UTC (rev 11403) @@ -0,0 +1,47 @@ +package org.hibernate.test.insertordering; + +import java.util.Date; + +/** + * {@inheritDoc} + * + * @author Steve Ebersole + */ +public class Membership { + private Long id; + private User user; + private Group group; + private Date activationDate; + + /** + * For persistence + */ + Membership() { + } + + public Membership(User user, Group group) { + this( user, group, new Date() ); + } + + public Membership(User user, Group group, Date activationDate) { + this.user = user; + this.group = group; + this.activationDate = activationDate; + } + + public Long getId() { + return id; + } + + public User getUser() { + return user; + } + + public Group getGroup() { + return group; + } + + public Date getActivationDate() { + return activationDate; + } +} Added: branches/Branch_3_2/Hibernate3/test/org/hibernate/test/insertordering/User.java =================================================================== --- branches/Branch_3_2/Hibernate3/test/org/hibernate/test/insertordering/User.java (rev 0) +++ branches/Branch_3_2/Hibernate3/test/org/hibernate/test/insertordering/User.java 2007-04-11 14:25:13 UTC (rev 11403) @@ -0,0 +1,44 @@ +package org.hibernate.test.insertordering; + +import java.util.Set; +import java.util.HashSet; +import java.util.Iterator; + +/** + * {@inheritDoc} + * + * @author Steve Ebersole + */ +public class User { + private Long id; + private String username; + private Set memberships = new HashSet(); + + /** + * for persistence + */ + User() { + } + + public User(String username) { + this.username = username; + } + + public Long getId() { + return id; + } + + public String getUsername() { + return username; + } + + public Iterator getMemberships() { + return memberships.iterator(); + } + + public Membership addMembership(Group group) { + Membership membership = new Membership( this, group ); + memberships.add( membership ); + return membership; + } +}