2:03 p.m.
Author: timfox
Date: 2009-12-02 15:03:15 -0500 (Wed, 02 Dec 2009)
New Revision: 8506
Added:
trunk/tests/src/org/hornetq/tests/unit/util/MemorySizeTest.java
Modified:
trunk/src/main/org/hornetq/core/server/impl/MessageReferenceImpl.java
trunk/src/main/org/hornetq/core/server/impl/ServerMessageImpl.java
trunk/src/main/org/hornetq/utils/TypedProperties.java
trunk/tests/src/org/hornetq/tests/integration/client/PagingTest.java
trunk/tests/src/org/hornetq/tests/integration/client/ProducerFlowControlTest.java
Log:
improved memory estimate
Modified: trunk/src/main/org/hornetq/core/server/impl/MessageReferenceImpl.java
===================================================================
--- trunk/src/main/org/hornetq/core/server/impl/MessageReferenceImpl.java 2009-12-02
16:16:19 UTC (rev 8505)
+++ trunk/src/main/org/hornetq/core/server/impl/MessageReferenceImpl.java 2009-12-02
20:03:15 UTC (rev 8506)
@@ -17,7 +17,7 @@
import org.hornetq.core.server.MessageReference;
import org.hornetq.core.server.Queue;
import org.hornetq.core.server.ServerMessage;
-import org.hornetq.utils.DataConstants;
+import org.hornetq.utils.MemorySize;
/**
* Implementation of a MessageReference
@@ -42,7 +42,26 @@
private final Queue queue;
// Static --------------------------------------------------------
+
+ private static final int memoryOffset;
+ static
+ {
+ // This is an estimate of how much memory a ServerMessageImpl takes up, exclusing
body and properties
+ // Note, it is only an estimate, it's not possible to be entirely sure with
Java
+ // This figure is calculated using the test utilities in
org.hornetq.tests.unit.util.sizeof
+ // The value is somewhat higher on 64 bit architectures, probably due to different
alignment
+
+ if (MemorySize.is64bitArch())
+ {
+ memoryOffset = 48;
+ }
+ else
+ {
+ memoryOffset = 32;
+ }
+ }
+
// Constructors --------------------------------------------------
public MessageReferenceImpl()
@@ -78,13 +97,7 @@
public static int getMemoryEstimate()
{
- // from few tests I have done, deliveryCount and scheduledDelivery will use two
longs (because of alignment)
- // and each of the references (messages and queue) will use the equivalent to two
longs (because of long
- // pointers).
- // Anyway.. this is just an estimate
-
- // TODO - doesn't the object itself have an overhead? - I thought was usually
one Long per Object?
- return DataConstants.SIZE_LONG * 4;
+ return memoryOffset;
}
public int getDeliveryCount()
Modified: trunk/src/main/org/hornetq/core/server/impl/ServerMessageImpl.java
===================================================================
--- trunk/src/main/org/hornetq/core/server/impl/ServerMessageImpl.java 2009-12-02 16:16:19
UTC (rev 8505)
+++ trunk/src/main/org/hornetq/core/server/impl/ServerMessageImpl.java 2009-12-02 20:03:15
UTC (rev 8506)
@@ -16,8 +16,6 @@
import java.io.InputStream;
import java.util.concurrent.atomic.AtomicInteger;
-import org.hornetq.core.buffers.HornetQBuffer;
-import org.hornetq.core.buffers.impl.ResetLimitWrappedHornetQBuffer;
import org.hornetq.core.logging.Logger;
import org.hornetq.core.message.impl.MessageImpl;
import org.hornetq.core.paging.PagingStore;
@@ -26,8 +24,8 @@
import org.hornetq.core.server.Queue;
import org.hornetq.core.server.ServerMessage;
import org.hornetq.utils.DataConstants;
+import org.hornetq.utils.MemorySize;
import org.hornetq.utils.SimpleString;
-import org.hornetq.utils.TypedProperties;
/**
*
@@ -49,6 +47,25 @@
private PagingStore pagingStore;
+ private static final int memoryOffset;
+
+ static
+ {
+ // This is an estimate of how much memory a ServerMessageImpl takes up, exclusing
body and properties
+ // Note, it is only an estimate, it's not possible to be entirely sure with
Java
+ // This figure is calculated using the test utilities in
org.hornetq.tests.unit.util.sizeof
+ // The value is somewhat higher on 64 bit architectures, probably due to different
alignment
+
+ if (MemorySize.is64bitArch())
+ {
+ memoryOffset = 352;
+ }
+ else
+ {
+ memoryOffset = 232;
+ }
+ }
+
/*
* Constructor for when reading from network
*/
@@ -62,7 +79,7 @@
public ServerMessageImpl(final long messageID, final int initialMessageBufferSize)
{
super(initialMessageBufferSize);
-
+
this.messageID = messageID;
}
@@ -144,44 +161,17 @@
{
return false;
}
-
- private volatile int memoryEstimate = -1;
-// public int getMemoryEstimate()
-// {
-// if (memoryEstimate == -1)
-// {
-// memoryEstimate =
-// DataConstants.SIZE_INT + // Object overhead
-// this.getHeadersAndPropertiesEncodeSize() +
-// buffer.capacity() +
-// DataConstants.SIZE_INT + // PagingStore reference
-// DataConstants.SIZE_INT + // DurableRefCount reference
-// DataConstants.SIZE_INT + // RefCount reference
-// DataConstants.SIZE_INT + // Reference to buffer
-// DataConstants.SIZE_INT + // Reference to bodyBuffer
-// DataConstants.SIZE_BOOLEAN + // bufferValid
-// DataConstants.SIZE_INT + // endOfBodyPosition
-// DataConstants.SIZE_INT + // endOfMessagePosition
-// DataConstants.SIZE_BOOLEAN + // copied
-// DataConstants.SIZE_BOOLEAN + // bufferUsed
-// DataConstants.SIZE_LONG; // A bit more due to alignment and fragmentation
-// }
-//
-// return memoryEstimate;
-// }
+ private volatile int memoryEstimate = -1;
public int getMemoryEstimate()
{
if (memoryEstimate == -1)
{
- // This is just an estimate...
- // due to memory alignments and JVM implementation this could be very
- // different from reality
- memoryEstimate = getEncodeSize() + (16 + 4) * 2 + 1;
+ memoryEstimate = memoryOffset + buffer.capacity() +
properties.getMemoryOffset();
}
-
- return memoryEstimate;
+
+ return this.memoryEstimate;
}
public ServerMessage copy(final long newID)
Modified: trunk/src/main/org/hornetq/utils/TypedProperties.java
===================================================================
--- trunk/src/main/org/hornetq/utils/TypedProperties.java 2009-12-02 16:16:19 UTC (rev
8505)
+++ trunk/src/main/org/hornetq/utils/TypedProperties.java 2009-12-02 20:03:15 UTC (rev
8506)
@@ -65,6 +65,15 @@
public TypedProperties()
{
}
+
+ public int getMemoryOffset()
+ {
+ //The estimate is basically the encode size + 2 object references for each entry in
the map
+ //Note we don't include the attributes or anything else since they already
included in the memory estimate
+ //of the ServerMessage
+
+ return properties == null ? 0 : size + 2 * DataConstants.SIZE_INT *
properties.size();
+ }
public TypedProperties(final TypedProperties other)
{
Modified: trunk/tests/src/org/hornetq/tests/integration/client/PagingTest.java
===================================================================
--- trunk/tests/src/org/hornetq/tests/integration/client/PagingTest.java 2009-12-02
16:16:19 UTC (rev 8505)
+++ trunk/tests/src/org/hornetq/tests/integration/client/PagingTest.java 2009-12-02
20:03:15 UTC (rev 8506)
@@ -131,10 +131,8 @@
producer.send(message);
}
- log.info("** sent messages");
+ session.close();
- session.close();
-
assertTrue("TotalMemory expected to be > 0 when it was " +
server.getPostOffice()
.getPagingManager()
.getTotalMemory(),
@@ -696,15 +694,15 @@
session.start();
- for (int i = 0; i < 9; i++)
+ for (int i = 0; i < 6; i++)
{
ClientMessage message2 = consumer.receive(RECEIVE_TIMEOUT);
-
+
assertNotNull(message2);
message2.acknowledge();
}
-
+
assertNull(consumer.receiveImmediate());
assertEquals(0, server.getPostOffice().getPagingManager().getTotalMemory());
@@ -720,15 +718,15 @@
producer.send(message);
}
- for (int i = 0; i < 9; i++)
- {
+ for (int i = 0; i < 6; i++)
+ {
ClientMessage message2 = consumer.receive(RECEIVE_TIMEOUT);
-
+
assertNotNull(message2);
message2.acknowledge();
}
-
+
assertNull(consumer.receiveImmediate());
session.close();
@@ -753,7 +751,7 @@
session.start();
- for (int i = 0; i < 9; i++)
+ for (int i = 0; i < 6; i++)
{
ClientMessage message2 = consumer.receive(RECEIVE_TIMEOUT);
Modified:
trunk/tests/src/org/hornetq/tests/integration/client/ProducerFlowControlTest.java
===================================================================
---
trunk/tests/src/org/hornetq/tests/integration/client/ProducerFlowControlTest.java 2009-12-02
16:16:19 UTC (rev 8505)
+++
trunk/tests/src/org/hornetq/tests/integration/client/ProducerFlowControlTest.java 2009-12-02
20:03:15 UTC (rev 8506)
@@ -263,8 +263,6 @@
{
handlers[i] = new MyHandler();
- log.info("created consumer");
-
ClientConsumer consumer = session.createConsumer(new SimpleString(queueName +
i));
consumer.setMessageHandler(handlers[i]);
@@ -341,7 +339,7 @@
HornetQServer server = createServer(false, isNetty());
AddressSettings addressSettings = new AddressSettings();
- addressSettings.setMaxSizeBytes(1024);
+ addressSettings.setMaxSizeBytes(4000);
addressSettings.setAddressFullMessagePolicy(AddressFullMessagePolicy.BLOCK);
HierarchicalRepository<AddressSettings> repos =
server.getAddressSettingsRepository();
@@ -351,7 +349,8 @@
ClientSessionFactory sf = createFactory(isNetty());
- sf.setProducerWindowSize(1024);
+ //Make sure the producer grabs all the credits
+ sf.setProducerWindowSize(4000);
sf.setConsumerWindowSize(1024);
sf.setAckBatchSize(1024);
@@ -384,7 +383,7 @@
while (waiting != 1 || (System.currentTimeMillis() - start) > 3000);
assertEquals(1, waiting);
-
+
byte[] bytes = new byte[0];
ClientMessage message = session.createClientMessage(false);
@@ -392,7 +391,7 @@
message.getBodyBuffer().writeBytes(bytes);
producer.send(message);
-
+
class SessionCloser implements Runnable
{
public void run()
@@ -422,9 +421,9 @@
ClientMessage message2 = session.createClientMessage(false);
message2.getBodyBuffer().writeBytes(bytes);
-
+
producer2.send(message2);
-
+
// Make sure it blocked until the first producer was closed
assertTrue(closer.closed);
@@ -448,7 +447,7 @@
HornetQServer server = createServer(false, isNetty());
AddressSettings addressSettings = new AddressSettings();
- addressSettings.setMaxSizeBytes(1024);
+ addressSettings.setMaxSizeBytes(4000);
addressSettings.setAddressFullMessagePolicy(AddressFullMessagePolicy.BLOCK);
HierarchicalRepository<AddressSettings> repos =
server.getAddressSettingsRepository();
@@ -458,7 +457,8 @@
ClientSessionFactory sf = createFactory(isNetty());
- sf.setProducerWindowSize(1024);
+ //Make sure producer grabs all the credits
+ sf.setProducerWindowSize(4000);
sf.setConsumerWindowSize(1024);
sf.setAckBatchSize(1024);
@@ -526,7 +526,7 @@
HornetQServer server = createServer(false, isNetty());
AddressSettings addressSettings = new AddressSettings();
- addressSettings.setMaxSizeBytes(1024);
+ addressSettings.setMaxSizeBytes(4000);
addressSettings.setAddressFullMessagePolicy(AddressFullMessagePolicy.BLOCK);
HierarchicalRepository<AddressSettings> repos =
server.getAddressSettingsRepository();
@@ -536,7 +536,8 @@
ClientSessionFactory sf = createFactory(isNetty());
- sf.setProducerWindowSize(1024);
+ //Make sure first producer grabs all the credits
+ sf.setProducerWindowSize(4000);
sf.setConsumerWindowSize(1024);
sf.setAckBatchSize(1024);
Added: trunk/tests/src/org/hornetq/tests/unit/util/MemorySizeTest.java
===================================================================
--- trunk/tests/src/org/hornetq/tests/unit/util/MemorySizeTest.java
(rev 0)
+++ trunk/tests/src/org/hornetq/tests/unit/util/MemorySizeTest.java 2009-12-02 20:03:15
UTC (rev 8506)
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2009 Red Hat, Inc.
+ * Red Hat licenses this file to you under the Apache License, version
+ * 2.0 (the "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package org.hornetq.tests.unit.util;
+
+import junit.framework.TestCase;
+
+import org.hornetq.core.logging.Logger;
+import org.hornetq.core.server.impl.MessageReferenceImpl;
+import org.hornetq.core.server.impl.ServerMessageImpl;
+import org.hornetq.utils.MemorySize;
+
+/**
+ * A MemorySizeTest
+ *
+ * @author Tim Fox
+ *
+ *
+ */
+public class MemorySizeTest extends TestCase
+{
+ private static final Logger log = Logger.getLogger(MemorySizeTest.class);
+
+ public void testObjectSizes() throws Exception
+ {
+ log.info("Server message size is " + MemorySize.calculateSize(new
MemorySize.ObjectFactory()
+ {
+ public Object createObject()
+ {
+ return new ServerMessageImpl(1, 1000);
+ }
+ }));
+
+ log.info("Message reference size is " + MemorySize.calculateSize(new
MemorySize.ObjectFactory()
+ {
+ public Object createObject()
+ {
+ return new MessageReferenceImpl();
+ }
+ }));
+ }
+}
+
+