Re: [infinispan-dev] Improved Marshaller

I just checked the following in trunk. Galder, you may want to do something similar for your JBMAR prototype. The changes are around InternalCacheEntry and now InternalCacheValue. Previously, the marshaller stored an InternalCacheEntry as [key, value, canExpire, created, lifespan, lastUsed, maxIdle], setting either of the last 4 elements to -1 if they are unused. In a system mainly comprising of mortal values, this is wasteful since each of the last 2 values - which would be unused - are longs and still written to. So this change identifies the InternalCacheEntry and writes just what is needed, hence the use of specific magic numbers for each of the InternalCacheEntry implementations. Using a different magic number for each entry type allows me to trim what is written. Same approach used for InternalCacheValue. Yes, even more ugly switch statements - and this is why we need JBMAR to clean this up - if and when it can match performance and compactness of the resulting byte array. :-)

Cheers Manik Begin forwarded message: > From: infinispan-commits(a)lists.jboss.org > Date: 8 May 2009 12:37:23 BST > To: infinispan-commits(a)lists.jboss.org > Subject: [infinispan-commits] Infinispan SVN: r234 - > trunk/core/src/main/java/org/infinispan/marshall. > Reply-To: infinispan-commits(a)lists.jboss.org > > Author: manik.surtani(a)jboss.com > Date: 2009-05-08 07:37:23 -0400 (Fri, 08 May 2009) > New Revision: 234 > > Modified: > trunk/core/src/main/java/org/infinispan/marshall/MarshallerImpl.java > Log: > Improved marshaller > > Modified: > trunk/core/src/main/java/org/infinispan/marshall/MarshallerImpl.java > =================================================================== > --- > trunk/core/src/main/java/org/infinispan/marshall/MarshallerImpl.java > 2009-05-08 11:36:59 UTC (rev 233) > +++ > trunk/core/src/main/java/org/infinispan/marshall/MarshallerImpl.java > 2009-05-08 11:37:23 UTC (rev 234) > @@ -26,8 +26,13 @@ > import org.infinispan.commands.RemoteCommandFactory; > import org.infinispan.commands.ReplicableCommand; > import org.infinispan.commands.write.WriteCommand; > +import org.infinispan.container.entries.ImmortalCacheEntry; > import org.infinispan.container.entries.InternalCacheEntry; > +import org.infinispan.container.entries.InternalCacheValue; > import org.infinispan.container.entries.InternalEntryFactory; > +import org.infinispan.container.entries.MortalCacheEntry; > +import org.infinispan.container.entries.TransientCacheEntry; > +import org.infinispan.container.entries.TransientMortalCacheEntry; > import org.infinispan.io.ByteBuffer; > import org.infinispan.io.ExposedByteArrayOutputStream; > import org.infinispan.remoting.responses.ExceptionResponse; > @@ -94,14 +99,23 @@ > protected static final int MAGICNUMBER_SINGLETON_LIST = 23; > protected static final int MAGICNUMBER_COMMAND = 24; > protected static final int MAGICNUMBER_TRANSACTION_LOG = 25; > - protected static final int MAGICNUMBER_INTERNAL_CACHED_ENTRY = 26; > > + // --- cache entries and values --- > + protected static final int MAGICNUMBER_ICE_IMMORTAL = 26; > + protected static final int MAGICNUMBER_ICE_MORTAL = 27; > + protected static final int MAGICNUMBER_ICE_TRANSIENT = 28; > + protected static final int MAGICNUMBER_ICE_TRANSIENT_MORTAL = 29; > + protected static final int MAGICNUMBER_ICV_IMMORTAL = 30; > + protected static final int MAGICNUMBER_ICV_MORTAL = 31; > + protected static final int MAGICNUMBER_ICV_TRANSIENT = 32; > + protected static final int MAGICNUMBER_ICV_TRANSIENT_MORTAL = 33; > + > // ---- responses > - protected static final int MAGICNUMBER_REQUEST_IGNORED_RESPONSE = 27; > - protected static final int MAGICNUMBER_EXTENDED_RESPONSE = 28; > - protected static final int MAGICNUMBER_EXCEPTION_RESPONSE = 29; > - protected static final int MAGICNUMBER_SUCCESSFUL_RESPONSE = 30; > - protected static final int MAGICNUMBER_UNSUCCESSFUL_RESPONSE = 31; > + protected static final int MAGICNUMBER_REQUEST_IGNORED_RESPONSE = 34; > + protected static final int MAGICNUMBER_EXTENDED_RESPONSE = 35; > + protected static final int MAGICNUMBER_EXCEPTION_RESPONSE = 36; > + protected static final int MAGICNUMBER_SUCCESSFUL_RESPONSE = 37; > + protected static final int MAGICNUMBER_UNSUCCESSFUL_RESPONSE = 38; > > protected static final int MAGICNUMBER_NULL = 99; > protected static final int MAGICNUMBER_SERIALIZABLE = 100; > @@ -173,19 +187,9 @@ > } else if (o instanceof Response) { > marshallResponse((Response) o, out, refMap); > } else if (o instanceof InternalCacheEntry) { > - out.writeByte(MAGICNUMBER_INTERNAL_CACHED_ENTRY); > - InternalCacheEntry ice = (InternalCacheEntry) o; > - marshallObject(ice.getKey(), out, refMap); > - marshallObject(ice.getValue(), out, refMap); > - if (ice.canExpire()) { > - out.writeBoolean(true); > - writeUnsignedLong(out, ice.getCreated()); > - out.writeLong(ice.getLifespan()); // could be negative > so should not use unsigned longs > - writeUnsignedLong(out, ice.getLastUsed()); > - out.writeLong(ice.getMaxIdle()); // could be negative > so should not use unsigned longs > - } else { > - out.writeBoolean(false); > - } > + marshallInternalCacheEntry((InternalCacheEntry) o, out, > refMap); > + } else if (o instanceof InternalCacheValue) { > + marshallInternalCacheValue((InternalCacheValue) o, out, > refMap); > } else if (o.getClass().equals(ArrayList.class)) { > out.writeByte(MAGICNUMBER_ARRAY_LIST); > marshallCollection((Collection) o, out, refMap); > @@ -252,7 +256,65 @@ > } > } > > + private void marshallInternalCacheEntry(InternalCacheEntry ice, > ObjectOutput out, Map<Object, Integer> refMap) throws IOException { > + if (ice.getClass().equals(ImmortalCacheEntry.class)) { > + out.writeByte(MAGICNUMBER_ICE_IMMORTAL); > + marshallObject(ice.getKey(), out, refMap); > + marshallObject(ice.getValue(), out, refMap); > > + } else if (ice.getClass().equals(MortalCacheEntry.class)) { > + out.writeByte(MAGICNUMBER_ICE_MORTAL); > + marshallObject(ice.getKey(), out, refMap); > + marshallObject(ice.getValue(), out, refMap); > + writeUnsignedLong(out, ice.getCreated()); > + out.writeLong(ice.getLifespan()); // could be negative so > should not use unsigned longs > + > + } else if (ice.getClass().equals(TransientCacheEntry.class)) { > + out.writeByte(MAGICNUMBER_ICE_TRANSIENT); > + marshallObject(ice.getKey(), out, refMap); > + marshallObject(ice.getValue(), out, refMap); > + writeUnsignedLong(out, ice.getLastUsed()); > + out.writeLong(ice.getMaxIdle()); // could be negative so > should not use unsigned longs > + > + } else if > (ice.getClass().equals(TransientMortalCacheEntry.class)) { > + out.writeByte(MAGICNUMBER_ICE_TRANSIENT_MORTAL); > + marshallObject(ice.getKey(), out, refMap); > + marshallObject(ice.getValue(), out, refMap); > + writeUnsignedLong(out, ice.getCreated()); > + out.writeLong(ice.getLifespan()); // could be negative so > should not use unsigned longs > + writeUnsignedLong(out, ice.getLastUsed()); > + out.writeLong(ice.getMaxIdle()); // could be negative so > should not use unsigned longs > + } > + } > + > + private void marshallInternalCacheValue(InternalCacheValue icv, > ObjectOutput out, Map<Object, Integer> refMap) throws IOException { > + if (icv.getClass().equals(ImmortalCacheEntry.class)) { > + out.writeByte(MAGICNUMBER_ICV_IMMORTAL); > + marshallObject(icv.getValue(), out, refMap); > + > + } else if (icv.getClass().equals(MortalCacheEntry.class)) { > + out.writeByte(MAGICNUMBER_ICV_MORTAL); > + marshallObject(icv.getValue(), out, refMap); > + writeUnsignedLong(out, icv.getCreated()); > + out.writeLong(icv.getLifespan()); // could be negative so > should not use unsigned longs > + > + } else if (icv.getClass().equals(TransientCacheEntry.class)) { > + out.writeByte(MAGICNUMBER_ICV_TRANSIENT); > + marshallObject(icv.getValue(), out, refMap); > + writeUnsignedLong(out, icv.getLastUsed()); > + out.writeLong(icv.getMaxIdle()); // could be negative so > should not use unsigned longs > + > + } else if > (icv.getClass().equals(TransientMortalCacheEntry.class)) { > + out.writeByte(MAGICNUMBER_ICV_TRANSIENT_MORTAL); > + marshallObject(icv.getValue(), out, refMap); > + writeUnsignedLong(out, icv.getCreated()); > + out.writeLong(icv.getLifespan()); // could be negative so > should not use unsigned longs > + writeUnsignedLong(out, icv.getLastUsed()); > + out.writeLong(icv.getMaxIdle()); // could be negative so > should not use unsigned longs > + } > + } > + > + > protected void marshallString(String s, ObjectOutput out) throws > IOException { > //StringUtil.saveString(out, s); > out.writeObject(s); > @@ -385,19 +447,16 @@ > MarshalledValue mv = new MarshalledValue(); > mv.readExternal(in); > return mv; > - case MAGICNUMBER_INTERNAL_CACHED_ENTRY: > - Object k = unmarshallObject(in, refMap); > - Object v = unmarshallObject(in, refMap); > - boolean canExpire = in.readBoolean(); > - if (canExpire) { > - long created = readUnsignedLong(in); > - long lifespan = in.readLong(); // could be negative so > should not use unsigned longs > - long lastUsed = readUnsignedLong(in); > - long maxIdle = in.readLong(); // could be negative so > should not use unsigned longs > - return InternalEntryFactory.create(k, v, created, > lifespan, lastUsed, maxIdle); > - } else { > - return InternalEntryFactory.create(k, v); > - } > + case MAGICNUMBER_ICE_IMMORTAL: > + case MAGICNUMBER_ICE_MORTAL: > + case MAGICNUMBER_ICE_TRANSIENT: > + case MAGICNUMBER_ICE_TRANSIENT_MORTAL: > + return unmarshallInternalCacheEntry(magicNumber, in, > refMap); > + case MAGICNUMBER_ICV_IMMORTAL: > + case MAGICNUMBER_ICV_MORTAL: > + case MAGICNUMBER_ICV_TRANSIENT: > + case MAGICNUMBER_ICV_TRANSIENT_MORTAL: > + return unmarshallInternalCacheValue(magicNumber, in, > refMap); > case MAGICNUMBER_REQUEST_IGNORED_RESPONSE: > case MAGICNUMBER_EXTENDED_RESPONSE: > case MAGICNUMBER_EXCEPTION_RESPONSE: > @@ -465,6 +524,49 @@ > throw new IOException("Unknown magic number " + magicNumber); > } > > + private InternalCacheEntry unmarshallInternalCacheEntry(byte > magic, ObjectInput in, UnmarshalledReferences refMap) throws > IOException, ClassNotFoundException { > + Object k = unmarshallObject(in, refMap); > + Object v = unmarshallObject(in, refMap); > + switch (magic) { > + case MAGICNUMBER_ICE_IMMORTAL: > + return InternalEntryFactory.create(k, v); > + case MAGICNUMBER_ICE_MORTAL: > + return InternalEntryFactory.create(k, v, > + readUnsignedLong(in), > (Long) unmarshallObject(in, refMap), > + -1, -1); > + case MAGICNUMBER_ICE_TRANSIENT: > + return InternalEntryFactory.create(k, v, > + -1, -1, > + readUnsignedLong(in), > (Long) unmarshallObject(in, refMap)); > + case MAGICNUMBER_ICE_TRANSIENT_MORTAL: > + return InternalEntryFactory.create(k, v, > + readUnsignedLong(in), > (Long) unmarshallObject(in, refMap), > + readUnsignedLong(in), > (Long) unmarshallObject(in, refMap)); > + } > + throw new IllegalArgumentException("Unknown magic number " + > magic); > + } > + > + private InternalCacheValue unmarshallInternalCacheValue(byte > magic, ObjectInput in, UnmarshalledReferences refMap) throws > IOException, ClassNotFoundException { > + Object v = unmarshallObject(in, refMap); > + switch (magic) { > + case MAGICNUMBER_ICE_IMMORTAL: > + return InternalEntryFactory.createValue(v); > + case MAGICNUMBER_ICE_MORTAL: > + return InternalEntryFactory.createValue(v, > + > readUnsignedLong(in), (Long) unmarshallObject(in, refMap), > + -1, -1); > + case MAGICNUMBER_ICE_TRANSIENT: > + return InternalEntryFactory.createValue(v, > + -1, -1, > + > readUnsignedLong(in), (Long) unmarshallObject(in, refMap)); > + case MAGICNUMBER_ICE_TRANSIENT_MORTAL: > + return InternalEntryFactory.createValue(v, > + > readUnsignedLong(in), (Long) unmarshallObject(in, refMap), > + > readUnsignedLong(in), (Long) unmarshallObject(in, refMap)); > + } > + throw new IllegalArgumentException("Unknown magic number " + > magic); > + } > + > private FastCopyHashMap unmarshallFastCopyHashMap(ObjectInput in, > UnmarshalledReferences refMap) throws IOException, > ClassNotFoundException { > FastCopyHashMap map = new FastCopyHashMap(); > populateFromStream(in, refMap, map); > > _______________________________________________ > infinispan-commits mailing list > infinispan-commits(a)lists.jboss.org > https://lists.jboss.org/mailman/listinfo/infinispan-commits -- Manik Surtani manik(a)jboss.org Lead, Infinispan Lead, JBoss Cache http://www.infinispan.org http://www.jbosscache.org ------------------------------------------------------------------------ _______________________________________________ infinispan-dev mailing list infinispan-dev(a)lists.jboss.org https://lists.jboss.org/mailman/listinfo/infinispan-dev