Author: manik.surtani(a)jboss.com Date: 2007-11-07 22:43:39 -0500 (Wed, 07 Nov 2007) New Revision: 4737 Modified: core/trunk/src/main/java/org/jboss/cache/marshall/CacheMarshaller200.java core/trunk/src/main/java/org/jboss/cache/marshall/CacheMarshaller210.java core/trunk/src/main/java/org/jboss/cache/marshall/VersionAwareMarshaller.java core/trunk/src/test/java/org/jboss/cache/marshall/CacheMarshaller210Test.java core/trunk/src/test/java/org/jboss/cache/marshall/CacheMarshallerTestBase.java Log: JBCACHE-1211 - somewhat better fix, using variable length ints and longs Modified: core/trunk/src/main/java/org/jboss/cache/marshall/CacheMarshaller200.java =================================================================== --- core/trunk/src/main/java/org/jboss/cache/marshall/CacheMarshaller200.java 2007-11-08 02:00:52 UTC (rev 4736) +++ core/trunk/src/main/java/org/jboss/cache/marshall/CacheMarshaller200.java 2007-11-08 03:43:39 UTC (rev 4737) @@ -302,7 +302,7 @@ else if (o instanceof DefaultDataVersion) { out.writeByte(MAGICNUMBER_DEFAULT_DATA_VERSION); - out.writeLong(((DefaultDataVersion) o).getRawVersion()); + marshallDefaultDataVersion((DefaultDataVersion) o, out); } else if (o.getClass().equals(ArrayList.class)) { @@ -466,7 +466,7 @@ private void marshallCollection(Collection c, ObjectOutputStream out, Map refMap) throws Exception { - out.writeInt(c.size()); + writeUnsignedInt(out, c.size()); for (Object o : c) { marshallObject(o, out, refMap); @@ -476,7 +476,7 @@ private void marshallMap(Map map, ObjectOutputStream out, Map<Object, Integer> refMap) throws Exception { int mapSize = map.size(); - out.writeInt(mapSize); + writeUnsignedInt(out, mapSize); if (mapSize == 0) return; for (Map.Entry me : (Set<Map.Entry>) map.entrySet()) @@ -548,7 +548,7 @@ retVal = unmarshallIpAddress(in); return retVal; case MAGICNUMBER_DEFAULT_DATA_VERSION: - retVal = new DefaultDataVersion(in.readLong()); + retVal = unmarshallDefaultDataVersion(in); return retVal; case MAGICNUMBER_ARRAY_LIST: return unmarshallArrayList(in, refMap); @@ -680,7 +680,7 @@ private List unmarshallArrayList(ObjectInputStream in, Map refMap) throws Exception { - int listSize = in.readInt(); + int listSize = readUnsignedInt(in); List list = new ArrayList(listSize); populateFromStream(in, refMap, list, listSize); return list; @@ -689,7 +689,7 @@ private List unmarshallLinkedList(ObjectInputStream in, Map refMap) throws Exception { List list = new LinkedList(); - populateFromStream(in, refMap, list, in.readInt()); + populateFromStream(in, refMap, list, readUnsignedInt(in)); return list; } @@ -704,8 +704,7 @@ { // read in as a HashMap first Map m = unmarshallHashMap(in, refMap); - MapCopy mc = new MapCopy(m); - return mc; + return new MapCopy(m); } private Map unmarshallTreeMap(ObjectInputStream in, Map refMap) throws Exception @@ -731,7 +730,7 @@ private void populateFromStream(ObjectInputStream in, Map refMap, Map mapToPopulate) throws Exception { - int size = in.readInt(); + int size = readUnsignedInt(in); for (int i = 0; i < size; i++) { mapToPopulate.put(unmarshallObject(in, refMap), unmarshallObject(in, refMap)); @@ -740,7 +739,7 @@ private void populateFromStream(ObjectInputStream in, Map refMap, Set setToPopulate) throws Exception { - int size = in.readInt(); + int size = readUnsignedInt(in); for (int i = 0; i < size; i++) { setToPopulate.add(unmarshallObject(in, refMap)); @@ -755,6 +754,16 @@ } } + protected void marshallDefaultDataVersion(DefaultDataVersion ddv, ObjectOutputStream out) throws Exception + { + writeUnsignedLong(out, ddv.getRawVersion()); + } + + protected DefaultDataVersion unmarshallDefaultDataVersion(ObjectInputStream in) throws Exception + { + return new DefaultDataVersion(readUnsignedLong(in)); + } + /** * Reads a reference from a given stream. * @param in the stream to read from @@ -776,4 +785,24 @@ { out.writeShort(reference); } + + protected int readUnsignedInt(ObjectInputStream in) throws IOException + { + return in.readInt(); + } + + protected void writeUnsignedInt(ObjectOutputStream out, int i) throws IOException + { + out.writeInt(i); + } + + protected long readUnsignedLong(ObjectInputStream in) throws IOException + { + return in.readLong(); + } + + protected void writeUnsignedLong(ObjectOutputStream out, long i) throws IOException + { + out.writeLong(i); + } } Modified: core/trunk/src/main/java/org/jboss/cache/marshall/CacheMarshaller210.java =================================================================== --- core/trunk/src/main/java/org/jboss/cache/marshall/CacheMarshaller210.java 2007-11-08 02:00:52 UTC (rev 4736) +++ core/trunk/src/main/java/org/jboss/cache/marshall/CacheMarshaller210.java 2007-11-08 03:43:39 UTC (rev 4737) @@ -8,8 +8,12 @@ /** * An evolution of {@link org.jboss.cache.marshall.CacheMarshaller200}, created to fix <a href="http://jira.jboss.org/jira/browse/JBCACHE-1211">JBCACH...;. + * <p/> + * To prevent ints taking too much space, they are written as variable-length ints. Details <a href="http://lucene.apache.org/java/docs/fileformats.html#VInt"... on VInts. * * @author Manik Surtani + * @see <a href="http://lucene.apache.org/java/docs/fileformats.html#VInt"... + * @see <a href="http://lucene.apache.org/java/docs/fileformats.html#VLong"... * @since 2.1.0 */ public class CacheMarshaller210 extends CacheMarshaller200 @@ -21,7 +25,8 @@ /** * This version of writeReference is written to solve JBCACHE-1211, where references are encoded as ints rather than shorts. - * @param out stream to write to + * + * @param out stream to write to * @param reference reference to write * @throws IOException propagated from OOS * @see <a href="http://jira.jboss.org/jira/browse/JBCACHE-1211">JBCACH... @@ -29,11 +34,12 @@ @Override protected void writeReference(ObjectOutputStream out, int reference) throws IOException { - out.writeInt(reference); + writeUnsignedInt(out, reference); } /** * This version of readReference is written to solve JBCACHE-1211, where references are encoded as ints rather than shorts. + * * @param in stream to read from * @return reference * @throws IOException propagated from OUS @@ -42,6 +48,78 @@ @Override protected int readReference(ObjectInputStream in) throws IOException { - return in.readInt(); + return readUnsignedInt(in); } + + /** + * Reads an int stored in variable-length format. Reads between one and + * five bytes. Smaller values take fewer bytes. Negative numbers are not + * supported. + */ + @Override + protected int readUnsignedInt(ObjectInputStream in) throws IOException + { + byte b = in.readByte(); + int i = b & 0x7F; + for (int shift = 7; (b & 0x80) != 0; shift += 7) + { + b = in.readByte(); + i |= (b & 0x7FL) << shift; + } + return i; + } + + /** + * Writes an int in a variable-length format. Writes between one and + * five bytes. Smaller values take fewer bytes. Negative numbers are not + * supported. + * + * @param i int to write + */ + @Override + protected void writeUnsignedInt(ObjectOutputStream out, int i) throws IOException + { + while ((i & ~0x7F) != 0) + { + out.writeByte((byte) ((i & 0x7f) | 0x80)); + i >>>= 7; + } + out.writeByte((byte) i); + } + + /** + * Reads an int stored in variable-length format. Reads between one and + * nine bytes. Smaller values take fewer bytes. Negative numbers are not + * supported. + */ + @Override + protected long readUnsignedLong(ObjectInputStream in) throws IOException + { + byte b = in.readByte(); + long i = b & 0x7F; + for (int shift = 7; (b & 0x80) != 0; shift += 7) + { + b = in.readByte(); + i |= (b & 0x7FL) << shift; + } + return i; + } + + /** + * Writes an int in a variable-length format. Writes between one and + * nine bytes. Smaller values take fewer bytes. Negative numbers are not + * supported. + * + * @param i int to write + */ + @Override + protected void writeUnsignedLong(ObjectOutputStream out, long i) throws IOException + { + while ((i & ~0x7F) != 0) + { + out.writeByte((byte) ((i & 0x7f) | 0x80)); + i >>>= 7; + } + out.writeByte((byte) i); + } } Modified: core/trunk/src/main/java/org/jboss/cache/marshall/VersionAwareMarshaller.java =================================================================== --- core/trunk/src/main/java/org/jboss/cache/marshall/VersionAwareMarshaller.java 2007-11-08 02:00:52 UTC (rev 4736) +++ core/trunk/src/main/java/org/jboss/cache/marshall/VersionAwareMarshaller.java 2007-11-08 03:43:39 UTC (rev 4737) @@ -191,7 +191,7 @@ switch (versionId) { case VERSION_200: - marshaller = marshallers.get(VERSION_210); + marshaller = marshallers.get(VERSION_200); if (marshaller == null) { marshaller = new CacheMarshaller200(manager, defaultInactive, useRegionBasedMarshalling); Modified: core/trunk/src/test/java/org/jboss/cache/marshall/CacheMarshaller210Test.java =================================================================== --- core/trunk/src/test/java/org/jboss/cache/marshall/CacheMarshaller210Test.java 2007-11-08 02:00:52 UTC (rev 4736) +++ core/trunk/src/test/java/org/jboss/cache/marshall/CacheMarshaller210Test.java 2007-11-08 03:43:39 UTC (rev 4737) @@ -3,6 +3,11 @@ import org.jboss.cache.Fqn; import org.testng.annotations.Test; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; import java.util.HashMap; import java.util.Map; @@ -40,4 +45,62 @@ { doMapTest(500000); } + + public void testVInts() throws IOException + { + + CacheMarshaller210 cm210 = (CacheMarshaller210) marshaller.defaultMarshaller; + CacheMarshaller200 cm200 = (CacheMarshaller200) marshaller.getMarshaller(20); + int[] ints = {2, 100, 500, 12000, 20000, 500000, 2000000, Integer.MAX_VALUE}; + + for (int i : ints) + { + System.out.println("CM200: Number of bytes (i="+i+") : " + getAndTestSize(cm200, i)); + System.out.println("CM210: Number of bytes (i="+i+") : " + getAndTestSize(cm210, i)); + } + } + + public void testVLongs() throws IOException + { + + CacheMarshaller210 cm210 = (CacheMarshaller210) marshaller.defaultMarshaller; + CacheMarshaller200 cm200 = (CacheMarshaller200) marshaller.getMarshaller(20); + long[] ints = {2, 100, 500, 12000, 20000, 500000, 2000000, Integer.MAX_VALUE, (long) Integer.MAX_VALUE + 500000L, Long.MAX_VALUE}; + + for (long i : ints) + { + System.out.println("CM200: Number of bytes (i="+i+") : " + getAndTestSize(cm200, i)); + System.out.println("CM210: Number of bytes (i="+i+") : " + getAndTestSize(cm210, i)); + } + } + + private int getAndTestSize(CacheMarshaller200 m, int i) throws IOException + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ObjectOutputStream oos = new ObjectOutputStream(baos); + m.writeUnsignedInt(oos, i); + oos.flush(); + oos.close(); + baos.flush(); + baos.close(); + byte[] bytes = baos.toByteArray(); + int byteL = bytes.length; + assert i == m.readUnsignedInt(new ObjectInputStream(new ByteArrayInputStream(bytes))); + return byteL; + } + + private int getAndTestSize(CacheMarshaller200 m, long i) throws IOException + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ObjectOutputStream oos = new ObjectOutputStream(baos); + m.writeUnsignedLong(oos, i); + oos.flush(); + oos.close(); + baos.flush(); + baos.close(); + byte[] bytes = baos.toByteArray(); + int byteL = bytes.length; + assert i == m.readUnsignedLong(new ObjectInputStream(new ByteArrayInputStream(bytes))); + return byteL; + } } Modified: core/trunk/src/test/java/org/jboss/cache/marshall/CacheMarshallerTestBase.java =================================================================== --- core/trunk/src/test/java/org/jboss/cache/marshall/CacheMarshallerTestBase.java 2007-11-08 02:00:52 UTC (rev 4736) +++ core/trunk/src/test/java/org/jboss/cache/marshall/CacheMarshallerTestBase.java 2007-11-08 03:43:39 UTC (rev 4737) @@ -92,7 +92,7 @@ assertEquals(expectedMarshallerClass, marshaller.getMarshaller(1).getClass()); assertEquals(expectedMarshallerClass, marshaller.getMarshaller(-1).getClass()); assertEquals(expectedMarshallerClass, marshaller.getMarshaller(0).getClass()); - assertEquals(expectedMarshallerClass, marshaller.getMarshaller(20).getClass()); + assertEquals(CacheMarshaller200.class, marshaller.getMarshaller(20).getClass()); assertEquals("One marshaller should be in the map by this stage", 1, marshaller.marshallers.size()); }