7:37 a.m.
Author: gaohoward
Date: 2010-11-17 08:37:28 -0500 (Wed, 17 Nov 2010)
New Revision: 9908
Modified:
branches/Branch_Large_Message_Compression/src/main/org/hornetq/utils/GZipUtil.java
branches/Branch_Large_Message_Compression/tests/src/org/hornetq/tests/unit/util/GZipUtilTest.java
Log:
more test
Modified:
branches/Branch_Large_Message_Compression/src/main/org/hornetq/utils/GZipUtil.java
===================================================================
---
branches/Branch_Large_Message_Compression/src/main/org/hornetq/utils/GZipUtil.java 2010-11-17
06:32:48 UTC (rev 9907)
+++
branches/Branch_Large_Message_Compression/src/main/org/hornetq/utils/GZipUtil.java 2010-11-17
13:37:28 UTC (rev 9908)
@@ -13,23 +13,17 @@
package org.hornetq.utils;
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
-import java.io.PipedInputStream;
-import java.io.PipedOutputStream;
import java.util.ArrayList;
import java.util.List;
-import java.util.concurrent.Executor;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import org.hornetq.api.core.HornetQException;
-import org.hornetq.core.logging.Logger;
/**
* A GZipUtil
@@ -41,132 +35,6 @@
public class GZipUtil
{
- private static final Logger log = Logger.getLogger(GZipUtil.class);
-
- /**
- * This will start a GZipOutputStream, using another thread through a Pipe
- * TODO: We would need an inverted GZipInputStream (that would compress on reading) to
avoid creating this thread (through an executor)
- * @param inputStreamParameter
- * @param compress = true if compressing, false if decompressing
- * @return
- * @throws HornetQException
- */
- public static InputStream pipeGZip(final InputStream inputStreamParameter, final
boolean compress, final Executor threadPool) throws HornetQException
- {
- final InputStream input;
- if (compress)
- {
- input = inputStreamParameter;
- }
- else
- {
- try
- {
- input = new GZIPInputStream(new BufferedInputStream(inputStreamParameter));
- }
- catch (IOException e)
- {
- throw new HornetQException(HornetQException.LARGE_MESSAGE_ERROR_BODY,
e.getMessage(), e);
- }
- }
-
- final PipedOutputStream pipedOut = new PipedOutputStream();
- final PipedInputStream pipedInput = new PipedInputStream();
- try
- {
- pipedOut.connect(pipedInput);
- }
- catch (IOException e)
- {
- throw new HornetQException(HornetQException.LARGE_MESSAGE_ERROR_BODY,
e.getMessage(), e);
- }
-
- threadPool.execute(new Runnable()
- {
-
- public void run()
- {
- byte readBytes[] = new byte[1024];
- int size = 0;
-
- try
- {
- OutputStream out;
- if (compress)
- {
- BufferedOutputStream buffOut = new BufferedOutputStream(pipedOut);
- out = new GZIPOutputStream(buffOut);
- }
- else
- {
- out = new BufferedOutputStream(pipedOut);
- }
- while ((size = input.read(readBytes)) > 0)
- {
-// System.out.println("Read " + size + " bytes on
compressing thread");
- out.write(readBytes, 0, size);
- }
- System.out.println("Finished compressing");
- out.close();
- }
- catch (Exception e)
- {
- log.warn(e.getMessage());
- try
- {
- pipedOut.close();
- }
- catch (Exception ignored)
- {
- }
- }
-
- }
- });
-
- return pipedInput;
- }
-
- public static void deZip(final InputStream input, final OutputStream output, final
Executor threadPool) throws HornetQException
- {
- threadPool.execute(new Runnable()
- {
-
- public void run()
- {
- byte readBytes[] = new byte[1024];
- int size = 0;
-
- OutputStream out = null;
-
- try
- {
- BufferedOutputStream buffOut = new BufferedOutputStream(output);
- out = new GZIPOutputStream(buffOut);
- while ((size = input.read(readBytes)) > 0)
- {
- System.out.println("Read " + size + " bytes on
compressing thread");
- out.write(readBytes, 0, size);
- }
- System.out.println("Finished compressing");
- out.close();
- }
- catch (Exception e)
- {
- log.warn(e.getMessage());
- try
- {
- out.close();
- }
- catch (Exception ignored)
- {
- }
- }
-
- }
- });
- }
-
public static InputStream createZipInputStream(InputStream input) throws
HornetQException
{
try
@@ -223,13 +91,15 @@
public void close() throws IOException
{
zipIn = new GZIPInputStream(receiver);
- int b = zipIn.read();
- int counter = 0;
- while (b != -1)
+ byte[] buffer = new byte[1024];
+ int n = zipIn.read(buffer);
+ while (n != -1)
{
- target.write(b);
- counter++;
- b = zipIn.read();
+ if (n > 0)
+ {
+ target.write(buffer, 0, n);
+ }
+ n = zipIn.read(buffer);
}
target.close();
}
Modified:
branches/Branch_Large_Message_Compression/tests/src/org/hornetq/tests/unit/util/GZipUtilTest.java
===================================================================
---
branches/Branch_Large_Message_Compression/tests/src/org/hornetq/tests/unit/util/GZipUtilTest.java 2010-11-17
06:32:48 UTC (rev 9907)
+++
branches/Branch_Large_Message_Compression/tests/src/org/hornetq/tests/unit/util/GZipUtilTest.java 2010-11-17
13:37:28 UTC (rev 9908)
@@ -16,12 +16,15 @@
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
+import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Random;
import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
import org.hornetq.core.logging.Logger;
import org.hornetq.tests.util.UnitTestCase;
+import org.hornetq.utils.GZipUtil;
import org.hornetq.utils.GZipUtil.GZipPipe;
/**
@@ -117,6 +120,116 @@
zippedFile.delete();
}
+ //create a 10M file, zip it into another file
+ //load it into an input stream and feed to the
+ //GZipOutput. Then compare the result
+ public void testUnzipFunction() throws Exception
+ {
+ this.recreateDirectory(this.getTestDir());
+
+ File originalFile = new File(this.getTestDir(),
"gzipUtilTest_file1.txt");
+ File zippedFile = new File(this.getTestDir(), "gzipUtilTest_file1.zip");
+ File unzippedFile = new File(this.getTestDir(),
"gzipUtilTest_unzipped.txt");
+
+ FileOutputStream originalOut = new FileOutputStream(originalFile);
+ FileOutputStream zippedOut = new FileOutputStream(zippedFile);
+
+ //now create the file
+ Random r = new Random();
+ final int size = 1024 * 10;
+ byte[] writeBuffer = new byte[1024];
+
+ for (int i = 0; i < size; i++)
+ {
+ int b = r.nextInt(256);
+ for (int j = 0; j < 1024; j++)
+ {
+ writeBuffer[j] = (byte)b;
+ }
+ originalOut.write(writeBuffer);
+ }
+ originalOut.close();
+
+ log.info("file created.");
+
+ //now zip it
+ GZIPOutputStream gzipOut = new GZIPOutputStream(zippedOut);
+ FileInputStream originalIn = new FileInputStream(originalFile);
+
+ byte[] buffer = new byte[2048];
+
+ int n = originalIn.read(buffer);
+ while (n != -1)
+ {
+ if (n > 0)
+ {
+ gzipOut.write(buffer, 0, n);
+ }
+ n = originalIn.read(buffer);
+ }
+ gzipOut.close();
+
+ log.info("file zipped.");
+
+ //get a zipped input stream
+ FileInputStream zippedInput = new FileInputStream(zippedFile);
+
+ FileOutputStream unzipOut = new FileOutputStream(unzippedFile);
+
+ OutputStream newOut = GZipUtil.createZipOutputStream(unzipOut);
+
+ n = zippedInput.read(buffer);
+ while (n != -1)
+ {
+ if (n > 0)
+ {
+ newOut.write(buffer, 0, n);
+ }
+ n = zippedInput.read(buffer);
+ }
+ newOut.close();
+
+ log.info("file unzipped");
+
+ //compare original and unzipped
+ FileInputStream originalInput = new FileInputStream(originalFile);
+ FileInputStream unzippedInput = new FileInputStream(unzippedFile);
+
+ ArrayList<Integer> fromZip = new ArrayList<Integer>();
+ ArrayList<Integer> original = new ArrayList<Integer>();
+
+ byte[] readBuffer = new byte[2048];
+ int count = originalInput.read(readBuffer);
+
+ while (count != -1)
+ {
+ for (int i = 0; i < count; i++)
+ {
+ original.add(readBuffer[i] & 0xFF);
+ }
+ count = originalInput.read(readBuffer);
+ }
+ originalInput.close();
+
+ count = unzippedInput.read(readBuffer);
+
+ while (count != -1)
+ {
+ for (int i = 0; i < count; i++)
+ {
+ fromZip.add(readBuffer[i] & 0xFF);
+ }
+ count = unzippedInput.read(readBuffer);
+ }
+ unzippedInput.close();
+
+ compareByteArray(fromZip, original);
+
+ originalFile.delete();
+ zippedFile.delete();
+ unzippedFile.delete();
+ }
+
private void compareByteArray(ArrayList<Integer> b1, ArrayList<Integer>
b2)
{
assertEquals(b1.size(), b2.size());