Author: gaohoward Date: 2010-11-17 08:37:28 -0500 (Wed, 17 Nov 2010) New Revision: 9908 Modified: branches/Branch_Large_Message_Compression/src/main/org/hornetq/utils/GZipUtil.java branches/Branch_Large_Message_Compression/tests/src/org/hornetq/tests/unit/util/GZipUtilTest.java Log: more test Modified: branches/Branch_Large_Message_Compression/src/main/org/hornetq/utils/GZipUtil.java =================================================================== --- branches/Branch_Large_Message_Compression/src/main/org/hornetq/utils/GZipUtil.java 2010-11-17 06:32:48 UTC (rev 9907) +++ branches/Branch_Large_Message_Compression/src/main/org/hornetq/utils/GZipUtil.java 2010-11-17 13:37:28 UTC (rev 9908) @@ -13,23 +13,17 @@ package org.hornetq.utils; -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.io.PipedInputStream; -import java.io.PipedOutputStream; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.Executor; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; import org.hornetq.api.core.HornetQException; -import org.hornetq.core.logging.Logger; /** * A GZipUtil @@ -41,132 +35,6 @@ public class GZipUtil { - private static final Logger log = Logger.getLogger(GZipUtil.class); - - /** - * This will start a GZipOutputStream, using another thread through a Pipe - * TODO: We would need an inverted GZipInputStream (that would compress on reading) to avoid creating this thread (through an executor) - * @param inputStreamParameter - * @param compress = true if compressing, false if decompressing - * @return - * @throws HornetQException - */ - public static InputStream pipeGZip(final InputStream inputStreamParameter, final boolean compress, final Executor threadPool) throws HornetQException - { - final InputStream input; - if (compress) - { - input = inputStreamParameter; - } - else - { - try - { - input = new GZIPInputStream(new BufferedInputStream(inputStreamParameter)); - } - catch (IOException e) - { - throw new HornetQException(HornetQException.LARGE_MESSAGE_ERROR_BODY, e.getMessage(), e); - } - } - - final PipedOutputStream pipedOut = new PipedOutputStream(); - final PipedInputStream pipedInput = new PipedInputStream(); - try - { - pipedOut.connect(pipedInput); - } - catch (IOException e) - { - throw new HornetQException(HornetQException.LARGE_MESSAGE_ERROR_BODY, e.getMessage(), e); - } - - threadPool.execute(new Runnable() - { - - public void run() - { - byte readBytes[] = new byte[1024]; - int size = 0; - - try - { - OutputStream out; - if (compress) - { - BufferedOutputStream buffOut = new BufferedOutputStream(pipedOut); - out = new GZIPOutputStream(buffOut); - } - else - { - out = new BufferedOutputStream(pipedOut); - } - while ((size = input.read(readBytes)) > 0) - { -// System.out.println("Read " + size + " bytes on compressing thread"); - out.write(readBytes, 0, size); - } - System.out.println("Finished compressing"); - out.close(); - } - catch (Exception e) - { - log.warn(e.getMessage()); - try - { - pipedOut.close(); - } - catch (Exception ignored) - { - } - } - - } - }); - - return pipedInput; - } - - public static void deZip(final InputStream input, final OutputStream output, final Executor threadPool) throws HornetQException - { - threadPool.execute(new Runnable() - { - - public void run() - { - byte readBytes[] = new byte[1024]; - int size = 0; - - OutputStream out = null; - - try - { - BufferedOutputStream buffOut = new BufferedOutputStream(output); - out = new GZIPOutputStream(buffOut); - while ((size = input.read(readBytes)) > 0) - { - System.out.println("Read " + size + " bytes on compressing thread"); - out.write(readBytes, 0, size); - } - System.out.println("Finished compressing"); - out.close(); - } - catch (Exception e) - { - log.warn(e.getMessage()); - try - { - out.close(); - } - catch (Exception ignored) - { - } - } - - } - }); - } - public static InputStream createZipInputStream(InputStream input) throws HornetQException { try @@ -223,13 +91,15 @@ public void close() throws IOException { zipIn = new GZIPInputStream(receiver); - int b = zipIn.read(); - int counter = 0; - while (b != -1) + byte[] buffer = new byte[1024]; + int n = zipIn.read(buffer); + while (n != -1) { - target.write(b); - counter++; - b = zipIn.read(); + if (n > 0) + { + target.write(buffer, 0, n); + } + n = zipIn.read(buffer); } target.close(); } Modified: branches/Branch_Large_Message_Compression/tests/src/org/hornetq/tests/unit/util/GZipUtilTest.java =================================================================== --- branches/Branch_Large_Message_Compression/tests/src/org/hornetq/tests/unit/util/GZipUtilTest.java 2010-11-17 06:32:48 UTC (rev 9907) +++ branches/Branch_Large_Message_Compression/tests/src/org/hornetq/tests/unit/util/GZipUtilTest.java 2010-11-17 13:37:28 UTC (rev 9908) @@ -16,12 +16,15 @@ import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; +import java.io.OutputStream; import java.util.ArrayList; import java.util.Random; import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; import org.hornetq.core.logging.Logger; import org.hornetq.tests.util.UnitTestCase; +import org.hornetq.utils.GZipUtil; import org.hornetq.utils.GZipUtil.GZipPipe; /** @@ -117,6 +120,116 @@ zippedFile.delete(); } + //create a 10M file, zip it into another file + //load it into an input stream and feed to the + //GZipOutput. Then compare the result + public void testUnzipFunction() throws Exception + { + this.recreateDirectory(this.getTestDir()); + + File originalFile = new File(this.getTestDir(), "gzipUtilTest_file1.txt"); + File zippedFile = new File(this.getTestDir(), "gzipUtilTest_file1.zip"); + File unzippedFile = new File(this.getTestDir(), "gzipUtilTest_unzipped.txt"); + + FileOutputStream originalOut = new FileOutputStream(originalFile); + FileOutputStream zippedOut = new FileOutputStream(zippedFile); + + //now create the file + Random r = new Random(); + final int size = 1024 * 10; + byte[] writeBuffer = new byte[1024]; + + for (int i = 0; i < size; i++) + { + int b = r.nextInt(256); + for (int j = 0; j < 1024; j++) + { + writeBuffer[j] = (byte)b; + } + originalOut.write(writeBuffer); + } + originalOut.close(); + + log.info("file created."); + + //now zip it + GZIPOutputStream gzipOut = new GZIPOutputStream(zippedOut); + FileInputStream originalIn = new FileInputStream(originalFile); + + byte[] buffer = new byte[2048]; + + int n = originalIn.read(buffer); + while (n != -1) + { + if (n > 0) + { + gzipOut.write(buffer, 0, n); + } + n = originalIn.read(buffer); + } + gzipOut.close(); + + log.info("file zipped."); + + //get a zipped input stream + FileInputStream zippedInput = new FileInputStream(zippedFile); + + FileOutputStream unzipOut = new FileOutputStream(unzippedFile); + + OutputStream newOut = GZipUtil.createZipOutputStream(unzipOut); + + n = zippedInput.read(buffer); + while (n != -1) + { + if (n > 0) + { + newOut.write(buffer, 0, n); + } + n = zippedInput.read(buffer); + } + newOut.close(); + + log.info("file unzipped"); + + //compare original and unzipped + FileInputStream originalInput = new FileInputStream(originalFile); + FileInputStream unzippedInput = new FileInputStream(unzippedFile); + + ArrayList<Integer> fromZip = new ArrayList<Integer>(); + ArrayList<Integer> original = new ArrayList<Integer>(); + + byte[] readBuffer = new byte[2048]; + int count = originalInput.read(readBuffer); + + while (count != -1) + { + for (int i = 0; i < count; i++) + { + original.add(readBuffer[i] & 0xFF); + } + count = originalInput.read(readBuffer); + } + originalInput.close(); + + count = unzippedInput.read(readBuffer); + + while (count != -1) + { + for (int i = 0; i < count; i++) + { + fromZip.add(readBuffer[i] & 0xFF); + } + count = unzippedInput.read(readBuffer); + } + unzippedInput.close(); + + compareByteArray(fromZip, original); + + originalFile.delete(); + zippedFile.delete(); + unzippedFile.delete(); + } + private void compareByteArray(ArrayList<Integer> b1, ArrayList<Integer> b2) { assertEquals(b1.size(), b2.size());