[hibernate-commits] Hibernate SVN: r14727 - search/trunk/src/java/org/hibernate/search/util.

[hibernate-commits at lists.jboss.org]

Author: sannegrinovero
Date: 2008-06-03 14:12:26 -0400 (Tue, 03 Jun 2008)
New Revision: 14727

Modified:
   search/trunk/src/java/org/hibernate/search/util/FileHelper.java
Log:
HSEARCH-205 Out of Memory on copy of large indexes

Modified: search/trunk/src/java/org/hibernate/search/util/FileHelper.java
===================================================================
--- search/trunk/src/java/org/hibernate/search/util/FileHelper.java	2008-06-03 08:24:26 UTC (rev 14726)
+++ search/trunk/src/java/org/hibernate/search/util/FileHelper.java	2008-06-03 18:12:26 UTC (rev 14727)
@@ -10,16 +10,29 @@
 import java.util.Arrays;
 import java.nio.channels.FileChannel;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 /**
  * @author Emmanuel Bernard
+ * @author Sanne Grinovero
  */
 public abstract class FileHelper {
+	
 	private static final int FAT_PRECISION = 2000;
+	private static final long DEFAULT_CHUNK_SIZE = 16 * 1024 * 1024; // 16 MB
+	private static final Logger log = LoggerFactory.getLogger( FileHelper.class );
+	
+	public static void synchronize(File source, File destination, boolean smart) throws IOException {
+		synchronize( source, destination, smart, DEFAULT_CHUNK_SIZE );
+	}
 
-	public static void synchronize(File source, File destination, boolean smart) throws IOException {
+	public static void synchronize(File source, File destination, boolean smart, long chunkSize) throws IOException {
 		if ( source.isDirectory() ) {
 			if ( ! destination.exists() ) {
-				destination.mkdirs();
+				if ( ! destination.mkdirs() ){
+					throw new IOException("Could not create path " + destination);
+				}
 			}
 			else if ( ! destination.isDirectory() ) {
 				throw new IOException("Source and Destination not of the same type:"
@@ -39,7 +52,7 @@
 			for (String fileName : sources) {
 				File srcFile = new File(source, fileName);
 				File destFile = new File(destination, fileName);
-				synchronize( srcFile, destFile, smart );
+				synchronize( srcFile, destFile, smart, chunkSize );
 			}
 		}
 		else {
@@ -51,36 +64,61 @@
 				long dts = destination.lastModified() / FAT_PRECISION;
 				//do not copy if smart and same timestamp and same length
 				if ( !smart || sts == 0 || sts != dts || source.length() != destination.length() ) {
-					copyFile(source, destination);
+					copyFile(source, destination, chunkSize);
 				}
 			}
 			else {
-				copyFile(source, destination);
+				copyFile(source, destination, chunkSize);
 			}
 		}
 	}
 
-	private static void copyFile(File srcFile, File destFile) throws IOException {
+	private static void copyFile(File srcFile, File destFile, long chunkSize) throws IOException {
 		FileInputStream is = null;
 		FileOutputStream os = null;
+		long startTime = System.currentTimeMillis();
 		try {
 			is = new FileInputStream(srcFile);
 			FileChannel iChannel = is.getChannel();
 			os = new FileOutputStream( destFile, false );
 			FileChannel oChannel = os.getChannel();
-			oChannel.transferFrom( iChannel, 0, srcFile.length() );
+			long doneBytes = 0L;
+			long todoBytes = srcFile.length();
+			while ( todoBytes != 0L ) {
+				long iterationBytes = Math.min( todoBytes, chunkSize );
+				long transferredLength = oChannel.transferFrom( iChannel, doneBytes, iterationBytes );
+				if ( iterationBytes != transferredLength ) {
+					throw new IOException( "Error during file transfer: expected "
+							+ iterationBytes + " bytes, only "+ transferredLength + " bytes copied." );
+				}
+				doneBytes += transferredLength;
+				todoBytes -= transferredLength;
+			}
 		}
 		finally {
 			if (is != null) is.close();
 			if (os != null) os.close();
 		}
-		destFile.setLastModified( srcFile.lastModified() );
+		long endTime = System.currentTimeMillis() - startTime;
+		System.out.println("copied in "+ endTime + " ms.");
+		boolean successTimestampOp = destFile.setLastModified( srcFile.lastModified() );
+		if ( ! successTimestampOp ) {
+			log.warn( "Could not change timestamp for " + destFile + 
+					". Index synchronization may be slow." );
+		}
 	}
 
 	public static void delete(File file) {
 		if ( file.isDirectory() ) {
-			for ( File subFile : file.listFiles() ) delete( subFile );
+			for ( File subFile : file.listFiles() ) {
+				delete( subFile );
+			}
 		}
-		if ( file.exists() ) file.delete();
+		if ( file.exists() ) {
+			if ( ! file.delete() ) {
+				log.error( "Could not delete " + file );
+			}
+		}
 	}
+	
 }