Author: sannegrinovero Date: 2008-06-05 19:41:49 -0400 (Thu, 05 Jun 2008) New Revision: 14744 Added: search/trunk/src/test/org/hibernate/search/test/directoryProvider/DirectoryProviderHelperTest.java Modified: search/trunk/doc/reference/en/modules/configuration.xml search/trunk/src/java/org/hibernate/search/store/DirectoryProviderHelper.java search/trunk/src/java/org/hibernate/search/store/FSMasterDirectoryProvider.java search/trunk/src/java/org/hibernate/search/store/FSSlaveDirectoryProvider.java search/trunk/src/java/org/hibernate/search/util/FileHelper.java search/trunk/src/test/org/hibernate/search/test/FSDirectoryTest.java Log: HSEARCH-206 Enable configuration parameters for file transfer options in JMS DirectoryProviders Modified: search/trunk/doc/reference/en/modules/configuration.xml =================================================================== --- search/trunk/doc/reference/en/modules/configuration.xml 2008-06-05 23:21:58 UTC (rev 14743) +++ search/trunk/doc/reference/en/modules/configuration.xml 2008-06-05 23:41:49 UTC (rev 14744) @@ -52,7 +52,10 @@ 3600 seconds - 60 minutes).</para><para>Note that the copy is based on an incremental copy mechanism reducing the average copy time.</para><para>DirectoryProvider typically used on the master - node in a JMS back end cluster.</para></entry> + node in a JMS back end cluster.</para><para>The <literal> + buffer_size_on_copy</literal> optimum depends + on your operating system and available RAM; most people reported + good results using values between 16 and 64MB.</para></entry> <entry><para><literal>indexBase</literal>: Base directory</para><para><literal>indexName</literal>: override @@ -63,7 +66,10 @@ The actual source directory name being <filename>&lt;sourceBase&gt;/&lt;source&gt;</filename> </para><para><literal>refresh</literal>: refresh period in second - (the copy will take place every refresh seconds).</para></entry> + (the copy will take place every refresh seconds).</para><para> + <literal>buffer_size_on_copy</literal>: The amount of + MegaBytes to move in a single low level copy instruction; + defaults to 16MB.</para></entry> </row> <row> @@ -77,7 +83,10 @@ information (default 3600 seconds - 60 minutes).</para><para>Note that the copy is based on an incremental copy mechanism reducing the average copy time.</para><para>DirectoryProvider typically - used on slave nodes using a JMS back end.</para></entry> + used on slave nodes using a JMS back end.</para><para>The <literal> + buffer_size_on_copy</literal> optimum depends + on your operating system and available RAM; most people reported + good results using values between 16 and 64MB.</para></entry> <entry><para><literal>indexBase</literal>: Base directory</para><para><literal>indexName</literal>: override @@ -88,7 +97,10 @@ The actual source directory name being <filename>&lt;sourceBase&gt;/&lt;source&gt;</filename> </para><para><literal>refresh</literal>: refresh period in second - (the copy will take place every refresh seconds).</para></entry> + (the copy will take place every refresh seconds).</para><para> + <literal>buffer_size_on_copy</literal>: The amount of + MegaBytes to move in a single low level copy instruction; + defaults to 16MB.</para></entry> </row> <row> Modified: search/trunk/src/java/org/hibernate/search/store/DirectoryProviderHelper.java =================================================================== --- search/trunk/src/java/org/hibernate/search/store/DirectoryProviderHelper.java 2008-06-05 23:21:58 UTC (rev 14743) +++ search/trunk/src/java/org/hibernate/search/store/DirectoryProviderHelper.java 2008-06-05 23:41:49 UTC (rev 14744) @@ -6,6 +6,7 @@ import java.io.IOException; import org.hibernate.search.SearchException; +import org.hibernate.search.util.FileHelper; import org.hibernate.annotations.common.util.StringHelper; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.index.IndexReader; @@ -23,6 +24,7 @@ private static final Logger log = LoggerFactory.getLogger( DirectoryProviderHelper.class ); private static final String ROOTINDEX_PROP_NAME = "sourceBase"; private static final String RELATIVEINDEX_PROP_NAME = "source"; + public static final String COPYBUFFERSIZE_PROP_NAME = "buffer_size_on_copy"; /** * Build a directory name out of a root and relative path, guessing the significant part @@ -142,5 +144,31 @@ log.debug( "Refresh period: {} seconds", period ); return period * 1000; //per second } + + /** + * Users may configure the number of KB to use as + * "chunk size" for large file copy operations performed + * by DirectoryProviders. + * @param directoryProviderName + * @param properties + * @return the number of Bytes to use as "chunk size" in file copy operations. + */ + public static long getCopyBufferSize(String directoryProviderName, Properties properties) { + String value = properties.getProperty( COPYBUFFERSIZE_PROP_NAME ); + long size = FileHelper.DEFAULT_COPY_BUFFER_SIZE; + if ( value != null ) { + try { + size = Long.parseLong( value ) * 1024 * 1024; //from MB to B. + } catch (NumberFormatException nfe) { + throw new SearchException( "Unable to initialize index " + + directoryProviderName +"; "+ COPYBUFFERSIZE_PROP_NAME + " is not numeric.", nfe ); + } + if ( size <= 0 ) { + throw new SearchException( "Unable to initialize index " + + directoryProviderName +"; "+ COPYBUFFERSIZE_PROP_NAME + " needs to be greater than zero."); + } + } + return size; + } } Modified: search/trunk/src/java/org/hibernate/search/store/FSMasterDirectoryProvider.java =================================================================== --- search/trunk/src/java/org/hibernate/search/store/FSMasterDirectoryProvider.java 2008-06-05 23:21:58 UTC (rev 14743) +++ search/trunk/src/java/org/hibernate/search/store/FSMasterDirectoryProvider.java 2008-06-05 23:41:49 UTC (rev 14744) @@ -38,6 +38,7 @@ private String indexName; private Timer timer; private SearchFactoryImplementor searchFactory; + private long copyChunkSize; //variables needed between initialize and start private File sourceDir; @@ -60,6 +61,7 @@ catch (IOException e) { throw new SearchException( "Unable to initialize index: " + directoryProviderName, e ); } + copyChunkSize = DirectoryProviderHelper.getCopyBufferSize( directoryProviderName, properties ); this.searchFactory = searchFactoryImplementor; } @@ -79,7 +81,7 @@ } String currentString = Integer.valueOf( current ).toString(); File subDir = new File( sourceDir, currentString ); - FileHelper.synchronize( indexDir, subDir, true ); + FileHelper.synchronize( indexDir, subDir, true, copyChunkSize ); new File( sourceDir, "current1 ").delete(); new File( sourceDir, "current2" ).delete(); //TODO small hole, no file can be found here @@ -176,11 +178,10 @@ int oldIndex = current; int index = current == 1 ? 2 : 1; - File destinationFile = new File(destination, Integer.valueOf(index).toString() ); - //TODO make smart a parameter + File destinationFile = new File( destination, Integer.valueOf(index).toString() ); try { log.trace( "Copying {} into {}", source, destinationFile ); - FileHelper.synchronize( source, destinationFile, true ); + FileHelper.synchronize( source, destinationFile, true, copyChunkSize ); current = index; } catch (IOException e) { Modified: search/trunk/src/java/org/hibernate/search/store/FSSlaveDirectoryProvider.java =================================================================== --- search/trunk/src/java/org/hibernate/search/store/FSSlaveDirectoryProvider.java 2008-06-05 23:21:58 UTC (rev 14743) +++ search/trunk/src/java/org/hibernate/search/store/FSSlaveDirectoryProvider.java 2008-06-05 23:41:49 UTC (rev 14744) @@ -38,7 +38,8 @@ private int current; private String indexName; private Timer timer; - + private long copyChunkSize; + //variables needed between initialize and start private File sourceIndexDir; private File indexDir; @@ -62,6 +63,7 @@ catch (IOException e) { throw new SearchException( "Unable to initialize index: " + directoryProviderName, e ); } + copyChunkSize = DirectoryProviderHelper.getCopyBufferSize( directoryProviderName, properties ); } public void start() { @@ -96,7 +98,8 @@ throw new AssertionFailure( "No current file marker found in source directory: " + sourceIndexDir.getPath() ); } try { - FileHelper.synchronize( new File( sourceIndexDir, String.valueOf(sourceCurrent) ), destinationFile, true); + FileHelper.synchronize( new File( sourceIndexDir, String.valueOf(sourceCurrent) ), + destinationFile, true, copyChunkSize ); } catch (IOException e) { throw new SearchException( "Unable to synchronize directory: " + indexName, e ); @@ -197,10 +200,9 @@ } File destinationFile = new File( destination, Integer.valueOf( index ).toString() ); - //TODO make smart a parameter try { log.trace( "Copying {} into {}", sourceFile, destinationFile ); - FileHelper.synchronize( sourceFile, destinationFile, true ); + FileHelper.synchronize( sourceFile, destinationFile, true, copyChunkSize ); current = index; } catch (IOException e) { Modified: search/trunk/src/java/org/hibernate/search/util/FileHelper.java =================================================================== --- search/trunk/src/java/org/hibernate/search/util/FileHelper.java 2008-06-05 23:21:58 UTC (rev 14743) +++ search/trunk/src/java/org/hibernate/search/util/FileHelper.java 2008-06-05 23:41:49 UTC (rev 14744) @@ -20,14 +20,18 @@ public abstract class FileHelper { private static final int FAT_PRECISION = 2000; - private static final long DEFAULT_CHUNK_SIZE = 16 * 1024 * 1024; // 16 MB + public static final long DEFAULT_COPY_BUFFER_SIZE = 16 * 1024 * 1024; // 16 MB private static final Logger log = LoggerFactory.getLogger( FileHelper.class ); public static void synchronize(File source, File destination, boolean smart) throws IOException { - synchronize( source, destination, smart, DEFAULT_CHUNK_SIZE ); + synchronize( source, destination, smart, DEFAULT_COPY_BUFFER_SIZE ); } public static void synchronize(File source, File destination, boolean smart, long chunkSize) throws IOException { + if ( chunkSize <= 0 ) { + log.warn( "Chunk size must be positive: using default value." ); + chunkSize = DEFAULT_COPY_BUFFER_SIZE; + } if ( source.isDirectory() ) { if ( ! destination.exists() ) { if ( ! destination.mkdirs() ){ Modified: search/trunk/src/test/org/hibernate/search/test/FSDirectoryTest.java =================================================================== --- search/trunk/src/test/org/hibernate/search/test/FSDirectoryTest.java 2008-06-05 23:21:58 UTC (rev 14743) +++ search/trunk/src/test/org/hibernate/search/test/FSDirectoryTest.java 2008-06-05 23:41:49 UTC (rev 14744) @@ -3,7 +3,6 @@ import java.io.File; import java.util.List; -import java.util.Properties; import org.apache.lucene.analysis.StopAnalyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer; @@ -21,7 +20,6 @@ import org.hibernate.event.PostUpdateEventListener; import org.hibernate.search.Environment; import org.hibernate.search.event.FullTextIndexEventListener; -import org.hibernate.search.store.DirectoryProviderHelper; import org.hibernate.search.store.FSDirectoryProvider; import org.hibernate.search.util.FileHelper; @@ -54,36 +52,6 @@ FileHelper.delete( sub ); } - public void testDirectoryProviderHelperMkdirsGetSource() throws Exception { - String root = "./testDir"; - String relative = "dir1/dir2/dir3"; - - Properties properties = new Properties(); - properties.put( "sourceBase", root ); - properties.put( "source", relative ); - - File rel = DirectoryProviderHelper.getSourceDirectory( "name", properties, true ); - - assertTrue( rel.exists() ); - - FileHelper.delete( new File( root ) ); - } - - public void testDirectoryProviderHelperMkdirsDetermineIndex() throws Exception { - String root = "./testDir/dir1/dir2"; - String relative = "dir3"; - - Properties properties = new Properties(); - properties.put( "indexBase", root ); - properties.put( "indexName", relative ); - - File f = DirectoryProviderHelper.getVerifiedIndexDir( "name", properties, true ); - - assertTrue( new File( root ).exists() ); - - FileHelper.delete( new File( "./testDir" ) ); - } - public void testEventIntegration() throws Exception { Session s = getSessions().openSession(); Added: search/trunk/src/test/org/hibernate/search/test/directoryProvider/DirectoryProviderHelperTest.java =================================================================== --- search/trunk/src/test/org/hibernate/search/test/directoryProvider/DirectoryProviderHelperTest.java (rev 0) +++ search/trunk/src/test/org/hibernate/search/test/directoryProvider/DirectoryProviderHelperTest.java 2008-06-05 23:41:49 UTC (rev 14744) @@ -0,0 +1,78 @@ +package org.hibernate.search.test.directoryProvider; + +import java.io.File; +import java.util.Properties; +import junit.framework.TestCase; +import org.hibernate.search.SearchException; +import org.hibernate.search.store.DirectoryProviderHelper; +import org.hibernate.search.util.FileHelper; + +/** + * @author Gavin King + * @author Sanne Grinovero + */ +public class DirectoryProviderHelperTest extends TestCase { + + public void testMkdirsDetermineIndex() throws Exception { + String root = "./testDir/dir1/dir2"; + String relative = "dir3"; + + Properties properties = new Properties(); + properties.put( "indexBase", root ); + properties.put( "indexName", relative ); + + File f = DirectoryProviderHelper.getVerifiedIndexDir( "name", properties, true ); + + assertTrue( new File( root ).exists() ); + + FileHelper.delete( new File( "./testDir" ) ); + } + + public void testMkdirsGetSource() throws Exception { + String root = "./testDir"; + String relative = "dir1/dir2/dir3"; + + Properties properties = new Properties(); + properties.put( "sourceBase", root ); + properties.put( "source", relative ); + + File rel = DirectoryProviderHelper.getSourceDirectory( "name", properties, true ); + + assertTrue( rel.exists() ); + + FileHelper.delete( new File( root ) ); + } + + public void testConfiguringCopyBufferSize() { + Properties prop = new Properties(); + long mB = 1024 * 1024; + + //default to FileHelper default: + assertEquals( FileHelper.DEFAULT_COPY_BUFFER_SIZE, DirectoryProviderHelper.getCopyBufferSize( "testIdx", prop ) ); + + //any value from MegaBytes: + prop.setProperty( DirectoryProviderHelper.COPYBUFFERSIZE_PROP_NAME, "4" ); + assertEquals( 4*mB, DirectoryProviderHelper.getCopyBufferSize( "testIdx", prop ) ); + prop.setProperty( DirectoryProviderHelper.COPYBUFFERSIZE_PROP_NAME, "1000" ); + assertEquals( 1000*mB, DirectoryProviderHelper.getCopyBufferSize( "testIdx", prop ) ); + + //invalid values + prop.setProperty( DirectoryProviderHelper.COPYBUFFERSIZE_PROP_NAME, "0" ); + boolean testOk = false; + try { + DirectoryProviderHelper.getCopyBufferSize( "testIdx", prop ); + } catch (SearchException e){ + testOk = true; + } + assertTrue( testOk ); + prop.setProperty( DirectoryProviderHelper.COPYBUFFERSIZE_PROP_NAME, "-100" ); + testOk = false; + try { + DirectoryProviderHelper.getCopyBufferSize( "testIdx", prop ); + } catch (SearchException e){ + testOk = true; + } + assertTrue( testOk ); + } + +}