[hibernate-commits] Hibernate SVN: r14744 - in search/trunk: src/java/org/hibernate/search/store and 3 other directories.
hibernate-commits at lists.jboss.org
hibernate-commits at lists.jboss.org
Thu Jun 5 19:41:50 EDT 2008
Author: sannegrinovero
Date: 2008-06-05 19:41:49 -0400 (Thu, 05 Jun 2008)
New Revision: 14744
Added:
search/trunk/src/test/org/hibernate/search/test/directoryProvider/DirectoryProviderHelperTest.java
Modified:
search/trunk/doc/reference/en/modules/configuration.xml
search/trunk/src/java/org/hibernate/search/store/DirectoryProviderHelper.java
search/trunk/src/java/org/hibernate/search/store/FSMasterDirectoryProvider.java
search/trunk/src/java/org/hibernate/search/store/FSSlaveDirectoryProvider.java
search/trunk/src/java/org/hibernate/search/util/FileHelper.java
search/trunk/src/test/org/hibernate/search/test/FSDirectoryTest.java
Log:
HSEARCH-206 Enable configuration parameters for file transfer options in JMS DirectoryProviders
Modified: search/trunk/doc/reference/en/modules/configuration.xml
===================================================================
--- search/trunk/doc/reference/en/modules/configuration.xml 2008-06-05 23:21:58 UTC (rev 14743)
+++ search/trunk/doc/reference/en/modules/configuration.xml 2008-06-05 23:41:49 UTC (rev 14744)
@@ -52,7 +52,10 @@
3600 seconds - 60 minutes).</para><para>Note that the copy is
based on an incremental copy mechanism reducing the average copy
time.</para><para>DirectoryProvider typically used on the master
- node in a JMS back end cluster.</para></entry>
+ node in a JMS back end cluster.</para><para>The <literal>
+ buffer_size_on_copy</literal> optimum depends
+ on your operating system and available RAM; most people reported
+ good results using values between 16 and 64MB.</para></entry>
<entry><para><literal>indexBase</literal>: Base
directory</para><para><literal>indexName</literal>: override
@@ -63,7 +66,10 @@
The actual source directory name being
<filename><sourceBase>/<source></filename>
</para><para><literal>refresh</literal>: refresh period in second
- (the copy will take place every refresh seconds).</para></entry>
+ (the copy will take place every refresh seconds).</para><para>
+ <literal>buffer_size_on_copy</literal>: The amount of
+ MegaBytes to move in a single low level copy instruction;
+ defaults to 16MB.</para></entry>
</row>
<row>
@@ -77,7 +83,10 @@
information (default 3600 seconds - 60 minutes).</para><para>Note
that the copy is based on an incremental copy mechanism reducing
the average copy time.</para><para>DirectoryProvider typically
- used on slave nodes using a JMS back end.</para></entry>
+ used on slave nodes using a JMS back end.</para><para>The <literal>
+ buffer_size_on_copy</literal> optimum depends
+ on your operating system and available RAM; most people reported
+ good results using values between 16 and 64MB.</para></entry>
<entry><para><literal>indexBase</literal>: Base
directory</para><para><literal>indexName</literal>: override
@@ -88,7 +97,10 @@
The actual source directory name being
<filename><sourceBase>/<source></filename>
</para><para><literal>refresh</literal>: refresh period in second
- (the copy will take place every refresh seconds).</para></entry>
+ (the copy will take place every refresh seconds).</para><para>
+ <literal>buffer_size_on_copy</literal>: The amount of
+ MegaBytes to move in a single low level copy instruction;
+ defaults to 16MB.</para></entry>
</row>
<row>
Modified: search/trunk/src/java/org/hibernate/search/store/DirectoryProviderHelper.java
===================================================================
--- search/trunk/src/java/org/hibernate/search/store/DirectoryProviderHelper.java 2008-06-05 23:21:58 UTC (rev 14743)
+++ search/trunk/src/java/org/hibernate/search/store/DirectoryProviderHelper.java 2008-06-05 23:41:49 UTC (rev 14744)
@@ -6,6 +6,7 @@
import java.io.IOException;
import org.hibernate.search.SearchException;
+import org.hibernate.search.util.FileHelper;
import org.hibernate.annotations.common.util.StringHelper;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.index.IndexReader;
@@ -23,6 +24,7 @@
private static final Logger log = LoggerFactory.getLogger( DirectoryProviderHelper.class );
private static final String ROOTINDEX_PROP_NAME = "sourceBase";
private static final String RELATIVEINDEX_PROP_NAME = "source";
+ public static final String COPYBUFFERSIZE_PROP_NAME = "buffer_size_on_copy";
/**
* Build a directory name out of a root and relative path, guessing the significant part
@@ -142,5 +144,31 @@
log.debug( "Refresh period: {} seconds", period );
return period * 1000; //per second
}
+
+ /**
+ * Users may configure the number of KB to use as
+ * "chunk size" for large file copy operations performed
+ * by DirectoryProviders.
+ * @param directoryProviderName
+ * @param properties
+ * @return the number of Bytes to use as "chunk size" in file copy operations.
+ */
+ public static long getCopyBufferSize(String directoryProviderName, Properties properties) {
+ String value = properties.getProperty( COPYBUFFERSIZE_PROP_NAME );
+ long size = FileHelper.DEFAULT_COPY_BUFFER_SIZE;
+ if ( value != null ) {
+ try {
+ size = Long.parseLong( value ) * 1024 * 1024; //from MB to B.
+ } catch (NumberFormatException nfe) {
+ throw new SearchException( "Unable to initialize index " +
+ directoryProviderName +"; "+ COPYBUFFERSIZE_PROP_NAME + " is not numeric.", nfe );
+ }
+ if ( size <= 0 ) {
+ throw new SearchException( "Unable to initialize index " +
+ directoryProviderName +"; "+ COPYBUFFERSIZE_PROP_NAME + " needs to be greater than zero.");
+ }
+ }
+ return size;
+ }
}
Modified: search/trunk/src/java/org/hibernate/search/store/FSMasterDirectoryProvider.java
===================================================================
--- search/trunk/src/java/org/hibernate/search/store/FSMasterDirectoryProvider.java 2008-06-05 23:21:58 UTC (rev 14743)
+++ search/trunk/src/java/org/hibernate/search/store/FSMasterDirectoryProvider.java 2008-06-05 23:41:49 UTC (rev 14744)
@@ -38,6 +38,7 @@
private String indexName;
private Timer timer;
private SearchFactoryImplementor searchFactory;
+ private long copyChunkSize;
//variables needed between initialize and start
private File sourceDir;
@@ -60,6 +61,7 @@
catch (IOException e) {
throw new SearchException( "Unable to initialize index: " + directoryProviderName, e );
}
+ copyChunkSize = DirectoryProviderHelper.getCopyBufferSize( directoryProviderName, properties );
this.searchFactory = searchFactoryImplementor;
}
@@ -79,7 +81,7 @@
}
String currentString = Integer.valueOf( current ).toString();
File subDir = new File( sourceDir, currentString );
- FileHelper.synchronize( indexDir, subDir, true );
+ FileHelper.synchronize( indexDir, subDir, true, copyChunkSize );
new File( sourceDir, "current1 ").delete();
new File( sourceDir, "current2" ).delete();
//TODO small hole, no file can be found here
@@ -176,11 +178,10 @@
int oldIndex = current;
int index = current == 1 ? 2 : 1;
- File destinationFile = new File(destination, Integer.valueOf(index).toString() );
- //TODO make smart a parameter
+ File destinationFile = new File( destination, Integer.valueOf(index).toString() );
try {
log.trace( "Copying {} into {}", source, destinationFile );
- FileHelper.synchronize( source, destinationFile, true );
+ FileHelper.synchronize( source, destinationFile, true, copyChunkSize );
current = index;
}
catch (IOException e) {
Modified: search/trunk/src/java/org/hibernate/search/store/FSSlaveDirectoryProvider.java
===================================================================
--- search/trunk/src/java/org/hibernate/search/store/FSSlaveDirectoryProvider.java 2008-06-05 23:21:58 UTC (rev 14743)
+++ search/trunk/src/java/org/hibernate/search/store/FSSlaveDirectoryProvider.java 2008-06-05 23:41:49 UTC (rev 14744)
@@ -38,7 +38,8 @@
private int current;
private String indexName;
private Timer timer;
-
+ private long copyChunkSize;
+
//variables needed between initialize and start
private File sourceIndexDir;
private File indexDir;
@@ -62,6 +63,7 @@
catch (IOException e) {
throw new SearchException( "Unable to initialize index: " + directoryProviderName, e );
}
+ copyChunkSize = DirectoryProviderHelper.getCopyBufferSize( directoryProviderName, properties );
}
public void start() {
@@ -96,7 +98,8 @@
throw new AssertionFailure( "No current file marker found in source directory: " + sourceIndexDir.getPath() );
}
try {
- FileHelper.synchronize( new File( sourceIndexDir, String.valueOf(sourceCurrent) ), destinationFile, true);
+ FileHelper.synchronize( new File( sourceIndexDir, String.valueOf(sourceCurrent) ),
+ destinationFile, true, copyChunkSize );
}
catch (IOException e) {
throw new SearchException( "Unable to synchronize directory: " + indexName, e );
@@ -197,10 +200,9 @@
}
File destinationFile = new File( destination, Integer.valueOf( index ).toString() );
- //TODO make smart a parameter
try {
log.trace( "Copying {} into {}", sourceFile, destinationFile );
- FileHelper.synchronize( sourceFile, destinationFile, true );
+ FileHelper.synchronize( sourceFile, destinationFile, true, copyChunkSize );
current = index;
}
catch (IOException e) {
Modified: search/trunk/src/java/org/hibernate/search/util/FileHelper.java
===================================================================
--- search/trunk/src/java/org/hibernate/search/util/FileHelper.java 2008-06-05 23:21:58 UTC (rev 14743)
+++ search/trunk/src/java/org/hibernate/search/util/FileHelper.java 2008-06-05 23:41:49 UTC (rev 14744)
@@ -20,14 +20,18 @@
public abstract class FileHelper {
private static final int FAT_PRECISION = 2000;
- private static final long DEFAULT_CHUNK_SIZE = 16 * 1024 * 1024; // 16 MB
+ public static final long DEFAULT_COPY_BUFFER_SIZE = 16 * 1024 * 1024; // 16 MB
private static final Logger log = LoggerFactory.getLogger( FileHelper.class );
public static void synchronize(File source, File destination, boolean smart) throws IOException {
- synchronize( source, destination, smart, DEFAULT_CHUNK_SIZE );
+ synchronize( source, destination, smart, DEFAULT_COPY_BUFFER_SIZE );
}
public static void synchronize(File source, File destination, boolean smart, long chunkSize) throws IOException {
+ if ( chunkSize <= 0 ) {
+ log.warn( "Chunk size must be positive: using default value." );
+ chunkSize = DEFAULT_COPY_BUFFER_SIZE;
+ }
if ( source.isDirectory() ) {
if ( ! destination.exists() ) {
if ( ! destination.mkdirs() ){
Modified: search/trunk/src/test/org/hibernate/search/test/FSDirectoryTest.java
===================================================================
--- search/trunk/src/test/org/hibernate/search/test/FSDirectoryTest.java 2008-06-05 23:21:58 UTC (rev 14743)
+++ search/trunk/src/test/org/hibernate/search/test/FSDirectoryTest.java 2008-06-05 23:41:49 UTC (rev 14744)
@@ -3,7 +3,6 @@
import java.io.File;
import java.util.List;
-import java.util.Properties;
import org.apache.lucene.analysis.StopAnalyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
@@ -21,7 +20,6 @@
import org.hibernate.event.PostUpdateEventListener;
import org.hibernate.search.Environment;
import org.hibernate.search.event.FullTextIndexEventListener;
-import org.hibernate.search.store.DirectoryProviderHelper;
import org.hibernate.search.store.FSDirectoryProvider;
import org.hibernate.search.util.FileHelper;
@@ -54,36 +52,6 @@
FileHelper.delete( sub );
}
- public void testDirectoryProviderHelperMkdirsGetSource() throws Exception {
- String root = "./testDir";
- String relative = "dir1/dir2/dir3";
-
- Properties properties = new Properties();
- properties.put( "sourceBase", root );
- properties.put( "source", relative );
-
- File rel = DirectoryProviderHelper.getSourceDirectory( "name", properties, true );
-
- assertTrue( rel.exists() );
-
- FileHelper.delete( new File( root ) );
- }
-
- public void testDirectoryProviderHelperMkdirsDetermineIndex() throws Exception {
- String root = "./testDir/dir1/dir2";
- String relative = "dir3";
-
- Properties properties = new Properties();
- properties.put( "indexBase", root );
- properties.put( "indexName", relative );
-
- File f = DirectoryProviderHelper.getVerifiedIndexDir( "name", properties, true );
-
- assertTrue( new File( root ).exists() );
-
- FileHelper.delete( new File( "./testDir" ) );
- }
-
public void testEventIntegration() throws Exception {
Session s = getSessions().openSession();
Added: search/trunk/src/test/org/hibernate/search/test/directoryProvider/DirectoryProviderHelperTest.java
===================================================================
--- search/trunk/src/test/org/hibernate/search/test/directoryProvider/DirectoryProviderHelperTest.java (rev 0)
+++ search/trunk/src/test/org/hibernate/search/test/directoryProvider/DirectoryProviderHelperTest.java 2008-06-05 23:41:49 UTC (rev 14744)
@@ -0,0 +1,78 @@
+package org.hibernate.search.test.directoryProvider;
+
+import java.io.File;
+import java.util.Properties;
+import junit.framework.TestCase;
+import org.hibernate.search.SearchException;
+import org.hibernate.search.store.DirectoryProviderHelper;
+import org.hibernate.search.util.FileHelper;
+
+/**
+ * @author Gavin King
+ * @author Sanne Grinovero
+ */
+public class DirectoryProviderHelperTest extends TestCase {
+
+ public void testMkdirsDetermineIndex() throws Exception {
+ String root = "./testDir/dir1/dir2";
+ String relative = "dir3";
+
+ Properties properties = new Properties();
+ properties.put( "indexBase", root );
+ properties.put( "indexName", relative );
+
+ File f = DirectoryProviderHelper.getVerifiedIndexDir( "name", properties, true );
+
+ assertTrue( new File( root ).exists() );
+
+ FileHelper.delete( new File( "./testDir" ) );
+ }
+
+ public void testMkdirsGetSource() throws Exception {
+ String root = "./testDir";
+ String relative = "dir1/dir2/dir3";
+
+ Properties properties = new Properties();
+ properties.put( "sourceBase", root );
+ properties.put( "source", relative );
+
+ File rel = DirectoryProviderHelper.getSourceDirectory( "name", properties, true );
+
+ assertTrue( rel.exists() );
+
+ FileHelper.delete( new File( root ) );
+ }
+
+ public void testConfiguringCopyBufferSize() {
+ Properties prop = new Properties();
+ long mB = 1024 * 1024;
+
+ //default to FileHelper default:
+ assertEquals( FileHelper.DEFAULT_COPY_BUFFER_SIZE, DirectoryProviderHelper.getCopyBufferSize( "testIdx", prop ) );
+
+ //any value from MegaBytes:
+ prop.setProperty( DirectoryProviderHelper.COPYBUFFERSIZE_PROP_NAME, "4" );
+ assertEquals( 4*mB, DirectoryProviderHelper.getCopyBufferSize( "testIdx", prop ) );
+ prop.setProperty( DirectoryProviderHelper.COPYBUFFERSIZE_PROP_NAME, "1000" );
+ assertEquals( 1000*mB, DirectoryProviderHelper.getCopyBufferSize( "testIdx", prop ) );
+
+ //invalid values
+ prop.setProperty( DirectoryProviderHelper.COPYBUFFERSIZE_PROP_NAME, "0" );
+ boolean testOk = false;
+ try {
+ DirectoryProviderHelper.getCopyBufferSize( "testIdx", prop );
+ } catch (SearchException e){
+ testOk = true;
+ }
+ assertTrue( testOk );
+ prop.setProperty( DirectoryProviderHelper.COPYBUFFERSIZE_PROP_NAME, "-100" );
+ testOk = false;
+ try {
+ DirectoryProviderHelper.getCopyBufferSize( "testIdx", prop );
+ } catch (SearchException e){
+ testOk = true;
+ }
+ assertTrue( testOk );
+ }
+
+}
More information about the hibernate-commits
mailing list