Author: epbernard Date: 2007-01-23 14:26:42 -0500 (Tue, 23 Jan 2007) New Revision: 11085 Added: branches/Branch_3_2/HibernateExt/metadata/src/java/org/hibernate/search/backend/impl/jms/JMSHibernateSearchController.java branches/Branch_3_2/HibernateExt/metadata/src/java/org/hibernate/search/store/FSMasterDirectoryProvider.java branches/Branch_3_2/HibernateExt/metadata/src/java/org/hibernate/search/store/FSSwitchableDirectoryProvider.java branches/Branch_3_2/HibernateExt/metadata/src/java/org/hibernate/search/util/FileHelper.java branches/Branch_3_2/HibernateExt/metadata/src/test/org/hibernate/search/test/util/ branches/Branch_3_2/HibernateExt/metadata/src/test/org/hibernate/search/test/util/FileHelperTest.java Modified: branches/Branch_3_2/HibernateExt/metadata/src/java/org/hibernate/search/SearchFactory.java branches/Branch_3_2/HibernateExt/metadata/src/java/org/hibernate/search/backend/impl/jms/JMSQueueWorker.java branches/Branch_3_2/HibernateExt/metadata/src/java/org/hibernate/search/store/DirectoryProvider.java branches/Branch_3_2/HibernateExt/metadata/src/java/org/hibernate/search/store/DirectoryProviderFactory.java branches/Branch_3_2/HibernateExt/metadata/src/java/org/hibernate/search/store/FSDirectoryProvider.java branches/Branch_3_2/HibernateExt/metadata/src/java/org/hibernate/search/store/RAMDirectoryProvider.java Log: ANN-519 pluggable worker ANN-523 JMS implementation Modified: branches/Branch_3_2/HibernateExt/metadata/src/java/org/hibernate/search/SearchFactory.java =================================================================== --- branches/Branch_3_2/HibernateExt/metadata/src/java/org/hibernate/search/SearchFactory.java 2007-01-23 19:24:37 UTC (rev 11084) +++ branches/Branch_3_2/HibernateExt/metadata/src/java/org/hibernate/search/SearchFactory.java 2007-01-23 19:26:42 UTC (rev 11085) @@ -82,7 +82,7 @@ if ( mappedClass != null ) { XClass mappedXClass = reflectionManager.toXClass( mappedClass ); if ( mappedXClass != null && mappedXClass.isAnnotationPresent( Indexed.class ) ) { - DirectoryProvider provider = factory.createDirectoryProvider( mappedXClass, cfg ); + DirectoryProvider provider = factory.createDirectoryProvider( mappedXClass, cfg, this ); if ( !lockableDirectoryProviders.containsKey( provider ) ) { lockableDirectoryProviders.put( provider, new ReentrantLock() ); } Added: branches/Branch_3_2/HibernateExt/metadata/src/java/org/hibernate/search/backend/impl/jms/JMSHibernateSearchController.java =================================================================== --- branches/Branch_3_2/HibernateExt/metadata/src/java/org/hibernate/search/backend/impl/jms/JMSHibernateSearchController.java (rev 0) +++ branches/Branch_3_2/HibernateExt/metadata/src/java/org/hibernate/search/backend/impl/jms/JMSHibernateSearchController.java 2007-01-23 19:26:42 UTC (rev 11085) @@ -0,0 +1,71 @@ +//$Id: $ +package org.hibernate.search.backend.impl.jms; + +import java.util.List; +import java.util.Properties; +import javax.jms.MessageListener; +import javax.jms.Message; +import javax.jms.ObjectMessage; +import javax.jms.JMSException; +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; + +import org.apache.commons.logging.LogFactory; +import org.apache.commons.logging.Log; +import org.hibernate.search.backend.Work; +import org.hibernate.search.backend.QueueWorker; +import org.hibernate.search.backend.impl.BatchedQueueWorker; +import org.hibernate.search.util.ContextHelper; +import org.hibernate.search.SearchFactory; +import org.hibernate.Session; +import org.hibernate.engine.SessionImplementor; + +/** + * @author Emmanuel Bernard + */ +public abstract class JMSHibernateSearchController implements MessageListener { + private static Log log = LogFactory.getLog( JMSHibernateSearchController.class ); + + protected abstract Session getSession(); + + public void onMessage(Message message) { + if (! (message instanceof ObjectMessage) ) { + log.error( "Incorrect message type: " + message.getClass() ); + return; + } + ObjectMessage objectMessage = (ObjectMessage) message; + List<Work> queue; + try { + queue = (List<Work>) objectMessage.getObject(); + } + catch (JMSException e) { + log.error( "Unable to retrieve object from message: " + message.getClass(), e ); + return; + } + catch( ClassCastException e ) { + log.error( "Illegal object retrieved from message", e ); + return; + } + QueueWorker worker = getWorker( queue ); + worker.run(); + } + + private QueueWorker getWorker(List<Work> queue) { + //FIXME casting sucks becasue we do not control what get session from + SearchFactory factory = ContextHelper.getSearchFactory( (SessionImplementor) getSession() ); + QueueWorker worker = new BatchedQueueWorker(); + worker.initialize( new Properties(), factory ); + worker.setQueue( queue ); + return worker; + } + + @PostConstruct + public void initialize() { + //init the source copy process + + } + @PreDestroy + public void shutdown() { + //stop the source copy process + } +} Modified: branches/Branch_3_2/HibernateExt/metadata/src/java/org/hibernate/search/backend/impl/jms/JMSQueueWorker.java =================================================================== --- branches/Branch_3_2/HibernateExt/metadata/src/java/org/hibernate/search/backend/impl/jms/JMSQueueWorker.java 2007-01-23 19:24:37 UTC (rev 11084) +++ branches/Branch_3_2/HibernateExt/metadata/src/java/org/hibernate/search/backend/impl/jms/JMSQueueWorker.java 2007-01-23 19:26:42 UTC (rev 11085) @@ -2,11 +2,9 @@ package org.hibernate.search.backend.impl.jms; import java.util.List; -import java.util.Map; import java.util.Properties; import java.util.HashSet; import java.util.Iterator; -import java.util.concurrent.locks.ReentrantLock; import java.io.Serializable; import javax.jms.QueueSender; @@ -22,8 +20,7 @@ import org.hibernate.search.backend.QueueWorker; import org.hibernate.search.backend.Work; -import org.hibernate.search.engine.DocumentBuilder; -import org.hibernate.search.store.DirectoryProvider; +import org.hibernate.search.SearchFactory; import org.hibernate.search.Environment; import org.hibernate.HibernateException; @@ -32,8 +29,6 @@ */ public class JMSQueueWorker implements QueueWorker { private List<Work> queue; - private Map<DirectoryProvider, ReentrantLock> lockableDirectoryProviders; - private Map<Class, DocumentBuilder<Object>> documentBuilders; private String jmsQueueName; private String jmsConnectionFactoryName; private static final String JNDI_PREFIX = Environment.WORKER_PREFIX + "jndi."; @@ -43,9 +38,9 @@ public void run() { resetJMSTools(); - QueueConnection cnn = null; - QueueSender sender = null; - QueueSession session = null; + QueueConnection cnn; + QueueSender sender; + QueueSession session; try { cnn = factory.createQueueConnection(); //TODO make transacted parameterized @@ -75,10 +70,7 @@ } } - public void initialize(Properties props, Map<Class, DocumentBuilder<Object>> documentBuilders, - Map<DirectoryProvider, ReentrantLock> lockableDirectoryProviders) { - this.documentBuilders = documentBuilders; - this.lockableDirectoryProviders = lockableDirectoryProviders; + public void initialize(Properties props, SearchFactory searchFactory) { this.properties = props; this.jmsConnectionFactoryName = props.getProperty( Environment.WORKER_PREFIX + "jms.connection_factory" ); this.jmsQueueName = props.getProperty( Environment.WORKER_PREFIX + "jms.queue" ); Modified: branches/Branch_3_2/HibernateExt/metadata/src/java/org/hibernate/search/store/DirectoryProvider.java =================================================================== --- branches/Branch_3_2/HibernateExt/metadata/src/java/org/hibernate/search/store/DirectoryProvider.java 2007-01-23 19:24:37 UTC (rev 11084) +++ branches/Branch_3_2/HibernateExt/metadata/src/java/org/hibernate/search/store/DirectoryProvider.java 2007-01-23 19:26:42 UTC (rev 11085) @@ -4,6 +4,7 @@ import java.util.Properties; import org.apache.lucene.store.Directory; +import org.hibernate.search.SearchFactory; /** @@ -21,7 +22,7 @@ /** * get the information to initialize the directory and build its hashCode */ - void initialize(String directoryProviderName, Properties properties); + void initialize(String directoryProviderName, Properties properties, SearchFactory searchFactory); /** * Returns an initialized Lucene Directory. This method call <b>must</b> be threadsafe Modified: branches/Branch_3_2/HibernateExt/metadata/src/java/org/hibernate/search/store/DirectoryProviderFactory.java =================================================================== --- branches/Branch_3_2/HibernateExt/metadata/src/java/org/hibernate/search/store/DirectoryProviderFactory.java 2007-01-23 19:24:37 UTC (rev 11084) +++ branches/Branch_3_2/HibernateExt/metadata/src/java/org/hibernate/search/store/DirectoryProviderFactory.java 2007-01-23 19:26:42 UTC (rev 11085) @@ -10,6 +10,7 @@ import org.hibernate.cfg.AnnotationConfiguration; import org.hibernate.cfg.Configuration; import org.hibernate.search.annotations.Indexed; +import org.hibernate.search.SearchFactory; import org.hibernate.mapping.PersistentClass; import org.hibernate.reflection.ReflectionManager; import org.hibernate.reflection.XClass; @@ -40,9 +41,7 @@ private static String DEFAULT_DIRECTORY_PROVIDER = FSDirectoryProvider.class.getName(); //TODO for the public? - //public DirectoryProvider<?> createDirectoryProvider(XClass entity, Configuration cfg) { - - public DirectoryProvider<?> createDirectoryProvider(XClass entity, Configuration cfg) { + public DirectoryProvider<?> createDirectoryProvider(XClass entity, Configuration cfg, SearchFactory searchFactory) { //get properties String directoryProviderName = getDirectoryProviderName( entity, cfg ); Properties indexProps = getDirectoryProperties( cfg, directoryProviderName ); @@ -63,7 +62,12 @@ catch (Exception e) { throw new HibernateException( "Unable to instanciate directory provider: " + className, e ); } - provider.initialize( directoryProviderName, indexProps ); + try { + provider.initialize( directoryProviderName, indexProps, searchFactory ); + } + catch (Exception e) { + throw new HibernateException( "Unable to initialize: " + directoryProviderName, e); + } int index = providers.indexOf( provider ); if ( index != -1 ) { //share the same Directory provider for the same underlying store @@ -93,7 +97,7 @@ return indexProps; } - public static String getDirectoryProviderName(XClass clazz, Configuration cfg) { + private static String getDirectoryProviderName(XClass clazz, Configuration cfg) { //yuk ReflectionManager reflectionManager = ( (AnnotationConfiguration) cfg ).createExtendedMappings().getReflectionManager(); Modified: branches/Branch_3_2/HibernateExt/metadata/src/java/org/hibernate/search/store/FSDirectoryProvider.java =================================================================== --- branches/Branch_3_2/HibernateExt/metadata/src/java/org/hibernate/search/store/FSDirectoryProvider.java 2007-01-23 19:24:37 UTC (rev 11084) +++ branches/Branch_3_2/HibernateExt/metadata/src/java/org/hibernate/search/store/FSDirectoryProvider.java 2007-01-23 19:26:42 UTC (rev 11085) @@ -3,7 +3,6 @@ import java.io.File; import java.io.IOException; -import java.text.MessageFormat; import java.util.Properties; import org.apache.commons.logging.Log; @@ -12,6 +11,8 @@ import org.apache.lucene.index.IndexWriter; import org.apache.lucene.store.FSDirectory; import org.hibernate.HibernateException; +import org.hibernate.search.util.DirectoryProviderHelper; +import org.hibernate.search.SearchFactory; /** * Use a Lucene FSDirectory @@ -26,24 +27,11 @@ private static Log log = LogFactory.getLog( FSDirectoryProvider.class ); private String indexName; - public void initialize(String directoryProviderName, Properties properties) { - String indexBase = properties.getProperty( "indexBase", "." ); - File indexDir = new File( indexBase ); - - if ( !( indexDir.exists() && indexDir.isDirectory() ) ) { - //TODO create the directory? - throw new HibernateException( MessageFormat.format( "Index directory does not exists: {0}", indexBase ) ); - } - if ( !indexDir.canWrite() ) { - throw new HibernateException( "Cannot write into index directory: " + indexBase ); - } - log.info( "Setting index dir to " + indexDir ); - - File file = new File( indexDir, directoryProviderName ); - + public void initialize(String directoryProviderName, Properties properties, SearchFactory searchFactory) { + File indexDir = DirectoryProviderHelper.determineIndexDir( directoryProviderName, properties ); try { - boolean create = !file.exists(); - indexName = file.getCanonicalPath(); + boolean create = !indexDir.exists(); + indexName = indexDir.getCanonicalPath(); directory = FSDirectory.getDirectory( indexName, create ); if ( create ) { IndexWriter iw = new IndexWriter( directory, new StandardAnalyzer(), create ); @@ -53,7 +41,6 @@ catch (IOException e) { throw new HibernateException( "Unable to initialize index: " + directoryProviderName, e ); } - } public FSDirectory getDirectory() { Added: branches/Branch_3_2/HibernateExt/metadata/src/java/org/hibernate/search/store/FSMasterDirectoryProvider.java =================================================================== --- branches/Branch_3_2/HibernateExt/metadata/src/java/org/hibernate/search/store/FSMasterDirectoryProvider.java (rev 0) +++ branches/Branch_3_2/HibernateExt/metadata/src/java/org/hibernate/search/store/FSMasterDirectoryProvider.java 2007-01-23 19:26:42 UTC (rev 11085) @@ -0,0 +1,193 @@ +//$Id: $ +package org.hibernate.search.store; + +import java.util.Timer; +import java.util.Properties; +import java.util.TimerTask; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.locks.Lock; +import java.io.File; +import java.io.IOException; + +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.hibernate.search.util.DirectoryProviderHelper; +import org.hibernate.search.util.FileHelper; +import org.hibernate.search.SearchFactory; +import org.hibernate.HibernateException; + +/** + * Use a Lucene FSDirectory + * The base directory is represented by hibernate.search.<index>.indexBase + * The index is created in <base directory>/<index name> + * + * @author Emmanuel Bernard + */ +public class FSMasterDirectoryProvider implements DirectoryProvider<FSDirectory> { + private static Log log = LogFactory.getLog( FSMasterDirectoryProvider.class ); + private FSDirectory directory; + private int current; + private String indexName; + private Timer timer; + private SearchFactory searchFactory; + + public void initialize(String directoryProviderName, Properties properties, SearchFactory searchFactory) { + //source guessing + String source = DirectoryProviderHelper.getSourceDirectory( "sourceBase", "source", properties ); + if (source == null) + throw new IllegalStateException("FSMasterDirectoryProvider requires a viable source directory"); + File indexDir = DirectoryProviderHelper.determineIndexDir( directoryProviderName, properties ); + String refreshPeriod = properties.getProperty( "refresh", "60" ); + long period = Long.parseLong( refreshPeriod ); + + try { + boolean create = !indexDir.exists(); + indexName = indexDir.getCanonicalPath(); + if (create) { + log.debug( "Index directory '" + indexName + "' will be initialized"); + indexDir.mkdir(); + } + directory = FSDirectory.getDirectory( indexName, create ); + if ( create ) { + IndexWriter iw = new IndexWriter( directory, new StandardAnalyzer(), create ); + iw.close(); + } + + //copy to source + if ( new File(source, "current1").exists() ) { + current = 2; + } + else if ( new File(source, "current2").exists() ) { + current = 1; + } + else { + log.debug( "Source directory for '" + indexName + "' will be initialized"); + current = 1; + } + String currentString = Integer.valueOf( current ).toString(); + File subDir = new File(source, currentString ); + FileHelper.synchronize( indexDir, subDir, true ); + new File(source, "current1").delete(); + new File(source, "current2").delete(); + //TODO small hole, no file can be found here + new File(source, "current" + currentString).createNewFile(); + log.debug( "Current directory: " + current); + } + catch (IOException e) { + throw new HibernateException( "Unable to initialize index: " + directoryProviderName, e ); + } + timer = new Timer(); + TimerTask task = new FSMasterDirectoryProvider.TriggerTask(indexName, source, this ); + timer.scheduleAtFixedRate( task, period, period ); + this.searchFactory = searchFactory; + } + + public FSDirectory getDirectory() { + return directory; + } + + @Override + public boolean equals(Object obj) { + // this code is actually broken since the value change after initialize call + // but from a practical POV this is fine since we only call this method + // after initialize call + if ( obj == this ) return true; + if ( obj == null || !( obj instanceof FSMasterDirectoryProvider ) ) return false; + return indexName.equals( ( (FSMasterDirectoryProvider) obj ).indexName ); + } + + @Override + public int hashCode() { + // this code is actually broken since the value change after initialize call + // but from a practical POV this is fine since we only call this method + // after initialize call + int hash = 11; + return 37 * hash + indexName.hashCode(); + } + + class TriggerTask extends TimerTask { + + private ExecutorService executor; + private FSMasterDirectoryProvider.CopyDirectory copyTask; + + public TriggerTask(String source, String destination, DirectoryProvider directoryProvider) { + executor = Executors.newSingleThreadExecutor(); + copyTask = new FSMasterDirectoryProvider.CopyDirectory( source, destination, directoryProvider ); + } + + public void run() { + if (!copyTask.inProgress) { + executor.execute( copyTask ); + } + else { + log.info( "Skipping directory synchronization, previous work still in progress: " + indexName); + } + } + } + + class CopyDirectory implements Runnable { + private String source; + private String destination; + private volatile boolean inProgress; + private Lock directoryProviderLock; + private DirectoryProvider directoryProvider; + + public CopyDirectory(String source, String destination, DirectoryProvider directoryProvider) { + this.source = source; + this.destination = destination; + this.directoryProvider = directoryProvider; + } + + public void run() { + //TODO get rid of current and use the marker file instead? + inProgress = true; + if (directoryProviderLock == null) { + directoryProviderLock = searchFactory.getLockableDirectoryProviders().get( directoryProvider ); + directoryProvider = null; + searchFactory = null; //get rid of any useless link (help hot redeployment?) + } + try { + directoryProviderLock.lock(); + int oldIndex = current; + int index = current == 1 ? 2 : 1; + File sourceFile = new File(source); + + File destinationFile = new File(destination, Integer.valueOf(index).toString() ); + //TODO make smart a parameter + try { + FileHelper.synchronize( sourceFile, destinationFile, true); + current = index; + } + catch (IOException e) { + //don't change current + log.error( "Unable to synchronize source of " + indexName, e); + inProgress = false; + return; + } + if ( ! new File(source, "current" + oldIndex).delete() ) { + log.warn( "Unable to remove previous marker file from source of " + indexName ); + } + try { + new File(source, "current" + index).createNewFile(); + } + catch( IOException e ) { + log.warn( "Unable to create current marker in source of " + indexName, e ); + } + } + finally { + directoryProviderLock.unlock(); + inProgress = false; + } + } + } + + public void finalize() throws Throwable { + super.finalize(); + timer.cancel(); + //TODO find a better cycle from Hibernate core + } +} Added: branches/Branch_3_2/HibernateExt/metadata/src/java/org/hibernate/search/store/FSSwitchableDirectoryProvider.java =================================================================== --- branches/Branch_3_2/HibernateExt/metadata/src/java/org/hibernate/search/store/FSSwitchableDirectoryProvider.java (rev 0) +++ branches/Branch_3_2/HibernateExt/metadata/src/java/org/hibernate/search/store/FSSwitchableDirectoryProvider.java 2007-01-23 19:26:42 UTC (rev 11085) @@ -0,0 +1,217 @@ +//$Id: $ +package org.hibernate.search.store; + +import java.util.Properties; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.io.File; +import java.io.IOException; + +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.hibernate.HibernateException; +import org.hibernate.AssertionFailure; +import org.hibernate.search.util.FileHelper; +import org.hibernate.search.util.DirectoryProviderHelper; +import org.hibernate.search.SearchFactory; + +/** + * Use a Lucene FSDirectory + * The base directory is represented by hibernate.search.<index>.indexBase + * The index is created in <base directory>/<index name> + * + * @author Emmanuel Bernard + */ +public class FSSwitchableDirectoryProvider implements DirectoryProvider<FSDirectory> { + private static Log log = LogFactory.getLog( FSSwitchableDirectoryProvider.class ); + private FSDirectory directory1; + private FSDirectory directory2; + private int current; + private String indexName; + private Timer timer; + + public void initialize(String directoryProviderName, Properties properties, SearchFactory searchFactory) { + //source guessing + String source = DirectoryProviderHelper.getSourceDirectory( "sourceBase", "source", properties ); + if (source == null) + throw new IllegalStateException("FSSwitchableDirectoryProvider requires a viable source directory"); + if ( ! new File(source, "current1").exists() && ! new File(source, "current2").exists() ) { + throw new IllegalStateException("No current marker in source directory"); + } + File indexDir = DirectoryProviderHelper.determineIndexDir( directoryProviderName, properties ); + String refreshPeriod = properties.getProperty( "refresh", "60" ); + long period = Long.parseLong( refreshPeriod ); + + try { + boolean create = !indexDir.exists(); + indexName = indexDir.getCanonicalPath(); + if (create) indexDir.mkdir(); + + File subDir = new File( indexName, "1" ); + create = ! subDir.exists(); + directory1 = FSDirectory.getDirectory( subDir.getCanonicalPath(), create ); + if ( create ) { + IndexWriter iw = new IndexWriter( directory1, new StandardAnalyzer(), create ); + iw.close(); + } + + subDir = new File( indexName, "2" ); + create = ! subDir.exists(); + directory2 = FSDirectory.getDirectory( subDir.getCanonicalPath(), create ); + if ( create ) { + IndexWriter iw = new IndexWriter( directory2, new StandardAnalyzer(), create ); + iw.close(); + } + File currentMarker = new File(indexName, "current1"); + File current2Marker = new File(indexName, "current2"); + if ( currentMarker.exists() ) { + current = 1; + } + else if ( current2Marker.exists() ) { + current = 2; + } + else { + //no default + log.debug( "Setting directory 1 as current"); + current = 1; + File sourceFile = new File(source); + File destinationFile = new File(indexName, Integer.valueOf(current).toString() ); + //TODO make smart a parameter + try { + FileHelper.synchronize( sourceFile, destinationFile, true); + } + catch (IOException e) { + throw new HibernateException("Umable to synchonize directory: " + indexName, e); + } + if (! currentMarker.createNewFile() ) { + throw new HibernateException("Unable to create the directory marker file: " + indexName); + } + } + log.debug( "Current directory: " + current); + } + catch (IOException e) { + throw new HibernateException( "Unable to initialize index: " + directoryProviderName, e ); + } + timer = new Timer(); + TimerTask task = new TriggerTask(source, indexName); + timer.scheduleAtFixedRate( task, period, period ); + } + + public FSDirectory getDirectory() { + if (current == 1) { + return directory1; + } + else if (current == 2) { + return directory2; + } + else { + throw new AssertionFailure("Illegal current directory: " + current); + } + } + + @Override + public boolean equals(Object obj) { + // this code is actually broken since the value change after initialize call + // but from a practical POV this is fine since we only call this method + // after initialize call + if ( obj == this ) return true; + if ( obj == null || !( obj instanceof FSSwitchableDirectoryProvider ) ) return false; + return indexName.equals( ( (FSSwitchableDirectoryProvider) obj ).indexName ); + } + + @Override + public int hashCode() { + // this code is actually broken since the value change after initialize call + // but from a practical POV this is fine since we only call this method + // after initialize call + int hash = 11; + return 37 * hash + indexName.hashCode(); + } + + class TriggerTask extends TimerTask { + + private ExecutorService executor; + private CopyDirectory copyTask; + + public TriggerTask(String source, String destination) { + executor = Executors.newSingleThreadExecutor(); + copyTask = new CopyDirectory( source, destination ); + } + + public void run() { + if (!copyTask.inProgress) { + executor.execute( copyTask ); + } + else { + log.info( "Skipping directory synchronization, previous work still in progress: " + indexName); + } + } + } + + class CopyDirectory implements Runnable { + private String source; + private String destination; + private volatile boolean inProgress; + + public CopyDirectory(String source, String destination) { + this.source = source; + this.destination = destination; + } + + public void run() { + try { + inProgress = true; + int oldIndex = current; + int index = current == 1 ? 2 : 1; + File sourceFile; + if ( new File( source, "current1" ).exists() ) { + sourceFile = new File(source, "1"); + } + else if ( new File( source, "current2" ).exists() ) { + sourceFile = new File(source, "2"); + } + else { + log.error("Unable to determine current in source directory"); + inProgress = false; + return; + } + + File destinationFile = new File(destination, Integer.valueOf(index).toString() ); + //TODO make smart a parameter + try { + FileHelper.synchronize( sourceFile, destinationFile, true); + current = index; + } + catch (IOException e) { + //don't change current + log.error( "Unable to synchronize " + indexName, e); + inProgress = false; + return; + } + if ( ! new File(indexName, "current" + oldIndex).delete() ) { + log.warn( "Unable to remove previous marker file in " + indexName ); + } + try { + new File(indexName, "current" + index).createNewFile(); + } + catch( IOException e ) { + log.warn( "Unable to create current marker file in " + indexName, e ); + } + } + finally { + inProgress = false; + } + } + } + + public void finalize() throws Throwable { + super.finalize(); + timer.cancel(); + //TODO find a better cycle from Hibernate core + } +} Modified: branches/Branch_3_2/HibernateExt/metadata/src/java/org/hibernate/search/store/RAMDirectoryProvider.java =================================================================== --- branches/Branch_3_2/HibernateExt/metadata/src/java/org/hibernate/search/store/RAMDirectoryProvider.java 2007-01-23 19:24:37 UTC (rev 11084) +++ branches/Branch_3_2/HibernateExt/metadata/src/java/org/hibernate/search/store/RAMDirectoryProvider.java 2007-01-23 19:26:42 UTC (rev 11085) @@ -8,6 +8,7 @@ import org.apache.lucene.index.IndexWriter; import org.apache.lucene.store.RAMDirectory; import org.hibernate.HibernateException; +import org.hibernate.search.SearchFactory; /** * Use a Lucene RAMDirectory @@ -20,7 +21,7 @@ private RAMDirectory directory; private String indexName; - public void initialize(String directoryProviderName, Properties properties) { + public void initialize(String directoryProviderName, Properties properties, SearchFactory searchFactory) { indexName = directoryProviderName; directory = new RAMDirectory(); try { Added: branches/Branch_3_2/HibernateExt/metadata/src/java/org/hibernate/search/util/FileHelper.java =================================================================== --- branches/Branch_3_2/HibernateExt/metadata/src/java/org/hibernate/search/util/FileHelper.java (rev 0) +++ branches/Branch_3_2/HibernateExt/metadata/src/java/org/hibernate/search/util/FileHelper.java 2007-01-23 19:26:42 UTC (rev 11085) @@ -0,0 +1,86 @@ +//$Id: $ +package org.hibernate.search.util; + +import java.io.File; +import java.io.IOException; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.util.Set; +import java.util.HashSet; +import java.util.Arrays; +import java.nio.channels.FileChannel; + +/** + * @author Emmanuel Bernard + */ +public abstract class FileHelper { + private static final int FAT_PRECISION = 2000; + + public static void synchronize(File source, File destination, boolean smart) throws IOException { + if ( source.isDirectory() ) { + if ( ! destination.exists() ) { + destination.mkdirs(); + } + else if ( ! destination.isDirectory() ) { + throw new IOException("Source and Destination not of the same type:" + + source.getCanonicalPath() + " , " + destination.getCanonicalPath() ); + } + String[] sources = source.list(); + Set<String> srcNames = new HashSet<String>( Arrays.asList( sources ) ); + String[] dests = destination.list(); + + //delete files not present in source + for (String fileName : dests) { + if ( ! srcNames.contains( fileName ) ) { + delete( new File(destination, fileName) ); + } + } + //copy each file from source + for (String fileName : sources) { + File srcFile = new File(source, fileName); + File destFile = new File(destination, fileName); + synchronize( srcFile, destFile, smart ); + } + } + else { + if ( destination.exists() && destination.isDirectory() ) { + delete( destination ); + } + if ( destination.exists() ) { + long sts = source.lastModified() / FAT_PRECISION; + long dts = destination.lastModified() / FAT_PRECISION; + //do not copy if smart and same timestamp and same length + if ( !smart || sts == 0 || sts != dts || source.length() != destination.length() ) { + copyFile(source, destination); + } + } + else { + copyFile(source, destination); + } + } + } + + private static void copyFile(File srcFile, File destFile) throws IOException { + FileInputStream is = null; + FileOutputStream os = null; + try { + is = new FileInputStream(srcFile); + FileChannel iChannel = is.getChannel(); + os = new FileOutputStream( destFile, false ); + FileChannel oChannel = os.getChannel(); + oChannel.transferFrom( iChannel, 0, srcFile.length() ); + } + finally { + if (is != null) is.close(); + if (os != null) os.close(); + } + destFile.setLastModified( srcFile.lastModified() ); + } + + public static void delete(File file) { + if ( file.isDirectory() ) { + for ( File subFile : file.listFiles() ) delete( subFile ); + } + if ( file.exists() ) file.delete(); + } +} Added: branches/Branch_3_2/HibernateExt/metadata/src/test/org/hibernate/search/test/util/FileHelperTest.java =================================================================== --- branches/Branch_3_2/HibernateExt/metadata/src/test/org/hibernate/search/test/util/FileHelperTest.java (rev 0) +++ branches/Branch_3_2/HibernateExt/metadata/src/test/org/hibernate/search/test/util/FileHelperTest.java 2007-01-23 19:26:42 UTC (rev 11085) @@ -0,0 +1,82 @@ +//$Id: $ +package org.hibernate.search.test.util; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; + +import junit.framework.TestCase; +import org.hibernate.search.util.FileHelper; + +/** + * @author Emmanuel Bernard + */ +public class FileHelperTest extends TestCase { + public void testTiti() throws Exception { + File titi = new File("file:/c:/titi", "file:/d:/toito"); + assertFalse ( titi.exists() ); + } + + protected void setUp() throws Exception { + super.setUp(); + File dir = new File("./filehelpersrc"); + dir.mkdir(); + String name = "a"; + createFile( dir, name ); + name = "b"; + createFile( dir, name ); + dir = new File(dir, "subdir"); + dir.mkdir(); + name = "c"; + createFile( dir, name ); + } + + private void createFile(File dir, String name) throws IOException { + File a = new File(dir, name); + a.createNewFile(); + FileOutputStream os = new FileOutputStream( a, false ); + os.write( 1 ); + os.write( 2 ); + os.write( 3 ); + os.flush(); + os.close(); + } + + protected void tearDown() throws Exception { + super.setUp(); + File dir = new File("./filehelpersrc"); + FileHelper.delete( dir ); + dir = new File("./filehelperdest"); + FileHelper.delete( dir ); + } + + public void testSynchronize() throws Exception { + File src = new File("./filehelpersrc"); + File dest = new File("./filehelpertest"); + FileHelper.synchronize( src, dest, true ); + File test = new File(dest, "b"); + assertTrue( test.exists() ); + test = new File( new File(dest, "subdir"), "c"); + assertTrue( test.exists() ); + + //change + Thread.sleep( 2*2000 ); + test = new File( src, "c"); + FileOutputStream os = new FileOutputStream( test, true ); + os.write( 1 ); + os.write( 2 ); + os.write( 3 ); + os.flush(); + os.close(); + File destTest = new File(dest, "c"); + assertNotSame( test.lastModified(), destTest.lastModified() ); + FileHelper.synchronize( src, dest, true ); + assertEquals( test.lastModified(), destTest.lastModified() ); + assertEquals( test.length(), destTest.length() ); + + //delete file + test.delete(); + FileHelper.synchronize( src, dest, true ); + assertTrue( ! destTest.exists() ); + } +}