Hibernate SVN: r14730 - search/trunk/src/java/org/hibernate/search/engine.
by hibernate-commits＠lists.jboss.org
Author: epbernard
Date: 2008-06-03 16:09:33 -0400 (Tue, 03 Jun 2008)
New Revision: 14730
Modified:
search/trunk/src/java/org/hibernate/search/engine/ObjectLoader.java
Log:
HSEARCH-196 ONFE not caught when oblectLoader.load is used
Modified: search/trunk/src/java/org/hibernate/search/engine/ObjectLoader.java
===================================================================
--- search/trunk/src/java/org/hibernate/search/engine/ObjectLoader.java 2008-06-03 19:50:49 UTC (rev 14729)
+++ search/trunk/src/java/org/hibernate/search/engine/ObjectLoader.java 2008-06-03 20:09:33 UTC (rev 14730)
@@ -22,7 +22,7 @@
public Object load(EntityInfo entityInfo) {
//be sure to get an initialized object
- Object maybeProxy = session.get( entityInfo.clazz, entityInfo.id );
+ Object maybeProxy = session.load( entityInfo.clazz, entityInfo.id );
try {
Hibernate.initialize( maybeProxy );
}
16 years, 6 months
Hibernate SVN: r14729 - validator/trunk/src/test/org/hibernate/validator/test/validators.
by hibernate-commits＠lists.jboss.org
Author: epbernard
Date: 2008-06-03 15:50:49 -0400 (Tue, 03 Jun 2008)
New Revision: 14729
Modified:
validator/trunk/src/test/org/hibernate/validator/test/validators/Car.java
validator/trunk/src/test/org/hibernate/validator/test/validators/DigitsTest.java
Log:
Add more test to DigitsValidator (testing strings)
Modified: validator/trunk/src/test/org/hibernate/validator/test/validators/Car.java
===================================================================
--- validator/trunk/src/test/org/hibernate/validator/test/validators/Car.java 2008-06-03 19:11:06 UTC (rev 14728)
+++ validator/trunk/src/test/org/hibernate/validator/test/validators/Car.java 2008-06-03 19:50:49 UTC (rev 14729)
@@ -24,5 +24,7 @@
@Digits(integerDigits = 1, fractionalDigits = 2)
public BigDecimal length;
@Digits(integerDigits = 2, fractionalDigits = 1)
- public Double gallons;
+ public Double gallons;
+ @Digits(integerDigits = 2, fractionalDigits = 1)
+ public String mpg;
}
Modified: validator/trunk/src/test/org/hibernate/validator/test/validators/DigitsTest.java
===================================================================
--- validator/trunk/src/test/org/hibernate/validator/test/validators/DigitsTest.java 2008-06-03 19:11:06 UTC (rev 14728)
+++ validator/trunk/src/test/org/hibernate/validator/test/validators/DigitsTest.java 2008-06-03 19:50:49 UTC (rev 14729)
@@ -24,13 +24,14 @@
car.insurances = new String[] { "random" };
car.length = new BigDecimal(10.2);
car.gallons = 100.3;
+ car.mpg = "EFG";
ClassValidator<Car> classValidator = new ClassValidator<Car>( Car.class );
InvalidValue[] invalidValues = classValidator.getInvalidValues( car );
- assertEquals( 2, invalidValues.length );
+ assertEquals( 3, invalidValues.length );
car.length = new BigDecimal(1.223); //more than 2
car.gallons = 10.300; //1 digit really so not invalid
invalidValues = classValidator.getInvalidValues( car );
- assertEquals( 1, invalidValues.length );
+ assertEquals( 2, invalidValues.length );
}
public void testApply() throws Exception {
16 years, 6 months
Hibernate SVN: r14728 - search/trunk/src/java/org/hibernate/search/util.
by hibernate-commits＠lists.jboss.org
Author: sannegrinovero
Date: 2008-06-03 15:11:06 -0400 (Tue, 03 Jun 2008)
New Revision: 14728
Modified:
search/trunk/src/java/org/hibernate/search/util/FileHelper.java
Log:
forgot to remove System.out for "HSEARCH-205 Out of Memory on copy of large indexes"
Modified: search/trunk/src/java/org/hibernate/search/util/FileHelper.java
===================================================================
--- search/trunk/src/java/org/hibernate/search/util/FileHelper.java 2008-06-03 18:12:26 UTC (rev 14727)
+++ search/trunk/src/java/org/hibernate/search/util/FileHelper.java 2008-06-03 19:11:06 UTC (rev 14728)
@@ -76,7 +76,6 @@
private static void copyFile(File srcFile, File destFile, long chunkSize) throws IOException {
FileInputStream is = null;
FileOutputStream os = null;
- long startTime = System.currentTimeMillis();
try {
is = new FileInputStream(srcFile);
FileChannel iChannel = is.getChannel();
@@ -99,8 +98,6 @@
if (is != null) is.close();
if (os != null) os.close();
}
- long endTime = System.currentTimeMillis() - startTime;
- System.out.println("copied in "+ endTime + " ms.");
boolean successTimestampOp = destFile.setLastModified( srcFile.lastModified() );
if ( ! successTimestampOp ) {
log.warn( "Could not change timestamp for " + destFile +
16 years, 6 months
Hibernate SVN: r14727 - search/trunk/src/java/org/hibernate/search/util.
by hibernate-commits＠lists.jboss.org
Author: sannegrinovero
Date: 2008-06-03 14:12:26 -0400 (Tue, 03 Jun 2008)
New Revision: 14727
Modified:
search/trunk/src/java/org/hibernate/search/util/FileHelper.java
Log:
HSEARCH-205 Out of Memory on copy of large indexes
Modified: search/trunk/src/java/org/hibernate/search/util/FileHelper.java
===================================================================
--- search/trunk/src/java/org/hibernate/search/util/FileHelper.java 2008-06-03 08:24:26 UTC (rev 14726)
+++ search/trunk/src/java/org/hibernate/search/util/FileHelper.java 2008-06-03 18:12:26 UTC (rev 14727)
@@ -10,16 +10,29 @@
import java.util.Arrays;
import java.nio.channels.FileChannel;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
* @author Emmanuel Bernard
+ * @author Sanne Grinovero
*/
public abstract class FileHelper {
+
private static final int FAT_PRECISION = 2000;
+ private static final long DEFAULT_CHUNK_SIZE = 16 * 1024 * 1024; // 16 MB
+ private static final Logger log = LoggerFactory.getLogger( FileHelper.class );
+
+ public static void synchronize(File source, File destination, boolean smart) throws IOException {
+ synchronize( source, destination, smart, DEFAULT_CHUNK_SIZE );
+ }
- public static void synchronize(File source, File destination, boolean smart) throws IOException {
+ public static void synchronize(File source, File destination, boolean smart, long chunkSize) throws IOException {
if ( source.isDirectory() ) {
if ( ! destination.exists() ) {
- destination.mkdirs();
+ if ( ! destination.mkdirs() ){
+ throw new IOException("Could not create path " + destination);
+ }
}
else if ( ! destination.isDirectory() ) {
throw new IOException("Source and Destination not of the same type:"
@@ -39,7 +52,7 @@
for (String fileName : sources) {
File srcFile = new File(source, fileName);
File destFile = new File(destination, fileName);
- synchronize( srcFile, destFile, smart );
+ synchronize( srcFile, destFile, smart, chunkSize );
}
}
else {
@@ -51,36 +64,61 @@
long dts = destination.lastModified() / FAT_PRECISION;
//do not copy if smart and same timestamp and same length
if ( !smart || sts == 0 || sts != dts || source.length() != destination.length() ) {
- copyFile(source, destination);
+ copyFile(source, destination, chunkSize);
}
}
else {
- copyFile(source, destination);
+ copyFile(source, destination, chunkSize);
}
}
}
- private static void copyFile(File srcFile, File destFile) throws IOException {
+ private static void copyFile(File srcFile, File destFile, long chunkSize) throws IOException {
FileInputStream is = null;
FileOutputStream os = null;
+ long startTime = System.currentTimeMillis();
try {
is = new FileInputStream(srcFile);
FileChannel iChannel = is.getChannel();
os = new FileOutputStream( destFile, false );
FileChannel oChannel = os.getChannel();
- oChannel.transferFrom( iChannel, 0, srcFile.length() );
+ long doneBytes = 0L;
+ long todoBytes = srcFile.length();
+ while ( todoBytes != 0L ) {
+ long iterationBytes = Math.min( todoBytes, chunkSize );
+ long transferredLength = oChannel.transferFrom( iChannel, doneBytes, iterationBytes );
+ if ( iterationBytes != transferredLength ) {
+ throw new IOException( "Error during file transfer: expected "
+ + iterationBytes + " bytes, only "+ transferredLength + " bytes copied." );
+ }
+ doneBytes += transferredLength;
+ todoBytes -= transferredLength;
+ }
}
finally {
if (is != null) is.close();
if (os != null) os.close();
}
- destFile.setLastModified( srcFile.lastModified() );
+ long endTime = System.currentTimeMillis() - startTime;
+ System.out.println("copied in "+ endTime + " ms.");
+ boolean successTimestampOp = destFile.setLastModified( srcFile.lastModified() );
+ if ( ! successTimestampOp ) {
+ log.warn( "Could not change timestamp for " + destFile +
+ ". Index synchronization may be slow." );
+ }
}
public static void delete(File file) {
if ( file.isDirectory() ) {
- for ( File subFile : file.listFiles() ) delete( subFile );
+ for ( File subFile : file.listFiles() ) {
+ delete( subFile );
+ }
}
- if ( file.exists() ) file.delete();
+ if ( file.exists() ) {
+ if ( ! file.delete() ) {
+ log.error( "Could not delete " + file );
+ }
+ }
}
+
}
16 years, 6 months
Hibernate SVN: r14725 - in search/trunk/src/java/org/hibernate/search: query and 1 other directory.
by hibernate-commits＠lists.jboss.org
Author: sannegrinovero
Date: 2008-06-02 20:34:56 -0400 (Mon, 02 Jun 2008)
New Revision: 14725
Modified:
search/trunk/src/java/org/hibernate/search/backend/LuceneIndexingParameters.java
search/trunk/src/java/org/hibernate/search/query/FullTextQueryImpl.java
Log:
trivial improvements about use of java.util Collections
Modified: search/trunk/src/java/org/hibernate/search/backend/LuceneIndexingParameters.java
===================================================================
--- search/trunk/src/java/org/hibernate/search/backend/LuceneIndexingParameters.java 2008-06-02 22:55:36 UTC (rev 14724)
+++ search/trunk/src/java/org/hibernate/search/backend/LuceneIndexingParameters.java 2008-06-03 00:34:56 UTC (rev 14725)
@@ -2,7 +2,7 @@
package org.hibernate.search.backend;
import java.io.Serializable;
-import java.util.HashMap;
+import java.util.EnumMap;
import java.util.Map;
import java.util.Properties;
@@ -81,7 +81,7 @@
private static final long serialVersionUID = -6121723702279869524L;
- final Map<IndexWriterSetting, Integer> parameters = new HashMap<IndexWriterSetting, Integer>();
+ final Map<IndexWriterSetting, Integer> parameters = new EnumMap<IndexWriterSetting, Integer>(IndexWriterSetting.class);
public ParameterSet(Properties prop, String paramName) {
//don't iterate on property entries as we know all the keys:
Modified: search/trunk/src/java/org/hibernate/search/query/FullTextQueryImpl.java
===================================================================
--- search/trunk/src/java/org/hibernate/search/query/FullTextQueryImpl.java 2008-06-02 22:55:36 UTC (rev 14724)
+++ search/trunk/src/java/org/hibernate/search/query/FullTextQueryImpl.java 2008-06-03 00:34:56 UTC (rev 14725)
@@ -66,7 +66,7 @@
//TODO implements setParameter()
public class FullTextQueryImpl extends AbstractQueryImpl implements FullTextQuery {
private final Logger log = LoggerFactory.getLogger( FullTextQueryImpl.class );
- private org.apache.lucene.search.Query luceneQuery;
+ private final org.apache.lucene.search.Query luceneQuery;
private Class[] classes;
private Set<Class> classesAndSubclasses;
//optimization: if we can avoid the filter clause (we can most of the time) do it as it has a significant perf impact
@@ -124,7 +124,7 @@
//find the directories
IndexSearcher searcher = buildSearcher( searchFactoryImplementor );
if ( searcher == null ) {
- return new IteratorImpl( new ArrayList<EntityInfo>( 0 ), noLoader );
+ return new IteratorImpl( Collections.EMPTY_LIST, noLoader );
}
try {
Hits hits = getHits( searcher );
@@ -234,7 +234,7 @@
SearchFactoryImplementor searchFactoryImplementor = ContextHelper.getSearchFactoryBySFI( session );
//find the directories
IndexSearcher searcher = buildSearcher( searchFactoryImplementor );
- if ( searcher == null ) return new ArrayList( 0 );
+ if ( searcher == null ) return Collections.EMPTY_LIST;
Hits hits;
try {
hits = getHits( searcher );
16 years, 6 months
Hibernate SVN: r14724 - search/trunk/src/test/org/hibernate/search/test/perf.
by hibernate-commits＠lists.jboss.org
Author: sannegrinovero
Date: 2008-06-02 18:55:36 -0400 (Mon, 02 Jun 2008)
New Revision: 14724
Modified:
search/trunk/src/test/org/hibernate/search/test/perf/IndexTestDontRun.java
search/trunk/src/test/org/hibernate/search/test/perf/SearcherThread.java
Log:
Improvements in performance test comparing to raw Lucene
Modified: search/trunk/src/test/org/hibernate/search/test/perf/IndexTestDontRun.java
===================================================================
--- search/trunk/src/test/org/hibernate/search/test/perf/IndexTestDontRun.java 2008-06-02 18:02:15 UTC (rev 14723)
+++ search/trunk/src/test/org/hibernate/search/test/perf/IndexTestDontRun.java 2008-06-02 22:55:36 UTC (rev 14724)
@@ -3,6 +3,10 @@
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
import junit.textui.TestRunner;
import org.apache.lucene.search.IndexSearcher;
@@ -18,15 +22,16 @@
* @author Emmanuel Bernard
*/
public class IndexTestDontRun extends SearchTestCase {
- private static boolean isLucene;
+
+ private static final int TOTAL_SEARCHES = 800;
+ private static final int SEARCH_THREADS = 100;
public static void main(String[] args) {
//isLucene = Boolean.parseBoolean( args[0] );
TestRunner.run( IndexTestDontRun.class );
-
}
- public void NonestInit() throws Exception {
+ public void notestInit() throws Exception {
long time = System.currentTimeMillis();
Session s = openSession();
Transaction tx = s.beginTransaction();
@@ -37,26 +42,39 @@
s.close();
System.out.println( " init time = " + ( System.currentTimeMillis() - time ) );
}
+
+ public void testPerformance() throws Exception {
+ measure(true);//JVM warmup
+ measure(false);//JVM warmup
+ long measureLucene = measure( true );
+ long measureSearch = measure( false );
+ System.out.println( "Totaltime Lucene = " + measureLucene );
+ System.out.println( "Totaltime Search = " + measureSearch );
+ }
- public void testPerf() throws Exception {
- boolean useLucene = false;
-
- List<SearcherThread> threads = new ArrayList<SearcherThread>( 100 );
- IndexSearcher indexsearcher = getNewSearcher();
- SearcherThread searcherThrea = new SearcherThread( 0, "name:maria OR description:long" + 0, getSessions(), indexsearcher, useLucene );
- searcherThrea.run();
- for (int i = 1; i <= 100; i++) {
+ public long measure(boolean plainLucene) throws Exception {
+ ThreadPoolExecutor threadPool = (ThreadPoolExecutor) Executors.newFixedThreadPool( SEARCH_THREADS );
+ threadPool.prestartAllCoreThreads();
+ CountDownLatch startSignal = new CountDownLatch(1);
+ List<SearcherThread> threadsList = new ArrayList<SearcherThread>( TOTAL_SEARCHES );
+ IndexSearcher indexSearcher = getNewSearcher();
+ for (int i = 0; i < TOTAL_SEARCHES; i++) {
// Create a thread and invoke it
- //if ( i % 100 == 0) indexsearcher = getNewSearcher();
- SearcherThread searcherThread = new SearcherThread( i, "name:maria OR description:long" + i, getSessions(), indexsearcher, useLucene );
- searcherThread.setDaemon( false );
- threads.add( searcherThread );
- searcherThread.start();
+ //if ( i % 100 == 0) indexSearcher = getNewSearcher();
+ SearcherThread searcherThread = new SearcherThread( i, "name:maria OR description:long" + i, getSessions(), indexSearcher, plainLucene, startSignal );
+ threadsList.add( searcherThread );
+ threadPool.execute( searcherThread );
}
- Thread.sleep( 5000 );
+ threadPool.shutdown();//required to enable awaitTermination functionality
+ startSignal.countDown();//start all created threads
+ boolean terminationOk = threadPool.awaitTermination( 60, TimeUnit.SECONDS );
+ if ( terminationOk==false ) {
+ System.out.println( "No enough time to complete the tests!" );
+ return 0;
+ }
long totalTime = 0;
- for (SearcherThread t : threads) totalTime += t.time;
- System.out.println( "Totaltime=" + totalTime );
+ for (SearcherThread t : threadsList) totalTime += t.getTime();
+ return totalTime;
}
private IndexSearcher getNewSearcher() throws IOException {
@@ -75,7 +93,6 @@
protected void configure(Configuration cfg) {
super.configure( cfg );
cfg.setProperty( "hibernate.search.default.directory_provider", FSDirectoryProvider.class.getName() );
- //cfg.setProperty( "hibernate.search.reader.strategy", DumbSharedReaderProvider.class.getName() );
-
+// cfg.setProperty( "hibernate.search.reader.strategy", DumbSharedReaderProvider.class.getName() );
}
}
Modified: search/trunk/src/test/org/hibernate/search/test/perf/SearcherThread.java
===================================================================
--- search/trunk/src/test/org/hibernate/search/test/perf/SearcherThread.java 2008-06-02 18:02:15 UTC (rev 14723)
+++ search/trunk/src/test/org/hibernate/search/test/perf/SearcherThread.java 2008-06-02 22:55:36 UTC (rev 14724)
@@ -3,6 +3,7 @@
import java.io.IOException;
import java.util.List;
import java.util.ArrayList;
+import java.util.concurrent.CountDownLatch;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.queryParser.ParseException;
@@ -11,9 +12,7 @@
import org.apache.lucene.search.Hits;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.store.Directory;
import org.hibernate.SessionFactory;
-import org.hibernate.classic.Session;
import org.hibernate.search.Search;
import org.hibernate.search.FullTextSession;
import org.hibernate.search.FullTextQuery;
@@ -23,27 +22,39 @@
/**
* @author Emmanuel Bernard
*/
-public class SearcherThread extends Thread {
- private static Logger log = LoggerFactory.getLogger( SearcherThread.class );
- private int threadId;
- private String queryString;
- private SessionFactory sf;
- private IndexSearcher indexsearcher;
- private boolean isLucene;
- public long time;
+public class SearcherThread implements Runnable {
+ private static final Logger log = LoggerFactory.getLogger( SearcherThread.class );
+ private final int threadId;
+ private final String queryString;
+ private final SessionFactory sf;
+ private final IndexSearcher indexsearcher;
+ private final boolean isLucene;
+ private final CountDownLatch startSignal;
+ private long time;
/**
- * Initialize with thread-id, querystring, indexsearcher
+ * Initialize with thread-id, queryString, indexSearcher
+ * @param startSignal
*/
- public SearcherThread(int threadId, String queryString, SessionFactory sf, IndexSearcher indexSearcher, boolean isLucene) {
+ public SearcherThread(int threadId, String queryString, SessionFactory sf, IndexSearcher indexSearcher, boolean isLucene, CountDownLatch startSignal) {
this.isLucene = isLucene;
this.threadId = threadId;
this.queryString = queryString;
this.sf = sf;
this.indexsearcher = indexSearcher;
+ this.startSignal = startSignal;
}
+ /**
+ * @see java.lang.Runnable#run()
+ */
public void run() {
+ try {
+ startSignal.await();
+ } catch (InterruptedException e) {
+ log.error( "tests canceled", e );
+ return;
+ }
if ( isLucene ) {
runLucene();
}
@@ -52,34 +63,22 @@
}
}
- /**
- * @see java.lang.Runnable#run()
- */
public void runLucene() {
-
try {
- QueryParser qp = new QueryParser( "t",
- new StandardAnalyzer() );
- qp.setLowercaseExpandedTerms( true );
- // Parse the query
- Query q = qp.parse( queryString );
- if ( q instanceof BooleanQuery ) {
- BooleanQuery
- .setMaxClauseCount( Integer.MAX_VALUE );
- }
+ Query q = getQuery();
long start = System.currentTimeMillis();
// Search
Hits hits = indexsearcher.search( q );
List<String> names = new ArrayList<String>(100);
- for (int i = 1 ; i <= 100 ; i++) {
+ for (int i = 0 ; i < 100 ; i++) {
names.add( hits.doc( i ).get( "name" ) );
}
+ int resultSize = hits.length();
long totalTime = System.currentTimeMillis() - start;
- log.error( "Lucene [ Thread-id : " + threadId + " ] Total time taken for search is : " + totalTime + "ms with total no. of matching records : " + hits.length() );
- time = totalTime;
+// log.error( "Lucene [ Thread-id : " + threadId + " ] Total time taken for search is : " + totalTime + "ms with total no. of matching records : " + hits.length() );
+ setTime( totalTime );
}
catch (ParseException e) {
- // TODO Auto-generated catch block
System.out.println( "[ Thread-id : " + threadId + " ] Parse Exception for queryString : " + queryString );
e.printStackTrace();
}
@@ -91,32 +90,33 @@
}
}
+ private Query getQuery() throws ParseException {
+ QueryParser qp = new QueryParser( "t", new StandardAnalyzer() );
+ qp.setLowercaseExpandedTerms( true );
+ // Parse the query
+ Query q = qp.parse( queryString );
+ if ( q instanceof BooleanQuery ) {
+ BooleanQuery.setMaxClauseCount( Integer.MAX_VALUE );
+ }
+ return q;
+ }
+
public void runHSearch() {
-
try {
- QueryParser qp = new QueryParser( "t",
- new StandardAnalyzer() );
- qp.setLowercaseExpandedTerms( true );
-
- // Parse the query
- Query q = qp.parse( queryString );
-
-
+ Query q = getQuery();
// Search
FullTextSession ftSession = Search.createFullTextSession( sf.openSession( ) );
-
final FullTextQuery textQuery = ftSession.createFullTextQuery( q, Boat.class )
.setMaxResults( 100 ).setProjection( "name" );
long start = System.currentTimeMillis();
List results = textQuery.list();
+ int resultSize = textQuery.getResultSize();
long totalTime = System.currentTimeMillis() - start;
ftSession.close();
-
- log.error( "HSearch [ Thread-id : " + threadId + " ] Total time taken for search is : " + totalTime + "ms with total no. of matching records : " + textQuery.getResultSize() );
- time = totalTime;
+// log.error( "HSearch [ Thread-id : " + threadId + " ] Total time taken for search is : " + totalTime + "ms with total no. of matching records : " + resultSize );
+ setTime( totalTime );
}
catch (ParseException e) {
- // TODO Auto-generated catch block
log.error( "[ Thread-id : " + threadId + " ] Parse Exception for queryString : " + queryString );
e.printStackTrace();
}
@@ -125,4 +125,13 @@
e.printStackTrace( );
}
}
+
+ public synchronized long getTime() {
+ return time;
+ }
+
+ public synchronized void setTime(long time) {
+ this.time = time;
+ }
+
}
16 years, 6 months
Hibernate SVN: r14723 - search/branches.
by hibernate-commits＠lists.jboss.org
Author: epbernard
Date: 2008-06-02 14:02:15 -0400 (Mon, 02 Jun 2008)
New Revision: 14723
Added:
search/branches/jboss_cache_integration/
Log:
Branch off at r14722 to start JBoss Cache integration
Copied: search/branches/jboss_cache_integration (from rev 14722, search/trunk)
16 years, 6 months