[jboss-cvs] JBoss Messaging SVN: r7516 - in trunk: native/bin and 17 other directories.
jboss-cvs-commits at lists.jboss.org
jboss-cvs-commits at lists.jboss.org
Thu Jul 2 17:36:16 EDT 2009
Author: clebert.suconic at jboss.com
Date: 2009-07-02 17:36:15 -0400 (Thu, 02 Jul 2009)
New Revision: 7516
Added:
trunk/src/main/org/jboss/messaging/core/journal/LoaderCallback.java
trunk/src/main/org/jboss/messaging/core/journal/impl/JournalCompactor.java
trunk/src/main/org/jboss/messaging/core/journal/impl/JournalReaderCallback.java
trunk/src/main/org/jboss/messaging/core/journal/impl/JournalReaderCallbackAbstract.java
trunk/src/main/org/jboss/messaging/core/journal/impl/JournalTransaction.java
trunk/src/main/org/jboss/messaging/core/journal/impl/TransactionCallback.java
trunk/tests/src/org/jboss/messaging/tests/integration/journal/AIOJournalCompactTest.java
trunk/tests/src/org/jboss/messaging/tests/integration/journal/NIOJournalCompactTest.java
Removed:
trunk/src/main/org/jboss/messaging/core/journal/LoadManager.java
Modified:
trunk/examples/core/perf/build.xml
trunk/native/bin/libJBMLibAIO32.so
trunk/native/bin/libJBMLibAIO64.so
trunk/native/src/JNICallbackAdapter.cpp
trunk/native/src/JNICallbackAdapter.h
trunk/native/src/JNI_AsynchronousFileImpl.cpp
trunk/native/src/Version.h
trunk/src/config/common/schema/jbm-configuration.xsd
trunk/src/main/org/jboss/messaging/core/asyncio/impl/AsynchronousFileImpl.java
trunk/src/main/org/jboss/messaging/core/config/Configuration.java
trunk/src/main/org/jboss/messaging/core/config/impl/ConfigurationImpl.java
trunk/src/main/org/jboss/messaging/core/config/impl/FileConfiguration.java
trunk/src/main/org/jboss/messaging/core/config/impl/Validators.java
trunk/src/main/org/jboss/messaging/core/journal/Journal.java
trunk/src/main/org/jboss/messaging/core/journal/SequentialFile.java
trunk/src/main/org/jboss/messaging/core/journal/SequentialFileFactory.java
trunk/src/main/org/jboss/messaging/core/journal/TestableJournal.java
trunk/src/main/org/jboss/messaging/core/journal/impl/AIOSequentialFile.java
trunk/src/main/org/jboss/messaging/core/journal/impl/AIOSequentialFileFactory.java
trunk/src/main/org/jboss/messaging/core/journal/impl/AbstractSequentialFactory.java
trunk/src/main/org/jboss/messaging/core/journal/impl/DummyCallback.java
trunk/src/main/org/jboss/messaging/core/journal/impl/JournalFile.java
trunk/src/main/org/jboss/messaging/core/journal/impl/JournalFileImpl.java
trunk/src/main/org/jboss/messaging/core/journal/impl/JournalImpl.java
trunk/src/main/org/jboss/messaging/core/journal/impl/NIOSequentialFile.java
trunk/src/main/org/jboss/messaging/core/persistence/impl/journal/JournalStorageManager.java
trunk/tests/config/ConfigurationTest-full-config.xml
trunk/tests/src/org/jboss/messaging/tests/integration/journal/NIOSequentialFileFactoryTest.java
trunk/tests/src/org/jboss/messaging/tests/integration/journal/RealAIOJournalImplTest.java
trunk/tests/src/org/jboss/messaging/tests/performance/journal/JournalImplTestUnit.java
trunk/tests/src/org/jboss/messaging/tests/performance/journal/RealJournalImplAIOTest.java
trunk/tests/src/org/jboss/messaging/tests/performance/journal/RealJournalImplNIOTest.java
trunk/tests/src/org/jboss/messaging/tests/stress/journal/AddAndRemoveStressTest.java
trunk/tests/src/org/jboss/messaging/tests/stress/journal/ValidateTransactionHealthTest.java
trunk/tests/src/org/jboss/messaging/tests/stress/journal/remote/RemoteJournalAppender.java
trunk/tests/src/org/jboss/messaging/tests/unit/core/config/impl/DefaultsFileConfigurationTest.java
trunk/tests/src/org/jboss/messaging/tests/unit/core/config/impl/FileConfigurationTest.java
trunk/tests/src/org/jboss/messaging/tests/unit/core/journal/impl/AlignedJournalImplTest.java
trunk/tests/src/org/jboss/messaging/tests/unit/core/journal/impl/JournalAsyncTest.java
trunk/tests/src/org/jboss/messaging/tests/unit/core/journal/impl/JournalImplTestBase.java
trunk/tests/src/org/jboss/messaging/tests/unit/core/journal/impl/JournalImplTestUnit.java
trunk/tests/src/org/jboss/messaging/tests/unit/core/journal/impl/ReclaimerTest.java
trunk/tests/src/org/jboss/messaging/tests/unit/core/journal/impl/SequentialFileFactoryTestBase.java
trunk/tests/src/org/jboss/messaging/tests/unit/core/journal/impl/fakes/FakeSequentialFileFactory.java
trunk/tests/src/org/jboss/messaging/tests/util/JournalExample.java
trunk/tests/src/org/jboss/messaging/tests/util/ListJournal.java
trunk/tests/src/org/jboss/messaging/tests/util/ServiceTestBase.java
Log:
JBMESSAGING-1504 - Full Journal Compacting
Modified: trunk/examples/core/perf/build.xml
===================================================================
--- trunk/examples/core/perf/build.xml 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/examples/core/perf/build.xml 2009-07-02 21:36:15 UTC (rev 7516)
@@ -35,8 +35,11 @@
<path id="extra.classpath">
<path location="server0"/>
</path>
+
+
+ <property name="file.name" value="perf.properties"/>
- <property name="perf.properties.file.name" value="perf.properties" />
+ <property name="perf.properties.file.name" value="${file.name}" />
<target name="runSender" depends="compile">
<java classname="org.jboss.core.example.PerfSender" fork="true" resultproperty="example-result">
@@ -78,4 +81,4 @@
</java>
</target>
-</project>
\ No newline at end of file
+</project>
Modified: trunk/native/bin/libJBMLibAIO32.so
===================================================================
(Binary files differ)
Modified: trunk/native/bin/libJBMLibAIO64.so
===================================================================
(Binary files differ)
Modified: trunk/native/src/JNICallbackAdapter.cpp
===================================================================
--- trunk/native/src/JNICallbackAdapter.cpp 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/native/src/JNICallbackAdapter.cpp 2009-07-02 21:36:15 UTC (rev 7516)
@@ -22,12 +22,15 @@
#include <iostream>
#include "JavaUtilities.h"
-JNICallbackAdapter::JNICallbackAdapter(AIOController * _controller, jobject _callback, jobject _fileController, jobject _bufferReference) : CallbackAdapter()
+jobject nullObj = NULL;
+
+JNICallbackAdapter::JNICallbackAdapter(AIOController * _controller, jobject _callback, jobject _fileController, jobject _bufferReference, short _isRead) : CallbackAdapter()
{
controller = _controller;
callback = _callback;
fileController = _fileController;
bufferReference = _bufferReference;
+ isRead = _isRead;
}
JNICallbackAdapter::~JNICallbackAdapter()
@@ -36,7 +39,7 @@
void JNICallbackAdapter::done(THREAD_CONTEXT threadContext)
{
- JNI_ENV(threadContext)->CallVoidMethod(fileController, controller->done, callback, bufferReference);
+ JNI_ENV(threadContext)->CallVoidMethod(fileController, controller->done, callback, isRead ? nullObj : bufferReference);
release(threadContext);
}
@@ -44,7 +47,7 @@
{
controller->log(threadContext, 0, "Libaio event generated errors, callback object was informed about it");
jstring strError = JNI_ENV(threadContext)->NewStringUTF(error.data());
- JNI_ENV(threadContext)->CallVoidMethod(fileController, controller->error, callback, (jint)errorCode, strError);
+ JNI_ENV(threadContext)->CallVoidMethod(fileController, controller->error, callback, isRead ? nullObj : bufferReference, (jint)errorCode, strError);
release(threadContext);
}
Modified: trunk/native/src/JNICallbackAdapter.h
===================================================================
--- trunk/native/src/JNICallbackAdapter.h 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/native/src/JNICallbackAdapter.h 2009-07-02 21:36:15 UTC (rev 7516)
@@ -34,6 +34,8 @@
jobject callback;
jobject fileController;
jobject bufferReference;
+ // Is this a read operation
+ short isRead;
void release(THREAD_CONTEXT threadContext)
{
@@ -47,7 +49,7 @@
public:
// _ob must be a global Reference (use createGloblReferente before calling the constructor)
- JNICallbackAdapter(AIOController * _controller, jobject _callback, jobject _fileController, jobject _bufferReference);
+ JNICallbackAdapter(AIOController * _controller, jobject _callback, jobject _fileController, jobject _bufferReference, short _isRead);
virtual ~JNICallbackAdapter();
void done(THREAD_CONTEXT threadContext);
Modified: trunk/native/src/JNI_AsynchronousFileImpl.cpp
===================================================================
--- trunk/native/src/JNI_AsynchronousFileImpl.cpp 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/native/src/JNI_AsynchronousFileImpl.cpp 2009-07-02 21:36:15 UTC (rev 7516)
@@ -57,7 +57,7 @@
controller->done = env->GetMethodID(clazz,"callbackDone","(Lorg/jboss/messaging/core/asyncio/AIOCallback;Ljava/nio/ByteBuffer;)V");
if (!controller->done) return 0;
- controller->error = env->GetMethodID(clazz, "callbackError", "(Lorg/jboss/messaging/core/asyncio/AIOCallback;ILjava/lang/String;)V");
+ controller->error = env->GetMethodID(clazz, "callbackError", "(Lorg/jboss/messaging/core/asyncio/AIOCallback;Ljava/nio/ByteBuffer;ILjava/lang/String;)V");
if (!controller->error) return 0;
jclass loggerClass = env->GetObjectClass(logger);
@@ -103,7 +103,7 @@
return;
}
- CallbackAdapter * adapter = new JNICallbackAdapter(controller, env->NewGlobalRef(callback), env->NewGlobalRef(objThis), env->NewGlobalRef(jbuffer));
+ CallbackAdapter * adapter = new JNICallbackAdapter(controller, env->NewGlobalRef(callback), env->NewGlobalRef(objThis), env->NewGlobalRef(jbuffer), true);
controller->fileOutput.read(env, position, (size_t)size, buffer, adapter);
}
@@ -186,7 +186,7 @@
}
- CallbackAdapter * adapter = new JNICallbackAdapter(controller, env->NewGlobalRef(callback), env->NewGlobalRef(objThis), env->NewGlobalRef(jbuffer));
+ CallbackAdapter * adapter = new JNICallbackAdapter(controller, env->NewGlobalRef(callback), env->NewGlobalRef(objThis), env->NewGlobalRef(jbuffer), false);
controller->fileOutput.write(env, position, (size_t)size, buffer, adapter);
}
Modified: trunk/native/src/Version.h
===================================================================
--- trunk/native/src/Version.h 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/native/src/Version.h 2009-07-02 21:36:15 UTC (rev 7516)
@@ -1,5 +1,5 @@
#ifndef _VERSION_NATIVE_AIO
-#define _VERSION_NATIVE_AIO 21
+#define _VERSION_NATIVE_AIO 22
#endif
Modified: trunk/src/config/common/schema/jbm-configuration.xsd
===================================================================
--- trunk/src/config/common/schema/jbm-configuration.xsd 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/src/config/common/schema/jbm-configuration.xsd 2009-07-02 21:36:15 UTC (rev 7516)
@@ -158,6 +158,10 @@
</xsd:element>
<xsd:element maxOccurs="1" minOccurs="0" name="journal-min-files" type="xsd:int">
</xsd:element>
+ <xsd:element maxOccurs="1" minOccurs="0" name="journal-compact-percentage" type="xsd:int">
+ </xsd:element>
+ <xsd:element maxOccurs="1" minOccurs="0" name="journal-compact-min-files" type="xsd:int">
+ </xsd:element>
<xsd:element maxOccurs="1" minOccurs="0" name="journal-max-aio" type="xsd:int">
</xsd:element>
<xsd:element maxOccurs="1" minOccurs="0" name="perf-blast-pages" type="xsd:int">
Modified: trunk/src/main/org/jboss/messaging/core/asyncio/impl/AsynchronousFileImpl.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/asyncio/impl/AsynchronousFileImpl.java 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/src/main/org/jboss/messaging/core/asyncio/impl/AsynchronousFileImpl.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -55,7 +55,7 @@
private static boolean loaded = false;
- private static int EXPECTED_NATIVE_VERSION = 21;
+ private static int EXPECTED_NATIVE_VERSION = 22;
public static void addMax(final int io)
{
@@ -289,11 +289,11 @@
}
catch (MessagingException e)
{
- callbackError(aioCallback, e.getCode(), e.getMessage());
+ callbackError(aioCallback, directByteBuffer, e.getCode(), e.getMessage());
}
catch (RuntimeException e)
{
- callbackError(aioCallback, MessagingException.INTERNAL_ERROR, e.getMessage());
+ callbackError(aioCallback, directByteBuffer, MessagingException.INTERNAL_ERROR, e.getMessage());
}
}
});
@@ -308,11 +308,11 @@
}
catch (MessagingException e)
{
- callbackError(aioCallback, e.getCode(), e.getMessage());
+ callbackError(aioCallback, directByteBuffer, e.getCode(), e.getMessage());
}
catch (RuntimeException e)
{
- callbackError(aioCallback, MessagingException.INTERNAL_ERROR, e.getMessage());
+ callbackError(aioCallback, directByteBuffer, MessagingException.INTERNAL_ERROR, e.getMessage());
}
}
@@ -418,7 +418,9 @@
writeSemaphore.release();
pendingWrites.down();
callback.done();
- if (bufferCallback != null)
+
+ // The buffer is not sent on callback for read operations
+ if (bufferCallback != null && buffer != null)
{
bufferCallback.bufferDone(buffer);
}
@@ -426,12 +428,18 @@
// Called by the JNI layer.. just ignore the
// warning
- private void callbackError(final AIOCallback callback, final int errorCode, final String errorMessage)
+ private void callbackError(final AIOCallback callback, final ByteBuffer buffer, final int errorCode, final String errorMessage)
{
log.warn("CallbackError: " + errorMessage);
writeSemaphore.release();
pendingWrites.down();
callback.onError(errorCode, errorMessage);
+
+ // The buffer is not sent on callback for read operations
+ if (bufferCallback != null && buffer != null)
+ {
+ bufferCallback.bufferDone(buffer);
+ }
}
private void pollEvents()
Modified: trunk/src/main/org/jboss/messaging/core/config/Configuration.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/config/Configuration.java 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/src/main/org/jboss/messaging/core/config/Configuration.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -185,6 +185,14 @@
int getJournalFileSize();
void setJournalFileSize(int size);
+
+ int getJournalCompactMinFiles();
+
+ void setJournalCompactMinFiles(int minFiles);
+
+ int getJournalCompactPercentage();
+
+ void setJournalCompactPercentage(int percentage);
int getJournalMinFiles();
Modified: trunk/src/main/org/jboss/messaging/core/config/impl/ConfigurationImpl.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/config/impl/ConfigurationImpl.java 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/src/main/org/jboss/messaging/core/config/impl/ConfigurationImpl.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -90,6 +90,10 @@
public static final boolean DEFAULT_JOURNAL_SYNC_NON_TRANSACTIONAL = false;
public static final int DEFAULT_JOURNAL_FILE_SIZE = 10485760;
+
+ public static final int DEFAULT_JOURNAL_COMPACT_MIN_FILES = 10;
+
+ public static final int DEFAULT_JOURNAL_COMPACT_PERCENTAGE = 30;
public static final int DEFAULT_JOURNAL_MIN_FILES = 2;
@@ -235,6 +239,11 @@
protected boolean journalSyncNonTransactional = DEFAULT_JOURNAL_SYNC_NON_TRANSACTIONAL;
+
+ protected int journalCompactMinFiles = DEFAULT_JOURNAL_COMPACT_MIN_FILES;
+
+ protected int journalCompactPercentage = DEFAULT_JOURNAL_COMPACT_PERCENTAGE;
+
protected int journalFileSize = DEFAULT_JOURNAL_FILE_SIZE;
protected int journalMinFiles = DEFAULT_JOURNAL_MIN_FILES;
@@ -886,4 +895,36 @@
cother.getManagementAddress().equals(getManagementAddress()) &&
cother.getGlobalPagingSize() == getGlobalPagingSize();
}
+
+ /* (non-Javadoc)
+ * @see org.jboss.messaging.core.config.Configuration#getJournalCompactMinFiles()
+ */
+ public int getJournalCompactMinFiles()
+ {
+ return journalCompactMinFiles;
+ }
+
+ /* (non-Javadoc)
+ * @see org.jboss.messaging.core.config.Configuration#getJournalCompactPercentage()
+ */
+ public int getJournalCompactPercentage()
+ {
+ return journalCompactPercentage;
+ }
+
+ /* (non-Javadoc)
+ * @see org.jboss.messaging.core.config.Configuration#setJournalCompactMinFiles()
+ */
+ public void setJournalCompactMinFiles(int minFiles)
+ {
+ this.journalCompactMinFiles = minFiles;
+ }
+
+ /* (non-Javadoc)
+ * @see org.jboss.messaging.core.config.Configuration#setJournalCompactPercentage()
+ */
+ public void setJournalCompactPercentage(int percentage)
+ {
+ this.journalCompactPercentage = percentage;
+ }
}
Modified: trunk/src/main/org/jboss/messaging/core/config/impl/FileConfiguration.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/config/impl/FileConfiguration.java 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/src/main/org/jboss/messaging/core/config/impl/FileConfiguration.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -30,6 +30,7 @@
import static org.jboss.messaging.core.config.impl.Validators.MINUS_ONE_OR_GT_ZERO;
import static org.jboss.messaging.core.config.impl.Validators.NOT_NULL_OR_EMPTY;
import static org.jboss.messaging.core.config.impl.Validators.NO_CHECK;
+import static org.jboss.messaging.core.config.impl.Validators.PERCENTAGE;
import static org.jboss.messaging.core.config.impl.Validators.THREAD_PRIORITY_RANGE;
import static org.jboss.messaging.utils.XMLConfigurationUtil.getBoolean;
import static org.jboss.messaging.utils.XMLConfigurationUtil.getDouble;
@@ -313,7 +314,11 @@
journalAIOBufferSize = getInteger(e, "journal-aio-buffer-size", DEFAULT_JOURNAL_AIO_BUFFER_SIZE, GT_ZERO);
journalMinFiles = getInteger(e, "journal-min-files", journalMinFiles, GT_ZERO);
+
+ journalCompactMinFiles = getInteger(e, "journal-compact-min-files", journalCompactMinFiles, GE_ZERO);
+ journalCompactPercentage = getInteger(e, "journal-compact-percentage", journalCompactPercentage, PERCENTAGE);
+
journalMaxAIO = getInteger(e, "journal-max-aio", journalMaxAIO, GT_ZERO);
logJournalWriteRate = getBoolean(e, "log-journal-write-rate", DEFAULT_JOURNAL_LOG_WRITE_RATE);
Modified: trunk/src/main/org/jboss/messaging/core/config/impl/Validators.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/config/impl/Validators.java 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/src/main/org/jboss/messaging/core/config/impl/Validators.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -78,11 +78,23 @@
}
else
{
- throw new IllegalArgumentException(format("%s must be greater than 0 (actual value: %s", name, val));
+ throw new IllegalArgumentException(format("%s must be greater than 0 (actual value: %s)", name, val));
}
}
};
+ public static Validator PERCENTAGE = new Validator()
+ {
+ public void validate(String name, Object value)
+ {
+ Number val = (Number)value;
+ if (val != null && val.intValue()< 0 || val.intValue() > 100)
+ {
+ throw new IllegalArgumentException(format("%s must be a valid percentual value between 0 and 100 (actual value: %s)", name, val));
+ }
+ }
+ };
+
public static Validator GE_ZERO = new Validator()
{
public void validate(String name, Object value)
@@ -94,7 +106,7 @@
}
else
{
- throw new IllegalArgumentException(format("%s must be greater or equals than 0 (actual value: %s",
+ throw new IllegalArgumentException(format("%s must be greater or equals than 0 (actual value: %s)",
name,
val));
}
Modified: trunk/src/main/org/jboss/messaging/core/journal/Journal.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/journal/Journal.java 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/src/main/org/jboss/messaging/core/journal/Journal.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -24,6 +24,7 @@
import java.util.List;
+import org.jboss.messaging.core.journal.impl.JournalFile;
import org.jboss.messaging.core.server.MessagingComponent;
/**
@@ -41,7 +42,7 @@
void appendAddRecord(long id, byte recordType, byte[] record, boolean sync) throws Exception;
void appendAddRecord(long id, byte recordType, EncodingSupport record, boolean sync) throws Exception;
-
+
void appendUpdateRecord(long id, byte recordType, byte[] record, boolean sync) throws Exception;
void appendUpdateRecord(long id, byte recordType, EncodingSupport record, boolean sync) throws Exception;
@@ -50,19 +51,19 @@
// Transactional operations
- void appendAddRecordTransactional(long txID, long id, byte recordType, byte[] record, boolean sync) throws Exception;
+ void appendAddRecordTransactional(long txID, long id, byte recordType, byte[] record) throws Exception;
- void appendAddRecordTransactional(long txID, long id, byte recordType, EncodingSupport record, boolean sync) throws Exception;
+ void appendAddRecordTransactional(long txID, long id, byte recordType, EncodingSupport record) throws Exception;
- void appendUpdateRecordTransactional(long txID, long id, byte recordType, byte[] record, boolean sync) throws Exception;
+ void appendUpdateRecordTransactional(long txID, long id, byte recordType, byte[] record) throws Exception;
- void appendUpdateRecordTransactional(long txID, long id, byte recordType, EncodingSupport record, boolean sync) throws Exception;
+ void appendUpdateRecordTransactional(long txID, long id, byte recordType, EncodingSupport record) throws Exception;
- void appendDeleteRecordTransactional(long txID, long id, byte[] record, boolean sync) throws Exception;
+ void appendDeleteRecordTransactional(long txID, long id, byte[] record) throws Exception;
- void appendDeleteRecordTransactional(long txID, long id, EncodingSupport record, boolean sync) throws Exception;
+ void appendDeleteRecordTransactional(long txID, long id, EncodingSupport record) throws Exception;
- void appendDeleteRecordTransactional(long txID, long id, boolean sync) throws Exception;
+ void appendDeleteRecordTransactional(long txID, long id) throws Exception;
void appendCommitRecord(long txID, boolean sync) throws Exception;
@@ -86,7 +87,24 @@
long load(List<RecordInfo> committedRecords, List<PreparedTransactionInfo> preparedTransactions) throws Exception;
int getAlignment() throws Exception;
-
+
void perfBlast(int pages) throws Exception;
+ /** This method is called automatically when a new file is opened */
+ void checkAndReclaimFiles() throws Exception;
+
+ /** This method check for the need of compacting based on the minCompactPercentage
+ * This method is usually called automatically when new files are opened
+ */
+ void checkCompact() throws Exception;
+
+ /**
+ * Eliminate deleted records of the journal.
+ * @throws Exception
+ */
+ void compact() throws Exception;
+
+
+ JournalFile[] getDataFiles();
+
}
Deleted: trunk/src/main/org/jboss/messaging/core/journal/LoadManager.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/journal/LoadManager.java 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/src/main/org/jboss/messaging/core/journal/LoadManager.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -1,39 +0,0 @@
-/*
- * JBoss, Home of Professional Open Source
- * Copyright 2005-2008, Red Hat Middleware LLC, and individual contributors
- * by the @authors tag. See the copyright.txt in the distribution for a
- * full listing of individual contributors.
- *
- * This is free software; you can redistribute it and/or modify it
- * under the terms of the GNU Lesser General Public License as
- * published by the Free Software Foundation; either version 2.1 of
- * the License, or (at your option) any later version.
- *
- * This software is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this software; if not, write to the Free
- * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
- * 02110-1301 USA, or see the FSF site: http://www.fsf.org.
- */
-
-package org.jboss.messaging.core.journal;
-
-/**
- *
- * @author <a href="mailto:clebert.suconic at jboss.com">Clebert Suconic</a>
- *
- */
-public interface LoadManager
-{
- void addRecord(RecordInfo info);
-
- void deleteRecord(long id);
-
- void updateRecord(RecordInfo info);
-
- void addPreparedTransaction(PreparedTransactionInfo preparedTransaction);
-}
Copied: trunk/src/main/org/jboss/messaging/core/journal/LoaderCallback.java (from rev 7512, branches/clebert_temp_expirement/src/main/org/jboss/messaging/core/journal/LoaderCallback.java)
===================================================================
--- trunk/src/main/org/jboss/messaging/core/journal/LoaderCallback.java (rev 0)
+++ trunk/src/main/org/jboss/messaging/core/journal/LoaderCallback.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -0,0 +1,39 @@
+/*
+ * JBoss, Home of Professional Open Source
+ * Copyright 2005-2008, Red Hat Middleware LLC, and individual contributors
+ * by the @authors tag. See the copyright.txt in the distribution for a
+ * full listing of individual contributors.
+ *
+ * This is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this software; if not, write to the Free
+ * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+ * 02110-1301 USA, or see the FSF site: http://www.fsf.org.
+ */
+
+package org.jboss.messaging.core.journal;
+
+/**
+ *
+ * @author <a href="mailto:clebert.suconic at jboss.com">Clebert Suconic</a>
+ *
+ */
+public interface LoaderCallback
+{
+ void addRecord(RecordInfo info);
+
+ void deleteRecord(long id);
+
+ void updateRecord(RecordInfo info);
+
+ void addPreparedTransaction(PreparedTransactionInfo preparedTransaction);
+}
Modified: trunk/src/main/org/jboss/messaging/core/journal/SequentialFile.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/journal/SequentialFile.java 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/src/main/org/jboss/messaging/core/journal/SequentialFile.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -42,6 +42,8 @@
void open() throws Exception;
boolean isOpen();
+
+ boolean exists();
/**
* For certain operations (like loading) we don't need open the file with full maxIO
@@ -79,6 +81,8 @@
long position() throws Exception;
void close() throws Exception;
+
+ void waitForClose() throws Exception;
void sync() throws Exception;
Modified: trunk/src/main/org/jboss/messaging/core/journal/SequentialFileFactory.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/journal/SequentialFileFactory.java 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/src/main/org/jboss/messaging/core/journal/SequentialFileFactory.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -40,13 +40,11 @@
List<String> listFiles(String extension) throws Exception;
boolean isSupportsCallbacks();
-
+
ByteBuffer newBuffer(int size);
void releaseBuffer(ByteBuffer buffer);
- void controlBuffersLifeCycle(boolean value);
-
/** The factory may need to do some initialization before the file is activated.
* this was added as a hook for AIO to initialize the Observer on TimedBuffer.
* It could be eventually done the same on NIO if we implement TimedBuffer on NIO */
@@ -71,5 +69,9 @@
* Create the directory if it doesn't exist yet
*/
void createDirs() throws Exception;
+
+ // used on tests only
+ void testFlush();
+
}
Modified: trunk/src/main/org/jboss/messaging/core/journal/TestableJournal.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/journal/TestableJournal.java 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/src/main/org/jboss/messaging/core/journal/TestableJournal.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -22,6 +22,8 @@
package org.jboss.messaging.core.journal;
+import org.jboss.messaging.core.journal.impl.JournalFile;
+
/**
*
* A TestableJournal
@@ -32,8 +34,6 @@
*/
public interface TestableJournal extends Journal
{
- void checkAndReclaimFiles() throws Exception;
-
int getDataFilesCount();
int getFreeFilesCount();
@@ -50,10 +50,6 @@
int getMinFiles();
- // boolean isSyncTransactional();
-
- // boolean isSyncNonTransactional();
-
String getFilePrefix();
String getFileExtension();
@@ -63,14 +59,14 @@
/** This method could be promoted to {@link Journal} interface when we decide to use the loadManager
* instead of load(List,List)
*/
- long load(LoadManager reloadManager) throws Exception;
+ long load(LoaderCallback reloadManager) throws Exception;
void forceMoveNextFile() throws Exception;
void setAutoReclaim(boolean autoReclaim);
boolean isAutoReclaim();
+
-
}
Modified: trunk/src/main/org/jboss/messaging/core/journal/impl/AIOSequentialFile.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/journal/impl/AIOSequentialFile.java 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/src/main/org/jboss/messaging/core/journal/impl/AIOSequentialFile.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -53,9 +53,9 @@
{
private static final Logger log = Logger.getLogger(AIOSequentialFile.class);
- private final String journalDir;
+ private final String directory;
- private final String fileName;
+ private File file;
private boolean opened = false;
@@ -89,7 +89,7 @@
public AIOSequentialFile(final SequentialFileFactory factory,
final int bufferSize,
final long bufferTimeoutMilliseconds,
- final String journalDir,
+ final String directory,
final String fileName,
final int maxIO,
final BufferCallback bufferCallback,
@@ -97,8 +97,8 @@
final Executor pollerExecutor)
{
this.factory = factory;
- this.journalDir = journalDir;
- this.fileName = fileName;
+ this.directory = directory;
+ file = new File(directory + "/" + fileName);
this.maxIO = maxIO;
this.bufferCallback = bufferCallback;
this.executor = executor;
@@ -116,6 +116,11 @@
return aioFile.getBlockSize();
}
+
+ public boolean exists()
+ {
+ return file.exists();
+ }
public int calculateBlockStart(final int position) throws Exception
{
@@ -166,14 +171,28 @@
while (!donelatch.await(60, TimeUnit.SECONDS))
{
- log.warn("Executor on file " + fileName + " couldn't complete its tasks in 60 seconds.",
- new Exception("Warning: Executor on file " + fileName + " couldn't complete its tasks in 60 seconds."));
+ log.warn("Executor on file " + file.getName() + " couldn't complete its tasks in 60 seconds.",
+ new Exception("Warning: Executor on file " + file.getName() + " couldn't complete its tasks in 60 seconds."));
}
aioFile.close();
aioFile = null;
+
+ this.notifyAll();
}
+ /* (non-Javadoc)
+ * @see org.jboss.messaging.core.journal.SequentialFile#waitForClose()
+ */
+ public synchronized void waitForClose() throws Exception
+ {
+ while (isOpen())
+ {
+ wait();
+ }
+ }
+
+
public void delete() throws Exception
{
if (aioFile != null)
@@ -182,7 +201,6 @@
aioFile = null;
}
- File file = new File(journalDir + "/" + fileName);
file.delete();
}
@@ -236,7 +254,7 @@
public String getFileName()
{
- return fileName;
+ return file.getName();
}
public void open() throws Exception
@@ -247,17 +265,22 @@
/* (non-Javadoc)
* @see org.jboss.messaging.core.journal.SequentialFile#renameTo(org.jboss.messaging.core.journal.SequentialFile)
*/
- public void renameTo(String fileName) throws Exception
+ public void renameTo(String newFileName) throws Exception
{
- throw new IllegalStateException("method rename not supported on AIO");
-
+ if (isOpen())
+ {
+ close();
+ }
+ File newFile = new File(directory + "/" + newFileName);
+ file.renameTo(newFile);
+ file = newFile;
}
public synchronized void open(final int currentMaxIO) throws Exception
{
opened = true;
aioFile = newFile();
- aioFile.open(journalDir + "/" + fileName, currentMaxIO);
+ aioFile.open(file.getAbsolutePath(), currentMaxIO);
position.set(0);
aioFile.setBufferCallback(bufferCallback);
this.fileSize = aioFile.size();
@@ -328,7 +351,7 @@
}
else
{
- write(bytes, false, DummyCallback.instance);
+ write(bytes, false, DummyCallback.getInstance());
}
}
@@ -358,7 +381,7 @@
}
else
{
- write(bytes, false, DummyCallback.instance);
+ write(bytes, false, DummyCallback.getInstance());
}
}
@@ -376,7 +399,7 @@
@Override
public String toString()
{
- return "AIOSequentialFile:" + journalDir + "/" + fileName;
+ return "AIOSequentialFile:" + file.getAbsolutePath();
}
// Public methods
@@ -514,9 +537,8 @@
public String toString()
{
- return "TimedBufferObserver on file (" + AIOSequentialFile.this.fileName + ")";
+ return "TimedBufferObserver on file (" + AIOSequentialFile.this.file.getName() + ")";
}
}
-
}
Modified: trunk/src/main/org/jboss/messaging/core/journal/impl/AIOSequentialFileFactory.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/journal/impl/AIOSequentialFileFactory.java 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/src/main/org/jboss/messaging/core/journal/impl/AIOSequentialFileFactory.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -111,6 +111,11 @@
}
}
+ public void testFlush()
+ {
+ timedBuffer.flush();
+ }
+
public void deactivate(SequentialFile file)
{
timedBuffer.flush();
@@ -140,18 +145,6 @@
return AsynchronousFileImpl.isLoaded();
}
- public void controlBuffersLifeCycle(boolean value)
- {
- if (value)
- {
- buffersControl.enable();
- }
- else
- {
- buffersControl.disable();
- }
- }
-
public ByteBuffer newBuffer(int size)
{
if (size % 512 != 0)
@@ -223,23 +216,10 @@
* and ready to be reused or GCed */
private final ConcurrentLinkedQueue<ByteBuffer> reuseBuffersQueue = new ConcurrentLinkedQueue<ByteBuffer>();
- /** During reload we may disable/enable buffer reuse */
- private boolean enabled = true;
-
private boolean stopped = false;
final BufferCallback callback = new LocalBufferCallback();
- public void enable()
- {
- this.enabled = true;
- }
-
- public void disable()
- {
- this.enabled = false;
- }
-
public ByteBuffer newBuffer(final int size)
{
// if a new buffer wasn't requested in 10 seconds, we clear the queue
@@ -314,26 +294,23 @@
synchronized (ReuseBuffersController.this)
{
- if (enabled)
+ if (stopped)
{
- if (stopped)
+ releaseBuffer(buffer);
+ }
+ else
+ {
+ bufferReuseLastTime = System.currentTimeMillis();
+
+ // If a buffer has any other than the configured bufferSize, the buffer
+ // will be just sent to GC
+ if (buffer.capacity() == bufferSize)
{
- releaseBuffer(buffer);
+ reuseBuffersQueue.offer(buffer);
}
else
{
- bufferReuseLastTime = System.currentTimeMillis();
-
- // If a buffer has any other than the configured bufferSize, the buffer
- // will be just sent to GC
- if (buffer.capacity() == bufferSize)
- {
- reuseBuffersQueue.offer(buffer);
- }
- else
- {
- releaseBuffer(buffer);
- }
+ releaseBuffer(buffer);
}
}
}
Modified: trunk/src/main/org/jboss/messaging/core/journal/impl/AbstractSequentialFactory.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/journal/impl/AbstractSequentialFactory.java 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/src/main/org/jboss/messaging/core/journal/impl/AbstractSequentialFactory.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -53,10 +53,6 @@
}
- public void controlBuffersLifeCycle(boolean value)
- {
- }
-
public void stop()
{
}
@@ -76,6 +72,10 @@
public void deactivate(SequentialFile file)
{
}
+
+ public void testFlush()
+ {
+ }
/**
* Create the directory if it doesn't exist yet
Modified: trunk/src/main/org/jboss/messaging/core/journal/impl/DummyCallback.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/journal/impl/DummyCallback.java 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/src/main/org/jboss/messaging/core/journal/impl/DummyCallback.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -35,7 +35,7 @@
*/
public class DummyCallback implements IOCallback
{
- static DummyCallback instance = new DummyCallback();
+ private static DummyCallback instance = new DummyCallback();
private static final Logger log = Logger.getLogger(SimpleWaitIOCallback.class);
Copied: trunk/src/main/org/jboss/messaging/core/journal/impl/JournalCompactor.java (from rev 7512, branches/clebert_temp_expirement/src/main/org/jboss/messaging/core/journal/impl/JournalCompactor.java)
===================================================================
--- trunk/src/main/org/jboss/messaging/core/journal/impl/JournalCompactor.java (rev 0)
+++ trunk/src/main/org/jboss/messaging/core/journal/impl/JournalCompactor.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -0,0 +1,683 @@
+/*
+ * JBoss, Home of Professional Open Source
+ * Copyright 2005-2009, Red Hat Middleware LLC, and individual contributors
+ * by the @authors tag. See the copyright.txt in the distribution for a
+ * full listing of individual contributors.
+ *
+ * This is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this software; if not, write to the Free
+ * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+ * 02110-1301 USA, or see the FSF site: http://www.fsf.org.
+ */
+
+package org.jboss.messaging.core.journal.impl;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.jboss.messaging.core.buffers.ChannelBuffer;
+import org.jboss.messaging.core.buffers.ChannelBuffers;
+import org.jboss.messaging.core.journal.RecordInfo;
+import org.jboss.messaging.core.journal.SequentialFile;
+import org.jboss.messaging.core.journal.SequentialFileFactory;
+import org.jboss.messaging.core.journal.impl.JournalImpl.JournalRecord;
+import org.jboss.messaging.core.logging.Logger;
+import org.jboss.messaging.core.remoting.spi.MessagingBuffer;
+import org.jboss.messaging.utils.ConcurrentHashSet;
+import org.jboss.messaging.utils.DataConstants;
+
+/**
+ * A JournalCompactor
+ *
+ * @author <a href="mailto:clebert.suconic at jboss.org">Clebert Suconic</a>
+ *
+ *
+ */
+public class JournalCompactor implements JournalReaderCallback
+{
+
+ private static final String FILE_COMPACT_CONTROL = "journal-rename-control.ctr";
+
+ private static final Logger log = Logger.getLogger(JournalCompactor.class);
+
+ private final JournalImpl journal;
+
+ private final SequentialFileFactory fileFactory;
+
+ private JournalFile currentFile;
+
+ private SequentialFile sequentialFile;
+
+ private int fileID;
+
+ private ChannelBuffer writingChannel;
+
+ private int nextOrderingID;
+
+ private final List<JournalFile> newDataFiles = new ArrayList<JournalFile>();
+
+ private final Set<Long> recordsSnapshot = new ConcurrentHashSet<Long>();;
+
+ // Snapshot of transactions that were pending when the compactor started
+ private final Map<Long, PendingTransaction> pendingTransactions = new ConcurrentHashMap<Long, PendingTransaction>();
+
+ private final Map<Long, JournalRecord> newRecords = new HashMap<Long, JournalRecord>();
+
+ private final Map<Long, JournalTransaction> newTransactions = new HashMap<Long, JournalTransaction>();
+
+ /** Commands that happened during compacting
+ * We can't process any counts during compacting, as we won't know in what files the records are taking place, so
+ * we cache those updates. As soon as we are done we take the right account. */
+ private final LinkedList<CompactCommand> pendingCommands = new LinkedList<CompactCommand>();
+
+ /**
+ * @param tmpRenameFile
+ * @param files
+ * @param newFiles
+ */
+ public static SequentialFile writeControlFile(final SequentialFileFactory fileFactory,
+ final List<JournalFile> files,
+ final List<JournalFile> newFiles) throws Exception
+ {
+
+ SequentialFile controlFile = fileFactory.createSequentialFile(FILE_COMPACT_CONTROL, 1);
+
+ try
+ {
+ controlFile.open(1);
+
+ ChannelBuffer renameBuffer = ChannelBuffers.dynamicBuffer(1);
+
+ renameBuffer.writeInt(-1);
+ renameBuffer.writeInt(-1);
+
+ MessagingBuffer filesToRename = ChannelBuffers.dynamicBuffer(1);
+
+ // DataFiles first
+
+ filesToRename.writeInt(files.size());
+
+ for (JournalFile file : files)
+ {
+ filesToRename.writeUTF(file.getFile().getFileName());
+ }
+
+ filesToRename.writeInt(newFiles.size());
+
+ for (JournalFile file : newFiles)
+ {
+ filesToRename.writeUTF(file.getFile().getFileName());
+ }
+
+ JournalImpl.writeAddRecord(-1,
+ 1,
+ (byte)0,
+ new JournalImpl.ByteArrayEncoding(filesToRename.array()),
+ JournalImpl.SIZE_ADD_RECORD + filesToRename.array().length,
+ renameBuffer);
+
+ ByteBuffer writeBuffer = fileFactory.newBuffer(renameBuffer.writerIndex());
+
+ writeBuffer.put(renameBuffer.array(), 0, renameBuffer.writerIndex());
+
+ writeBuffer.rewind();
+
+ controlFile.write(writeBuffer, true);
+
+ return controlFile;
+ }
+ finally
+ {
+ controlFile.close();
+ }
+ }
+
+ public static SequentialFile readControlFile(final SequentialFileFactory fileFactory,
+ final List<String> dataFiles,
+ final List<String> newFiles) throws Exception
+ {
+ SequentialFile controlFile = fileFactory.createSequentialFile(FILE_COMPACT_CONTROL, 1);
+
+ if (controlFile.exists())
+ {
+ JournalFile file = new JournalFileImpl(controlFile, -1);
+
+ final ArrayList<RecordInfo> records = new ArrayList<RecordInfo>();
+
+ JournalImpl.readJournalFile(fileFactory, file, new JournalReaderCallbackAbstract()
+ {
+ @Override
+ public void onReadAddRecord(final RecordInfo info) throws Exception
+ {
+ records.add(info);
+ }
+ });
+
+ if (records.size() == 0)
+ {
+ return null;
+ }
+ else
+ {
+ ChannelBuffer input = ChannelBuffers.wrappedBuffer(records.get(0).data);
+
+ int numberDataFiles = input.readInt();
+
+ for (int i = 0; i < numberDataFiles; i++)
+ {
+ dataFiles.add(input.readUTF());
+ }
+
+ int numberNewFiles = input.readInt();
+
+ for (int i = 0; i < numberNewFiles; i++)
+ {
+ newFiles.add(input.readUTF());
+ }
+
+ }
+
+ return controlFile;
+ }
+ else
+ {
+ return null;
+ }
+ }
+
+ public List<JournalFile> getNewDataFiles()
+ {
+ return newDataFiles;
+ }
+
+ public Map<Long, JournalRecord> getNewRecords()
+ {
+ return newRecords;
+ }
+
+ public Map<Long, JournalTransaction> getNewTransactions()
+ {
+ return newTransactions;
+ }
+
+ public JournalCompactor(final SequentialFileFactory fileFactory,
+ final JournalImpl journal,
+ final Set<Long> recordsSnapshot,
+ final int firstFileID)
+ {
+ this.fileFactory = fileFactory;
+ this.journal = journal;
+ this.recordsSnapshot.addAll(recordsSnapshot);
+ nextOrderingID = firstFileID;
+ }
+
+ /** This methods informs the Compactor about the existence of a pending (non committed) transaction */
+ public void addPendingTransaction(final long transactionID, final long ids[])
+ {
+ pendingTransactions.put(transactionID, new PendingTransaction(ids));
+ }
+
+ /**
+ * @param id
+ * @param journalTransaction
+ */
+ public void addCommandCommit(final JournalTransaction liveTransaction, final JournalFile currentFile)
+ {
+ pendingCommands.add(new CommitCompactCommand(liveTransaction, currentFile));
+
+ long ids[] = liveTransaction.getPositiveArray();
+
+ PendingTransaction oldTransaction = pendingTransactions.get(liveTransaction.getId());
+ long ids2[] = null;
+
+ if (oldTransaction != null)
+ {
+ ids2 = oldTransaction.pendingIDs;
+ }
+
+ /** If a delete comes for these records, while the compactor still working, we need to be able to take them into account for later deletes
+ * instead of throwing exceptions about non existent records */
+ if (ids != null)
+ {
+ for (long id : ids)
+ {
+ recordsSnapshot.add(id);
+ }
+ }
+
+ if (ids2 != null)
+ {
+ for (long id : ids2)
+ {
+ recordsSnapshot.add(id);
+ }
+ }
+ }
+
+ public void addCommandRollback(final JournalTransaction liveTransaction, final JournalFile currentFile)
+ {
+ pendingCommands.add(new RollbackCompactCommand(liveTransaction, currentFile));
+ }
+
+ /**
+ * @param id
+ * @param usedFile
+ */
+ public void addCommandDelete(final long id, final JournalFile usedFile)
+ {
+ pendingCommands.add(new DeleteCompactCommand(id, usedFile));
+ }
+
+ /**
+ * @param id
+ * @param usedFile
+ */
+ public void addCommandUpdate(final long id, final JournalFile usedFile, final int size)
+ {
+ pendingCommands.add(new UpdateCompactCommand(id, usedFile, size));
+ }
+
+ public boolean lookupRecord(final long id)
+ {
+ return recordsSnapshot.contains(id);
+ }
+
+ private void checkSize(final int size) throws Exception
+ {
+ if (writingChannel == null)
+ {
+ openFile();
+ }
+ else
+ {
+ if (writingChannel.writerIndex() + size > writingChannel.capacity())
+ {
+ openFile();
+ }
+ }
+ }
+
+ /** Write pending output into file */
+ public void flush() throws Exception
+ {
+ if (writingChannel != null)
+ {
+ sequentialFile.position(0);
+ sequentialFile.write(writingChannel.toByteBuffer(), true);
+ sequentialFile.close();
+ newDataFiles.add(currentFile);
+ }
+
+ writingChannel = null;
+ }
+
+ /**
+ * Replay pending counts that happened during compacting
+ */
+ public void replayPendingCommands()
+ {
+ for (CompactCommand command : pendingCommands)
+ {
+ try
+ {
+ command.execute();
+ }
+ catch (Exception e)
+ {
+ log.warn("Error replaying pending commands after compacting", e);
+ }
+ }
+
+ pendingCommands.clear();
+ }
+
+ // JournalReaderCallback implementation -------------------------------------------
+
+ public void onReadAddRecord(final RecordInfo info) throws Exception
+ {
+ if (recordsSnapshot.contains(info.id))
+ {
+ int size = JournalImpl.SIZE_ADD_RECORD + info.data.length;
+
+ checkSize(size);
+
+ JournalImpl.writeAddRecord(fileID,
+ info.id,
+ info.getUserRecordType(),
+ new JournalImpl.ByteArrayEncoding(info.data),
+ size,
+ writingChannel);
+
+ newRecords.put(info.id, new JournalRecord(currentFile, size));
+ }
+ }
+
+ public void onReadAddRecordTX(final long transactionID, final RecordInfo info) throws Exception
+ {
+ if (pendingTransactions.get(transactionID) != null)
+ {
+ JournalTransaction newTransaction = getNewJournalTransaction(transactionID);
+
+ int size = JournalImpl.SIZE_ADD_RECORD_TX + info.data.length;
+
+ checkSize(size);
+
+ newTransaction.addPositive(currentFile, info.id, size);
+
+ JournalImpl.writeAddRecordTX(fileID,
+ transactionID,
+ info.id,
+ info.getUserRecordType(),
+ new JournalImpl.ByteArrayEncoding(info.data),
+ size,
+ writingChannel);
+ }
+ else
+ {
+ // Will try it as a regular record, the method addRecord will validate if this is a live record or not
+ onReadAddRecord(info);
+ }
+ }
+
+ public void onReadCommitRecord(final long transactionID, final int numberOfRecords) throws Exception
+ {
+ if (pendingTransactions.get(transactionID) != null)
+ {
+ // Sanity check, this should never happen
+ throw new IllegalStateException("Inconsistency during compacting: CommitRecord ID = " + transactionID +
+ " for an already committed transaction during compacting");
+ }
+ }
+
+ public void onReadDeleteRecord(final long recordID) throws Exception
+ {
+ if (newRecords.get(recordID) != null)
+ {
+ // Sanity check, it should never happen
+ throw new IllegalStateException("Inconsistency during compacting: Delete record being read on an existent record");
+ }
+
+ }
+
+ public void onReadDeleteRecordTX(final long transactionID, final RecordInfo info) throws Exception
+ {
+ if (pendingTransactions.get(transactionID) != null)
+ {
+ JournalTransaction newTransaction = getNewJournalTransaction(transactionID);
+
+ int size = JournalImpl.SIZE_DELETE_RECORD_TX + info.data.length;
+
+ checkSize(size);
+
+ JournalImpl.writeDeleteRecordTransactional(fileID,
+ transactionID,
+ info.id,
+ new JournalImpl.ByteArrayEncoding(info.data),
+ size,
+ writingChannel);
+
+ newTransaction.addNegative(currentFile, info.id);
+ }
+ // else.. nothing to be done
+ }
+
+ public void markAsDataFile(final JournalFile file)
+ {
+ // nothing to be done here
+ }
+
+ public void onReadPrepareRecord(final long transactionID, final byte[] extraData, final int numberOfRecords) throws Exception
+ {
+ if (pendingTransactions.get(transactionID) != null)
+ {
+
+ JournalTransaction newTransaction = getNewJournalTransaction(transactionID);
+
+ int size = JournalImpl.SIZE_COMPLETE_TRANSACTION_RECORD + extraData.length + DataConstants.SIZE_INT;
+
+ checkSize(size);
+
+ JournalImpl.writeTransaction(fileID,
+ JournalImpl.PREPARE_RECORD,
+ transactionID,
+ newTransaction,
+ new JournalImpl.ByteArrayEncoding(extraData),
+ size,
+ newTransaction.getCounter(currentFile),
+ writingChannel);
+
+ newTransaction.prepare(currentFile);
+
+ }
+ }
+
+ public void onReadRollbackRecord(final long transactionID) throws Exception
+ {
+ if (pendingTransactions.get(transactionID) != null)
+ {
+ // Sanity check, this should never happen
+ throw new IllegalStateException("Inconsistency during compacting: RollbackRecord ID = " + transactionID +
+ " for an already rolled back transaction during compacting");
+ }
+ }
+
+ public void onReadUpdateRecord(final RecordInfo info) throws Exception
+ {
+ if (recordsSnapshot.contains(info.id))
+ {
+ int size = JournalImpl.SIZE_UPDATE_RECORD + info.data.length;
+
+ checkSize(size);
+
+ JournalRecord newRecord = newRecords.get(info.id);
+
+ if (newRecord == null)
+ {
+ log.warn("Couldn't find addRecord information for record " + info.id + " during compacting");
+ }
+ else
+ {
+ newRecord.addUpdateFile(currentFile, size);
+ }
+
+ JournalImpl.writeUpdateRecord(fileID,
+ info.id,
+ info.userRecordType,
+ new JournalImpl.ByteArrayEncoding(info.data),
+ size,
+ writingChannel);
+
+ }
+ }
+
+ public void onReadUpdateRecordTX(final long transactionID, final RecordInfo info) throws Exception
+ {
+ if (pendingTransactions.get(transactionID) != null)
+ {
+ JournalTransaction newTransaction = getNewJournalTransaction(transactionID);
+
+ int size = JournalImpl.SIZE_UPDATE_RECORD_TX + info.data.length;
+
+ checkSize(size);
+
+ JournalImpl.writeUpdateRecordTX(fileID,
+ transactionID,
+ info.id,
+ info.userRecordType,
+ new JournalImpl.ByteArrayEncoding(info.data),
+ size,
+ writingChannel);
+
+ newTransaction.addPositive(currentFile, info.id, size);
+ }
+ else
+ {
+ onReadUpdateRecord(info);
+ }
+ }
+
+ /**
+ * @param transactionID
+ * @return
+ */
+ private JournalTransaction getNewJournalTransaction(final long transactionID)
+ {
+ JournalTransaction newTransaction = newTransactions.get(transactionID);
+ if (newTransaction == null)
+ {
+ newTransaction = new JournalTransaction(transactionID, journal);
+ newTransactions.put(transactionID, newTransaction);
+ }
+ return newTransaction;
+ }
+
+ /**
+ * @throws Exception
+ */
+ private void openFile() throws Exception
+ {
+ flush();
+
+ ByteBuffer bufferWrite = fileFactory.newBuffer(journal.getFileSize());
+ writingChannel = ChannelBuffers.wrappedBuffer(bufferWrite);
+
+ currentFile = journal.getFile(false, false, false);
+ sequentialFile = currentFile.getFile();
+ sequentialFile.renameTo(sequentialFile.getFileName() + ".cmp");
+
+ log.info("Writing compacted data into " + sequentialFile.getFileName());
+ sequentialFile.open(1);
+ fileID = nextOrderingID++;
+ currentFile = new JournalFileImpl(sequentialFile, fileID);
+
+ writingChannel.writeInt(fileID);
+ }
+
+ private static abstract class CompactCommand
+ {
+ abstract void execute() throws Exception;
+ }
+
+ private class DeleteCompactCommand extends CompactCommand
+ {
+ long id;
+
+ JournalFile usedFile;
+
+ public DeleteCompactCommand(final long id, final JournalFile usedFile)
+ {
+ this.id = id;
+ this.usedFile = usedFile;
+ }
+
+ @Override
+ void execute() throws Exception
+ {
+ JournalRecord deleteRecord = journal.getRecords().remove(id);
+ deleteRecord.delete(usedFile);
+ }
+ }
+
+ private static class PendingTransaction
+ {
+ long pendingIDs[];
+
+ PendingTransaction(final long ids[])
+ {
+ pendingIDs = ids;
+ }
+
+ }
+
+ private class UpdateCompactCommand extends CompactCommand
+ {
+ private long id;
+
+ private JournalFile usedFile;
+
+ private final int size;
+
+ public UpdateCompactCommand(final long id, final JournalFile usedFile, final int size)
+ {
+ this.id = id;
+ this.usedFile = usedFile;
+ this.size = size;
+ }
+
+ @Override
+ void execute() throws Exception
+ {
+ JournalRecord updateRecord = journal.getRecords().get(id);
+ updateRecord.addUpdateFile(usedFile, size);
+ }
+ }
+
+ private class CommitCompactCommand extends CompactCommand
+ {
+ private final JournalTransaction liveTransaction;
+
+ /** File containing the commit record */
+ private final JournalFile commitFile;
+
+ public CommitCompactCommand(final JournalTransaction liveTransaction, final JournalFile commitFile)
+ {
+ this.liveTransaction = liveTransaction;
+ this.commitFile = commitFile;
+ }
+
+ @Override
+ void execute() throws Exception
+ {
+ JournalTransaction newTransaction = newTransactions.get(liveTransaction.getId());
+ if (newTransaction != null)
+ {
+ liveTransaction.merge(newTransaction);
+ liveTransaction.commit(commitFile);
+ }
+ newTransactions.remove(liveTransaction.getId());
+ }
+ }
+
+ private class RollbackCompactCommand extends CompactCommand
+ {
+ private final JournalTransaction liveTransaction;
+
+ /** File containing the commit record */
+ private final JournalFile rollbackFile;
+
+ public RollbackCompactCommand(final JournalTransaction liveTransaction, final JournalFile rollbackFile)
+ {
+ this.liveTransaction = liveTransaction;
+ this.rollbackFile = rollbackFile;
+ }
+
+ @Override
+ void execute() throws Exception
+ {
+ JournalTransaction newTransaction = newTransactions.get(liveTransaction.getId());
+ if (newTransaction != null)
+ {
+ liveTransaction.merge(newTransaction);
+ liveTransaction.rollback(rollbackFile);
+ }
+ newTransactions.remove(liveTransaction.getId());
+ }
+ }
+
+}
Modified: trunk/src/main/org/jboss/messaging/core/journal/impl/JournalFile.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/journal/impl/JournalFile.java 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/src/main/org/jboss/messaging/core/journal/impl/JournalFile.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -35,6 +35,10 @@
*/
public interface JournalFile
{
+
+ /** Used during compacting (clearing counters) */
+ void clearCounts();
+
int getNegCount(JournalFile file);
void incNegCount(JournalFile file);
@@ -44,14 +48,20 @@
void incPosCount();
void decPosCount();
-
+
+ void addSize(int bytes);
+
+ void decSize(int bytes);
+
+ int getLiveSize();
+
void setCanReclaim(boolean canDelete);
boolean isCanReclaim();
long getOffset();
+
+ int getFileID();
- int getOrderingID();
-
SequentialFile getFile();
}
Modified: trunk/src/main/org/jboss/messaging/core/journal/impl/JournalFileImpl.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/journal/impl/JournalFileImpl.java 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/src/main/org/jboss/messaging/core/journal/impl/JournalFileImpl.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -44,23 +44,32 @@
private final SequentialFile file;
- private final int orderingID;
+ private final int fileID;
private long offset;
-
+
private final AtomicInteger posCount = new AtomicInteger(0);
+
+ private final AtomicInteger liveBytes = new AtomicInteger(0);
private boolean canReclaim;
private final Map<JournalFile, AtomicInteger> negCounts = new ConcurrentHashMap<JournalFile, AtomicInteger>();
-
- public JournalFileImpl(final SequentialFile file, final int orderingID)
+
+ public JournalFileImpl(final SequentialFile file, final int fileID)
{
this.file = file;
- this.orderingID = orderingID;
+ this.fileID = fileID;
}
+ public void clearCounts()
+ {
+ negCounts.clear();
+ posCount.set(0);
+ liveBytes.set(0);
+ }
+
public int getPosCount()
{
return posCount.intValue();
@@ -104,7 +113,7 @@
{
posCount.decrementAndGet();
}
-
+
public void extendOffset(final int delta)
{
offset += delta;
@@ -115,9 +124,9 @@
return offset;
}
- public int getOrderingID()
+ public int getFileID()
{
- return orderingID;
+ return fileID;
}
public void setOffset(final long offset)
@@ -170,4 +179,29 @@
return count;
}
+ /* (non-Javadoc)
+ * @see org.jboss.messaging.core.journal.impl.JournalFile#addSize(int)
+ */
+ public void addSize(int bytes)
+ {
+ liveBytes.addAndGet(bytes);
+ }
+
+ /* (non-Javadoc)
+ * @see org.jboss.messaging.core.journal.impl.JournalFile#decSize(int)
+ */
+ public void decSize(int bytes)
+ {
+ liveBytes.addAndGet(-bytes);
+ }
+
+ /* (non-Javadoc)
+ * @see org.jboss.messaging.core.journal.impl.JournalFile#getSize()
+ */
+ public int getLiveSize()
+ {
+ return liveBytes.get();
+ }
+
+
}
Modified: trunk/src/main/org/jboss/messaging/core/journal/impl/JournalImpl.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/journal/impl/JournalImpl.java 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/src/main/org/jboss/messaging/core/journal/impl/JournalImpl.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -26,7 +26,6 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
@@ -47,13 +46,14 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.jboss.messaging.core.buffers.ChannelBuffer;
import org.jboss.messaging.core.buffers.ChannelBuffers;
-import org.jboss.messaging.core.exception.MessagingException;
import org.jboss.messaging.core.journal.EncodingSupport;
import org.jboss.messaging.core.journal.IOCallback;
-import org.jboss.messaging.core.journal.LoadManager;
+import org.jboss.messaging.core.journal.LoaderCallback;
import org.jboss.messaging.core.journal.PreparedTransactionInfo;
import org.jboss.messaging.core.journal.RecordInfo;
import org.jboss.messaging.core.journal.SequentialFile;
@@ -61,8 +61,10 @@
import org.jboss.messaging.core.journal.TestableJournal;
import org.jboss.messaging.core.logging.Logger;
import org.jboss.messaging.core.remoting.spi.MessagingBuffer;
+import org.jboss.messaging.utils.DataConstants;
import org.jboss.messaging.utils.Pair;
import org.jboss.messaging.utils.VariableLatch;
+import org.jboss.messaging.utils.concurrent.LinkedBlockingDeque;
/**
*
@@ -71,7 +73,7 @@
* <p>WIKI Page: <a href="http://wiki.jboss.org/wiki/JBossMessaging2Journal"> http://wiki.jboss.org/wiki/JBossMessaging2Journal</a></p>
*
*
- * <p>Look at {@link JournalImpl#load(LoadManager)} for the file layout
+ * <p>Look at {@link JournalImpl#load(LoaderCallback)} for the file layout
*
* @author <a href="mailto:tim.fox at jboss.com">Tim Fox</a>
*
@@ -94,6 +96,9 @@
private static final boolean trace = log.isTraceEnabled();
+ /** This is to be set to true at DEBUG & development only */
+ private static final boolean LOAD_TRACE = false;
+
// This method exists just to make debug easier.
// I could replace log.trace by log.info temporarily while I was debugging
// Journal
@@ -104,47 +109,54 @@
// The sizes of primitive types
- private static final int SIZE_LONG = 8;
-
- private static final int SIZE_INT = 4;
-
- private static final int SIZE_BYTE = 1;
-
public static final int MIN_FILE_SIZE = 1024;
- public static final int SIZE_HEADER = 4;
+ public static final int SIZE_HEADER = DataConstants.SIZE_INT;
- public static final int BASIC_SIZE = SIZE_BYTE + SIZE_INT + SIZE_INT;
+ public static final int BASIC_SIZE = DataConstants.SIZE_BYTE + DataConstants.SIZE_INT + DataConstants.SIZE_INT;
- public static final int SIZE_ADD_RECORD = BASIC_SIZE + SIZE_LONG + SIZE_BYTE + SIZE_INT /* + record.length */;
+ public static final int SIZE_ADD_RECORD = BASIC_SIZE + DataConstants.SIZE_LONG +
+ DataConstants.SIZE_BYTE +
+ DataConstants.SIZE_INT /* + record.length */;
// Record markers - they must be all unique
public static final byte ADD_RECORD = 11;
- public static final byte SIZE_UPDATE_RECORD = BASIC_SIZE + SIZE_LONG + SIZE_BYTE + SIZE_INT /* + record.length */;
+ public static final byte SIZE_UPDATE_RECORD = BASIC_SIZE + DataConstants.SIZE_LONG +
+ DataConstants.SIZE_BYTE +
+ DataConstants.SIZE_INT /* + record.length */;
public static final byte UPDATE_RECORD = 12;
- public static final int SIZE_ADD_RECORD_TX = BASIC_SIZE + SIZE_LONG + SIZE_BYTE + SIZE_LONG + SIZE_INT /* + record.length */;
+ public static final int SIZE_ADD_RECORD_TX = BASIC_SIZE + DataConstants.SIZE_LONG +
+ DataConstants.SIZE_BYTE +
+ DataConstants.SIZE_LONG +
+ DataConstants.SIZE_INT /* + record.length */;
public static final byte ADD_RECORD_TX = 13;
- public static final int SIZE_UPDATE_RECORD_TX = BASIC_SIZE + SIZE_LONG + SIZE_BYTE + SIZE_LONG + SIZE_INT /* + record.length */;
+ public static final int SIZE_UPDATE_RECORD_TX = BASIC_SIZE + DataConstants.SIZE_LONG +
+ DataConstants.SIZE_BYTE +
+ DataConstants.SIZE_LONG +
+ DataConstants.SIZE_INT /* + record.length */;
public static final byte UPDATE_RECORD_TX = 14;
- public static final int SIZE_DELETE_RECORD_TX = BASIC_SIZE + SIZE_LONG + SIZE_LONG + SIZE_INT /* + record.length */;
+ public static final int SIZE_DELETE_RECORD_TX = BASIC_SIZE + DataConstants.SIZE_LONG +
+ DataConstants.SIZE_LONG +
+ DataConstants.SIZE_INT /* + record.length */;
public static final byte DELETE_RECORD_TX = 15;
- public static final int SIZE_DELETE_RECORD = BASIC_SIZE + SIZE_LONG;
+ public static final int SIZE_DELETE_RECORD = BASIC_SIZE + DataConstants.SIZE_LONG;
public static final byte DELETE_RECORD = 16;
- public static final int SIZE_COMPLETE_TRANSACTION_RECORD = BASIC_SIZE + SIZE_INT + SIZE_LONG /* + NumerOfElements*SIZE_INT*2 */;
+ public static final int SIZE_COMPLETE_TRANSACTION_RECORD = BASIC_SIZE + DataConstants.SIZE_LONG +
+ DataConstants.SIZE_INT;
- public static final int SIZE_PREPARE_RECORD = SIZE_COMPLETE_TRANSACTION_RECORD + SIZE_INT;
+ public static final int SIZE_PREPARE_RECORD = SIZE_COMPLETE_TRANSACTION_RECORD + DataConstants.SIZE_INT;
public static final byte PREPARE_RECORD = 17;
@@ -152,7 +164,7 @@
public static final byte COMMIT_RECORD = 18;
- public static final int SIZE_ROLLBACK_RECORD = BASIC_SIZE + SIZE_LONG;
+ public static final int SIZE_ROLLBACK_RECORD = BASIC_SIZE + DataConstants.SIZE_LONG;
public static final byte ROLLBACK_RECORD = 19;
@@ -160,9 +172,9 @@
// Attributes ----------------------------------------------------
- private boolean autoReclaim = true;
+ private volatile boolean autoReclaim = true;
- private final AtomicInteger nextOrderingId = new AtomicInteger(0);
+ private final AtomicInteger nextFileID = new AtomicInteger(0);
// used for Asynchronous IO only (ignored on NIO).
private final int maxAIO;
@@ -171,28 +183,41 @@
private final int minFiles;
+ private final float compactPercentage;
+
+ private final int compactMinFiles;
+
private final SequentialFileFactory fileFactory;
public final String filePrefix;
public final String fileExtension;
- private final Queue<JournalFile> dataFiles = new ConcurrentLinkedQueue<JournalFile>();
+ private final LinkedBlockingDeque<JournalFile> dataFiles = new LinkedBlockingDeque<JournalFile>();
private final Queue<JournalFile> freeFiles = new ConcurrentLinkedQueue<JournalFile>();
private final BlockingQueue<JournalFile> openedFiles = new LinkedBlockingQueue<JournalFile>();
- private final ConcurrentMap<Long, PosFiles> posFilesMap = new ConcurrentHashMap<Long, PosFiles>();
+ // Compacting may replace this structure
+ private final ConcurrentMap<Long, JournalRecord> records = new ConcurrentHashMap<Long, JournalRecord>();
- private final ConcurrentMap<Long, JournalTransaction> transactionInfos = new ConcurrentHashMap<Long, JournalTransaction>();
+ // Compacting may replace this structure
+ private final ConcurrentMap<Long, JournalTransaction> transactions = new ConcurrentHashMap<Long, JournalTransaction>();
- private final ConcurrentMap<Long, TransactionCallback> transactionCallbacks = new ConcurrentHashMap<Long, TransactionCallback>();
+ // This will be set only while the JournalCompactor is being executed
+ private volatile JournalCompactor compactor;
+
+ // Latch used to wait compactor finish, to make sure we won't stop the journal with the compactor running
+ private final VariableLatch compactorWait = new VariableLatch();
private ExecutorService filesExecutor = null;
private final Semaphore lock = new Semaphore(1);
+ /** We don't lock the journal while compacting, however we need to lock it while taking and updating snapshots */
+ private final ReadWriteLock compactingLock = new ReentrantReadWriteLock();
+
private volatile JournalFile currentFile;
private volatile int state;
@@ -203,6 +228,8 @@
public JournalImpl(final int fileSize,
final int minFiles,
+ final int compactMinFiles,
+ final int compactPercentage,
final SequentialFileFactory fileFactory,
final String filePrefix,
final String fileExtension,
@@ -239,6 +266,22 @@
throw new IllegalStateException("maxAIO should aways be a positive number");
}
+ if (compactPercentage < 0 || compactPercentage > 100)
+ {
+ throw new IllegalArgumentException("Compact Percentage out of range");
+ }
+
+ if (compactPercentage == 0)
+ {
+ this.compactPercentage = 0;
+ }
+ else
+ {
+ this.compactPercentage = (float)compactPercentage / 100f;
+ }
+
+ this.compactMinFiles = compactMinFiles;
+
this.fileSize = fileSize;
this.minFiles = minFiles;
@@ -252,6 +295,24 @@
this.maxAIO = maxAIO;
}
+ // Public methods (used by package members such as JournalCompactor) (these methods are not part of the JournalImpl
+ // interface)
+
+ public Map<Long, JournalRecord> getRecords()
+ {
+ return records;
+ }
+
+ public JournalFile getCurrentFile()
+ {
+ return currentFile;
+ }
+
+ public JournalCompactor getCompactor()
+ {
+ return this.compactor;
+ }
+
// Journal implementation
// ----------------------------------------------------------------
@@ -267,32 +328,35 @@
throw new IllegalStateException("Journal must be loaded first");
}
- int recordLength = record.getEncodeSize();
+ IOCallback callback = null;
- int size = SIZE_ADD_RECORD + recordLength;
+ compactingLock.readLock().lock();
- ChannelBuffer bb = newBuffer(size);
+ try
+ {
+ int size = SIZE_ADD_RECORD + record.getEncodeSize();
- bb.writeByte(ADD_RECORD);
- bb.writeInt(-1); // skip ID part
- bb.writeLong(id);
- bb.writeInt(recordLength);
- bb.writeByte(recordType);
- record.encode(bb);
- bb.writeInt(size);
+ ChannelBuffer bb = newBuffer(size);
- IOCallback callback = getSyncCallback(sync);
+ writeAddRecord(-1, id, recordType, record, size, bb); // fileID will be filled later
- lock.acquire();
- try
- {
- JournalFile usedFile = appendRecord(bb, sync, callback);
+ callback = getSyncCallback(sync);
- posFilesMap.put(id, new PosFiles(usedFile));
+ lock.acquire();
+ try
+ {
+ JournalFile usedFile = appendRecord(bb, false, sync, null, callback);
+
+ records.put(id, new JournalRecord(usedFile, size));
+ }
+ finally
+ {
+ lock.release();
+ }
}
finally
{
- lock.release();
+ compactingLock.readLock().unlock();
}
if (callback != null)
@@ -313,37 +377,55 @@
throw new IllegalStateException("Journal must be loaded first");
}
- PosFiles posFiles = posFilesMap.get(id);
+ IOCallback callback = null;
- if (posFiles == null)
+ compactingLock.readLock().lock();
+
+ try
{
- throw new IllegalStateException("Cannot find add info " + id);
- }
- int size = SIZE_UPDATE_RECORD + record.getEncodeSize();
+ JournalRecord posFiles = records.get(id);
- ChannelBuffer bb = newBuffer(size);
+ if (posFiles == null)
+ {
+ if (!(compactor != null && compactor.lookupRecord(id)))
+ {
+ throw new IllegalStateException("Cannot find add info " + id);
+ }
+ }
- bb.writeByte(UPDATE_RECORD);
- bb.writeInt(-1); // skip ID part
- bb.writeLong(id);
- bb.writeInt(record.getEncodeSize());
- bb.writeByte(recordType);
- record.encode(bb);
- bb.writeInt(size);
+ int size = SIZE_UPDATE_RECORD + record.getEncodeSize();
- IOCallback callback = getSyncCallback(sync);
+ ChannelBuffer bb = newBuffer(size);
- lock.acquire();
- try
- {
- JournalFile usedFile = appendRecord(bb, sync, callback);
+ writeUpdateRecord(-1, id, recordType, record, size, bb);
- posFiles.addUpdateFile(usedFile);
+ callback = getSyncCallback(sync);
+
+ lock.acquire();
+ try
+ {
+ JournalFile usedFile = appendRecord(bb, false, sync, null, callback);
+
+ // record== null here could only mean there is a compactor, and computing the delete should be done after
+ // compacting is done
+ if (posFiles == null)
+ {
+ compactor.addCommandUpdate(id, usedFile, size);
+ }
+ else
+ {
+ posFiles.addUpdateFile(usedFile, size);
+ }
+ }
+ finally
+ {
+ lock.release();
+ }
}
finally
{
- lock.release();
+ compactingLock.readLock().unlock();
}
if (callback != null)
@@ -359,34 +441,59 @@
throw new IllegalStateException("Journal must be loaded first");
}
- PosFiles posFiles = posFilesMap.remove(id);
+ compactingLock.readLock().lock();
- if (posFiles == null)
+ IOCallback callback = null;
+
+ try
{
- throw new IllegalStateException("Cannot find add info " + id);
- }
- int size = SIZE_DELETE_RECORD;
+ JournalRecord record = records.remove(id);
- ChannelBuffer bb = newBuffer(size);
+ if (record == null)
+ {
+ if (!(compactor != null && compactor.lookupRecord(id)))
+ {
+ throw new IllegalStateException("Cannot find add info " + id);
+ }
+ }
- bb.writeByte(DELETE_RECORD);
- bb.writeInt(-1); // skip ID part
- bb.writeLong(id);
- bb.writeInt(size);
+ int size = SIZE_DELETE_RECORD;
- IOCallback callback = getSyncCallback(sync);
+ ChannelBuffer bb = newBuffer(size);
- lock.acquire();
- try
- {
- JournalFile usedFile = appendRecord(bb, sync, callback);
+ bb.writeByte(DELETE_RECORD);
+ bb.writeInt(-1); // skip ID part
+ bb.writeLong(id);
+ bb.writeInt(size);
- posFiles.addDelete(usedFile);
+ callback = getSyncCallback(sync);
+
+ lock.acquire();
+ try
+ {
+ JournalFile usedFile = appendRecord(bb, false, sync, null, callback);
+
+ // record== null here could only mean there is a compactor, and computing the delete should be done after
+ // compacting is done
+ if (record == null)
+ {
+ compactor.addCommandDelete(id, usedFile);
+ }
+ else
+ {
+ record.delete(usedFile);
+ }
+
+ }
+ finally
+ {
+ lock.release();
+ }
}
finally
{
- lock.release();
+ compactingLock.readLock().unlock();
}
if (callback != null)
@@ -395,181 +502,147 @@
}
}
- public void appendAddRecordTransactional(final long txID,
- final long id,
- final byte recordType,
- final byte[] record,
- final boolean sync) throws Exception
+ public void appendAddRecordTransactional(final long txID, final long id, final byte recordType, final byte[] record) throws Exception
{
- appendAddRecordTransactional(txID, id, recordType, new ByteArrayEncoding(record), sync);
+ appendAddRecordTransactional(txID, id, recordType, new ByteArrayEncoding(record));
}
public void appendAddRecordTransactional(final long txID,
final long id,
final byte recordType,
- final EncodingSupport record,
- final boolean sync) throws Exception
+ final EncodingSupport record) throws Exception
{
if (state != STATE_LOADED)
{
throw new IllegalStateException("Journal must be loaded first");
}
- int recordLength = record.getEncodeSize();
+ compactingLock.readLock().lock();
- int size = SIZE_ADD_RECORD_TX + recordLength;
+ try
+ {
- ChannelBuffer bb = newBuffer(size);
+ int size = SIZE_ADD_RECORD_TX + record.getEncodeSize();
- bb.writeByte(ADD_RECORD_TX);
- bb.writeInt(-1); // skip ID part
- bb.writeLong(txID);
- bb.writeLong(id);
- bb.writeInt(recordLength);
- bb.writeByte(recordType);
- record.encode(bb);
- bb.writeInt(size);
+ ChannelBuffer bb = newBuffer(size);
- lock.acquire();
- try
- {
- JournalFile usedFile = appendRecord(bb, false, getTransactionCallback(txID, sync));
+ writeAddRecordTX(-1, txID, id, recordType, record, size, bb);
JournalTransaction tx = getTransactionInfo(txID);
- tx.addPositive(usedFile, id);
+ lock.acquire();
+ try
+ {
+ JournalFile usedFile = appendRecord(bb, false, false, tx, null);
+
+ tx.addPositive(usedFile, id, size);
+ }
+ finally
+ {
+ lock.release();
+ }
}
finally
{
- lock.release();
+ compactingLock.readLock().unlock();
}
}
public void appendUpdateRecordTransactional(final long txID,
final long id,
final byte recordType,
- final byte[] record,
- final boolean sync) throws Exception
+ final byte[] record) throws Exception
{
- appendUpdateRecordTransactional(txID, id, recordType, new ByteArrayEncoding(record), sync);
+ appendUpdateRecordTransactional(txID, id, recordType, new ByteArrayEncoding(record));
}
public void appendUpdateRecordTransactional(final long txID,
final long id,
final byte recordType,
- final EncodingSupport record,
- final boolean sync) throws Exception
+ final EncodingSupport record) throws Exception
{
if (state != STATE_LOADED)
{
throw new IllegalStateException("Journal must be loaded first");
}
- int size = SIZE_UPDATE_RECORD_TX + record.getEncodeSize();
+ compactingLock.readLock().lock();
- ChannelBuffer bb = newBuffer(size);
-
- bb.writeByte(UPDATE_RECORD_TX);
- bb.writeInt(-1); // skip ID part
- bb.writeLong(txID);
- bb.writeLong(id);
- bb.writeInt(record.getEncodeSize());
- bb.writeByte(recordType);
- record.encode(bb);
- bb.writeInt(size);
-
- lock.acquire();
try
{
- JournalFile usedFile = appendRecord(bb, false, getTransactionCallback(txID, sync));
+ int size = SIZE_UPDATE_RECORD_TX + record.getEncodeSize();
+
+ ChannelBuffer bb = newBuffer(size);
+
+ writeUpdateRecordTX(-1, txID, id, recordType, record, size, bb);
+
JournalTransaction tx = getTransactionInfo(txID);
- tx.addPositive(usedFile, id);
+ lock.acquire();
+ try
+ {
+ JournalFile usedFile = appendRecord(bb, false, false, tx, null);
+
+ tx.addPositive(usedFile, id, size);
+ }
+ finally
+ {
+ lock.release();
+ }
}
finally
{
- lock.release();
+ compactingLock.readLock().unlock();
}
}
- public void appendDeleteRecordTransactional(final long txID, final long id, final byte[] record, final boolean sync) throws Exception
+ public void appendDeleteRecordTransactional(final long txID, final long id, final byte[] record) throws Exception
{
- appendDeleteRecordTransactional(txID, id, new ByteArrayEncoding(record), sync);
+ appendDeleteRecordTransactional(txID, id, new ByteArrayEncoding(record));
}
- public void appendDeleteRecordTransactional(final long txID,
- final long id,
- final EncodingSupport record,
- final boolean sync) throws Exception
+ public void appendDeleteRecordTransactional(final long txID, final long id, final EncodingSupport record) throws Exception
{
if (state != STATE_LOADED)
{
throw new IllegalStateException("Journal must be loaded first");
}
- int size = SIZE_DELETE_RECORD_TX + (record != null ? record.getEncodeSize() : 0);
+ compactingLock.readLock().lock();
- ChannelBuffer bb = newBuffer(size);
-
- bb.writeByte(DELETE_RECORD_TX);
- bb.writeInt(-1); // skip ID part
- bb.writeLong(txID);
- bb.writeLong(id);
- bb.writeInt(record != null ? record.getEncodeSize() : 0);
- if (record != null)
- {
- record.encode(bb);
- }
- bb.writeInt(size);
-
- lock.acquire();
try
{
- JournalFile usedFile = appendRecord(bb, false, getTransactionCallback(txID, sync));
+ int size = SIZE_DELETE_RECORD_TX + record.getEncodeSize();
+ ChannelBuffer bb = newBuffer(size);
+
+ writeDeleteRecordTransactional(-1, txID, id, record, size, bb);
+
JournalTransaction tx = getTransactionInfo(txID);
- tx.addNegative(usedFile, id);
+ lock.acquire();
+ try
+ {
+ JournalFile usedFile = appendRecord(bb, false, false, tx, null);
+
+ tx.addNegative(usedFile, id);
+ }
+ finally
+ {
+ lock.release();
+ }
}
finally
{
- lock.release();
+ compactingLock.readLock().unlock();
}
}
- public void appendDeleteRecordTransactional(final long txID, final long id, final boolean sync) throws Exception
+ public void appendDeleteRecordTransactional(final long txID, final long id) throws Exception
{
- if (state != STATE_LOADED)
- {
- throw new IllegalStateException("Journal must be loaded first");
- }
-
- int size = SIZE_DELETE_RECORD_TX;
-
- ChannelBuffer bb = newBuffer(size);
-
- bb.writeByte(DELETE_RECORD_TX);
- bb.writeInt(-1); // skip ID part
- bb.writeLong(txID);
- bb.writeLong(id);
- bb.writeInt(0);
- bb.writeInt(size);
-
- lock.acquire();
- try
- {
- JournalFile usedFile = appendRecord(bb, false, getTransactionCallback(txID, sync));
-
- JournalTransaction tx = getTransactionInfo(txID);
-
- tx.addNegative(usedFile, id);
- }
- finally
- {
- lock.release();
- }
+ appendDeleteRecordTransactional(txID, id, NullEncoding.instance);
}
/**
@@ -594,41 +667,55 @@
JournalTransaction tx = getTransactionInfo(txID);
- ChannelBuffer bb = writeTransaction(PREPARE_RECORD, txID, tx, transactionData);
+ if (sync)
+ {
+ tx.syncPreviousFiles(fileFactory.isSupportsCallbacks(), currentFile);
+ }
- IOCallback callback = getTransactionCallback(txID, sync);
+ compactingLock.readLock().lock();
- lock.acquire();
try
{
- JournalFile usedFile = appendRecord(bb, sync, callback);
- tx.prepare(usedFile);
+ int size = SIZE_COMPLETE_TRANSACTION_RECORD + transactionData.getEncodeSize() + DataConstants.SIZE_INT;
+ ChannelBuffer bb = newBuffer(size);
+
+ writeTransaction(-1, PREPARE_RECORD, txID, tx, transactionData, size, -1, bb);
+
+ lock.acquire();
+ try
+ {
+ JournalFile usedFile = appendRecord(bb, true, sync, tx, null);
+
+ tx.prepare(usedFile);
+ }
+ finally
+ {
+ lock.release();
+ }
+
}
finally
{
- lock.release();
+ compactingLock.readLock().unlock();
}
// We should wait this outside of the lock, to increase throughput
- if (callback != null)
- {
- callback.waitCompletion();
- }
+ tx.waitCompletion();
}
/**
* <p>A transaction record (Commit or Prepare), will hold the number of elements the transaction has on each file.</p>
- * <p>For example, a transaction was spread along 3 journal files with 10 records on each file.
- * (What could happen if there are too many records, or if an user event delayed records to come in time to a single file).</p>
+ * <p>For example, a transaction was spread along 3 journal files with 10 pendingTransactions on each file.
+ * (What could happen if there are too many pendingTransactions, or if an user event delayed pendingTransactions to come in time to a single file).</p>
* <p>The element-summary will then have</p>
* <p>FileID1, 10</p>
* <p>FileID2, 10</p>
* <p>FileID3, 10</p>
*
* <br>
- * <p> During the load, the transaction needs to have 30 records spread across the files as originally written.</p>
- * <p> If for any reason there are missing records, that means the transaction was not completed and we should ignore the whole transaction </p>
+ * <p> During the load, the transaction needs to have 30 pendingTransactions spread across the files as originally written.</p>
+ * <p> If for any reason there are missing pendingTransactions, that means the transaction was not completed and we should ignore the whole transaction </p>
* <p> We can't just use a global counter as reclaiming could delete files after the transaction was successfully committed.
* That also means not having a whole file on journal-reload doesn't mean we have to invalidate the transaction </p>
*
@@ -641,37 +728,45 @@
throw new IllegalStateException("Journal must be loaded first");
}
- JournalTransaction tx = transactionInfos.remove(txID);
+ JournalTransaction tx = transactions.remove(txID);
- if (tx == null)
+ compactingLock.readLock().lock();
+
+ try
{
- throw new IllegalStateException("Cannot find tx with id " + txID);
- }
- ChannelBuffer bb = writeTransaction(COMMIT_RECORD, txID, tx, null);
+ if (tx == null)
+ {
+ throw new IllegalStateException("Cannot find tx with id " + txID);
+ }
- IOCallback callback = getTransactionCallback(txID, sync);
+ ChannelBuffer bb = newBuffer(SIZE_COMPLETE_TRANSACTION_RECORD);
- lock.acquire();
- try
- {
- JournalFile usedFile = appendRecord(bb, sync, callback);
+ writeTransaction(-1, COMMIT_RECORD, txID, tx, null, SIZE_COMPLETE_TRANSACTION_RECORD, -1, bb);
- transactionCallbacks.remove(txID);
+ lock.acquire();
+ try
+ {
+ JournalFile usedFile = appendRecord(bb, true, sync, tx, null);
- tx.commit(usedFile);
+ tx.commit(usedFile);
+ }
+ finally
+ {
+ lock.release();
+ }
+
}
finally
{
- lock.release();
+ compactingLock.readLock().unlock();
}
- // We should wait this outside of the lock, to increase throuput
- if (callback != null)
+ if (sync)
{
- callback.waitCompletion();
+ // We should wait this outside of the lock, to increase throuput
+ tx.waitCompletion();
}
-
}
public void appendRollbackRecord(final long txID, final boolean sync) throws Exception
@@ -681,48 +776,57 @@
throw new IllegalStateException("Journal must be loaded first");
}
- JournalTransaction tx = transactionInfos.remove(txID);
+ compactingLock.readLock().lock();
- if (tx == null)
+ JournalTransaction tx = null;
+
+ try
{
- throw new IllegalStateException("Cannot find tx with id " + txID);
- }
+ tx = transactions.remove(txID);
- int size = SIZE_ROLLBACK_RECORD;
+ if (tx == null)
+ {
+ throw new IllegalStateException("Cannot find tx with id " + txID);
+ }
- ChannelBuffer bb = newBuffer(size);
+ int size = SIZE_ROLLBACK_RECORD;
- bb.writeByte(ROLLBACK_RECORD);
- bb.writeInt(-1); // skip ID part
- bb.writeLong(txID);
- bb.writeInt(size);
+ ChannelBuffer bb = newBuffer(size);
- IOCallback callback = getTransactionCallback(txID, sync);
+ bb.writeByte(ROLLBACK_RECORD);
+ bb.writeInt(-1); // skip ID part
+ bb.writeLong(txID);
+ bb.writeInt(size);
- lock.acquire();
- try
- {
- JournalFile usedFile = appendRecord(bb, sync, callback);
+ lock.acquire();
+ try
+ {
+ JournalFile usedFile = appendRecord(bb, false, sync, tx, null);
- transactionCallbacks.remove(txID);
+ tx.rollback(usedFile);
+ }
+ finally
+ {
+ lock.release();
+ }
- tx.rollback(usedFile);
}
finally
{
- lock.release();
+ compactingLock.readLock().unlock();
}
// We should wait this outside of the lock, to increase throuput
- if (callback != null)
+
+ if (sync)
{
- callback.waitCompletion();
+ tx.waitCompletion();
}
}
/**
- * @see JournalImpl#load(LoadManager)
+ * @see JournalImpl#load(LoaderCallback)
*/
public synchronized long load(final List<RecordInfo> committedRecords,
final List<PreparedTransactionInfo> preparedTransactions) throws Exception
@@ -730,7 +834,7 @@
final Set<Long> recordsToDelete = new HashSet<Long>();
final List<RecordInfo> records = new ArrayList<RecordInfo>();
- long maxID = load(new LoadManager()
+ long maxID = load(new LoaderCallback()
{
public void addPreparedTransaction(PreparedTransactionInfo preparedTransaction)
{
@@ -764,8 +868,214 @@
return maxID;
}
- private boolean isInvalidSize(int bufferPos, int size)
+ /**
+ *
+ * Note: This method can't be called from the main executor, as it will invoke other methods depending on it.
+ */
+ public void compact() throws Exception
{
+ if (compactor != null)
+ {
+ throw new IllegalStateException("There is pending compacting operation");
+ }
+
+ ArrayList<JournalFile> dataFilesToProcess = new ArrayList<JournalFile>(dataFiles.size());
+
+ boolean previousReclaimValue = autoReclaim;
+
+ try
+ {
+
+ // We need to guarantee that the journal is frozen for this short time
+ // We don't freeze the journal as we compact, only for the short time where we replace records
+ compactingLock.writeLock().lock();
+ try
+ {
+
+ // We need to move to the next file, as we need a clear start for negatives and positives counts
+ moveNextFile();
+
+ autoReclaim = false;
+
+ // Take the snapshots and replace the structures
+
+ dataFilesToProcess.addAll(dataFiles);
+
+ dataFiles.clear();
+
+ this.compactor = new JournalCompactor(fileFactory,
+ this,
+ this.records.keySet(),
+ dataFilesToProcess.get(0).getFileID());
+
+ for (Map.Entry<Long, JournalTransaction> entry : transactions.entrySet())
+ {
+ compactor.addPendingTransaction(entry.getKey(), entry.getValue().getPositiveArray());
+ entry.getValue().setCompacting();
+ }
+
+ // We will calculate the new records during compacting, what will take the position the records will take
+ // after compacting
+ this.records.clear();
+ }
+ finally
+ {
+ compactingLock.writeLock().unlock();
+ }
+
+ Collections.sort(dataFilesToProcess, new JournalFileComparator());
+
+ // Read the files, and use the JournalCompactor class to create the new outputFiles, and the new collections as
+ // well
+ JournalFile previousFile = null;
+ for (final JournalFile file : dataFilesToProcess)
+ {
+ if (previousFile != null)
+ {
+ if (file.getFileID() < previousFile.getFileID())
+ {
+ // Sanity check, this should never happen
+ throw new IllegalStateException("DataFiles out of order!");
+ }
+ }
+ previousFile = file;
+
+ log.info("Compacting file " + file.getFile().getFileName() + ", internalID = " + file.getFileID());
+ readJournalFile(fileFactory, file, compactor);
+ }
+
+ compactor.flush();
+
+ // pointcut for tests
+ // We need to test concurrent updates on the journal, as the compacting is being performed.
+ // Usually tests will use this to hold the compacting while other structures are being updated.
+ onCompactDone();
+
+ SequentialFile controlFile = createControlFile(dataFilesToProcess, compactor.getNewDataFiles());
+
+ List<JournalFile> newDatafiles = null;
+
+ JournalCompactor localCompactor = compactor;
+
+ compactingLock.writeLock().lock();
+ try
+ {
+ // Need to clear the compactor here, or the replay commands will send commands back (infinite loop)
+ this.compactor = null;
+
+ newDatafiles = localCompactor.getNewDataFiles();
+
+ // Restore newRecords created during compacting
+ for (Map.Entry<Long, JournalRecord> newRecordEntry : localCompactor.getNewRecords().entrySet())
+ {
+ records.put(newRecordEntry.getKey(), newRecordEntry.getValue());
+ }
+
+ // Restore compacted dataFiles
+ for (int i = newDatafiles.size() - 1; i >= 0; i--)
+ {
+ JournalFile fileToAdd = newDatafiles.get(i);
+ if (trace)
+ {
+ trace("Adding file " + fileToAdd + " back as datafile");
+ }
+ dataFiles.addFirst(fileToAdd);
+ }
+
+ // Replay pending commands (including updates, deletes and commits)
+
+ localCompactor.replayPendingCommands();
+
+ // Merge transactions back after compacting
+ // This has to be done after the replay pending commands, as we need to delete committs that happened during
+ // the compacting
+
+ for (JournalTransaction newTransaction : localCompactor.getNewTransactions().values())
+ {
+ if (trace)
+ {
+ trace("Merging pending transaction " + newTransaction + " after compacting to the journal");
+ }
+ JournalTransaction liveTransaction = this.transactions.get(newTransaction.getId());
+ if (liveTransaction == null)
+ {
+ log.warn("Inconsistency: Can't merge transaction " + newTransaction.getId() +
+ " back into JournalTransactions");
+ }
+ else
+ {
+ liveTransaction.merge(newTransaction);
+ }
+ }
+ }
+ finally
+ {
+ compactingLock.writeLock().unlock();
+ }
+
+ // At this point the journal is unlocked. We keep renaming files while the journal is already operational
+ renameFiles(dataFilesToProcess, newDatafiles);
+ deleteControlFile(controlFile);
+
+ }
+ finally
+ {
+ // An Exception was probably thrown, and the compactor was not cleared
+ if (compactor != null)
+ {
+ try
+ {
+ compactor.flush();
+ }
+ catch (Throwable ignored)
+ {
+ }
+
+ compactor = null;
+ }
+ autoReclaim = previousReclaimValue;
+ }
+
+ }
+
+ protected void deleteControlFile(SequentialFile controlFile) throws Exception
+ {
+ controlFile.delete();
+ }
+
+ /** being protected as testcases can override this method */
+ protected void renameFiles(List<JournalFile> oldFiles, List<JournalFile> newFiles) throws Exception
+ {
+ for (JournalFile file : oldFiles)
+ {
+ dataFiles.remove(file);
+ freeFiles.add(reinitializeFile(file));
+ }
+
+ for (JournalFile file : newFiles)
+ {
+ String newName = file.getFile().getFileName();
+ newName = newName.substring(0, newName.lastIndexOf(".cmp"));
+ file.getFile().renameTo(newName);
+ }
+
+ }
+
+ /** This is an interception point for testcases, when the compacted files are written, before replacing the data structures */
+ protected void onCompactDone()
+ {
+ }
+
+ /**
+ * @throws Exception
+ */
+ protected SequentialFile createControlFile(List<JournalFile> files, List<JournalFile> newFiles) throws Exception
+ {
+ return JournalCompactor.writeControlFile(fileFactory, files, newFiles);
+ }
+
+ private static boolean isInvalidSize(int fileSize, int bufferPos, int size)
+ {
if (size < 0)
{
return true;
@@ -813,16 +1123,16 @@
* <p> * FileID and NumberOfElements are the transaction summary, and they will be repeated (N)umberOfFiles times </p>
*
* */
- public synchronized long load(final LoadManager loadManager) throws Exception
+ public synchronized long load(final LoaderCallback loadManager) throws Exception
{
if (state != STATE_STARTED)
{
throw new IllegalStateException("Journal must be in started state");
}
- fileFactory.controlBuffersLifeCycle(false);
+ checkControlFile();
- final Map<Long, TransactionHolder> transactions = new LinkedHashMap<Long, TransactionHolder>();
+ final Map<Long, TransactionHolder> loadTransactions = new LinkedHashMap<Long, TransactionHolder>();
final List<JournalFile> orderedFiles = orderFiles();
@@ -836,12 +1146,12 @@
final AtomicBoolean hasData = new AtomicBoolean(false);
- int resultLastPost = readJournalFile(file, new JournalReader()
+ int resultLastPost = readJournalFile(fileFactory, file, new JournalReaderCallback()
{
- public void addRecord(RecordInfo info) throws Exception
+ public void onReadAddRecord(RecordInfo info) throws Exception
{
- if (trace)
+ if (trace && LOAD_TRACE)
{
trace("AddRecord: " + info);
}
@@ -849,12 +1159,12 @@
loadManager.addRecord(info);
- posFilesMap.put(info.id, new PosFiles(file));
+ records.put(info.id, new JournalRecord(file, info.data.length + SIZE_ADD_RECORD));
}
- public void updateRecord(RecordInfo info) throws Exception
+ public void onReadUpdateRecord(RecordInfo info) throws Exception
{
- if (trace)
+ if (trace && LOAD_TRACE)
{
trace("UpdateRecord: " + info);
}
@@ -862,7 +1172,7 @@
loadManager.updateRecord(info);
- PosFiles posFiles = posFilesMap.get(info.id);
+ JournalRecord posFiles = records.get(info.id);
if (posFiles != null)
{
@@ -870,13 +1180,13 @@
// have been deleted
// just leaving some updates in this file
- posFiles.addUpdateFile(file);
+ posFiles.addUpdateFile(file, info.data.length + SIZE_UPDATE_RECORD);
}
}
- public void deleteRecord(long recordID) throws Exception
+ public void onReadDeleteRecord(long recordID) throws Exception
{
- if (trace)
+ if (trace && LOAD_TRACE)
{
trace("DeleteRecord: " + recordID);
}
@@ -884,118 +1194,118 @@
loadManager.deleteRecord(recordID);
- PosFiles posFiles = posFilesMap.remove(recordID);
+ JournalRecord posFiles = records.remove(recordID);
if (posFiles != null)
{
- posFiles.addDelete(file);
+ posFiles.delete(file);
}
}
- public void updateRecordTX(long transactionID, RecordInfo info) throws Exception
+ public void onReadUpdateRecordTX(long transactionID, RecordInfo info) throws Exception
{
- addRecordTX(transactionID, info);
+ onReadAddRecordTX(transactionID, info);
}
- public void addRecordTX(long transactionID, RecordInfo info) throws Exception
+ public void onReadAddRecordTX(long transactionID, RecordInfo info) throws Exception
{
- if (trace)
+ if (trace && LOAD_TRACE)
{
trace((info.isUpdate ? "updateRecordTX: " : "addRecordTX: ") + info + ", txid = " + transactionID);
}
hasData.set(true);
- TransactionHolder tx = transactions.get(transactionID);
+ TransactionHolder tx = loadTransactions.get(transactionID);
if (tx == null)
{
tx = new TransactionHolder(transactionID);
- transactions.put(transactionID, tx);
+ loadTransactions.put(transactionID, tx);
}
tx.recordInfos.add(info);
- JournalTransaction tnp = transactionInfos.get(transactionID);
+ JournalTransaction tnp = transactions.get(transactionID);
if (tnp == null)
{
- tnp = new JournalTransaction();
+ tnp = new JournalTransaction(info.id, JournalImpl.this);
- transactionInfos.put(transactionID, tnp);
+ transactions.put(transactionID, tnp);
}
- tnp.addPositive(file, info.id);
+ tnp.addPositive(file, info.id, info.data.length + SIZE_ADD_RECORD_TX);
}
- public void deleteRecordTX(long transactionID, RecordInfo info) throws Exception
+ public void onReadDeleteRecordTX(long transactionID, RecordInfo info) throws Exception
{
- if (trace)
+ if (trace && LOAD_TRACE)
{
trace("DeleteRecordTX: " + transactionID + " info = " + info);
}
hasData.set(true);
- TransactionHolder tx = transactions.get(transactionID);
+ TransactionHolder tx = loadTransactions.get(transactionID);
if (tx == null)
{
tx = new TransactionHolder(transactionID);
- transactions.put(transactionID, tx);
+ loadTransactions.put(transactionID, tx);
}
tx.recordsToDelete.add(info);
- JournalTransaction tnp = transactionInfos.get(transactionID);
+ JournalTransaction tnp = transactions.get(transactionID);
if (tnp == null)
{
- tnp = new JournalTransaction();
+ tnp = new JournalTransaction(transactionID, JournalImpl.this);
- transactionInfos.put(transactionID, tnp);
+ transactions.put(transactionID, tnp);
}
tnp.addNegative(file, info.id);
}
- public void prepareRecord(long transactionID, byte[] extraData, Pair<Integer, Integer>[] summary) throws Exception
+ public void onReadPrepareRecord(long transactionID, byte[] extraData, int numberOfRecords) throws Exception
{
- if (trace)
+ if (trace && LOAD_TRACE)
{
trace("prepareRecordTX: txid = " + transactionID);
}
hasData.set(true);
- TransactionHolder tx = transactions.get(transactionID);
+ TransactionHolder tx = loadTransactions.get(transactionID);
if (tx == null)
{
// The user could choose to prepare empty transactions
tx = new TransactionHolder(transactionID);
- transactions.put(transactionID, tx);
+ loadTransactions.put(transactionID, tx);
}
tx.prepared = true;
tx.extraData = extraData;
- JournalTransaction journalTransaction = transactionInfos.get(transactionID);
+ JournalTransaction journalTransaction = transactions.get(transactionID);
if (journalTransaction == null)
{
- journalTransaction = new JournalTransaction();
+ journalTransaction = new JournalTransaction(transactionID, JournalImpl.this);
- transactionInfos.put(transactionID, journalTransaction);
+ transactions.put(transactionID, journalTransaction);
}
- boolean healthy = checkTransactionHealth(journalTransaction, orderedFiles, summary);
+ boolean healthy = checkTransactionHealth(file, journalTransaction, orderedFiles, numberOfRecords);
if (healthy)
{
@@ -1008,14 +1318,14 @@
}
}
- public void commitRecord(long transactionID, Pair<Integer, Integer>[] summary) throws Exception
+ public void onReadCommitRecord(long transactionID, int numberOfRecords) throws Exception
{
- if (trace)
+ if (trace && LOAD_TRACE)
{
trace("commitRecord: txid = " + transactionID);
}
- TransactionHolder tx = transactions.remove(transactionID);
+ TransactionHolder tx = loadTransactions.remove(transactionID);
// The commit could be alone on its own journal-file and the
// whole transaction body was reclaimed but not the
@@ -1026,14 +1336,14 @@
// ignore this
if (tx != null)
{
- JournalTransaction journalTransaction = transactionInfos.remove(transactionID);
+ JournalTransaction journalTransaction = transactions.remove(transactionID);
if (journalTransaction == null)
{
throw new IllegalStateException("Cannot find tx " + transactionID);
}
- boolean healthy = checkTransactionHealth(journalTransaction, orderedFiles, summary);
+ boolean healthy = checkTransactionHealth(file, journalTransaction, orderedFiles, numberOfRecords);
if (healthy)
{
@@ -1069,14 +1379,14 @@
}
- public void rollbackRecord(long transactionID) throws Exception
+ public void onReadRollbackRecord(long transactionID) throws Exception
{
- if (trace)
+ if (trace && LOAD_TRACE)
{
trace("rollbackRecord: txid = " + transactionID);
}
- TransactionHolder tx = transactions.remove(transactionID);
+ TransactionHolder tx = loadTransactions.remove(transactionID);
// The rollback could be alone on its own journal-file and the
// whole transaction body was reclaimed but the commit-record
@@ -1084,7 +1394,7 @@
// point
if (tx != null)
{
- JournalTransaction tnp = transactionInfos.remove(transactionID);
+ JournalTransaction tnp = transactions.remove(transactionID);
if (tnp == null)
{
@@ -1101,7 +1411,7 @@
public void markAsDataFile(JournalFile file)
{
- if (trace)
+ if (trace && LOAD_TRACE)
{
trace("Marking " + file + " as data file");
}
@@ -1123,8 +1433,6 @@
}
}
- fileFactory.controlBuffersLifeCycle(true);
-
// Create any more files we need
// FIXME - size() involves a scan
@@ -1135,7 +1443,7 @@
for (int i = 0; i < filesToCreate; i++)
{
// Keeping all files opened can be very costly (mainly on AIO)
- freeFiles.add(createFile(false));
+ freeFiles.add(createFile(false, false, true));
}
}
@@ -1163,20 +1471,20 @@
{
currentFile = freeFiles.remove();
- openFile(currentFile);
+ openFile(currentFile, true);
}
fileFactory.activate(currentFile.getFile());
pushOpenedFile();
- for (TransactionHolder transaction : transactions.values())
+ for (TransactionHolder transaction : loadTransactions.values())
{
if (!transaction.prepared || transaction.invalid)
{
log.warn("Uncommitted transaction with id " + transaction.transactionID + " found and discarded");
- JournalTransaction transactionInfo = transactionInfos.get(transaction.transactionID);
+ JournalTransaction transactionInfo = transactions.get(transaction.transactionID);
if (transactionInfo == null)
{
@@ -1187,7 +1495,7 @@
transactionInfo.forget();
// Remove the transactionInfo
- transactionInfos.remove(transaction.transactionID);
+ transactions.remove(transaction.transactionID);
}
else
{
@@ -1208,6 +1516,62 @@
return maxID;
}
+ /**
+ * @return
+ * @throws Exception
+ */
+ private void checkControlFile() throws Exception
+ {
+ ArrayList<String> dataFiles = new ArrayList<String>();
+ ArrayList<String> newFiles = new ArrayList<String>();
+
+ SequentialFile controlFile = JournalCompactor.readControlFile(fileFactory, dataFiles, newFiles);
+ if (controlFile != null)
+ {
+ log.info("Journal Compactor was interrupted during renaming phase, renaming files");
+
+ for (String dataFile : dataFiles)
+ {
+ SequentialFile file = fileFactory.createSequentialFile(dataFile, 1);
+ log.info("Removing old compacted file" + file.getFileName());
+ if (file.exists())
+ {
+ file.delete();
+ }
+ }
+
+ for (String newFile : newFiles)
+ {
+ SequentialFile file = fileFactory.createSequentialFile(newFile, 1);
+ log.info("Renaming file " + file.getFileName() + " as an part of the data files");
+ if (file.exists())
+ {
+ final String originalName = file.getFileName();
+ final String newName = originalName.substring(0, originalName.lastIndexOf(".cmp"));
+ file.renameTo(newName);
+ }
+ }
+
+ controlFile.delete();
+ }
+
+ List<String> leftFiles = fileFactory.listFiles(this.getFileExtension() + ".cmp");
+
+ if (leftFiles.size() > 0)
+ {
+ log.warn("Compacted files were left unnatended on journal directory, deleting invalid files now");
+
+ for (String fileToDelete : leftFiles)
+ {
+ log.warn("Deleting unnatended file " + fileToDelete);
+ SequentialFile file = fileFactory.createSequentialFile(fileToDelete, 1);
+ file.delete();
+ }
+ }
+
+ return;
+ }
+
public int getAlignment() throws Exception
{
return fileFactory.getAlignment();
@@ -1239,6 +1603,8 @@
file.getPosCount() +
" reclaimStatus = " +
file.isCanReclaim() +
+ " live size = " +
+ file.getLiveSize() +
"\n");
if (file instanceof JournalFileImpl)
{
@@ -1275,9 +1641,11 @@
* It will call waitComplete on every transaction, so any assertions on the file system will be correct after this */
public void debugWait() throws Exception
{
- for (TransactionCallback callback : transactionCallbacks.values())
+ fileFactory.testFlush();
+
+ for (JournalTransaction tx : transactions.values())
{
- callback.waitCompletion();
+ tx.waitCallbacks();
}
if (filesExecutor != null && !filesExecutor.isShutdown())
@@ -1301,37 +1669,108 @@
public void checkAndReclaimFiles() throws Exception
{
- checkReclaimStatus();
+ // We can't start compacting while compacting is working
+ compactingLock.readLock().lock();
+ try
+ {
+ checkReclaimStatus();
- for (JournalFile file : dataFiles)
- {
- if (file.isCanReclaim())
+ for (JournalFile file : dataFiles)
{
- // File can be reclaimed or deleted
+ if (file.isCanReclaim())
+ {
+ // File can be reclaimed or deleted
- if (trace)
- {
- trace("Reclaiming file " + file);
+ if (trace)
+ {
+ trace("Reclaiming file " + file);
+ }
+
+ if (!dataFiles.remove(file))
+ {
+ log.warn("Could not remove file " + file);
+ }
+
+ // FIXME - size() involves a scan!!!
+ if (freeFiles.size() + dataFiles.size() + 1 + openedFiles.size() < minFiles)
+ {
+ // Re-initialise it
+
+ JournalFile jf = reinitializeFile(file);
+
+ freeFiles.add(jf);
+ }
+ else
+ {
+ file.getFile().open(1);
+
+ file.getFile().delete();
+ }
}
+ }
+ }
+ finally
+ {
+ compactingLock.readLock().unlock();
+ }
+ }
- dataFiles.remove(file);
+ public void checkCompact() throws Exception
+ {
+ if (compactMinFiles == 0)
+ {
+ // compacting is disabled
+ return;
+ }
- // FIXME - size() involves a scan!!!
- if (freeFiles.size() + dataFiles.size() + 1 + openedFiles.size() < minFiles)
- {
- // Re-initialise it
+ JournalFile[] dataFiles = getDataFiles();
- JournalFile jf = reinitializeFile(file);
+ long totalLiveSize = 0;
- freeFiles.add(jf);
- }
- else
+ for (JournalFile file : dataFiles)
+ {
+ totalLiveSize += file.getLiveSize();
+ }
+
+ long totalBytes = (long)dataFiles.length * (long)fileSize;
+
+ long compactMargin = (long)(totalBytes * compactPercentage);
+
+ if (totalLiveSize < compactMargin && compactor == null && dataFiles.length > compactMinFiles)
+ {
+
+ log.info("Compacting being started, numberOfDataFiles = " + dataFiles.length +
+ ", liveSize = " +
+ totalLiveSize +
+ ", margin to start compacting = " +
+ compactMargin);
+
+ compactorWait.up();
+
+ // We can't use the executor for the compacting... or we would lock files opening and creation (besides other
+ // operations)
+ // that would freeze the journal while compacting
+ Thread t = new Thread()
+ {
+ public void run()
{
- file.getFile().open(1);
- file.getFile().delete();
+ try
+ {
+ JournalImpl.this.compact();
+ }
+ catch (Exception e)
+ {
+ log.error(e.getMessage(), e);
+ }
+ finally
+ {
+ compactorWait.down();
+ }
}
- }
+ };
+
+ t.start();
}
}
@@ -1340,6 +1779,11 @@
return dataFiles.size();
}
+ public JournalFile[] getDataFiles()
+ {
+ return (JournalFile[])dataFiles.toArray(new JournalFile[dataFiles.size()]);
+ }
+
public int getFreeFilesCount()
{
return freeFiles.size();
@@ -1352,7 +1796,7 @@
public int getIDMapSize()
{
- return posFilesMap.size();
+ return records.size();
}
public int getFileSize()
@@ -1441,6 +1885,11 @@
{
log.warn("Couldn't stop journal executor after 60 seconds");
}
+
+ while (!compactorWait.waitCompletion(60000))
+ {
+ log.warn("Waiting the compactor to finish its operations");
+ }
fileFactory.stop();
@@ -1467,6 +1916,13 @@
finally
{
lock.release();
+ try
+ {
+ compactingLock.writeLock().unlock();
+ }
+ catch (Throwable ignored)
+ {
+ }
}
}
@@ -1478,15 +1934,13 @@
private void checkReclaimStatus() throws Exception
{
- JournalFile[] files = new JournalFile[dataFiles.size()];
-
- reclaimer.scan(dataFiles.toArray(files));
+ reclaimer.scan(getDataFiles());
}
// Discard the old JournalFile and set it with a new ID
private JournalFile reinitializeFile(final JournalFile file) throws Exception
{
- int newOrderingID = generateOrderingID();
+ int newFileID = generateFileID();
SequentialFile sf = file.getFile();
@@ -1494,15 +1948,15 @@
sf.position(0);
- ByteBuffer bb = fileFactory.newBuffer(SIZE_INT);
+ ByteBuffer bb = fileFactory.newBuffer(SIZE_HEADER);
- bb.putInt(newOrderingID);
+ bb.putInt(newFileID);
bb.rewind();
sf.write(bb, true);
- JournalFile jf = new JournalFileImpl(sf, newOrderingID);
+ JournalFile jf = new JournalFileImpl(sf, newFileID);
sf.position(bb.limit());
@@ -1511,343 +1965,332 @@
return jf;
}
-
- private int readJournalFile(JournalFile file, JournalReader reader) throws Exception
+ public static int readJournalFile(SequentialFileFactory fileFactory, JournalFile file, JournalReaderCallback reader) throws Exception
{
- ByteBuffer wholeFileBuffer = fileFactory.newBuffer(fileSize);
file.getFile().open(1);
-
- int bytesRead = file.getFile().read(wholeFileBuffer);
-
- if (bytesRead != fileSize)
+ ByteBuffer wholeFileBuffer = null;
+ try
{
- // FIXME - We should extract everything we can from this file
- // and then we shouldn't ever reuse this file on reclaiming (instead
- // reclaim on different size files would aways throw the file away)
- // rather than throw ISE!
- // We don't want to leave the user with an unusable system
- throw new IllegalStateException("File is wrong size " + bytesRead +
- " expected " +
- fileSize +
- " : " +
- file.getFile().getFileName());
- }
- wholeFileBuffer.position(0);
+ wholeFileBuffer = fileFactory.newBuffer((int)file.getFile().size());
- // First long is the ordering timestamp, we just jump its position
- wholeFileBuffer.position(SIZE_HEADER);
+ final int journalFileSize = file.getFile().read(wholeFileBuffer);
- int lastDataPos = SIZE_HEADER;
+ if (journalFileSize != file.getFile().size())
+ {
+ throw new RuntimeException("Invalid read! The system couldn't read the entire file into memory");
+ }
- while (wholeFileBuffer.hasRemaining())
- {
- final int pos = wholeFileBuffer.position();
+ wholeFileBuffer.position(0);
- byte recordType = wholeFileBuffer.get();
+ // First long is the ordering timestamp, we just jump its position
+ wholeFileBuffer.position(SIZE_HEADER);
- if (recordType < ADD_RECORD || recordType > ROLLBACK_RECORD)
- {
- // I - We scan for any valid record on the file. If a hole
- // happened on the middle of the file we keep looking until all
- // the possibilities are gone
- continue;
- }
+ int lastDataPos = SIZE_HEADER;
- if (isInvalidSize(wholeFileBuffer.position(), SIZE_INT))
+ while (wholeFileBuffer.hasRemaining())
{
- reader.markAsDataFile(file);
+ final int pos = wholeFileBuffer.position();
- wholeFileBuffer.position(pos + 1);
- // II - Ignore this record, lets keep looking
- continue;
- }
+ byte recordType = wholeFileBuffer.get();
- // III - Every record has the file-id.
- // This is what supports us from not re-filling the whole file
- int readFileId = wholeFileBuffer.getInt();
+ if (recordType < ADD_RECORD || recordType > ROLLBACK_RECORD)
+ {
+ // I - We scan for any valid record on the file. If a hole
+ // happened on the middle of the file we keep looking until all
+ // the possibilities are gone
+ continue;
+ }
- long transactionID = 0;
+ if (isInvalidSize(journalFileSize, wholeFileBuffer.position(), DataConstants.SIZE_INT))
+ {
+ reader.markAsDataFile(file);
- if (isTransaction(recordType))
- {
- if (isInvalidSize(wholeFileBuffer.position(), SIZE_LONG))
- {
wholeFileBuffer.position(pos + 1);
- reader.markAsDataFile(file);
+ // II - Ignore this record, lets keep looking
continue;
}
- transactionID = wholeFileBuffer.getLong();
- }
+ // III - Every record has the file-id.
+ // This is what supports us from not re-filling the whole file
+ int readFileId = wholeFileBuffer.getInt();
- long recordID = 0;
+ long transactionID = 0;
- if (!isCompleteTransaction(recordType))
- {
- if (isInvalidSize(wholeFileBuffer.position(), SIZE_LONG))
+ if (isTransaction(recordType))
{
- wholeFileBuffer.position(pos + 1);
- reader.markAsDataFile(file);
- continue;
+ if (isInvalidSize(journalFileSize, wholeFileBuffer.position(), DataConstants.SIZE_LONG))
+ {
+ wholeFileBuffer.position(pos + 1);
+ reader.markAsDataFile(file);
+ continue;
+ }
+
+ transactionID = wholeFileBuffer.getLong();
}
- recordID = wholeFileBuffer.getLong();
- }
+ long recordID = 0;
- // We use the size of the record to validate the health of the
- // record.
- // (V) We verify the size of the record
+ // If prepare or commit
+ if (!isCompleteTransaction(recordType))
+ {
+ if (isInvalidSize(journalFileSize, wholeFileBuffer.position(), DataConstants.SIZE_LONG))
+ {
+ wholeFileBuffer.position(pos + 1);
+ reader.markAsDataFile(file);
+ continue;
+ }
- // The variable record portion used on Updates and Appends
- int variableSize = 0;
+ recordID = wholeFileBuffer.getLong();
+ }
- // Used to hold extra data on transaction prepares
- int preparedTransactionExtraDataSize = 0;
+ // We use the size of the record to validate the health of the
+ // record.
+ // (V) We verify the size of the record
- byte userRecordType = 0;
+ // The variable record portion used on Updates and Appends
+ int variableSize = 0;
- byte record[] = null;
+ // Used to hold extra data on transaction prepares
+ int preparedTransactionExtraDataSize = 0;
- if (isContainsBody(recordType))
- {
- if (isInvalidSize(wholeFileBuffer.position(), SIZE_INT))
- {
- wholeFileBuffer.position(pos + 1);
- reader.markAsDataFile(file);
- continue;
- }
+ byte userRecordType = 0;
- variableSize = wholeFileBuffer.getInt();
+ byte record[] = null;
- if (isInvalidSize(wholeFileBuffer.position(), variableSize))
+ if (isContainsBody(recordType))
{
- wholeFileBuffer.position(pos + 1);
- continue;
- }
+ if (isInvalidSize(journalFileSize, wholeFileBuffer.position(), DataConstants.SIZE_INT))
+ {
+ wholeFileBuffer.position(pos + 1);
+ reader.markAsDataFile(file);
+ continue;
+ }
- if (recordType != DELETE_RECORD_TX)
- {
- userRecordType = wholeFileBuffer.get();
- }
+ variableSize = wholeFileBuffer.getInt();
- record = new byte[variableSize];
+ if (isInvalidSize(journalFileSize, wholeFileBuffer.position(), variableSize))
+ {
+ wholeFileBuffer.position(pos + 1);
+ continue;
+ }
- wholeFileBuffer.get(record);
- }
+ if (recordType != DELETE_RECORD_TX)
+ {
+ userRecordType = wholeFileBuffer.get();
+ }
- if (recordType == PREPARE_RECORD || recordType == COMMIT_RECORD)
- {
- if (recordType == PREPARE_RECORD)
- {
- // Add the variable size required for preparedTransactions
- preparedTransactionExtraDataSize = wholeFileBuffer.getInt();
+ record = new byte[variableSize];
+
+ wholeFileBuffer.get(record);
}
- // Both commit and record contain the recordSummary, and this is
- // used to calculate the record-size on both record-types
- variableSize += wholeFileBuffer.getInt() * SIZE_INT * 2;
- }
- int recordSize = getRecordSize(recordType);
+ // Case this is a transaction, this will contain the number of pendingTransactions on a transaction, at the
+ // currentFile
+ int transactionCheckNumberOfRecords = 0;
- // VI - this is completing V, We will validate the size at the end
- // of the record,
- // But we avoid buffer overflows by damaged data
- if (isInvalidSize(pos, recordSize + variableSize + preparedTransactionExtraDataSize))
- {
- // Avoid a buffer overflow caused by damaged data... continue
- // scanning for more records...
- trace("Record at position " + pos +
- " recordType = " +
- recordType +
- " file:" +
- file.getFile().getFileName() +
- " recordSize: " +
- recordSize +
- " variableSize: " +
- variableSize +
- " preparedTransactionExtraDataSize: " +
- preparedTransactionExtraDataSize +
- " is corrupted and it is being ignored (II)");
- // If a file has damaged records, we make it a dataFile, and the
- // next reclaiming will fix it
- reader.markAsDataFile(file);
- wholeFileBuffer.position(pos + 1);
+ if (recordType == PREPARE_RECORD || recordType == COMMIT_RECORD)
+ {
+ if (isInvalidSize(journalFileSize, wholeFileBuffer.position(), DataConstants.SIZE_INT))
+ {
+ wholeFileBuffer.position(pos + 1);
+ continue;
+ }
- continue;
- }
+ transactionCheckNumberOfRecords = wholeFileBuffer.getInt();
- int oldPos = wholeFileBuffer.position();
+ if (recordType == PREPARE_RECORD)
+ {
+ // Add the variable size required for preparedTransactions
+ preparedTransactionExtraDataSize = wholeFileBuffer.getInt();
+ }
+ variableSize = 0;
+ }
- wholeFileBuffer.position(pos + variableSize + recordSize + preparedTransactionExtraDataSize - SIZE_INT);
+ int recordSize = getRecordSize(recordType);
- int checkSize = wholeFileBuffer.getInt();
+ // VI - this is completing V, We will validate the size at the end
+ // of the record,
+ // But we avoid buffer overflows by damaged data
+ if (isInvalidSize(journalFileSize, pos, recordSize + variableSize + preparedTransactionExtraDataSize))
+ {
+ // Avoid a buffer overflow caused by damaged data... continue
+ // scanning for more pendingTransactions...
+ trace("Record at position " + pos +
+ " recordType = " +
+ recordType +
+ " file:" +
+ file.getFile().getFileName() +
+ " recordSize: " +
+ recordSize +
+ " variableSize: " +
+ variableSize +
+ " preparedTransactionExtraDataSize: " +
+ preparedTransactionExtraDataSize +
+ " is corrupted and it is being ignored (II)");
+ // If a file has damaged pendingTransactions, we make it a dataFile, and the
+ // next reclaiming will fix it
+ reader.markAsDataFile(file);
+ wholeFileBuffer.position(pos + 1);
- // VII - The checkSize at the end has to match with the size
- // informed at the beggining.
- // This is like testing a hash for the record. (We could replace the
- // checkSize by some sort of calculated hash)
- if (checkSize != variableSize + recordSize + preparedTransactionExtraDataSize)
- {
- trace("Record at position " + pos +
- " recordType = " +
- recordType +
- " file:" +
- file.getFile().getFileName() +
- " is corrupted and it is being ignored (III)");
+ continue;
+ }
- // If a file has damaged records, we make it a dataFile, and the
- // next reclaiming will fix it
- reader.markAsDataFile(file);
+ int oldPos = wholeFileBuffer.position();
- wholeFileBuffer.position(pos + SIZE_BYTE);
+ wholeFileBuffer.position(pos + variableSize +
+ recordSize +
+ preparedTransactionExtraDataSize -
+ DataConstants.SIZE_INT);
- continue;
- }
+ int checkSize = wholeFileBuffer.getInt();
- // This record is from a previous file-usage. The file was
- // reused and we need to ignore this record
- if (readFileId != file.getOrderingID())
- {
- // If a file has damaged records, we make it a dataFile, and the
- // next reclaiming will fix it
- reader.markAsDataFile(file);
+ // VII - The checkSize at the end has to match with the size
+ // informed at the beggining.
+ // This is like testing a hash for the record. (We could replace the
+ // checkSize by some sort of calculated hash)
+ if (checkSize != variableSize + recordSize + preparedTransactionExtraDataSize)
+ {
+ trace("Record at position " + pos +
+ " recordType = " +
+ recordType +
+ " file:" +
+ file.getFile().getFileName() +
+ " is corrupted and it is being ignored (III)");
- continue;
- }
+ // If a file has damaged pendingTransactions, we make it a dataFile, and the
+ // next reclaiming will fix it
+ reader.markAsDataFile(file);
- wholeFileBuffer.position(oldPos);
+ wholeFileBuffer.position(pos + DataConstants.SIZE_BYTE);
- // At this point everything is checked. So we relax and just load
- // the data now.
+ continue;
+ }
- switch (recordType)
- {
- case ADD_RECORD:
+ // This record is from a previous file-usage. The file was
+ // reused and we need to ignore this record
+ if (readFileId != file.getFileID())
{
- reader.addRecord(new RecordInfo(recordID, userRecordType, record, false));
- break;
+ // If a file has damaged pendingTransactions, we make it a dataFile, and the
+ // next reclaiming will fix it
+ reader.markAsDataFile(file);
+
+ continue;
}
-
- case UPDATE_RECORD:
- {
- reader.updateRecord(new RecordInfo(recordID, userRecordType, record, true));
- break;
- }
-
- case DELETE_RECORD:
- {
- reader.deleteRecord(recordID);
- break;
- }
- case ADD_RECORD_TX:
+ wholeFileBuffer.position(oldPos);
+
+ // At this point everything is checked. So we relax and just load
+ // the data now.
+
+ switch (recordType)
{
- reader.addRecordTX(transactionID, new RecordInfo(recordID, userRecordType, record, false));
- break;
- }
-
- case UPDATE_RECORD_TX:
- {
- reader.updateRecordTX(transactionID, new RecordInfo(recordID, userRecordType, record, true));
- break;
- }
-
- case DELETE_RECORD_TX:
- {
- reader.deleteRecordTX(transactionID, new RecordInfo(recordID, (byte)0, record, true));
- break;
- }
-
- case PREPARE_RECORD:
- {
+ case ADD_RECORD:
+ {
+ reader.onReadAddRecord(new RecordInfo(recordID, userRecordType, record, false));
+ break;
+ }
- byte extraData[] = new byte[preparedTransactionExtraDataSize];
+ case UPDATE_RECORD:
+ {
+ reader.onReadUpdateRecord(new RecordInfo(recordID, userRecordType, record, true));
+ break;
+ }
- wholeFileBuffer.get(extraData);
+ case DELETE_RECORD:
+ {
+ reader.onReadDeleteRecord(recordID);
+ break;
+ }
- // Pair <FileID, NumberOfElements>
- Pair<Integer, Integer>[] summary = readTransactionalElementsSummary(variableSize, wholeFileBuffer);
+ case ADD_RECORD_TX:
+ {
+ reader.onReadAddRecordTX(transactionID, new RecordInfo(recordID, userRecordType, record, false));
+ break;
+ }
- reader.prepareRecord(transactionID, extraData, summary);
+ case UPDATE_RECORD_TX:
+ {
+ reader.onReadUpdateRecordTX(transactionID, new RecordInfo(recordID, userRecordType, record, true));
+ break;
+ }
- break;
- }
- case COMMIT_RECORD:
- {
- // We need to read it even if transaction was not found, or
- // the reading checks would fail
- // Pair <OrderId, NumberOfElements>
- Pair<Integer, Integer>[] summary = readTransactionalElementsSummary(variableSize, wholeFileBuffer);
+ case DELETE_RECORD_TX:
+ {
+ reader.onReadDeleteRecordTX(transactionID, new RecordInfo(recordID, (byte)0, record, true));
+ break;
+ }
- reader.commitRecord(transactionID, summary);
- break;
- }
- case ROLLBACK_RECORD:
- {
- reader.rollbackRecord(transactionID);
- break;
- }
- default:
- {
- throw new IllegalStateException("Journal " + file.getFile().getFileName() +
- " is corrupt, invalid record type " +
- recordType);
- }
- }
+ case PREPARE_RECORD:
+ {
- checkSize = wholeFileBuffer.getInt();
+ byte extraData[] = new byte[preparedTransactionExtraDataSize];
- // This is a sanity check about the loading code itself.
- // If this checkSize doesn't match, it means the reading method is
- // not doing what it was supposed to do
- if (checkSize != variableSize + recordSize + preparedTransactionExtraDataSize)
- {
- throw new IllegalStateException("Internal error on loading file. Position doesn't match with checkSize, file = " + file.getFile() +
- ", pos = " +
- pos);
- }
+ wholeFileBuffer.get(extraData);
- lastDataPos = wholeFileBuffer.position();
+ reader.onReadPrepareRecord(transactionID, extraData, transactionCheckNumberOfRecords);
- }
+ break;
+ }
+ case COMMIT_RECORD:
+ {
- fileFactory.releaseBuffer(wholeFileBuffer);
+ reader.onReadCommitRecord(transactionID, transactionCheckNumberOfRecords);
+ break;
+ }
+ case ROLLBACK_RECORD:
+ {
+ reader.onReadRollbackRecord(transactionID);
+ break;
+ }
+ default:
+ {
+ throw new IllegalStateException("Journal " + file.getFile().getFileName() +
+ " is corrupt, invalid record type " +
+ recordType);
+ }
+ }
- file.getFile().close();
+ checkSize = wholeFileBuffer.getInt();
- return lastDataPos;
+ // This is a sanity check about the loading code itself.
+ // If this checkSize doesn't match, it means the reading method is
+ // not doing what it was supposed to do
+ if (checkSize != variableSize + recordSize + preparedTransactionExtraDataSize)
+ {
+ throw new IllegalStateException("Internal error on loading file. Position doesn't match with checkSize, file = " + file.getFile() +
+ ", pos = " +
+ pos);
+ }
- }
+ lastDataPos = wholeFileBuffer.position();
-
- /** It will read the elements-summary back from the commit/prepare transaction
- * Pair<FileID, Counter> */
- @SuppressWarnings("unchecked")
- // See comment on the method body
- private Pair<Integer, Integer>[] readTransactionalElementsSummary(final int variableSize, final ByteBuffer bb)
- {
- int numberOfFiles = variableSize / (SIZE_INT * 2);
+ }
- // This line aways show an annoying compilation-warning, the
- // SupressWarning is to avoid a warning about this cast
- Pair<Integer, Integer> values[] = new Pair[numberOfFiles];
+ return lastDataPos;
+ }
- for (int i = 0; i < numberOfFiles; i++)
+ finally
{
- values[i] = new Pair<Integer, Integer>(bb.getInt(), bb.getInt());
+ if (wholeFileBuffer != null)
+ {
+ fileFactory.releaseBuffer(wholeFileBuffer);
+ }
+
+ try
+ {
+ file.getFile().close();
+ }
+ catch (Throwable ignored)
+ {
+ }
}
-
- return values;
}
/**
* <p> Check for holes on the transaction (a commit written but with an incomplete transaction) </p>
* <p>This method will validate if the transaction (PREPARE/COMMIT) is complete as stated on the COMMIT-RECORD.</p>
- * <p> We record a summary about the records on the journal file on COMMIT and PREPARE.
- * When we load the records we build a new summary and we check the original summary to the current summary.
- * This method is basically verifying if the entire transaction is being loaded </p>
*
* <p>Look at the javadoc on {@link JournalImpl#appendCommitRecord(long)} about how the transaction-summary is recorded</p>
*
@@ -1856,67 +2299,26 @@
* @param recordedSummary
* @return
*/
- private boolean checkTransactionHealth(final JournalTransaction journalTransaction,
+ private boolean checkTransactionHealth(final JournalFile currentFile,
+ final JournalTransaction journalTransaction,
final List<JournalFile> orderedFiles,
- final Pair<Integer, Integer>[] recordedSummary)
+ final int numberOfRecords)
{
- boolean healthy = true;
-
- // (I) First we get the summary of what we really have on the files now:
-
- // FileID, NumberOfElements
- Map<Integer, AtomicInteger> currentSummary = journalTransaction.getElementsSummary();
-
- // (II) We compare the recorded summary on the commit, against to the
- // reality on the files
- for (Pair<Integer, Integer> ref : recordedSummary)
- {
- AtomicInteger counter = currentSummary.get(ref.a);
-
- if (counter == null)
- {
- for (JournalFile lookupFile : orderedFiles)
- {
- if (lookupFile.getOrderingID() == ref.a)
- {
- // (III) oops, we were expecting at least one record on this
- // file.
- // The file still exists and no records were found.
- // That means the transaction crashed before complete,
- // so this transaction is broken and needs to be ignored.
- // This is probably a hole caused by a crash during commit.
- healthy = false;
- break;
- }
- }
- }
- else
- {
- // (IV) Missing a record... Transaction was not completed as stated.
- // we will ignore the whole transaction
- // This is probably a hole caused by a crash during commit/prepare.
- if (counter.get() != ref.b)
- {
- healthy = false;
- break;
- }
- }
- }
- return healthy;
+ return journalTransaction.getCounter(currentFile) == numberOfRecords;
}
/**
* <p>A transaction record (Commit or Prepare), will hold the number of elements the transaction has on each file.</p>
- * <p>For example, a transaction was spread along 3 journal files with 10 records on each file.
- * (What could happen if there are too many records, or if an user event delayed records to come in time to a single file).</p>
+ * <p>For example, a transaction was spread along 3 journal files with 10 pendingTransactions on each file.
+ * (What could happen if there are too many pendingTransactions, or if an user event delayed pendingTransactions to come in time to a single file).</p>
* <p>The element-summary will then have</p>
* <p>FileID1, 10</p>
* <p>FileID2, 10</p>
* <p>FileID3, 10</p>
*
* <br>
- * <p> During the load, the transaction needs to have 30 records spread across the files as originally written.</p>
- * <p> If for any reason there are missing records, that means the transaction was not completed and we should ignore the whole transaction </p>
+ * <p> During the load, the transaction needs to have 30 pendingTransactions spread across the files as originally written.</p>
+ * <p> If for any reason there are missing pendingTransactions, that means the transaction was not completed and we should ignore the whole transaction </p>
* <p> We can't just use a global counter as reclaiming could delete files after the transaction was successfully committed.
* That also means not having a whole file on journal-reload doesn't mean we have to invalidate the transaction </p>
*
@@ -1927,63 +2329,176 @@
* @return
* @throws Exception
*/
- private ChannelBuffer writeTransaction(final byte recordType,
- final long txID,
- final JournalTransaction tx,
- final EncodingSupport transactionData) throws Exception
+ public static void writeTransaction(final int fileID,
+ final byte recordType,
+ final long txID,
+ final JournalTransaction tx,
+ final EncodingSupport transactionData,
+ final int size,
+ final int numberOfRecords,
+ final ChannelBuffer bb) throws Exception
{
- int size = SIZE_COMPLETE_TRANSACTION_RECORD + tx.getElementsSummary().size() *
- SIZE_INT *
- 2 +
- (transactionData != null ? transactionData.getEncodeSize() + SIZE_INT : 0);
-
- ChannelBuffer bb = newBuffer(size);
-
bb.writeByte(recordType);
- bb.writeInt(-1); // skip ID part
+ bb.writeInt(fileID); // skip ID part
bb.writeLong(txID);
+ bb.writeInt(numberOfRecords); // skip number of pendingTransactions part
if (transactionData != null)
{
bb.writeInt(transactionData.getEncodeSize());
}
- bb.writeInt(tx.getElementsSummary().size());
-
if (transactionData != null)
{
transactionData.encode(bb);
}
- for (Map.Entry<Integer, AtomicInteger> entry : tx.getElementsSummary().entrySet())
+ bb.writeInt(size);
+ }
+
+ /**
+ * @param txID
+ * @param id
+ * @param recordType
+ * @param record
+ * @param size
+ * @param bb
+ */
+ public static void writeUpdateRecordTX(final int fileID,
+ final long txID,
+ final long id,
+ final byte recordType,
+ final EncodingSupport record,
+ int size,
+ ChannelBuffer bb)
+ {
+ bb.writeByte(UPDATE_RECORD_TX);
+ bb.writeInt(fileID);
+ bb.writeLong(txID);
+ bb.writeLong(id);
+ bb.writeInt(record.getEncodeSize());
+ bb.writeByte(recordType);
+ record.encode(bb);
+ bb.writeInt(size);
+ }
+
+ /**
+ * @param id
+ * @param recordType
+ * @param record
+ * @param size
+ * @param bb
+ */
+ public static void writeUpdateRecord(final int fileId,
+ final long id,
+ final byte recordType,
+ final EncodingSupport record,
+ int size,
+ ChannelBuffer bb)
+ {
+ bb.writeByte(UPDATE_RECORD);
+ bb.writeInt(fileId);
+ bb.writeLong(id);
+ bb.writeInt(record.getEncodeSize());
+ bb.writeByte(recordType);
+ record.encode(bb);
+ bb.writeInt(size);
+ }
+
+ /**
+ * @param id
+ * @param recordType
+ * @param record
+ * @param size
+ * @param bb
+ */
+ public static void writeAddRecord(final int fileId,
+ final long id,
+ final byte recordType,
+ final EncodingSupport record,
+ int size,
+ ChannelBuffer bb)
+ {
+ bb.writeByte(ADD_RECORD);
+ bb.writeInt(fileId);
+ bb.writeLong(id);
+ bb.writeInt(record.getEncodeSize());
+ bb.writeByte(recordType);
+ record.encode(bb);
+ bb.writeInt(size);
+ }
+
+ /**
+ * @param txID
+ * @param id
+ * @param record
+ * @param size
+ * @param bb
+ */
+ public static void writeDeleteRecordTransactional(final int fileID,
+ final long txID,
+ final long id,
+ final EncodingSupport record,
+ int size,
+ ChannelBuffer bb)
+ {
+ bb.writeByte(DELETE_RECORD_TX);
+ bb.writeInt(fileID);
+ bb.writeLong(txID);
+ bb.writeLong(id);
+ bb.writeInt(record != null ? record.getEncodeSize() : 0);
+ if (record != null)
{
- bb.writeInt(entry.getKey());
- bb.writeInt(entry.getValue().get());
+ record.encode(bb);
}
+ bb.writeInt(size);
+ }
+ /**
+ * @param txID
+ * @param id
+ * @param recordType
+ * @param record
+ * @param recordLength
+ * @param size
+ * @param bb
+ */
+ public static void writeAddRecordTX(final int fileID,
+ final long txID,
+ final long id,
+ final byte recordType,
+ final EncodingSupport record,
+ int size,
+ ChannelBuffer bb)
+ {
+ bb.writeByte(ADD_RECORD_TX);
+ bb.writeInt(fileID);
+ bb.writeLong(txID);
+ bb.writeLong(id);
+ bb.writeInt(record.getEncodeSize());
+ bb.writeByte(recordType);
+ record.encode(bb);
bb.writeInt(size);
-
- return bb;
}
- private boolean isTransaction(final byte recordType)
+ private static boolean isTransaction(final byte recordType)
{
return recordType == ADD_RECORD_TX || recordType == UPDATE_RECORD_TX ||
recordType == DELETE_RECORD_TX ||
isCompleteTransaction(recordType);
}
- private boolean isCompleteTransaction(final byte recordType)
+ private static boolean isCompleteTransaction(final byte recordType)
{
return recordType == COMMIT_RECORD || recordType == PREPARE_RECORD || recordType == ROLLBACK_RECORD;
}
- private boolean isContainsBody(final byte recordType)
+ private static boolean isContainsBody(final byte recordType)
{
return recordType >= ADD_RECORD && recordType <= DELETE_RECORD_TX;
}
- private int getRecordSize(final byte recordType)
+ private static int getRecordSize(final byte recordType)
{
// The record size (without the variable portion)
int recordSize = 0;
@@ -2041,22 +2556,22 @@
file.open(1);
- ByteBuffer bb = fileFactory.newBuffer(SIZE_INT);
+ ByteBuffer bb = fileFactory.newBuffer(SIZE_HEADER);
file.read(bb);
- int orderingID = bb.getInt();
+ int fileID = bb.getInt();
fileFactory.releaseBuffer(bb);
bb = null;
- if (nextOrderingId.get() < orderingID)
+ if (nextFileID.get() < fileID)
{
- nextOrderingId.set(orderingID);
+ nextFileID.set(fileID);
}
- orderedFiles.add(new JournalFileImpl(file, orderingID));
+ orderedFiles.add(new JournalFileImpl(file, fileID));
file.close();
}
@@ -2064,17 +2579,6 @@
// Now order them by ordering id - we can't use the file name for ordering
// since we can re-use dataFiles
- class JournalFileComparator implements Comparator<JournalFile>
- {
- public int compare(final JournalFile f1, final JournalFile f2)
- {
- int id1 = f1.getOrderingID();
- int id2 = f2.getOrderingID();
-
- return id1 < id2 ? -1 : id1 == id2 ? 0 : 1;
- }
- }
-
Collections.sort(orderedFiles, new JournalFileComparator());
return orderedFiles;
@@ -2082,8 +2586,14 @@
/**
* Note: You should aways guarantee locking the semaphore lock.
+ *
+ * @param completeTransaction If the appendRecord is for a prepare or commit, where we should update the number of pendingTransactions on the current file
* */
- private JournalFile appendRecord(final MessagingBuffer bb, final boolean sync, final IOCallback callback) throws Exception
+ private JournalFile appendRecord(final MessagingBuffer bb,
+ final boolean completeTransaction,
+ final boolean sync,
+ final JournalTransaction tx,
+ IOCallback callback) throws Exception
{
try
{
@@ -2119,13 +2629,45 @@
if (currentFile == null)
{
- throw new IllegalStateException("Current file = null");
+ throw new NullPointerException("Current file = null");
}
- bb.writerIndex(SIZE_BYTE);
+ if (tx != null)
+ {
+ if (callback != null)
+ {
+ // sanity check, it should not happen.
+ throw new IllegalArgumentException("Invalid callback parameter. Use of tx is mutually exclusive with the callback");
+ }
- bb.writeInt(currentFile.getOrderingID());
+ // The callback of a transaction has to be taken inside the lock,
+ // when we guarantee the currentFile will not be changed,
+ // since we individualize the callback per file
+ if (fileFactory.isSupportsCallbacks())
+ {
+ callback = tx.getCallback(currentFile);
+ }
+ if (sync)
+ {
+ // 99 % of the times this will be already synced, as previous files should be closed already.
+ // This is to have 100% guarantee the transaction will be persisted and no loss of information would
+ // happen
+ tx.syncPreviousFiles(fileFactory.isSupportsCallbacks(), currentFile);
+ }
+
+ // We need to add the number of records on currentFile if prepare or commit
+ if (completeTransaction)
+ {
+ // Filling the number of pendingTransactions at the current file
+ tx.fillNumberOfRecords(currentFile, bb);
+ }
+ }
+
+ // Adding fileID
+ bb.writerIndex(DataConstants.SIZE_BYTE);
+ bb.writeInt(currentFile.getFileID());
+
if (callback != null)
{
currentFile.getFile().write(bb, sync, callback);
@@ -2150,11 +2692,11 @@
* @return
* @throws Exception
*/
- private JournalFile createFile(final boolean keepOpened) throws Exception
+ private JournalFile createFile(final boolean keepOpened, final boolean multiAIO, final boolean fill) throws Exception
{
- int orderingID = generateOrderingID();
+ int fileID = generateFileID();
- String fileName = filePrefix + "-" + orderingID + "." + fileExtension;
+ String fileName = filePrefix + "-" + fileID + "." + fileExtension;
if (trace)
{
@@ -2163,19 +2705,29 @@
SequentialFile sequentialFile = fileFactory.createSequentialFile(fileName, maxAIO);
- sequentialFile.open();
+ if (multiAIO)
+ {
+ sequentialFile.open();
+ }
+ else
+ {
+ sequentialFile.open(1);
+ }
- sequentialFile.fill(0, fileSize, FILL_CHARACTER);
+ if (fill)
+ {
+ sequentialFile.fill(0, fileSize, FILL_CHARACTER);
- ByteBuffer bb = fileFactory.newBuffer(SIZE_INT);
+ ByteBuffer bb = fileFactory.newBuffer(SIZE_HEADER);
- bb.putInt(orderingID);
+ bb.putInt(fileID);
- bb.rewind();
+ bb.rewind();
- sequentialFile.write(bb, true);
+ sequentialFile.write(bb, true);
+ }
- JournalFile info = new JournalFileImpl(sequentialFile, orderingID);
+ JournalFile info = new JournalFileImpl(sequentialFile, fileID);
if (!keepOpened)
{
@@ -2185,25 +2737,38 @@
return info;
}
- private void openFile(final JournalFile file) throws Exception
+ private void openFile(final JournalFile file, final boolean multiAIO) throws Exception
{
- file.getFile().open();
+ if (multiAIO)
+ {
+ file.getFile().open();
+ }
+ else
+ {
+ file.getFile().open(1);
+ }
file.getFile().position(file.getFile().calculateBlockStart(SIZE_HEADER));
}
- private int generateOrderingID()
+ private int generateFileID()
{
- return nextOrderingId.incrementAndGet();
+ return nextFileID.incrementAndGet();
}
// You need to guarantee lock.acquire() before calling this method
private void moveNextFile() throws InterruptedException
{
+ // Asynchronously close the file
closeFile(currentFile);
currentFile = enqueueOpenFile();
+ if (trace)
+ {
+ trace("moveNextFile: " + currentFile.getFile().getFileName());
+ }
+
fileFactory.activate(currentFile.getFile());
}
@@ -2250,6 +2815,20 @@
}
}
});
+ filesExecutor.execute(new Runnable()
+ {
+ public void run()
+ {
+ try
+ {
+ checkCompact();
+ }
+ catch (Exception e)
+ {
+ log.error(e.getMessage(), e);
+ }
+ }
+ });
}
JournalFile nextFile = null;
@@ -2272,6 +2851,17 @@
* */
private void pushOpenedFile() throws Exception
{
+ JournalFile nextOpenedFile = getFile(true, true, true);
+
+ openedFiles.offer(nextOpenedFile);
+ }
+
+ /**
+ * @return
+ * @throws Exception
+ */
+ JournalFile getFile(boolean keepOpened, boolean multiAIO, boolean fill) throws Exception
+ {
JournalFile nextOpenedFile = null;
try
{
@@ -2283,19 +2873,23 @@
if (nextOpenedFile == null)
{
- nextOpenedFile = createFile(true);
+ nextOpenedFile = createFile(keepOpened, multiAIO, fill);
}
else
{
- openFile(nextOpenedFile);
+ if (keepOpened)
+ {
+ openFile(nextOpenedFile, multiAIO);
+ }
}
-
- openedFiles.offer(nextOpenedFile);
+ return nextOpenedFile;
}
private void closeFile(final JournalFile file)
{
fileFactory.deactivate(file.getFile());
+ dataFiles.add(file);
+
filesExecutor.execute(new Runnable()
{
public void run()
@@ -2308,20 +2902,19 @@
{
log.warn(e.getMessage(), e);
}
- dataFiles.add(file);
}
});
}
private JournalTransaction getTransactionInfo(final long txID)
{
- JournalTransaction tx = transactionInfos.get(txID);
+ JournalTransaction tx = transactions.get(txID);
if (tx == null)
{
- tx = new JournalTransaction();
+ tx = new JournalTransaction(txID, this);
- JournalTransaction trans = transactionInfos.putIfAbsent(txID, tx);
+ JournalTransaction trans = transactions.putIfAbsent(txID, tx);
if (trans != null)
{
@@ -2351,39 +2944,6 @@
}
}
- private IOCallback getTransactionCallback(final long transactionId, final boolean sync) throws MessagingException
- {
- if (sync && fileFactory.isSupportsCallbacks())
- {
- TransactionCallback callback = transactionCallbacks.get(transactionId);
-
- if (callback == null)
- {
- callback = new TransactionCallback();
-
- TransactionCallback callbackCheck = transactionCallbacks.putIfAbsent(transactionId, callback);
-
- if (callbackCheck != null)
- {
- callback = callbackCheck;
- }
- }
-
- if (callback.errorMessage != null)
- {
- throw new MessagingException(callback.errorCode, callback.errorMessage);
- }
-
- callback.countUp();
-
- return callback;
- }
- else
- {
- return null;
- }
- }
-
public ChannelBuffer newBuffer(final int size)
{
return ChannelBuffers.buffer(size);
@@ -2392,82 +2952,55 @@
// Inner classes
// ---------------------------------------------------------------------------
- // Just encapsulates the VariableLatch waiting for transaction completions
- // Used if the SequentialFile supports Callbacks
- private static class TransactionCallback implements IOCallback
+ /**
+ * This holds the relationship a record has with other files in regard to reference counting.
+ * Note: This class used to be called PosFiles
+ *
+ * Used on the ref-count for reclaiming */
+ public static class JournalRecord
{
- private final VariableLatch countLatch = new VariableLatch();
+ private final JournalFile addFile;
- private volatile String errorMessage = null;
+ private final int size;
- private volatile int errorCode = 0;
+ private List<Pair<JournalFile, Integer>> updateFiles;
- public void countUp()
+ JournalRecord(final JournalFile addFile, final int size)
{
- countLatch.up();
- }
+ this.addFile = addFile;
- public void done()
- {
- countLatch.down();
- }
+ this.size = size;
- public void waitCompletion() throws InterruptedException
- {
- countLatch.waitCompletion();
+ addFile.incPosCount();
- if (errorMessage != null)
- {
- throw new IllegalStateException("Error on Transaction: " + errorCode + " - " + errorMessage);
- }
+ addFile.addSize(size);
}
- public void onError(final int errorCode, final String errorMessage)
+ void addUpdateFile(final JournalFile updateFile, final int size)
{
- this.errorMessage = errorMessage;
-
- this.errorCode = errorCode;
-
- countLatch.down();
- }
-
- }
-
- /** Used on the ref-count for reclaiming */
- private static class PosFiles
- {
- private final JournalFile addFile;
-
- private List<JournalFile> updateFiles;
-
- PosFiles(final JournalFile addFile)
- {
- this.addFile = addFile;
-
- addFile.incPosCount();
- }
-
- void addUpdateFile(final JournalFile updateFile)
- {
if (updateFiles == null)
{
- updateFiles = new ArrayList<JournalFile>();
+ updateFiles = new ArrayList<Pair<JournalFile, Integer>>();
}
- updateFiles.add(updateFile);
+ updateFiles.add(new Pair<JournalFile, Integer>(updateFile, size));
updateFile.incPosCount();
+
+ updateFile.addSize(size);
}
- void addDelete(final JournalFile file)
+ void delete(final JournalFile file)
{
file.incNegCount(addFile);
+ addFile.decSize(size);
if (updateFiles != null)
{
- for (JournalFile jf : updateFiles)
+ for (Pair<JournalFile, Integer> updFile : updateFiles)
{
- file.incNegCount(jf);
+ file.incNegCount(updFile.a);
+ updFile.a.decSize(updFile.b);
}
}
}
@@ -2475,14 +3008,14 @@
public String toString()
{
StringBuffer buffer = new StringBuffer();
- buffer.append("PosFiles(add=" + addFile.getFile().getFileName());
+ buffer.append("JournalRecord(add=" + addFile.getFile().getFileName());
if (updateFiles != null)
{
- for (JournalFile update : updateFiles)
+ for (Pair<JournalFile, Integer> update : updateFiles)
{
- buffer.append(", update=" + update.getFile().getFileName());
+ buffer.append(", update=" + update.a.getFile().getFileName());
}
}
@@ -2493,165 +3026,27 @@
}
}
- private class JournalTransaction
+ private static class NullEncoding implements EncodingSupport
{
- private List<Pair<JournalFile, Long>> pos;
- private List<Pair<JournalFile, Long>> neg;
+ static NullEncoding instance = new NullEncoding();
- private Set<JournalFile> transactionPos;
-
- // Map of file id to number of elements participating on the transaction
- // in that file
- // Used to verify completion on reload
- private final Map<Integer, AtomicInteger> numberOfElementsPerFile = new HashMap<Integer, AtomicInteger>();
-
- public Map<Integer, AtomicInteger> getElementsSummary()
+ public void decode(MessagingBuffer buffer)
{
- return numberOfElementsPerFile;
}
- public void addPositive(final JournalFile file, final long id)
+ public void encode(MessagingBuffer buffer)
{
- getCounter(file).incrementAndGet();
-
- addTXPosCount(file);
-
- if (pos == null)
- {
- pos = new ArrayList<Pair<JournalFile, Long>>();
- }
-
- pos.add(new Pair<JournalFile, Long>(file, id));
}
- public void addNegative(final JournalFile file, final long id)
+ public int getEncodeSize()
{
- getCounter(file).incrementAndGet();
-
- addTXPosCount(file);
-
- if (neg == null)
- {
- neg = new ArrayList<Pair<JournalFile, Long>>();
- }
-
- neg.add(new Pair<JournalFile, Long>(file, id));
+ return 0;
}
- public void commit(final JournalFile file)
- {
- if (pos != null)
- {
- for (Pair<JournalFile, Long> p : pos)
- {
- PosFiles posFiles = posFilesMap.get(p.b);
-
- if (posFiles == null)
- {
- posFiles = new PosFiles(p.a);
-
- posFilesMap.put(p.b, posFiles);
- }
- else
- {
- posFiles.addUpdateFile(p.a);
- }
- }
- }
-
- if (neg != null)
- {
- for (Pair<JournalFile, Long> n : neg)
- {
- PosFiles posFiles = posFilesMap.remove(n.b);
-
- if (posFiles != null)
- {
- posFiles.addDelete(n.a);
- }
- }
- }
-
- // Now add negs for the pos we added in each file in which there were
- // transactional operations
-
- for (JournalFile jf : transactionPos)
- {
- file.incNegCount(jf);
- }
- }
-
- public void rollback(final JournalFile file)
- {
- // Now add negs for the pos we added in each file in which there were
- // transactional operations
- // Note that we do this on rollback as we do on commit, since we need
- // to ensure the file containing
- // the rollback record doesn't get deleted before the files with the
- // transactional operations are deleted
- // Otherwise we may run into problems especially with XA where we are
- // just left with a prepare when the tx
- // has actually been rolled back
-
- for (JournalFile jf : transactionPos)
- {
- file.incNegCount(jf);
- }
- }
-
- public void prepare(final JournalFile file)
- {
- // We don't want the prepare record getting deleted before time
-
- addTXPosCount(file);
- }
-
- public void forget()
- {
- // The transaction was not committed or rolled back in the file, so we
- // reverse any pos counts we added
-
- for (JournalFile jf : transactionPos)
- {
- jf.decPosCount();
- }
- }
-
- private void addTXPosCount(final JournalFile file)
- {
- if (transactionPos == null)
- {
- transactionPos = new HashSet<JournalFile>();
- }
-
- if (!transactionPos.contains(file))
- {
- transactionPos.add(file);
-
- // We add a pos for the transaction itself in the file - this
- // prevents any transactional operations
- // being deleted before a commit or rollback is written
- file.incPosCount();
- }
- }
-
- private AtomicInteger getCounter(final JournalFile file)
- {
- AtomicInteger value = numberOfElementsPerFile.get(file.getOrderingID());
-
- if (value == null)
- {
- value = new AtomicInteger();
- numberOfElementsPerFile.put(file.getOrderingID(), value);
- }
-
- return value;
- }
-
}
- private static class ByteArrayEncoding implements EncodingSupport
+ public static class ByteArrayEncoding implements EncodingSupport
{
final byte[] data;
@@ -2701,6 +3096,17 @@
}
+ private static class JournalFileComparator implements Comparator<JournalFile>
+ {
+ public int compare(final JournalFile f1, final JournalFile f2)
+ {
+ int id1 = f1.getFileID();
+ int id2 = f2.getFileID();
+
+ return id1 < id2 ? -1 : id1 == id2 ? 0 : 1;
+ }
+ }
+
private class PerfBlast extends Thread
{
private final int pages;
@@ -2720,7 +3126,7 @@
for (int i = 0; i < pages; i++)
{
- appendRecord(bb, false, null);
+ appendRecord(bb, false, false, null, null);
}
lock.release();
@@ -2732,61 +3138,4 @@
}
}
- private static interface JournalReader
- {
- void addRecord(RecordInfo info) throws Exception;
-
- /**
- * @param recordInfo
- * @throws Exception
- */
- void updateRecord(RecordInfo recordInfo) throws Exception;
-
- /**
- * @param recordID
- */
- void deleteRecord(long recordID) throws Exception;
-
- /**
- * @param transactionID
- * @param recordInfo
- * @throws Exception
- */
- void addRecordTX(long transactionID, RecordInfo recordInfo) throws Exception;
-
- /**
- * @param transactionID
- * @param recordInfo
- * @throws Exception
- */
- void updateRecordTX(long transactionID, RecordInfo recordInfo) throws Exception;
-
- /**
- * @param transactionID
- * @param recordInfo
- */
- void deleteRecordTX(long transactionID, RecordInfo recordInfo) throws Exception;
-
- /**
- * @param transactionID
- * @param extraData
- * @param summaryData
- */
- void prepareRecord(long transactionID, byte[] extraData, Pair<Integer, Integer>[] summary) throws Exception;
-
- /**
- * @param transactionID
- * @param summaryData
- */
- void commitRecord(long transactionID, Pair<Integer, Integer>[] summary) throws Exception;
-
- /**
- * @param transactionID
- */
- void rollbackRecord(long transactionID) throws Exception;
-
- public void markAsDataFile(JournalFile file);
-
- }
-
}
Copied: trunk/src/main/org/jboss/messaging/core/journal/impl/JournalReaderCallback.java (from rev 7512, branches/clebert_temp_expirement/src/main/org/jboss/messaging/core/journal/impl/JournalReaderCallback.java)
===================================================================
--- trunk/src/main/org/jboss/messaging/core/journal/impl/JournalReaderCallback.java (rev 0)
+++ trunk/src/main/org/jboss/messaging/core/journal/impl/JournalReaderCallback.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -0,0 +1,91 @@
+/*
+ * JBoss, Home of Professional Open Source
+ * Copyright 2005-2009, Red Hat Middleware LLC, and individual contributors
+ * by the @authors tag. See the copyright.txt in the distribution for a
+ * full listing of individual contributors.
+ *
+ * This is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this software; if not, write to the Free
+ * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+ * 02110-1301 USA, or see the FSF site: http://www.fsf.org.
+ */
+
+
+package org.jboss.messaging.core.journal.impl;
+
+import org.jboss.messaging.core.journal.RecordInfo;
+
+/**
+ * A JournalReader
+ *
+ * @author <a href="mailto:clebert.suconic at jboss.org">Clebert Suconic</a>
+ *
+ *
+ */
+public interface JournalReaderCallback
+{
+ void onReadAddRecord(RecordInfo info) throws Exception;
+
+ /**
+ * @param recordInfo
+ * @throws Exception
+ */
+ void onReadUpdateRecord(RecordInfo recordInfo) throws Exception;
+
+ /**
+ * @param recordID
+ */
+ void onReadDeleteRecord(long recordID) throws Exception;
+
+ /**
+ * @param transactionID
+ * @param recordInfo
+ * @throws Exception
+ */
+ void onReadAddRecordTX(long transactionID, RecordInfo recordInfo) throws Exception;
+
+ /**
+ * @param transactionID
+ * @param recordInfo
+ * @throws Exception
+ */
+ void onReadUpdateRecordTX(long transactionID, RecordInfo recordInfo) throws Exception;
+
+ /**
+ * @param transactionID
+ * @param recordInfo
+ */
+ void onReadDeleteRecordTX(long transactionID, RecordInfo recordInfo) throws Exception;
+
+ /**
+ * @param transactionID
+ * @param extraData
+ * @param summaryData
+ */
+ void onReadPrepareRecord(long transactionID, byte[] extraData, int numberOfRecords) throws Exception;
+
+ /**
+ * @param transactionID
+ * @param summaryData
+ */
+ void onReadCommitRecord(long transactionID, int numberOfRecords) throws Exception;
+
+ /**
+ * @param transactionID
+ */
+ void onReadRollbackRecord(long transactionID) throws Exception;
+
+ public void markAsDataFile(JournalFile file);
+
+
+}
Copied: trunk/src/main/org/jboss/messaging/core/journal/impl/JournalReaderCallbackAbstract.java (from rev 7512, branches/clebert_temp_expirement/src/main/org/jboss/messaging/core/journal/impl/JournalReaderCallbackAbstract.java)
===================================================================
--- trunk/src/main/org/jboss/messaging/core/journal/impl/JournalReaderCallbackAbstract.java (rev 0)
+++ trunk/src/main/org/jboss/messaging/core/journal/impl/JournalReaderCallbackAbstract.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -0,0 +1,78 @@
+/*
+ * JBoss, Home of Professional Open Source
+ * Copyright 2005-2009, Red Hat Middleware LLC, and individual contributors
+ * by the @authors tag. See the copyright.txt in the distribution for a
+ * full listing of individual contributors.
+ *
+ * This is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this software; if not, write to the Free
+ * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+ * 02110-1301 USA, or see the FSF site: http://www.fsf.org.
+ */
+
+
+package org.jboss.messaging.core.journal.impl;
+
+import org.jboss.messaging.core.journal.RecordInfo;
+
+/**
+ * A JournalReaderCallbackAbstract
+ *
+ * @author <mailto:clebert.suconic at jboss.org">Clebert Suconic</a>
+ *
+ *
+ */
+public class JournalReaderCallbackAbstract implements JournalReaderCallback
+{
+
+ public void markAsDataFile(JournalFile file)
+ {
+ }
+
+ public void onReadAddRecord(RecordInfo info) throws Exception
+ {
+ }
+
+ public void onReadAddRecordTX(long transactionID, RecordInfo recordInfo) throws Exception
+ {
+ }
+
+ public void onReadCommitRecord(long transactionID, int numberOfRecords) throws Exception
+ {
+ }
+
+ public void onReadDeleteRecord(long recordID) throws Exception
+ {
+ }
+
+ public void onReadDeleteRecordTX(long transactionID, RecordInfo recordInfo) throws Exception
+ {
+ }
+
+ public void onReadPrepareRecord(long transactionID, byte[] extraData, int numberOfRecords) throws Exception
+ {
+ }
+
+ public void onReadRollbackRecord(long transactionID) throws Exception
+ {
+ }
+
+ public void onReadUpdateRecord(RecordInfo recordInfo) throws Exception
+ {
+ }
+
+ public void onReadUpdateRecordTX(long transactionID, RecordInfo recordInfo) throws Exception
+ {
+ }
+
+}
Copied: trunk/src/main/org/jboss/messaging/core/journal/impl/JournalTransaction.java (from rev 7512, branches/clebert_temp_expirement/src/main/org/jboss/messaging/core/journal/impl/JournalTransaction.java)
===================================================================
--- trunk/src/main/org/jboss/messaging/core/journal/impl/JournalTransaction.java (rev 0)
+++ trunk/src/main/org/jboss/messaging/core/journal/impl/JournalTransaction.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -0,0 +1,515 @@
+/*
+ * JBoss, Home of Professional Open Source
+ * Copyright 2005-2009, Red Hat Middleware LLC, and individual contributors
+ * by the @authors tag. See the copyright.txt in the distribution for a
+ * full listing of individual contributors.
+ *
+ * This is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this software; if not, write to the Free
+ * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+ * 02110-1301 USA, or see the FSF site: http://www.fsf.org.
+ */
+
+package org.jboss.messaging.core.journal.impl;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.jboss.messaging.core.exception.MessagingException;
+import org.jboss.messaging.core.remoting.spi.MessagingBuffer;
+import org.jboss.messaging.utils.DataConstants;
+import org.jboss.messaging.utils.Pair;
+
+/**
+ * A JournalTransaction
+ *
+ * @author <a href="mailto:clebert.suconic at jboss.org">Clebert Suconic</a>
+ *
+ *
+ */
+public class JournalTransaction
+{
+
+ private final JournalImpl journal;
+
+ private List<JournalUpdate> pos;
+
+ private List<JournalUpdate> neg;
+
+ private final long id;
+
+ // All the files this transaction is touching on.
+ // We can't have those files being reclaimed if there is a pending transaction
+ private Set<JournalFile> pendingFiles;
+
+ private TransactionCallback currentCallback;
+
+ private boolean compacting = false;
+
+ private Map<JournalFile, TransactionCallback> callbackList;
+
+ private JournalFile lastFile = null;
+
+ private final AtomicInteger counter = new AtomicInteger();
+
+ public JournalTransaction(final long id, final JournalImpl journal)
+ {
+ this.id = id;
+ this.journal = journal;
+ }
+
+ /**
+ * @return the id
+ */
+ public long getId()
+ {
+ return id;
+ }
+
+ public int getCounter(final JournalFile file)
+ {
+ return internalgetCounter(file).intValue();
+ }
+
+ public void incCounter(final JournalFile file)
+ {
+ internalgetCounter(file).incrementAndGet();
+ }
+
+ public long[] getPositiveArray()
+ {
+ if (pos == null)
+ {
+ return new long[0];
+ }
+ else
+ {
+ int i = 0;
+ long[] ids = new long[pos.size()];
+ for (JournalUpdate el : pos)
+ {
+ ids[i++] = el.getId();
+ }
+ return ids;
+ }
+ }
+
+ public void setCompacting()
+ {
+ compacting = true;
+
+ // Everything is cleared on the transaction...
+ // since we are compacting, everything is at the compactor's level
+ clear();
+ }
+
+ /** This is used to merge transactions from compacting */
+ public void merge(JournalTransaction other)
+ {
+ if (other.pos != null)
+ {
+ if (pos == null)
+ {
+ pos = new ArrayList<JournalUpdate>();
+ }
+
+ pos.addAll(other.pos);
+ }
+
+ if (other.neg != null)
+ {
+ if (neg == null)
+ {
+ neg = new ArrayList<JournalUpdate>();
+ }
+
+ neg.addAll(other.neg);
+ }
+
+ if (other.pendingFiles != null)
+ {
+ if (pendingFiles == null)
+ {
+ pendingFiles = new HashSet<JournalFile>();
+ }
+
+ pendingFiles.addAll(other.pendingFiles);
+ }
+
+ this.compacting = false;
+ }
+
+ /**
+ *
+ */
+ public void clear()
+ {
+ // / Compacting is recreating all the previous files and everything
+ // / so we just clear the list of previous files, previous pos and previous adds
+ // / The transaction may be working at the top from now
+
+ if (pendingFiles != null)
+ {
+ pendingFiles.clear();
+ }
+
+ if (callbackList != null)
+ {
+ callbackList.clear();
+ }
+
+ if (pos != null)
+ {
+ pos.clear();
+ }
+
+ if (neg != null)
+ {
+ neg.clear();
+ }
+
+ counter.set(0);
+
+ lastFile = null;
+
+ currentCallback = null;
+ }
+
+ /**
+ * @param currentFile
+ * @param bb
+ */
+ public void fillNumberOfRecords(final JournalFile currentFile, final MessagingBuffer bb)
+ {
+ bb.writerIndex(DataConstants.SIZE_BYTE + DataConstants.SIZE_INT + DataConstants.SIZE_LONG);
+
+ bb.writeInt(getCounter(currentFile));
+
+ }
+
+ /** 99.99 % of the times previous files will be already synced, since they are scheduled to be closed.
+ * Because of that, this operation should be almost very fast.*/
+ public void syncPreviousFiles(final boolean callbacks, final JournalFile currentFile) throws Exception
+ {
+ if (callbacks)
+ {
+ if (callbackList != null)
+ {
+ for (Map.Entry<JournalFile, TransactionCallback> entry : callbackList.entrySet())
+ {
+ if (entry.getKey() != currentFile)
+ {
+ entry.getValue().waitCompletion();
+ }
+ }
+ }
+ }
+ else
+ {
+ for (JournalFile file : pendingFiles)
+ {
+ if (file != currentFile)
+ {
+ file.getFile().waitForClose();
+ }
+ }
+ }
+ }
+
+ /**
+ * @return
+ */
+ public TransactionCallback getCallback(final JournalFile file) throws Exception
+ {
+ if (callbackList == null)
+ {
+ callbackList = new HashMap<JournalFile, TransactionCallback>();
+ }
+
+ currentCallback = callbackList.get(file);
+
+ if (currentCallback == null)
+ {
+ currentCallback = new TransactionCallback();
+ callbackList.put(file, currentCallback);
+ }
+
+ if (currentCallback.getErrorMessage() != null)
+ {
+ throw new MessagingException(currentCallback.getErrorCode(), currentCallback.getErrorMessage());
+ }
+
+ currentCallback.countUp();
+
+ return currentCallback;
+ }
+
+ public void addPositive(final JournalFile file, final long id, final int size)
+ {
+ incCounter(file);
+
+ addFile(file);
+
+ if (pos == null)
+ {
+ pos = new ArrayList<JournalUpdate>();
+ }
+
+ pos.add(new JournalUpdate(file, id, size));
+ }
+
+ public void addNegative(final JournalFile file, final long id)
+ {
+ incCounter(file);
+
+ addFile(file);
+
+ if (neg == null)
+ {
+ neg = new ArrayList<JournalUpdate>();
+ }
+
+ neg.add(new JournalUpdate(file, id, 0));
+ }
+
+ /**
+ * The caller of this method needs to guarantee lock.acquire at the journal. (unless this is being called from load what is a single thread process).
+ * */
+ public void commit(final JournalFile file)
+ {
+ JournalCompactor compactor = journal.getCompactor();
+
+ if (compacting)
+ {
+ compactor.addCommandCommit(this, file);
+ }
+ else
+ {
+
+ if (pos != null)
+ {
+ for (JournalUpdate trUpdate : pos)
+ {
+ JournalImpl.JournalRecord posFiles = journal.getRecords().get(trUpdate.id);
+
+ if (compactor != null && compactor.lookupRecord(trUpdate.id))
+ {
+ // This is a case where the transaction was opened after compacting was started,
+ // but the commit arrived while compacting was working
+ // We need to cache the counter update, so compacting will take the correct files when it is done
+ compactor.addCommandUpdate(trUpdate.id, trUpdate.file, trUpdate.size);
+ }
+ else if (posFiles == null)
+ {
+ posFiles = new JournalImpl.JournalRecord(trUpdate.file, trUpdate.size);
+
+ journal.getRecords().put(trUpdate.id, posFiles);
+ }
+ else
+ {
+ posFiles.addUpdateFile(trUpdate.file, trUpdate.size);
+ }
+ }
+ }
+
+ if (neg != null)
+ {
+ for (JournalUpdate trDelete : neg)
+ {
+ JournalImpl.JournalRecord posFiles = journal.getRecords().remove(trDelete.id);
+
+ if (posFiles != null)
+ {
+ posFiles.delete(trDelete.file);
+ }
+ else if (compactor != null && compactor.lookupRecord(trDelete.id))
+ {
+ // This is a case where the transaction was opened after compacting was started,
+ // but the commit arrived while compacting was working
+ // We need to cache the counter update, so compacting will take the correct files when it is done
+ compactor.addCommandDelete(trDelete.id, trDelete.file);
+ }
+ }
+ }
+
+ // Now add negs for the pos we added in each file in which there were
+ // transactional operations
+
+ for (JournalFile jf : pendingFiles)
+ {
+ file.incNegCount(jf);
+ }
+ }
+ }
+
+ public void waitCallbacks() throws Exception
+ {
+ if (callbackList != null)
+ {
+ for (TransactionCallback callback : callbackList.values())
+ {
+ callback.waitCompletion();
+ }
+ }
+ }
+
+ /** Wait completion at the latest file only */
+ public void waitCompletion() throws Exception
+ {
+ if (currentCallback != null)
+ {
+ currentCallback.waitCompletion();
+ }
+ }
+
+ /**
+ * The caller of this method needs to guarantee lock.acquire before calling this method if being used outside of the lock context.
+ * or else potFilesMap could be affected
+ * */
+ public void rollback(final JournalFile file)
+ {
+ JournalCompactor compactor = journal.getCompactor();
+
+ if (compacting && compactor != null)
+ {
+ compactor.addCommandRollback(this, file);
+ }
+ else
+ {
+ // Now add negs for the pos we added in each file in which there were
+ // transactional operations
+ // Note that we do this on rollback as we do on commit, since we need
+ // to ensure the file containing
+ // the rollback record doesn't get deleted before the files with the
+ // transactional operations are deleted
+ // Otherwise we may run into problems especially with XA where we are
+ // just left with a prepare when the tx
+ // has actually been rolled back
+
+ for (JournalFile jf : pendingFiles)
+ {
+ file.incNegCount(jf);
+ }
+ }
+ }
+
+ /**
+ * The caller of this method needs to guarantee lock.acquire before calling this method if being used outside of the lock context.
+ * or else potFilesMap could be affected
+ * */
+ public void prepare(final JournalFile file)
+ {
+ // We don't want the prepare record getting deleted before time
+
+ addFile(file);
+ }
+
+ /** Used by load, when the transaction was not loaded correctly */
+ public void forget()
+ {
+ // The transaction was not committed or rolled back in the file, so we
+ // reverse any pos counts we added
+ for (JournalFile jf : pendingFiles)
+ {
+ jf.decPosCount();
+ }
+
+ }
+
+ public String toString()
+ {
+ return "JournalTransaction(" + this.id + ")";
+ }
+
+ private AtomicInteger internalgetCounter(final JournalFile file)
+ {
+ if (lastFile != file)
+
+ {
+ lastFile = file;
+ counter.set(0);
+ }
+ return counter;
+ }
+
+ private void addFile(final JournalFile file)
+ {
+ if (pendingFiles == null)
+ {
+ pendingFiles = new HashSet<JournalFile>();
+ }
+
+ if (!pendingFiles.contains(file))
+ {
+ pendingFiles.add(file);
+
+ // We add a pos for the transaction itself in the file - this
+ // prevents any transactional operations
+ // being deleted before a commit or rollback is written
+ file.incPosCount();
+ }
+ }
+
+ static class JournalUpdate
+ {
+ JournalFile file;
+
+ long id;
+
+ int size;
+
+
+ /**
+ * @param file
+ * @param id
+ * @param size
+ */
+ public JournalUpdate(JournalFile file, long id, int size)
+ {
+ super();
+ this.file = file;
+ this.id = id;
+ this.size = size;
+ }
+
+ /**
+ * @return the file
+ */
+ public JournalFile getFile()
+ {
+ return file;
+ }
+
+ /**
+ * @return the id
+ */
+ public long getId()
+ {
+ return id;
+ }
+
+ /**
+ * @return the size
+ */
+ public int getSize()
+ {
+ return size;
+ }
+
+ }
+}
Modified: trunk/src/main/org/jboss/messaging/core/journal/impl/NIOSequentialFile.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/journal/impl/NIOSequentialFile.java 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/src/main/org/jboss/messaging/core/journal/impl/NIOSequentialFile.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -46,7 +46,7 @@
private static final Logger log = Logger.getLogger(NIOSequentialFile.class);
private File file;
-
+
private long fileSize = 0;
private final String directory;
@@ -54,7 +54,7 @@
private FileChannel channel;
private RandomAccessFile rfile;
-
+
private final AtomicLong position = new AtomicLong(0);
public NIOSequentialFile(final String directory, final String fileName)
@@ -63,11 +63,16 @@
file = new File(directory + "/" + fileName);
}
+ public boolean exists()
+ {
+ return file.exists();
+ }
+
public int getAlignment()
{
return 1;
}
-
+
public void flush()
{
}
@@ -76,7 +81,7 @@
{
return position;
}
-
+
public boolean fits(final int size)
{
return this.position.get() + size <= fileSize;
@@ -97,7 +102,7 @@
rfile = new RandomAccessFile(file, "rw");
channel = rfile.getChannel();
-
+
fileSize = channel.size();
}
@@ -124,12 +129,20 @@
channel.force(false);
channel.position(0);
-
+
fileSize = channel.size();
}
- public void close() throws Exception
+ public synchronized void waitForClose() throws Exception
{
+ while (isOpen())
+ {
+ wait();
+ }
+ }
+
+ public synchronized void close() throws Exception
+ {
if (channel != null)
{
channel.close();
@@ -143,6 +156,8 @@
channel = null;
rfile = null;
+
+ notifyAll();
}
public void delete() throws Exception
@@ -184,7 +199,6 @@
}
-
public void write(final MessagingBuffer bytes, final boolean sync) throws Exception
{
write(ByteBuffer.wrap(bytes.array()), sync);
@@ -202,7 +216,7 @@
channel.write(bytes);
if (sync)
- {
+ {
sync();
}
}
@@ -212,9 +226,9 @@
try
{
position.addAndGet(bytes.limit());
-
+
channel.write(bytes);
-
+
if (sync)
{
sync();
@@ -234,7 +248,10 @@
public void sync() throws Exception
{
- channel.force(false);
+ if (channel != null)
+ {
+ channel.force(false);
+ }
}
public long size() throws Exception
Copied: trunk/src/main/org/jboss/messaging/core/journal/impl/TransactionCallback.java (from rev 7512, branches/clebert_temp_expirement/src/main/org/jboss/messaging/core/journal/impl/TransactionCallback.java)
===================================================================
--- trunk/src/main/org/jboss/messaging/core/journal/impl/TransactionCallback.java (rev 0)
+++ trunk/src/main/org/jboss/messaging/core/journal/impl/TransactionCallback.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -0,0 +1,92 @@
+/*
+ * JBoss, Home of Professional Open Source
+ * Copyright 2005-2009, Red Hat Middleware LLC, and individual contributors
+ * by the @authors tag. See the copyright.txt in the distribution for a
+ * full listing of individual contributors.
+ *
+ * This is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this software; if not, write to the Free
+ * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+ * 02110-1301 USA, or see the FSF site: http://www.fsf.org.
+ */
+
+
+package org.jboss.messaging.core.journal.impl;
+
+import org.jboss.messaging.core.journal.IOCallback;
+import org.jboss.messaging.utils.VariableLatch;
+
+/**
+ * A TransactionCallback
+ *
+ * @author <a href="mailto:clebert.suconic at jboss.org">Clebert Suconic</a>
+ *
+ *
+ */
+public class TransactionCallback implements IOCallback
+{
+ private final VariableLatch countLatch = new VariableLatch();
+
+ private volatile String errorMessage = null;
+
+ private volatile int errorCode = 0;
+
+ public void countUp()
+ {
+ countLatch.up();
+ }
+
+ public void done()
+ {
+ countLatch.down();
+ }
+
+ public void waitCompletion() throws InterruptedException
+ {
+ countLatch.waitCompletion();
+
+ if (errorMessage != null)
+ {
+ throw new IllegalStateException("Error on Transaction: " + errorCode + " - " + errorMessage);
+ }
+ }
+
+ public void onError(final int errorCode, final String errorMessage)
+ {
+ this.errorMessage = errorMessage;
+
+ this.errorCode = errorCode;
+
+ countLatch.down();
+ }
+
+ /**
+ * @return the errorMessage
+ */
+ public String getErrorMessage()
+ {
+ return errorMessage;
+ }
+
+ /**
+ * @return the errorCode
+ */
+ public int getErrorCode()
+ {
+ return errorCode;
+ }
+
+
+
+
+}
Modified: trunk/src/main/org/jboss/messaging/core/persistence/impl/journal/JournalStorageManager.java
===================================================================
--- trunk/src/main/org/jboss/messaging/core/persistence/impl/journal/JournalStorageManager.java 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/src/main/org/jboss/messaging/core/persistence/impl/journal/JournalStorageManager.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -141,11 +141,11 @@
private final boolean syncTransactional;
private final boolean syncNonTransactional;
-
+
private final int perfBlastPages;
public JournalStorageManager(final Configuration config, final Executor executor)
- {
+ {
this.executor = executor;
if (config.getJournalType() != JournalType.NIO && config.getJournalType() != JournalType.ASYNCIO)
@@ -164,7 +164,14 @@
SequentialFileFactory bindingsFF = new NIOSequentialFileFactory(bindingsDir);
- bindingsJournal = new JournalImpl(1024 * 1024, 2, bindingsFF, "jbm-bindings", "bindings", 1);
+ bindingsJournal = new JournalImpl(1024 * 1024,
+ 2,
+ config.getJournalCompactMinFiles(),
+ config.getJournalCompactPercentage(),
+ bindingsFF,
+ "jbm-bindings",
+ "bindings",
+ 1);
String journalDir = config.getJournalDirectory();
@@ -210,7 +217,9 @@
}
messageJournal = new JournalImpl(config.getJournalFileSize(),
- config.getJournalMinFiles(),
+ config.getJournalMinFiles(),
+ config.getJournalCompactMinFiles(),
+ config.getJournalCompactPercentage(),
journalFF,
"jbm-data",
"jbm",
@@ -221,7 +230,7 @@
checkAndCreateDir(largeMessagesDirectory, config.isCreateJournalDir());
largeMessagesFactory = new NIOSequentialFileFactory(config.getLargeMessagesDirectory());
-
+
perfBlastPages = config.getJournalPerfBlastPages();
}
@@ -243,14 +252,14 @@
}
public long generateUniqueID()
- {
+ {
long id = idGenerator.generateID();
return id;
}
public long getCurrentUniqueID()
- {
+ {
return idGenerator.getCurrentID();
}
@@ -340,16 +349,11 @@
messageJournal.appendAddRecordTransactional(txID,
message.getMessageID(),
ADD_LARGE_MESSAGE,
- new LargeMessageEncoding(((LargeServerMessage)message)),
- syncTransactional);
+ new LargeMessageEncoding(((LargeServerMessage)message)));
}
else
{
- messageJournal.appendAddRecordTransactional(txID,
- message.getMessageID(),
- ADD_MESSAGE,
- message,
- syncTransactional);
+ messageJournal.appendAddRecordTransactional(txID, message.getMessageID(), ADD_MESSAGE, message);
}
}
@@ -360,7 +364,7 @@
{
// Instead of updating the record, we delete the old one as that is
// better for reclaiming
- messageJournal.appendDeleteRecordTransactional(txID, pageTransaction.getRecordID(), syncTransactional);
+ messageJournal.appendDeleteRecordTransactional(txID, pageTransaction.getRecordID());
}
pageTransaction.setRecordID(generateUniqueID());
@@ -368,31 +372,22 @@
messageJournal.appendAddRecordTransactional(txID,
pageTransaction.getRecordID(),
PAGE_TRANSACTION,
- pageTransaction,
- syncTransactional);
+ pageTransaction);
}
public void storeReferenceTransactional(final long txID, final long queueID, final long messageID) throws Exception
{
- messageJournal.appendUpdateRecordTransactional(txID,
- messageID,
- ADD_REF,
- new RefEncoding(queueID),
- syncTransactional);
+ messageJournal.appendUpdateRecordTransactional(txID, messageID, ADD_REF, new RefEncoding(queueID));
}
public void storeAcknowledgeTransactional(final long txID, final long queueID, final long messageID) throws Exception
{
- messageJournal.appendUpdateRecordTransactional(txID,
- messageID,
- ACKNOWLEDGE_REF,
- new RefEncoding(queueID),
- syncTransactional);
+ messageJournal.appendUpdateRecordTransactional(txID, messageID, ACKNOWLEDGE_REF, new RefEncoding(queueID));
}
public void deletePageTransactional(final long txID, final long recordID) throws Exception
{
- messageJournal.appendDeleteRecordTransactional(txID, recordID, syncTransactional);
+ messageJournal.appendDeleteRecordTransactional(txID, recordID);
}
public void updateScheduledDeliveryTimeTransactional(final long txID, final MessageReference ref) throws Exception
@@ -403,13 +398,12 @@
messageJournal.appendUpdateRecordTransactional(txID,
ref.getMessage().getMessageID(),
SET_SCHEDULED_DELIVERY_TIME,
- encoding,
- syncTransactional);
+ encoding);
}
public void deleteMessageTransactional(final long txID, final long queueID, final long messageID) throws Exception
{
- messageJournal.appendDeleteRecordTransactional(txID, messageID, new DeleteEncoding(queueID), syncTransactional);
+ messageJournal.appendDeleteRecordTransactional(txID, messageID, new DeleteEncoding(queueID));
}
public void prepare(final long txID, final Xid xid) throws Exception
@@ -434,7 +428,7 @@
{
DuplicateIDEncoding encoding = new DuplicateIDEncoding(address, duplID);
- messageJournal.appendAddRecordTransactional(txID, recordID, DUPLICATE_ID, encoding, syncTransactional);
+ messageJournal.appendAddRecordTransactional(txID, recordID, DUPLICATE_ID, encoding);
}
public void updateDuplicateIDTransactional(final long txID,
@@ -444,12 +438,12 @@
{
DuplicateIDEncoding encoding = new DuplicateIDEncoding(address, duplID);
- messageJournal.appendUpdateRecordTransactional(txID, recordID, DUPLICATE_ID, encoding, syncTransactional);
+ messageJournal.appendUpdateRecordTransactional(txID, recordID, DUPLICATE_ID, encoding);
}
public void deleteDuplicateIDTransactional(long txID, long recordID) throws Exception
{
- messageJournal.appendDeleteRecordTransactional(txID, recordID, syncTransactional);
+ messageJournal.appendDeleteRecordTransactional(txID, recordID);
}
// Other operations
@@ -697,7 +691,7 @@
}
loadPreparedTransactions(pagingManager, resourceManager, queues, preparedTransactions, duplicateIDMap);
-
+
if (perfBlastPages != -1)
{
messageJournal.perfBlast(perfBlastPages);
@@ -896,7 +890,7 @@
public void addQueueBinding(final Binding binding) throws Exception
{
Queue queue = (Queue)binding.getBindable();
-
+
Filter filter = queue.getFilter();
SimpleString filterString = filter == null ? null : filter.getFilterString();
@@ -966,7 +960,7 @@
throw new IllegalStateException("Invalid record type " + rec);
}
}
-
+
idGenerator.setID(lastID + 1);
}
@@ -985,7 +979,7 @@
bindingsJournal.start();
messageJournal.start();
-
+
started = true;
}
Modified: trunk/tests/config/ConfigurationTest-full-config.xml
===================================================================
--- trunk/tests/config/ConfigurationTest-full-config.xml 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/tests/config/ConfigurationTest-full-config.xml 2009-07-02 21:36:15 UTC (rev 7516)
@@ -38,6 +38,8 @@
<create-journal-dir>false</create-journal-dir>
<journal-type>NIO</journal-type>
<journal-aio-flush-on-sync>true</journal-aio-flush-on-sync>
+ <journal-compact-min-files>123</journal-compact-min-files>
+ <journal-compact-percentage>33</journal-compact-percentage>
<journal-aio-buffer-timeout>1000</journal-aio-buffer-timeout>
<journal-aio-buffer-size>10000</journal-aio-buffer-size>
<journal-sync-transactional>false</journal-sync-transactional>
Copied: trunk/tests/src/org/jboss/messaging/tests/integration/journal/AIOJournalCompactTest.java (from rev 7512, branches/clebert_temp_expirement/tests/src/org/jboss/messaging/tests/integration/journal/AIOJournalCompactTest.java)
===================================================================
--- trunk/tests/src/org/jboss/messaging/tests/integration/journal/AIOJournalCompactTest.java (rev 0)
+++ trunk/tests/src/org/jboss/messaging/tests/integration/journal/AIOJournalCompactTest.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -0,0 +1,79 @@
+/*
+ * JBoss, Home of Professional Open Source
+ * Copyright 2005-2009, Red Hat Middleware LLC, and individual contributors
+ * by the @authors tag. See the copyright.txt in the distribution for a
+ * full listing of individual contributors.
+ *
+ * This is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this software; if not, write to the Free
+ * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+ * 02110-1301 USA, or see the FSF site: http://www.fsf.org.
+ */
+
+
+package org.jboss.messaging.tests.integration.journal;
+
+import java.io.File;
+
+import org.jboss.messaging.core.config.impl.ConfigurationImpl;
+import org.jboss.messaging.core.journal.SequentialFileFactory;
+import org.jboss.messaging.core.journal.impl.AIOSequentialFileFactory;
+
+/**
+ * A AIOJournalCompactTest
+ *
+ * @author <a href="mailto:clebert.suconic at jboss.com">Clebert Suconic</a>
+ *
+ *
+ */
+public class AIOJournalCompactTest extends NIOJournalCompactTest
+{
+
+ // Constants -----------------------------------------------------
+
+ // Attributes ----------------------------------------------------
+
+ // Static --------------------------------------------------------
+
+ // Constructors --------------------------------------------------
+
+ // Public --------------------------------------------------------
+
+ // Package protected ---------------------------------------------
+
+ // Protected -----------------------------------------------------
+
+ @Override
+ protected SequentialFileFactory getFileFactory() throws Exception
+ {
+ File file = new File(getTestDir());
+
+ deleteDirectory(file);
+
+ file.mkdir();
+
+ return new AIOSequentialFileFactory(getTestDir(),
+ ConfigurationImpl.DEFAULT_JOURNAL_AIO_BUFFER_SIZE,
+ 1000000,
+ true,
+ false
+ );
+ }
+
+
+
+ // Private -------------------------------------------------------
+
+ // Inner classes -------------------------------------------------
+
+}
Copied: trunk/tests/src/org/jboss/messaging/tests/integration/journal/NIOJournalCompactTest.java (from rev 7512, branches/clebert_temp_expirement/tests/src/org/jboss/messaging/tests/integration/journal/NIOJournalCompactTest.java)
===================================================================
--- trunk/tests/src/org/jboss/messaging/tests/integration/journal/NIOJournalCompactTest.java (rev 0)
+++ trunk/tests/src/org/jboss/messaging/tests/integration/journal/NIOJournalCompactTest.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -0,0 +1,807 @@
+/*
+ * JBoss, Home of Professional Open Source Copyright 2005-2008, Red Hat
+ * Middleware LLC, and individual contributors by the @authors tag. See the
+ * copyright.txt in the distribution for a full listing of individual
+ * contributors.
+ *
+ * This is free software; you can redistribute it and/or modify it under the
+ * terms of the GNU Lesser General Public License as published by the Free
+ * Software Foundation; either version 2.1 of the License, or (at your option)
+ * any later version.
+ *
+ * This software is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this software; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA, or see the FSF
+ * site: http://www.fsf.org.
+ */
+
+package org.jboss.messaging.tests.integration.journal;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+
+import org.jboss.messaging.core.journal.SequentialFile;
+import org.jboss.messaging.core.journal.SequentialFileFactory;
+import org.jboss.messaging.core.journal.impl.JournalCompactor;
+import org.jboss.messaging.core.journal.impl.JournalFile;
+import org.jboss.messaging.core.journal.impl.JournalFileImpl;
+import org.jboss.messaging.core.journal.impl.JournalImpl;
+import org.jboss.messaging.core.journal.impl.NIOSequentialFileFactory;
+import org.jboss.messaging.core.logging.Logger;
+import org.jboss.messaging.tests.unit.core.journal.impl.JournalImplTestBase;
+import org.jboss.messaging.tests.unit.core.journal.impl.fakes.SimpleEncoding;
+import org.jboss.messaging.utils.IDGenerator;
+import org.jboss.messaging.utils.Pair;
+import org.jboss.messaging.utils.TimeAndCounterIDGenerator;
+
+/**
+ *
+ * A JournalImplTestBase
+ *
+ * @author <a href="mailto:clebert.suconic at jboss.com">Clebert Suconic</a>
+ *
+ */
+public class NIOJournalCompactTest extends JournalImplTestBase
+{
+ private final Logger log = Logger.getLogger(this.getClass());
+
+ private static final int NUMBER_OF_RECORDS = 1000;
+
+ IDGenerator idGenerator = new TimeAndCounterIDGenerator();
+
+ // General tests
+ // =============
+
+ public void testControlFile() throws Exception
+ {
+ ArrayList<JournalFile> dataFiles = new ArrayList<JournalFile>();
+
+ for (int i = 0; i < 5; i++)
+ {
+ SequentialFile file = fileFactory.createSequentialFile("file-" + i + ".tst", 1);
+ dataFiles.add(new JournalFileImpl(file, 0));
+ }
+
+ ArrayList<JournalFile> newFiles = new ArrayList<JournalFile>();
+
+ for (int i = 0; i < 3; i++)
+ {
+ SequentialFile file = fileFactory.createSequentialFile("file-" + i + ".tst.new", 1);
+ newFiles.add(new JournalFileImpl(file, 0));
+ }
+
+ JournalCompactor.writeControlFile(fileFactory, dataFiles, newFiles);
+
+ ArrayList<String> strDataFiles = new ArrayList<String>();
+
+ ArrayList<String> strNewFiles = new ArrayList<String>();
+
+ assertNotNull(JournalCompactor.readControlFile(fileFactory, strDataFiles, strNewFiles));
+
+ assertEquals(dataFiles.size(), strDataFiles.size());
+ assertEquals(newFiles.size(), strNewFiles.size());
+
+ Iterator<String> iterDataFiles = strDataFiles.iterator();
+ for (JournalFile file : dataFiles)
+ {
+ assertEquals(file.getFile().getFileName(), iterDataFiles.next());
+ }
+
+ assertFalse(iterDataFiles.hasNext());
+
+ Iterator<String> iterNewFiles = strNewFiles.iterator();
+ for (JournalFile file : newFiles)
+ {
+ assertEquals(file.getFile().getFileName(), iterNewFiles.next());
+ }
+ }
+
+ public void testCrashRenamingFiles() throws Exception
+ {
+ internalCompactTest(false, false, true, false, false, false, false, false, false, false, true, false, false);
+ }
+
+ public void testCrashDuringCompacting() throws Exception
+ {
+ internalCompactTest(false, false, true, false, false, false, false, false, false, false, false, false, false);
+ }
+
+ public void testCompactwithPendingXACommit() throws Exception
+ {
+ internalCompactTest(true, false, false, false, false, false, false, true, false, false, true, true, true);
+ }
+
+ public void testCompactwithPendingXAPrepareAndCommit() throws Exception
+ {
+ internalCompactTest(false, true, false, false, false, false, false, true, false, false, true, true, true);
+ }
+
+ public void testCompactwithPendingXAPrepareAndDelayedCommit() throws Exception
+ {
+ internalCompactTest(false, true, false, false, false, false, false, true, false, true, true, true, true);
+ }
+
+ public void testCompactwithPendingCommit() throws Exception
+ {
+ internalCompactTest(true, false, false, false, false, false, false, true, false, false, true, true, true);
+ }
+
+ public void testCompactwithDelayedCommit() throws Exception
+ {
+ internalCompactTest(false, true, false, false, false, false, false, true, false, true, true, true, true);
+ }
+
+ public void testCompactwithPendingCommitFollowedByDelete() throws Exception
+ {
+ internalCompactTest(false, false, false, false, false, false, false, true, true, false, true, true, true);
+ }
+
+ public void testCompactwithConcurrentUpdateAndDeletes() throws Exception
+ {
+ internalCompactTest(false, false, true, false, true, true, false, false, false, false, true, true, true);
+ tearDown();
+ setUp();
+ internalCompactTest(false, false, true, false, true, false, true, false, false, false, true, true, true);
+ }
+
+ public void testCompactwithConcurrentDeletes() throws Exception
+ {
+ internalCompactTest(false, false, true, false, false, true, false, false, false, false, true, true, true);
+ tearDown();
+ setUp();
+ internalCompactTest(false, false, true, false, false, false, true, false, false, false, true, true, true);
+ }
+
+ public void testCompactwithConcurrentUpdates() throws Exception
+ {
+ internalCompactTest(false, false, true, false, true, false, false, false, false, false, true, true, true);
+ }
+
+ public void testCompactWithConcurrentAppend() throws Exception
+ {
+ internalCompactTest(false, false, true, true, false, false, false, false, false, false, true, true, true);
+ }
+
+ private void internalCompactTest(final boolean preXA, // prepare before compact
+ final boolean postXA, // prepare after compact
+ final boolean regularAdd,
+ final boolean performAppend,
+ final boolean performUpdate,
+ boolean performDelete,
+ boolean performNonTransactionalDelete,
+ final boolean pendingTransactions,
+ final boolean deleteTransactRecords,
+ final boolean delayCommit,
+ final boolean createControlFile,
+ final boolean deleteControlFile,
+ final boolean renameFilesAfterCompacting) throws Exception
+ {
+ if (performNonTransactionalDelete)
+ {
+ performDelete = false;
+ }
+ if (performDelete)
+ {
+ performNonTransactionalDelete = false;
+ }
+
+ setup(50, 60 * 1024, true);
+
+ ArrayList<Long> liveIDs = new ArrayList<Long>();
+
+ ArrayList<Pair<Long, Long>> transactedRecords = new ArrayList<Pair<Long, Long>>();
+
+ final CountDownLatch latchDone = new CountDownLatch(1);
+ final CountDownLatch latchWait = new CountDownLatch(1);
+ journal = new JournalImpl(fileSize, minFiles, 0, 0, fileFactory, filePrefix, fileExtension, maxAIO)
+ {
+
+ @Override
+ protected SequentialFile createControlFile(List<JournalFile> files, List<JournalFile> newFiles) throws Exception
+ {
+ if (createControlFile)
+ {
+ return super.createControlFile(files, newFiles);
+ }
+ else
+ {
+ throw new IllegalStateException("Simulating a crash during compact creation");
+ }
+ }
+
+ @Override
+ protected void deleteControlFile(SequentialFile controlFile) throws Exception
+ {
+ if (deleteControlFile)
+ {
+ super.deleteControlFile(controlFile);
+ }
+ }
+
+ @Override
+ protected void renameFiles(List<JournalFile> oldFiles, List<JournalFile> newFiles) throws Exception
+ {
+ if (renameFilesAfterCompacting)
+ {
+ super.renameFiles(oldFiles, newFiles);
+ }
+ }
+
+ @Override
+ public void onCompactDone()
+ {
+ latchDone.countDown();
+ System.out.println("Waiting on Compact");
+ try
+ {
+ latchWait.await();
+ }
+ catch (InterruptedException e)
+ {
+ e.printStackTrace();
+ }
+ System.out.println("Done");
+ }
+ };
+
+ journal.setAutoReclaim(false);
+
+ startJournal();
+ load();
+
+ long transactionID = 0;
+
+ if (regularAdd)
+ {
+
+ for (int i = 0; i < NUMBER_OF_RECORDS / 2; i++)
+ {
+ add(i);
+ if (i % 10 == 0 && i > 0)
+ {
+ journal.forceMoveNextFile();
+ }
+ update(i);
+ }
+
+ for (int i = NUMBER_OF_RECORDS / 2; i < NUMBER_OF_RECORDS; i++)
+ {
+
+ addTx(transactionID, i);
+ updateTx(transactionID, i);
+ if (i % 10 == 0)
+ {
+ journal.forceMoveNextFile();
+ }
+ commit(transactionID++);
+ update(i);
+ }
+ }
+
+ if (pendingTransactions)
+ {
+ for (long i = 0; i < 100; i++)
+ {
+ long recordID = idGenerator.generateID();
+ addTx(transactionID, recordID);
+ updateTx(transactionID, recordID);
+ if (preXA)
+ {
+ prepare(transactionID, new SimpleEncoding(10, (byte)0));
+ }
+ transactedRecords.add(new Pair<Long, Long>(transactionID++, recordID));
+ }
+ }
+
+ System.out.println("Number of Files: " + journal.getDataFilesCount());
+
+ if (regularAdd)
+ {
+ for (int i = 0; i < NUMBER_OF_RECORDS; i++)
+ {
+ if (!(i % 10 == 0))
+ {
+ delete(i);
+ }
+ else
+ {
+ liveIDs.add((long)i);
+ }
+ }
+ }
+
+ journal.forceMoveNextFile();
+
+ Thread t = new Thread()
+ {
+ @Override
+ public void run()
+ {
+ try
+ {
+ journal.compact();
+ }
+ catch (Exception e)
+ {
+ e.printStackTrace();
+ }
+ }
+ };
+
+ t.start();
+
+ latchDone.await();
+
+ int nextID = NUMBER_OF_RECORDS;
+
+ if (performAppend)
+ {
+ for (int i = 0; i < 50; i++)
+ {
+ add(nextID++);
+ if (i % 10 == 0)
+ {
+ journal.forceMoveNextFile();
+ }
+ }
+
+ for (int i = 0; i < 50; i++)
+ {
+ // A Total new transaction (that was created after the compact started) to add new record while compacting
+ // is still working
+ addTx(transactionID, nextID++);
+ commit(transactionID++);
+ if (i % 10 == 0)
+ {
+ journal.forceMoveNextFile();
+ }
+ }
+ }
+
+ if (performUpdate)
+ {
+ int count = 0;
+ for (Long liveID : liveIDs)
+ {
+ if (count++ % 2 == 0)
+ {
+ update(liveID);
+ }
+ else
+ {
+ // A Total new transaction (that was created after the compact started) to update a record that is being
+ // compacted
+ updateTx(transactionID, liveID);
+ commit(transactionID++);
+ }
+ }
+ }
+
+ if (performDelete)
+ {
+ int count = 0;
+ for (long liveID : liveIDs)
+ {
+ if (count++ % 2 == 0)
+ {
+ System.out.println("Deleting no trans " + liveID);
+ delete(liveID);
+ }
+ else
+ {
+ System.out.println("Deleting TX " + liveID);
+ // A Total new transaction (that was created after the compact started) to delete a record that is being
+ // compacted
+ deleteTx(transactionID, liveID);
+ commit(transactionID++);
+ }
+
+ System.out.println("Deletes are going into " + ((JournalImpl)journal).getCurrentFile());
+ }
+ }
+
+ if (performNonTransactionalDelete)
+ {
+ for (long liveID : liveIDs)
+ {
+ delete(liveID);
+ }
+ }
+
+ if (pendingTransactions && !delayCommit)
+ {
+ for (Pair<Long, Long> tx : transactedRecords)
+ {
+ if (postXA)
+ {
+ prepare(tx.a, new SimpleEncoding(10, (byte)0));
+ }
+ if (tx.a % 2 == 0)
+ {
+ commit(tx.a);
+
+ if (deleteTransactRecords)
+ {
+ delete(tx.b);
+ }
+ }
+ else
+ {
+ rollback(tx.a);
+ }
+ }
+ }
+
+ /** Some independent adds and updates */
+ for (int i = 0; i < 1000; i++)
+ {
+ long id = idGenerator.generateID();
+ add(id);
+ delete(id);
+
+ if (i % 100 == 0)
+ {
+ journal.forceMoveNextFile();
+ }
+ }
+ journal.forceMoveNextFile();
+
+ latchWait.countDown();
+
+ t.join();
+
+ if (pendingTransactions && delayCommit)
+ {
+ for (Pair<Long, Long> tx : transactedRecords)
+ {
+ if (postXA)
+ {
+ prepare(tx.a, new SimpleEncoding(10, (byte)0));
+ }
+ if (tx.a % 2 == 0)
+ {
+ commit(tx.a);
+
+ if (deleteTransactRecords)
+ {
+ delete(tx.b);
+ }
+ }
+ else
+ {
+ rollback(tx.a);
+ }
+ }
+ }
+
+ add(idGenerator.generateID());
+
+ if (createControlFile && deleteControlFile && renameFilesAfterCompacting)
+ {
+ journal.compact();
+ }
+
+ stopJournal();
+ createJournal();
+ startJournal();
+ loadAndCheck();
+
+ }
+
+ public void testSimpleCompacting() throws Exception
+ {
+ setup(2, 60 * 1024, true);
+
+ createJournal();
+ startJournal();
+ load();
+
+ int NUMBER_OF_RECORDS = 1000;
+
+ // add and remove some data to force reclaiming
+ {
+ ArrayList<Long> ids = new ArrayList<Long>();
+ for (int i = 0; i < NUMBER_OF_RECORDS; i++)
+ {
+ long id = idGenerator.generateID();
+ ids.add(id);
+ add(id);
+ if (i > 0 && i % 100 == 0)
+ {
+ journal.forceMoveNextFile();
+ }
+ }
+
+ for (Long id : ids)
+ {
+ delete(id);
+ }
+
+ journal.forceMoveNextFile();
+
+ journal.checkAndReclaimFiles();
+ }
+
+ long transactionID = 0;
+
+ for (int i = 0; i < NUMBER_OF_RECORDS / 2; i++)
+ {
+ add(i);
+ if (i % 10 == 0 && i > 0)
+ {
+ journal.forceMoveNextFile();
+ }
+ update(i);
+ }
+
+ for (int i = NUMBER_OF_RECORDS / 2; i < NUMBER_OF_RECORDS; i++)
+ {
+
+ addTx(transactionID, i);
+ updateTx(transactionID, i);
+ if (i % 10 == 0)
+ {
+ journal.forceMoveNextFile();
+ }
+ commit(transactionID++);
+ update(i);
+ }
+
+ System.out.println("Number of Files: " + journal.getDataFilesCount());
+
+ for (int i = 0; i < NUMBER_OF_RECORDS; i++)
+ {
+ if (!(i % 10 == 0))
+ {
+ delete(i);
+ }
+ }
+
+ journal.forceMoveNextFile();
+
+ System.out.println("Number of Files: " + journal.getDataFilesCount());
+
+ System.out.println("Before compact ****************************");
+ System.out.println(journal.debug());
+ System.out.println("*****************************************");
+
+ journal.compact();
+
+ add(idGenerator.generateID());
+
+ journal.compact();
+
+ stopJournal();
+ createJournal();
+ startJournal();
+ loadAndCheck();
+
+ }
+
+ public void testLiveSize() throws Exception
+ {
+ setup(2, 60 * 1024, true);
+
+ createJournal();
+ startJournal();
+ loadAndCheck();
+
+ ArrayList<Long> listToDelete = new ArrayList<Long>();
+
+ ArrayList<Integer> expectedSizes = new ArrayList<Integer>();
+
+ for (int i = 0; i < 10; i++)
+ {
+ long id = idGenerator.generateID();
+ listToDelete.add(id);
+
+ expectedSizes.add(recordLength + JournalImpl.SIZE_ADD_RECORD);
+
+ add(id);
+ journal.forceMoveNextFile();
+ update(id);
+
+ expectedSizes.add(recordLength + JournalImpl.SIZE_UPDATE_RECORD);
+ journal.forceMoveNextFile();
+ }
+
+ System.out.println("DataFiles = " + journal.getDataFilesCount());
+
+ JournalFile files[] = journal.getDataFiles();
+
+ for (JournalFile file : files)
+ {
+ System.out.println("Size: " + file.getLiveSize());
+ }
+
+ stopJournal();
+ createJournal();
+ startJournal();
+ loadAndCheck();
+
+ journal.forceMoveNextFile();
+
+ System.out.println("DataFiles = " + journal.getDataFilesCount());
+
+ JournalFile files2[] = journal.getDataFiles();
+
+ assertEquals(files.length, files2.length);
+
+ for (JournalFile file : files2)
+ {
+ System.out.println("Size: " + file.getLiveSize());
+ }
+
+ for (int i = 0; i < files.length; i++)
+ {
+ assertEquals(expectedSizes.get(i).intValue(), files[i].getLiveSize());
+ assertEquals(expectedSizes.get(i).intValue(), files2[i].getLiveSize());
+ }
+
+ for (long id : listToDelete)
+ {
+ delete(id);
+ }
+
+ journal.forceMoveNextFile();
+
+ JournalFile files3[] = journal.getDataFiles();
+
+ for (JournalFile file : files3)
+ {
+ System.out.println("Size: " + file.getLiveSize());
+ }
+
+ for (JournalFile file : files3)
+ {
+ assertEquals(0, file.getLiveSize());
+ }
+
+ stopJournal();
+ createJournal();
+ startJournal();
+ loadAndCheck();
+
+ files3 = journal.getDataFiles();
+
+ for (JournalFile file : files3)
+ {
+ assertEquals(0, file.getLiveSize());
+ }
+
+ }
+
+ public void testLiveSizeTransactional() throws Exception
+ {
+ setup(2, 60 * 1024, true);
+
+ createJournal();
+ startJournal();
+ loadAndCheck();
+
+ ArrayList<Long> listToDelete = new ArrayList<Long>();
+
+ ArrayList<Integer> expectedSizes = new ArrayList<Integer>();
+
+ for (int i = 0; i < 10; i++)
+ {
+ long tx = idGenerator.generateID();
+ long id = idGenerator.generateID();
+ listToDelete.add(id);
+
+ addTx(tx, id);
+
+ // Append Record Transaction will make the recordSize as exactly recordLength (discounting SIZE_ADD_RECORD_TX)
+ expectedSizes.add(recordLength);
+ journal.forceMoveNextFile();
+
+ updateTx(tx, id);
+ // uPDATE Record Transaction will make the recordSize as exactly recordLength (discounting SIZE_ADD_RECORD_TX)
+ expectedSizes.add(recordLength);
+
+ journal.forceMoveNextFile();
+ expectedSizes.add(0);
+
+ commit(tx);
+
+ journal.forceMoveNextFile();
+ }
+
+ System.out.println("DataFiles = " + journal.getDataFilesCount());
+
+ JournalFile files[] = journal.getDataFiles();
+
+ for (JournalFile file : files)
+ {
+ System.out.println("Size: " + file.getLiveSize());
+ }
+
+ stopJournal();
+ createJournal();
+ startJournal();
+ loadAndCheck();
+
+ journal.forceMoveNextFile();
+
+ System.out.println("DataFiles = " + journal.getDataFilesCount());
+
+ JournalFile files2[] = journal.getDataFiles();
+
+ assertEquals(files.length, files2.length);
+
+ for (JournalFile file : files2)
+ {
+ System.out.println("Size: " + file.getLiveSize());
+ }
+
+ for (int i = 0; i < files.length; i++)
+ {
+ assertEquals(expectedSizes.get(i).intValue(), files[i].getLiveSize());
+ assertEquals(expectedSizes.get(i).intValue(), files2[i].getLiveSize());
+ }
+
+ long tx = idGenerator.generateID();
+ for (long id : listToDelete)
+ {
+ deleteTx(tx, id);
+ }
+ commit(tx);
+
+ journal.forceMoveNextFile();
+
+ JournalFile files3[] = journal.getDataFiles();
+
+ for (JournalFile file : files3)
+ {
+ System.out.println("Size: " + file.getLiveSize());
+ }
+
+ for (JournalFile file : files3)
+ {
+ assertEquals(0, file.getLiveSize());
+ }
+
+ stopJournal();
+ createJournal();
+ startJournal();
+ loadAndCheck();
+
+ files3 = journal.getDataFiles();
+
+ for (JournalFile file : files3)
+ {
+ assertEquals(0, file.getLiveSize());
+ }
+
+ }
+
+ @Override
+ protected void setUp() throws Exception
+ {
+ super.setUp();
+
+ File file = new File(getTestDir());
+
+ deleteDirectory(file);
+
+ file.mkdir();
+ }
+
+ /* (non-Javadoc)
+ * @see org.jboss.messaging.tests.unit.core.journal.impl.JournalImplTestBase#getFileFactory()
+ */
+ @Override
+ protected SequentialFileFactory getFileFactory() throws Exception
+ {
+ return new NIOSequentialFileFactory(getTestDir());
+ }
+
+}
Modified: trunk/tests/src/org/jboss/messaging/tests/integration/journal/NIOSequentialFileFactoryTest.java
===================================================================
--- trunk/tests/src/org/jboss/messaging/tests/integration/journal/NIOSequentialFileFactoryTest.java 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/tests/src/org/jboss/messaging/tests/integration/journal/NIOSequentialFileFactoryTest.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -37,14 +37,13 @@
*/
public class NIOSequentialFileFactoryTest extends SequentialFileFactoryTestBase
{
- protected String journalDir = System.getProperty("user.home") + "/journal-test";
@Override
protected void setUp() throws Exception
{
super.setUp();
- File file = new File(journalDir);
+ File file = new File(getTestDir());
deleteDirectory(file);
@@ -54,7 +53,7 @@
@Override
protected SequentialFileFactory createFactory()
{
- return new NIOSequentialFileFactory(journalDir);
+ return new NIOSequentialFileFactory(getTestDir());
}
}
Modified: trunk/tests/src/org/jboss/messaging/tests/integration/journal/RealAIOJournalImplTest.java
===================================================================
--- trunk/tests/src/org/jboss/messaging/tests/integration/journal/RealAIOJournalImplTest.java 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/tests/src/org/jboss/messaging/tests/integration/journal/RealAIOJournalImplTest.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -70,7 +70,12 @@
file.mkdir();
- return new AIOSequentialFileFactory(getTestDir());
+ return new AIOSequentialFileFactory(getTestDir(),
+ ConfigurationImpl.DEFAULT_JOURNAL_AIO_BUFFER_SIZE,
+ 1000000,
+ true,
+ false
+ );
}
Modified: trunk/tests/src/org/jboss/messaging/tests/performance/journal/JournalImplTestUnit.java
===================================================================
--- trunk/tests/src/org/jboss/messaging/tests/performance/journal/JournalImplTestUnit.java 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/tests/src/org/jboss/messaging/tests/performance/journal/JournalImplTestUnit.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -200,7 +200,7 @@
public void testSpeedTransactional() throws Exception
{
- Journal journal = new JournalImpl(10 * 1024 * 1024, 10, getFileFactory(), "jbm-data", "jbm", 5000);
+ Journal journal = new JournalImpl(10 * 1024 * 1024, 10, 0, 0, getFileFactory(), "jbm-data", "jbm", 5000);
journal.start();
@@ -221,7 +221,7 @@
long startTrans = System.currentTimeMillis();
for (int j = 0; j < 1000; j++)
{
- journal.appendAddRecordTransactional(i, count++, (byte)0, data, true);
+ journal.appendAddRecordTransactional(i, count++, (byte)0, data);
}
journal.appendCommitRecord(i, true);
@@ -261,7 +261,7 @@
log.debug("num Files=" + numFiles);
- Journal journal = new JournalImpl(10 * 1024 * 1024, numFiles, getFileFactory(), "jbm-data", "jbm", 5000);
+ Journal journal = new JournalImpl(10 * 1024 * 1024, numFiles, 0, 0, getFileFactory(), "jbm-data", "jbm", 5000);
journal.start();
@@ -285,7 +285,7 @@
journal.stop();
- journal = new JournalImpl(10 * 1024 * 1024, numFiles, getFileFactory(), "jbm-data", "jbm", 5000);
+ journal = new JournalImpl(10 * 1024 * 1024, numFiles, 0, 0, getFileFactory(), "jbm-data", "jbm", 5000);
journal.start();
journal.load(new ArrayList<RecordInfo>(), null);
Modified: trunk/tests/src/org/jboss/messaging/tests/performance/journal/RealJournalImplAIOTest.java
===================================================================
--- trunk/tests/src/org/jboss/messaging/tests/performance/journal/RealJournalImplAIOTest.java 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/tests/src/org/jboss/messaging/tests/performance/journal/RealJournalImplAIOTest.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -23,7 +23,6 @@
package org.jboss.messaging.tests.performance.journal;
import java.io.File;
-import java.util.concurrent.Executors;
import org.jboss.messaging.core.asyncio.impl.AsynchronousFileImpl;
import org.jboss.messaging.core.journal.SequentialFileFactory;
@@ -41,8 +40,6 @@
public class RealJournalImplAIOTest extends JournalImplTestUnit
{
private static final Logger log = Logger.getLogger(RealJournalImplAIOTest.class);
-
- protected String journalDir = System.getProperty("user.home") + "/journal-test";
@Override
protected void setUp() throws Exception
@@ -60,15 +57,15 @@
protected SequentialFileFactory getFileFactory() throws Exception
{
- File file = new File(journalDir);
+ File file = new File(getTestDir());
- log.debug("deleting directory " + journalDir);
+ log.debug("deleting directory " + file);
deleteDirectory(file);
file.mkdir();
- return new AIOSequentialFileFactory(journalDir);
+ return new AIOSequentialFileFactory(getTestDir());
}
}
Modified: trunk/tests/src/org/jboss/messaging/tests/performance/journal/RealJournalImplNIOTest.java
===================================================================
--- trunk/tests/src/org/jboss/messaging/tests/performance/journal/RealJournalImplNIOTest.java 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/tests/src/org/jboss/messaging/tests/performance/journal/RealJournalImplNIOTest.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -39,20 +39,18 @@
public class RealJournalImplNIOTest extends JournalImplTestUnit
{
private static final Logger log = Logger.getLogger(RealJournalImplNIOTest.class);
-
- protected String journalDir = System.getProperty("user.home") + "/journal-test";
protected SequentialFileFactory getFileFactory() throws Exception
{
- File file = new File(journalDir);
+ File file = new File(getTestDir());
- log.debug("deleting directory " + journalDir);
+ log.debug("deleting directory " + getTestDir());
deleteDirectory(file);
file.mkdir();
- return new NIOSequentialFileFactory(journalDir);
+ return new NIOSequentialFileFactory(getTestDir());
}
Modified: trunk/tests/src/org/jboss/messaging/tests/stress/journal/AddAndRemoveStressTest.java
===================================================================
--- trunk/tests/src/org/jboss/messaging/tests/stress/journal/AddAndRemoveStressTest.java 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/tests/src/org/jboss/messaging/tests/stress/journal/AddAndRemoveStressTest.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -25,7 +25,7 @@
import java.io.File;
import java.util.ArrayList;
-import org.jboss.messaging.core.journal.LoadManager;
+import org.jboss.messaging.core.journal.LoaderCallback;
import org.jboss.messaging.core.journal.PreparedTransactionInfo;
import org.jboss.messaging.core.journal.RecordInfo;
import org.jboss.messaging.core.journal.SequentialFileFactory;
@@ -44,7 +44,7 @@
// Constants -----------------------------------------------------
- private static final LoadManager dummyLoader = new LoadManager()
+ private static final LoaderCallback dummyLoader = new LoaderCallback()
{
public void addPreparedTransaction(final PreparedTransactionInfo preparedTransaction)
@@ -65,7 +65,7 @@
};
private static final long NUMBER_OF_MESSAGES = 210000l;
-
+
private static final int NUMBER_OF_FILES_ON_JOURNAL = 6;
// Attributes ----------------------------------------------------
@@ -76,12 +76,18 @@
// Public --------------------------------------------------------
-
public void testInsertAndLoad() throws Exception
{
SequentialFileFactory factory = new AIOSequentialFileFactory(getTestDir());
- JournalImpl impl = new JournalImpl(10 * 1024 * 1024, NUMBER_OF_FILES_ON_JOURNAL, factory, "jbm", "jbm", 1000);
+ JournalImpl impl = new JournalImpl(10 * 1024 * 1024,
+ NUMBER_OF_FILES_ON_JOURNAL,
+ 0,
+ 0,
+ factory,
+ "jbm",
+ "jbm",
+ 1000);
impl.start();
@@ -99,7 +105,7 @@
impl.stop();
factory = new AIOSequentialFileFactory(getTestDir());
- impl = new JournalImpl(10 * 1024 * 1024, NUMBER_OF_FILES_ON_JOURNAL, factory, "jbm", "jbm", 1000);
+ impl = new JournalImpl(10 * 1024 * 1024, NUMBER_OF_FILES_ON_JOURNAL, 0, 0, factory, "jbm", "jbm", 1000);
impl.start();
@@ -118,7 +124,7 @@
impl.stop();
factory = new AIOSequentialFileFactory(getTestDir());
- impl = new JournalImpl(10 * 1024 * 1024, NUMBER_OF_FILES_ON_JOURNAL, factory, "jbm", "jbm", 1000);
+ impl = new JournalImpl(10 * 1024 * 1024, NUMBER_OF_FILES_ON_JOURNAL, 0, 0, factory, "jbm", "jbm", 1000);
impl.start();
@@ -126,7 +132,7 @@
ArrayList<PreparedTransactionInfo> trans = new ArrayList<PreparedTransactionInfo>();
impl.load(info, trans);
-
+
impl.forceMoveNextFile();
if (info.size() > 0)
@@ -136,7 +142,7 @@
assertEquals(0, info.size());
assertEquals(0, trans.size());
-
+
assertEquals(0, impl.getDataFilesCount());
}
@@ -145,7 +151,14 @@
{
SequentialFileFactory factory = new AIOSequentialFileFactory(getTestDir());
- JournalImpl impl = new JournalImpl(10 * 1024 * 1024, NUMBER_OF_FILES_ON_JOURNAL, factory, "jbm", "jbm", 1000);
+ JournalImpl impl = new JournalImpl(10 * 1024 * 1024,
+ NUMBER_OF_FILES_ON_JOURNAL,
+ 0,
+ 0,
+ factory,
+ "jbm",
+ "jbm",
+ 1000);
impl.start();
@@ -164,7 +177,7 @@
impl.stop();
factory = new AIOSequentialFileFactory(getTestDir());
- impl = new JournalImpl(10 * 1024 * 1024, 10, factory, "jbm", "jbm", 1000);
+ impl = new JournalImpl(10 * 1024 * 1024, 10, 0, 0, factory, "jbm", "jbm", 1000);
impl.start();
@@ -183,7 +196,7 @@
impl.stop();
factory = new AIOSequentialFileFactory(getTestDir());
- impl = new JournalImpl(10 * 1024 * 1024, NUMBER_OF_FILES_ON_JOURNAL, factory, "jbm", "jbm", 1000);
+ impl = new JournalImpl(10 * 1024 * 1024, NUMBER_OF_FILES_ON_JOURNAL, 0, 0, factory, "jbm", "jbm", 1000);
impl.start();
@@ -196,7 +209,7 @@
{
System.out.println("Info ID: " + info.get(0).id);
}
-
+
impl.forceMoveNextFile();
impl.checkAndReclaimFiles();
@@ -216,7 +229,7 @@
protected void setUp() throws Exception
{
super.setUp();
-
+
File file = new File(getTestDir());
deleteDirectory(file);
file.mkdirs();
Modified: trunk/tests/src/org/jboss/messaging/tests/stress/journal/ValidateTransactionHealthTest.java
===================================================================
--- trunk/tests/src/org/jboss/messaging/tests/stress/journal/ValidateTransactionHealthTest.java 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/tests/src/org/jboss/messaging/tests/stress/journal/ValidateTransactionHealthTest.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -26,7 +26,7 @@
import java.nio.ByteBuffer;
import org.jboss.messaging.core.asyncio.impl.AsynchronousFileImpl;
-import org.jboss.messaging.core.journal.LoadManager;
+import org.jboss.messaging.core.journal.LoaderCallback;
import org.jboss.messaging.core.journal.PreparedTransactionInfo;
import org.jboss.messaging.core.journal.RecordInfo;
import org.jboss.messaging.core.journal.impl.JournalImpl;
@@ -181,7 +181,7 @@
// Inner classes -------------------------------------------------
- class Loader implements LoadManager
+ class Loader implements LoaderCallback
{
int numberOfPreparedTransactions = 0;
Modified: trunk/tests/src/org/jboss/messaging/tests/stress/journal/remote/RemoteJournalAppender.java
===================================================================
--- trunk/tests/src/org/jboss/messaging/tests/stress/journal/remote/RemoteJournalAppender.java 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/tests/src/org/jboss/messaging/tests/stress/journal/remote/RemoteJournalAppender.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -26,7 +26,7 @@
import java.util.concurrent.atomic.AtomicLong;
import org.jboss.messaging.core.config.impl.ConfigurationImpl;
-import org.jboss.messaging.core.journal.LoadManager;
+import org.jboss.messaging.core.journal.LoaderCallback;
import org.jboss.messaging.core.journal.PreparedTransactionInfo;
import org.jboss.messaging.core.journal.RecordInfo;
import org.jboss.messaging.core.journal.SequentialFileFactory;
@@ -41,23 +41,22 @@
*/
public class RemoteJournalAppender
{
-
+
// Constants -----------------------------------------------------
-
+
public static final int OK = 10;
+
// Attributes ----------------------------------------------------
-
+
// Static --------------------------------------------------------
-
+
public static void main(String args[]) throws Exception
{
-
+
if (args.length != 5)
{
- System.err
- .println("Use: java -cp <classpath> "
- + RemoteJournalAppender.class.getCanonicalName()
- + " aio|nio <journalDirectory> <NumberOfElements> <TransactionSize> <NumberOfThreads>");
+ System.err.println("Use: java -cp <classpath> " + RemoteJournalAppender.class.getCanonicalName() +
+ " aio|nio <journalDirectory> <NumberOfElements> <TransactionSize> <NumberOfThreads>");
System.exit(-1);
}
System.out.println("Running");
@@ -66,54 +65,53 @@
long numberOfElements = Long.parseLong(args[2]);
int transactionSize = Integer.parseInt(args[3]);
int numberOfThreads = Integer.parseInt(args[4]);
-
try
{
- appendData(journalType, journalDir,
- numberOfElements, transactionSize, numberOfThreads);
-
+ appendData(journalType, journalDir, numberOfElements, transactionSize, numberOfThreads);
+
}
catch (Exception e)
{
e.printStackTrace(System.out);
System.exit(-1);
}
-
+
System.exit(OK);
}
- public static JournalImpl appendData(String journalType, String journalDir,
- long numberOfElements, int transactionSize, int numberOfThreads) throws Exception
+ public static JournalImpl appendData(String journalType,
+ String journalDir,
+ long numberOfElements,
+ int transactionSize,
+ int numberOfThreads) throws Exception
{
final JournalImpl journal = createJournal(journalType, journalDir);
-
+
journal.start();
- journal.load(new LoadManager()
+ journal.load(new LoaderCallback()
{
-
- public void addPreparedTransaction(
- PreparedTransactionInfo preparedTransaction)
+
+ public void addPreparedTransaction(PreparedTransactionInfo preparedTransaction)
{
}
-
+
public void addRecord(RecordInfo info)
{
}
-
+
public void deleteRecord(long id)
{
}
-
+
public void updateRecord(RecordInfo info)
{
}
});
-
-
+
LocalThreads threads[] = new LocalThreads[numberOfThreads];
final AtomicLong sequenceTransaction = new AtomicLong();
-
+
for (int i = 0; i < numberOfThreads; i++)
{
threads[i] = new LocalThreads(journal, numberOfElements, transactionSize, sequenceTransaction);
@@ -121,33 +119,38 @@
}
Exception e = null;
- for (LocalThreads t: threads)
+ for (LocalThreads t : threads)
{
t.join();
-
+
if (t.e != null)
{
e = t.e;
}
}
-
+
if (e != null)
{
throw e;
}
-
-
+
return journal;
}
public static JournalImpl createJournal(String journalType, String journalDir)
{
- JournalImpl journal = new JournalImpl(10485760, 2, getFactory(journalType, journalDir), "journaltst", "tst", 500);
+ JournalImpl journal = new JournalImpl(10485760,
+ 2,
+ 0,
+ 0,
+ getFactory(journalType, journalDir),
+ "journaltst",
+ "tst",
+ 500);
return journal;
}
-
- public static SequentialFileFactory getFactory(String factoryType,
- String directory)
+
+ public static SequentialFileFactory getFactory(String factoryType, String directory)
{
if (factoryType.equals("aio"))
{
@@ -162,29 +165,31 @@
return new NIOSequentialFileFactory(directory);
}
}
+
// Constructors --------------------------------------------------
-
+
// Public --------------------------------------------------------
-
+
// Package protected ---------------------------------------------
-
+
// Protected -----------------------------------------------------
-
-
+
// Private -------------------------------------------------------
-
+
// Inner classes -------------------------------------------------
-
-
+
static class LocalThreads extends Thread
{
final JournalImpl journal;
+
final long numberOfElements;
+
final int transactionSize;
+
final AtomicLong nextID;
-
+
Exception e;
-
+
public LocalThreads(JournalImpl journal, long numberOfElements, int transactionSize, AtomicLong nextID)
{
super();
@@ -194,29 +199,26 @@
this.nextID = nextID;
}
-
-
-
public void run()
{
try
{
int transactionCounter = 0;
-
+
long transactionId = nextID.incrementAndGet();
-
+
for (long i = 0; i < numberOfElements; i++)
{
-
+
long id = nextID.incrementAndGet();
-
- ByteBuffer buffer = ByteBuffer.allocate(512*3);
+
+ ByteBuffer buffer = ByteBuffer.allocate(512 * 3);
buffer.putLong(id);
-
+
if (transactionSize != 0)
{
- journal.appendAddRecordTransactional(transactionId, id, (byte)99, buffer.array(), false);
-
+ journal.appendAddRecordTransactional(transactionId, id, (byte)99, buffer.array());
+
if (++transactionCounter == transactionSize)
{
System.out.println("Commit transaction " + transactionId);
@@ -230,12 +232,12 @@
journal.appendAddRecord(id, (byte)99, buffer.array(), false);
}
}
-
+
if (transactionCounter != 0)
{
journal.appendCommitRecord(transactionId, true);
}
-
+
if (transactionSize == 0)
{
journal.debugWait();
@@ -245,8 +247,8 @@
{
this.e = e;
}
-
+
}
}
-
+
}
Modified: trunk/tests/src/org/jboss/messaging/tests/unit/core/config/impl/DefaultsFileConfigurationTest.java
===================================================================
--- trunk/tests/src/org/jboss/messaging/tests/unit/core/config/impl/DefaultsFileConfigurationTest.java 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/tests/src/org/jboss/messaging/tests/unit/core/config/impl/DefaultsFileConfigurationTest.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -106,6 +106,10 @@
assertEquals(ConfigurationImpl.DEFAULT_JOURNAL_FILE_SIZE, conf.getJournalFileSize());
+ assertEquals(ConfigurationImpl.DEFAULT_JOURNAL_COMPACT_MIN_FILES, conf.getJournalCompactMinFiles());
+
+ assertEquals(ConfigurationImpl.DEFAULT_JOURNAL_COMPACT_PERCENTAGE, conf.getJournalCompactPercentage());
+
assertEquals(ConfigurationImpl.DEFAULT_JOURNAL_MIN_FILES, conf.getJournalMinFiles());
assertEquals(ConfigurationImpl.DEFAULT_JOURNAL_MAX_AIO, conf.getJournalMaxAIO());
Modified: trunk/tests/src/org/jboss/messaging/tests/unit/core/config/impl/FileConfigurationTest.java
===================================================================
--- trunk/tests/src/org/jboss/messaging/tests/unit/core/config/impl/FileConfigurationTest.java 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/tests/src/org/jboss/messaging/tests/unit/core/config/impl/FileConfigurationTest.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -86,6 +86,8 @@
assertEquals(true, conf.isJournalSyncNonTransactional());
assertEquals(12345678, conf.getJournalFileSize());
assertEquals(100, conf.getJournalMinFiles());
+ assertEquals(123, conf.getJournalCompactMinFiles());
+ assertEquals(33, conf.getJournalCompactPercentage());
assertEquals(56546, conf.getJournalMaxAIO());
assertEquals("largemessagesdir", conf.getLargeMessagesDirectory());
Modified: trunk/tests/src/org/jboss/messaging/tests/unit/core/journal/impl/AlignedJournalImplTest.java
===================================================================
--- trunk/tests/src/org/jboss/messaging/tests/unit/core/journal/impl/AlignedJournalImplTest.java 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/tests/src/org/jboss/messaging/tests/unit/core/journal/impl/AlignedJournalImplTest.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -31,7 +31,7 @@
import java.util.concurrent.atomic.AtomicInteger;
import org.jboss.messaging.core.journal.EncodingSupport;
-import org.jboss.messaging.core.journal.LoadManager;
+import org.jboss.messaging.core.journal.LoaderCallback;
import org.jboss.messaging.core.journal.PreparedTransactionInfo;
import org.jboss.messaging.core.journal.RecordInfo;
import org.jboss.messaging.core.journal.SequentialFile;
@@ -52,7 +52,7 @@
// Constants -----------------------------------------------------
- private static final LoadManager dummyLoader = new LoadManager()
+ private static final LoaderCallback dummyLoader = new LoaderCallback()
{
public void addPreparedTransaction(PreparedTransactionInfo preparedTransaction)
@@ -146,7 +146,7 @@
try
{
- journalImpl = new JournalImpl(2000, 2, factory, "tt", "tt", 1000);
+ journalImpl = new JournalImpl(2000, 2, 0, 0, factory, "tt", "tt", 1000);
fail("Supposed to throw an exception");
}
catch (Exception ignored)
@@ -159,7 +159,7 @@
{
final int JOURNAL_SIZE = 1060;
- setupJournal(JOURNAL_SIZE, 10);
+ setupAndLoadJournal(JOURNAL_SIZE, 10);
journalImpl.appendAddRecord(13, (byte)14, new SimpleEncoding(1, (byte)15), false);
@@ -167,7 +167,7 @@
journalImpl.checkAndReclaimFiles();
- setupJournal(JOURNAL_SIZE, 10);
+ setupAndLoadJournal(JOURNAL_SIZE, 10);
assertEquals(1, records.size());
@@ -186,7 +186,7 @@
final int JOURNAL_SIZE = 1060;
- setupJournal(JOURNAL_SIZE, 10);
+ setupAndLoadJournal(JOURNAL_SIZE, 10);
assertEquals(0, records.size());
assertEquals(0, transactions.size());
@@ -207,7 +207,7 @@
journalImpl.appendAddRecord(i * 100l, (byte)i, support, false);
}
- setupJournal(JOURNAL_SIZE, 1024);
+ setupAndLoadJournal(JOURNAL_SIZE, 1024);
assertEquals(50, records.size());
@@ -236,7 +236,7 @@
journalImpl.appendUpdateRecord(i * 100l, (byte)i, bytes, false);
}
- setupJournal(JOURNAL_SIZE, 1024);
+ setupAndLoadJournal(JOURNAL_SIZE, 1024);
i = 0;
for (RecordInfo recordItem : records)
@@ -275,7 +275,7 @@
{
final int JOURNAL_SIZE = 10000;
- setupJournal(JOURNAL_SIZE, 100);
+ setupAndLoadJournal(JOURNAL_SIZE, 100);
journalImpl.setAutoReclaim(false);
@@ -309,7 +309,7 @@
journalImpl.debugWait();
- setupJournal(JOURNAL_SIZE, 100);
+ setupAndLoadJournal(JOURNAL_SIZE, 100);
assertEquals(10, records.size());
@@ -321,7 +321,7 @@
{
final int JOURNAL_SIZE = 10000;
- setupJournal(JOURNAL_SIZE, 1);
+ setupAndLoadJournal(JOURNAL_SIZE, 1);
journalImpl.setAutoReclaim(false);
@@ -359,7 +359,7 @@
assertEquals(3, factory.listFiles("tt").size());
- setupJournal(JOURNAL_SIZE, 1);
+ setupAndLoadJournal(JOURNAL_SIZE, 1);
assertEquals(1, records.size());
@@ -385,14 +385,14 @@
{
final int JOURNAL_SIZE = 2000;
- setupJournal(JOURNAL_SIZE, 100);
+ setupAndLoadJournal(JOURNAL_SIZE, 100);
assertEquals(0, records.size());
assertEquals(0, transactions.size());
- journalImpl.appendAddRecordTransactional(1, 1, (byte)1, new SimpleEncoding(1, (byte)1), false);
+ journalImpl.appendAddRecordTransactional(1, 1, (byte)1, new SimpleEncoding(1, (byte)1));
- setupJournal(JOURNAL_SIZE, 100);
+ setupAndLoadJournal(JOURNAL_SIZE, 100);
assertEquals(0, records.size());
assertEquals(0, transactions.size());
@@ -409,7 +409,7 @@
log.warn(e);
}
- setupJournal(JOURNAL_SIZE, 100);
+ setupAndLoadJournal(JOURNAL_SIZE, 100);
assertEquals(0, records.size());
assertEquals(0, transactions.size());
@@ -420,7 +420,7 @@
{
final int JOURNAL_SIZE = 1100;
- setupJournal(JOURNAL_SIZE, 100);
+ setupAndLoadJournal(JOURNAL_SIZE, 100);
journalImpl.setAutoReclaim(false);
@@ -429,7 +429,7 @@
for (int i = 0; i < 10; i++)
{
- journalImpl.appendAddRecordTransactional(77l, 1, (byte)1, new SimpleEncoding(1, (byte)1), false);
+ journalImpl.appendAddRecordTransactional(77l, 1, (byte)1, new SimpleEncoding(1, (byte)1));
journalImpl.forceMoveNextFile();
}
@@ -437,11 +437,11 @@
assertEquals(12, factory.listFiles("tt").size());
- journalImpl.appendAddRecordTransactional(78l, 1, (byte)1, new SimpleEncoding(1, (byte)1), false);
+ journalImpl.appendAddRecordTransactional(78l, 1, (byte)1, new SimpleEncoding(1, (byte)1));
assertEquals(12, factory.listFiles("tt").size());
- setupJournal(JOURNAL_SIZE, 100);
+ setupAndLoadJournal(JOURNAL_SIZE, 100);
assertEquals(0, records.size());
assertEquals(0, transactions.size());
@@ -458,7 +458,7 @@
log.debug("Expected exception " + e, e);
}
- setupJournal(JOURNAL_SIZE, 100);
+ setupAndLoadJournal(JOURNAL_SIZE, 100);
assertEquals(0, records.size());
assertEquals(0, transactions.size());
@@ -471,14 +471,14 @@
{
final int JOURNAL_SIZE = 2000;
- setupJournal(JOURNAL_SIZE, 100);
+ setupAndLoadJournal(JOURNAL_SIZE, 100);
assertEquals(0, records.size());
assertEquals(0, transactions.size());
for (int i = 0; i < 10; i++)
{
- journalImpl.appendAddRecordTransactional(1, i, (byte)1, new SimpleEncoding(1, (byte)1), false);
+ journalImpl.appendAddRecordTransactional(1, i, (byte)1, new SimpleEncoding(1, (byte)1));
journalImpl.forceMoveNextFile();
}
@@ -488,7 +488,7 @@
assertEquals(12, factory.listFiles("tt").size());
- setupJournal(JOURNAL_SIZE, 100);
+ setupAndLoadJournal(JOURNAL_SIZE, 100);
assertEquals(10, records.size());
assertEquals(0, transactions.size());
@@ -501,7 +501,7 @@
for (int i = 0; i < 10; i++)
{
- journalImpl.appendDeleteRecordTransactional(2l, i, false);
+ journalImpl.appendDeleteRecordTransactional(2l, i);
journalImpl.forceMoveNextFile();
}
@@ -517,7 +517,7 @@
assertEquals(1, journalImpl.getDataFilesCount());
- setupJournal(JOURNAL_SIZE, 100);
+ setupAndLoadJournal(JOURNAL_SIZE, 100);
assertEquals(1, journalImpl.getDataFilesCount());
@@ -528,7 +528,7 @@
{
final int JOURNAL_SIZE = 2000;
- setupJournal(JOURNAL_SIZE, 100);
+ setupAndLoadJournal(JOURNAL_SIZE, 100);
assertEquals(0, records.size());
assertEquals(0, transactions.size());
@@ -536,13 +536,13 @@
journalImpl.appendAddRecordTransactional(1l,
2l,
(byte)3,
- new SimpleEncoding(1900 - JournalImpl.SIZE_ADD_RECORD_TX, (byte)4), false);
+ new SimpleEncoding(1900 - JournalImpl.SIZE_ADD_RECORD_TX, (byte)4));
journalImpl.appendCommitRecord(1l, false);
journalImpl.debugWait();
- setupJournal(JOURNAL_SIZE, 100);
+ setupAndLoadJournal(JOURNAL_SIZE, 100);
assertEquals(1, records.size());
@@ -552,23 +552,22 @@
{
final int JOURNAL_SIZE = 2000;
- setupJournal(JOURNAL_SIZE, 100);
+ setupAndLoadJournal(JOURNAL_SIZE, 100);
assertEquals(2, factory.listFiles("tt").size());
assertEquals(0, records.size());
assertEquals(0, transactions.size());
- for (int i = 0; i < 20; i++)
+ for (int i = 0; i < 2; i++)
{
- journalImpl.appendAddRecordTransactional(1l, i, (byte)0, new SimpleEncoding(1, (byte)15), false);
- journalImpl.forceMoveNextFile();
+ journalImpl.appendAddRecordTransactional(1l, i, (byte)0, new SimpleEncoding(1, (byte)15));
}
- journalImpl.forceMoveNextFile();
-
journalImpl.appendCommitRecord(1l, false);
+ System.out.println("Files = " + factory.listFiles("tt"));
+
SequentialFile file = factory.createSequentialFile("tt-1.tt", 1);
file.open();
@@ -601,7 +600,7 @@
file.close();
- setupJournal(JOURNAL_SIZE, 100);
+ setupAndLoadJournal(JOURNAL_SIZE, 100);
assertEquals(0, records.size());
@@ -615,9 +614,9 @@
public void testPartiallyBrokenFile() throws Exception
{
- final int JOURNAL_SIZE = 2000;
+ final int JOURNAL_SIZE = 20000;
- setupJournal(JOURNAL_SIZE, 100);
+ setupAndLoadJournal(JOURNAL_SIZE, 100);
assertEquals(2, factory.listFiles("tt").size());
@@ -626,13 +625,10 @@
for (int i = 0; i < 20; i++)
{
- journalImpl.appendAddRecordTransactional(1l, i, (byte)0, new SimpleEncoding(1, (byte)15), false);
- journalImpl.appendAddRecordTransactional(2l, i + 20l, (byte)0, new SimpleEncoding(1, (byte)15), false);
- journalImpl.forceMoveNextFile();
+ journalImpl.appendAddRecordTransactional(1l, i, (byte)0, new SimpleEncoding(1, (byte)15));
+ journalImpl.appendAddRecordTransactional(2l, i + 20l, (byte)0, new SimpleEncoding(1, (byte)15));
}
- journalImpl.forceMoveNextFile();
-
journalImpl.appendCommitRecord(1l, false);
journalImpl.appendCommitRecord(2l, false);
@@ -669,27 +665,23 @@
file.close();
- setupJournal(JOURNAL_SIZE, 100);
+ setupAndLoadJournal(JOURNAL_SIZE, 100);
assertEquals(20, records.size());
journalImpl.checkAndReclaimFiles();
- assertEquals(20, journalImpl.getDataFilesCount());
-
- assertEquals(22, factory.listFiles("tt").size());
-
}
public void testReduceFreeFiles() throws Exception
{
final int JOURNAL_SIZE = 2000;
- setupJournal(JOURNAL_SIZE, 100, 10);
+ setupAndLoadJournal(JOURNAL_SIZE, 100, 10);
assertEquals(10, factory.listFiles("tt").size());
- setupJournal(JOURNAL_SIZE, 100, 2);
+ setupAndLoadJournal(JOURNAL_SIZE, 100, 2);
assertEquals(10, factory.listFiles("tt").size());
@@ -699,7 +691,7 @@
journalImpl.forceMoveNextFile();
}
- setupJournal(JOURNAL_SIZE, 100, 2);
+ setupAndLoadJournal(JOURNAL_SIZE, 100, 2);
assertEquals(10, records.size());
@@ -714,7 +706,7 @@
journalImpl.checkAndReclaimFiles();
- setupJournal(JOURNAL_SIZE, 100, 2);
+ setupAndLoadJournal(JOURNAL_SIZE, 100, 2);
assertEquals(0, records.size());
@@ -725,7 +717,7 @@
{
final int JOURNAL_SIZE = 2000;
- setupJournal(JOURNAL_SIZE, 100);
+ setupAndLoadJournal(JOURNAL_SIZE, 100);
assertEquals(2, factory.listFiles("tt").size());
@@ -734,18 +726,14 @@
for (int i = 0; i < 10; i++)
{
- journalImpl.appendAddRecordTransactional(1l, i, (byte)0, new SimpleEncoding(1, (byte)15), false);
- journalImpl.forceMoveNextFile();
+ journalImpl.appendAddRecordTransactional(1l, i, (byte)0, new SimpleEncoding(1, (byte)15));
}
for (int i = 10; i < 20; i++)
{
- journalImpl.appendAddRecordTransactional(1l, i, (byte)0, new SimpleEncoding(1, (byte)15), false);
- journalImpl.forceMoveNextFile();
+ journalImpl.appendAddRecordTransactional(1l, i, (byte)0, new SimpleEncoding(1, (byte)15));
}
- journalImpl.forceMoveNextFile();
-
journalImpl.appendCommitRecord(1l, false);
SequentialFile file = factory.createSequentialFile("tt-1.tt", 1);
@@ -773,7 +761,7 @@
file.close();
- setupJournal(JOURNAL_SIZE, 100);
+ setupAndLoadJournal(JOURNAL_SIZE, 100);
assertEquals(0, records.size());
@@ -789,14 +777,14 @@
{
final int JOURNAL_SIZE = 20000;
- setupJournal(JOURNAL_SIZE, 100);
+ setupAndLoadJournal(JOURNAL_SIZE, 100);
assertEquals(0, records.size());
assertEquals(0, transactions.size());
for (int i = 0; i < 10; i++)
{
- journalImpl.appendAddRecordTransactional(1l, i, (byte)0, new SimpleEncoding(1, (byte)15), false);
+ journalImpl.appendAddRecordTransactional(1l, i, (byte)0, new SimpleEncoding(1, (byte)15));
}
journalImpl.forceMoveNextFile();
@@ -807,7 +795,7 @@
for (int i = 0; i < 10; i++)
{
- journalImpl.appendDeleteRecordTransactional(2l, i, false);
+ journalImpl.appendDeleteRecordTransactional(2l, i);
}
journalImpl.appendCommitRecord(2l, false);
@@ -821,7 +809,7 @@
journalImpl.forceMoveNextFile();
journalImpl.checkAndReclaimFiles();
- setupJournal(JOURNAL_SIZE, 100);
+ setupAndLoadJournal(JOURNAL_SIZE, 100);
assertEquals(1, records.size());
}
@@ -830,7 +818,7 @@
{
final int JOURNAL_SIZE = 20000;
- setupJournal(JOURNAL_SIZE, 100);
+ setupAndLoadJournal(JOURNAL_SIZE, 100);
assertEquals(0, records.size());
assertEquals(0, transactions.size());
@@ -841,7 +829,7 @@
{
journalImpl.forceMoveNextFile();
}
- journalImpl.appendAddRecordTransactional(1l, i, (byte)0, new SimpleEncoding(1, (byte)15), false);
+ journalImpl.appendAddRecordTransactional(1l, i, (byte)0, new SimpleEncoding(1, (byte)15));
}
journalImpl.appendCommitRecord(1l, false);
@@ -852,14 +840,14 @@
{
journalImpl.forceMoveNextFile();
}
- journalImpl.appendDeleteRecordTransactional(2l, i, false);
+ journalImpl.appendDeleteRecordTransactional(2l, i);
}
journalImpl.appendCommitRecord(2l, false);
journalImpl.forceMoveNextFile();
journalImpl.checkAndReclaimFiles();
- setupJournal(JOURNAL_SIZE, 100);
+ setupAndLoadJournal(JOURNAL_SIZE, 100);
assertEquals(40, records.size());
@@ -869,7 +857,7 @@
{
final int JOURNAL_SIZE = 3 * 1024;
- setupJournal(JOURNAL_SIZE, 1);
+ setupAndLoadJournal(JOURNAL_SIZE, 1);
assertEquals(0, records.size());
assertEquals(0, transactions.size());
@@ -878,13 +866,13 @@
journalImpl.appendAddRecord(10l, (byte)0, new SimpleEncoding(10, (byte)0), false);
- journalImpl.appendDeleteRecordTransactional(1l, 10l, new SimpleEncoding(100, (byte)'j'), false);
+ journalImpl.appendDeleteRecordTransactional(1l, 10l, new SimpleEncoding(100, (byte)'j'));
journalImpl.appendPrepareRecord(1, xid, false);
journalImpl.debugWait();
- setupJournal(JOURNAL_SIZE, 1);
+ setupAndLoadJournal(JOURNAL_SIZE, 1);
assertEquals(1, transactions.size());
assertEquals(1, transactions.get(0).recordsToDelete.size());
@@ -911,7 +899,7 @@
journalImpl.debugWait();
- setupJournal(JOURNAL_SIZE, 1);
+ setupAndLoadJournal(JOURNAL_SIZE, 1);
assertEquals(0, transactions.size());
assertEquals(0, records.size());
@@ -922,14 +910,14 @@
{
final int JOURNAL_SIZE = 3 * 1024;
- setupJournal(JOURNAL_SIZE, 1);
+ setupAndLoadJournal(JOURNAL_SIZE, 1);
assertEquals(0, records.size());
assertEquals(0, transactions.size());
for (int i = 0; i < 10; i++)
{
- journalImpl.appendAddRecordTransactional(1, i, (byte)1, new SimpleEncoding(50, (byte)1), false);
+ journalImpl.appendAddRecordTransactional(1, i, (byte)1, new SimpleEncoding(50, (byte)1));
journalImpl.forceMoveNextFile();
}
@@ -941,7 +929,7 @@
assertEquals(12, factory.listFiles("tt").size());
- setupJournal(JOURNAL_SIZE, 1024);
+ setupAndLoadJournal(JOURNAL_SIZE, 1024);
assertEquals(0, records.size());
assertEquals(1, transactions.size());
@@ -960,7 +948,7 @@
journalImpl.appendCommitRecord(1l, false);
- setupJournal(JOURNAL_SIZE, 1024);
+ setupAndLoadJournal(JOURNAL_SIZE, 1024);
assertEquals(10, records.size());
@@ -968,14 +956,14 @@
for (int i = 0; i < 10; i++)
{
- journalImpl.appendDeleteRecordTransactional(2l, i, false);
+ journalImpl.appendDeleteRecordTransactional(2l, i);
}
SimpleEncoding xid2 = new SimpleEncoding(15, (byte)2);
journalImpl.appendPrepareRecord(2l, xid2, false);
- setupJournal(JOURNAL_SIZE, 1);
+ setupAndLoadJournal(JOURNAL_SIZE, 1);
assertEquals(1, transactions.size());
@@ -992,7 +980,7 @@
journalImpl.appendCommitRecord(2l, false);
- setupJournal(JOURNAL_SIZE, 1);
+ setupAndLoadJournal(JOURNAL_SIZE, 1);
assertEquals(0, records.size());
assertEquals(0, transactions.size());
@@ -1011,20 +999,19 @@
{
final int JOURNAL_SIZE = 3000;
- setupJournal(JOURNAL_SIZE, 100);
+ setupAndLoadJournal(JOURNAL_SIZE, 100);
assertEquals(0, records.size());
assertEquals(0, transactions.size());
for (int i = 0; i < 10; i++)
{
- journalImpl.appendAddRecordTransactional(1, i, (byte)1, new SimpleEncoding(50, (byte)1), false);
- journalImpl.forceMoveNextFile();
+ journalImpl.appendAddRecordTransactional(1, i, (byte)1, new SimpleEncoding(50, (byte)1));
}
journalImpl.appendPrepareRecord(1l, new SimpleEncoding(13, (byte)0), false);
- setupJournal(JOURNAL_SIZE, 100);
+ setupAndLoadJournal(JOURNAL_SIZE, 100);
assertEquals(0, records.size());
assertEquals(1, transactions.size());
@@ -1053,7 +1040,7 @@
file.close();
- setupJournal(JOURNAL_SIZE, 100);
+ setupAndLoadJournal(JOURNAL_SIZE, 100);
assertEquals(0, records.size());
assertEquals(0, transactions.size());
@@ -1063,11 +1050,11 @@
{
final int JOURNAL_SIZE = 2000;
- setupJournal(JOURNAL_SIZE, 1);
+ setupAndLoadJournal(JOURNAL_SIZE, 1);
for (int i = 0; i < 10; i++)
{
- journalImpl.appendAddRecordTransactional(1l, i, (byte)0, new SimpleEncoding(1, (byte)0), false);
+ journalImpl.appendAddRecordTransactional(1l, i, (byte)0, new SimpleEncoding(1, (byte)0));
journalImpl.forceMoveNextFile();
}
@@ -1079,7 +1066,7 @@
assertEquals(0, journalImpl.getDataFilesCount());
- setupJournal(JOURNAL_SIZE, 1);
+ setupAndLoadJournal(JOURNAL_SIZE, 1);
assertEquals(0, journalImpl.getDataFilesCount());
@@ -1092,20 +1079,20 @@
{
final int JOURNAL_SIZE = 512 * 4;
- setupJournal(JOURNAL_SIZE, 512);
+ setupAndLoadJournal(JOURNAL_SIZE, 512);
for (int i = 0; i < 10; i++)
{
- journalImpl.appendAddRecordTransactional(1l, i, (byte)0, new SimpleEncoding(1, (byte)0), false);
+ journalImpl.appendAddRecordTransactional(1l, i, (byte)0, new SimpleEncoding(1, (byte)0));
}
journalImpl.appendCommitRecord(1l, false);
- setupJournal(JOURNAL_SIZE, 100);
+ setupAndLoadJournal(JOURNAL_SIZE, 100);
assertEquals(10, records.size());
- setupJournal(JOURNAL_SIZE, 1);
+ setupAndLoadJournal(JOURNAL_SIZE, 1);
assertEquals(10, records.size());
}
@@ -1115,20 +1102,20 @@
{
final int JOURNAL_SIZE = 512 * 4;
- setupJournal(JOURNAL_SIZE, 1);
+ setupAndLoadJournal(JOURNAL_SIZE, 1);
for (int i = 0; i < 10; i++)
{
- journalImpl.appendAddRecordTransactional(1l, i, (byte)0, new SimpleEncoding(1, (byte)0), false);
+ journalImpl.appendAddRecordTransactional(1l, i, (byte)0, new SimpleEncoding(1, (byte)0));
}
journalImpl.appendCommitRecord(1l, false);
- setupJournal(JOURNAL_SIZE, 100);
+ setupAndLoadJournal(JOURNAL_SIZE, 100);
assertEquals(10, records.size());
- setupJournal(JOURNAL_SIZE, 512);
+ setupAndLoadJournal(JOURNAL_SIZE, 512);
assertEquals(10, records.size());
}
@@ -1137,7 +1124,7 @@
{
final int JOURNAL_SIZE = 512 * 4;
- setupJournal(JOURNAL_SIZE, 1);
+ setupAndLoadJournal(JOURNAL_SIZE, 1);
journalImpl.appendPrepareRecord(2l, new SimpleEncoding(10, (byte)'j'), false);
@@ -1145,7 +1132,7 @@
journalImpl.appendAddRecord(1l, (byte)0, new SimpleEncoding(10, (byte)'k'), false);
- setupJournal(JOURNAL_SIZE, 1);
+ setupAndLoadJournal(JOURNAL_SIZE, 1);
assertEquals(1, journalImpl.getDataFilesCount());
@@ -1153,7 +1140,7 @@
journalImpl.forceMoveNextFile();
- setupJournal(JOURNAL_SIZE, 1);
+ setupAndLoadJournal(JOURNAL_SIZE, 1);
assertEquals(1, journalImpl.getDataFilesCount());
@@ -1165,7 +1152,7 @@
journalImpl.forceMoveNextFile();
- setupJournal(JOURNAL_SIZE, 0);
+ setupAndLoadJournal(JOURNAL_SIZE, 0);
journalImpl.forceMoveNextFile();
journalImpl.debugWait();
@@ -1180,7 +1167,7 @@
{
final int JOURNAL_SIZE = 10 * 1024;
- setupJournal(JOURNAL_SIZE, 1);
+ setupAndLoadJournal(JOURNAL_SIZE, 1);
assertEquals(0, records.size());
assertEquals(0, transactions.size());
@@ -1203,7 +1190,7 @@
latchStart.await();
for (int i = 0; i < NUMBER_OF_ELEMENTS; i++)
{
- journalImpl.appendAddRecordTransactional(i, i, (byte)1, new SimpleEncoding(50, (byte)1), false);
+ journalImpl.appendAddRecordTransactional(i, i, (byte)1, new SimpleEncoding(50, (byte)1));
journalImpl.appendCommitRecord(i, false);
queueDelete.offer(i);
}
@@ -1268,7 +1255,7 @@
{
SequentialFileFactory factory = new FakeSequentialFileFactory(512, false);
- JournalImpl impl = new JournalImpl(512 + 512 * 3, 20, factory, "jbm", "jbm", 1000);
+ JournalImpl impl = new JournalImpl(512 + 512 * 3, 20, 0, 0, factory, "jbm", "jbm", 1000);
impl.start();
@@ -1281,7 +1268,7 @@
impl.stop();
- impl = new JournalImpl(512 + 1024 + 512, 20, factory, "jbm", "jbm", 1000);
+ impl = new JournalImpl(512 + 1024 + 512, 20, 0, 0, factory, "jbm", "jbm", 1000);
impl.start();
impl.load(dummyLoader);
@@ -1296,7 +1283,7 @@
impl.stop();
- impl = new JournalImpl(512 + 1024 + 512, 20, factory, "jbm", "jbm", 1000);
+ impl = new JournalImpl(512 + 1024 + 512, 20, 0, 0, factory, "jbm", "jbm", 1000);
impl.start();
ArrayList<RecordInfo> info = new ArrayList<RecordInfo>();
@@ -1341,18 +1328,18 @@
{
}
}
-
+
super.tearDown();
}
// Private -------------------------------------------------------
- private void setupJournal(final int journalSize, final int alignment) throws Exception
+ private void setupAndLoadJournal(final int journalSize, final int alignment) throws Exception
{
- setupJournal(journalSize, alignment, 2);
+ setupAndLoadJournal(journalSize, alignment, 2);
}
- private void setupJournal(final int journalSize, final int alignment, final int numberOfMinimalFiles) throws Exception
+ private void setupAndLoadJournal(final int journalSize, final int alignment, final int numberOfMinimalFiles) throws Exception
{
if (factory == null)
{
@@ -1364,7 +1351,7 @@
journalImpl.stop();
}
- journalImpl = new JournalImpl(journalSize, numberOfMinimalFiles, factory, "tt", "tt", 1000);
+ journalImpl = new JournalImpl(journalSize, numberOfMinimalFiles, 0, 0, factory, "tt", "tt", 1000);
journalImpl.start();
Modified: trunk/tests/src/org/jboss/messaging/tests/unit/core/journal/impl/JournalAsyncTest.java
===================================================================
--- trunk/tests/src/org/jboss/messaging/tests/unit/core/journal/impl/JournalAsyncTest.java 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/tests/src/org/jboss/messaging/tests/unit/core/journal/impl/JournalAsyncTest.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -85,7 +85,7 @@
{
for (int i = 0; i < 10; i++)
{
- journalImpl.appendAddRecordTransactional(1l, i, (byte)1, new SimpleEncoding(1, (byte)0), true);
+ journalImpl.appendAddRecordTransactional(1l, i, (byte)1, new SimpleEncoding(1, (byte)0));
}
latch.countDown();
@@ -147,7 +147,7 @@
{
for (int i = 0; i < 10; i++)
{
- journalImpl.appendAddRecordTransactional(1l, i, (byte)1, new SimpleEncoding(1, (byte)0), true);
+ journalImpl.appendAddRecordTransactional(1l, i, (byte)1, new SimpleEncoding(1, (byte)0));
}
journalImpl.appendRollbackRecord(1l, true);
@@ -211,7 +211,7 @@
{
for (int i = 0; i < 10; i++)
{
- journalImpl.appendAddRecordTransactional(1l, i, (byte)1, new SimpleEncoding(1, (byte)0), true);
+ journalImpl.appendAddRecordTransactional(1l, i, (byte)1, new SimpleEncoding(1, (byte)0));
}
journalImpl.appendCommitRecord(1l, true);
@@ -268,7 +268,7 @@
factory.setHoldCallbacks(true, null);
factory.setGenerateErrors(true);
- journalImpl.appendAddRecordTransactional(1l, 1, (byte)1, new SimpleEncoding(1, (byte)0), true);
+ journalImpl.appendAddRecordTransactional(1l, 1, (byte)1, new SimpleEncoding(1, (byte)0));
factory.flushAllCallbacks();
@@ -277,7 +277,7 @@
try
{
- journalImpl.appendAddRecordTransactional(1l, 2, (byte)1, new SimpleEncoding(1, (byte)0), true);
+ journalImpl.appendAddRecordTransactional(1l, 2, (byte)1, new SimpleEncoding(1, (byte)0));
fail("Exception expected"); // An exception already happened in one
// of the elements on this transaction.
// We can't accept any more elements on
@@ -340,7 +340,7 @@
{
}
}
-
+
super.tearDown();
}
@@ -357,7 +357,7 @@
journalImpl.stop();
}
- journalImpl = new JournalImpl(journalSize, numberOfMinimalFiles, factory, "tt", "tt", 1000);
+ journalImpl = new JournalImpl(journalSize, numberOfMinimalFiles, 0, 0, factory, "tt", "tt", 1000);
journalImpl.start();
Modified: trunk/tests/src/org/jboss/messaging/tests/unit/core/journal/impl/JournalImplTestBase.java
===================================================================
--- trunk/tests/src/org/jboss/messaging/tests/unit/core/journal/impl/JournalImplTestBase.java 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/tests/src/org/jboss/messaging/tests/unit/core/journal/impl/JournalImplTestBase.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -38,7 +38,6 @@
import org.jboss.messaging.core.journal.TestableJournal;
import org.jboss.messaging.core.journal.impl.JournalImpl;
import org.jboss.messaging.core.logging.Logger;
-import org.jboss.messaging.tests.util.RandomUtil;
import org.jboss.messaging.tests.util.UnitTestCase;
/**
@@ -67,7 +66,7 @@
protected int fileSize;
protected boolean sync;
-
+
protected String filePrefix = "jbm";
protected String fileExtension = "jbm";
@@ -105,7 +104,7 @@
journal = null;
assertEquals(0, AsynchronousFileImpl.getTotalMaxIO());
-
+
super.tearDown();
}
@@ -144,7 +143,7 @@
public void createJournal() throws Exception
{
- journal = new JournalImpl(fileSize, minFiles, fileFactory, filePrefix, fileExtension, maxAIO);
+ journal = new JournalImpl(fileSize, minFiles, 0, 0, fileFactory, filePrefix, fileExtension, maxAIO);
journal.setAutoReclaim(false);
}
@@ -168,7 +167,7 @@
journal.stop();
}
-
+
protected void loadAndCheck() throws Exception
{
loadAndCheck(false);
@@ -183,7 +182,7 @@
journal.load(committedRecords, preparedTransactions);
checkRecordsEquivalent(records, committedRecords);
-
+
if (printDebugJournal)
{
printJournalLists(records, committedRecords);
@@ -270,7 +269,7 @@
// SIZE_BYTE
byte[] record = generateRecord(recordLength - JournalImpl.SIZE_ADD_RECORD_TX);
- journal.appendAddRecordTransactional(txID, element, (byte)0, record, sync);
+ journal.appendAddRecordTransactional(txID, element, (byte)0, record);
tx.records.add(new RecordInfo(element, (byte)0, record, false));
@@ -287,7 +286,7 @@
{
byte[] updateRecord = generateRecord(recordLength - JournalImpl.SIZE_UPDATE_RECORD_TX);
- journal.appendUpdateRecordTransactional(txID, element, (byte)0, updateRecord, sync);
+ journal.appendUpdateRecordTransactional(txID, element, (byte)0, updateRecord);
tx.records.add(new RecordInfo(element, (byte)0, updateRecord, true));
}
@@ -300,7 +299,7 @@
for (long element : arguments)
{
- journal.appendDeleteRecordTransactional(txID, element, sync);
+ journal.appendDeleteRecordTransactional(txID, element);
tx.deletes.add(new RecordInfo(element, (byte)0, null, true));
}
@@ -499,7 +498,8 @@
byte[] record = new byte[length];
for (int i = 0; i < length; i++)
{
- record[i] = RandomUtil.randomByte();
+ // record[i] = RandomUtil.randomByte();
+ record[i] = getSamplebyte(i);
}
return record;
}
Modified: trunk/tests/src/org/jboss/messaging/tests/unit/core/journal/impl/JournalImplTestUnit.java
===================================================================
--- trunk/tests/src/org/jboss/messaging/tests/unit/core/journal/impl/JournalImplTestUnit.java 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/tests/src/org/jboss/messaging/tests/unit/core/journal/impl/JournalImplTestUnit.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -128,7 +128,7 @@
{
try
{
- new JournalImpl(JournalImpl.MIN_FILE_SIZE - 1, 10, fileFactory, filePrefix, fileExtension, 1);
+ new JournalImpl(JournalImpl.MIN_FILE_SIZE - 1, 10, 0, 0, fileFactory, filePrefix, fileExtension, 1);
fail("Should throw exception");
}
@@ -139,7 +139,7 @@
try
{
- new JournalImpl(10 * 1024, 1, fileFactory, filePrefix, fileExtension, 1);
+ new JournalImpl(10 * 1024, 1, 0, 0, fileFactory, filePrefix, fileExtension, 1);
fail("Should throw exception");
}
@@ -150,7 +150,7 @@
try
{
- new JournalImpl(10 * 1024, 10, null, filePrefix, fileExtension, 1);
+ new JournalImpl(10 * 1024, 10, 0, 0, null, filePrefix, fileExtension, 1);
fail("Should throw exception");
}
@@ -161,7 +161,7 @@
try
{
- new JournalImpl(10 * 1024, 10, fileFactory, null, fileExtension, 1);
+ new JournalImpl(10 * 1024, 10, 0, 0, fileFactory, null, fileExtension, 1);
fail("Should throw exception");
}
@@ -172,7 +172,7 @@
try
{
- new JournalImpl(10 * 1024, 10, fileFactory, filePrefix, null, 1);
+ new JournalImpl(10 * 1024, 10, 0, 0, fileFactory, filePrefix, null, 1);
fail("Should throw exception");
}
@@ -183,7 +183,7 @@
try
{
- new JournalImpl(10 * 1024, 10, fileFactory, filePrefix, null, 0);
+ new JournalImpl(10 * 1024, 10, 0, 0, fileFactory, filePrefix, null, 0);
fail("Should throw exception");
}
@@ -1665,7 +1665,6 @@
assertEquals(1, journal.getIDMapSize());
}
-
public void testPrepareNoReclaim() throws Exception
{
setup(2, calculateRecordSize(JournalImpl.SIZE_HEADER, getAlignment()) + calculateRecordSize(recordLength,
@@ -2316,6 +2315,41 @@
loadAndCheck();
}
+ public void testSimpleAddTXReload() throws Exception
+ {
+ setup(2, 10 * 1024, true);
+ createJournal();
+ startJournal();
+ load();
+ addTx(1, 1);
+ commit(1);
+
+ stopJournal();
+ createJournal();
+ startJournal();
+ loadAndCheck();
+
+ }
+
+ public void testSimpleAddTXXAReload() throws Exception
+ {
+ setup(2, 10 * 1024, true);
+ createJournal();
+ startJournal();
+ load();
+ addTx(1, 1);
+
+ EncodingSupport xid = new SimpleEncoding(10, (byte)'p');
+
+ prepare(1, xid);
+
+ stopJournal();
+ createJournal();
+ startJournal();
+ loadAndCheck();
+
+ }
+
public void testAddUpdateDeleteTransactionalRestartAndContinue() throws Exception
{
setup(10, 10 * 1024, true);
@@ -2944,7 +2978,7 @@
createJournal();
startJournal();
load();
-
+
int transactionID = 0;
for (int i = 0; i < 100; i++)
@@ -2962,7 +2996,6 @@
{
addTx(transactionID, i);
- updateTx(i + 100);
if (i % 10 == 0 && i > 0)
{
journal.forceMoveNextFile();
@@ -2991,25 +3024,40 @@
delete(i);
}
+ System.out.println("After delete ****************************");
+ System.out.println(journal.debug());
+ System.out.println("*****************************************");
+
for (int i = 100; i < 200; i++)
{
updateTx(transactionID, i);
}
+ System.out.println("After updatetx ****************************");
+ System.out.println(journal.debug());
+ System.out.println("*****************************************");
+
journal.forceMoveNextFile();
-
+
commit(transactionID++);
-
+
+ System.out.println("After commit ****************************");
+ System.out.println(journal.debug());
+ System.out.println("*****************************************");
+
for (int i = 100; i < 200; i++)
{
updateTx(transactionID, i);
deleteTx(transactionID, i);
}
-
+
+ System.out.println("After delete ****************************");
+ System.out.println(journal.debug());
+ System.out.println("*****************************************");
+
commit(transactionID++);
-
-
- System.out.println("Before reclaim ****************************");
+
+ System.out.println("Before reclaim/after commit ****************************");
System.out.println(journal.debug());
System.out.println("*****************************************");
@@ -3021,7 +3069,7 @@
System.out.println("After reclaim ****************************");
System.out.println(journal.debug());
System.out.println("*****************************************");
-
+
journal.forceMoveNextFile();
journal.checkAndReclaimFiles();
Modified: trunk/tests/src/org/jboss/messaging/tests/unit/core/journal/impl/ReclaimerTest.java
===================================================================
--- trunk/tests/src/org/jboss/messaging/tests/unit/core/journal/impl/ReclaimerTest.java 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/tests/src/org/jboss/messaging/tests/unit/core/journal/impl/ReclaimerTest.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -762,7 +762,7 @@
return 0;
}
- public int getOrderingID()
+ public int getFileID()
{
return 0;
}
@@ -863,5 +863,64 @@
{
return transactionIDs;
}
+
+ /* (non-Javadoc)
+ * @see org.jboss.messaging.core.journal.impl.JournalFile#decPendingTransaction()
+ */
+ public void decPendingTransaction()
+ {
+ }
+
+ /* (non-Javadoc)
+ * @see org.jboss.messaging.core.journal.impl.JournalFile#getPendingTransactions()
+ */
+ public int getPendingTransactions()
+ {
+ return 0;
+ }
+
+ /* (non-Javadoc)
+ * @see org.jboss.messaging.core.journal.impl.JournalFile#incPendingTransaction()
+ */
+ public void incPendingTransaction()
+ {
+ }
+
+ /* (non-Javadoc)
+ * @see org.jboss.messaging.core.journal.impl.JournalFile#getOrderingID()
+ */
+ public int getOrderingID()
+ {
+ return 0;
+ }
+
+ /* (non-Javadoc)
+ * @see org.jboss.messaging.core.journal.impl.JournalFile#clearCounts()
+ */
+ public void clearCounts()
+ {
+ }
+
+ /* (non-Javadoc)
+ * @see org.jboss.messaging.core.journal.impl.JournalFile#addSize(int)
+ */
+ public void addSize(int bytes)
+ {
+ }
+
+ /* (non-Javadoc)
+ * @see org.jboss.messaging.core.journal.impl.JournalFile#decSize(int)
+ */
+ public void decSize(int bytes)
+ {
+ }
+
+ /* (non-Javadoc)
+ * @see org.jboss.messaging.core.journal.impl.JournalFile#getSize()
+ */
+ public int getLiveSize()
+ {
+ return 0;
+ }
}
}
Modified: trunk/tests/src/org/jboss/messaging/tests/unit/core/journal/impl/SequentialFileFactoryTestBase.java
===================================================================
--- trunk/tests/src/org/jboss/messaging/tests/unit/core/journal/impl/SequentialFileFactoryTestBase.java 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/tests/src/org/jboss/messaging/tests/unit/core/journal/impl/SequentialFileFactoryTestBase.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -177,6 +177,38 @@
}
+ public void testRename() throws Exception
+ {
+ SequentialFile sf = factory.createSequentialFile("test1.jbm", 1);
+
+ sf.open();
+
+ List<String> fileNames = factory.listFiles("jbm");
+
+ assertEquals(1, fileNames.size());
+
+ assertTrue(fileNames.contains("test1.jbm"));
+
+ sf.renameTo("test1.cmp");
+
+ fileNames = factory.listFiles("cmp");
+
+ assertEquals(1, fileNames.size());
+
+ assertTrue(fileNames.contains("test1.cmp"));
+
+ sf.delete();
+
+ fileNames = factory.listFiles("jbm");
+
+ assertEquals(0, fileNames.size());
+
+ fileNames = factory.listFiles("cmp");
+
+ assertEquals(0, fileNames.size());
+
+ }
+
// TODO: RE-ENABLE THIS
// public void testWriteandRead() throws Exception
// {
Modified: trunk/tests/src/org/jboss/messaging/tests/unit/core/journal/impl/fakes/FakeSequentialFileFactory.java
===================================================================
--- trunk/tests/src/org/jboss/messaging/tests/unit/core/journal/impl/fakes/FakeSequentialFileFactory.java 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/tests/src/org/jboss/messaging/tests/unit/core/journal/impl/fakes/FakeSequentialFileFactory.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -87,7 +87,7 @@
{
FakeSequentialFile sf = fileMap.get(fileName);
- if (sf == null)
+ if (sf == null || sf.data == null)
{
sf = newSequentialFile(fileName);
@@ -332,7 +332,7 @@
this.fileName = fileName;
}
- public void close() throws Exception
+ public synchronized void close() throws Exception
{
open = false;
@@ -340,15 +340,24 @@
{
data.position(0);
}
+
+ this.notifyAll();
}
+ public synchronized void waitForClose() throws Exception
+ {
+ while (open)
+ {
+ this.wait();
+ }
+ }
+
public void delete() throws Exception
{
if (!open)
{
- throw new IllegalStateException("Is closed");
+ close();
}
- close();
fileMap.remove(fileName);
}
@@ -602,7 +611,7 @@
public void write(MessagingBuffer bytes, boolean sync, IOCallback callback) throws Exception
{
write(ByteBuffer.wrap(bytes.array()), sync, callback);
-
+
}
/* (non-Javadoc)
@@ -613,6 +622,16 @@
write(ByteBuffer.wrap(bytes.array()), sync);
}
+ /* (non-Javadoc)
+ * @see org.jboss.messaging.core.journal.SequentialFile#exists()
+ */
+ public boolean exists()
+ {
+ FakeSequentialFile file = fileMap.get(fileName);
+
+ return file != null && file.data != null && file.data.capacity() > 0;
+ }
+
}
/* (non-Javadoc)
@@ -680,4 +699,11 @@
{
}
+ /* (non-Javadoc)
+ * @see org.jboss.messaging.core.journal.SequentialFileFactory#testFlush()
+ */
+ public void testFlush()
+ {
+ }
+
}
Modified: trunk/tests/src/org/jboss/messaging/tests/util/JournalExample.java
===================================================================
--- trunk/tests/src/org/jboss/messaging/tests/util/JournalExample.java 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/tests/src/org/jboss/messaging/tests/util/JournalExample.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -70,8 +70,10 @@
SequentialFileFactory fileFactory = new AIOSequentialFileFactory("/tmp"); // any dir you want
// SequentialFileFactory fileFactory = new NIOSequentialFileFactory("/tmp"); // any dir you want
JournalImpl journalExample = new JournalImpl(10 * 1024 * 1024, // 10M.. we believe that's the usual cilinder
- // bufferSize.. not an exact science here
+ // bufferSize.. not an exact science here
2, // number of files pre-allocated
+ 0,
+ 0,
fileFactory, // AIO or NIO
"exjournal", // file name
"dat", // extension
@@ -147,7 +149,7 @@
0,
1,
2,
- 5 }, true);
+ 5 });
}
// After this is complete, you're sure the records are there
Modified: trunk/tests/src/org/jboss/messaging/tests/util/ListJournal.java
===================================================================
--- trunk/tests/src/org/jboss/messaging/tests/util/ListJournal.java 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/tests/src/org/jboss/messaging/tests/util/ListJournal.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -67,6 +67,8 @@
JournalImpl journal = new JournalImpl(fileConf.getJournalFileSize(),
fileConf.getJournalMinFiles(),
+ 0,
+ 0,
new NIOSequentialFileFactory(fileConf.getJournalDirectory()),
"jbm-data",
"jbm",
Modified: trunk/tests/src/org/jboss/messaging/tests/util/ServiceTestBase.java
===================================================================
--- trunk/tests/src/org/jboss/messaging/tests/util/ServiceTestBase.java 2009-07-02 16:25:39 UTC (rev 7515)
+++ trunk/tests/src/org/jboss/messaging/tests/util/ServiceTestBase.java 2009-07-02 21:36:15 UTC (rev 7516)
@@ -128,6 +128,8 @@
config.setBindingsDirectory(getBindingsDir());
config.setJournalType(JournalType.ASYNCIO);
config.setLargeMessagesDirectory(getLargeMessagesDir());
+ config.setJournalCompactMinFiles(0);
+ config.setJournalCompactPercentage(0);
return config;
}
@@ -275,7 +277,10 @@
configuration.setJournalType(JournalType.ASYNCIO);
configuration.setPagingDirectory(getPageDir(index, false));
configuration.setLargeMessagesDirectory(getLargeMessagesDir(index, false));
+ configuration.setJournalCompactMinFiles(0);
+ configuration.setJournalCompactPercentage(0);
+
configuration.getAcceptorConfigurations().clear();
for (String acceptor : acceptors)
@@ -298,6 +303,8 @@
configuration.setJournalFileSize(100 * 1024);
configuration.setPagingDirectory(getPageDir());
configuration.setLargeMessagesDirectory(getLargeMessagesDir());
+ configuration.setJournalCompactMinFiles(0);
+ configuration.setJournalCompactPercentage(0);
configuration.setJournalType(JournalType.ASYNCIO);
More information about the jboss-cvs-commits
mailing list