rhmessaging-commits
September 2007
rhmessaging-commits@lists.jboss.org
- 1 participants
- 51 discussions
rhmessaging commits: r954 - store/trunk/cpp/tests/jrnl.
by rhmessaging-commits＠lists.jboss.org
Author: kpvdr
Date: 2007-09-30 10:33:44 -0400 (Sun, 30 Sep 2007)
New Revision: 954
Modified:
store/trunk/cpp/tests/jrnl/Makefile.rtest
store/trunk/cpp/tests/jrnl/janalyze.py
store/trunk/cpp/tests/jrnl/rtest
Log:
Minor bugfixes in tools/scripts
Modified: store/trunk/cpp/tests/jrnl/Makefile.rtest
===================================================================
--- store/trunk/cpp/tests/jrnl/Makefile.rtest 2007-09-28 20:56:24 UTC (rev 953)
+++ store/trunk/cpp/tests/jrnl/Makefile.rtest 2007-09-30 14:33:44 UTC (rev 954)
@@ -41,7 +41,7 @@
SHELL = /bin/bash
-QPID_HOME_DIR = $(HOME)/rhm/qpid
+QPID_HOME_DIR = $(HOME)/qpid
RHM_JRNL_SRC_DIR = ../../lib/jrnl
RHM_JRNL_DOC_DIR = ../../docs
Modified: store/trunk/cpp/tests/jrnl/janalyze.py
===================================================================
--- store/trunk/cpp/tests/jrnl/janalyze.py 2007-09-28 20:56:24 UTC (rev 953)
+++ store/trunk/cpp/tests/jrnl/janalyze.py 2007-09-30 14:33:44 UTC (rev 954)
@@ -373,8 +373,15 @@
def run(self):
- start_info = self.analyze_files()
- stop = self.advance_file(*start_info)
+ try:
+ start_info = self.analyze_files()
+ stop = self.advance_file(*start_info)
+ except Exception:
+ print 'WARNING: All journal files are empty.'
+ if self.num_msgs > 0:
+ raise Exception('All journal files are empty.')
+ else:
+ stop = True
while not stop:
warn = ''
if file_full(self.f):
@@ -528,7 +535,7 @@
print 'Usage: ftest test-file test-number'
raise Exception('Illegal argument value')
-if len(sys.argv) != 0:
+if len(sys.argv) == 1:
m = Main(None, None)
m.run()
else:
Modified: store/trunk/cpp/tests/jrnl/rtest
===================================================================
--- store/trunk/cpp/tests/jrnl/rtest 2007-09-28 20:56:24 UTC (rev 953)
+++ store/trunk/cpp/tests/jrnl/rtest 2007-09-30 14:33:44 UTC (rev 954)
@@ -30,6 +30,7 @@
NUM_JFILES=8
VG_NORM_FILESIZE=11
+#VG_NORM_FILESIZE=18 # RHEL5 triggers extra valgrind messages when pthreads are in use
# Write test
W_DO_TEST=T
@@ -367,7 +368,7 @@
fi
lc=`wc -l < ${TEMP_V_FILE}`
if (( ${lc} != ${VG_NORM_FILESIZE} )); then
- echo "${VALGRIND} ${TEST_PROG} -t ${test_num} exposed errors; exiting."
+ echo "${VALGRIND} ${TEST_PROG} ${test_file} ${test_num} exposed errors; exiting."
echo
cat ${TEMP_V_FILE}
return 5
17 years, 5 months
- 1
- 0
rhmessaging commits: r953 - store/trunk/cpp/lib.
by rhmessaging-commits＠lists.jboss.org
Author: cctrieloff
Date: 2007-09-28 16:56:24 -0400 (Fri, 28 Sep 2007)
New Revision: 953
Modified:
store/trunk/cpp/lib/BdbMessageStore.cpp
store/trunk/cpp/lib/BdbMessageStore.h
store/trunk/cpp/lib/TxnCtxt.h
Log:
- txn support for async IO
- some refactor to reduce line count.
- TODO
- pass xid list to jc recover when it supports it
- read msg list back from jc for indoubt txns and populate.
- move the xid jc functions.
Modified: store/trunk/cpp/lib/BdbMessageStore.cpp
===================================================================
--- store/trunk/cpp/lib/BdbMessageStore.cpp 2007-09-28 16:08:00 UTC (rev 952)
+++ store/trunk/cpp/lib/BdbMessageStore.cpp 2007-09-28 20:56:24 UTC (rev 953)
@@ -30,7 +30,6 @@
#include "BindingDbt.h"
#include "IdPairDbt.h"
#include "StringDbt.h"
-#include <jrnl/jcntl.hpp>
using namespace rhm::bdbstore;
using namespace qpid::broker;
@@ -44,7 +43,9 @@
static const u_int8_t MESSAGE_MESSAGE = 1;
static const u_int8_t BASIC_MESSAGE = 2;
+unsigned int TxnCtxt::count = 0;
+
BdbMessageStore::BdbMessageStore(const char* envpath) : env(0),
queueDb(&env, 0),
exchangeDb(&env, 0),
@@ -111,13 +112,11 @@
}
txn->commit(0);
-
- if (usingJrnl())
- {
- try{
+ if (usingJrnl()) {
+ try{
journal::jdir::delete_dir(getJrnlBaseDir(),true);
}
- catch ( journal::jexception& e) {
+ catch ( journal::jexception& e) {
std::string str;
THROW_STORE_EXCEPTION("Truncate clean up failed: " +e.to_string(str) );
}
@@ -129,31 +128,24 @@
if (queue.getPersistenceId()) {
THROW_STORE_EXCEPTION("Queue already created: " + queue.getName());
}
-
-
- if (usingJrnl())
- {
+ if (usingJrnl()) {
journal::jcntl* jQueue = new journal::jcntl(queue.getName(), getJrnlDir(queue), string("JournalData"));
queue.setExternalQueueStore(dynamic_cast<ExternalQueueStore*>(jQueue));
-
- try
- {
- // init will create the deque's for the init...
- jQueue->initialize();
-
- } catch (journal::jexception& e) {
- std::string s;
+ try {
+ // init will create the deque's for the init...
+ jQueue->initialize();
+ } catch (journal::jexception& e) {
+ std::string s;
THROW_STORE_EXCEPTION(e.to_string(s) + queue.getName());
}
+ }
- }
-
try {
if (!create(queueDb, queueIdSequence, queue)) {
- THROW_STORE_EXCEPTION("Queue already exists: " + queue.getName());
+ THROW_STORE_EXCEPTION("Queue already exists: " + queue.getName());
}
} catch (DbException& e) {
- THROW_STORE_EXCEPTION_2("Error creating queue named " + queue.getName(), e);
+ THROW_STORE_EXCEPTION_2("Error creating queue named " + queue.getName(), e);
}
}
@@ -412,7 +404,7 @@
try {
while (read) {
-std:: cout << "loop -- uses fixed size -> FIX <-" << std::endl;
+//std:: cout << "loop -- uses fixed size -> FIX <-" << std::endl;
// const iores read_data(void** dbuf, size_t& dbsize, data_tok* const dtok)
rhm::journal::iores res = jc->read_data_record(&buff, buffSize, &dtokp);
@@ -422,23 +414,24 @@
switch (res)
{
case rhm::journal::RHM_IORES_SUCCESS:{
- msg_count++;
- char* data = buff;
- unsigned headerSize = Buffer(data, preambleLength).getLong();
- Buffer headerBuff(data+ preambleLength, headerSize); /// do we want read size or header size ????
+ msg_count++;
+ char* data = buff;
+ unsigned headerSize = Buffer(data, preambleLength).getLong();
+ Buffer headerBuff(data+ preambleLength, headerSize); /// do we want read size or header size ????
- RecoverableMessage::shared_ptr msg = recovery.recoverMessage(headerBuff);
- msg->setPersistenceId(dtokp.rid());
+ RecoverableMessage::shared_ptr msg = recovery.recoverMessage(headerBuff);
+ msg->setPersistenceId(dtokp.rid());
- u_int32_t contentOffset = headerSize + preambleLength;
- u_int64_t contentSize = readSize - contentOffset;
-
- if (msg->loadContent(contentSize)) {
+ u_int32_t contentOffset = headerSize + preambleLength;
+ u_int64_t contentSize = readSize - contentOffset;
+ if (msg->loadContent(contentSize)) {
//now read the content
Buffer contentBuff(data + contentOffset, contentSize);
msg->decodeContent(contentBuff);
- }
- if (PreparedTransaction::isLocked(locked, queue->getPersistenceId(), dtokp.rid())) {
+ }
+ // TODO - change to prep list based on reading state from journal
+ // -- add to prepared.enqued list..
+ if (PreparedTransaction::isLocked(locked, queue->getPersistenceId(), dtokp.rid())) {
prepared[dtokp.rid()] = msg;
} else {
queue->recover(msg);
@@ -448,9 +441,9 @@
maxMessageId = dtokp.rid();
}
- dtokp.reset();
- dtokp.set_wstate(rhm::journal::data_tok::ENQ);
- break;
+ dtokp.reset();
+ dtokp.set_wstate(rhm::journal::data_tok::ENQ);
+ break;
}
case rhm::journal::RHM_IORES_AIO_WAIT:
if (++aio_sleep_cnt > MAX_AIO_SLEEPS)
@@ -552,22 +545,25 @@
void BdbMessageStore::recoverXids(txn_list& txns)
{
std::set<string> prepared;
- std::set<string> known;
+ collectPreparedXids(prepared);
- readXids(prepareXidDb, prepared);
- readXids(enqueueXidDb, known);
- readXids(dequeueXidDb, known);
+ if (!usingJrnl()){
+ std::set<string> known;
+ readXids(enqueueXidDb, known);
+ readXids(dequeueXidDb, known);
- //abort all known but unprepared xids:
- for (std::set<string>::iterator i = known.begin(); i != known.end(); i++) {
- if (prepared.find(*i) == prepared.end()) {
- TPCTxnCtxt txn(*i);
- completed(txn, dequeueXidDb, enqueueXidDb);
- }
- }
-
+ //abort all known but unprepared xids:
+ for (std::set<string>::iterator i = known.begin(); i != known.end(); i++) {
+ if (prepared.find(*i) == prepared.end()) {
+ TPCTxnCtxt txn(*i);
+ completed(txn, dequeueXidDb, enqueueXidDb);
+ }
+ }
+ }
txn_lock_map enqueues;
txn_lock_map dequeues;
+
+ //this will be empty when using the journal -- TODO to get from journal.
readLockedMappings(enqueueXidDb, enqueues);
readLockedMappings(dequeueXidDb, dequeues);
@@ -625,7 +621,7 @@
try {
Dbt key (&messageId, sizeof(messageId));
messageId = messageIdSequence.next();
- store(NULL, txn.get(), key, msg, true);
+ store(NULL, &txn, key, msg, true);
msg.setPersistenceId(messageId);
txn.commit();
} catch (std::exception& e) {
@@ -748,20 +744,23 @@
if (messageId == 0) {
messageId = messageIdSequence.next();
msg.setPersistenceId(messageId);
- newId = true;
- }
- store(&queue, txn->get(), key, msg, newId);
+ newId = true;
+ }
+ store(&queue, txn, key, msg, newId);
- if (!usingJrnl()){
+ if (usingJrnl()){
+ // add queue* to the txn map..
+ if (ctxt) txn->addXidRecord(queue);
+ }else{
msg.enqueueComplete(); // set enqueued for ack
put(mappingDb, txn->get(), key, value);
- }
- // cct if using Journal do we need to wait for IO to complete before calling thus???
- // set enqueue comple on callback msg.enqueueComplete();
- if (txn->isTPC()) {
- record2pcOp(enqueueXidDb, dynamic_cast<TPCTxnCtxt&>(*txn), messageId, queueId);
- }
+ // cct if using Journal do we need to wait for IO to complete before calling thus???
+ // set enqueue comple on callback msg.enqueueComplete();
+ if (txn->isTPC()) {
+ record2pcOp(enqueueXidDb, dynamic_cast<TPCTxnCtxt&>(*txn), messageId, queueId);
+ }
+ }
if (!ctxt) txn->commit();
} catch (std::exception& e) {
@@ -771,7 +770,7 @@
}
void BdbMessageStore::store(const PersistableQueue* queue,
- DbTxn* txn, Dbt& messageId,
+ TxnCtxt* txn, Dbt& messageId,
PersistableMessage& message,
bool newId)
{
@@ -789,7 +788,7 @@
if ( queue && usingJrnl()){
dtokp = new journal::data_tok;
- // cct TODO -- delete this in the callback...
+ // deleted this in the callback...
dtokp->setSourceMessage (&message);
dtokp->set_rid(message.getPersistenceId()); // set the messageID into the Journal header (record-id)
@@ -798,7 +797,7 @@
while (!written)
{
journal::jcntl* jc = static_cast<journal::jcntl*>(queue->getExternalQueueStore());
- rhm::journal::iores eres = jc->enqueue_data_record(buff, size, dtokp);
+ rhm::journal::iores eres = jc->enqueue_data_record(buff, size, dtokp/*, txn->getXid(), false*/);
switch (eres)
{
case rhm::journal::RHM_IORES_SUCCESS:
@@ -828,7 +827,7 @@
/// cct message db
if (newId){ // only store in Bd if first time message is stored
Dbt data(buff,size);
- messageDb.put(txn, &messageId, &data, DB_NOOVERWRITE);
+ messageDb.put(txn->get(), &messageId, &data, DB_NOOVERWRITE);
}
}
}catch ( journal::jexception& e) {
@@ -862,22 +861,23 @@
}
try {
- if (txn->isTPC()) {
+
+ if (usingJrnl()){
+ // add queue* to the txn map..
+ if (ctxt) txn->addXidRecord(queue);
+ async_dequeue(ctxt, msg, queue);
+
+ } else if (txn->isTPC()) {
//if this is part of a 2pc transaction, then only record the dequeue now,
//it will be applied on commit
record2pcOp(dequeueXidDb, dynamic_cast<TPCTxnCtxt&>(*txn), messageId, queueId);
} else {
Dbt key (&messageId, sizeof(messageId));
Dbt value (&queueId, sizeof(queueId));
-
- if (usingJrnl()){
- async_dequeue(ctxt, msg, queue);
- }else{
- if (dequeue(txn->get(), key, value)) {
- msg.setPersistenceId(0);//clear id as we have now removed the message from the store
- msg.dequeueComplete(); // set dequeued for ack
- }
- }
+ if (dequeue(txn->get(), key, value)) {
+ msg.setPersistenceId(0);//clear id as we have now removed the message from the store
+ msg.dequeueComplete(); // set dequeued for ack
+ }
}
if (!ctxt) txn->commit();
@@ -890,7 +890,7 @@
}
}
-void BdbMessageStore::async_dequeue(TransactionContext* /*ctxt*/, PersistableMessage& msg, const PersistableQueue& queue)
+void BdbMessageStore::async_dequeue(TransactionContext* ctxt, PersistableMessage& msg, const PersistableQueue& queue)
{
unsigned aio_sleep_cnt = 0;
bool written = false;
@@ -900,11 +900,17 @@
ddtokp->set_dequeue_rid(messageIdSequence.next());
ddtokp->set_wstate(journal::data_tok::ENQ);
journal::jcntl* jc = static_cast<journal::jcntl*>(queue.getExternalQueueStore());
+ string tid;
+ if (ctxt){
+ TxnCtxt* txn = check(ctxt);
+ tid = txn->getXid();
+ }
+
while (!written)
{
rhm::journal::iores dres;
try {
- dres = jc->dequeue_data_record(ddtokp);
+ dres = jc->dequeue_data_record(ddtokp, tid);
} catch (rhm::journal::jexception& e) {
std::string str;
delete ddtokp;
@@ -982,23 +988,24 @@
try {
- //scroll through all records matching xid in apply and dequeue
- //using the message and queue id encoded in each value
- Cursor c;
- c.open(apply, txn.get());
-
StringDbt key(txn.getXid());
- IdPairDbt value;
+ if (!usingJrnl()){
+ //scroll through all records matching xid in apply and dequeue
+ //using the message and queue id encoded in each value
+ Cursor c;
+ c.open(apply, txn.get());
+ IdPairDbt value;
- for (int status = c->get(&key, &value, DB_SET); status == 0; status = c->get(&key, &value, DB_NEXT_DUP)) {
- dequeue(txn.get(), value.message, value.queue);
- }
- c.close();
+ for (int status = c->get(&key, &value, DB_SET); status == 0; status = c->get(&key, &value, DB_NEXT_DUP)) {
+ dequeue(txn.get(), value.message, value.queue);
+ }
+ c.close();
- //delete all records matching xid
+ //delete all records matching xid
+ discard.del(txn.get(), &key, 0);
+ apply.del(txn.get(), &key, 0);
+ }
prepareXidDb.del(txn.get(), &key, 0);
- discard.del(txn.get(), &key, 0);
- apply.del(txn.get(), &key, 0);
txn.commit();
} catch (std::exception& e) {
@@ -1033,6 +1040,8 @@
Dbt key ((void*) xid.data(), xid.length());
Dbt value(&dummy, sizeof(dummy));
+ // make sure all the data is written to disk before returning
+ txn->sync();
prepareXidDb.put(txn->get(), &key, &value, 0);
txn->commit();
Modified: store/trunk/cpp/lib/BdbMessageStore.h
===================================================================
--- store/trunk/cpp/lib/BdbMessageStore.h 2007-09-28 16:08:00 UTC (rev 952)
+++ store/trunk/cpp/lib/BdbMessageStore.h 2007-09-28 20:56:24 UTC (rev 953)
@@ -39,15 +39,12 @@
#include <iostream>
#include <boost/format.hpp>
#include <boost/ptr_container/ptr_list.hpp>
+#include <jrnl/jcntl.hpp>
namespace rhm {
namespace bdbstore {
using std::string;
- #define MAX_AIO_SLEEPS 300
- #define AIO_SLEEP_TIME 1000
-
-
/**
* An implementation of the MessageStore interface based on Berkeley DB
*/
@@ -89,7 +86,7 @@
void readXids(Db& db, std::set<string>& xids);
void readLockedMappings(Db& db, txn_lock_map& mappings);
TxnCtxt* check(qpid::broker::TransactionContext* ctxt);
- void store(const qpid::broker::PersistableQueue* queue, DbTxn* txn,
+ void store(const qpid::broker::PersistableQueue* queue, TxnCtxt* txn,
Dbt& messageId,
qpid::broker::PersistableMessage& message,
bool newId);
@@ -111,13 +108,14 @@
void open(Db& db, DbTxn* txn, const char* file, bool dupKey);
- // journal functions
- void createJrnlQueue(const qpid::broker::PersistableQueue& queue);
- string getJrnlDir(const qpid::broker::PersistableQueue& queue); //for exmaple /var/rhm/ + queueDir/
- string getJrnlDir(const char* queueName);
- inline bool usingJrnl() {return false;} // make configurable
- string getJrnlBaseDir();
+ // journal functions
+ void createJrnlQueue(const qpid::broker::PersistableQueue& queue);
+ string getJrnlDir(const qpid::broker::PersistableQueue& queue); //for exmaple /var/rhm/ + queueDir/
+ string getJrnlDir(const char* queueName);
+ static inline bool usingJrnl() {return false;} // make configurable
+ string getJrnlBaseDir();
+
public:
BdbMessageStore(const char* envpath = 0);
~BdbMessageStore();
Modified: store/trunk/cpp/lib/TxnCtxt.h
===================================================================
--- store/trunk/cpp/lib/TxnCtxt.h 2007-09-28 16:08:00 UTC (rev 952)
+++ store/trunk/cpp/lib/TxnCtxt.h 2007-09-28 20:56:24 UTC (rev 953)
@@ -26,21 +26,101 @@
#include "db-inc.h"
#include <qpid/broker/MessageStore.h>
+#include <qpid/sys/Mutex.h>
+#include <boost/shared_ptr.hpp>
+#include <vector>
+#include <jrnl/jcntl.hpp>
+#include <boost/format.hpp>
namespace rhm{
namespace bdbstore{
+// find a better place to put these
+#define MAX_AIO_SLEEPS 1000
+#define AIO_SLEEP_TIME 1000
+
+
class TxnCtxt : public qpid::broker::TransactionContext
{
- DbTxn* txn;
+private:
+ typedef std::set<const qpid::broker::PersistableQueue*> ipqdef;
+ ipqdef impactedQueues; // list of Queues used in the txn
+ static unsigned int count;
+ mutable qpid::sys::Mutex Lock;
+
+ unsigned int getCount() {
+ qpid::sys::Mutex::ScopedLock locker(Lock);
+ return ++count;
+ }
+ /**
+ * local txn id, if non XA.
+ */
+ std::string tid;
+ DbTxn* txn;
+
+ void completeTXN(bool commit){
+ for (TxnCtxt::ipqdef::iterator i = impactedQueues.begin(); i != impactedQueues.end(); i++) {
+ journal::jcntl* jc = static_cast<journal::jcntl*>((*i)->getExternalQueueStore());
+ if (jc) /* if using journal */
+ if (commit)
+ jc->commit_dtx(getXid());
+ else
+ jc->abort_dtx(getXid());
+ }
+ deleteXidRecord();
+ sync();
+ }
+
public:
- TxnCtxt() : txn(0) {}
+
+ TxnCtxt() : txn(0) {
+ tid = "rhm-tid" + getCount();
+ }
+
+ /**
+ * Call to make sure all the data for this txn is written to safe store
+ *
+ *@return if the data sucessfully synced.
+ */
+ void sync(){
+ bool allWritten = true;
+ bool firstloop = true;
+ unsigned aio_sleep_cnt = 0;
+ while (!allWritten){
+ if (!firstloop) ::usleep(AIO_SLEEP_TIME);
+ allWritten = true;
+ unsigned qcnt = 0;
+ for (TxnCtxt::ipqdef::iterator i = impactedQueues.begin(); i != impactedQueues.end(); i++) {
+ qcnt ++;
+ journal::jcntl* jc = static_cast<journal::jcntl*>((*i)->getExternalQueueStore());
+ if (jc && !(jc->is_dtx_synced(getXid())))
+ {
+ if (firstloop)
+ jc->flush();
+ allWritten = false;
+ jc->get_wr_events();
+ }
+ }
+ firstloop = false;
+ if (++aio_sleep_cnt > MAX_AIO_SLEEPS*qcnt)
+ {
+ THROW_STORE_EXCEPTION("Store error, disk time out on sync for:" + getXid());
+ }
+ }
+ }
+
virtual ~TxnCtxt() { if(txn) abort(); }
void begin(DbEnv& env){ env.txn_begin(0, &txn, 0); }
- void commit(){ txn->commit(0); txn = 0; }
- void abort(){ txn->abort(); txn = 0; }
+ void commit(){ txn->commit(0); completeTXN(true); txn = 0; sync();}
+ void abort(){ txn->abort(); completeTXN(false); txn = 0; sync();}
DbTxn* get(){ return txn; }
virtual bool isTPC() { return false; }
+ virtual const std::string& getXid() { return tid; }
+
+ void deleteXidRecord(){ impactedQueues.clear(); }
+ void addXidRecord(const qpid::broker::PersistableQueue& queue){
+ impactedQueues.insert(&queue); }
+
};
class TPCTxnCtxt : public TxnCtxt, public qpid::broker::TPCTransactionContext
@@ -49,7 +129,7 @@
public:
TPCTxnCtxt(const std::string& _xid) : xid(_xid) {}
virtual bool isTPC() { return true; }
- const std::string& getXid() { return xid; }
+ virtual const std::string& getXid() { return xid; }
};
}}
17 years, 5 months
- 1
- 0
rhmessaging commits: r952 - store/trunk/cpp/tests/jrnl.
by rhmessaging-commits＠lists.jboss.org
Author: kpvdr
Date: 2007-09-28 12:08:00 -0400 (Fri, 28 Sep 2007)
New Revision: 952
Modified:
store/trunk/cpp/tests/jrnl/Makefile.rtest
Log:
Fixed makefile typo
Modified: store/trunk/cpp/tests/jrnl/Makefile.rtest
===================================================================
--- store/trunk/cpp/tests/jrnl/Makefile.rtest 2007-09-28 16:04:41 UTC (rev 951)
+++ store/trunk/cpp/tests/jrnl/Makefile.rtest 2007-09-28 16:08:00 UTC (rev 952)
@@ -94,7 +94,7 @@
@echo " jrtest: Make the journal test suite for read tests"
@echo " jwtest: Make the journal test suite for write tests"
@echo " jrwtest: Make the journal test suite for simultaneous write/read tests"
- @echo " jwperf:" Make the jorunal test suite for performance write tests"
+ @echo " jwperf: Make the jorunal test suite for performance write tests"
@echo " --- Testing options ---"
@echo " check: Run complete regression test suite; compiles with special flags"
@echo " --- Documentation options ---"
17 years, 5 months
- 1
- 0
rhmessaging commits: r951 - store/trunk/cpp/tests/jrnl.
by rhmessaging-commits＠lists.jboss.org
Author: kpvdr
Date: 2007-09-28 12:04:41 -0400 (Fri, 28 Sep 2007)
New Revision: 951
Modified:
store/trunk/cpp/tests/jrnl/Makefile.rtest
store/trunk/cpp/tests/jrnl/jtest.cpp
Log:
Modifications to support performance tests in jtest.cpp
Modified: store/trunk/cpp/tests/jrnl/Makefile.rtest
===================================================================
--- store/trunk/cpp/tests/jrnl/Makefile.rtest 2007-09-28 16:02:33 UTC (rev 950)
+++ store/trunk/cpp/tests/jrnl/Makefile.rtest 2007-09-28 16:04:41 UTC (rev 951)
@@ -36,11 +36,12 @@
# read. Use together with RHM_JOWRITE if test reads from previously written
# journal. Ensure journal contains valid entries in all files. NOTE: ONLY USE
# FOR PERFORMANCE TESTS!
+# RHM_TIMERS: Enable and show times for initialize and run of a test.
# Prefix each define with -D. (eg RHM_DEFINES = -DRHM_JOWRITE -DRHM_WRONLY)
SHELL = /bin/bash
-QPID_HOME_DIR = $(HOME)/qpid
+QPID_HOME_DIR = $(HOME)/rhm/qpid
RHM_JRNL_SRC_DIR = ../../lib/jrnl
RHM_JRNL_DOC_DIR = ../../docs
@@ -93,6 +94,7 @@
@echo " jrtest: Make the journal test suite for read tests"
@echo " jwtest: Make the journal test suite for write tests"
@echo " jrwtest: Make the journal test suite for simultaneous write/read tests"
+ @echo " jwperf:" Make the jorunal test suite for performance write tests"
@echo " --- Testing options ---"
@echo " check: Run complete regression test suite; compiles with special flags"
@echo " --- Documentation options ---"
@@ -116,6 +118,9 @@
jrwtest: RHM_DEFINES = -DRHM_CLEAN -DRHM_TESTVALS
jrwtest: $(JTEST_FILES)
+jwperf: RHM_DEFINES = -DRHM_WRONLY -DRHM_TIMERS
+jwperf: $(JTEST_FILES)
+
check:
@./rtest
Modified: store/trunk/cpp/tests/jrnl/jtest.cpp
===================================================================
--- store/trunk/cpp/tests/jrnl/jtest.cpp 2007-09-28 16:02:33 UTC (rev 950)
+++ store/trunk/cpp/tests/jrnl/jtest.cpp 2007-09-28 16:04:41 UTC (rev 951)
@@ -53,6 +53,9 @@
cout << "jtest: starting test " << tnum << endl;
jtest t;
t.initialize(tnum, test_csv_file);
+#ifdef RHM_TIMERS
+ usleep(1000000); // Sleep for 1 sec so disks can rest after initialize
+#endif
t.run();
t.report();
t.finalize();
@@ -118,7 +121,9 @@
void
jtest::initialize(const targs& ta) throw (rhm::journal::jexception)
{
-// gettimeofday(_start_time, _tz_ptr);
+#ifdef RHM_TIMERS
+ gettimeofday(_start_time, _tz_ptr);
+#endif
p_args = new msg_producer::_p_args(_jc, ta._num_msgs, ta._min_msg_size, ta._max_msg_size,
ta._auto_deq);
@@ -142,15 +147,19 @@
_dtok_master_list.push_back(dtp);
}
-// gettimeofday(_end_time, _tz_ptr);
-// string str;
-// cout << "initialize(): " << report_time(str) << endl;
+#ifdef RHM_TIMERS
+ gettimeofday(_end_time, _tz_ptr);
+ string str;
+ cout << "initialize(): " << report_time(str) << endl;
+#endif
}
void
jtest::run() throw (rhm::journal::jexception)
{
+#ifdef RHM_TIMERS
gettimeofday(_start_time, _tz_ptr);
+#endif
// === Do something spectacular here... ===
@@ -191,9 +200,11 @@
// === End: Do something spectacular here... ===
+#ifdef RHM_TIMERS
gettimeofday(_end_time, _tz_ptr);
string str;
-// cout << "run(): " << report_time(str) << endl;
+ cout << "run(): " << report_time(str) << endl;
+#endif
}
void
@@ -405,7 +416,10 @@
#ifdef RHM_RDONLY
cout << "*** WARNING: Compiled with RHM_RDONLY defined." << endl;
#endif
-#if defined(RHM_CLEAN) || defined(RHM_TESTVALS) || defined(RHM_JOWRITE) || defined(RHM_WRONLY) || defined(RHM_RDONLY)
+#ifdef RHM_TIMERS
+ cout << "*** WARNING: Compiled with RHM_TIMERS defined." << endl;
+#endif
+#if defined(RHM_CLEAN) || defined(RHM_TESTVALS) || defined(RHM_JOWRITE) || defined(RHM_WRONLY) || defined(RHM_RDONLY) || defined(RHM_TIMERS)
cout << endl;
#endif
17 years, 5 months
- 1
- 0
rhmessaging commits: r950 - in store/trunk/cpp: tests/jrnl and 1 other directory.
by rhmessaging-commits＠lists.jboss.org
Author: kpvdr
Date: 2007-09-28 12:02:33 -0400 (Fri, 28 Sep 2007)
New Revision: 950
Added:
store/trunk/cpp/tests/jrnl/janalyze.py
Removed:
store/trunk/cpp/tests/jrnl/ftest.py
Modified:
store/trunk/cpp/lib/jrnl/jcntl.hpp
store/trunk/cpp/tests/jrnl/Makefile.am
store/trunk/cpp/tests/jrnl/rtest
store/trunk/cpp/tests/jrnl/tests.ods
store/trunk/cpp/tests/jrnl/wtests.csv
Log:
Updated python analysis tool to handle XID
Modified: store/trunk/cpp/lib/jrnl/jcntl.hpp
===================================================================
--- store/trunk/cpp/lib/jrnl/jcntl.hpp 2007-09-27 19:29:25 UTC (rev 949)
+++ store/trunk/cpp/lib/jrnl/jcntl.hpp 2007-09-28 16:02:33 UTC (rev 950)
@@ -295,8 +295,6 @@
* \param tot_data_len Total data length.
* \param this_data_len Amount to be written in this enqueue operation.
* \param dtokp Pointer to data token which contains the details of the enqueue operation.
- * \param xid String containing xid. An empty string (i.e. length=0) will be considered
- * non-transactional.
* \param transient Flag indicating transient persistence (ie, ignored on recover).
*
* \exception TODO
@@ -304,6 +302,19 @@
const iores enqueue_data_record(const void* const data_buff, const size_t tot_data_len,
const size_t this_data_len, data_tok* dtokp, const bool transient = false)
throw (jexception);
+ /**
+ * \brief Enqueue data.
+ *
+ * \param data_buff Pointer to data to be enqueued for this enqueue operation.
+ * \param tot_data_len Total data length.
+ * \param this_data_len Amount to be written in this enqueue operation.
+ * \param dtokp Pointer to data token which contains the details of the enqueue operation.
+ * \param xid String containing xid. An empty string (i.e. length=0) will be considered
+ * non-transactional.
+ * \param transient Flag indicating transient persistence (ie, ignored on recover).
+ *
+ * \exception TODO
+ */
const iores enqueue_data_record(const void* const data_buff, const size_t tot_data_len,
const size_t this_data_len, data_tok* dtokp, const std::string& xid,
const bool transient = false) throw (jexception);
@@ -403,12 +414,27 @@
*
* \param dtokp Pointer to data_tok instance for this data, used to track state of data
* through journal.
+ *
+ * \exception TODO
+ */
+ const iores dequeue_data_record(data_tok* const dtokp) throw (jexception);
+
+ /**
+ * \brief Dequeues (marks as no longer needed) data record in journal.
+ *
+ * Dequeues (marks as no longer needed) data record in journal as part of a transaction.
+ * Note that it is possible to use the same data token instance used to enqueue this data;
+ * it contains the RID needed to correctly mark this data as dequeued in the journal.
+ * Otherwise the RID of the record to be dequeued and the write state of ENQ must be
+ * manually set in a new or reset instance of data_tok.
+ *
+ * \param dtokp Pointer to data_tok instance for this data, used to track state of data
+ * through journal.
* \param xid String containing xid. An empty string (i.e. length=0) will be considered
* non-transactional.
*
* \exception TODO
*/
- const iores dequeue_data_record(data_tok* const dtokp) throw (jexception);
const iores dequeue_data_record(data_tok* const dtokp, const std::string& xid)
throw (jexception);
Modified: store/trunk/cpp/tests/jrnl/Makefile.am
===================================================================
--- store/trunk/cpp/tests/jrnl/Makefile.am 2007-09-27 19:29:25 UTC (rev 949)
+++ store/trunk/cpp/tests/jrnl/Makefile.am 2007-09-28 16:02:33 UTC (rev 950)
@@ -50,4 +50,4 @@
libdlclose_noop_la_SOURCES = ../dlclose_noop.c
libdlclose_noop_la_LDFLAGS = -module -rpath $(abs_builddir)
-EXTRA_DIST = $(TESTS) rtest ftest.py tests.ods rtests.csv rwtests.csv wtests.csv
+EXTRA_DIST = $(TESTS) rtest janalyze.py tests.ods rtests.csv rwtests.csv wtests.csv
Deleted: store/trunk/cpp/tests/jrnl/ftest.py
===================================================================
--- store/trunk/cpp/tests/jrnl/ftest.py 2007-09-27 19:29:25 UTC (rev 949)
+++ store/trunk/cpp/tests/jrnl/ftest.py 2007-09-28 16:02:33 UTC (rev 950)
@@ -1,433 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright (C) 2007 Red Hat Inc.
-#
-# This file is part of Red Hat Messaging.
-#
-# Red Hat Messaging is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
-# USA
-#
-# The GNU Lesser General Public License is available in the file COPYING.
-
-import sys
-from struct import unpack, calcsize
-from time import gmtime, strftime
-
-dblk_size = 128
-sblk_size = 4 * dblk_size
-file_size = (3072 + 1) * sblk_size
-num_files = 8
-hdr_ver = 1
-
-def load(f, klass):
- args = load_args(f, klass)
- subclass = klass.descriminate(args)
- result = subclass(*args)
- if subclass != klass:
- result.init(f, *load_args(f, subclass))
- result.skip(f)
- return result;
-
-def load_args(f, klass):
- size = calcsize(klass.format)
- foffs = f.tell(),
- bin = f.read(size)
- if len(bin) != size:
- raise Exception("end of file")
- return foffs + unpack(klass.format, bin)
-
-def size_blks(size, blk_size):
- return (size + blk_size - 1)/blk_size
-
-def rem_in_blk(f, blk_size):
- foffs = f.tell()
- return (size_blks(f.tell(), blk_size) * blk_size) - foffs;
-
-def compl(n):
- return ~n & 0xffffffff
-
-def file_full(f):
- return f.tell() >= file_size
-
-
-class Sizeable:
-
- def size(self):
- classes = [self.__class__]
-
- size = 0
- while classes:
- cls = classes.pop()
- if hasattr(cls, "format"):
- size += calcsize(cls.format)
- classes.extend(cls.__bases__)
-
- return size
-
-class Hdr(Sizeable):
-
- format = '=4sBBHQ'
-
- @staticmethod
- def descriminate(args):
- return CLASSES.get(args[1][-1], Hdr)
-
- def __init__(self, foffs, magic, ver, end, flags, rid):
- self.foffs = foffs
- self.magic = magic
- self.ver = ver
- self.end = end
- self.flags = flags
- self.rid = rid
- if self.magic[-1] not in ['0x00', 'd', 'e', 'f', 'x']:
- error = 3
-
- def __str__(self):
- if self.empty():
- return '0x%08x: <empty>' % (self.foffs)
- if self.magic[-1] == 'x':
- return '0x%08x: \"%s\"' % (self.foffs, self.magic)
- if self.magic[-1] in ['d', 'e', 'f', 'x']:
- return '0x%08x: \"%s\" v=%d e=%d f=0x%04x rid=%d' % (self.foffs, self.magic, self.ver, self.end, self.flags, self.rid)
- return '0x%08x: <error, unknown magic \"%s\" (possible overwrite boundary?)>' % (self.foffs, self.magic)
-
- def empty(self):
- return self.magic == '\x00'*4
-
- def skip(self, f):
- f.read(rem_in_blk(f, dblk_size))
-
- def check(self):
- if self.empty() or self.magic[-1] not in ['d', 'e', 'f', 'x']:
- return True
- if self.ver != hdr_ver and self.magic[-1] != 'x':
- raise Exception('%s: Invalid header version: found %d, expected %d.' % (self, self.ver, hdr_ver))
- return False
-
-
-
-class FileHdr(Hdr):
-
- format = '=I4x3Q'
-
- def init(self, f, foffs, fid, fro, time_sec, time_ns):
- self.fid = fid
- self.fro = fro
- self.time_sec = time_sec
- self.time_ns = time_ns
-
- def __str__(self):
- return '%s fid=%d fro=0x%08x t=%s' % (Hdr.__str__(self), self.fid, self.fro, self.timestamp_str())
-
- def skip(self, f):
- f.read(rem_in_blk(f, sblk_size))
-
- def timestamp(self):
- return (self.time_sec, self.time_ns)
-
- def timestamp_str(self):
- ts = gmtime(self.time_sec)
- fstr = '%%a %%b %%d %%H:%%M:%%S.%d %%Y' % (self.time_ns)
- return strftime(fstr, ts)
-
-
-class DeqHdr(Hdr):
-
- format = '=QQ'
-
- def init(self, f, foffs, deq_rid, xid):
- self.deq_rid = deq_rid
- self.xid = xid
-
- def __str__(self):
- return '%s d=%d x=%d' % (Hdr.__str__(self), self.deq_rid, self.xid)
-
-
-class RecTail(Sizeable):
-
- format = '=IQ'
-
- def __init__(self, foffs, magic_inv, rid):
- self.foffs = foffs
- self.magic_inv = magic_inv
- self.rid = rid
-
- def __str__(self):
- return '0x%04x rid=%d' % (self.magic_inv, self.rid)
-
- def skip(self, f):
- f.read(rem_in_blk(f, dblk_size))
-
-
-class EnqRec(Hdr):
-
- format = '=QQ'
-
- def init(self, f, foffs, xid, dsize):
- self.xid = xid
- self.dsize = dsize
- self.data = None
- self.enq_tail = None
- self.data_complete = False
- self.tail_complete = False
- self.tail_bin = None
- self.tail_offs = 0
- self.load(f)
-
- def load(self, f):
- if not self.data_complete:
- self.load_data(f)
- if self.data_complete and not self.tail_complete:
- self.load_tail(f)
- return self.complete()
-
- def load_data(self, f):
- if self.data == None:
- loaded = 0
- else:
- loaded = len(self.data)
- foverflow = f.tell() + self.dsize - loaded > file_size
- if foverflow:
- rsize = file_size - f.tell()
- else:
- rsize = self.dsize - loaded
- bin = f.read(rsize)
- if self.data == None:
- self.data = unpack('%ds' % (rsize), bin)[0]
- else:
- self.data = self.data + unpack('%ds' % (rsize), bin)[0]
- self.data_complete = not foverflow
- return self.data_complete
-
- def load_tail(self, f):
- if self.tail_bin == None:
- self.tail_offs = f.tell()
- self.tail_bin = f.read(calcsize(RecTail.format))
- if len(self.tail_bin) != calcsize(RecTail.format):
- return
- else:
- self.tail_bin += f.read(calcsize(RecTail.format) - len(self.tail_bin))
- self.enq_tail = RecTail(self.tail_offs, *unpack(RecTail.format, self.tail_bin))
- magic_int = 0
- inv_magic_int_1 = 0
- for i in range(0,4):
- magic_int = (magic_int << 8) + ord(self.magic[3-i]) # Little endian only
- if self.enq_tail.magic_inv != compl(magic_int) or self.enq_tail.rid != self.rid:
- print " > %s" % self
- raise Exception('Invalid enqueue record tail (magic=0x%08x; rid=%d) at 0x%08x' % (self.enq_tail.magic_inv, self.enq_tail.rid, self.enq_tail.foffs))
- self.enq_tail.skip(f)
- self.tail_complete = True
-
- def complete(self):
- return self.data_complete and self.tail_complete
-
- def __str__(self):
- if len(self.data) > 50:
- dstr = self.data[:20] + ' ... ' + self.data[-20:]
- else:
- dstr = self.data
- if self.enq_tail == None:
- return '%s s=%d data=\"%s\" [no tail]' % (Hdr.__str__(self), self.dsize, dstr)
- return '%s x=%d s=%d data=\"%s\" %s' % (Hdr.__str__(self), self.xid, self.dsize, dstr, self.enq_tail)
-
-
-class Main:
-
- def __init__(self, tfile, tnum):
- tparams = self.get_test(tfile, tnum)
- if tparams == None:
- raise Exception('Test %d not found in file %s' % (tnum, tfile))
- self.num_msgs = tparams['num_msgs']
- if tparams['min_size'] == tparams['max_size']:
- self.msg_len = tparams['max_size']
- else:
- self.msg_len = 0
- self.auto_deq = tparams['auto_deq']
- self.file_start = 0
- self.file_num = 0
- self.fro = 0x200
- self.enqueued = {}
- self.msg_cnt = 0
- self.fhdr = None
- self.f = None
- self.first_rec = False
- self.last_file = False
- self.last_rid = -1
-
-
- def run(self):
- for msg_num in range(self.num_msgs):
- start_info = self.analyze_files()
- stop = self.advance_file(*start_info)
- while not stop:
- warn = ''
- if file_full(self.f):
- stop = self.advance_file()
- if stop:
- break
- hdr = load(self.f, Hdr)
- if hdr.empty():
- stop = True;
- break
- if hdr.check():
- stop = True;
- if self.first_rec:
- if self.fhdr.fro != hdr.foffs:
- raise Exception('File header first record offset mismatch: fro=0x%08x; rec_offs=0x%08x' % (self.fhdr.fro, hdr.foffs))
- else:
- print ' * fro ok: 0x%08x' % self.fhdr.fro
- self.first_rec = False
- if isinstance(hdr, EnqRec) and not stop:
- while not hdr.complete():
- stop = self.advance_file()
- if stop:
- break
- hdr.load(self.f)
- if self.msg_len > 0 and len(hdr.data) != self.msg_len:
- raise Exception('Message length (%d) incorrect: expected %d' % (len(hdr.data), self.msg_len))
- stop = not self.check_rid(hdr)
- if stop:
- warn = ' (WARNING: rid out of order, last rid = %d - could be overwrite boundary.)' % hdr.rid
- else:
- self.msg_cnt += 1
- if self.auto_deq:
- self.enqueued[hdr.rid] = hdr
- elif isinstance(hdr, DeqHdr) and not stop:
- if self.auto_deq:
- if hdr.deq_rid in self.enqueued:
- del self.enqueued[hdr.deq_rid]
- else:
- warn = ' (WARNING: dequeue rid %d not found in enqueued records)' % hdr.deq_rid
- stop = not self.check_rid(hdr)
- if stop:
- warn = ' (WARNING: rid out of order, last rid = %d - could be overwrite boundary.)' % hdr.rid
- else:
- warn = 'WARNING: Dequeue record rid=%d found in non-dequeue test - ignoring.' % hdr.rid
- print ' > %s%s' % (hdr, warn)
- if not stop:
- stop = (self.last_file and hdr.check()) or hdr.empty() or self.fhdr.empty() or msg_num >= self.num_msgs
- if stop:
- break
-
- def report(self):
- if self.msg_cnt != self.num_msgs:
- print 'WARNING: Found %d messages; %d expected.' % (self.msg_cnt, self.num_msgs)
- if len(self.enqueued) > 0:
- print 'Remaining enqueued records: ', len(self.enqueued)
- for h in self.enqueued:
- print self.enqueued[h]
- print 'WARNING: Enqueue-Dequeue mismatch, %d enqueued records remain.' % len(self.enqueued)
- print 'Test passed; %d records processed.' % self.msg_cnt
-
- def advance_file(self, *start_info):
- seek_flag = False
- if len(start_info) == 3:
- self.file_start = self.file_num = start_info[0]
- self.fro = start_info[2]
- seek_flag = True
- if self.f != None and file_full(self.f):
- self.file_num = self.incr_fnum()
- if self.file_num == self.file_start:
- return True
- if self.file_start == 0:
- self.last_file = self.file_num == num_files - 1
- else:
- self.last_file = self.file_num == self.file_start - 1
- if self.file_num < 0 or self.file_num >= num_files:
- raise Exception('Bad file number %d' % self.file_num)
- file_name = 'jdata/test.%04d.jdat' % self.file_num
- self.f = open(file_name)
- self.fhdr = load(self.f, Hdr)
- if seek_flag and self.f.tell() != self.fro:
- self.f.seek(self.fro)
- self.first_rec = True
- print file_name, ": ", self.fhdr
- return False
-
- def incr_fnum(self):
- self.file_num += 1
- if self.file_num >= num_files:
- self.file_num = 0;
- return self.file_num
-
- def check_rid(self, hdr):
- if self.last_rid != -1 and hdr.rid != self.last_rid + 1:
- return False
- self.last_rid = hdr.rid
- return True
-
- def get_test(self, filename, tnum):
- f=open(filename, 'r')
- for l in f:
- sl = l.strip().split(',')
- if len(sl[1]) > 0: #Comments are in col 0, remaining cols are empty
- try:
- if (int(sl[0]) == tnum):
- return { 'num_msgs':int(sl[1]),
- 'min_size':int(sl[2]),
- 'max_size':int(sl[3]),
- 'auto_deq':sl[4] != 'FALSE' }
- except Exception:
- pass
- return None
-
- def analyze_files(self):
- fname = ''
- fnum = -1
- rid = -1
- fro = -1
- tss = ''
- print 'Analyzing journal files:'
- for i in range(0, num_files):
- file_name = 'jdata/test.%04d.jdat' % i
- f = open(file_name)
- fhdr = load(f, Hdr)
- if fhdr.empty():
- break
- if (rid == -1 or fhdr.rid < rid) and fhdr.fro > 0:
- fname = file_name
- fnum = i
- rid = fhdr.rid
- fro = fhdr.fro
- tss = fhdr.timestamp_str()
- print ' %s: rid=%d, fro=0x%08x ts=%s' % (file_name, fhdr.rid, fhdr.fro, fhdr.timestamp_str())
- if fnum < 0 or rid < 0 or fro < 0:
- raise Exception('All journal files empty')
- print 'Lowest rid found in file %s: rid=%d, fro=0x%08x ts=%s' % (fname, rid, fro, tss)
- return (fnum, rid, fro)
-
-
-CLASSES = {
- "e": EnqRec,
- "d": DeqHdr,
- "f": FileHdr
-}
-
-if len(sys.argv) != 3:
- print 'Incorrect number of command-line arguments (expected 2, found %d).' % (len(sys.argv) - 1)
- print 'Usage: ftest test-file test-number'
- raise Exception('Bad command-line arguments')
-if not sys.argv[2].isdigit():
- print 'Illegal argument (\"%s\"): test-number must be a number' % a
- print 'Usage: ftest test-file test-number'
- raise Exception('Illegal argument value')
-
-tnum = int(sys.argv[2])
-print 'Test number: %d' % tnum
-print 'Test file: %s' %sys.argv[1]
-
-m = Main(sys.argv[1], tnum)
-m.run()
-m.report()
Copied: store/trunk/cpp/tests/jrnl/janalyze.py (from rev 947, store/trunk/cpp/tests/jrnl/ftest.py)
===================================================================
--- store/trunk/cpp/tests/jrnl/janalyze.py (rev 0)
+++ store/trunk/cpp/tests/jrnl/janalyze.py 2007-09-28 16:02:33 UTC (rev 950)
@@ -0,0 +1,540 @@
+#!/usr/bin/env python
+
+# Copyright (C) 2007 Red Hat Inc.
+#
+# This file is part of Red Hat Messaging.
+#
+# Red Hat Messaging is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
+# USA
+#
+# The GNU Lesser General Public License is available in the file COPYING.
+
+import sys
+from struct import unpack, calcsize
+from time import gmtime, strftime
+
+dblk_size = 128
+sblk_size = 4 * dblk_size
+file_size = (3072 + 1) * sblk_size
+num_files = 8
+hdr_ver = 1
+
+def load(f, klass):
+ args = load_args(f, klass)
+ subclass = klass.descriminate(args)
+ result = subclass(*args)
+ if subclass != klass:
+ result.init(f, *load_args(f, subclass))
+ result.skip(f)
+ return result;
+
+def load_args(f, klass):
+ size = calcsize(klass.format)
+ foffs = f.tell(),
+ bin = f.read(size)
+ if len(bin) != size:
+ raise Exception("end of file")
+ return foffs + unpack(klass.format, bin)
+
+def size_blks(size, blk_size):
+ return (size + blk_size - 1)/blk_size
+
+def rem_in_blk(f, blk_size):
+ foffs = f.tell()
+ return (size_blks(f.tell(), blk_size) * blk_size) - foffs;
+
+def file_full(f):
+ return f.tell() >= file_size
+
+def print_xid(xidsize, xid):
+ if xid == None:
+ if xidsize > 0:
+ raise Exception('Inconsistent XID size: xidsize=%d, xid=None' % xidsize)
+ return ''
+ if len(xid) > 25:
+ xidstr = xid[:10] + ' ... ' + xid[-10:]
+ else:
+ xidstr = xid
+ if xidsize != len(xid):
+ raise Exception('Inconsistent XID size: xidsize=%d, xid(%d)=\"%s\"' % (xidsize, len(xid), xidstr))
+ return 'xid(%d)=\"%s\" ' % (xidsize, xidstr)
+
+def print_data(dsize, data):
+ if len(data) > 25:
+ datastr = data[:10] + ' ... ' + data[-10:]
+ else:
+ datastr = data
+ if dsize != len(data):
+ raise Exception('Inconsistent data size: dsize=%d, data(%d)=\"%s\"' % (dsize, len(data), datastr))
+ return 'data(%d)=\"%s\" ' % (dsize, datastr)
+
+def inv_str(s):
+ si = ''
+ for i in range(0,len(s)):
+ si += chr(~ord(s[i]) & 0xff)
+ return si
+
+def load_file_data(f, size, data):
+ if size == 0:
+ return (data, True)
+ if data == None:
+ loaded = 0
+ else:
+ loaded = len(data)
+ foverflow = f.tell() + size - loaded > file_size
+ if foverflow:
+ rsize = file_size - f.tell()
+ else:
+ rsize = size - loaded
+ bin = f.read(rsize)
+ if data == None:
+ data = unpack('%ds' % (rsize), bin)[0]
+ else:
+ data = data + unpack('%ds' % (rsize), bin)[0]
+ return (data, not foverflow)
+
+def load_tail(self, f):
+ if self.tail_bin == None:
+ self.tail_offs = f.tell()
+ self.tail_bin = f.read(calcsize(RecTail.format))
+ if len(self.tail_bin) != calcsize(RecTail.format):
+ return
+ else:
+ self.tail_bin += f.read(calcsize(RecTail.format) - len(self.tail_bin))
+ self.enq_tail = RecTail(self.tail_offs, *unpack(RecTail.format, self.tail_bin))
+ if self.enq_tail.magic_inv != inv_str(self.magic) or self.enq_tail.rid != self.rid:
+ print " > %s" % self
+ raise Exception('Invalid enqueue record tail (magic=0x%08x; rid=%d) at 0x%08x' % (self.enq_tail.magic_inv, self.enq_tail.rid, self.enq_tail.foffs))
+ self.enq_tail.skip(f)
+ self.tail_complete = True
+
+
+class Sizeable:
+
+ def size(self):
+ classes = [self.__class__]
+
+ size = 0
+ while classes:
+ cls = classes.pop()
+ if hasattr(cls, "format"):
+ size += calcsize(cls.format)
+ classes.extend(cls.__bases__)
+
+ return size
+
+class Hdr(Sizeable):
+
+ format = '=4sBBHQ'
+
+ @staticmethod
+ def descriminate(args):
+ return CLASSES.get(args[1][-1], Hdr)
+
+ def __init__(self, foffs, magic, ver, end, flags, rid):
+ self.foffs = foffs
+ self.magic = magic
+ self.ver = ver
+ self.end = end
+ self.flags = flags
+ self.rid = rid
+ if self.magic[-1] not in ['0x00', 'a', 'c', 'd', 'e', 'f', 'x']:
+ error = 3
+
+ def __str__(self):
+ if self.empty():
+ return '0x%08x: <empty>' % (self.foffs)
+ if self.magic[-1] == 'x':
+ return '0x%08x: \"%s\"' % (self.foffs, self.magic)
+ if self.magic[-1] in ['a', 'c', 'd', 'e', 'f', 'x']:
+ return '0x%08x: \"%s\" v=%d e=%d f=0x%04x rid=%d' % (self.foffs, self.magic, self.ver, self.end, self.flags, self.rid)
+ return '0x%08x: <error, unknown magic \"%s\" (possible overwrite boundary?)>' % (self.foffs, self.magic)
+
+ def empty(self):
+ return self.magic == '\x00'*4
+
+ def skip(self, f):
+ f.read(rem_in_blk(f, dblk_size))
+
+ def check(self):
+ if self.empty() or self.magic[-1] not in ['a', 'c', 'd', 'e', 'f', 'x']:
+ return True
+ if self.ver != hdr_ver and self.magic[-1] != 'x':
+ raise Exception('%s: Invalid header version: found %d, expected %d.' % (self, self.ver, hdr_ver))
+ return False
+
+
+
+class FileHdr(Hdr):
+
+ format = '=I4x3Q'
+
+ def init(self, f, foffs, fid, fro, time_sec, time_ns):
+ self.fid = fid
+ self.fro = fro
+ self.time_sec = time_sec
+ self.time_ns = time_ns
+
+ def __str__(self):
+ return '%s fid=%d fro=0x%08x t=%s' % (Hdr.__str__(self), self.fid, self.fro, self.timestamp_str())
+
+ def skip(self, f):
+ f.read(rem_in_blk(f, sblk_size))
+
+ def timestamp(self):
+ return (self.time_sec, self.time_ns)
+
+ def timestamp_str(self):
+ ts = gmtime(self.time_sec)
+ fstr = '%%a %%b %%d %%H:%%M:%%S.%d %%Y' % (self.time_ns)
+ return strftime(fstr, ts)
+
+
+class DeqHdr(Hdr):
+
+ format = '=QQ'
+
+ def init(self, f, foffs, deq_rid, xidsize):
+ self.deq_rid = deq_rid
+ self.xidsize = xidsize
+ self.xid = None
+ self.deq_tail = None
+ self.xid_complete = False
+ self.tail_complete = False
+ self.load(f)
+
+ def load(self, f):
+ if self.xidsize == 0:
+ self.xid_complete = True
+ self.tail_complete = True
+ else:
+ if not self.xid_complete:
+ ret = load_file_data(f, self.xidsize, self.xid)
+ self.xid = ret[0]
+ self.xid_complete = ret[1]
+ if self.xid_complete and not self.tail_complete:
+ ret = load_file_data(f, calcsize(RecTail.format), self.tail_bin)
+ self.tail_bin = ret[0]
+ if ret[1]:
+ self.enq_tail = RecTail(self.tail_offs, *unpack(RecTail.format, self.tail_bin))
+ if self.enq_tail.magic_inv != inv_str(self.magic) or self.enq_tail.rid != self.rid:
+ print " > %s" % self
+ raise Exception('Invalid enqueue record tail (magic=0x%08x; rid=%d) at 0x%08x' % (self.enq_tail.magic_inv, self.enq_tail.rid, self.enq_tail.foffs))
+ self.enq_tail.skip(f)
+ self.tail_complete = ret[1]
+ return self.complete()
+
+ def complete(self):
+ return self.xid_complete and self.tail_complete
+
+ def __str__(self):
+ return '%s %sdrid=%d' % (Hdr.__str__(self), print_xid(self.xidsize, self.xid), self.deq_rid)
+
+
+class TxHdr(Hdr):
+
+ format = '=Q'
+
+ def init(self, f, foffs, xidsize):
+ self.xidsize = xid
+ self.xid = None
+ self.tx_tail = None
+ self.xid_complete = False
+ self.tail_complete = False
+ self.load(f)
+
+ def load(self, f):
+ if not self.xid_complete:
+ ret = load_file_data(f, self.xidsize, self.xid)
+ self.xid = ret[0]
+ self.xid_complete = ret[1]
+ if self.xid_complete and not self.tail_complete:
+ ret = load_file_data(f, calcsize(RecTail.format), self.tail_bin)
+ self.tail_bin = ret[0]
+ if ret[1]:
+ self.enq_tail = RecTail(self.tail_offs, *unpack(RecTail.format, self.tail_bin))
+ if self.enq_tail.magic_inv != inv_str(self.magic) or self.enq_tail.rid != self.rid:
+ print " > %s" % self
+ raise Exception('Invalid enqueue record tail (magic=0x%08x; rid=%d) at 0x%08x' % (self.enq_tail.magic_inv, self.enq_tail.rid, self.enq_tail.foffs))
+ self.enq_tail.skip(f)
+ self.tail_complete = ret[1]
+ return self.complete()
+
+ def complete(self):
+ return self.xid_complete and self.tail_complete
+
+ def __str__(self):
+ return '%s x=%d xid=\"%s\"' % (Hdr.__str__(self), self.xidsize, self.xid)
+
+
+class RecTail(Sizeable):
+
+ format = '=4sQ'
+
+ def __init__(self, foffs, magic_inv, rid):
+ self.foffs = foffs
+ self.magic_inv = magic_inv
+ self.rid = rid
+
+ def __str__(self):
+ magic = inv_str(self.magic_inv)
+ return '[\"%s\" rid=%d]' % (magic, self.rid)
+
+ def skip(self, f):
+ f.read(rem_in_blk(f, dblk_size))
+
+
+class EnqRec(Hdr):
+
+ format = '=QQ'
+
+ def init(self, f, foffs, xidsize, dsize):
+ self.xidsize = xidsize
+ self.dsize = dsize
+ self.xid = None
+ self.data = None
+ self.enq_tail = None
+ self.xid_complete = False
+ self.data_complete = False
+ self.tail_complete = False
+ self.tail_bin = None
+ self.tail_offs = 0
+ self.load(f)
+
+ def load(self, f):
+ if not self.xid_complete:
+ ret = load_file_data(f, self.xidsize, self.xid)
+ self.xid = ret[0]
+ self.xid_complete = ret[1]
+ if self.xid_complete and not self.data_complete:
+ ret = load_file_data(f, self.dsize, self.data)
+ self.data = ret[0]
+ self.data_complete = ret[1]
+ if self.data_complete and not self.tail_complete:
+ ret = load_file_data(f, calcsize(RecTail.format), self.tail_bin)
+ self.tail_bin = ret[0]
+ if ret[1]:
+ self.enq_tail = RecTail(self.tail_offs, *unpack(RecTail.format, self.tail_bin))
+ if self.enq_tail.magic_inv != inv_str(self.magic) or self.enq_tail.rid != self.rid:
+ print " > %s" % self
+ raise Exception('Invalid enqueue record tail (magic=0x%08x; rid=%d) at 0x%08x' % (self.enq_tail.magic_inv, self.enq_tail.rid, self.enq_tail.foffs))
+ self.enq_tail.skip(f)
+ self.tail_complete = ret[1]
+ return self.complete()
+
+ def complete(self):
+ return self.xid_complete and self.data_complete and self.tail_complete
+
+ def __str__(self):
+ if self.enq_tail == None:
+ return '%s %s [no tail]' % (Hdr.__str__(self), self.dsize, dstr)
+ return '%s %s%s %s' % (Hdr.__str__(self), print_xid(self.xidsize, self.xid), print_data(self.dsize, self.data), self.enq_tail)
+
+
+class Main:
+
+ def __init__(self, tfile, tnum):
+ if tfile != None:
+ tparams = self.get_test(tfile, tnum)
+ if tparams == None:
+ raise Exception('Test %d not found in file %s' % (tnum, tfile))
+ self.num_msgs = tparams['num_msgs']
+ if tparams['min_size'] == tparams['max_size']:
+ self.msg_len = tparams['max_size']
+ else:
+ self.msg_len = 0
+ self.auto_deq = tparams['auto_deq']
+ else:
+ self.num_msgs = 0
+ self.msg_len = 0
+ self.auto_deq = False
+ self.file_start = 0
+ self.file_num = 0
+ self.fro = 0x200
+ self.enqueued = {}
+ self.msg_cnt = 0
+ self.fhdr = None
+ self.f = None
+ self.first_rec = False
+ self.last_file = False
+ self.last_rid = -1
+
+
+ def run(self):
+ start_info = self.analyze_files()
+ stop = self.advance_file(*start_info)
+ while not stop:
+ warn = ''
+ if file_full(self.f):
+ stop = self.advance_file()
+ if stop:
+ break
+ hdr = load(self.f, Hdr)
+ if hdr.empty():
+ stop = True;
+ break
+ if hdr.check():
+ stop = True;
+ if self.first_rec:
+ if self.fhdr.fro != hdr.foffs:
+ raise Exception('File header first record offset mismatch: fro=0x%08x; rec_offs=0x%08x' % (self.fhdr.fro, hdr.foffs))
+ else:
+ print ' * fro ok: 0x%08x' % self.fhdr.fro
+ self.first_rec = False
+ if isinstance(hdr, EnqRec) and not stop:
+ while not hdr.complete():
+ stop = self.advance_file()
+ if stop:
+ break
+ hdr.load(self.f)
+ if self.msg_len > 0 and len(hdr.data) != self.msg_len:
+ raise Exception('Message length (%d) incorrect: expected %d' % (len(hdr.data), self.msg_len))
+ stop = not self.check_rid(hdr)
+ if stop:
+ warn = ' (WARNING: rid out of order, last rid = %d - could be overwrite boundary.)' % hdr.rid
+ else:
+ self.msg_cnt += 1
+ if self.auto_deq:
+ self.enqueued[hdr.rid] = hdr
+ elif isinstance(hdr, DeqHdr) and not stop:
+ if self.auto_deq:
+ if hdr.deq_rid in self.enqueued:
+ del self.enqueued[hdr.deq_rid]
+ else:
+ warn = ' (WARNING: dequeue rid %d not found in enqueued records)' % hdr.deq_rid
+ stop = not self.check_rid(hdr)
+ if stop:
+ warn = ' (WARNING: rid out of order, last rid = %d - could be overwrite boundary.)' % hdr.rid
+ else:
+ warn = 'WARNING: Dequeue record rid=%d found in non-dequeue test - ignoring.' % hdr.rid
+ print ' > %s%s' % (hdr, warn)
+ if not stop:
+ stop = (self.last_file and hdr.check()) or hdr.empty() or self.fhdr.empty()
+
+ def report(self):
+ if self.num_msgs > 0 and self.msg_cnt != self.num_msgs:
+ print 'WARNING: Found %d messages; %d expected.' % (self.msg_cnt, self.num_msgs)
+ if len(self.enqueued) > 0:
+ print 'Remaining enqueued records: ', len(self.enqueued)
+ for h in self.enqueued:
+ print self.enqueued[h]
+ print 'WARNING: Enqueue-Dequeue mismatch, %d enqueued records remain.' % len(self.enqueued)
+ print 'Test passed; %d records processed.' % self.msg_cnt
+
+ def advance_file(self, *start_info):
+ seek_flag = False
+ if len(start_info) == 3:
+ self.file_start = self.file_num = start_info[0]
+ self.fro = start_info[2]
+ seek_flag = True
+ if self.f != None and file_full(self.f):
+ self.file_num = self.incr_fnum()
+ if self.file_num == self.file_start:
+ return True
+ if self.file_start == 0:
+ self.last_file = self.file_num == num_files - 1
+ else:
+ self.last_file = self.file_num == self.file_start - 1
+ if self.file_num < 0 or self.file_num >= num_files:
+ raise Exception('Bad file number %d' % self.file_num)
+ file_name = 'jdata/test.%04d.jdat' % self.file_num
+ self.f = open(file_name)
+ self.fhdr = load(self.f, Hdr)
+ if seek_flag and self.f.tell() != self.fro:
+ self.f.seek(self.fro)
+ self.first_rec = True
+ print file_name, ": ", self.fhdr
+ return False
+
+ def incr_fnum(self):
+ self.file_num += 1
+ if self.file_num >= num_files:
+ self.file_num = 0;
+ return self.file_num
+
+ def check_rid(self, hdr):
+ if self.last_rid != -1 and hdr.rid != self.last_rid + 1:
+ return False
+ self.last_rid = hdr.rid
+ return True
+
+ def get_test(self, filename, tnum):
+ f=open(filename, 'r')
+ for l in f:
+ sl = l.strip().split(',')
+ if len(sl[1]) > 0: #Comments are in col 0, remaining cols are empty
+ try:
+ if (int(sl[0]) == tnum):
+ return { 'num_msgs':int(sl[1]),
+ 'min_size':int(sl[2]),
+ 'max_size':int(sl[3]),
+ 'auto_deq':sl[4] != 'FALSE' }
+ except Exception:
+ pass
+ return None
+
+ def analyze_files(self):
+ fname = ''
+ fnum = -1
+ rid = -1
+ fro = -1
+ tss = ''
+ print 'Analyzing journal files:'
+ for i in range(0, num_files):
+ file_name = 'jdata/test.%04d.jdat' % i
+ f = open(file_name)
+ fhdr = load(f, Hdr)
+ if fhdr.empty():
+ break
+ if (rid == -1 or fhdr.rid < rid) and fhdr.fro > 0:
+ fname = file_name
+ fnum = i
+ rid = fhdr.rid
+ fro = fhdr.fro
+ tss = fhdr.timestamp_str()
+ print ' %s: rid=%d, fro=0x%08x ts=%s' % (file_name, fhdr.rid, fhdr.fro, fhdr.timestamp_str())
+ if fnum < 0 or rid < 0 or fro < 0:
+ raise Exception('All journal files empty')
+ print ' Lowest rid found in file %s: rid=%d, fro=0x%08x ts=%s' % (fname, rid, fro, tss)
+ return (fnum, rid, fro)
+
+
+CLASSES = {
+ "a": TxHdr,
+ "c": TxHdr,
+ "d": DeqHdr,
+ "e": EnqRec,
+ "f": FileHdr
+}
+
+if len(sys.argv) != 1 and len(sys.argv) != 3:
+ print 'Incorrect number of command-line arguments (expected 0 or 2, found %d).' % (len(sys.argv) - 1)
+ print 'Usage: ftest test-file test-number'
+ raise Exception('Bad command-line arguments')
+if len(sys.argv) == 3 and not sys.argv[2].isdigit():
+ print 'Illegal argument (\"%s\"): test-number must be a number' % a
+ print 'Usage: ftest test-file test-number'
+ raise Exception('Illegal argument value')
+
+if len(sys.argv) != 0:
+ m = Main(None, None)
+ m.run()
+else:
+ tnum = int(sys.argv[2])
+ print 'Test number: %d' % tnum
+ print 'Test file: %s' %sys.argv[1]
+ m = Main(sys.argv[1], tnum)
+ m.run()
+ m.report()
Modified: store/trunk/cpp/tests/jrnl/rtest
===================================================================
--- store/trunk/cpp/tests/jrnl/rtest 2007-09-27 19:29:25 UTC (rev 949)
+++ store/trunk/cpp/tests/jrnl/rtest 2007-09-28 16:02:33 UTC (rev 950)
@@ -55,7 +55,7 @@
RM=rm
RM_DIR="${RM} -rf"
TEST_PROG="./jtest"
-CHK_PROG="./ftest.py"
+CHK_PROG="./janalyze.py"
VALGRIND="valgrind -q --track-fds=yes --leak-check=full --leak-resolution=high --show-reachable=yes"
#VALGRIND="valgrind -q --track-fds=yes --leak-check=full --leak-resolution=high --show-reachable=yes --suppressions=/usr/lib/valgrind/glibc-2.5.supp"
MAKE="make -f Makefile.rtest"
Modified: store/trunk/cpp/tests/jrnl/tests.ods
===================================================================
(Binary files differ)
Modified: store/trunk/cpp/tests/jrnl/wtests.csv
===================================================================
--- store/trunk/cpp/tests/jrnl/wtests.csv 2007-09-27 19:29:25 UTC (rev 949)
+++ store/trunk/cpp/tests/jrnl/wtests.csv 2007-09-28 16:02:33 UTC (rev 950)
@@ -102,6 +102,6 @@
80,30000,0,38356,TRUE,,"300 dblks max"
81,10000,0,127956,TRUE,,"1000 dblks max"
,,,,,,
-"STANDARD PERFORMANCE BENCHMARK: 10,000,000 writes, data=220b (2 dblks)",,,,,,
-82,10000000,220,220,FALSE,,"2 dblks"
-83,10000000,220,220,TRUE,,"2 dblks"
+"STANDARD PERFORMANCE BENCHMARK: 10,000,000 writes, data=212b (2 dblks)",,,,,,
+82,10000000,212,212,FALSE,,"2 dblks"
+83,10000000,212,212,TRUE,,"2 dblks"
17 years, 5 months
- 1
- 0
rhmessaging commits: r949 - in store/trunk/cpp: tests and 1 other directory.
by rhmessaging-commits＠lists.jboss.org
Author: kpvdr
Date: 2007-09-27 15:29:25 -0400 (Thu, 27 Sep 2007)
New Revision: 949
Modified:
store/trunk/cpp/lib/jrnl/data_tok.cpp
store/trunk/cpp/lib/jrnl/data_tok.hpp
store/trunk/cpp/lib/jrnl/deq_rec.cpp
store/trunk/cpp/lib/jrnl/deq_rec.hpp
store/trunk/cpp/lib/jrnl/dtx_rec.cpp
store/trunk/cpp/lib/jrnl/dtx_rec.hpp
store/trunk/cpp/lib/jrnl/enq_rec.cpp
store/trunk/cpp/lib/jrnl/enq_rec.hpp
store/trunk/cpp/lib/jrnl/jcntl.cpp
store/trunk/cpp/lib/jrnl/jcntl.hpp
store/trunk/cpp/lib/jrnl/jerrno.cpp
store/trunk/cpp/lib/jrnl/jerrno.hpp
store/trunk/cpp/lib/jrnl/jrec.hpp
store/trunk/cpp/lib/jrnl/pmgr.hpp
store/trunk/cpp/lib/jrnl/rmgr.cpp
store/trunk/cpp/lib/jrnl/rmgr.hpp
store/trunk/cpp/lib/jrnl/wmgr.cpp
store/trunk/cpp/lib/jrnl/wmgr.hpp
store/trunk/cpp/tests/persistence.py
Log:
Corrections to class jcntl transactions API; new class deq_rec.encode() now in use.
Modified: store/trunk/cpp/lib/jrnl/data_tok.cpp
===================================================================
--- store/trunk/cpp/lib/jrnl/data_tok.cpp 2007-09-27 15:18:31 UTC (rev 948)
+++ store/trunk/cpp/lib/jrnl/data_tok.cpp 2007-09-27 19:29:25 UTC (rev 949)
@@ -87,6 +87,8 @@
return "ENQ";
case DEQ_CACHED:
return "DEQ_CACHED";
+ case DEQ_PART:
+ return "DEQ_PART";
case DEQ_SUBM:
return "DEQ_SUBM";
case DEQ:
Modified: store/trunk/cpp/lib/jrnl/data_tok.hpp
===================================================================
--- store/trunk/cpp/lib/jrnl/data_tok.hpp 2007-09-27 15:18:31 UTC (rev 948)
+++ store/trunk/cpp/lib/jrnl/data_tok.hpp 2007-09-27 19:29:25 UTC (rev 949)
@@ -75,6 +75,7 @@
ENQ_SUBM, ///< Data block enqueue submitted to AIO
ENQ, ///< Data block enqueue AIO write complete (enqueue complete)
DEQ_CACHED, ///< Data block dequeue written to page cache
+ DEQ_PART, ///< Data block part-submitted to AIO, waiting for page buffer to free up
DEQ_SUBM, ///< Data block dequeue submitted to AIO
DEQ ///< Data block dequeue AIO write complete (dequeue complete)
};
Modified: store/trunk/cpp/lib/jrnl/deq_rec.cpp
===================================================================
--- store/trunk/cpp/lib/jrnl/deq_rec.cpp 2007-09-27 15:18:31 UTC (rev 948)
+++ store/trunk/cpp/lib/jrnl/deq_rec.cpp 2007-09-27 19:29:25 UTC (rev 949)
@@ -43,12 +43,18 @@
{
deq_rec::deq_rec():
- _deq_hdr(),
+ _deq_hdr(RHM_JDAT_DEQ_MAGIC, 0, 0, 0, RHM_JDAT_VERSION),
_xid(),
_deq_tail()
{}
-deq_rec::deq_rec(const u_int64_t rid, const u_int64_t drid, const std::string xid):
+deq_rec::deq_rec(const u_int64_t rid, const u_int64_t drid):
+ _deq_hdr(RHM_JDAT_DEQ_MAGIC, rid, drid, 0, RHM_JDAT_VERSION),
+ _xid(),
+ _deq_tail(_deq_hdr._hdr)
+{}
+
+deq_rec::deq_rec(const u_int64_t rid, const u_int64_t drid, const std::string& xid):
_deq_hdr(RHM_JDAT_DEQ_MAGIC, rid, drid, xid.size(), RHM_JDAT_VERSION),
_xid(xid),
_deq_tail(_deq_hdr._hdr)
@@ -57,6 +63,24 @@
deq_rec::~deq_rec()
{}
+void
+deq_rec::reset(const u_int64_t rid, const u_int64_t drid)
+{
+ _deq_hdr._hdr._rid = rid;
+ _deq_hdr._deq_rid = drid;
+ _deq_hdr._xidsize = 0;
+ _deq_tail._rid = rid;
+}
+
+void
+deq_rec::reset(const u_int64_t rid, const u_int64_t drid, const std::string& xid)
+{
+ _deq_hdr._hdr._rid = rid;
+ _deq_hdr._deq_rid = drid;
+ _deq_hdr._xidsize = xid.size();
+ _deq_tail._rid = rid;
+}
+
u_int32_t
deq_rec::encode(void* wptr, u_int32_t rec_offs_dblks, u_int32_t max_size_dblks) throw (jexception)
{
Modified: store/trunk/cpp/lib/jrnl/deq_rec.hpp
===================================================================
--- store/trunk/cpp/lib/jrnl/deq_rec.hpp 2007-09-27 15:18:31 UTC (rev 948)
+++ store/trunk/cpp/lib/jrnl/deq_rec.hpp 2007-09-27 19:29:25 UTC (rev 949)
@@ -61,9 +61,12 @@
public:
deq_rec();
- deq_rec(const u_int64_t rid, const u_int64_t drid, const std::string xid);
+ deq_rec(const u_int64_t rid, const u_int64_t drid);
+ deq_rec(const u_int64_t rid, const u_int64_t drid, const std::string& xid);
~deq_rec();
+ void reset(const u_int64_t rid, const u_int64_t drid);
+ void reset(const u_int64_t rid, const u_int64_t drid, const std::string& xid);
u_int32_t encode(void* wptr, u_int32_t rec_offs_dblks, u_int32_t max_size_dblks)
throw (jexception);
u_int32_t decode(hdr& h, void* rptr, u_int32_t rec_offs_dblks, u_int32_t max_size_dblks)
@@ -74,9 +77,9 @@
const size_t rec_size() const;
private:
- virtual void chk_hdr() const throw (jexception) = 0;
- virtual void chk_hdr(u_int64_t rid) const throw (jexception) = 0;
- virtual void chk_tail() const throw (jexception) = 0;
+ virtual void chk_hdr() const throw (jexception);
+ virtual void chk_hdr(u_int64_t rid) const throw (jexception);
+ virtual void chk_tail() const throw (jexception);
}; // class deq_rec
} // namespace journal
Modified: store/trunk/cpp/lib/jrnl/dtx_rec.cpp
===================================================================
--- store/trunk/cpp/lib/jrnl/dtx_rec.cpp 2007-09-27 15:18:31 UTC (rev 948)
+++ store/trunk/cpp/lib/jrnl/dtx_rec.cpp 2007-09-27 19:29:25 UTC (rev 949)
@@ -48,7 +48,13 @@
_dtx_tail()
{}
-dtx_rec::dtx_rec(const u_int32_t magic, const u_int64_t rid, const std::string xid):
+dtx_rec::dtx_rec(const u_int32_t magic, const u_int64_t rid):
+ _dtx_hdr(magic, rid, 0, RHM_JDAT_VERSION),
+ _xid(),
+ _dtx_tail(_dtx_hdr._hdr)
+{}
+
+dtx_rec::dtx_rec(const u_int32_t magic, const u_int64_t rid, const std::string& xid):
_dtx_hdr(magic, rid, xid.size(), RHM_JDAT_VERSION),
_xid(xid),
_dtx_tail(_dtx_hdr._hdr)
@@ -57,6 +63,22 @@
dtx_rec::~dtx_rec()
{}
+void
+dtx_rec::reset(const u_int64_t rid)
+{
+ _dtx_hdr._hdr._rid = rid;
+ _dtx_hdr._xidsize = 0;
+ _dtx_tail._rid = rid;
+}
+
+void
+dtx_rec::reset(const u_int64_t rid, const std::string& xid)
+{
+ _dtx_hdr._hdr._rid = rid;
+ _dtx_hdr._xidsize = xid.size();
+ _dtx_tail._rid = rid;
+}
+
u_int32_t
dtx_rec::encode(void* wptr, u_int32_t rec_offs_dblks, u_int32_t max_size_dblks) throw (jexception)
{
Modified: store/trunk/cpp/lib/jrnl/dtx_rec.hpp
===================================================================
--- store/trunk/cpp/lib/jrnl/dtx_rec.hpp 2007-09-27 15:18:31 UTC (rev 948)
+++ store/trunk/cpp/lib/jrnl/dtx_rec.hpp 2007-09-27 19:29:25 UTC (rev 949)
@@ -61,9 +61,12 @@
public:
dtx_rec();
- dtx_rec(const u_int32_t magic, const u_int64_t rid, const std::string xid);
+ dtx_rec(const u_int32_t magic, const u_int64_t rid);
+ dtx_rec(const u_int32_t magic, const u_int64_t rid, const std::string& xid);
~dtx_rec();
+ void reset(const u_int64_t rid);
+ void reset(const u_int64_t rid, const std::string& xid);
u_int32_t encode(void* wptr, u_int32_t rec_offs_dblks, u_int32_t max_size_dblks)
throw (jexception);
u_int32_t decode(hdr& h, void* rptr, u_int32_t rec_offs_dblks, u_int32_t max_size_dblks)
Modified: store/trunk/cpp/lib/jrnl/enq_rec.cpp
===================================================================
--- store/trunk/cpp/lib/jrnl/enq_rec.cpp 2007-09-27 15:18:31 UTC (rev 948)
+++ store/trunk/cpp/lib/jrnl/enq_rec.cpp 2007-09-27 19:29:25 UTC (rev 949)
@@ -70,6 +70,21 @@
JRNL_DBLK_SIZE)
{}
+// Constructor used for transactional write operations, where dbuf contains data to be written.
+enq_rec::enq_rec(const u_int64_t rid, const void* const dbuf, const size_t dlen,
+ const std::string& xid):
+ jrec(), // superclass
+ _enq_hdr(RHM_JDAT_ENQ_MAGIC, rid, xid.size(), dlen, RHM_JDAT_VERSION),
+ _xid(xid),
+ _data(dbuf),
+ _buff(NULL),
+ _enq_tail(_enq_hdr._hdr),
+ _max_data_size(0),
+ _data_size(dlen),
+ _rec_size(size_dblks(enq_hdr::size() + _enq_hdr._xidsize + dlen + rec_tail::size()) *
+ JRNL_DBLK_SIZE)
+{}
+
// Constructor used for read operations, where buf contains preallocated space
// to receive data.
enq_rec::enq_rec(void* const buf, const size_t bufsize):
@@ -110,6 +125,23 @@
_rec_size = size_dblks(dlen + enq_hdr::size() + rec_tail::size()) * JRNL_DBLK_SIZE;
}
+// Prepare instance for use in writing transactional data to journal, where dbuf contains data to
+// be written.
+void
+enq_rec::reset(const u_int64_t rid, const void* const dbuf, const size_t dlen,
+ const std::string& xid)
+{
+ _enq_hdr._hdr._rid = rid;
+ _enq_hdr._xidsize = xid.size();
+ _enq_hdr._dsize = dlen;
+ _data = dbuf;
+ _buff = NULL;
+ _enq_tail._rid = rid;
+ _max_data_size = 0;
+ _data_size = dlen;
+ _rec_size = size_dblks(dlen + enq_hdr::size() + rec_tail::size()) * JRNL_DBLK_SIZE;
+}
+
// Prepare instance for use in reading data from journal, where buf contains preallocated space
// to receive data.
void
Modified: store/trunk/cpp/lib/jrnl/enq_rec.hpp
===================================================================
--- store/trunk/cpp/lib/jrnl/enq_rec.hpp 2007-09-27 15:18:31 UTC (rev 948)
+++ store/trunk/cpp/lib/jrnl/enq_rec.hpp 2007-09-27 19:29:25 UTC (rev 949)
@@ -67,7 +67,7 @@
public:
/**
* \brief Default constructor; must be used in conjunction with reset() to prepare
- * instance for use with write or read operations.
+ * instance for use with write or read operations.
*/
enq_rec();
@@ -77,8 +77,15 @@
enq_rec(const u_int64_t rid, const void* const dbuf, const size_t dlen);
/**
+ * \brief Constructor used for transactional write operations, where mbuf contains data to
+ * be written.
+ */
+ enq_rec(const u_int64_t rid, const void* const dbuf, const size_t dlen,
+ const std::string& xid);
+
+ /**
* \brief Constructor used for read operations, where buf contains preallocated space
- * to receive data.
+ * to receive data.
*/
enq_rec(void* const buf, const size_t bufsize);
@@ -89,6 +96,8 @@
// Prepare instance for use in writing data to journal
void reset(const u_int64_t rid, const void* const dbuf, const size_t dlen);
+ void reset(const u_int64_t rid, const void* const dbuf, const size_t dlen,
+ const std::string& xid);
// Prepare instance for use in reading data from journal
void reset(void* const buf, const size_t bufsize);
@@ -103,7 +112,6 @@
const size_t data_size() const;
const size_t xid_size() const;
const size_t rec_size() const;
- inline const u_int32_t rec_size_dblks() const { return size_dblks(rec_size()); }
inline const u_int64_t rid() const { return _enq_hdr._hdr._rid; }
void set_rid(const u_int64_t rid);
Modified: store/trunk/cpp/lib/jrnl/jcntl.cpp
===================================================================
--- store/trunk/cpp/lib/jrnl/jcntl.cpp 2007-09-27 15:18:31 UTC (rev 948)
+++ store/trunk/cpp/lib/jrnl/jcntl.cpp 2007-09-27 19:29:25 UTC (rev 949)
@@ -202,7 +202,15 @@
const iores
jcntl::enqueue_data_record(const void* const /*data_buff*/, const size_t /*tot_data_len*/,
- const size_t /*this_data_len*/, data_tok* /*dtokp*/, const std::string /*xid*/,
+ const size_t /*this_data_len*/, data_tok* /*dtokp*/, const bool /*transient*/)
+ throw (jexception)
+{
+ return RHM_IORES_NOTIMPL;
+}
+
+const iores
+jcntl::enqueue_data_record(const void* const /*data_buff*/, const size_t /*tot_data_len*/,
+ const size_t /*this_data_len*/, data_tok* /*dtokp*/, const std::string& /*xid*/,
const bool /*transient*/) throw (jexception)
{
return RHM_IORES_NOTIMPL;
@@ -230,20 +238,27 @@
}
const iores
-jcntl::dequeue_data_record(data_tok* const dtokp, const std::string /*xid*/) throw (jexception)
+jcntl::dequeue_data_record(data_tok* const dtokp) throw (jexception)
{
check_wstatus("dequeue_data");
return _wmgr.dequeue(dtokp);
}
const iores
-jcntl::abort_dtx(const std::string /*xid*/) throw (jexception)
+jcntl::dequeue_data_record(data_tok* const dtokp, const std::string& /*xid*/) throw (jexception)
{
+ check_wstatus("dequeue_data");
+ return _wmgr.dequeue(dtokp);
+}
+
+const iores
+jcntl::abort_dtx(const std::string& /*xid*/) throw (jexception)
+{
return RHM_IORES_NOTIMPL;
}
const iores
-jcntl::commit_dtx(const std::string /*xid*/) throw (jexception)
+jcntl::commit_dtx(const std::string& /*xid*/) throw (jexception)
{
return RHM_IORES_NOTIMPL;
}
@@ -545,9 +560,5 @@
}
-// Static instance of empty string used as default XID parameter and which signifies no XID.
-const std::string jcntl::no_xid;
-
-
} // namespace journal
} // namespace rhm
Modified: store/trunk/cpp/lib/jrnl/jcntl.hpp
===================================================================
--- store/trunk/cpp/lib/jrnl/jcntl.hpp 2007-09-27 15:18:31 UTC (rev 948)
+++ store/trunk/cpp/lib/jrnl/jcntl.hpp 2007-09-27 19:29:25 UTC (rev 949)
@@ -142,8 +142,6 @@
std::deque<rhm::journal::data_tok*> _aio_wr_cmpl_dtok_list; ///< Internally mamanged deque
public:
- static const std::string no_xid;
-
/**
* \brief Journal constructor.
*
@@ -304,7 +302,10 @@
* \exception TODO
*/
const iores enqueue_data_record(const void* const data_buff, const size_t tot_data_len,
- const size_t this_data_len, data_tok* dtokp, const std::string xid = no_xid,
+ const size_t this_data_len, data_tok* dtokp, const bool transient = false)
+ throw (jexception);
+ const iores enqueue_data_record(const void* const data_buff, const size_t tot_data_len,
+ const size_t this_data_len, data_tok* dtokp, const std::string& xid,
const bool transient = false) throw (jexception);
/**
@@ -407,7 +408,8 @@
*
* \exception TODO
*/
- const iores dequeue_data_record(data_tok* const dtokp, const std::string xid = no_xid)
+ const iores dequeue_data_record(data_tok* const dtokp) throw (jexception);
+ const iores dequeue_data_record(data_tok* const dtokp, const std::string& xid)
throw (jexception);
/**
@@ -421,7 +423,7 @@
*
* \exception TODO
*/
- const iores abort_dtx(const std::string xid) throw (jexception);
+ const iores abort_dtx(const std::string& xid) throw (jexception);
/**
* \brief Commit the transaction for all records enqueued or dequeued with the matching xid.
@@ -434,7 +436,7 @@
*
* \exception TODO
*/
- const iores commit_dtx(const std::string xid) throw (jexception);
+ const iores commit_dtx(const std::string& xid) throw (jexception);
/**
* \brief Check whether all the enqueue records for the given xid have reached disk.
@@ -443,7 +445,7 @@
*
* \exception TODO
*/
- const bool is_dtx_synced(const std::string xid) throw (jexception);
+ const bool is_dtx_synced(const std::string& xid) throw (jexception);
/**
* \brief Forces a check for returned AIO write events.
Modified: store/trunk/cpp/lib/jrnl/jerrno.cpp
===================================================================
--- store/trunk/cpp/lib/jrnl/jerrno.cpp 2007-09-27 15:18:31 UTC (rev 948)
+++ store/trunk/cpp/lib/jrnl/jerrno.cpp 2007-09-27 19:29:25 UTC (rev 949)
@@ -93,6 +93,7 @@
const u_int32_t jerrno::JERR_WMGR_BADPGSTATE = 0x0801;
const u_int32_t jerrno::JERR_WMGR_BADDTOKSTATE = 0x0802;
const u_int32_t jerrno::JERR_WMGR_ENQDISCONT = 0x0803;
+const u_int32_t jerrno::JERR_WMGR_DEQDISCONT = 0x0804; ///< Deq. new dtok when previous part compl.
// class rmgr
const u_int32_t jerrno::JERR_RMGR_UNKNOWNMAGIC = 0x0900;
@@ -182,6 +183,8 @@
"Data token in illegal state for operation.");
_err_map[JERR_WMGR_ENQDISCONT] = std::string("JERR_WMGR_ENQDISCONT: "
"Enqueued new dtok when previous enqueue returned partly completed (state ENQ_PART).");
+ _err_map[JERR_WMGR_DEQDISCONT] = std::string("JERR_WMGR_DEQDISCONT: "
+ "Dequeued new dtok when previous dequeue returned partly completed (state DEQ_PART).");
// class rmgr
_err_map[JERR_RMGR_UNKNOWNMAGIC] = std::string("JERR_RMGR_UNKNOWNMAGIC: "
Modified: store/trunk/cpp/lib/jrnl/jerrno.hpp
===================================================================
--- store/trunk/cpp/lib/jrnl/jerrno.hpp 2007-09-27 15:18:31 UTC (rev 948)
+++ store/trunk/cpp/lib/jrnl/jerrno.hpp 2007-09-27 19:29:25 UTC (rev 949)
@@ -110,6 +110,7 @@
static const u_int32_t JERR_WMGR_BADPGSTATE; ///< Page buffer in illegal state.
static const u_int32_t JERR_WMGR_BADDTOKSTATE; ///< Data token in illegal state.
static const u_int32_t JERR_WMGR_ENQDISCONT; ///< Enq. new dtok when previous part compl.
+ static const u_int32_t JERR_WMGR_DEQDISCONT; ///< Deq. new dtok when previous part compl.
// class rmgr
static const u_int32_t JERR_RMGR_UNKNOWNMAGIC; ///< Found record with unknown magic
Modified: store/trunk/cpp/lib/jrnl/jrec.hpp
===================================================================
--- store/trunk/cpp/lib/jrnl/jrec.hpp 2007-09-27 15:18:31 UTC (rev 948)
+++ store/trunk/cpp/lib/jrnl/jrec.hpp 2007-09-27 19:29:25 UTC (rev 949)
@@ -153,6 +153,7 @@
virtual const size_t data_size() const = 0;
virtual const size_t xid_size() const = 0;
virtual const size_t rec_size() const = 0;
+ inline virtual const u_int32_t rec_size_dblks() const { return size_dblks(rec_size()); }
static const u_int32_t size_dblks(const size_t size);
static const u_int32_t size_sblks(const size_t size);
static const u_int32_t size_blks(const size_t size, const size_t blksize);
Modified: store/trunk/cpp/lib/jrnl/pmgr.hpp
===================================================================
--- store/trunk/cpp/lib/jrnl/pmgr.hpp 2007-09-27 15:18:31 UTC (rev 948)
+++ store/trunk/cpp/lib/jrnl/pmgr.hpp 2007-09-27 19:29:25 UTC (rev 949)
@@ -45,9 +45,11 @@
#include <deque>
#include <libaio.h>
#include <jrnl/aio_cb.hpp>
-#include <jrnl/enq_rec.hpp>
#include <jrnl/data_tok.hpp>
+#include <jrnl/deq_rec.hpp>
+#include <jrnl/dtx_rec.hpp>
#include <jrnl/enq_map.hpp>
+#include <jrnl/enq_rec.hpp>
#include <jrnl/nlfh.hpp>
namespace rhm
@@ -125,7 +127,11 @@
u_int32_t _pg_cntr; ///< Page counter; determines if file rotation req'd
u_int32_t _pg_offset_dblks; ///< Page offset (used so far) in data blocks
u_int32_t _aio_evt_rem; ///< Remaining AIO events
- enq_rec _enq_rec; ///< Data record into/from which data is encoded/decoded
+
+ enq_rec _enq_rec; ///< Enqueue record used for encoding/decoding
+ deq_rec _deq_rec; ///< Dequeue record used for encoding/decoding
+ dtx_rec _dtx_rec; ///< Transaction record used for encoding/decoding
+
// TODO: move _cb down to wmgr, it is the only class that uses it There is no need for
// read callbacks based on AIO. - (check this asertion)
aio_cb _cb; ///< Callback function pointer for AIO events
Modified: store/trunk/cpp/lib/jrnl/rmgr.cpp
===================================================================
--- store/trunk/cpp/lib/jrnl/rmgr.cpp 2007-09-27 15:18:31 UTC (rev 948)
+++ store/trunk/cpp/lib/jrnl/rmgr.cpp 2007-09-27 19:29:25 UTC (rev 949)
@@ -290,11 +290,11 @@
{
if (_page_cb_arr[_pg_index]._state != AIO_COMPLETE)
{
-//std::cout << " U=" << page_state_str(_page_cb_arr[_pg_index]._state) << std::flush;
+//std::cout << " S=" << page_state_str(_page_cb_arr[_pg_index]._state) << std::flush;
aio_cycle(); // check if any AIOs have returned
return RHM_IORES_AIO_WAIT;
}
-//std::cout << " V" << dtokp->dblocks_read() << "," << dblks_rem() << std::flush;
+//std::cout << " T" << dtokp->dblocks_read() << "," << dblks_rem() << std::flush;
// Read data from this page, first block will have header and data size.
u_int32_t dblks_rd = _enq_rec.decode(h, rptr, dtokp->dblocks_read(), dblks_rem());
@@ -305,7 +305,7 @@
// If data still incomplete, move to next page and decode again
while (dtokp->dblocks_read() < _enq_rec.rec_size_dblks())
{
-//std::cout << " W" << std::flush;
+//std::cout << " U" << std::flush;
rotate_page();
if (_page_cb_arr[_pg_index]._state != AIO_COMPLETE)
{
@@ -322,7 +322,7 @@
// If we have finished with this page, rotate it
if (dblks_rem() == 0)
-//{std::cout << " X" << std::flush;
+//{std::cout << " V" << std::flush;
rotate_page();
//}
@@ -332,10 +332,58 @@
return RHM_IORES_SUCCESS;
}
+const iores
+rmgr::read_deq(hdr& h, void* rptr, data_tok* dtokp) throw (jexception)
+{
+ if (_page_cb_arr[_pg_index]._state != AIO_COMPLETE)
+ {
+//std::cout << " W=" << page_state_str(_page_cb_arr[_pg_index]._state) << std::flush;
+ aio_cycle(); // check if any AIOs have returned
+ return RHM_IORES_AIO_WAIT;
+ }
+//std::cout << " X" << dtokp->dblocks_read() << "," << dblks_rem() << std::flush;
+
+ // Read data from this page, first block will have header and data size.
+ u_int32_t dblks_rd = _deq_rec.decode(h, rptr, dtokp->dblocks_read(), dblks_rem());
+ dtokp->incr_dblocks_read(dblks_rd);
+
+ _pg_offset_dblks += dblks_rd;
+
+ // If data still incomplete, move to next page and decode again
+ // TODO
+ while (dtokp->dblocks_read() < _enq_rec.rec_size_dblks())
+ {
+//std::cout << " Y" << std::flush;
+ rotate_page();
+ if (_page_cb_arr[_pg_index]._state != AIO_COMPLETE)
+ {
+ dtokp->set_rstate(data_tok::READ_PART);
+ dtokp->set_dsize(_enq_rec.data_size());
+ return RHM_IORES_AIO_WAIT;
+ }
+
+ rptr = (void*)((char*)_page_ptr_arr[_pg_index]);
+ dblks_rd = _enq_rec.decode(h, rptr, dtokp->dblocks_read(), dblks_rem());
+ dtokp->incr_dblocks_read(dblks_rd);
+ _pg_offset_dblks += dblks_rd;
+ }
+
+ // If we have finished with this page, rotate it
+ if (dblks_rem() == 0)
+//{std::cout << " Z" << std::flush;
+ rotate_page();
+//}
+
+ // Set the record size in dtokp
+ dtokp->set_rstate(data_tok::READ);
+ dtokp->set_dsize(_enq_rec.data_size());
+ return RHM_IORES_SUCCESS;
+}
+
+// *** DEPRECATED ***, to be removed, use read_deq()
void
rmgr::consume_deq() throw (jexception)
{
- // TODO: Check: Assumption - dequeue record fills one dblk, but dblk size is tunable.
_pg_offset_dblks++;
if (dblks_rem() == 0)
rotate_page();
Modified: store/trunk/cpp/lib/jrnl/rmgr.hpp
===================================================================
--- store/trunk/cpp/lib/jrnl/rmgr.hpp 2007-09-27 15:18:31 UTC (rev 948)
+++ store/trunk/cpp/lib/jrnl/rmgr.hpp 2007-09-27 19:29:25 UTC (rev 949)
@@ -75,7 +75,8 @@
private:
void initialize() throw (jexception);
const iores read_enq(hdr& h, void* rptr, data_tok* dtok) throw (jexception);
- void consume_deq() throw (jexception);
+ const iores read_deq(hdr& h, void* rptr, data_tok* dtok) throw (jexception);
+ void consume_deq() throw (jexception); // DEPRECATED, to be removed, use read_deq()
void consume_filler() throw (jexception);
const iores skip(data_tok* dtokp) throw (jexception);
void aio_cycle() throw (jexception);
Modified: store/trunk/cpp/lib/jrnl/wmgr.cpp
===================================================================
--- store/trunk/cpp/lib/jrnl/wmgr.cpp 2007-09-27 15:18:31 UTC (rev 948)
+++ store/trunk/cpp/lib/jrnl/wmgr.cpp 2007-09-27 19:29:25 UTC (rev 949)
@@ -49,7 +49,8 @@
_max_io_wait_us(0),
_fhdr_buff(NULL),
_cached_offset_dblks(0),
- _enq_busy(false)
+ _enq_busy(false),
+ _deq_busy(false)
{}
wmgr::wmgr(jcntl* jc, enq_map& emap, wrfc& wrfc, std::deque<data_tok*>* const dtokl,
@@ -60,7 +61,8 @@
_max_io_wait_us(max_iowait_us),
_fhdr_buff(NULL),
_cached_offset_dblks(0),
- _enq_busy(false)
+ _enq_busy(false),
+ _deq_busy(false)
{}
wmgr::~wmgr()
@@ -86,6 +88,9 @@
const iores
wmgr::enqueue(const void* const mbuf, const size_t dlen, data_tok* dtok) throw (jexception)
{
+ if (_deq_busy)
+ return RHM_IORES_BUSY;
+
iores res = pre_write_check(true, dtok);
if (res != RHM_IORES_SUCCESS)
return res;
@@ -215,6 +220,24 @@
if (res != RHM_IORES_SUCCESS)
return res;
+ bool cont = false;
+ if (_deq_busy) // If dequeue() exited last time with RHM_IORES_FULL or RHM_IORES_AIO_WAIT
+ {
+ if (dtok->wstate() == data_tok::DEQ_PART)
+ cont = true;
+ else
+ {
+ std::stringstream ss;
+ ss << "This data_tok: id=" << dtok->id() << " state=" << dtok->wstate_str();
+ throw jexception(jerrno::JERR_WMGR_DEQDISCONT, ss.str(), "wmgr", "dequeue");
+ }
+ }
+ else
+ {
+ _deq_busy = true;
+ dtok->set_dblocks_written(0); // Reset dblks_written from enqueue op
+ }
+
u_int64_t rid;
if (dtok->getSourceMessage())
{
@@ -224,34 +247,129 @@
else
rid = _wrfc.get_incr_rid();
-
- //***
- // NOTE: ASSUMPTION: sizeof(deq_hdr) <= JRNL_DBLK_SIZE
- // This encoding is a simplification: it assumes deq_hdr (currently 20 bytes) fits inside
- // one dblk. The dblk is a tunable parameter, but is unlikely to go lower than deq_hdr
- // (currently dblk = 128 bytes). JRNL_DBLK_SIZE must be a power of 2.
- // IF JRNL_DBLK_SIZE IS SET TO < 32 (i.e. 16 OR LESS) BYTES, THIS ENCODING WILL FAIL!
- //***
- deq_hdr dhdr(RHM_JDAT_DEQ_MAGIC, rid, dtok->rid(), 0, RHM_JDAT_VERSION);
- void* wptr = (void*)((char*)_page_ptr_arr[_pg_index] + _pg_offset_dblks * JRNL_DBLK_SIZE);
- ::memcpy(wptr, &dhdr, sizeof(dhdr));
-#ifdef RHM_CLEAN
- ::memset((char*)wptr + sizeof(dhdr), RHM_CLEAN_CHAR, JRNL_DBLK_SIZE - sizeof(dhdr));
+ _deq_rec.reset(rid, dtok->rid());
+ bool done = false;
+ while (!done)
+ {
+ assert(_pg_offset_dblks < JRNL_WMGR_PAGE_SIZE * JRNL_SBLK_SIZE);
+ void* wptr = (void*)((char*)_page_ptr_arr[_pg_index] + _pg_offset_dblks * JRNL_DBLK_SIZE);
+ u_int32_t data_offs_dblks = dtok->dblocks_written();
+ u_int32_t ret = _deq_rec.encode(wptr, data_offs_dblks,
+ (JRNL_WMGR_PAGE_SIZE * JRNL_SBLK_SIZE) - _pg_offset_dblks);
+#if !(defined(RHM_WRONLY) || defined(RHM_RDONLY))
+ if (data_offs_dblks == 0)
+ {
+ u_int16_t fid = _emap.get_remove_fid(dtok->rid());
+ _wrfc.decr_enqcnt(fid);
+ }
#endif
-#if !(defined RHM_WRONLY || defined RHM_RDONLY)
- u_int16_t fid = _emap.get_remove_fid(dtok->rid());
- _wrfc.decr_enqcnt(fid);
-#endif
- _pg_offset_dblks++;
- _cached_offset_dblks++;
- // TODO: Incorrect - must set state to DEQ_CACHED; DEQ_SUBM is set when AIO returns.
- dtok->set_wstate(data_tok::DEQ_SUBM);
- _page_cb_arr[_pg_index]._pdtokl->push_back(dtok);
- if (_wrfc.empty()) // Has the file_hdr been written?
- write_fhdr(rid, _wrfc.index(), JRNL_SBLK_SIZE * JRNL_DBLK_SIZE);
- if (_pg_offset_dblks >= JRNL_WMGR_PAGE_SIZE * JRNL_SBLK_SIZE)
- res = flush();
+ _pg_offset_dblks += ret;
+ _cached_offset_dblks += ret;
+ dtok->incr_dblocks_written(ret);
+ // Is the encoding of this record complete?
+ if (dtok->dblocks_written() >= _deq_rec.rec_size_dblks())
+ {
+ // TODO: Incorrect - must set state to ENQ_CACHED; ENQ_SUBM is set when AIO returns.
+ dtok->set_wstate(data_tok::DEQ_SUBM);
+ _page_cb_arr[_pg_index]._pdtokl->push_back(dtok);
+ done = true;
+ }
+
+ // Has the file header been written (i.e. write pointers still at 0)?
+ if (_wrfc.empty())
+ {
+ u_int32_t rec_dblks_rem = _enq_rec.rec_size_dblks() - data_offs_dblks;
+ bool file_fit = rec_dblks_rem <= JRNL_FILE_SIZE * JRNL_SBLK_SIZE;
+ bool file_full = rec_dblks_rem == JRNL_FILE_SIZE * JRNL_SBLK_SIZE;
+ size_t fro = 0;
+ if (cont)
+ {
+ if (file_fit && !file_full)
+ fro = (rec_dblks_rem + JRNL_SBLK_SIZE) * JRNL_DBLK_SIZE;
+ }
+ else
+ fro = JRNL_SBLK_SIZE * JRNL_DBLK_SIZE;
+ write_fhdr(rid, _wrfc.index(), fro);
+ }
+
+ // Is the page full? If so, flush.
+ if (_pg_offset_dblks >= JRNL_WMGR_PAGE_SIZE * JRNL_SBLK_SIZE)
+ {
+ res = write_flush();
+ assert(res == RHM_IORES_SUCCESS);
+
+ if (_page_cb_arr[_pg_index]._state == AIO_PENDING && !done)
+ {
+ res = RHM_IORES_AIO_WAIT;
+ dtok->set_wstate(data_tok::DEQ_PART);
+ done = true;
+ }
+
+ // File full?
+ if (_pg_cntr >= (JRNL_FILE_SIZE / JRNL_WMGR_PAGE_SIZE))
+ {
+ iores rfres = rotate_file();
+ if (rfres != RHM_IORES_SUCCESS)
+ res = rfres;
+ if (!done)
+ {
+ if (rfres == RHM_IORES_SUCCESS)
+ cont = true;
+ else
+ {
+ // Set last data_tok in page only to state ENQ_PART
+ dtok->set_wstate(data_tok::ENQ_PART);
+ done = true;
+ }
+ }
+ }
+ }
+ }
+ if (dtok->wstate() >= data_tok::DEQ_SUBM)
+ _deq_busy = false;
+
+// iores res = pre_write_check(false, dtok);
+// if (res != RHM_IORES_SUCCESS)
+// return res;
+//
+// u_int64_t rid;
+//
+// if (dtok->getSourceMessage())
+// {
+// rid = dtok->dequeue_rid();
+// assert(rid != 0);
+// }
+// else
+// rid = _wrfc.get_incr_rid();
+//
+// //***
+// // NOTE: ASSUMPTION: sizeof(deq_hdr) <= JRNL_DBLK_SIZE
+// // This encoding is a simplification: it assumes deq_hdr (currently 20 bytes) fits inside
+// // one dblk. The dblk is a tunable parameter, but is unlikely to go lower than deq_hdr
+// // (currently dblk = 128 bytes). JRNL_DBLK_SIZE must be a power of 2.
+// // IF JRNL_DBLK_SIZE IS SET TO < 32 (i.e. 16 OR LESS) BYTES, THIS ENCODING WILL FAIL!
+// //***
+// deq_hdr dhdr(RHM_JDAT_DEQ_MAGIC, rid, dtok->rid(), 0, RHM_JDAT_VERSION);
+// void* wptr = (void*)((char*)_page_ptr_arr[_pg_index] + _pg_offset_dblks * JRNL_DBLK_SIZE);
+// ::memcpy(wptr, &dhdr, sizeof(dhdr));
+// #ifdef RHM_CLEAN
+// ::memset((char*)wptr + sizeof(dhdr), RHM_CLEAN_CHAR, JRNL_DBLK_SIZE - sizeof(dhdr));
+// #endif
+// #if !(defined RHM_WRONLY || defined RHM_RDONLY)
+// u_int16_t fid = _emap.get_remove_fid(dtok->rid());
+// _wrfc.decr_enqcnt(fid);
+// #endif
+// _pg_offset_dblks++;
+// _cached_offset_dblks++;
+// // TODO: Incorrect - must set state to DEQ_CACHED; DEQ_SUBM is set when AIO returns.
+// dtok->set_wstate(data_tok::DEQ_SUBM);
+// _page_cb_arr[_pg_index]._pdtokl->push_back(dtok);
+// if (_wrfc.empty()) // Has the file_hdr been written?
+// write_fhdr(rid, _wrfc.index(), JRNL_SBLK_SIZE * JRNL_DBLK_SIZE);
+// if (_pg_offset_dblks >= JRNL_WMGR_PAGE_SIZE * JRNL_SBLK_SIZE)
+// res = flush();
+
return res;
}
Modified: store/trunk/cpp/lib/jrnl/wmgr.hpp
===================================================================
--- store/trunk/cpp/lib/jrnl/wmgr.hpp 2007-09-27 15:18:31 UTC (rev 948)
+++ store/trunk/cpp/lib/jrnl/wmgr.hpp 2007-09-27 19:29:25 UTC (rev 949)
@@ -77,6 +77,7 @@
std::deque<data_tok*> _ddtokl; ///< Deferred dequeue data_tok list
// TODO: Convert _enq_busy into a proper threadsafe lock
bool _enq_busy; ///< Flag true if enqueue is in progress
+ bool _deq_busy; ///< Flag true if dequeue is in progress
public:
wmgr(jcntl* jc, enq_map& emap, wrfc& wrfc);
Modified: store/trunk/cpp/tests/persistence.py
===================================================================
--- store/trunk/cpp/tests/persistence.py 2007-09-27 15:18:31 UTC (rev 948)
+++ store/trunk/cpp/tests/persistence.py 2007-09-27 19:29:25 UTC (rev 949)
@@ -354,6 +354,12 @@
self.channel.session_close()
+
+ # Crude fix to wait for thread in client to exit after return from session_close()
+ # Reduces occurrences of "Unhandled exception in thread" messages after each test
+ import time
+ time.sleep(1)
+
return True
17 years, 5 months
- 1
- 0
rhmessaging commits: r948 - in store/trunk/cpp: lib/jrnl and 1 other directories.
by rhmessaging-commits＠lists.jboss.org
Author: kpvdr
Date: 2007-09-27 11:18:31 -0400 (Thu, 27 Sep 2007)
New Revision: 948
Modified:
store/trunk/cpp/lib/BdbMessageStore.cpp
store/trunk/cpp/lib/jrnl/deq_rec.cpp
store/trunk/cpp/lib/jrnl/deq_rec.hpp
store/trunk/cpp/lib/jrnl/dtx_rec.cpp
store/trunk/cpp/lib/jrnl/dtx_rec.hpp
store/trunk/cpp/lib/jrnl/enq_rec.cpp
store/trunk/cpp/lib/jrnl/enq_rec.hpp
store/trunk/cpp/lib/jrnl/jcntl.cpp
store/trunk/cpp/lib/jrnl/jcntl.hpp
store/trunk/cpp/lib/jrnl/jrec.cpp
store/trunk/cpp/lib/jrnl/jrec.hpp
store/trunk/cpp/tests/jrnl/JournalSystemTests.cpp
store/trunk/cpp/tests/jrnl/msg_consumer.cpp
store/trunk/cpp/tests/jrnl/msg_producer.cpp
Log:
Encode and decode paths for enq_rec, dec_rec, dtx_rec completed. The encoding of xids not yet tested, however, but all tests which do not have xid still ok.
Modified: store/trunk/cpp/lib/BdbMessageStore.cpp
===================================================================
--- store/trunk/cpp/lib/BdbMessageStore.cpp 2007-09-25 20:25:49 UTC (rev 947)
+++ store/trunk/cpp/lib/BdbMessageStore.cpp 2007-09-27 15:18:31 UTC (rev 948)
@@ -415,7 +415,7 @@
std:: cout << "loop -- uses fixed size -> FIX <-" << std::endl;
// const iores read_data(void** dbuf, size_t& dbsize, data_tok* const dtok)
- rhm::journal::iores res = jc->read_next_data_record(&buff, buffSize, &dtokp);
+ rhm::journal::iores res = jc->read_data_record(&buff, buffSize, &dtokp);
readSize = dtokp.dsize();
assert(readSize < buffSize); /// fail safe for hack...
Modified: store/trunk/cpp/lib/jrnl/deq_rec.cpp
===================================================================
--- store/trunk/cpp/lib/jrnl/deq_rec.cpp 2007-09-25 20:25:49 UTC (rev 947)
+++ store/trunk/cpp/lib/jrnl/deq_rec.cpp 2007-09-27 15:18:31 UTC (rev 948)
@@ -32,11 +32,310 @@
#include <jrnl/deq_rec.hpp>
+#include <assert.h>
+#include <iomanip>
+#include <sstream>
+#include <jrnl/jerrno.hpp>
+
namespace rhm
{
namespace journal
{
+deq_rec::deq_rec():
+ _deq_hdr(),
+ _xid(),
+ _deq_tail()
+{}
+deq_rec::deq_rec(const u_int64_t rid, const u_int64_t drid, const std::string xid):
+ _deq_hdr(RHM_JDAT_DEQ_MAGIC, rid, drid, xid.size(), RHM_JDAT_VERSION),
+ _xid(xid),
+ _deq_tail(_deq_hdr._hdr)
+{}
+
+deq_rec::~deq_rec()
+{}
+
+u_int32_t
+deq_rec::encode(void* wptr, u_int32_t rec_offs_dblks, u_int32_t max_size_dblks) throw (jexception)
+{
+ assert(wptr != NULL);
+ assert(max_size_dblks > 0);
+ assert(_xid.size() == _deq_hdr._xidsize);
+
+ size_t rec_offs = rec_offs_dblks * JRNL_DBLK_SIZE;
+ size_t rem = max_size_dblks * JRNL_DBLK_SIZE;
+ size_t wr_cnt = 0;
+ if (rec_offs_dblks) // Continuation of split dequeue record (over 2 or more pages)
+ {
+ if (size_dblks(rec_size()) - rec_offs_dblks > max_size_dblks) // Further split required
+ {
+ rec_offs -= sizeof(_deq_hdr);
+ size_t wsize = _deq_hdr._xidsize > rec_offs ? _deq_hdr._xidsize - rec_offs : 0;
+ size_t wsize2 = wsize;
+ if (wsize)
+ {
+ if (wsize > rem)
+ wsize = rem;
+ ::memcpy(wptr, (const char*)_xid.c_str() + rec_offs, wsize);
+ wr_cnt += wsize;
+ rem -= wsize;
+ }
+ rec_offs -= _deq_hdr._xidsize - wsize2;
+ if (rem)
+ {
+ wsize = sizeof(_deq_tail) > rec_offs ? sizeof(_deq_tail) - rec_offs : 0;
+ wsize2 = wsize;
+ if (wsize)
+ {
+ if (wsize > rem)
+ wsize = rem;
+ ::memcpy((char*)wptr + wr_cnt, (char*)&_deq_tail + rec_offs, wsize);
+ wr_cnt += wsize;
+ rem -= wsize;
+ }
+ rec_offs -= sizeof(_deq_tail) - wsize2;
+ }
+ assert(rem == 0);
+ assert(rec_offs == 0);
+ }
+ else // No further split required
+ {
+ rec_offs -= sizeof(_deq_hdr);
+ size_t wsize = _deq_hdr._xidsize > rec_offs ? _deq_hdr._xidsize - rec_offs : 0;
+ if (wsize)
+ {
+ ::memcpy(wptr, _xid.c_str() + rec_offs, wsize);
+ wr_cnt += wsize;
+ }
+ rec_offs -= _deq_hdr._xidsize - wsize;
+ wsize = sizeof(_deq_tail) > rec_offs ? sizeof(_deq_tail) - rec_offs : 0;
+ if (wsize)
+ {
+ ::memcpy((char*)wptr + wr_cnt, (char*)&_deq_tail + rec_offs, wsize);
+ wr_cnt += wsize;
+#ifdef RHM_CLEAN
+ ::memset((char*)wptr + wr_cnt, RHM_CLEAN_CHAR,
+ (size_dblks(rec_size()) * JRNL_DBLK_SIZE) -
+ (rec_offs_dblks * JRNL_DBLK_SIZE) - wr_cnt);
+#endif
+ }
+ rec_offs -= sizeof(_deq_tail) - wsize;
+ assert(rec_offs == 0);
+ }
+ }
+ else // Start at beginning of data record
+ {
+ // Assumption: the header will always fit into the first dblk
+ ::memcpy(wptr, (void*)&_deq_hdr, sizeof(_deq_hdr));
+ wr_cnt = sizeof(_deq_hdr);
+ if (size_dblks(rec_size()) > max_size_dblks) // Split required - can only occur with xid
+ {
+ size_t wsize;
+ rem -= sizeof(_deq_hdr);
+ if (rem)
+ {
+ wsize = rem >= _deq_hdr._xidsize ? _deq_hdr._xidsize : rem;
+ ::memcpy((char*)wptr + wr_cnt, _xid.c_str(), wsize);
+ wr_cnt += wsize;
+ rem -= wsize;
+ }
+ if (rem)
+ {
+ wsize = rem >= sizeof(_deq_tail) ? sizeof(_deq_tail) : rem;
+ ::memcpy((char*)wptr + wr_cnt, (void*)&_deq_tail, wsize);
+ wr_cnt += wsize;
+ rem -= wsize;
+ }
+ assert(rem == 0);
+ }
+ else // No split required
+ {
+ if (_deq_hdr._xidsize)
+ {
+ ::memcpy((char*)wptr + wr_cnt, _xid.c_str(), _deq_hdr._xidsize);
+ wr_cnt += _deq_hdr._xidsize;
+ ::memcpy((char*)wptr + wr_cnt, (void*)&_deq_tail, sizeof(_deq_tail));
+ wr_cnt += sizeof(_deq_tail);
+ }
+#ifdef RHM_CLEAN
+ ::memset((char*)wptr + wr_cnt, RHM_CLEAN_CHAR,
+ (size_dblks(rec_size()) * JRNL_DBLK_SIZE) - wr_cnt);
+#endif
+ }
+ }
+ return size_dblks(wr_cnt);
+}
+
+u_int32_t
+deq_rec::decode(hdr& h, void* rptr, u_int32_t rec_offs_dblks, u_int32_t max_size_dblks)
+ throw (jexception)
+{
+ assert(rptr != NULL);
+ assert(max_size_dblks > 0);
+
+ size_t rd_cnt = 0;
+ if (rec_offs_dblks) // Continuation of record on new page
+ {
+ const u_int32_t hdr_xid_dblks = size_dblks(deq_hdr::size() + _deq_hdr._xidsize);
+ const u_int32_t hdr_xid_tail_dblks = size_dblks(enq_hdr::size() + _deq_hdr._xidsize +
+ rec_tail::size());
+ const size_t rec_offs = rec_offs_dblks * JRNL_DBLK_SIZE;
+
+ if (hdr_xid_tail_dblks - rec_offs_dblks <= max_size_dblks)
+ {
+ // Remainder of xid fits within this page
+ if (rec_offs - deq_hdr::size() < _deq_hdr._xidsize)
+ {
+ // Part of xid still outstanding, copy remainder of xid and tail
+ const size_t xid_offs = rec_offs - deq_hdr::size();
+ const size_t xid_rem = _deq_hdr._xidsize - xid_offs;
+ _xid.append((char*)rptr, xid_rem);
+ rd_cnt = xid_rem;
+ ::memcpy((void*)&_deq_tail, ((char*)rptr + rd_cnt), sizeof(_deq_tail));
+ chk_tail();
+ rd_cnt += sizeof(_deq_tail);
+ }
+ else
+ {
+ // Tail or part of tail only outstanding, complete tail
+ const size_t tail_offs = rec_offs - deq_hdr::size() - _deq_hdr._xidsize;
+ const size_t tail_rem = rec_tail::size() - tail_offs;
+ ::memcpy((char*)&_deq_tail + tail_offs, rptr, tail_rem);
+ chk_tail();
+ rd_cnt = tail_rem;
+ }
+ }
+ else if (hdr_xid_dblks - rec_offs_dblks <= max_size_dblks)
+ {
+ // Remainder of xid fits within this page, tail split
+ const size_t xid_offs = rec_offs - deq_hdr::size();
+ const size_t xid_rem = _deq_hdr._xidsize - xid_offs;
+ _xid.append((char*)rptr, xid_rem);
+ rd_cnt += xid_rem;
+ const size_t tail_rem = (max_size_dblks * JRNL_DBLK_SIZE) - rd_cnt;
+ if (tail_rem)
+ {
+ ::memcpy((void*)&_deq_tail, ((char*)rptr + xid_rem), tail_rem);
+ rd_cnt += tail_rem;
+ }
+ }
+ else
+ {
+ // Remainder of xid split
+ const size_t xid_cp_size = (max_size_dblks * JRNL_DBLK_SIZE);
+ _xid.append((char*)rptr, xid_cp_size);
+ rd_cnt += xid_cp_size;
+ }
+ }
+ else // Start of record
+ {
+ // Get and check header
+ _deq_hdr._hdr.copy(h);
+ rd_cnt = sizeof(hdr);
+ _deq_hdr._deq_rid = *(u_int64_t*)((char*)rptr + rd_cnt);
+ rd_cnt += sizeof(u_int64_t);
+#if defined(JRNL_BIG_ENDIAN) && defined(JRNL_32_BIT)
+ rd_cnt += sizeof(u_int32_t); // Filler 0
+#endif
+ _deq_hdr._xidsize = *(size_t*)((char*)rptr + rd_cnt);
+ rd_cnt = _deq_hdr.size();
+ chk_hdr();
+ if (_deq_hdr._xidsize)
+ {
+ const u_int32_t hdr_xid_dblks = size_dblks(deq_hdr::size() + _deq_hdr._xidsize);
+ const u_int32_t hdr_xid_tail_dblks = size_dblks(enq_hdr::size() + _deq_hdr._xidsize +
+ rec_tail::size());
+
+ // Check if record (header + xid + tail) fits within this page, we can check the
+ // tail before the expense of copying data to memory
+ if (hdr_xid_tail_dblks <= max_size_dblks)
+ {
+ // Entire header, xid and tail fits within this page
+ ::memcpy((void*)&_deq_tail, (char*)rptr + rd_cnt + _deq_hdr._xidsize,
+ sizeof(_deq_tail));
+ chk_tail();
+ _xid.assign((char*)rptr + rd_cnt, _deq_hdr._xidsize);
+ rd_cnt += _deq_hdr._xidsize + sizeof(_deq_tail);
+ }
+ else if (hdr_xid_dblks <= max_size_dblks)
+ {
+ // Entire header and xid fit within this page, tail split
+ _xid.assign((char*)rptr + rd_cnt, _deq_hdr._xidsize);
+ rd_cnt += _deq_hdr._xidsize;
+ const size_t tail_rem = (max_size_dblks * JRNL_DBLK_SIZE) - rd_cnt;
+ if (tail_rem)
+ {
+ ::memcpy((void*)&_deq_tail, (char*)rptr + rd_cnt, tail_rem);
+ rd_cnt += tail_rem;
+ }
+ }
+ else
+ {
+ // Header fits within this page, xid split
+ const size_t xid_cp_size = (max_size_dblks * JRNL_DBLK_SIZE) - rd_cnt;
+ _xid.assign((char*)rptr + rd_cnt, xid_cp_size);
+ rd_cnt += xid_cp_size;
+ }
+ }
+ }
+ return size_dblks(rd_cnt);
+}
+
+std::string&
+deq_rec::str(std::string& str) const
+{
+ std::stringstream ss;
+ ss << "deq_rec: m=" << _deq_hdr._hdr._magic;
+ ss << " v=" << (int)_deq_hdr._hdr._version;
+ ss << " rid=" << _deq_hdr._hdr._rid;
+ ss << " dren=" << _deq_hdr._deq_rid;
+ if (_deq_hdr._xidsize)
+ ss << " xid=\"" << _xid << "\"";
+ str.append(ss.str());
+ return str;
+}
+
+const size_t
+deq_rec::xid_size() const
+{
+ return _deq_hdr._xidsize;
+}
+
+const size_t
+deq_rec::rec_size() const
+{
+ return deq_hdr::size() + (_deq_hdr._xidsize ? _deq_hdr._xidsize + rec_tail::size() : 0);
+}
+
+void
+deq_rec::chk_hdr() const throw (jexception)
+{
+ jrec::chk_hdr(_deq_hdr._hdr);
+ if (_deq_hdr._hdr._magic != RHM_JDAT_DEQ_MAGIC)
+ {
+ std::stringstream ss;
+ ss << std::hex << std::setfill('0');
+ ss << "deq magic: rid=0x" << std::setw(16) << _deq_hdr._hdr._rid;
+ ss << ": expected=0x" << std::setw(8) << RHM_JDAT_DEQ_MAGIC;
+ ss << " read=0x" << std::setw(2) << (int)_deq_hdr._hdr._magic;
+ throw jexception(jerrno::JERR_DREC_INVRHDR, ss.str(), "deq_rec", "chk_hdr");
+ }
+}
+
+void
+deq_rec::chk_hdr(u_int64_t rid) const throw (jexception)
+{
+ chk_hdr();
+ jrec::chk_rid(_deq_hdr._hdr, rid);
+}
+
+void
+deq_rec::chk_tail() const throw (jexception)
+{
+ jrec::chk_tail(_deq_tail, _deq_hdr._hdr);
+}
+
} // namespace journal
} // namespace rhm
Modified: store/trunk/cpp/lib/jrnl/deq_rec.hpp
===================================================================
--- store/trunk/cpp/lib/jrnl/deq_rec.hpp 2007-09-25 20:25:49 UTC (rev 947)
+++ store/trunk/cpp/lib/jrnl/deq_rec.hpp 2007-09-27 15:18:31 UTC (rev 948)
@@ -54,6 +54,29 @@
*/
class deq_rec : public jrec
{
+ private:
+ deq_hdr _deq_hdr; ///< Dequeue header
+ std::string _xid; ///< XID - empty string means no XID is set
+ rec_tail _deq_tail; ///< Record tail, only encoded if XID is present
+
+ public:
+ deq_rec();
+ deq_rec(const u_int64_t rid, const u_int64_t drid, const std::string xid);
+ ~deq_rec();
+
+ u_int32_t encode(void* wptr, u_int32_t rec_offs_dblks, u_int32_t max_size_dblks)
+ throw (jexception);
+ u_int32_t decode(hdr& h, void* rptr, u_int32_t rec_offs_dblks, u_int32_t max_size_dblks)
+ throw (jexception);
+ std::string& str(std::string& str) const;
+ inline const size_t data_size() const { return 0; } // This record never carries data
+ const size_t xid_size() const;
+ const size_t rec_size() const;
+
+ private:
+ virtual void chk_hdr() const throw (jexception) = 0;
+ virtual void chk_hdr(u_int64_t rid) const throw (jexception) = 0;
+ virtual void chk_tail() const throw (jexception) = 0;
}; // class deq_rec
} // namespace journal
Modified: store/trunk/cpp/lib/jrnl/dtx_rec.cpp
===================================================================
--- store/trunk/cpp/lib/jrnl/dtx_rec.cpp 2007-09-25 20:25:49 UTC (rev 947)
+++ store/trunk/cpp/lib/jrnl/dtx_rec.cpp 2007-09-27 15:18:31 UTC (rev 948)
@@ -32,11 +32,308 @@
#include <jrnl/dtx_rec.hpp>
+#include <assert.h>
+#include <iomanip>
+#include <sstream>
+#include <jrnl/jerrno.hpp>
+
namespace rhm
{
namespace journal
{
+dtx_rec::dtx_rec():
+ _dtx_hdr(),
+ _xid(),
+ _dtx_tail()
+{}
+dtx_rec::dtx_rec(const u_int32_t magic, const u_int64_t rid, const std::string xid):
+ _dtx_hdr(magic, rid, xid.size(), RHM_JDAT_VERSION),
+ _xid(xid),
+ _dtx_tail(_dtx_hdr._hdr)
+{}
+
+dtx_rec::~dtx_rec()
+{}
+
+u_int32_t
+dtx_rec::encode(void* wptr, u_int32_t rec_offs_dblks, u_int32_t max_size_dblks) throw (jexception)
+{
+ assert(wptr != NULL);
+ assert(max_size_dblks > 0);
+ assert(_xid.size() == _dtx_hdr._xidsize);
+ assert(_dtx_hdr._xidsize > 0);
+
+ size_t rec_offs = rec_offs_dblks * JRNL_DBLK_SIZE;
+ size_t rem = max_size_dblks * JRNL_DBLK_SIZE;
+ size_t wr_cnt = 0;
+ if (rec_offs_dblks) // Continuation of split dequeue record (over 2 or more pages)
+ {
+ if (size_dblks(rec_size()) - rec_offs_dblks > max_size_dblks) // Further split required
+ {
+ rec_offs -= sizeof(_dtx_hdr);
+ size_t wsize = _dtx_hdr._xidsize > rec_offs ? _dtx_hdr._xidsize - rec_offs : 0;
+ size_t wsize2 = wsize;
+ if (wsize)
+ {
+ if (wsize > rem)
+ wsize = rem;
+ ::memcpy(wptr, (const char*)_xid.c_str() + rec_offs, wsize);
+ wr_cnt += wsize;
+ rem -= wsize;
+ }
+ rec_offs -= _dtx_hdr._xidsize - wsize2;
+ if (rem)
+ {
+ wsize = sizeof(_dtx_tail) > rec_offs ? sizeof(_dtx_tail) - rec_offs : 0;
+ wsize2 = wsize;
+ if (wsize)
+ {
+ if (wsize > rem)
+ wsize = rem;
+ ::memcpy((char*)wptr + wr_cnt, (char*)&_dtx_tail + rec_offs, wsize);
+ wr_cnt += wsize;
+ rem -= wsize;
+ }
+ rec_offs -= sizeof(_dtx_tail) - wsize2;
+ }
+ assert(rem == 0);
+ assert(rec_offs == 0);
+ }
+ else // No further split required
+ {
+ rec_offs -= sizeof(_dtx_hdr);
+ size_t wsize = _dtx_hdr._xidsize > rec_offs ? _dtx_hdr._xidsize - rec_offs : 0;
+ if (wsize)
+ {
+ ::memcpy(wptr, _xid.c_str() + rec_offs, wsize);
+ wr_cnt += wsize;
+ }
+ rec_offs -= _dtx_hdr._xidsize - wsize;
+ wsize = sizeof(_dtx_tail) > rec_offs ? sizeof(_dtx_tail) - rec_offs : 0;
+ if (wsize)
+ {
+ ::memcpy((char*)wptr + wr_cnt, (char*)&_dtx_tail + rec_offs, wsize);
+ wr_cnt += wsize;
+#ifdef RHM_CLEAN
+ ::memset((char*)wptr + wr_cnt, RHM_CLEAN_CHAR,
+ (size_dblks(rec_size()) * JRNL_DBLK_SIZE) -
+ (rec_offs_dblks * JRNL_DBLK_SIZE) - wr_cnt);
+#endif
+ }
+ rec_offs -= sizeof(_dtx_tail) - wsize;
+ assert(rec_offs == 0);
+ }
+ }
+ else // Start at beginning of data record
+ {
+ // Assumption: the header will always fit into the first dblk
+ ::memcpy(wptr, (void*)&_dtx_hdr, sizeof(_dtx_hdr));
+ wr_cnt = sizeof(_dtx_hdr);
+ if (size_dblks(rec_size()) > max_size_dblks) // Split required
+ {
+ size_t wsize;
+ rem -= sizeof(_dtx_hdr);
+ if (rem)
+ {
+ wsize = rem >= _dtx_hdr._xidsize ? _dtx_hdr._xidsize : rem;
+ ::memcpy((char*)wptr + wr_cnt, _xid.c_str(), wsize);
+ wr_cnt += wsize;
+ rem -= wsize;
+ }
+ if (rem)
+ {
+ wsize = rem >= sizeof(_dtx_tail) ? sizeof(_dtx_tail) : rem;
+ ::memcpy((char*)wptr + wr_cnt, (void*)&_dtx_tail, wsize);
+ wr_cnt += wsize;
+ rem -= wsize;
+ }
+ assert(rem == 0);
+ }
+ else // No split required
+ {
+ if (_dtx_hdr._xidsize)
+ {
+ ::memcpy((char*)wptr + wr_cnt, _xid.c_str(), _dtx_hdr._xidsize);
+ wr_cnt += _dtx_hdr._xidsize;
+ ::memcpy((char*)wptr + wr_cnt, (void*)&_dtx_tail, sizeof(_dtx_tail));
+ wr_cnt += sizeof(_dtx_tail);
+ }
+#ifdef RHM_CLEAN
+ ::memset((char*)wptr + wr_cnt, RHM_CLEAN_CHAR,
+ (size_dblks(rec_size()) * JRNL_DBLK_SIZE) - wr_cnt);
+#endif
+ }
+ }
+ return size_dblks(wr_cnt);
+}
+
+u_int32_t
+dtx_rec::decode(hdr& h, void* rptr, u_int32_t rec_offs_dblks, u_int32_t max_size_dblks)
+ throw (jexception)
+{
+ assert(rptr != NULL);
+ assert(max_size_dblks > 0);
+
+ size_t rd_cnt = 0;
+ if (rec_offs_dblks) // Continuation of record on new page
+ {
+ const u_int32_t hdr_xid_dblks = size_dblks(deq_hdr::size() + _dtx_hdr._xidsize);
+ const u_int32_t hdr_xid_tail_dblks = size_dblks(enq_hdr::size() + _dtx_hdr._xidsize +
+ rec_tail::size());
+ const size_t rec_offs = rec_offs_dblks * JRNL_DBLK_SIZE;
+
+ if (hdr_xid_tail_dblks - rec_offs_dblks <= max_size_dblks)
+ {
+ // Remainder of xid fits within this page
+ if (rec_offs - deq_hdr::size() < _dtx_hdr._xidsize)
+ {
+ // Part of xid still outstanding, copy remainder of xid and tail
+ const size_t xid_offs = rec_offs - deq_hdr::size();
+ const size_t xid_rem = _dtx_hdr._xidsize - xid_offs;
+ _xid.append((char*)rptr, xid_rem);
+ rd_cnt = xid_rem;
+ ::memcpy((void*)&_dtx_tail, ((char*)rptr + rd_cnt), sizeof(_dtx_tail));
+ chk_tail();
+ rd_cnt += sizeof(_dtx_tail);
+ }
+ else
+ {
+ // Tail or part of tail only outstanding, complete tail
+ const size_t tail_offs = rec_offs - deq_hdr::size() - _dtx_hdr._xidsize;
+ const size_t tail_rem = rec_tail::size() - tail_offs;
+ ::memcpy((char*)&_dtx_tail + tail_offs, rptr, tail_rem);
+ chk_tail();
+ rd_cnt = tail_rem;
+ }
+ }
+ else if (hdr_xid_dblks - rec_offs_dblks <= max_size_dblks)
+ {
+ // Remainder of xid fits within this page, tail split
+ const size_t xid_offs = rec_offs - deq_hdr::size();
+ const size_t xid_rem = _dtx_hdr._xidsize - xid_offs;
+ _xid.append((char*)rptr, xid_rem);
+ rd_cnt += xid_rem;
+ const size_t tail_rem = (max_size_dblks * JRNL_DBLK_SIZE) - rd_cnt;
+ if (tail_rem)
+ {
+ ::memcpy((void*)&_dtx_tail, ((char*)rptr + xid_rem), tail_rem);
+ rd_cnt += tail_rem;
+ }
+ }
+ else
+ {
+ // Remainder of xid split
+ const size_t xid_cp_size = (max_size_dblks * JRNL_DBLK_SIZE);
+ _xid.append((char*)rptr, xid_cp_size);
+ rd_cnt += xid_cp_size;
+ }
+ }
+ else // Start of record
+ {
+ // Get and check header
+ _dtx_hdr._hdr.copy(h);
+ rd_cnt = sizeof(hdr);
+#if defined(JRNL_BIG_ENDIAN) && defined(JRNL_32_BIT)
+ rd_cnt += sizeof(u_int32_t); // Filler 0
+#endif
+ _dtx_hdr._xidsize = *(size_t*)((char*)rptr + rd_cnt);
+ rd_cnt = _dtx_hdr.size();
+ chk_hdr();
+ const u_int32_t hdr_xid_dblks = size_dblks(deq_hdr::size() + _dtx_hdr._xidsize);
+ const u_int32_t hdr_xid_tail_dblks = size_dblks(enq_hdr::size() + _dtx_hdr._xidsize +
+ rec_tail::size());
+
+ // Check if record (header + xid + tail) fits within this page, we can check the
+ // tail before the expense of copying data to memory
+ if (hdr_xid_tail_dblks <= max_size_dblks)
+ {
+ // Entire header, xid and tail fits within this page
+ ::memcpy((void*)&_dtx_tail, (char*)rptr + rd_cnt + _dtx_hdr._xidsize,
+ sizeof(_dtx_tail));
+ chk_tail();
+ _xid.assign((char*)rptr + rd_cnt, _dtx_hdr._xidsize);
+ rd_cnt += _dtx_hdr._xidsize + sizeof(_dtx_tail);
+ }
+ else if (hdr_xid_dblks <= max_size_dblks)
+ {
+ // Entire header and xid fit within this page, tail split
+ _xid.assign((char*)rptr + rd_cnt, _dtx_hdr._xidsize);
+ rd_cnt += _dtx_hdr._xidsize;
+ const size_t tail_rem = (max_size_dblks * JRNL_DBLK_SIZE) - rd_cnt;
+ if (tail_rem)
+ {
+ ::memcpy((void*)&_dtx_tail, (char*)rptr + rd_cnt, tail_rem);
+ rd_cnt += tail_rem;
+ }
+ }
+ else
+ {
+ // Header fits within this page, xid split
+ const size_t xid_cp_size = (max_size_dblks * JRNL_DBLK_SIZE) - rd_cnt;
+ _xid.assign((char*)rptr + rd_cnt, xid_cp_size);
+ rd_cnt += xid_cp_size;
+ }
+ }
+ return size_dblks(rd_cnt);
+}
+
+std::string&
+dtx_rec::str(std::string& str) const
+{
+ std::stringstream ss;
+ if (_dtx_hdr._hdr._magic == RHM_JDAT_DTXA_MAGIC)
+ ss << "dtxa_rec: m=" << _dtx_hdr._hdr._magic;
+ else
+ ss << "dtxc_rec: m=" << _dtx_hdr._hdr._magic;
+ ss << " v=" << (int)_dtx_hdr._hdr._version;
+ ss << " rid=" << _dtx_hdr._hdr._rid;
+ ss << " xid=\"" << _xid << "\"";
+ str.append(ss.str());
+ return str;
+}
+
+const size_t
+dtx_rec::xid_size() const
+{
+ return _dtx_hdr._xidsize;
+}
+
+const size_t
+dtx_rec::rec_size() const
+{
+ return deq_hdr::size() + _dtx_hdr._xidsize + rec_tail::size();
+}
+
+void
+dtx_rec::chk_hdr() const throw (jexception)
+{
+ jrec::chk_hdr(_dtx_hdr._hdr);
+ if (_dtx_hdr._hdr._magic != RHM_JDAT_DTXA_MAGIC && _dtx_hdr._hdr._magic != RHM_JDAT_DTXC_MAGIC)
+ {
+ std::stringstream ss;
+ ss << std::hex << std::setfill('0');
+ ss << "dtx magic: rid=0x" << std::setw(16) << _dtx_hdr._hdr._rid;
+ ss << ": expected=(0x" << std::setw(8) << RHM_JDAT_DTXA_MAGIC;
+ ss << " or 0x" << RHM_JDAT_DTXC_MAGIC;
+ ss << ") read=0x" << std::setw(2) << (int)_dtx_hdr._hdr._magic;
+ throw jexception(jerrno::JERR_DREC_INVRHDR, ss.str(), "dtx_rec", "chk_hdr");
+ }
+}
+
+void
+dtx_rec::chk_hdr(u_int64_t rid) const throw (jexception)
+{
+ chk_hdr();
+ jrec::chk_rid(_dtx_hdr._hdr, rid);
+}
+
+void
+dtx_rec::chk_tail() const throw (jexception)
+{
+ jrec::chk_tail(_dtx_tail, _dtx_hdr._hdr);
+}
+
} // namespace journal
} // namespace rhm
Modified: store/trunk/cpp/lib/jrnl/dtx_rec.hpp
===================================================================
--- store/trunk/cpp/lib/jrnl/dtx_rec.hpp 2007-09-25 20:25:49 UTC (rev 947)
+++ store/trunk/cpp/lib/jrnl/dtx_rec.hpp 2007-09-27 15:18:31 UTC (rev 948)
@@ -54,6 +54,29 @@
*/
class dtx_rec : public jrec
{
+ private:
+ dtx_hdr _dtx_hdr; ///< DTX header
+ std::string _xid; ///< XID
+ rec_tail _dtx_tail; ///< Record tail
+
+ public:
+ dtx_rec();
+ dtx_rec(const u_int32_t magic, const u_int64_t rid, const std::string xid);
+ ~dtx_rec();
+
+ u_int32_t encode(void* wptr, u_int32_t rec_offs_dblks, u_int32_t max_size_dblks)
+ throw (jexception);
+ u_int32_t decode(hdr& h, void* rptr, u_int32_t rec_offs_dblks, u_int32_t max_size_dblks)
+ throw (jexception);
+ std::string& str(std::string& str) const;
+ inline const size_t data_size() const { return 0; } // This record never carries data
+ const size_t xid_size() const;
+ const size_t rec_size() const;
+
+ private:
+ void chk_hdr() const throw (jexception);
+ void chk_hdr(u_int64_t rid) const throw (jexception);
+ void chk_tail() const throw (jexception);
}; // class dtx_rec
} // namespace journal
Modified: store/trunk/cpp/lib/jrnl/enq_rec.cpp
===================================================================
--- store/trunk/cpp/lib/jrnl/enq_rec.cpp 2007-09-25 20:25:49 UTC (rev 947)
+++ store/trunk/cpp/lib/jrnl/enq_rec.cpp 2007-09-27 15:18:31 UTC (rev 948)
@@ -47,6 +47,7 @@
enq_rec::enq_rec():
jrec(), // superclass
_enq_hdr(RHM_JDAT_ENQ_MAGIC, 0, 0, 0, RHM_JDAT_VERSION),
+ _xid(),
_data(NULL),
_buff(NULL),
_enq_tail(_enq_hdr._hdr),
@@ -59,12 +60,14 @@
enq_rec::enq_rec(const u_int64_t rid, const void* const dbuf, const size_t dlen):
jrec(), // superclass
_enq_hdr(RHM_JDAT_ENQ_MAGIC, rid, 0, dlen, RHM_JDAT_VERSION),
+ _xid(),
_data(dbuf),
_buff(NULL),
_enq_tail(_enq_hdr._hdr),
_max_data_size(0),
_data_size(dlen),
- _rec_size(size_dblks(dlen + enq_hdr::size() + rec_tail::size()) * JRNL_DBLK_SIZE)
+ _rec_size(size_dblks(enq_hdr::size() + _enq_hdr._xidsize + dlen + rec_tail::size()) *
+ JRNL_DBLK_SIZE)
{}
// Constructor used for read operations, where buf contains preallocated space
@@ -72,6 +75,7 @@
enq_rec::enq_rec(void* const buf, const size_t bufsize):
jrec(), // superclass
_enq_hdr(RHM_JDAT_ENQ_MAGIC, 0, 0, bufsize, RHM_JDAT_VERSION),
+ _xid(),
_data(NULL),
_buff(buf),
_enq_tail(_enq_hdr._hdr),
@@ -127,6 +131,7 @@
{
assert(wptr != NULL);
assert(max_size_dblks > 0);
+ assert(_xid.size() == _enq_hdr._xidsize);
size_t rec_offs = rec_offs_dblks * JRNL_DBLK_SIZE;
size_t rem = max_size_dblks * JRNL_DBLK_SIZE;
@@ -135,19 +140,18 @@
{
if (size_dblks(rec_size()) - rec_offs_dblks > max_size_dblks) // Further split required
{
- // TODO: The header will never be split as it will always be less than one dblk.
- // Remove this check...
- size_t wsize = sizeof(_enq_hdr) > rec_offs ? sizeof(_enq_hdr) - rec_offs : 0;
+ rec_offs -= sizeof(_enq_hdr);
+ size_t wsize = _enq_hdr._xidsize > rec_offs ? _enq_hdr._xidsize - rec_offs : 0;
size_t wsize2 = wsize;
if (wsize)
{
if (wsize > rem)
wsize = rem;
- ::memcpy(wptr, (char*)&_enq_hdr + rec_offs, wsize);
+ ::memcpy(wptr, (const char*)_xid.c_str() + rec_offs, wsize);
wr_cnt = wsize;
rem -= wsize;
}
- rec_offs -= sizeof(_enq_hdr) - wsize2;
+ rec_offs -= _enq_hdr._xidsize - wsize2;
if (rem)
{
wsize = _data_size > rec_offs ? _data_size - rec_offs : 0;
@@ -181,15 +185,14 @@
}
else // No further split required
{
- // TODO: The header will never be split as it will always be less than one dblk.
- // Remove this check...
- size_t wsize = sizeof(_enq_hdr) > rec_offs ? sizeof(_enq_hdr) - rec_offs : 0;
+ rec_offs -= sizeof(_enq_hdr);
+ size_t wsize = _enq_hdr._xidsize > rec_offs ? _enq_hdr._xidsize - rec_offs : 0;
if (wsize)
{
- ::memcpy(wptr, (char*)&_enq_hdr + rec_offs, wsize);
- wr_cnt = wsize;
+ ::memcpy(wptr, _xid.c_str() + rec_offs, wsize);
+ wr_cnt += wsize;
}
- rec_offs -= sizeof(_enq_hdr) - wsize;
+ rec_offs -= _enq_hdr._xidsize - wsize;
wsize = _data_size > rec_offs ? _data_size - rec_offs : 0;
if (wsize)
{
@@ -213,14 +216,22 @@
}
else // Start at beginning of data record
{
+ // Assumption: the header will always fit into the first dblk
+ ::memcpy(wptr, (void*)&_enq_hdr, sizeof(_enq_hdr));
+ wr_cnt = sizeof(_enq_hdr);
if (size_dblks(rec_size()) > max_size_dblks) // Split required
{
- size_t wsize = rem >= sizeof(_enq_hdr) ? sizeof(_enq_hdr) : rem;
- ::memcpy(wptr, (void*)&_enq_hdr, wsize);
- wr_cnt = wsize;
- rem -= wsize;
+ size_t wsize;
+ rem -= sizeof(_enq_hdr);
if (rem)
{
+ wsize = rem >= _enq_hdr._xidsize ? _enq_hdr._xidsize : rem;
+ ::memcpy((char*)wptr + wr_cnt, _xid.c_str(), wsize);
+ wr_cnt += wsize;
+ rem -= wsize;
+ }
+ if (rem)
+ {
wsize = rem >= _data_size ? _data_size : rem;
::memcpy((char*)wptr + wr_cnt, _data, wsize);
wr_cnt += wsize;
@@ -237,8 +248,11 @@
}
else // No split required
{
- ::memcpy(wptr, (void*)&_enq_hdr, sizeof(_enq_hdr));
- wr_cnt = sizeof(_enq_hdr);
+ if (_enq_hdr._xidsize)
+ {
+ ::memcpy((char*)wptr + wr_cnt, _xid.c_str(), _xid.size());
+ wr_cnt += _xid.size();
+ }
::memcpy((char*)wptr + wr_cnt, _data, _data_size);
wr_cnt += _data_size;
::memcpy((char*)wptr + wr_cnt, (void*)&_enq_tail, sizeof(_enq_tail));
@@ -277,7 +291,7 @@
::memcpy((char*)_buff + data_offs, rptr, data_rem);
rd_cnt += data_rem;
::memcpy((void*)&_enq_tail, ((char*)rptr + data_rem), sizeof(_enq_tail));
- chk_tail(_enq_tail, _enq_hdr);
+ chk_tail();
rd_cnt += sizeof(_enq_tail);
}
else
@@ -286,7 +300,7 @@
const size_t tail_offs = rec_offs - enq_hdr::size() - _data_size;
const size_t tail_rem = rec_tail::size() - tail_offs;
::memcpy((char*)&_enq_tail + tail_offs, rptr, tail_rem);
- chk_tail(_enq_tail, _enq_hdr);
+ chk_tail();
rd_cnt = tail_rem;
}
}
@@ -316,32 +330,35 @@
{
// Get and check header
_enq_hdr._hdr.copy(h);
- _enq_hdr._xidsize = *(size_t*)((char*)rptr + sizeof(hdr));
- _enq_hdr._dsize = *(size_t*)((char*)rptr + sizeof(hdr) +
+ rd_cnt = sizeof(hdr);
#if defined(JRNL_BIG_ENDIAN) && defined(JRNL_32_BIT)
- sizeof(u_int32_t) +
+ rd_cnt += sizeof(u_int32_t); // Filler 0
#endif
- sizeof(size_t)
+ _enq_hdr._xidsize = *(size_t*)((char*)rptr + rd_cnt);
+ rd_cnt += sizeof(size_t);
#if defined(JRNL_LITTLE_ENDIAN) && defined(JRNL_32_BIT)
- + sizeof(u_int32_t)
+ rd_cnt += sizeof(u_int32_t); // Filler 0
#endif
- );
+#if defined(JRNL_BIG_ENDIAN) && defined(JRNL_32_BIT)
+ rd_cnt += sizeof(u_int32_t); // Filler 1
+#endif
+ _enq_hdr._dsize = *(size_t*)((char*)rptr + rd_cnt);
rd_cnt = _enq_hdr.size();
- chk_hdr(_enq_hdr);
+ chk_hdr();
_data_size = _enq_hdr._dsize;
- const u_int32_t hdr_data_dblks = size_dblks(enq_hdr::size() + _data_size);
- const u_int32_t hdr_data_tail_dblks = size_dblks(enq_hdr::size() + _data_size +
- rec_tail::size());
if (_data_size)
{
+ const u_int32_t hdr_data_dblks = size_dblks(enq_hdr::size() + _data_size);
+ const u_int32_t hdr_data_tail_dblks = size_dblks(enq_hdr::size() + _data_size +
+ rec_tail::size());
// Check if record (header + data + tail) fits within this page, we can check the
// tail before the expense of copying data to memory
if (hdr_data_tail_dblks <= max_size_dblks)
{
// Entire header, data and tail fits within this page
::memcpy((void*)&_enq_tail, (char*)rptr + rd_cnt + _data_size, sizeof(_enq_tail));
- chk_tail(_enq_tail, _enq_hdr);
+ chk_tail();
::memcpy(_buff, (char*)rptr + rd_cnt, _data_size);
rd_cnt += _data_size + sizeof(_enq_tail);
}
@@ -376,6 +393,8 @@
ss << "enq_rec: m=" << _enq_hdr._hdr._magic;
ss << " v=" << (int)_enq_hdr._hdr._version;
ss << " rid=" << _enq_hdr._hdr._rid;
+ if (_xid.size())
+ ss << " xid=\"" << _xid << "\"";
ss << " len=" << _enq_hdr._dsize;
str.append(ss.str());
return str;
@@ -388,9 +407,15 @@
}
const size_t
+enq_rec::xid_size() const
+{
+ return _xid.size();
+}
+
+const size_t
enq_rec::rec_size() const
{
- return _data_size + enq_hdr::size() + rec_tail::size();
+ return enq_hdr::size() + _xid.size() + _data_size + rec_tail::size();
}
void
@@ -401,78 +426,40 @@
}
void
-enq_rec::chk_hdr(enq_hdr& hdr, u_int64_t rid, bool enq) const throw (jexception)
+enq_rec::chk_hdr() const throw (jexception)
{
- chk_hdr(hdr, enq);
- if (hdr._hdr._rid != rid)
+ jrec::chk_hdr(_enq_hdr._hdr);
+ if (_enq_hdr._hdr._magic != RHM_JDAT_ENQ_MAGIC)
{
std::stringstream ss;
ss << std::hex << std::setfill('0');
- ss << "rid mismatch: expected=0x" << std::setw(16) << rid;
- ss << " read=0x" << std::setw(16) << hdr._hdr._rid;
+ ss << "enq magic: rid=0x" << std::setw(16) << _enq_hdr._hdr._rid;
+ ss << ": expected=0x" << std::setw(8) << RHM_JDAT_ENQ_MAGIC;
+ ss << " read=0x" << std::setw(2) << (int)_enq_hdr._hdr._magic;
throw jexception(jerrno::JERR_DREC_INVRHDR, ss.str(), "enq_rec", "chk_hdr");
}
-}
-
-void
-enq_rec::chk_hdr(enq_hdr& hdr, bool enq) const throw (jexception)
-{
- if (enq)
+ if (_enq_hdr._dsize > _max_data_size)
{
- if (hdr._hdr._magic != RHM_JDAT_ENQ_MAGIC)
- {
- std::stringstream ss;
- ss << std::hex << std::setfill('0');
- ss << "enq magic: rid=0x" << std::setw(16) << hdr._hdr._rid;
- ss << ": expected=0x" << std::setw(8) << RHM_JDAT_ENQ_MAGIC;
- ss << " read=0x" << std::setw(2) << (int)hdr._hdr._magic;
- throw jexception(jerrno::JERR_DREC_INVRHDR, ss.str(), "enq_rec", "chk_hdr");
- }
- }
- else
- {
- if (hdr._hdr._magic != RHM_JDAT_DEQ_MAGIC)
- {
- std::stringstream ss;
- ss << std::hex << std::setfill('0');
- ss << "deq magic: rid=0x" << std::setw(16) << hdr._hdr._rid;
- ss << ": expected=0x" << std::setw(8) << RHM_JDAT_DEQ_MAGIC;
- ss << " read=0x" << std::setw(2) << (int)hdr._hdr._magic;
- throw jexception(jerrno::JERR_DREC_INVRHDR, ss.str(), "enq_rec", "chk_hdr");
- }
- }
- if (hdr._hdr._version != RHM_JDAT_VERSION)
- {
std::stringstream ss;
ss << std::hex << std::setfill('0');
- ss << "version: rid=0x" << std::setw(16) << hdr._hdr._rid;
- ss << ": expected=0x" << std::setw(2) << (int)RHM_JDAT_VERSION;
- ss << " read=0x" << std::setw(2) << (int)hdr._hdr._version;
- throw jexception(jerrno::JERR_DREC_INVRHDR, ss.str(), "enq_rec", "chk_hdr");
- }
-#if defined (JRNL_LITTLE_ENDIAN)
- u_int8_t endian_flag = RHM_LENDIAN_FLAG;
-#else
- u_int8_t endian_flag = RHM_BENDIAN_FLAG;
-#endif
- if (hdr._hdr._eflag != endian_flag)
- {
- std::stringstream ss;
- ss << std::hex << std::setfill('0');
- ss << "endian_flag: rid=" << std::setw(16) << hdr._hdr._rid;
- ss << ": expected=0x" << std::setw(2) << (int)endian_flag;
- ss << " read=0x" << std::setw(2) << (int)hdr._hdr._eflag;
- throw jexception(jerrno::JERR_DREC_INVRHDR, ss.str(), "enq_rec", "chk_hdr");
- }
- if (hdr._dsize > _max_data_size)
- {
- std::stringstream ss;
- ss << std::hex << std::setfill('0');
- ss << "data_size: rid=" << std::setw(16) << hdr._hdr._rid;
- ss << std::dec << ": buff_size=" << _max_data_size << " data_size=" << hdr._dsize;
+ ss << "data_size: rid=" << std::setw(16) << _enq_hdr._hdr._rid;
+ ss << std::dec << ": buff_size=" << _max_data_size << " data_size=" << _enq_hdr._dsize;
throw jexception(jerrno::JERR_DREC_BUFFSIZE, ss.str(), "enq_rec", "chk_hdr");
}
}
+void
+enq_rec::chk_hdr(u_int64_t rid) const throw (jexception)
+{
+ chk_hdr();
+ jrec::chk_rid(_enq_hdr._hdr, rid);
+}
+
+void
+enq_rec::chk_tail() const throw (jexception)
+{
+ jrec::chk_tail(_enq_tail, _enq_hdr._hdr);
+}
+
} // namespace journal
} // namespace rhm
Modified: store/trunk/cpp/lib/jrnl/enq_rec.hpp
===================================================================
--- store/trunk/cpp/lib/jrnl/enq_rec.hpp 2007-09-25 20:25:49 UTC (rev 947)
+++ store/trunk/cpp/lib/jrnl/enq_rec.hpp 2007-09-27 15:18:31 UTC (rev 948)
@@ -56,8 +56,9 @@
{
private:
enq_hdr _enq_hdr;
- const void* _data; ///< Pointer to data to be written
- void* _buff; ///< Pointer to buffer to receive read data
+ std::string _xid; ///< XID
+ const void* _data; ///< Pointer to data to be written to disk
+ void* _buff; ///< Pointer to buffer to receive data read from disk
rec_tail _enq_tail;
size_t _max_data_size; ///< Max buffer size for decoding into during read
size_t _data_size; ///< Size of data (bytes)
@@ -80,6 +81,10 @@
* to receive data.
*/
enq_rec(void* const buf, const size_t bufsize);
+
+ /**
+ * \brief Destructor
+ */
~enq_rec();
// Prepare instance for use in writing data to journal
@@ -96,14 +101,16 @@
inline const size_t max_rec_size() const
{ return _max_data_size + enq_hdr::size() + rec_tail::size(); }
const size_t data_size() const;
+ const size_t xid_size() const;
const size_t rec_size() const;
inline const u_int32_t rec_size_dblks() const { return size_dblks(rec_size()); }
inline const u_int64_t rid() const { return _enq_hdr._hdr._rid; }
void set_rid(const u_int64_t rid);
private:
- void chk_hdr(enq_hdr& hdr, u_int64_t rid, bool enq = true) const throw (jexception);
- void chk_hdr(enq_hdr& hdr, bool enq = true) const throw (jexception);
+ void chk_hdr() const throw (jexception);
+ void chk_hdr(u_int64_t rid) const throw (jexception);
+ void chk_tail() const throw (jexception);
}; // class enq_rec
} // namespace journal
Modified: store/trunk/cpp/lib/jrnl/jcntl.cpp
===================================================================
--- store/trunk/cpp/lib/jrnl/jcntl.cpp 2007-09-25 20:25:49 UTC (rev 947)
+++ store/trunk/cpp/lib/jrnl/jcntl.cpp 2007-09-27 15:18:31 UTC (rev 948)
@@ -209,20 +209,20 @@
}
const iores
-jcntl::get_next_data_record(const void** const /*data*/, const size_t& /*dsize*/,
- const size_t& /*dsize_avail*/) throw (jexception)
+jcntl::get_data_record(const u_int64_t& /*rid*/, const size_t& /*dsize*/, const size_t& /*dsize_avail*/,
+ const void** const /*data*/, bool /*auto_discard*/) throw (jexception)
{
return RHM_IORES_NOTIMPL;
}
const iores
-jcntl::discard_next_data_record(data_tok* /*dtokp*/) throw (jexception)
+jcntl::discard_data_record() throw (jexception)
{
return RHM_IORES_NOTIMPL;
}
const iores
-jcntl::read_next_data_record(void* const dbuf, const size_t dbsize, data_tok* const dtokp)
+jcntl::read_data_record(void* const dbuf, const size_t dbsize, data_tok* const dtokp)
throw (jexception)
{
check_rstatus("read_data");
Modified: store/trunk/cpp/lib/jrnl/jcntl.hpp
===================================================================
--- store/trunk/cpp/lib/jrnl/jcntl.hpp 2007-09-25 20:25:49 UTC (rev 947)
+++ store/trunk/cpp/lib/jrnl/jcntl.hpp 2007-09-27 15:18:31 UTC (rev 948)
@@ -137,12 +137,13 @@
rmgr _rmgr; ///< Read page manager which manages AIO
wmgr _wmgr; ///< Write page manager which manages AIO
rcvdat _rcvdat; ///< Recovery data used for recovery
- static const std::string no_xid;
std::deque<rhm::journal::data_tok*> _aio_rd_cmpl_dtok_list; ///< Internally mamanged deque
std::deque<rhm::journal::data_tok*> _aio_wr_cmpl_dtok_list; ///< Internally mamanged deque
public:
+ static const std::string no_xid;
+
/**
* \brief Journal constructor.
*
@@ -281,13 +282,16 @@
*
* Example: If a write of 99 kB is divided into three equal parts, then the following states
* and returns would characterize a successful operation:
+ * <pre>
* dtok. dtok. dtok.
- * operation return wstate() dsize() written() comment
+ * Pperation Return wstate() dsize() written() Comment
+ * -----------------+--------+--------+-------+---------+------------------------------------
* NONE 0 0 Value of dtok before op
* edr(99000, 33000) SUCCESS ENQ_PART 99000 33000 Enqueue part 1
* edr(99000, 33000) AIO_WAIT ENQ_PART 99000 50000 Enqueue part 2, not completed
* edr(99000, 33000) SUCCESS ENQ_PART 99000 66000 Enqueue part 2 again
* edr(99000, 33000) SUCCESS ENQ 99000 99000 Enqueue part 3
+ * </pre>
*
* \param data_buff Pointer to data to be enqueued for this enqueue operation.
* \param tot_data_len Total data length.
@@ -306,27 +310,63 @@
/**
* \brief Retrieve details of next record to be read without consuming the record.
*
- * Retrieve details of next record to be read without consuming the record. A pointer to
- * the data is returned, along with the data size and available data size. If a large record
- * should span more than one page of the read page cache, then if the following page is still
- * waiting for AIO return, then only a portion of the total data will be available for
- * consumption. If dsize_avail < dsize, then a subsequent call will update dsize_avail if
- * more pages have returned from AIO.
+ * Retrieve information about current read record. A pointer to the data is returned, along
+ * with the data size and available data size. Data is considered "available" when the AIO
+ * operations to fill page-cache pages from disk have returned, and is ready for consumption.
*
- * Note that using this function does not consume the record, it merely makes the content
- * of the read page cache available through a pointer. To mark this record as consumed,
- * discard_next_data_record() must be called.
+ * If <i>dsize_avail</i> < <i>dsize</i>, then not all of the data is available or part of
+ * the data is in non-contiguous memory, and a subsequent call will update both the pointer
+ * and <i>dsize_avail</i> if more pages have returned from AIO.
*
- * \param data Pointer to data pointer which will point to the first byte of the next record
- * data.
+ * The <i>dsize_avail</i> parameter will return the amount of data from this record that is
+ * available in the page cache as contiguous memory, even if it spans page cache boundaries.
+ * However, if a record spans the end of the page cache and continues at the beginning, even
+ * if both parts are ready for consumption, then this must be divided into at least two
+ * get_data_record() operations, as the data is contained in at least two non-contiguous
+ * segments of the page cache.
+ *
+ * Once all the available data for a record is exposed, it can not be read again using
+ * this function. It must be consumed prior to getting the next record. This can be done by
+ * calling discard_data_record() or read_data_record(). However, if parameter
+ * <i>auto_discard</i> is set to <b><i>true</i></b>, then this record will be automatically
+ * consumed when the entire record has become available whithout having to explicitly call
+ * discard_next_data_record() or read_data_record().
+ *
+ * If the current record is an open transactional record, then it cannot be read until it is
+ * committed. If it is aborted, it can never be read. Under this condition, get_data_record()
+ * will return RHM_IORES_TXPENDING, the data pointer will be set to NULL and all data
+ * lengths will be set to 0.
+ *
+ * Example: Read a record of 30k. Assume a read page cache of 10 pages of size 10k starting
+ * at address base_ptr (page0 = base_ptr, page1 = page_ptr+10k, etc.). The first 15k of
+ * the record falls at the end of the page cache, the remaining 15k folded to the beginning.
+ * The current page (page 8) containing 5k is available, the remaining pages which contain
+ * this record are pending AIO return:
+ * <pre>
+ * call dsize
+ * no. dsize avail data ptr Return Comment
+ * ----+-----+-----+------------+--------+--------------------------------------------------
+ * 1 30k 5k base_ptr+85k SUCCESS Initial call, read first 5k
+ * 2 30k 0k base_ptr+90k AIO_WAIT AIO still pending; no further pages avail
+ * 3 30k 10k base_ptr+90k SUCCESS AIO now returned; now read till end of page cache
+ * 4 30k 15k base_ptr SUCCESS data_ptr now pointing to start of page cache
+ * </pre>
+ *
+ * \param rid Reference that returns the record ID (rid)
* \param dsize Reference that returns the total data size of the record data .
* \param dsize_avail Reference that returns the amount of the data that is available for
* consumption.
+ * \param data Pointer to data pointer which will point to the first byte of the next record
+ * data.
+ * \param auto_discard If <b><i>true</i></b>, automatically discard the record being read if
+ * the entire record is avialable (i.e. dsize == dsize_avail). Otherwise
+ * discard_next_data_record() must be explicitly called.
*
* \exception TODO
*/
- const iores get_next_data_record(const void** const data, const size_t& dsize,
- const size_t& dsize_avail) throw (jexception);
+ const iores get_data_record(const u_int64_t& rid, const size_t& dsize,
+ const size_t& dsize_avail, const void** const data, bool auto_discard = false)
+ throw (jexception);
/**
* \brief Discard (skip) next record to be read without reading or retrieving it.
@@ -335,7 +375,7 @@
*
* \exception TODO
*/
- const iores discard_next_data_record(data_tok* dtokp) throw (jexception);
+ const iores discard_data_record() throw (jexception);
/**
* \brief Reads data from the journal.
@@ -348,8 +388,8 @@
*
* \exception TODO
*/
- const iores read_next_data_record(void* const dbuf, const size_t dbsize,
- data_tok* const dtokp) throw (jexception);
+ const iores read_data_record(void* const dbuf, const size_t dbsize, data_tok* const dtokp)
+ throw (jexception);
/**
* \brief Dequeues (marks as no longer needed) data record in journal.
Modified: store/trunk/cpp/lib/jrnl/jrec.cpp
===================================================================
--- store/trunk/cpp/lib/jrnl/jrec.cpp 2007-09-25 20:25:49 UTC (rev 947)
+++ store/trunk/cpp/lib/jrnl/jrec.cpp 2007-09-27 15:18:31 UTC (rev 948)
@@ -64,21 +64,70 @@
}
void
-jrec::chk_tail(rec_tail& tail, enq_hdr& hdr) const throw (jexception)
+jrec::chk_hdr(const hdr& hdr) throw (jexception)
{
- std::stringstream ss;
- if (tail._xmagic != ~hdr._hdr._magic)
+ if (hdr._magic == 0)
{
+ std::stringstream ss;
ss << std::hex << std::setfill('0');
- ss << "magic: rid=0x" << std::setw(16) << hdr._hdr._rid;
- ss << ": expected=0x" << std::setw(8) << ~hdr._hdr._magic;
+ ss << "enq magic NULL: rid=0x" << std::setw(16) << hdr._rid;
+ throw jexception(jerrno::JERR_DREC_INVRHDR, ss.str(), "jrec", "chk_hdr");
+ }
+ if (hdr._version != RHM_JDAT_VERSION)
+ {
+ std::stringstream ss;
+ ss << std::hex << std::setfill('0');
+ ss << "version: rid=0x" << std::setw(16) << hdr._rid;
+ ss << ": expected=0x" << std::setw(2) << (int)RHM_JDAT_VERSION;
+ ss << " read=0x" << std::setw(2) << (int)hdr._version;
+ throw jexception(jerrno::JERR_DREC_INVRHDR, ss.str(), "jrec", "chk_hdr");
+ }
+#if defined (JRNL_LITTLE_ENDIAN)
+ u_int8_t endian_flag = RHM_LENDIAN_FLAG;
+#else
+ u_int8_t endian_flag = RHM_BENDIAN_FLAG;
+#endif
+ if (hdr._eflag != endian_flag)
+ {
+ std::stringstream ss;
+ ss << std::hex << std::setfill('0');
+ ss << "endian_flag: rid=" << std::setw(16) << hdr._rid;
+ ss << ": expected=0x" << std::setw(2) << (int)endian_flag;
+ ss << " read=0x" << std::setw(2) << (int)hdr._eflag;
+ throw jexception(jerrno::JERR_DREC_INVRHDR, ss.str(), "jrec", "chk_hdr");
+ }
+}
+
+void
+jrec::chk_rid(const hdr& hdr, const u_int64_t rid) throw (jexception)
+{
+ if (hdr._rid != rid)
+ {
+ std::stringstream ss;
+ ss << std::hex << std::setfill('0');
+ ss << "rid mismatch: expected=0x" << std::setw(16) << rid;
+ ss << " read=0x" << std::setw(16) << hdr._rid;
+ throw jexception(jerrno::JERR_DREC_INVRHDR, ss.str(), "jrec", "chk_hdr");
+ }
+}
+
+void
+jrec::chk_tail(const rec_tail& tail, const hdr& hdr) throw (jexception)
+{
+ if (tail._xmagic != ~hdr._magic)
+ {
+ std::stringstream ss;
+ ss << std::hex << std::setfill('0');
+ ss << "magic: rid=0x" << std::setw(16) << hdr._rid;
+ ss << ": expected=0x" << std::setw(8) << ~hdr._magic;
ss << " read=0x" << std::setw(8) << tail._xmagic;
throw jexception(jerrno::JERR_JREC_INVRTAIL, ss.str(), "jrec", "chk_tail");
}
- if (tail._rid != hdr._hdr._rid)
+ if (tail._rid != hdr._rid)
{
+ std::stringstream ss;
ss << std::hex << std::setfill('0');
- ss << "rid: rid=0x" << std::setw(16) << hdr._hdr._rid;
+ ss << "rid: rid=0x" << std::setw(16) << hdr._rid;
ss << ": read=0x" << std::setw(16) << tail._rid;
throw jexception(jerrno::JERR_JREC_INVRTAIL, ss.str(), "jrec", "chk_tail");
}
Modified: store/trunk/cpp/lib/jrnl/jrec.hpp
===================================================================
--- store/trunk/cpp/lib/jrnl/jrec.hpp 2007-09-25 20:25:49 UTC (rev 947)
+++ store/trunk/cpp/lib/jrnl/jrec.hpp 2007-09-27 15:18:31 UTC (rev 948)
@@ -151,15 +151,19 @@
virtual std::string& str(std::string& str) const = 0;
virtual const size_t data_size() const = 0;
+ virtual const size_t xid_size() const = 0;
virtual const size_t rec_size() const = 0;
static const u_int32_t size_dblks(const size_t size);
static const u_int32_t size_sblks(const size_t size);
static const u_int32_t size_blks(const size_t size, const size_t blksize);
protected:
- virtual void chk_hdr(enq_hdr& hdr, u_int64_t rid, bool enq = true)
- const throw (jexception) = 0;
- virtual void chk_tail(rec_tail& tail, enq_hdr& hdr) const throw (jexception);
+ virtual void chk_hdr() const throw (jexception) = 0;
+ virtual void chk_hdr(u_int64_t rid) const throw (jexception) = 0;
+ virtual void chk_tail() const throw (jexception) = 0;
+ static void chk_hdr(const hdr& hdr) throw (jexception);
+ static void chk_rid(const hdr& hdr, u_int64_t rid) throw (jexception);
+ static void chk_tail(const rec_tail& tail, const hdr& hdr) throw (jexception);
}; // class jrec
} // namespace journal
Modified: store/trunk/cpp/tests/jrnl/JournalSystemTests.cpp
===================================================================
--- store/trunk/cpp/tests/jrnl/JournalSystemTests.cpp 2007-09-25 20:25:49 UTC (rev 947)
+++ store/trunk/cpp/tests/jrnl/JournalSystemTests.cpp 2007-09-27 15:18:31 UTC (rev 948)
@@ -746,7 +746,7 @@
dtp->set_wstate(rhm::journal::data_tok::ENQ);
unsigned aio_sleep_cnt = 0;
- while (handle_jcntl_response(jc->read_next_data_record(buff, MAX_MSG_SIZE, dtp), jc,
+ while (handle_jcntl_response(jc->read_data_record(buff, MAX_MSG_SIZE, dtp), jc,
aio_sleep_cnt, dtp));
buff[dtp->dsize()] = '\0';
return buff;
Modified: store/trunk/cpp/tests/jrnl/msg_consumer.cpp
===================================================================
--- store/trunk/cpp/tests/jrnl/msg_consumer.cpp 2007-09-25 20:25:49 UTC (rev 947)
+++ store/trunk/cpp/tests/jrnl/msg_consumer.cpp 2007-09-27 15:18:31 UTC (rev 948)
@@ -107,7 +107,7 @@
{
if (dtokp->wstate() >= rhm::journal::data_tok::ENQ)
{
- rhm::journal::iores res = _jcntl.read_next_data_record((void* const)_msg_buff,
+ rhm::journal::iores res = _jcntl.read_data_record((void* const)_msg_buff,
buffSize, dtokp);
rhm::journal::data_tok::read_state rs = dtokp->rstate();
rhm::journal::data_tok::write_state ws = dtokp->wstate();
Modified: store/trunk/cpp/tests/jrnl/msg_producer.cpp
===================================================================
--- store/trunk/cpp/tests/jrnl/msg_producer.cpp 2007-09-25 20:25:49 UTC (rev 947)
+++ store/trunk/cpp/tests/jrnl/msg_producer.cpp 2007-09-27 15:18:31 UTC (rev 948)
@@ -81,8 +81,6 @@
{
_num_msgs = numMsgs;
_auto_dequeue = auto_dequeue;
-// for (u_int32_t i=0; i<_num_msgs; i++)
-// _dtok_master_list.push_back(new rhm::journal::data_tok);
}
void
@@ -103,15 +101,6 @@
_tot_dsize = 0;
_aio_cmpl_dtok_list.clear();
_dd_dtok_list.clear();
-// for (u_int32_t i=0; i<_dtok_master_list.size(); i++)
-// {
-// if (_dtok_master_list[i])
-// {
-// delete _dtok_master_list[i];
-// _dtok_master_list[i] = NULL;
-// }
-// }
-// _dtok_master_list.clear();
}
u_int32_t
@@ -122,8 +111,8 @@
if (!_num_msgs)
return 0;
if (maxMsgSize > _msg_buff_size)
- throw rhm::journal::jexception(EXCEPTION_BASE+0, "Message size exceeds internal buffer limit",
- "msg_producer", "produce");
+ throw rhm::journal::jexception(EXCEPTION_BASE+0,
+ "Message size exceeds internal buffer limit", "msg_producer", "produce");
{
//std::cout << "[" << _num_msgs << "]" << std::flush;
for(u_int32_t msgCntr = 0; msgCntr < _num_msgs && !_interrupt_flag; msgCntr++)
@@ -277,10 +266,8 @@
msg_producer::send_deferred_dequeues(rhm::journal::jcntl& jc)
{
std::deque<rhm::journal::data_tok*>::iterator ditr = _dd_dtok_list.begin();
-// while (!_dd_dtok_list.empty())
while (ditr != _dd_dtok_list.end())
{
-// rhm::journal::data_tok* ddtokp = _dd_dtok_list.front();
rhm::journal::data_tok* ddtokp = *ditr;
#ifndef RHM_WRONLY
@@ -318,7 +305,6 @@
" dres=" << iores_str[dres] << std::flush;
}
}
-// _dd_dtok_list.pop_front();
ditr = _dd_dtok_list.erase(ditr);
}
#ifndef RHM_WRONLY
17 years, 5 months
- 1
- 0
rhmessaging commits: r947 - in store/trunk/cpp: lib/jrnl and 1 other directories.
by rhmessaging-commits＠lists.jboss.org
Author: kpvdr
Date: 2007-09-25 16:25:49 -0400 (Tue, 25 Sep 2007)
New Revision: 947
Added:
store/trunk/cpp/lib/jrnl/deq_rec.cpp
store/trunk/cpp/lib/jrnl/deq_rec.hpp
store/trunk/cpp/lib/jrnl/dtx_rec.cpp
store/trunk/cpp/lib/jrnl/dtx_rec.hpp
store/trunk/cpp/lib/jrnl/enq_rec.cpp
store/trunk/cpp/lib/jrnl/enq_rec.hpp
Removed:
store/trunk/cpp/lib/jrnl/data_rec.cpp
store/trunk/cpp/lib/jrnl/data_rec.hpp
Modified:
store/trunk/cpp/lib/Makefile.am
store/trunk/cpp/lib/jrnl/file_hdr.cpp
store/trunk/cpp/lib/jrnl/file_hdr.hpp
store/trunk/cpp/lib/jrnl/jcfg.hpp
store/trunk/cpp/lib/jrnl/jcntl.cpp
store/trunk/cpp/lib/jrnl/jrec.cpp
store/trunk/cpp/lib/jrnl/jrec.hpp
store/trunk/cpp/lib/jrnl/pmgr.cpp
store/trunk/cpp/lib/jrnl/pmgr.hpp
store/trunk/cpp/lib/jrnl/rmgr.cpp
store/trunk/cpp/lib/jrnl/wmgr.cpp
store/trunk/cpp/tests/jrnl/Makefile.rtest
store/trunk/cpp/tests/jrnl/ftest.py
store/trunk/cpp/tests/jrnl/msg_consumer.cpp
store/trunk/cpp/tests/jrnl/rtest
store/trunk/cpp/tests/jrnl/rtests.csv
store/trunk/cpp/tests/jrnl/rwtests.csv
store/trunk/cpp/tests/jrnl/tests.ods
store/trunk/cpp/tests/jrnl/wtests.csv
Log:
Added XID fields to journal disk records, new dtx_rec and deq_rec classes (still shells).
Modified: store/trunk/cpp/lib/Makefile.am
===================================================================
--- store/trunk/cpp/lib/Makefile.am 2007-09-25 19:25:41 UTC (rev 946)
+++ store/trunk/cpp/lib/Makefile.am 2007-09-25 20:25:49 UTC (rev 947)
@@ -33,10 +33,11 @@
StoreException.h \
StringDbt.h \
TxnCtxt.h \
- jrnl/data_rec.cpp \
- jrnl/data_rec.hpp \
jrnl/data_tok.cpp \
+ jrnl/deq_rec.cpp \
+ jrnl/dtx_rec.cpp \
jrnl/enq_map.cpp \
+ jrnl/enq_rec.cpp \
jrnl/file_hdr.cpp \
jrnl/jcntl.cpp \
jrnl/jdir.cpp \
@@ -52,9 +53,11 @@
jrnl/wmgr.cpp \
jrnl/wrfc.cpp \
jrnl/aio_cb.hpp \
- jrnl/data_rec.hpp \
jrnl/data_tok.hpp \
+ jrnl/deq_rec.hpp \
+ jrnl/dtx_rec.hpp \
jrnl/enq_map.hpp \
+ jrnl/enq_rec.hpp \
jrnl/file_hdr.hpp \
jrnl/jcfg.hpp \
jrnl/jcntl.hpp \
Deleted: store/trunk/cpp/lib/jrnl/data_rec.cpp
===================================================================
--- store/trunk/cpp/lib/jrnl/data_rec.cpp 2007-09-25 19:25:41 UTC (rev 946)
+++ store/trunk/cpp/lib/jrnl/data_rec.cpp 2007-09-25 20:25:49 UTC (rev 947)
@@ -1,467 +0,0 @@
-/**
-* \file data_rec.cpp
-*
-* Red Hat Messaging - Message Journal
-*
-* This file contains the code for the rhm::journal::data_rec (journal data
-* record) class. See comments in file data_rec.hpp for details.
-*
-* \author Kim van der Riet
-*
-* Copyright 2007 Red Hat, Inc.
-*
-* This file is part of Red Hat Messaging.
-*
-* Red Hat Messaging is free software; you can redistribute it and/or
-* modify it under the terms of the GNU Lesser General Public
-* License as published by the Free Software Foundation; either
-* version 2.1 of the License, or (at your option) any later version.
-*
-* This library is distributed in the hope that it will be useful,
-* but WITHOUT ANY WARRANTY; without even the implied warranty of
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-* Lesser General Public License for more details.
-*
-* You should have received a copy of the GNU Lesser General Public
-* License along with this library; if not, write to the Free Software
-* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
-* USA
-*
-* The GNU Lesser General Public License is available in the file COPYING.
-*/
-
-#include <jrnl/data_rec.hpp>
-
-#include <assert.h>
-#include <iomanip>
-#include <sstream>
-#include <jrnl/jerrno.hpp>
-
-namespace rhm
-{
-namespace journal
-{
-
-// Default constructor; must be used in conjunction with reset() to prepare
-// instance for use with write or read operations
-data_rec::data_rec():
- jrec(), // superclass
- _enq_hdr(RHM_JDAT_ENQ_MAGIC, 0, 0, RHM_JDAT_VERSION),
- _data(NULL),
- _buff(NULL),
- _enq_tail(_enq_hdr),
- _max_data_size(0),
- _data_size(0),
- _rec_size(0)
-{}
-
-// Constructor used for write operations, where dbuf contains data to be written.
-data_rec::data_rec(const u_int64_t rid, const void* const dbuf, const size_t dlen):
- jrec(), // superclass
- _enq_hdr(RHM_JDAT_ENQ_MAGIC, rid, dlen, RHM_JDAT_VERSION),
- _data(dbuf),
- _buff(NULL),
- _enq_tail(_enq_hdr),
- _max_data_size(0),
- _data_size(dlen),
- _rec_size(size_dblks(dlen + enq_hdr::size() + enq_tail::size()) * JRNL_DBLK_SIZE)
-{}
-
-// Constructor used for read operations, where buf contains preallocated space
-// to receive data.
-data_rec::data_rec(void* const buf, const size_t bufsize):
- jrec(), // superclass
- _enq_hdr(RHM_JDAT_ENQ_MAGIC, 0, bufsize, RHM_JDAT_VERSION),
- _data(NULL),
- _buff(buf),
- _enq_tail(_enq_hdr),
- _max_data_size(bufsize),
- _data_size(0),
- _rec_size(0)
-{}
-
-data_rec::~data_rec()
-{
-#ifdef RHM_NOAIO
- if (_wbuff)
- {
- ::free(_wbuff);
- _wbuff = NULL;
- }
-#endif
-}
-
-// Prepare instance for use in writing data to journal, where dbuf contains data to be written.
-void
-data_rec::reset(const u_int64_t rid, const void* const dbuf, const size_t dlen)
-{
- _enq_hdr._hdr._rid = rid;
- _enq_hdr._dsize = dlen;
- _data = dbuf;
- _buff = NULL;
- _enq_tail._rid = rid;
- _max_data_size = 0;
- _data_size = dlen;
- _rec_size = size_dblks(dlen + enq_hdr::size() + enq_tail::size()) * JRNL_DBLK_SIZE;
-}
-
-// Prepare instance for use in reading data from journal, where buf contains preallocated space
-// to receive data.
-void
-data_rec::reset(void* const buf, const size_t bufsize)
-{
- _enq_hdr._hdr._rid = 0;
- _enq_hdr._dsize = bufsize;
- _data = NULL;
- _buff = buf;
- _enq_tail._rid = 0;
- _max_data_size = bufsize;
- _data_size = 0;
- _rec_size = 0;
-}
-
-u_int32_t
-data_rec::encode(void* wptr, u_int32_t rec_offs_dblks, u_int32_t max_size_dblks) throw (jexception)
-{
- assert(wptr != NULL);
- assert(max_size_dblks > 0);
-
- size_t rec_offs = rec_offs_dblks * JRNL_DBLK_SIZE;
- size_t rem = max_size_dblks * JRNL_DBLK_SIZE;
- size_t wr_cnt = 0;
- if (rec_offs_dblks) // Continuation of split data record (over 2 or more pages)
- {
- if (size_dblks(rec_size()) - rec_offs_dblks > max_size_dblks) // Further split required
- {
- // TODO: The header will never be split as it will always be less than one dblk.
- // Remove this check...
- size_t wsize = sizeof(_enq_hdr) > rec_offs ? sizeof(_enq_hdr) - rec_offs : 0;
- size_t wsize2 = wsize;
- if (wsize)
- {
- if (wsize > rem)
- wsize = rem;
- ::memcpy(wptr, (char*)&_enq_hdr + rec_offs, wsize);
- wr_cnt = wsize;
- rem -= wsize;
- }
- rec_offs -= sizeof(_enq_hdr) - wsize2;
- if (rem)
- {
- wsize = _data_size > rec_offs ? _data_size - rec_offs : 0;
- wsize2 = wsize;
- if (wsize)
- {
- if (wsize > rem)
- wsize = rem;
- ::memcpy((char*)wptr + wr_cnt, (char*)_data + rec_offs, wsize);
- wr_cnt += wsize;
- rem -= wsize;
- }
- rec_offs -= _data_size - wsize2;
- }
- if (rem)
- {
- wsize = sizeof(_enq_tail) > rec_offs ? sizeof(_enq_tail) - rec_offs : 0;
- wsize2 = wsize;
- if (wsize)
- {
- if (wsize > rem)
- wsize = rem;
- ::memcpy((char*)wptr + wr_cnt, (char*)&_enq_tail + rec_offs, wsize);
- wr_cnt += wsize;
- rem -= wsize;
- }
- rec_offs -= sizeof(_enq_tail) - wsize2;
- }
- assert(rem == 0);
- assert(rec_offs == 0);
- }
- else // No further split required
- {
- // TODO: The header will never be split as it will always be less than one dblk.
- // Remove this check...
- size_t wsize = sizeof(_enq_hdr) > rec_offs ? sizeof(_enq_hdr) - rec_offs : 0;
- if (wsize)
- {
- ::memcpy(wptr, (char*)&_enq_hdr + rec_offs, wsize);
- wr_cnt = wsize;
- }
- rec_offs -= sizeof(_enq_hdr) - wsize;
- wsize = _data_size > rec_offs ? _data_size - rec_offs : 0;
- if (wsize)
- {
- ::memcpy((char*)wptr + wr_cnt, (char*)_data + rec_offs, wsize);
- wr_cnt += wsize;
- }
- rec_offs -= _data_size - wsize;
- wsize = sizeof(_enq_tail) > rec_offs ? sizeof(_enq_tail) - rec_offs : 0;
- if (wsize)
- {
- ::memcpy((char*)wptr + wr_cnt, (char*)&_enq_tail + rec_offs, wsize);
- wr_cnt += wsize;
-#ifdef RHM_CLEAN
- ::memset((char*)wptr + wr_cnt, RHM_CLEAN_CHAR,
- _rec_size - (rec_offs_dblks * JRNL_DBLK_SIZE) - wr_cnt);
-#endif
- }
- rec_offs -= sizeof(_enq_tail) - wsize;
- assert(rec_offs == 0);
- }
- }
- else // Start at beginning of data record
- {
- if (size_dblks(rec_size()) > max_size_dblks) // Split required
- {
- size_t wsize = rem >= sizeof(_enq_hdr) ? sizeof(_enq_hdr) : rem;
- ::memcpy(wptr, (void*)&_enq_hdr, wsize);
- wr_cnt = wsize;
- rem -= wsize;
- if (rem)
- {
- wsize = rem >= _data_size ? _data_size : rem;
- ::memcpy((char*)wptr + wr_cnt, _data, wsize);
- wr_cnt += wsize;
- rem -= wsize;
- }
- if (rem)
- {
- wsize = rem >= sizeof(_enq_tail) ? sizeof(_enq_tail) : rem;
- ::memcpy((char*)wptr + wr_cnt, (void*)&_enq_tail, wsize);
- wr_cnt += wsize;
- rem -= wsize;
- }
- assert(rem == 0);
- }
- else // No split required
- {
- ::memcpy(wptr, (void*)&_enq_hdr, sizeof(_enq_hdr));
- wr_cnt = sizeof(_enq_hdr);
- ::memcpy((char*)wptr + wr_cnt, _data, _data_size);
- wr_cnt += _data_size;
- ::memcpy((char*)wptr + wr_cnt, (void*)&_enq_tail, sizeof(_enq_tail));
- wr_cnt += sizeof(_enq_tail);
-#ifdef RHM_CLEAN
- ::memset((char*)wptr + wr_cnt, RHM_CLEAN_CHAR, _rec_size - wr_cnt);
-#endif
- }
- }
- return size_dblks(wr_cnt);
-}
-
-u_int32_t
-data_rec::decode(hdr& h, void* rptr, u_int32_t rec_offs_dblks, u_int32_t max_size_dblks)
- throw (jexception)
-{
- assert(rptr != NULL);
- assert(max_size_dblks > 0);
-
- size_t rd_cnt = 0;
- if (rec_offs_dblks) // Continuation of record on new page
- {
- const u_int32_t hdr_data_dblks = size_dblks(enq_hdr::size() + _data_size);
- const u_int32_t hdr_data_tail_dblks = size_dblks(enq_hdr::size() + _data_size +
- enq_tail::size());
- const size_t rec_offs = rec_offs_dblks * JRNL_DBLK_SIZE;
-
- if (hdr_data_tail_dblks - rec_offs_dblks <= max_size_dblks)
- {
- // Remainder of record fits within this page
- if (rec_offs - enq_hdr::size() < _data_size)
- {
- // Data still outstanding, copy remainder of data and tail
- const size_t data_offs = rec_offs - enq_hdr::size();
- const size_t data_rem = _data_size - data_offs;
- ::memcpy((char*)_buff + data_offs, rptr, data_rem);
- rd_cnt += data_rem;
- ::memcpy((void*)&_enq_tail, ((char*)rptr + data_rem), sizeof(_enq_tail));
- chk_tail(_enq_tail, _enq_hdr);
- rd_cnt += sizeof(_enq_tail);
- }
- else
- {
- // Tail or part of tail only outstanding, complete tail
- const size_t tail_offs = rec_offs - enq_hdr::size() - _data_size;
- const size_t tail_rem = enq_tail::size() - tail_offs;
- ::memcpy((char*)&_enq_tail + tail_offs, rptr, tail_rem);
- chk_tail(_enq_tail, _enq_hdr);
- rd_cnt = tail_rem;
- }
- }
- else if (hdr_data_dblks - rec_offs_dblks <= max_size_dblks)
- {
- // Remainder of record data fits within this page, tail split
- const size_t data_offs = rec_offs - enq_hdr::size();
- const size_t data_rem = _data_size - data_offs;
- ::memcpy((char*)_buff + data_offs, rptr, data_rem);
- rd_cnt += data_rem;
- const size_t tail_rem = (max_size_dblks * JRNL_DBLK_SIZE) - rd_cnt;
- if (tail_rem)
- {
- ::memcpy((void*)&_enq_tail, ((char*)rptr + data_rem), tail_rem);
- rd_cnt += tail_rem;
- }
- }
- else
- {
- // Remainder of record data split
- const size_t data_cp_size = (max_size_dblks * JRNL_DBLK_SIZE);
- ::memcpy((char*)_buff + rec_offs - enq_hdr::size(), rptr, data_cp_size);
- rd_cnt += data_cp_size;
- }
- }
- else // Start of record
- {
- // Get and check header
- _enq_hdr._hdr.copy(h);
- _enq_hdr._dsize = *(size_t*)((char*)rptr + sizeof(hdr));
- rd_cnt = _enq_hdr.size();
- chk_hdr(_enq_hdr);
- _data_size = _enq_hdr._dsize;
- const u_int32_t hdr_data_dblks = size_dblks(enq_hdr::size() + _data_size);
- const u_int32_t hdr_data_tail_dblks = size_dblks(enq_hdr::size() + _data_size +
- enq_tail::size());
-
- if (_data_size)
- {
- // Check if record (header + data + tail) fits within this page, we can check the
- // tail before the expense of copying data to memory
- if (hdr_data_tail_dblks <= max_size_dblks)
- {
- // Entire header, data and tail fits within this page
- ::memcpy((void*)&_enq_tail, (char*)rptr + rd_cnt + _data_size, sizeof(_enq_tail));
- chk_tail(_enq_tail, _enq_hdr);
- ::memcpy(_buff, (char*)rptr + rd_cnt, _data_size);
- rd_cnt += _data_size + sizeof(_enq_tail);
- }
- else if (hdr_data_dblks <= max_size_dblks)
- {
- // Entire header and data fit within this page, tail split
- ::memcpy(_buff, (char*)rptr + rd_cnt, _data_size);
- rd_cnt += _data_size;
- const size_t tail_rem = (max_size_dblks * JRNL_DBLK_SIZE) - rd_cnt;
- if (tail_rem)
- {
- ::memcpy((void*)&_enq_tail, (char*)rptr + rd_cnt, tail_rem);
- rd_cnt += tail_rem;
- }
- }
- else
- {
- // Header fits within this page, record data split
- const size_t data_cp_size = (max_size_dblks * JRNL_DBLK_SIZE) - rd_cnt;
- ::memcpy(_buff, (char*)rptr + rd_cnt, data_cp_size);
- rd_cnt += data_cp_size;
- }
- }
- }
- return size_dblks(rd_cnt);
-}
-
-std::string&
-data_rec::str(std::string& str) const
-{
- std::stringstream ss;
- ss << "data_rec: m=" << _enq_hdr._hdr._magic;
- ss << " v=" << (int)_enq_hdr._hdr._version;
- ss << " rid=" << _enq_hdr._hdr._rid;
- ss << " len=" << _enq_hdr._dsize;
- str.append(ss.str());
- return str;
-}
-
-const size_t
-data_rec::data_size() const
-{
- return _data_size;
-}
-
-const size_t
-data_rec::rec_size() const
-{
- return _data_size + enq_hdr::size() + enq_tail::size();
-}
-
-void
-data_rec::set_rid(const u_int64_t rid)
-{
- _enq_hdr._hdr._rid = rid;
- _enq_tail._rid = rid;
-}
-
-void
-data_rec::chk_hdr(enq_hdr& hdr, u_int64_t rid, bool enq) const throw (jexception)
-{
- chk_hdr(hdr, enq);
- if (hdr._hdr._rid != rid)
- {
- std::stringstream ss;
- ss << std::hex << std::setfill('0');
- ss << "rid mismatch: expected=0x" << std::setw(16) << rid;
- ss << " read=0x" << std::setw(16) << hdr._hdr._rid;
- throw jexception(jerrno::JERR_DREC_INVRHDR, ss.str(), "data_rec", "chk_hdr");
- }
-}
-
-void
-data_rec::chk_hdr(enq_hdr& hdr, bool enq) const throw (jexception)
-{
- if (enq)
- {
- if (hdr._hdr._magic != RHM_JDAT_ENQ_MAGIC)
- {
- std::stringstream ss;
- ss << std::hex << std::setfill('0');
- ss << "enq magic: rid=0x" << std::setw(16) << hdr._hdr._rid;
- ss << ": expected=0x" << std::setw(8) << RHM_JDAT_ENQ_MAGIC;
- ss << " read=0x" << std::setw(2) << (int)hdr._hdr._magic;
- throw jexception(jerrno::JERR_DREC_INVRHDR, ss.str(), "data_rec", "chk_hdr");
- }
- }
- else
- {
- if (hdr._hdr._magic != RHM_JDAT_DEQ_MAGIC)
- {
- std::stringstream ss;
- ss << std::hex << std::setfill('0');
- ss << "deq magic: rid=0x" << std::setw(16) << hdr._hdr._rid;
- ss << ": expected=0x" << std::setw(8) << RHM_JDAT_DEQ_MAGIC;
- ss << " read=0x" << std::setw(2) << (int)hdr._hdr._magic;
- throw jexception(jerrno::JERR_DREC_INVRHDR, ss.str(), "data_rec", "chk_hdr");
- }
- }
- if (hdr._hdr._version != RHM_JDAT_VERSION)
- {
- std::stringstream ss;
- ss << std::hex << std::setfill('0');
- ss << "version: rid=0x" << std::setw(16) << hdr._hdr._rid;
- ss << ": expected=0x" << std::setw(2) << (int)RHM_JDAT_VERSION;
- ss << " read=0x" << std::setw(2) << (int)hdr._hdr._version;
- throw jexception(jerrno::JERR_DREC_INVRHDR, ss.str(), "data_rec", "chk_hdr");
- }
-#if defined (JRNL_LITTLE_ENDIAN)
- u_int8_t endian_flag = RHM_LENDIAN_FLAG;
-#else
- u_int8_t endian_flag = RHM_BENDIAN_FLAG;
-#endif
- if (hdr._hdr._eflag != endian_flag)
- {
- std::stringstream ss;
- ss << std::hex << std::setfill('0');
- ss << "endian_flag: rid=" << std::setw(16) << hdr._hdr._rid;
- ss << ": expected=0x" << std::setw(2) << (int)endian_flag;
- ss << " read=0x" << std::setw(2) << (int)hdr._hdr._eflag;
- throw jexception(jerrno::JERR_DREC_INVRHDR, ss.str(), "data_rec", "chk_hdr");
- }
- if (hdr._dsize > _max_data_size)
- {
- std::stringstream ss;
- ss << std::hex << std::setfill('0');
- ss << "data_size: rid=" << std::setw(16) << hdr._hdr._rid;
- ss << std::dec << ": buff_size=" << _max_data_size << " data_size=" << hdr._dsize;
- throw jexception(jerrno::JERR_DREC_BUFFSIZE, ss.str(), "data_rec", "chk_hdr");
- }
-}
-
-} // namespace journal
-} // namespace rhm
Deleted: store/trunk/cpp/lib/jrnl/data_rec.hpp
===================================================================
--- store/trunk/cpp/lib/jrnl/data_rec.hpp 2007-09-25 19:25:41 UTC (rev 946)
+++ store/trunk/cpp/lib/jrnl/data_rec.hpp 2007-09-25 20:25:49 UTC (rev 947)
@@ -1,113 +0,0 @@
-/**
-* \file data_rec.hpp
-*
-* Red Hat Messaging - Message Journal
-*
-* This file contains the code for the rhm::journal::data_rec (journal data
-* record) class. See class documentation for details.
-*
-* \author Kim van der Riet
-*
-* Copyright 2007 Red Hat, Inc.
-*
-* This file is part of Red Hat Messaging.
-*
-* Red Hat Messaging is free software; you can redistribute it and/or
-* modify it under the terms of the GNU Lesser General Public
-* License as published by the Free Software Foundation; either
-* version 2.1 of the License, or (at your option) any later version.
-*
-* This library is distributed in the hope that it will be useful,
-* but WITHOUT ANY WARRANTY; without even the implied warranty of
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-* Lesser General Public License for more details.
-*
-* You should have received a copy of the GNU Lesser General Public
-* License along with this library; if not, write to the Free Software
-* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
-* USA
-*
-* The GNU Lesser General Public License is available in the file COPYING.
-*/
-
-#ifndef rhm_journal_data_rec_hpp
-#define rhm_journal_data_rec_hpp
-
-namespace rhm
-{
-namespace journal
-{
-class data_rec;
-}
-}
-
-#include <jrnl/jrec.hpp>
-
-namespace rhm
-{
-namespace journal
-{
-
- /**
- * \class data_rec
- * \brief Class to handle a single journal data record. These are the records
- * contained in the jouranl data files.
- */
- class data_rec : public jrec
- {
- private:
- enq_hdr _enq_hdr;
- const void* _data; ///< Pointer to data to be written
- void* _buff; ///< Pointer to buffer to receive read data
- enq_tail _enq_tail;
- size_t _max_data_size; ///< Max buffer size for decoding into during read
- size_t _data_size; ///< Size of data (bytes)
- size_t _rec_size; ///< Size of data blocks required for record (bytes)
-
- public:
- /**
- * \brief Default constructor; must be used in conjunction with reset() to prepare
- * instance for use with write or read operations.
- */
- data_rec();
-
- /**
- * \brief Constructor used for write operations, where mbuf contains data to be written.
- */
- data_rec(const u_int64_t rid, const void* const dbuf, const size_t dlen);
-
- /**
- * \brief Constructor used for read operations, where buf contains preallocated space
- * to receive data.
- */
- data_rec(void* const buf, const size_t bufsize);
- ~data_rec();
-
- // Prepare instance for use in writing data to journal
- void reset(const u_int64_t rid, const void* const dbuf, const size_t dlen);
- // Prepare instance for use in reading data from journal
- void reset(void* const buf, const size_t bufsize);
-
- u_int32_t encode(void* wptr, u_int32_t rec_offs_dblks, u_int32_t max_size_dblks)
- throw (jexception);
- u_int32_t decode(hdr& h, void* rptr, u_int32_t rec_offs_dblks, u_int32_t max_size_dblks)
- throw (jexception);
- std::string& str(std::string& str) const;
- inline const size_t max_data_size() const { return _max_data_size; }
- inline const size_t max_rec_size() const
- { return _max_data_size + enq_hdr::size() + enq_tail::size(); }
- const size_t data_size() const;
- const size_t rec_size() const;
- inline const u_int32_t rec_size_dblks() const { return size_dblks(rec_size()); }
- inline const u_int64_t rid() const { return _enq_hdr._hdr._rid; }
- void set_rid(const u_int64_t rid);
-
- private:
- void chk_hdr(enq_hdr& hdr, u_int64_t rid, bool enq = true) const throw (jexception);
- void chk_hdr(enq_hdr& hdr, bool enq = true) const throw (jexception);
- }; // class data_rec
-
-} // namespace journal
-} // namespace rhm
-
-#endif // ifndef rhm_journal_data_rec_hpp
Added: store/trunk/cpp/lib/jrnl/deq_rec.cpp
===================================================================
--- store/trunk/cpp/lib/jrnl/deq_rec.cpp (rev 0)
+++ store/trunk/cpp/lib/jrnl/deq_rec.cpp 2007-09-25 20:25:49 UTC (rev 947)
@@ -0,0 +1,42 @@
+/**
+* \file deq_rec.cpp
+*
+* Red Hat Messaging - Message Journal
+*
+* This file contains the code for the rhm::journal::deq_rec (journal dequeue
+* record) class. See comments in file deq_rec.hpp for details.
+*
+* \author Kim van der Riet
+*
+* Copyright 2007 Red Hat, Inc.
+*
+* This file is part of Red Hat Messaging.
+*
+* Red Hat Messaging is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License as published by the Free Software Foundation; either
+* version 2.1 of the License, or (at your option) any later version.
+*
+* This library is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+* Lesser General Public License for more details.
+*
+* You should have received a copy of the GNU Lesser General Public
+* License along with this library; if not, write to the Free Software
+* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
+* USA
+*
+* The GNU Lesser General Public License is available in the file COPYING.
+*/
+
+#include <jrnl/deq_rec.hpp>
+
+namespace rhm
+{
+namespace journal
+{
+
+
+} // namespace journal
+} // namespace rhm
Added: store/trunk/cpp/lib/jrnl/deq_rec.hpp
===================================================================
--- store/trunk/cpp/lib/jrnl/deq_rec.hpp (rev 0)
+++ store/trunk/cpp/lib/jrnl/deq_rec.hpp 2007-09-25 20:25:49 UTC (rev 947)
@@ -0,0 +1,62 @@
+/**
+* \file deq_rec.hpp
+*
+* Red Hat Messaging - Message Journal
+*
+* This file contains the code for the rhm::journal::deq_rec (journal dequeue
+* record) class. See class documentation for details.
+*
+* \author Kim van der Riet
+*
+* Copyright 2007 Red Hat, Inc.
+*
+* This file is part of Red Hat Messaging.
+*
+* Red Hat Messaging is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License as published by the Free Software Foundation; either
+* version 2.1 of the License, or (at your option) any later version.
+*
+* This library is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+* Lesser General Public License for more details.
+*
+* You should have received a copy of the GNU Lesser General Public
+* License along with this library; if not, write to the Free Software
+* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
+* USA
+*
+* The GNU Lesser General Public License is available in the file COPYING.
+*/
+
+#ifndef rhm_journal_deq_rec_hpp
+#define rhm_journal_deq_rec_hpp
+
+namespace rhm
+{
+namespace journal
+{
+class deq_rec;
+}
+}
+
+#include <jrnl/jrec.hpp>
+
+namespace rhm
+{
+namespace journal
+{
+
+ /**
+ * \class deq_rec
+ * \brief Class to handle a single journal dequeue record.
+ */
+ class deq_rec : public jrec
+ {
+ }; // class deq_rec
+
+} // namespace journal
+} // namespace rhm
+
+#endif // ifndef rhm_journal_deq_rec_hpp
Added: store/trunk/cpp/lib/jrnl/dtx_rec.cpp
===================================================================
--- store/trunk/cpp/lib/jrnl/dtx_rec.cpp (rev 0)
+++ store/trunk/cpp/lib/jrnl/dtx_rec.cpp 2007-09-25 20:25:49 UTC (rev 947)
@@ -0,0 +1,42 @@
+/**
+* \file dtx_rec.cpp
+*
+* Red Hat Messaging - Message Journal
+*
+* This file contains the code for the rhm::journal::dtx_rec (journal dequeue
+* record) class. See comments in file dtx_rec.hpp for details.
+*
+* \author Kim van der Riet
+*
+* Copyright 2007 Red Hat, Inc.
+*
+* This file is part of Red Hat Messaging.
+*
+* Red Hat Messaging is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License as published by the Free Software Foundation; either
+* version 2.1 of the License, or (at your option) any later version.
+*
+* This library is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+* Lesser General Public License for more details.
+*
+* You should have received a copy of the GNU Lesser General Public
+* License along with this library; if not, write to the Free Software
+* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
+* USA
+*
+* The GNU Lesser General Public License is available in the file COPYING.
+*/
+
+#include <jrnl/dtx_rec.hpp>
+
+namespace rhm
+{
+namespace journal
+{
+
+
+} // namespace journal
+} // namespace rhm
Added: store/trunk/cpp/lib/jrnl/dtx_rec.hpp
===================================================================
--- store/trunk/cpp/lib/jrnl/dtx_rec.hpp (rev 0)
+++ store/trunk/cpp/lib/jrnl/dtx_rec.hpp 2007-09-25 20:25:49 UTC (rev 947)
@@ -0,0 +1,62 @@
+/**
+* \file dtx_rec.hpp
+*
+* Red Hat Messaging - Message Journal
+*
+* This file contains the code for the rhm::journal::dtx_rec (journal data
+* record) class. See class documentation for details.
+*
+* \author Kim van der Riet
+*
+* Copyright 2007 Red Hat, Inc.
+*
+* This file is part of Red Hat Messaging.
+*
+* Red Hat Messaging is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License as published by the Free Software Foundation; either
+* version 2.1 of the License, or (at your option) any later version.
+*
+* This library is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+* Lesser General Public License for more details.
+*
+* You should have received a copy of the GNU Lesser General Public
+* License along with this library; if not, write to the Free Software
+* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
+* USA
+*
+* The GNU Lesser General Public License is available in the file COPYING.
+*/
+
+#ifndef rhm_journal_dtx_rec_hpp
+#define rhm_journal_dtx_rec_hpp
+
+namespace rhm
+{
+namespace journal
+{
+class dtx_rec;
+}
+}
+
+#include <jrnl/jrec.hpp>
+
+namespace rhm
+{
+namespace journal
+{
+
+ /**
+ * \class dtx_rec
+ * \brief Class to handle a single journal DTX commit or abort record.
+ */
+ class dtx_rec : public jrec
+ {
+ }; // class dtx_rec
+
+} // namespace journal
+} // namespace rhm
+
+#endif // ifndef rhm_journal_dtx_rec_hpp
Copied: store/trunk/cpp/lib/jrnl/enq_rec.cpp (from rev 946, store/trunk/cpp/lib/jrnl/data_rec.cpp)
===================================================================
--- store/trunk/cpp/lib/jrnl/enq_rec.cpp (rev 0)
+++ store/trunk/cpp/lib/jrnl/enq_rec.cpp 2007-09-25 20:25:49 UTC (rev 947)
@@ -0,0 +1,478 @@
+/**
+* \file enq_rec.cpp
+*
+* Red Hat Messaging - Message Journal
+*
+* This file contains the code for the rhm::journal::enq_rec (journal enqueue
+* record) class. See comments in file enq_rec.hpp for details.
+*
+* \author Kim van der Riet
+*
+* Copyright 2007 Red Hat, Inc.
+*
+* This file is part of Red Hat Messaging.
+*
+* Red Hat Messaging is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License as published by the Free Software Foundation; either
+* version 2.1 of the License, or (at your option) any later version.
+*
+* This library is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+* Lesser General Public License for more details.
+*
+* You should have received a copy of the GNU Lesser General Public
+* License along with this library; if not, write to the Free Software
+* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
+* USA
+*
+* The GNU Lesser General Public License is available in the file COPYING.
+*/
+
+#include <jrnl/enq_rec.hpp>
+
+#include <assert.h>
+#include <iomanip>
+#include <sstream>
+#include <jrnl/jerrno.hpp>
+
+namespace rhm
+{
+namespace journal
+{
+
+// Default constructor; must be used in conjunction with reset() to prepare
+// instance for use with write or read operations
+enq_rec::enq_rec():
+ jrec(), // superclass
+ _enq_hdr(RHM_JDAT_ENQ_MAGIC, 0, 0, 0, RHM_JDAT_VERSION),
+ _data(NULL),
+ _buff(NULL),
+ _enq_tail(_enq_hdr._hdr),
+ _max_data_size(0),
+ _data_size(0),
+ _rec_size(0)
+{}
+
+// Constructor used for write operations, where dbuf contains data to be written.
+enq_rec::enq_rec(const u_int64_t rid, const void* const dbuf, const size_t dlen):
+ jrec(), // superclass
+ _enq_hdr(RHM_JDAT_ENQ_MAGIC, rid, 0, dlen, RHM_JDAT_VERSION),
+ _data(dbuf),
+ _buff(NULL),
+ _enq_tail(_enq_hdr._hdr),
+ _max_data_size(0),
+ _data_size(dlen),
+ _rec_size(size_dblks(dlen + enq_hdr::size() + rec_tail::size()) * JRNL_DBLK_SIZE)
+{}
+
+// Constructor used for read operations, where buf contains preallocated space
+// to receive data.
+enq_rec::enq_rec(void* const buf, const size_t bufsize):
+ jrec(), // superclass
+ _enq_hdr(RHM_JDAT_ENQ_MAGIC, 0, 0, bufsize, RHM_JDAT_VERSION),
+ _data(NULL),
+ _buff(buf),
+ _enq_tail(_enq_hdr._hdr),
+ _max_data_size(bufsize),
+ _data_size(0),
+ _rec_size(0)
+{}
+
+enq_rec::~enq_rec()
+{
+#ifdef RHM_NOAIO
+ if (_wbuff)
+ {
+ ::free(_wbuff);
+ _wbuff = NULL;
+ }
+#endif
+}
+
+// Prepare instance for use in writing data to journal, where dbuf contains data to be written.
+void
+enq_rec::reset(const u_int64_t rid, const void* const dbuf, const size_t dlen)
+{
+ _enq_hdr._hdr._rid = rid;
+ _enq_hdr._xidsize = 0;
+ _enq_hdr._dsize = dlen;
+ _data = dbuf;
+ _buff = NULL;
+ _enq_tail._rid = rid;
+ _max_data_size = 0;
+ _data_size = dlen;
+ _rec_size = size_dblks(dlen + enq_hdr::size() + rec_tail::size()) * JRNL_DBLK_SIZE;
+}
+
+// Prepare instance for use in reading data from journal, where buf contains preallocated space
+// to receive data.
+void
+enq_rec::reset(void* const buf, const size_t bufsize)
+{
+ _enq_hdr._hdr._rid = 0;
+ _enq_hdr._xidsize = 0;
+ _enq_hdr._dsize = bufsize;
+ _data = NULL;
+ _buff = buf;
+ _enq_tail._rid = 0;
+ _max_data_size = bufsize;
+ _data_size = 0;
+ _rec_size = 0;
+}
+
+u_int32_t
+enq_rec::encode(void* wptr, u_int32_t rec_offs_dblks, u_int32_t max_size_dblks) throw (jexception)
+{
+ assert(wptr != NULL);
+ assert(max_size_dblks > 0);
+
+ size_t rec_offs = rec_offs_dblks * JRNL_DBLK_SIZE;
+ size_t rem = max_size_dblks * JRNL_DBLK_SIZE;
+ size_t wr_cnt = 0;
+ if (rec_offs_dblks) // Continuation of split data record (over 2 or more pages)
+ {
+ if (size_dblks(rec_size()) - rec_offs_dblks > max_size_dblks) // Further split required
+ {
+ // TODO: The header will never be split as it will always be less than one dblk.
+ // Remove this check...
+ size_t wsize = sizeof(_enq_hdr) > rec_offs ? sizeof(_enq_hdr) - rec_offs : 0;
+ size_t wsize2 = wsize;
+ if (wsize)
+ {
+ if (wsize > rem)
+ wsize = rem;
+ ::memcpy(wptr, (char*)&_enq_hdr + rec_offs, wsize);
+ wr_cnt = wsize;
+ rem -= wsize;
+ }
+ rec_offs -= sizeof(_enq_hdr) - wsize2;
+ if (rem)
+ {
+ wsize = _data_size > rec_offs ? _data_size - rec_offs : 0;
+ wsize2 = wsize;
+ if (wsize)
+ {
+ if (wsize > rem)
+ wsize = rem;
+ ::memcpy((char*)wptr + wr_cnt, (char*)_data + rec_offs, wsize);
+ wr_cnt += wsize;
+ rem -= wsize;
+ }
+ rec_offs -= _data_size - wsize2;
+ }
+ if (rem)
+ {
+ wsize = sizeof(_enq_tail) > rec_offs ? sizeof(_enq_tail) - rec_offs : 0;
+ wsize2 = wsize;
+ if (wsize)
+ {
+ if (wsize > rem)
+ wsize = rem;
+ ::memcpy((char*)wptr + wr_cnt, (char*)&_enq_tail + rec_offs, wsize);
+ wr_cnt += wsize;
+ rem -= wsize;
+ }
+ rec_offs -= sizeof(_enq_tail) - wsize2;
+ }
+ assert(rem == 0);
+ assert(rec_offs == 0);
+ }
+ else // No further split required
+ {
+ // TODO: The header will never be split as it will always be less than one dblk.
+ // Remove this check...
+ size_t wsize = sizeof(_enq_hdr) > rec_offs ? sizeof(_enq_hdr) - rec_offs : 0;
+ if (wsize)
+ {
+ ::memcpy(wptr, (char*)&_enq_hdr + rec_offs, wsize);
+ wr_cnt = wsize;
+ }
+ rec_offs -= sizeof(_enq_hdr) - wsize;
+ wsize = _data_size > rec_offs ? _data_size - rec_offs : 0;
+ if (wsize)
+ {
+ ::memcpy((char*)wptr + wr_cnt, (char*)_data + rec_offs, wsize);
+ wr_cnt += wsize;
+ }
+ rec_offs -= _data_size - wsize;
+ wsize = sizeof(_enq_tail) > rec_offs ? sizeof(_enq_tail) - rec_offs : 0;
+ if (wsize)
+ {
+ ::memcpy((char*)wptr + wr_cnt, (char*)&_enq_tail + rec_offs, wsize);
+ wr_cnt += wsize;
+#ifdef RHM_CLEAN
+ ::memset((char*)wptr + wr_cnt, RHM_CLEAN_CHAR,
+ _rec_size - (rec_offs_dblks * JRNL_DBLK_SIZE) - wr_cnt);
+#endif
+ }
+ rec_offs -= sizeof(_enq_tail) - wsize;
+ assert(rec_offs == 0);
+ }
+ }
+ else // Start at beginning of data record
+ {
+ if (size_dblks(rec_size()) > max_size_dblks) // Split required
+ {
+ size_t wsize = rem >= sizeof(_enq_hdr) ? sizeof(_enq_hdr) : rem;
+ ::memcpy(wptr, (void*)&_enq_hdr, wsize);
+ wr_cnt = wsize;
+ rem -= wsize;
+ if (rem)
+ {
+ wsize = rem >= _data_size ? _data_size : rem;
+ ::memcpy((char*)wptr + wr_cnt, _data, wsize);
+ wr_cnt += wsize;
+ rem -= wsize;
+ }
+ if (rem)
+ {
+ wsize = rem >= sizeof(_enq_tail) ? sizeof(_enq_tail) : rem;
+ ::memcpy((char*)wptr + wr_cnt, (void*)&_enq_tail, wsize);
+ wr_cnt += wsize;
+ rem -= wsize;
+ }
+ assert(rem == 0);
+ }
+ else // No split required
+ {
+ ::memcpy(wptr, (void*)&_enq_hdr, sizeof(_enq_hdr));
+ wr_cnt = sizeof(_enq_hdr);
+ ::memcpy((char*)wptr + wr_cnt, _data, _data_size);
+ wr_cnt += _data_size;
+ ::memcpy((char*)wptr + wr_cnt, (void*)&_enq_tail, sizeof(_enq_tail));
+ wr_cnt += sizeof(_enq_tail);
+#ifdef RHM_CLEAN
+ ::memset((char*)wptr + wr_cnt, RHM_CLEAN_CHAR, _rec_size - wr_cnt);
+#endif
+ }
+ }
+ return size_dblks(wr_cnt);
+}
+
+u_int32_t
+enq_rec::decode(hdr& h, void* rptr, u_int32_t rec_offs_dblks, u_int32_t max_size_dblks)
+ throw (jexception)
+{
+ assert(rptr != NULL);
+ assert(max_size_dblks > 0);
+
+ size_t rd_cnt = 0;
+ if (rec_offs_dblks) // Continuation of record on new page
+ {
+ const u_int32_t hdr_data_dblks = size_dblks(enq_hdr::size() + _data_size);
+ const u_int32_t hdr_data_tail_dblks = size_dblks(enq_hdr::size() + _data_size +
+ rec_tail::size());
+ const size_t rec_offs = rec_offs_dblks * JRNL_DBLK_SIZE;
+
+ if (hdr_data_tail_dblks - rec_offs_dblks <= max_size_dblks)
+ {
+ // Remainder of record fits within this page
+ if (rec_offs - enq_hdr::size() < _data_size)
+ {
+ // Data still outstanding, copy remainder of data and tail
+ const size_t data_offs = rec_offs - enq_hdr::size();
+ const size_t data_rem = _data_size - data_offs;
+ ::memcpy((char*)_buff + data_offs, rptr, data_rem);
+ rd_cnt += data_rem;
+ ::memcpy((void*)&_enq_tail, ((char*)rptr + data_rem), sizeof(_enq_tail));
+ chk_tail(_enq_tail, _enq_hdr);
+ rd_cnt += sizeof(_enq_tail);
+ }
+ else
+ {
+ // Tail or part of tail only outstanding, complete tail
+ const size_t tail_offs = rec_offs - enq_hdr::size() - _data_size;
+ const size_t tail_rem = rec_tail::size() - tail_offs;
+ ::memcpy((char*)&_enq_tail + tail_offs, rptr, tail_rem);
+ chk_tail(_enq_tail, _enq_hdr);
+ rd_cnt = tail_rem;
+ }
+ }
+ else if (hdr_data_dblks - rec_offs_dblks <= max_size_dblks)
+ {
+ // Remainder of record data fits within this page, tail split
+ const size_t data_offs = rec_offs - enq_hdr::size();
+ const size_t data_rem = _data_size - data_offs;
+ ::memcpy((char*)_buff + data_offs, rptr, data_rem);
+ rd_cnt += data_rem;
+ const size_t tail_rem = (max_size_dblks * JRNL_DBLK_SIZE) - rd_cnt;
+ if (tail_rem)
+ {
+ ::memcpy((void*)&_enq_tail, ((char*)rptr + data_rem), tail_rem);
+ rd_cnt += tail_rem;
+ }
+ }
+ else
+ {
+ // Remainder of record data split
+ const size_t data_cp_size = (max_size_dblks * JRNL_DBLK_SIZE);
+ ::memcpy((char*)_buff + rec_offs - enq_hdr::size(), rptr, data_cp_size);
+ rd_cnt += data_cp_size;
+ }
+ }
+ else // Start of record
+ {
+ // Get and check header
+ _enq_hdr._hdr.copy(h);
+ _enq_hdr._xidsize = *(size_t*)((char*)rptr + sizeof(hdr));
+ _enq_hdr._dsize = *(size_t*)((char*)rptr + sizeof(hdr) +
+#if defined(JRNL_BIG_ENDIAN) && defined(JRNL_32_BIT)
+ sizeof(u_int32_t) +
+#endif
+ sizeof(size_t)
+#if defined(JRNL_LITTLE_ENDIAN) && defined(JRNL_32_BIT)
+ + sizeof(u_int32_t)
+#endif
+ );
+ rd_cnt = _enq_hdr.size();
+ chk_hdr(_enq_hdr);
+ _data_size = _enq_hdr._dsize;
+ const u_int32_t hdr_data_dblks = size_dblks(enq_hdr::size() + _data_size);
+ const u_int32_t hdr_data_tail_dblks = size_dblks(enq_hdr::size() + _data_size +
+ rec_tail::size());
+
+ if (_data_size)
+ {
+ // Check if record (header + data + tail) fits within this page, we can check the
+ // tail before the expense of copying data to memory
+ if (hdr_data_tail_dblks <= max_size_dblks)
+ {
+ // Entire header, data and tail fits within this page
+ ::memcpy((void*)&_enq_tail, (char*)rptr + rd_cnt + _data_size, sizeof(_enq_tail));
+ chk_tail(_enq_tail, _enq_hdr);
+ ::memcpy(_buff, (char*)rptr + rd_cnt, _data_size);
+ rd_cnt += _data_size + sizeof(_enq_tail);
+ }
+ else if (hdr_data_dblks <= max_size_dblks)
+ {
+ // Entire header and data fit within this page, tail split
+ ::memcpy(_buff, (char*)rptr + rd_cnt, _data_size);
+ rd_cnt += _data_size;
+ const size_t tail_rem = (max_size_dblks * JRNL_DBLK_SIZE) - rd_cnt;
+ if (tail_rem)
+ {
+ ::memcpy((void*)&_enq_tail, (char*)rptr + rd_cnt, tail_rem);
+ rd_cnt += tail_rem;
+ }
+ }
+ else
+ {
+ // Header fits within this page, record data split
+ const size_t data_cp_size = (max_size_dblks * JRNL_DBLK_SIZE) - rd_cnt;
+ ::memcpy(_buff, (char*)rptr + rd_cnt, data_cp_size);
+ rd_cnt += data_cp_size;
+ }
+ }
+ }
+ return size_dblks(rd_cnt);
+}
+
+std::string&
+enq_rec::str(std::string& str) const
+{
+ std::stringstream ss;
+ ss << "enq_rec: m=" << _enq_hdr._hdr._magic;
+ ss << " v=" << (int)_enq_hdr._hdr._version;
+ ss << " rid=" << _enq_hdr._hdr._rid;
+ ss << " len=" << _enq_hdr._dsize;
+ str.append(ss.str());
+ return str;
+}
+
+const size_t
+enq_rec::data_size() const
+{
+ return _data_size;
+}
+
+const size_t
+enq_rec::rec_size() const
+{
+ return _data_size + enq_hdr::size() + rec_tail::size();
+}
+
+void
+enq_rec::set_rid(const u_int64_t rid)
+{
+ _enq_hdr._hdr._rid = rid;
+ _enq_tail._rid = rid;
+}
+
+void
+enq_rec::chk_hdr(enq_hdr& hdr, u_int64_t rid, bool enq) const throw (jexception)
+{
+ chk_hdr(hdr, enq);
+ if (hdr._hdr._rid != rid)
+ {
+ std::stringstream ss;
+ ss << std::hex << std::setfill('0');
+ ss << "rid mismatch: expected=0x" << std::setw(16) << rid;
+ ss << " read=0x" << std::setw(16) << hdr._hdr._rid;
+ throw jexception(jerrno::JERR_DREC_INVRHDR, ss.str(), "enq_rec", "chk_hdr");
+ }
+}
+
+void
+enq_rec::chk_hdr(enq_hdr& hdr, bool enq) const throw (jexception)
+{
+ if (enq)
+ {
+ if (hdr._hdr._magic != RHM_JDAT_ENQ_MAGIC)
+ {
+ std::stringstream ss;
+ ss << std::hex << std::setfill('0');
+ ss << "enq magic: rid=0x" << std::setw(16) << hdr._hdr._rid;
+ ss << ": expected=0x" << std::setw(8) << RHM_JDAT_ENQ_MAGIC;
+ ss << " read=0x" << std::setw(2) << (int)hdr._hdr._magic;
+ throw jexception(jerrno::JERR_DREC_INVRHDR, ss.str(), "enq_rec", "chk_hdr");
+ }
+ }
+ else
+ {
+ if (hdr._hdr._magic != RHM_JDAT_DEQ_MAGIC)
+ {
+ std::stringstream ss;
+ ss << std::hex << std::setfill('0');
+ ss << "deq magic: rid=0x" << std::setw(16) << hdr._hdr._rid;
+ ss << ": expected=0x" << std::setw(8) << RHM_JDAT_DEQ_MAGIC;
+ ss << " read=0x" << std::setw(2) << (int)hdr._hdr._magic;
+ throw jexception(jerrno::JERR_DREC_INVRHDR, ss.str(), "enq_rec", "chk_hdr");
+ }
+ }
+ if (hdr._hdr._version != RHM_JDAT_VERSION)
+ {
+ std::stringstream ss;
+ ss << std::hex << std::setfill('0');
+ ss << "version: rid=0x" << std::setw(16) << hdr._hdr._rid;
+ ss << ": expected=0x" << std::setw(2) << (int)RHM_JDAT_VERSION;
+ ss << " read=0x" << std::setw(2) << (int)hdr._hdr._version;
+ throw jexception(jerrno::JERR_DREC_INVRHDR, ss.str(), "enq_rec", "chk_hdr");
+ }
+#if defined (JRNL_LITTLE_ENDIAN)
+ u_int8_t endian_flag = RHM_LENDIAN_FLAG;
+#else
+ u_int8_t endian_flag = RHM_BENDIAN_FLAG;
+#endif
+ if (hdr._hdr._eflag != endian_flag)
+ {
+ std::stringstream ss;
+ ss << std::hex << std::setfill('0');
+ ss << "endian_flag: rid=" << std::setw(16) << hdr._hdr._rid;
+ ss << ": expected=0x" << std::setw(2) << (int)endian_flag;
+ ss << " read=0x" << std::setw(2) << (int)hdr._hdr._eflag;
+ throw jexception(jerrno::JERR_DREC_INVRHDR, ss.str(), "enq_rec", "chk_hdr");
+ }
+ if (hdr._dsize > _max_data_size)
+ {
+ std::stringstream ss;
+ ss << std::hex << std::setfill('0');
+ ss << "data_size: rid=" << std::setw(16) << hdr._hdr._rid;
+ ss << std::dec << ": buff_size=" << _max_data_size << " data_size=" << hdr._dsize;
+ throw jexception(jerrno::JERR_DREC_BUFFSIZE, ss.str(), "enq_rec", "chk_hdr");
+ }
+}
+
+} // namespace journal
+} // namespace rhm
Copied: store/trunk/cpp/lib/jrnl/enq_rec.hpp (from rev 946, store/trunk/cpp/lib/jrnl/data_rec.hpp)
===================================================================
--- store/trunk/cpp/lib/jrnl/enq_rec.hpp (rev 0)
+++ store/trunk/cpp/lib/jrnl/enq_rec.hpp 2007-09-25 20:25:49 UTC (rev 947)
@@ -0,0 +1,112 @@
+/**
+* \file enq_rec.hpp
+*
+* Red Hat Messaging - Message Journal
+*
+* This file contains the code for the rhm::journal::enq_rec (journal enqueue
+* record) class. See class documentation for details.
+*
+* \author Kim van der Riet
+*
+* Copyright 2007 Red Hat, Inc.
+*
+* This file is part of Red Hat Messaging.
+*
+* Red Hat Messaging is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License as published by the Free Software Foundation; either
+* version 2.1 of the License, or (at your option) any later version.
+*
+* This library is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+* Lesser General Public License for more details.
+*
+* You should have received a copy of the GNU Lesser General Public
+* License along with this library; if not, write to the Free Software
+* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
+* USA
+*
+* The GNU Lesser General Public License is available in the file COPYING.
+*/
+
+#ifndef rhm_journal_enq_rec_hpp
+#define rhm_journal_enq_rec_hpp
+
+namespace rhm
+{
+namespace journal
+{
+class enq_rec;
+}
+}
+
+#include <jrnl/jrec.hpp>
+
+namespace rhm
+{
+namespace journal
+{
+
+ /**
+ * \class enq_rec
+ * \brief Class to handle a single journal enqueue record.
+ */
+ class enq_rec : public jrec
+ {
+ private:
+ enq_hdr _enq_hdr;
+ const void* _data; ///< Pointer to data to be written
+ void* _buff; ///< Pointer to buffer to receive read data
+ rec_tail _enq_tail;
+ size_t _max_data_size; ///< Max buffer size for decoding into during read
+ size_t _data_size; ///< Size of data (bytes)
+ size_t _rec_size; ///< Size of data blocks required for record (bytes)
+
+ public:
+ /**
+ * \brief Default constructor; must be used in conjunction with reset() to prepare
+ * instance for use with write or read operations.
+ */
+ enq_rec();
+
+ /**
+ * \brief Constructor used for write operations, where mbuf contains data to be written.
+ */
+ enq_rec(const u_int64_t rid, const void* const dbuf, const size_t dlen);
+
+ /**
+ * \brief Constructor used for read operations, where buf contains preallocated space
+ * to receive data.
+ */
+ enq_rec(void* const buf, const size_t bufsize);
+ ~enq_rec();
+
+ // Prepare instance for use in writing data to journal
+ void reset(const u_int64_t rid, const void* const dbuf, const size_t dlen);
+ // Prepare instance for use in reading data from journal
+ void reset(void* const buf, const size_t bufsize);
+
+ u_int32_t encode(void* wptr, u_int32_t rec_offs_dblks, u_int32_t max_size_dblks)
+ throw (jexception);
+ u_int32_t decode(hdr& h, void* rptr, u_int32_t rec_offs_dblks, u_int32_t max_size_dblks)
+ throw (jexception);
+ std::string& str(std::string& str) const;
+ inline const size_t max_data_size() const { return _max_data_size; }
+ inline const size_t max_rec_size() const
+ { return _max_data_size + enq_hdr::size() + rec_tail::size(); }
+ const size_t data_size() const;
+ const size_t rec_size() const;
+ inline const u_int32_t rec_size_dblks() const { return size_dblks(rec_size()); }
+ inline const u_int64_t rid() const { return _enq_hdr._hdr._rid; }
+ void set_rid(const u_int64_t rid);
+
+ private:
+ void chk_hdr(enq_hdr& hdr, u_int64_t rid, bool enq = true) const throw (jexception);
+ void chk_hdr(enq_hdr& hdr, bool enq = true) const throw (jexception);
+ }; // class enq_rec
+
+} // namespace journal
+} // namespace rhm
+
+#endif // ifndef rhm_journal_enq_rec_hpp
Modified: store/trunk/cpp/lib/jrnl/file_hdr.cpp
===================================================================
--- store/trunk/cpp/lib/jrnl/file_hdr.cpp 2007-09-25 19:25:41 UTC (rev 946)
+++ store/trunk/cpp/lib/jrnl/file_hdr.cpp 2007-09-25 20:25:49 UTC (rev 947)
@@ -42,6 +42,8 @@
namespace journal
{
+// ***** struct hdr *****
+
hdr::hdr():
_magic(0),
_version(0),
@@ -83,6 +85,27 @@
_rid = 0;
}
+
+// ***** struct rec_tail *****
+
+rec_tail::rec_tail():
+ _xmagic(0xffffff), // 1's complement of 0
+ _rid(0)
+{}
+
+rec_tail::rec_tail(const hdr& _hdr):
+ _xmagic(~_hdr._magic),
+ _rid(_hdr._rid)
+{}
+
+rec_tail::rec_tail(const u_int32_t xmagic, const u_int64_t rid):
+ _xmagic(xmagic),
+ _rid(rid)
+{}
+
+
+// ***** struct file_hdr *****
+
file_hdr::file_hdr():
_hdr(),
_fid(0),
@@ -173,54 +196,101 @@
_ts_nsec = ts.tv_nsec;
}
+
+// ***** struct enq_hdr *****
+
enq_hdr::enq_hdr():
_hdr(),
#if defined(JRNL_BIG_ENDIAN) && defined(JRNL_32_BIT)
_filler0(0),
#endif
+ _xidsize(0),
+#if defined(JRNL_LITTLE_ENDIAN) && defined(JRNL_32_BIT)
+ _filler0(0),
+#endif
+#if defined(JRNL_BIG_ENDIAN) && defined(JRNL_32_BIT)
+ _filler1(0),
+#endif
_dsize(0)
#if defined(JRNL_LITTLE_ENDIAN) && defined(JRNL_32_BIT)
- , _filler0(0)
+ , _filler1(0)
#endif
{}
-enq_hdr::enq_hdr(const u_int32_t magic, const u_int64_t rid, const size_t dsize,
- const u_int8_t version, const u_int8_t uflag):
- _hdr(magic, rid, version, uflag),
+enq_hdr::enq_hdr(const u_int32_t magic, const u_int64_t rid, const size_t xidsize,
+ const size_t dsize, const u_int8_t version, const bool transient):
+ _hdr(magic, rid, version, transient?ENQ_HDR_TRANSIENT_MASK:0),
#if defined(JRNL_BIG_ENDIAN) && defined(JRNL_32_BIT)
_filler0(0),
#endif
+ _xidsize(xidsize),
+#if defined(JRNL_LITTLE_ENDIAN) && defined(JRNL_32_BIT)
+ _filler0(0),
+#endif
+#if defined(JRNL_BIG_ENDIAN) && defined(JRNL_32_BIT)
+ _filler1(0),
+#endif
_dsize(dsize)
#if defined(JRNL_LITTLE_ENDIAN) && defined(JRNL_32_BIT)
- , _filler0(0)
+ , _filler1(0)
#endif
{}
+const u_int32_t enq_hdr::ENQ_HDR_TRANSIENT_MASK = 0x1;
-enq_tail::enq_tail():
- _xmagic(0xffffff), // 1's complement of 0
- _rid(0)
-{}
-enq_tail::enq_tail(const enq_hdr& rhdr):
- _xmagic(~rhdr._hdr._magic),
- _rid(rhdr._hdr._rid)
+// ***** struct deq_hdr *****
+
+deq_hdr::deq_hdr():
+ _hdr(),
+ _deq_rid(0),
+#if defined(JRNL_BIG_ENDIAN) && defined(JRNL_32_BIT)
+ _filler0(0),
+#endif
+ _xidsize(0)
+#if defined(JRNL_LITTLE_ENDIAN) && defined(JRNL_32_BIT)
+ , _filler0(0)
+#endif
{}
-enq_tail::enq_tail(const u_int32_t xmagic, const u_int64_t rid):
- _xmagic(xmagic),
- _rid(rid)
+deq_hdr::deq_hdr(const u_int32_t magic, const u_int64_t rid, const u_int64_t deq_rid,
+ const size_t xidsize, const u_int8_t version, const u_int8_t uflag):
+ _hdr(magic, rid, version, uflag),
+ _deq_rid(deq_rid),
+#if defined(JRNL_BIG_ENDIAN) && defined(JRNL_32_BIT)
+ _filler0(0),
+#endif
+ _xidsize(xidsize)
+#if defined(JRNL_LITTLE_ENDIAN) && defined(JRNL_32_BIT)
+ , _filler0(0)
+#endif
{}
-deq_hdr::deq_hdr():
+
+// ***** struct dtx_hdr *****
+
+
+dtx_hdr::dtx_hdr():
_hdr(),
- _deq_rid(0)
+#if defined(JRNL_BIG_ENDIAN) && defined(JRNL_32_BIT)
+ _filler0(0),
+#endif
+ _xidsize(0)
+#if defined(JRNL_LITTLE_ENDIAN) && defined(JRNL_32_BIT)
+ , _filler0(0)
+#endif
{}
-deq_hdr::deq_hdr(const u_int32_t magic, const u_int64_t rid, const u_int64_t deq_rid,
- const u_int8_t version, const u_int8_t uflag):
+dtx_hdr::dtx_hdr(const u_int32_t magic, const u_int64_t rid, const size_t xidsize,
+ const u_int8_t version, const u_int8_t uflag):
_hdr(magic, rid, version, uflag),
- _deq_rid(deq_rid)
+#if defined(JRNL_BIG_ENDIAN) && defined(JRNL_32_BIT)
+ _filler0(0),
+#endif
+ _xidsize(xidsize)
+#if defined(JRNL_LITTLE_ENDIAN) && defined(JRNL_32_BIT)
+ , _filler0(0)
+#endif
{}
} // namespace journal
Modified: store/trunk/cpp/lib/jrnl/file_hdr.hpp
===================================================================
--- store/trunk/cpp/lib/jrnl/file_hdr.hpp 2007-09-25 19:25:41 UTC (rev 946)
+++ store/trunk/cpp/lib/jrnl/file_hdr.hpp 2007-09-25 20:25:49 UTC (rev 947)
@@ -53,7 +53,6 @@
#pragma pack(1)
/**
- * \struct hdr
* \brief Struct for data common to the head of all journal files and records.
* This includes identification for the file type, the encoding version, endian
* indicator and a record ID.
@@ -64,7 +63,7 @@
* +---+---+---+---+---+---+---+---+
* | magic | v | e | flags |
* +---+---+---+---+---+---+---+---+
- * | rec-id |
+ * | rid |
* +---+---+---+---+---+---+---+---+
* v = file version (If the format or encoding of this file changes, then this
* number should be incremented)
@@ -114,7 +113,54 @@
/**
- * \struct file_hdr
+ * \brief Struct for data common to the tail of all records. The magic number
+ * used here is the binary inverse (1's complement) of the magic used in the
+ * record header; this minimizes possible confusion with other headers that may
+ * be present during recovery.
+ *
+ * This header precedes all records in journal files.
+ *
+ * Record header info in binary format (12 bytes):
+ * <pre>
+ * 0 7
+ * +---+---+---+---+---+---+---+---+
+ * | ~(magic) | rid |
+ * +---+---+---+---+---+---+---+---+
+ * | rid (con't) |
+ * +---+---+---+---+
+ * </pre>
+ */
+ struct rec_tail
+ {
+ u_int32_t _xmagic; ///< Binary inverse (1's complement) of hdr magic number
+ u_int64_t _rid; ///< ID (rotating 64-bit counter)
+
+
+ /**
+ * \brief Default constructor, which sets all values to 0.
+ */
+ rec_tail();
+
+
+ /**
+ * \brief Convenience constructor which initializes values during construction from
+ * existing enq_hdr instance.
+ */
+ rec_tail(const hdr& _hdr);
+
+ /**
+ * \brief Convenience constructor which initializes values during construction.
+ */
+ rec_tail(const u_int32_t xmagic, const u_int64_t rid);
+
+ /**
+ * \brief Returns the size of the header in bytes.
+ */
+ inline static const size_t size() { return sizeof(rec_tail); }
+ };
+
+
+ /**
* \brief Struct for data common to the head of all journal files. In addition to
* the common data, this includes the record ID and offset of the first record in
* the file.
@@ -129,7 +175,7 @@
* +---+---+---+---+---+---+---+---+ -+
* | magic | v | e | flags | |
* +---+---+---+---+---+---+---+---+ | struct hdr
- * | first rec-id in file | |
+ * | first rid in file | |
* +---+---+---+---+---+---+---+---+ -+
* | fid | reserved (0) |
* +---+---+---+---+---+---+---+---+
@@ -209,20 +255,23 @@
/**
- * \struct enq_hdr
- * \brief Struct for data common to the head of all records. In addition to
- * the common data, this includes the data blob size.
+ * \brief Struct for enqueue record.
*
- * This header precedes all records in journal files.
+ * Struct for enqueue record. In addition to the common data, this header includes both the
+ * xid and data blob sizes.
*
- * Record header info in binary format (24 bytes):
+ * This header precedes all enqueue data in journal files.
+ *
+ * Record header info in binary format (32 bytes):
* <pre>
* 0 7
* +---+---+---+---+---+---+---+---+ -+
* | magic | v | e | flags | |
* +---+---+---+---+---+---+---+---+ | struct hdr
- * | rec-id | |
+ * | rid | |
* +---+---+---+---+---+---+---+---+ -+
+ * | xidsize |
+ * +---+---+---+---+---+---+---+---+
* | dsize |
* +---+---+---+---+---+---+---+---+
* v = file version (If the format or encoding of this file changes, then this
@@ -241,10 +290,18 @@
#if defined(JRNL_BIG_ENDIAN) && defined(JRNL_32_BIT)
u_int32_t _filler0; ///< Big-endian filler for 32-bit size_t
#endif
- size_t _dsize; ///< Record data size
+ size_t _xidsize; ///< XID size
#if defined(JRNL_LITTLE_ENDIAN) && defined(JRNL_32_BIT)
u_int32_t _filler0; ///< Little-endian filler for 32-bit size_t
#endif
+#if defined(JRNL_BIG_ENDIAN) && defined(JRNL_32_BIT)
+ u_int32_t _filler1; ///< Big-endian filler for 32-bit size_t
+#endif
+ size_t _dsize; ///< Record data size
+#if defined(JRNL_LITTLE_ENDIAN) && defined(JRNL_32_BIT)
+ u_int32_t _filler1; ///< Little-endian filler for 32-bit size_t
+#endif
+ static const u_int32_t ENQ_HDR_TRANSIENT_MASK;
/**
* \brief Default constructor, which sets all values to 0.
@@ -254,8 +311,10 @@
/**
* \brief Convenience constructor which initializes values during construction.
*/
- enq_hdr(const u_int32_t magic, const u_int64_t rid, const size_t dsize,
- const u_int8_t version = 0, const u_int8_t uflag = 0);
+ enq_hdr(const u_int32_t magic, const u_int64_t rid, const size_t xidsize,
+ const size_t dsize, const u_int8_t version = 0, const bool transient = false);
+
+ inline const bool is_transient() { return _hdr._uflag & ENQ_HDR_TRANSIENT_MASK; }
/**
* \brief Returns the size of the header in bytes.
@@ -265,71 +324,89 @@
/**
- * \struct enq_tail
- * \brief Struct for data common to the tail of all records. The magic number
- * used here is the binary inverse (1's complement) of the magic used in the
- * record header; this minimizes possible confusion with other headers that may
- * be present during recovery.
+ * \brief Struct for dequeue record.
*
- * This header precedes all records in journal files.
+ * Struct for dequeue record. If this record has a non-zero xidsize field (i.e., there is a
+ * valid XID), then this header is followed by the XID of xidsize bytes and a rec_tail. If,
+ * on the other hand, this record has a zero xidsize (i.e., there is no XID), then the rec_tail
+ * is absent.
*
- * Record header info in binary format (12 bytes):
+ * Note that this record had its own rid distinct from the rid of the record it is dequeueing.
+ * The rid field below is the rid of the dequeue record itself; the deq-rid field is the rid of a
+ * previous enqueue record being dequeued by this record.
+ *
+ * Record header info in binary format (24 bytes):
* <pre>
* 0 7
+ * +---+---+---+---+---+---+---+---+ -+
+ * | magic | v | e | flags | |
+ * +---+---+---+---+---+---+---+---+ | struct hdr
+ * | rid | |
+ * +---+---+---+---+---+---+---+---+ -+
+ * | deq-rid |
* +---+---+---+---+---+---+---+---+
- * | ~(magic) | rec-id |
+ * | xidsize |
* +---+---+---+---+---+---+---+---+
- * |rec-id (con't) |
- * +---+---+---+---+
+ * v = file version (If the format or encoding of this file changes, then this
+ * number should be incremented)
+ * e = endian flag, false (0x00) for little endian, true (0x01) for big endian
* </pre>
+ *
+ * Note that journal files should be transferable between 32- and 64-bit
+ * hardware of the same endianness, but not between hardware of opposite
+ * entianness without some sort of binary conversion utility. Thus buffering
+ * will be needed for types that change size between 32- and 64-bit compiles.
*/
- struct enq_tail
+ struct deq_hdr
{
- u_int32_t _xmagic; ///< Binary inverse (1's complement) of hdr magic number
- u_int64_t _rid; ///< ID (rotating 64-bit counter)
+ hdr _hdr; ///< Common header
+ u_int64_t _deq_rid; ///< Record ID of dequeued record
+#if defined(JRNL_BIG_ENDIAN) && defined(JRNL_32_BIT)
+ u_int32_t _filler0; ///< Big-endian filler for 32-bit size_t
+#endif
+ size_t _xidsize; ///< XID size
+#if defined(JRNL_LITTLE_ENDIAN) && defined(JRNL_32_BIT)
+ u_int32_t _filler0; ///< Little-endian filler for 32-bit size_t
+#endif
-
/**
* \brief Default constructor, which sets all values to 0.
*/
- enq_tail();
+ deq_hdr();
-
/**
- * \brief Convenience constructor which initializes values during construction from
- * existing enq_hdr instance.
- */
- enq_tail(const enq_hdr& rhdr);
-
- /**
* \brief Convenience constructor which initializes values during construction.
*/
- enq_tail(const u_int32_t xmagic, const u_int64_t rid);
+ deq_hdr(const u_int32_t magic, const u_int64_t rid, const u_int64_t deq_rid,
+ const size_t xidsize, const u_int8_t version = 0, const u_int8_t uflag = 0);
/**
* \brief Returns the size of the header in bytes.
*/
- inline static const size_t size() { return sizeof(enq_tail); }
+ inline static const size_t size() { return sizeof(deq_hdr); }
};
/**
- * \struct deq_hdr
- * \brief Struct for dequeue record. This record carries no data or tail,
- * and consists of this struct only. The record-id (part of hdr struct) is
- * the record id for this struct which is counted as a record.
+ * \brief Struct for DTX commit and abort records.
*
- * This header precedes all records in journal files.
+ * Struct for DTX commit and abort records. Only the magic distinguishes between them. Since
+ * this record must be used in the context of a valid XID, the xidsize field must not be zero.
+ * Immediately following this record is the XID itself which is xidsize bytes long, followed by
+ * a rec_tail.
*
+ * Note that this record had its own rid distinct from the rids of the record(s) making up the
+ * transaction it is committing or aborting.
+ *
* Record header info in binary format (24 bytes):
* <pre>
* 0 7
* +---+---+---+---+---+---+---+---+ -+
* | magic | v | e | flags | |
* +---+---+---+---+---+---+---+---+ | struct hdr
- * | rec-id | |
+ * | rid | |
* +---+---+---+---+---+---+---+---+ -+
- * | deq-rec-id |
+ * | xidsize |
* +---+---+---+---+---+---+---+---+
* v = file version (If the format or encoding of this file changes, then this
* number should be incremented)
@@ -341,26 +418,32 @@
* entianness without some sort of binary conversion utility. Thus buffering
* will be needed for types that change size between 32- and 64-bit compiles.
*/
- struct deq_hdr
+ struct dtx_hdr
{
hdr _hdr; ///< Common header
- u_int64_t _deq_rid; ///< Record ID of dequeued record
+#if defined(JRNL_BIG_ENDIAN) && defined(JRNL_32_BIT)
+ u_int32_t _filler0; ///< Big-endian filler for 32-bit size_t
+#endif
+ size_t _xidsize; ///< XID size
+#if defined(JRNL_LITTLE_ENDIAN) && defined(JRNL_32_BIT)
+ u_int32_t _filler0; ///< Little-endian filler for 32-bit size_t
+#endif
/**
* \brief Default constructor, which sets all values to 0.
*/
- deq_hdr();
+ dtx_hdr();
/**
* \brief Convenience constructor which initializes values during construction.
*/
- deq_hdr(const u_int32_t magic, const u_int64_t rid, const u_int64_t deq_rid,
+ dtx_hdr(const u_int32_t magic, const u_int64_t rid, const size_t xidsize,
const u_int8_t version = 0, const u_int8_t uflag = 0);
/**
* \brief Returns the size of the header in bytes.
*/
- inline static const size_t size() { return sizeof(deq_hdr); }
+ inline static const size_t size() { return sizeof(dtx_hdr); }
};
#pragma pack()
Modified: store/trunk/cpp/lib/jrnl/jcfg.hpp
===================================================================
--- store/trunk/cpp/lib/jrnl/jcfg.hpp 2007-09-25 19:25:41 UTC (rev 946)
+++ store/trunk/cpp/lib/jrnl/jcfg.hpp 2007-09-25 20:25:49 UTC (rev 947)
@@ -75,9 +75,11 @@
#define JRNL_INFO_EXTENSION "jinf" ///< Extension for journal info files
#define JRNL_DATA_EXTENSION "jdat" ///< Extension for journal data files
+#define RHM_JDAT_DTXA_MAGIC 0x614d4852 ///< ("RHMa" in little endian) Magic for dtx abort hdrs
+#define RHM_JDAT_DTXC_MAGIC 0x634d4852 ///< ("RHMc" in little endian) Magic for dtx commit hdrs
+#define RHM_JDAT_DEQ_MAGIC 0x644d4852 ///< ("RHMd" in little endian) Magic for deq rec hdrs
+#define RHM_JDAT_ENQ_MAGIC 0x654d4852 ///< ("RHMe" in little endian) Magic for enq rec hdrs
#define RHM_JDAT_FILE_MAGIC 0x664d4852 ///< ("RHMf" in little endian) Magic for file hdrs
-#define RHM_JDAT_ENQ_MAGIC 0x654d4852 ///< ("RHMe" in little endian) Magic for enq rec hdrs
-#define RHM_JDAT_DEQ_MAGIC 0x644d4852 ///< ("RHMd" in little endian) Magic for deq rec hdrs
#define RHM_JDAT_EMPTY_MAGIC 0x784d4852 ///< ("RHMx" in little endian) Magic for empty dblk
#define RHM_JDAT_VERSION 0x01 ///< Version (of file layout)
#define RHM_CLEAN_CHAR 0xff ///< Char used to clear empty space on disk
Modified: store/trunk/cpp/lib/jrnl/jcntl.cpp
===================================================================
--- store/trunk/cpp/lib/jrnl/jcntl.cpp 2007-09-25 19:25:41 UTC (rev 946)
+++ store/trunk/cpp/lib/jrnl/jcntl.cpp 2007-09-25 20:25:49 UTC (rev 947)
@@ -395,10 +395,18 @@
{
case RHM_JDAT_ENQ_MAGIC:
{
+ size_t xidsize = 0;
size_t recsize = 0;
#if defined(JRNL_BIG_ENDIAN) && defined(JRNL_32_BIT)
jifs.ignore(sizeof(u_int32_t));
#endif
+ jifs.read((char*)&xidsize, sizeof(size_t));
+#if defined(JRNL_LITTLE_ENDIAN) && defined(JRNL_32_BIT)
+ jifs.ignore(sizeof(u_int32_t));
+#endif
+#if defined(JRNL_BIG_ENDIAN) && defined(JRNL_32_BIT)
+ jifs.ignore(sizeof(u_int32_t));
+#endif
jifs.read((char*)&recsize, sizeof(size_t));
#if defined(JRNL_LITTLE_ENDIAN) && defined(JRNL_32_BIT)
jifs.ignore(sizeof(u_int32_t));
@@ -409,7 +417,7 @@
rd._h_rid = h._rid;
//std::cout << " e" << h._rid;
u_int32_t rec_dblks = jrec::size_dblks((size_t)recsize + sizeof(enq_hdr) +
- sizeof(enq_tail));
+ sizeof(rec_tail));
foffs += rec_dblks * JRNL_DBLK_SIZE;
jifs.seekg(foffs);
}
@@ -417,7 +425,15 @@
case RHM_JDAT_DEQ_MAGIC:
{
u_int64_t drid = 0;
+ size_t xidsize = 0;
jifs.read((char*)&drid, sizeof(u_int64_t));
+#if defined(JRNL_BIG_ENDIAN) && defined(JRNL_32_BIT)
+ jifs.ignore(sizeof(u_int32_t));
+#endif
+ jifs.read((char*)&xidsize, sizeof(size_t));
+#if defined(JRNL_LITTLE_ENDIAN) && defined(JRNL_32_BIT)
+ jifs.ignore(sizeof(u_int32_t));
+#endif
try
{
_emap.get_remove_fid(drid);
Modified: store/trunk/cpp/lib/jrnl/jrec.cpp
===================================================================
--- store/trunk/cpp/lib/jrnl/jrec.cpp 2007-09-25 19:25:41 UTC (rev 946)
+++ store/trunk/cpp/lib/jrnl/jrec.cpp 2007-09-25 20:25:49 UTC (rev 947)
@@ -64,7 +64,7 @@
}
void
-jrec::chk_tail(enq_tail& tail, enq_hdr& hdr) const throw (jexception)
+jrec::chk_tail(rec_tail& tail, enq_hdr& hdr) const throw (jexception)
{
std::stringstream ss;
if (tail._xmagic != ~hdr._hdr._magic)
Modified: store/trunk/cpp/lib/jrnl/jrec.hpp
===================================================================
--- store/trunk/cpp/lib/jrnl/jrec.hpp 2007-09-25 19:25:41 UTC (rev 946)
+++ store/trunk/cpp/lib/jrnl/jrec.hpp 2007-09-25 20:25:49 UTC (rev 947)
@@ -159,7 +159,7 @@
protected:
virtual void chk_hdr(enq_hdr& hdr, u_int64_t rid, bool enq = true)
const throw (jexception) = 0;
- virtual void chk_tail(enq_tail& tail, enq_hdr& hdr) const throw (jexception);
+ virtual void chk_tail(rec_tail& tail, enq_hdr& hdr) const throw (jexception);
}; // class jrec
} // namespace journal
Modified: store/trunk/cpp/lib/jrnl/pmgr.cpp
===================================================================
--- store/trunk/cpp/lib/jrnl/pmgr.cpp 2007-09-25 19:25:41 UTC (rev 946)
+++ store/trunk/cpp/lib/jrnl/pmgr.cpp 2007-09-25 20:25:49 UTC (rev 947)
@@ -109,7 +109,7 @@
_pg_cntr(0),
_pg_offset_dblks(0),
_aio_evt_rem(0),
- _data_rec(),
+ _enq_rec(),
_cb(NULL)
{
initialize();
Modified: store/trunk/cpp/lib/jrnl/pmgr.hpp
===================================================================
--- store/trunk/cpp/lib/jrnl/pmgr.hpp 2007-09-25 19:25:41 UTC (rev 946)
+++ store/trunk/cpp/lib/jrnl/pmgr.hpp 2007-09-25 20:25:49 UTC (rev 947)
@@ -45,7 +45,7 @@
#include <deque>
#include <libaio.h>
#include <jrnl/aio_cb.hpp>
-#include <jrnl/data_rec.hpp>
+#include <jrnl/enq_rec.hpp>
#include <jrnl/data_tok.hpp>
#include <jrnl/enq_map.hpp>
#include <jrnl/nlfh.hpp>
@@ -60,13 +60,13 @@
*/
enum _iores
{
- RHM_IORES_SUCCESS = 0, ///< Success: IO operation completed noramlly.
- RHM_IORES_AIO_WAIT, ///< IO operation suspended as all pages in cache are waiting for AIO.
- RHM_IORES_EMPTY, ///< During read operations, nothing further is available to read.
- RHM_IORES_FULL, ///< During write operations, the journal files are full.
- RHM_IORES_BUSY, ///< Another blocking operation is in progress.
- RHM_IORES_TXPENDING, ///< Operation blocked by pending transaction.
- RHM_IORES_NOTIMPL ///< Not yet implemented.
+ RHM_IORES_SUCCESS = 0, ///< Success: IO operation completed noramlly.
+ RHM_IORES_AIO_WAIT, ///< IO operation suspended - all pages are waiting for AIO.
+ RHM_IORES_EMPTY, ///< During read operations, nothing further is available to read.
+ RHM_IORES_FULL, ///< During write operations, the journal files are full.
+ RHM_IORES_BUSY, ///< Another blocking operation is in progress.
+ RHM_IORES_TXPENDING, ///< Operation blocked by pending transaction.
+ RHM_IORES_NOTIMPL ///< Not yet implemented.
};
typedef _iores iores;
@@ -125,7 +125,7 @@
u_int32_t _pg_cntr; ///< Page counter; determines if file rotation req'd
u_int32_t _pg_offset_dblks; ///< Page offset (used so far) in data blocks
u_int32_t _aio_evt_rem; ///< Remaining AIO events
- data_rec _data_rec; ///< Data record into/from which data is encoded/decoded
+ enq_rec _enq_rec; ///< Data record into/from which data is encoded/decoded
// TODO: move _cb down to wmgr, it is the only class that uses it There is no need for
// read callbacks based on AIO. - (check this asertion)
aio_cb _cb; ///< Callback function pointer for AIO events
Modified: store/trunk/cpp/lib/jrnl/rmgr.cpp
===================================================================
--- store/trunk/cpp/lib/jrnl/rmgr.cpp 2007-09-25 19:25:41 UTC (rev 946)
+++ store/trunk/cpp/lib/jrnl/rmgr.cpp 2007-09-25 20:25:49 UTC (rev 947)
@@ -130,7 +130,7 @@
case RHM_JDAT_ENQ_MAGIC:
{
//std::cout << " E" << std::flush;
- _data_rec.reset(mbuf, mbsize); // sets enqueue rec size
+ _enq_rec.reset(mbuf, mbsize); // sets enqueue rec size
// Check if RID of this rec is still enqueued, if so read it, else skip
#ifdef RHM_RDONLY
bool is_enq = true;
@@ -189,7 +189,11 @@
{
//std::cout << " %" << std::flush;
// skip this record, it is already dequeued
- u_int64_t* sizep = (u_int64_t*)((char*)rptr + sizeof(hdr));
+ u_int64_t* sizep = (u_int64_t*)((char*)rptr + sizeof(hdr) +
+#if defined(JRNL_32_BIT)
+ sizeof(u_int32_t) + // filler
+#endif
+ sizeof(size_t));
//u_int32_t dblks = jrec::size_dblks((size_t)*sizep + sizeof(enq_hdr) + sizeof(enq_tail));
//std::cout << "{" << *sizep << "=" << dblks << "d}" << std::flush;
dtokp->set_dsize((size_t)*sizep);
@@ -293,25 +297,25 @@
//std::cout << " V" << dtokp->dblocks_read() << "," << dblks_rem() << std::flush;
// Read data from this page, first block will have header and data size.
- u_int32_t dblks_rd = _data_rec.decode(h, rptr, dtokp->dblocks_read(), dblks_rem());
+ u_int32_t dblks_rd = _enq_rec.decode(h, rptr, dtokp->dblocks_read(), dblks_rem());
dtokp->incr_dblocks_read(dblks_rd);
_pg_offset_dblks += dblks_rd;
// If data still incomplete, move to next page and decode again
- while (dtokp->dblocks_read() < _data_rec.rec_size_dblks())
+ while (dtokp->dblocks_read() < _enq_rec.rec_size_dblks())
{
//std::cout << " W" << std::flush;
rotate_page();
if (_page_cb_arr[_pg_index]._state != AIO_COMPLETE)
{
dtokp->set_rstate(data_tok::READ_PART);
- dtokp->set_dsize(_data_rec.data_size());
+ dtokp->set_dsize(_enq_rec.data_size());
return RHM_IORES_AIO_WAIT;
}
rptr = (void*)((char*)_page_ptr_arr[_pg_index]);
- dblks_rd = _data_rec.decode(h, rptr, dtokp->dblocks_read(), dblks_rem());
+ dblks_rd = _enq_rec.decode(h, rptr, dtokp->dblocks_read(), dblks_rem());
dtokp->incr_dblocks_read(dblks_rd);
_pg_offset_dblks += dblks_rd;
}
@@ -324,7 +328,7 @@
// Set the record size in dtokp
dtokp->set_rstate(data_tok::READ);
- dtokp->set_dsize(_data_rec.data_size());
+ dtokp->set_dsize(_enq_rec.data_size());
return RHM_IORES_SUCCESS;
}
@@ -349,7 +353,7 @@
const iores
rmgr::skip(data_tok* dtokp) throw (jexception)
{
- u_int32_t dsize_dblks = jrec::size_dblks(dtokp->dsize() + sizeof(enq_hdr) + sizeof(enq_tail));
+ u_int32_t dsize_dblks = jrec::size_dblks(dtokp->dsize() + sizeof(enq_hdr) + sizeof(rec_tail));
u_int32_t tot_dblk_cnt = dtokp->rstate() == data_tok::SKIP_PART ? dtokp->dblocks_read() : 0;
//std::cout << " S" << tot_dblk_cnt << std::flush;
while (true)
@@ -367,15 +371,15 @@
dtokp->incr_dblocks_read(this_dblk_cnt);
_pg_offset_dblks += this_dblk_cnt;
tot_dblk_cnt += this_dblk_cnt;
- if (tot_dblk_cnt < _data_rec.rec_size_dblks())
+ if (tot_dblk_cnt < _enq_rec.rec_size_dblks())
{
-//std::cout << " * t=" << tot_dblk_cnt << " rs=" << _data_rec.rec_size_dblks() << std::flush;
+//std::cout << " * t=" << tot_dblk_cnt << " rs=" << _enq_rec.rec_size_dblks() << std::flush;
rotate_page();
if (_page_cb_arr[_pg_index]._state != AIO_COMPLETE)
{
dtokp->set_rstate(data_tok::SKIP_PART);
// Use data_tok::dblocks_proc field to save how many skip bloks still to go...
- dtokp->set_dblocks_read(_data_rec.rec_size_dblks() - tot_dblk_cnt);
+ dtokp->set_dblocks_read(_enq_rec.rec_size_dblks() - tot_dblk_cnt);
return RHM_IORES_AIO_WAIT;
}
//std::cout << " !" << std::flush;
Modified: store/trunk/cpp/lib/jrnl/wmgr.cpp
===================================================================
--- store/trunk/cpp/lib/jrnl/wmgr.cpp 2007-09-25 19:25:41 UTC (rev 946)
+++ store/trunk/cpp/lib/jrnl/wmgr.cpp 2007-09-25 20:25:49 UTC (rev 947)
@@ -114,7 +114,7 @@
else
rid = cont ? _wrfc.rid() - 1 : _wrfc.get_incr_rid();
- _data_rec.reset(rid, mbuf, dlen);
+ _enq_rec.reset(rid, mbuf, dlen);
if (!cont)
dtok->set_rid(rid);
bool done = false;
@@ -123,7 +123,7 @@
assert(_pg_offset_dblks < JRNL_WMGR_PAGE_SIZE * JRNL_SBLK_SIZE);
void* wptr = (void*)((char*)_page_ptr_arr[_pg_index] + _pg_offset_dblks * JRNL_DBLK_SIZE);
u_int32_t data_offs_dblks = dtok->dblocks_written();
- u_int32_t ret = _data_rec.encode(wptr, data_offs_dblks,
+ u_int32_t ret = _enq_rec.encode(wptr, data_offs_dblks,
(JRNL_WMGR_PAGE_SIZE * JRNL_SBLK_SIZE) - _pg_offset_dblks);
#if !(defined(RHM_WRONLY) || defined(RHM_RDONLY))
if (data_offs_dblks == 0)
@@ -137,7 +137,7 @@
dtok->incr_dblocks_written(ret);
// Is the encoding of this record complete?
- if (dtok->dblocks_written() >= _data_rec.rec_size_dblks())
+ if (dtok->dblocks_written() >= _enq_rec.rec_size_dblks())
{
// TODO: Incorrect - must set state to ENQ_CACHED; ENQ_SUBM is set when AIO returns.
dtok->set_wstate(data_tok::ENQ_SUBM);
@@ -153,7 +153,7 @@
// Has the file header been written (i.e. write pointers still at 0)?
if (_wrfc.empty())
{
- u_int32_t rec_dblks_rem = _data_rec.rec_size_dblks() - data_offs_dblks;
+ u_int32_t rec_dblks_rem = _enq_rec.rec_size_dblks() - data_offs_dblks;
bool file_fit = rec_dblks_rem <= JRNL_FILE_SIZE * JRNL_SBLK_SIZE;
bool file_full = rec_dblks_rem == JRNL_FILE_SIZE * JRNL_SBLK_SIZE;
size_t fro = 0;
@@ -232,7 +232,7 @@
// (currently dblk = 128 bytes). JRNL_DBLK_SIZE must be a power of 2.
// IF JRNL_DBLK_SIZE IS SET TO < 32 (i.e. 16 OR LESS) BYTES, THIS ENCODING WILL FAIL!
//***
- deq_hdr dhdr(RHM_JDAT_DEQ_MAGIC, rid, dtok->rid(), RHM_JDAT_VERSION);
+ deq_hdr dhdr(RHM_JDAT_DEQ_MAGIC, rid, dtok->rid(), 0, RHM_JDAT_VERSION);
void* wptr = (void*)((char*)_page_ptr_arr[_pg_index] + _pg_offset_dblks * JRNL_DBLK_SIZE);
::memcpy(wptr, &dhdr, sizeof(dhdr));
#ifdef RHM_CLEAN
Modified: store/trunk/cpp/tests/jrnl/Makefile.rtest
===================================================================
--- store/trunk/cpp/tests/jrnl/Makefile.rtest 2007-09-25 19:25:41 UTC (rev 946)
+++ store/trunk/cpp/tests/jrnl/Makefile.rtest 2007-09-25 20:25:49 UTC (rev 947)
@@ -55,7 +55,9 @@
data_tok.o \
file_hdr.o \
jrec.o \
- data_rec.o \
+ enq_rec.o \
+ deq_rec.o \
+ dtx_rec.o \
nlfh.o \
lfh.o \
rrfc.o \
@@ -123,6 +125,8 @@
clean-data:
@rm -rf jdata
+
+clean-test-data:
@rm -rf rd_test_jrnls
-clean-all: clean clean-data
+clean-all: clean clean-data clean-test-data
Modified: store/trunk/cpp/tests/jrnl/ftest.py
===================================================================
--- store/trunk/cpp/tests/jrnl/ftest.py 2007-09-25 19:25:41 UTC (rev 946)
+++ store/trunk/cpp/tests/jrnl/ftest.py 2007-09-25 20:25:49 UTC (rev 947)
@@ -29,6 +29,7 @@
sblk_size = 4 * dblk_size
file_size = (3072 + 1) * sblk_size
num_files = 8
+hdr_ver = 1
def load(f, klass):
args = load_args(f, klass)
@@ -91,7 +92,7 @@
self.flags = flags
self.rid = rid
if self.magic[-1] not in ['0x00', 'd', 'e', 'f', 'x']:
- error = 3;
+ error = 3
def __str__(self):
if self.empty():
@@ -111,6 +112,8 @@
def check(self):
if self.empty() or self.magic[-1] not in ['d', 'e', 'f', 'x']:
return True
+ if self.ver != hdr_ver and self.magic[-1] != 'x':
+ raise Exception('%s: Invalid header version: found %d, expected %d.' % (self, self.ver, hdr_ver))
return False
@@ -142,16 +145,17 @@
class DeqHdr(Hdr):
- format = '=Q'
+ format = '=QQ'
- def init(self, f, foffs, deq_rid):
+ def init(self, f, foffs, deq_rid, xid):
self.deq_rid = deq_rid
+ self.xid = xid
def __str__(self):
- return '%s d=%d' % (Hdr.__str__(self), self.deq_rid)
+ return '%s d=%d x=%d' % (Hdr.__str__(self), self.deq_rid, self.xid)
-class EnqTail(Sizeable):
+class RecTail(Sizeable):
format = '=IQ'
@@ -169,9 +173,10 @@
class EnqRec(Hdr):
- format = '=Q'
+ format = '=QQ'
- def init(self, f, foffs, dsize):
+ def init(self, f, foffs, xid, dsize):
+ self.xid = xid
self.dsize = dsize
self.data = None
self.enq_tail = None
@@ -209,12 +214,12 @@
def load_tail(self, f):
if self.tail_bin == None:
self.tail_offs = f.tell()
- self.tail_bin = f.read(calcsize(EnqTail.format))
- if len(self.tail_bin) != calcsize(EnqTail.format):
+ self.tail_bin = f.read(calcsize(RecTail.format))
+ if len(self.tail_bin) != calcsize(RecTail.format):
return
else:
- self.tail_bin += f.read(calcsize(EnqTail.format) - len(self.tail_bin))
- self.enq_tail = EnqTail(self.tail_offs, *unpack(EnqTail.format, self.tail_bin))
+ self.tail_bin += f.read(calcsize(RecTail.format) - len(self.tail_bin))
+ self.enq_tail = RecTail(self.tail_offs, *unpack(RecTail.format, self.tail_bin))
magic_int = 0
inv_magic_int_1 = 0
for i in range(0,4):
@@ -235,7 +240,7 @@
dstr = self.data
if self.enq_tail == None:
return '%s s=%d data=\"%s\" [no tail]' % (Hdr.__str__(self), self.dsize, dstr)
- return '%s s=%d data=\"%s\" %s' % (Hdr.__str__(self), self.dsize, dstr, self.enq_tail)
+ return '%s x=%d s=%d data=\"%s\" %s' % (Hdr.__str__(self), self.xid, self.dsize, dstr, self.enq_tail)
class Main:
@@ -284,7 +289,7 @@
else:
print ' * fro ok: 0x%08x' % self.fhdr.fro
self.first_rec = False
- if isinstance(hdr, EnqRec):
+ if isinstance(hdr, EnqRec) and not stop:
while not hdr.complete():
stop = self.advance_file()
if stop:
@@ -299,7 +304,7 @@
self.msg_cnt += 1
if self.auto_deq:
self.enqueued[hdr.rid] = hdr
- elif isinstance(hdr, DeqHdr):
+ elif isinstance(hdr, DeqHdr) and not stop:
if self.auto_deq:
if hdr.deq_rid in self.enqueued:
del self.enqueued[hdr.deq_rid]
Modified: store/trunk/cpp/tests/jrnl/msg_consumer.cpp
===================================================================
--- store/trunk/cpp/tests/jrnl/msg_consumer.cpp 2007-09-25 19:25:41 UTC (rev 946)
+++ store/trunk/cpp/tests/jrnl/msg_consumer.cpp 2007-09-25 20:25:49 UTC (rev 947)
@@ -71,13 +71,6 @@
msg_consumer::initialize(const u_int32_t numMsgs)
{
_num_msgs = numMsgs;
-// for (u_int32_t i=0; i<_num_msgs; i++)
-// {
-// rhm::journal::data_tok* dtp = new rhm::journal::data_tok;
-// dtp->set_wstate(rhm::journal::data_tok::ENQ); // Assume all msgs enqueued OK
-// _dtok_master_list.push_back(dtp);
-//
-// }
}
void
@@ -93,15 +86,6 @@
_tot_dblks = 0;
_tot_dsize = 0;
_aio_cmpl_dtok_list.clear();
-// for (u_int32_t i=0; i<_dtok_master_list.size(); i++)
-// {
-// if (_dtok_master_list[i])
-// {
-// delete _dtok_master_list[i];
-// _dtok_master_list[i] = NULL;
-// }
-// }
-// _dtok_master_list.clear();
}
u_int32_t
Modified: store/trunk/cpp/tests/jrnl/rtest
===================================================================
--- store/trunk/cpp/tests/jrnl/rtest 2007-09-25 19:25:41 UTC (rev 946)
+++ store/trunk/cpp/tests/jrnl/rtest 2007-09-25 20:25:49 UTC (rev 947)
@@ -290,7 +290,8 @@
# Remove all previous data and build
echo -n "Clearing previous build and test files..."
- ${MAKE} clean-all
+ ${MAKE} clean
+ ${MAKE} clean-data
echo " ok"
# Make for write pipeline tests
Modified: store/trunk/cpp/tests/jrnl/rtests.csv
===================================================================
--- store/trunk/cpp/tests/jrnl/rtests.csv 2007-09-25 19:25:41 UTC (rev 946)
+++ store/trunk/cpp/tests/jrnl/rtests.csv 2007-09-25 20:25:49 UTC (rev 947)
@@ -8,65 +8,65 @@
4,10,10,10,TRUE,,"10-byte message"
,,,,,,
"Transition from one d-block to two per message",,,,,,
-5,10,92,92,FALSE,,"1 dblk exact fit"
-6,10,93,93,FALSE,,"1 dblk + 1 byte"
-7,10,92,92,TRUE,,"1 dblk exact fit"
-8,10,93,93,TRUE,,"1 dblk + 1 byte"
+5,10,84,84,FALSE,,"1 dblk exact fit"
+6,10,85,85,FALSE,,"1 dblk + 1 byte"
+7,10,84,84,TRUE,,"1 dblk exact fit"
+8,10,85,85,TRUE,,"1 dblk + 1 byte"
,,,,,,
"Transition from one s-block to two per message",,,,,,
-9,10,476,476,FALSE,,"1 sblk exact fit"
-10,10,477,477,FALSE,,"1 sblk + 1 byte"
-11,10,476,476,TRUE,,"1 sblk exact fit"
-12,10,477,477,TRUE,,"1 sblk + 1 byte"
+9,10,468,468,FALSE,,"1 sblk exact fit"
+10,10,469,469,FALSE,,"1 sblk + 1 byte"
+11,10,468,468,TRUE,,"1 sblk exact fit"
+12,10,469,469,TRUE,,"1 sblk + 1 byte"
,,,,,,
"Transition from first page to second",,,,,,
-13,8,8156,8156,FALSE,,"8 * 1/8 page; Total = 1 page exact fit"
-14,9,8156,8156,FALSE,,"9 * 1/8 page"
-15,8,8157,8157,FALSE,,"8 * (1/8 page + 1 byte)"
-16,8,8028,8028,TRUE,,"8 * (1/8 page – 1 dblk for deq record); Total = 1 page exact fit with deqs"
-17,9,8028,8028,TRUE,,"9 * (1/8 page – 1 dblk for deq record)"
-18,8,8029,8029,TRUE,,"8 * (1/8 page – 1 dblk for deq record + 1 byte)"
+13,8,8148,8148,FALSE,,"8 * 1/8 page; Total = 1 page exact fit"
+14,9,8148,8148,FALSE,,"9 * 1/8 page"
+15,8,8149,8149,FALSE,,"8 * (1/8 page + 1 byte)"
+16,8,8020,8020,TRUE,,"8 * (1/8 page – 1 dblk for deq record); Total = 1 page exact fit with deqs"
+17,9,8020,8020,TRUE,,"9 * (1/8 page – 1 dblk for deq record)"
+18,8,8021,8021,TRUE,,"8 * (1/8 page – 1 dblk for deq record + 1 byte)"
,,,,,,
"Page cache rollover (from last page back to page 0) - no dequeues as exact sizes are needed and dequeues are non-deterministic",,,,,,
-19,16,65500,65500,FALSE,,"16 * (1 page exact fit); Total = entire cache exactly"
-20,17,65500,65500,FALSE,,"17 * (1 page exact fit); Total = entire cache + reuse of 1 page"
-21,11,98268,98268,FALSE,,"11 * 1.5 pages"
-22,11,98140,98140,TRUE,,"11 * (1.5 pages including 1 sblk for deq record)"
+19,16,65492,65492,FALSE,,"16 * (1 page exact fit); Total = entire cache exactly"
+20,17,65492,65492,FALSE,,"17 * (1 page exact fit); Total = entire cache + reuse of 1 page"
+21,11,98260,98260,FALSE,,"11 * 1.5 pages"
+22,11,98132,98132,TRUE,,"11 * (1.5 pages including 1 sblk for deq record)"
,,,,,,
"File transition (from file 0000 to 0001) - no dequeues as exact sizes are needed and dequeues are non-deterministic",,,,,,
-23,24,65500,65500,FALSE,,"24 * (1 page exact fit); Total = entire file exactly"
-24,25,65500,65500,FALSE,,"25 * (1 page exact fit); Total = entire file + 1 page"
-25,10,163804,163804,FALSE,,"10 * (2.5 pages); Total = entire file + 2 pages"
-26,10,163676,163676,TRUE,,"10 * (2.5 pages including 1 sblk for deq record); Total = entire file + 2 pages"
+23,24,65492,65492,FALSE,,"24 * (1 page exact fit); Total = entire file exactly"
+24,25,65492,65492,FALSE,,"25 * (1 page exact fit); Total = entire file + 1 page"
+25,10,163796,163796,FALSE,,"10 * (2.5 pages); Total = entire file + 2 pages"
+26,10,163668,163668,TRUE,,"10 * (2.5 pages including 1 sblk for deq record); Total = entire file + 2 pages"
,,,,,,
"File rollover (from file 0007 to 0000) - RHM_WRONLY req'd for auto-dequeue == FALSE",,,,,,
-27,16,786396,786396,FALSE,,"16 * (12 pages = ½ file); Total = 8 files exactly"
+27,16,786388,786388,FALSE,,"16 * (12 pages = ½ file); Total = 8 files exactly"
,,,,,,
"Multi-page messages (large messages) - tests various paths in encoder; no dequeues required to test this functionality.",,,,,,
-28,16,65500,65500,FALSE,,"16 * (1 page exact fit)"
-29,16,65501,65501,FALSE,,"16 * (1 page + 1 byte): tail split"
-30,16,65511,65511,FALSE,,"16 * (1 page + 11 bytes): tail split"
-31,16,65512,65512,FALSE,,"16 * (1 page + 12 bytes): tail separated exactly"
-32,16,65513,65513,FALSE,,"16 * (1 page + 13 bytes): data split"
-33,16,131036,131036,FALSE,,"16 * (2 pages exact fit)"
-34,16,131037,131037,FALSE,,"16 * (2 pages + 1 byte): tail split"
-35,16,131047,131047,FALSE,,"16 * (2 pages + 11 bytes): tail split"
-36,16,131048,131048,FALSE,,"16 * (2 pages + 12 bytes): tail separated exactly"
-37,16,131049,131049,FALSE,,"16 * (2 pages + 13 bytes) data split"
-38,16,262108,262108,FALSE,,"16 * (4 pages exact fit)"
-39,16,262109,262109,FALSE,,"16 * (4 pages + 1 byte: tail split)"
-40,16,262119,262119,FALSE,,"16 * (4 pages + 1 byte: tail split)"
-41,16,262120,262120,FALSE,,"16 * (4 pages + 12 bytes: tail separated)"
-42,16,262121,262121,FALSE,,"16 * (4 pages + 13 bytes: data split)"
-43,16,229340,229340,FALSE,,"16 * (3.5 pages)"
-44,16,229341,229341,FALSE,,"16 * (3.5 pages + 1 byte)"
+28,16,65492,65492,FALSE,,"16 * (1 page exact fit)"
+29,16,65493,65493,FALSE,,"16 * (1 page + 1 byte): tail split"
+30,16,65503,65503,FALSE,,"16 * (1 page + 11 bytes): tail split"
+31,16,65504,65504,FALSE,,"16 * (1 page + 12 bytes): tail separated exactly"
+32,16,65505,65505,FALSE,,"16 * (1 page + 13 bytes): data split"
+33,16,131028,131028,FALSE,,"16 * (2 pages exact fit)"
+34,16,131029,131029,FALSE,,"16 * (2 pages + 1 byte): tail split"
+35,16,131039,131039,FALSE,,"16 * (2 pages + 11 bytes): tail split"
+36,16,131040,131040,FALSE,,"16 * (2 pages + 12 bytes): tail separated exactly"
+37,16,131041,131041,FALSE,,"16 * (2 pages + 13 bytes) data split"
+38,16,262100,262100,FALSE,,"16 * (4 pages exact fit)"
+39,16,262101,262101,FALSE,,"16 * (4 pages + 1 byte: tail split)"
+40,16,262111,262111,FALSE,,"16 * (4 pages + 1 byte: tail split)"
+41,16,262112,262112,FALSE,,"16 * (4 pages + 12 bytes: tail separated)"
+42,16,262113,262113,FALSE,,"16 * (4 pages + 13 bytes: data split)"
+43,16,229332,229332,FALSE,,"16 * (3.5 pages)"
+44,16,229333,229333,FALSE,,"16 * (3.5 pages + 1 byte)"
,,,,,,
"These set up journals for circular tests (repeatedly reading same journal) Make sure value testing is off!",,,,,,
-45,98304,0,92,FALSE,,"1 dblk no dequeues"
-46,49152,0,92,TRUE,,"1 dblk with dequeues"
-47,49152,0,220,FALSE,,"2 dblk no dequeues"
-48,24576,0,220,TRUE,,"2 dblk with dequeues"
-49,32768,220,220,TRUE,,"2 dblk fixed with dequeues"
+45,98304,0,84,FALSE,,"1 dblk no dequeues"
+46,49152,0,84,TRUE,,"1 dblk with dequeues"
+47,49152,0,212,FALSE,,"2 dblk no dequeues"
+48,24576,0,212,TRUE,,"2 dblk with dequeues"
+49,32768,212,212,TRUE,,"2 dblk fixed with dequeues"
,,,,,,
"Circular tests",,,,,,
50,10000000,0,0,FALSE,,"Read 10,000,000 messages from one of the circular test journals above"
Modified: store/trunk/cpp/tests/jrnl/rwtests.csv
===================================================================
--- store/trunk/cpp/tests/jrnl/rwtests.csv 2007-09-25 19:25:41 UTC (rev 946)
+++ store/trunk/cpp/tests/jrnl/rwtests.csv 2007-09-25 20:25:49 UTC (rev 947)
@@ -8,61 +8,61 @@
4,10,10,10,TRUE,,"10-byte message"
,,,,,,
"Transition from one d-block to two per message",,,,,,
-5,10,92,92,FALSE,,"1 dblk exact fit"
-6,10,93,93,FALSE,,"1 dblk + 1 byte"
-7,10,92,92,TRUE,,"1 dblk exact fit"
-8,10,93,93,TRUE,,"1 dblk + 1 byte"
+5,10,84,84,FALSE,,"1 dblk exact fit"
+6,10,85,85,FALSE,,"1 dblk + 1 byte"
+7,10,84,84,TRUE,,"1 dblk exact fit"
+8,10,85,85,TRUE,,"1 dblk + 1 byte"
,,,,,,
"Transition from one s-block to two per message",,,,,,
-9,10,476,476,FALSE,,"1 sblk exact fit"
-10,10,477,477,FALSE,,"1 sblk + 1 byte"
-11,10,476,476,TRUE,,"1 sblk exact fit"
-12,10,477,477,TRUE,,"1 sblk + 1 byte"
+9,10,468,468,FALSE,,"1 sblk exact fit"
+10,10,469,469,FALSE,,"1 sblk + 1 byte"
+11,10,468,468,TRUE,,"1 sblk exact fit"
+12,10,469,469,TRUE,,"1 sblk + 1 byte"
,,,,,,
"Transition from first page to second",,,,,,
-13,8,4060,4060,FALSE,,"8 * 1/8 page; Total = 1 page exact fit"
-14,9,4060,4060,FALSE,,"9 * 1/8 page"
-15,8,4061,4061,FALSE,,"8 * (1/8 page + 1 byte)"
-16,8,3932,3932,TRUE,,"8 * (1/8 page - 1 dblk for deq record); Total = 1 page exact fit with deqs"
-17,9,3932,3932,TRUE,,"9 * (1/8 page - 1 dblk for deq record)"
-18,8,3933,3933,TRUE,,"8 * (1/8 page - 1 dblk for deq record + 1 byte)"
+13,8,4052,4052,FALSE,,"8 * 1/8 page; Total = 1 page exact fit"
+14,9,4052,4052,FALSE,,"9 * 1/8 page"
+15,8,4053,4053,FALSE,,"8 * (1/8 page + 1 byte)"
+16,8,3924,3924,TRUE,,"8 * (1/8 page - 1 dblk for deq record); Total = 1 page exact fit with deqs"
+17,9,3924,3924,TRUE,,"9 * (1/8 page - 1 dblk for deq record)"
+18,8,3925,3925,TRUE,,"8 * (1/8 page - 1 dblk for deq record + 1 byte)"
,,,,,,
"Page cache rollover (from page 32 back to page 0) - no dequeues as exact sizes are needed and dequeues are non-deterministic",,,,,,
-19,32,32732,32732,FALSE,,"32 * (1 page exact fit); Total = entire cache exactly"
-20,33,32732,32732,FALSE,,"33 * (1 page exact fit); Total = entire cache + reuse of 1 page"
-21,22,49116,49116,FALSE,,"22 * 1.5 pages"
-22,22,48988,48988,TRUE,,"22 * (1.5 pages including 1 sblk for deq record)"
+19,32,32724,32724,FALSE,,"32 * (1 page exact fit); Total = entire cache exactly"
+20,33,32724,32724,FALSE,,"33 * (1 page exact fit); Total = entire cache + reuse of 1 page"
+21,22,49108,49108,FALSE,,"22 * 1.5 pages"
+22,22,48980,48980,TRUE,,"22 * (1.5 pages including 1 sblk for deq record)"
,,,,,,
"File transition (from file 0000 to 0001) - no dequeues as exact sizes are needed and dequeues are non-deterministic",,,,,,
-23,48,32732,32732,FALSE,,"48 * (1 page exact fit); Total = entire file exactly"
-24,49,32732,32732,FALSE,,"49 * (1 page exact fit); Total = entire file + 1 page"
-25,20,81884,81884,FALSE,,"20 * (2.5 pages); Total = entire file + 2 pages"
-26,20,81756,81756,TRUE,,"20 * (2.5 pages including 1 sblk for deq record); Total = entire file + 2 pages"
+23,48,32724,32724,FALSE,,"48 * (1 page exact fit); Total = entire file exactly"
+24,49,32724,32724,FALSE,,"49 * (1 page exact fit); Total = entire file + 1 page"
+25,20,81876,81876,FALSE,,"20 * (2.5 pages); Total = entire file + 2 pages"
+26,20,81748,81748,TRUE,,"20 * (2.5 pages including 1 sblk for deq record); Total = entire file + 2 pages"
,,,,,,
"File rollover (from file 0007 to 0000) - RHM_WRONLY req'd for auto-dequeue == FALSE",,,,,,
-27,16,786396,786396,FALSE,,"16 * (24 pages = ½ file); Total = 8 files exactly"
-28,16,786268,786268,TRUE,,"16 * (24 pages = ½ file); Total = 8 files exactly"
-29,17,786268,786268,TRUE,,"17 * (24 pages = ½ file); Total = 8 files + file 0 overwritten by ½ file"
-30,16,786269,786269,TRUE,,"16 * (24 pages + 1 byte); Total = 8 files + file 0 overwritten by 16 sblks"
-31,32,786268,786268,TRUE,,"32 * (24 pages = ½ file); Total = 16 files exactly, all files overwritten once"
-32,33,786268,786268,TRUE,,"33 * (24 pages = ½ file); Total = 16 ½ files, all files overwritten once + file 0 overwritten again by ½ file"
-33,32,786269,786269,TRUE,,"32 * (24 pages + 1 byte); All files overwritten once + file 0 overwritten again by 32 sblks"
+27,16,786388,786388,FALSE,,"16 * (24 pages = ½ file); Total = 8 files exactly"
+28,16,786260,786260,TRUE,,"16 * (24 pages = ½ file); Total = 8 files exactly"
+29,17,786260,786260,TRUE,,"17 * (24 pages = ½ file); Total = 8 files + file 0 overwritten by ½ file"
+30,16,786261,786261,TRUE,,"16 * (24 pages + 1 byte); Total = 8 files + file 0 overwritten by 16 sblks"
+31,32,786260,786260,TRUE,,"32 * (24 pages = ½ file); Total = 16 files exactly, all files overwritten once"
+32,33,786260,786260,TRUE,,"33 * (24 pages = ½ file); Total = 16 ½ files, all files overwritten once + file 0 overwritten again by ½ file"
+33,32,786261,786261,TRUE,,"32 * (24 pages + 1 byte); All files overwritten once + file 0 overwritten again by 32 sblks"
,,,,,,
"Multi-page messages (large messages) - tests various paths in encoder; no dequeues required to test this functionality.",,,,,,
-34,16,32732,32732,FALSE,,"16 * (1 page exact fit)"
-35,16,32733,32733,FALSE,,"16 * (1 page + 1 byte): tail split"
-36,16,32743,32743,FALSE,,"16 * (1 page + 11 bytes): tail split"
-37,16,32744,32744,FALSE,,"16 * (1 page + 12 bytes): tail separated exactly"
-38,16,32745,32745,FALSE,,"16 * (1 page + 13 bytes): data split"
-39,16,65500,65500,FALSE,,"16 * (2 pages exact fit)"
-40,16,65501,65501,FALSE,,"16 * (2 pages + 1 byte): tail split"
-41,16,65511,65511,FALSE,,"16 * (2 pages + 11 bytes): tail split"
-42,16,65512,65512,FALSE,,"16 * (2 pages + 12 bytes): tail separated exactly"
-43,16,65513,65513,FALSE,,"16 * (2 pages + 13 bytes) data split"
-44,16,131036,131036,FALSE,,"16 * (4 pages exact fit)"
-45,16,131037,131037,FALSE,,"16 * (4 pages + 1 byte: tail split)"
-46,16,131047,131047,FALSE,,"16 * (4 pages + 1 byte: tail split)"
-47,16,131048,131048,FALSE,,"16 * (4 pages + 12 bytes: tail separated)"
-48,16,131049,131049,FALSE,,"16 * (4 pages + 13 bytes: data split)"
-49,16,114652,114652,FALSE,,"16 * (3.5 pages)"
-50,16,114653,114653,FALSE,,"16 * (3.5 pages + 1 byte)"
+34,16,32724,32724,FALSE,,"16 * (1 page exact fit)"
+35,16,32725,32725,FALSE,,"16 * (1 page + 1 byte): tail split"
+36,16,32735,32735,FALSE,,"16 * (1 page + 11 bytes): tail split"
+37,16,32736,32736,FALSE,,"16 * (1 page + 12 bytes): tail separated exactly"
+38,16,32737,32737,FALSE,,"16 * (1 page + 13 bytes): data split"
+39,16,65492,65492,FALSE,,"16 * (2 pages exact fit)"
+40,16,65493,65493,FALSE,,"16 * (2 pages + 1 byte): tail split"
+41,16,65503,65503,FALSE,,"16 * (2 pages + 11 bytes): tail split"
+42,16,65504,65504,FALSE,,"16 * (2 pages + 12 bytes): tail separated exactly"
+43,16,65505,65505,FALSE,,"16 * (2 pages + 13 bytes) data split"
+44,16,131028,131028,FALSE,,"16 * (4 pages exact fit)"
+45,16,131029,131029,FALSE,,"16 * (4 pages + 1 byte: tail split)"
+46,16,131039,131039,FALSE,,"16 * (4 pages + 1 byte: tail split)"
+47,16,131040,131040,FALSE,,"16 * (4 pages + 12 bytes: tail separated)"
+48,16,131041,131041,FALSE,,"16 * (4 pages + 13 bytes: data split)"
+49,16,114644,114644,FALSE,,"16 * (3.5 pages)"
+50,16,114645,114645,FALSE,,"16 * (3.5 pages + 1 byte)"
Modified: store/trunk/cpp/tests/jrnl/tests.ods
===================================================================
(Binary files differ)
Modified: store/trunk/cpp/tests/jrnl/wtests.csv
===================================================================
--- store/trunk/cpp/tests/jrnl/wtests.csv 2007-09-25 19:25:41 UTC (rev 946)
+++ store/trunk/cpp/tests/jrnl/wtests.csv 2007-09-25 20:25:49 UTC (rev 947)
@@ -8,99 +8,99 @@
4,10,10,10,TRUE,,"10-byte message"
,,,,,,
"Transition from one d-block to two per message",,,,,,
-5,10,92,92,FALSE,,"1 dblk exact fit"
-6,10,93,93,FALSE,,"1 dblk + 1 byte"
-7,10,92,92,TRUE,,"1 dblk exact fit"
-8,10,93,93,TRUE,,"1 dblk + 1 byte"
+5,10,84,84,FALSE,,"1 dblk exact fit"
+6,10,85,85,FALSE,,"1 dblk + 1 byte"
+7,10,84,84,TRUE,,"1 dblk exact fit"
+8,10,85,85,TRUE,,"1 dblk + 1 byte"
,,,,,,
"Transition from one s-block to two per message",,,,,,
-9,10,476,476,FALSE,,"1 sblk exact fit"
-10,10,477,477,FALSE,,"1 sblk + 1 byte"
-11,10,476,476,TRUE,,"1 sblk exact fit"
-12,10,477,477,TRUE,,"1 sblk + 1 byte"
+9,10,468,468,FALSE,,"1 sblk exact fit"
+10,10,469,469,FALSE,,"1 sblk + 1 byte"
+11,10,468,468,TRUE,,"1 sblk exact fit"
+12,10,469,469,TRUE,,"1 sblk + 1 byte"
,,,,,,
"Transition from first page to second",,,,,,
-13,8,4060,4060,FALSE,,"8 * 1/8 page; Total = 1 page exact fit"
-14,9,4060,4060,FALSE,,"9 * 1/8 page"
-15,8,4061,4061,FALSE,,"8 * (1/8 page + 1 byte)"
-16,8,3932,3932,TRUE,,"8 * (1/8 page - 1 dblk for deq record); Total = 1 page exact fit with deqs"
-17,9,3932,3932,TRUE,,"9 * (1/8 page - 1 dblk for deq record)"
-18,8,3933,3933,TRUE,,"8 * (1/8 page - 1 dblk for deq record + 1 byte)"
+13,8,4052,4052,FALSE,,"8 * 1/8 page; Total = 1 page exact fit"
+14,9,4052,4052,FALSE,,"9 * 1/8 page"
+15,8,4053,4053,FALSE,,"8 * (1/8 page + 1 byte)"
+16,8,3924,3924,TRUE,,"8 * (1/8 page - 1 dblk for deq record); Total = 1 page exact fit with deqs"
+17,9,3924,3924,TRUE,,"9 * (1/8 page - 1 dblk for deq record)"
+18,8,3925,3925,TRUE,,"8 * (1/8 page - 1 dblk for deq record + 1 byte)"
,,,,,,
"Page cache rollover (from page 32 back to page 0) - no dequeues as exact sizes are needed and dequeues are non-deterministic",,,,,,
-19,32,32732,32732,FALSE,,"32 * (1 page exact fit); Total = entire cache exactly"
-20,33,32732,32732,FALSE,,"33 * (1 page exact fit); Total = entire cache + reuse of 1 page"
-21,22,49116,49116,FALSE,,"22 * 1.5 pages"
-22,22,48988,48988,TRUE,,"22 * (1.5 pages including 1 sblk for deq record)"
+19,32,32724,32724,FALSE,,"32 * (1 page exact fit); Total = entire cache exactly"
+20,33,32724,32724,FALSE,,"33 * (1 page exact fit); Total = entire cache + reuse of 1 page"
+21,22,49108,49108,FALSE,,"22 * 1.5 pages"
+22,22,48980,48980,TRUE,,"22 * (1.5 pages including 1 sblk for deq record)"
,,,,,,
"File transition (from file 0000 to 0001) - no dequeues as exact sizes are needed and dequeues are non-deterministic",,,,,,
-23,48,32732,32732,FALSE,,"48 * (1 page exact fit); Total = entire file exactly"
-24,49,32732,32732,FALSE,,"49 * (1 page exact fit); Total = entire file + 1 page"
-25,20,81884,81884,FALSE,,"20 * (2.5 pages); Total = entire file + 2 pages"
-26,20,81756,81756,TRUE,,"20 * (2.5 pages including 1 sblk for deq record); Total = entire file + 2 pages"
+23,48,32724,32724,FALSE,,"48 * (1 page exact fit); Total = entire file exactly"
+24,49,32724,32724,FALSE,,"49 * (1 page exact fit); Total = entire file + 1 page"
+25,20,81876,81876,FALSE,,"20 * (2.5 pages); Total = entire file + 2 pages"
+26,20,81748,81748,TRUE,,"20 * (2.5 pages including 1 sblk for deq record); Total = entire file + 2 pages"
,,,,,,
"File rollover (from file 0007 to 0000) - RHM_WRONLY req'd for auto-dequeue == FALSE",,,,,,
-27,16,786396,786396,FALSE,,"16 * (24 pages = ½ file); Total = 8 files exactly"
-28,17,786396,786396,FALSE,,"17 * (24 pages = ½ file); Total = 8 files + file 0 overwritten by ½ file"
-29,16,786397,786397,FALSE,,"16 * (24 pages + 1 byte); Total = 8 files + file 0 overwritten by 16 sblks"
-30,32,786396,786396,FALSE,,"32 * (24 pages = ½ file); Total = 16 files exactly, all files overwritten once"
-31,33,786396,786396,FALSE,,"33 * (24 pages = ½ file); Total = 16 ½ files, all files overwritten once + file 0 overwritten again by ½ file"
-32,32,786397,786397,FALSE,,"32 * (24 pages + 1 byte); All files overwritten once + file 0 overwritten again by 32 sblks"
+27,16,786388,786388,FALSE,,"16 * (24 pages = ½ file); Total = 8 files exactly"
+28,17,786388,786388,FALSE,,"17 * (24 pages = ½ file); Total = 8 files + file 0 overwritten by ½ file"
+29,16,786389,786389,FALSE,,"16 * (24 pages + 1 byte); Total = 8 files + file 0 overwritten by 16 sblks"
+30,32,786388,786388,FALSE,,"32 * (24 pages = ½ file); Total = 16 files exactly, all files overwritten once"
+31,33,786388,786388,FALSE,,"33 * (24 pages = ½ file); Total = 16 ½ files, all files overwritten once + file 0 overwritten again by ½ file"
+32,32,786389,786389,FALSE,,"32 * (24 pages + 1 byte); All files overwritten once + file 0 overwritten again by 32 sblks"
,,,,,,
"Multi-page messages (large messages) - tests various paths in encoder; no dequeues required to test this functionality.",,,,,,
-33,16,32732,32732,FALSE,,"16 * (1 page exact fit)"
-34,16,32733,32733,FALSE,,"16 * (1 page + 1 byte): tail split"
-35,16,32743,32743,FALSE,,"16 * (1 page + 11 bytes): tail split"
-36,16,32744,32744,FALSE,,"16 * (1 page + 12 bytes): tail separated exactly"
-37,16,32745,32745,FALSE,,"16 * (1 page + 13 bytes): data split"
-38,16,65500,65500,FALSE,,"16 * (2 pages exact fit)"
-39,16,65501,65501,FALSE,,"16 * (2 pages + 1 byte): tail split"
-40,16,65511,65511,FALSE,,"16 * (2 pages + 11 bytes): tail split"
-41,16,65512,65512,FALSE,,"16 * (2 pages + 12 bytes): tail separated exactly"
-42,16,65513,65513,FALSE,,"16 * (2 pages + 13 bytes) data split"
-43,16,131036,131036,FALSE,,"16 * (4 pages exact fit)"
-44,16,131037,131037,FALSE,,"16 * (4 pages + 1 byte: tail split)"
-45,16,131047,131047,FALSE,,"16 * (4 pages + 1 byte: tail split)"
-46,16,131048,131048,FALSE,,"16 * (4 pages + 12 bytes: tail separated)"
-47,16,131049,131049,FALSE,,"16 * (4 pages + 13 bytes: data split)"
-48,16,114652,114652,FALSE,,"16 * (3.5 pages)"
-49,16,114653,114653,FALSE,,"16 * (3.5 pages + 1 byte)"
+33,16,32724,32724,FALSE,,"16 * (1 page exact fit)"
+34,16,32725,32725,FALSE,,"16 * (1 page + 1 byte): tail split"
+35,16,32735,32735,FALSE,,"16 * (1 page + 11 bytes): tail split"
+36,16,32736,32736,FALSE,,"16 * (1 page + 12 bytes): tail separated exactly"
+37,16,32737,32737,FALSE,,"16 * (1 page + 13 bytes): data split"
+38,16,65492,65492,FALSE,,"16 * (2 pages exact fit)"
+39,16,65493,65493,FALSE,,"16 * (2 pages + 1 byte): tail split"
+40,16,65503,65503,FALSE,,"16 * (2 pages + 11 bytes): tail split"
+41,16,65504,65504,FALSE,,"16 * (2 pages + 12 bytes): tail separated exactly"
+42,16,65505,65505,FALSE,,"16 * (2 pages + 13 bytes) data split"
+43,16,131028,131028,FALSE,,"16 * (4 pages exact fit)"
+44,16,131029,131029,FALSE,,"16 * (4 pages + 1 byte: tail split)"
+45,16,131039,131039,FALSE,,"16 * (4 pages + 1 byte: tail split)"
+46,16,131040,131040,FALSE,,"16 * (4 pages + 12 bytes: tail separated)"
+47,16,131041,131041,FALSE,,"16 * (4 pages + 13 bytes: data split)"
+48,16,114644,114644,FALSE,,"16 * (3.5 pages)"
+49,16,114645,114645,FALSE,,"16 * (3.5 pages + 1 byte)"
,,,,,,
"Large (multi-megabyte) messages - RHM_WRONLY req'd for auto-dequeue == FALSE",,,,,,
-50,32,1572828,1572828,FALSE,,"32 * (48 pages = 1 file exactly)"
-51,32,1572829,1572829,FALSE,,"32 * (48 pages + 1 byte)"
-52,32,1605596,1605596,FALSE,,"32 * (49 pages = 1 file + 1 page)"
-53,16,3145692,3145692,FALSE,,"16 * (96 pages = 2 files exactly)"
-54,16,3145693,3145693,FALSE,,"16 * (96 pages + 1 byte)"
-55,16,3178460,3178460,FALSE,,"16 * (97 pages = 2 files + 1 page"
-56,8,6291420,6291420,FALSE,,"8 * (192 pages = 4 files exactly)"
-57,8,6291421,6291421,FALSE,,"8 * (192 pages + 1 byte)"
-58,8,6324188,6324188,FALSE,,"8 * (193 pages = 4 files + 1 page)"
-59,32,1572700,1572700,TRUE,,"32 * (48 pages including 1 sblk for deq record = 1 file exactly)"
-60,32,1572701,1572701,TRUE,,"32 * (48 pages including 1 sblk for deq record + 1 byte)"
-61,32,1605468,1605468,TRUE,,"32 * (49 pages including 1 sblk for deq record = 1 file + 1 page)"
-62,16,3145564,3145564,TRUE,,"16 * (96 pages including 1 sblk for deq record = 2 files exactly)"
-63,16,3145565,3145565,TRUE,,"16 * (96 pages including 1 sblk for deq record + 1 byte)"
-64,16,3178332,3178332,TRUE,,"16 * (97 pages including 1 sblk for deq record = 2 files + 1 page"
-65,8,6291292,6291292,TRUE,,"8 * (192 pages including 1 sblk for deq record = 4 files exactly)"
-66,8,6291293,6291293,TRUE,,"8 * (192 pages including 1 sblk for deq record + 1 byte)"
-67,8,6324060,6324060,TRUE,,"8 * (193 pages including 1 sblk for deq record = 4 files + 1 page)"
+50,32,1572820,1572820,FALSE,,"32 * (48 pages = 1 file exactly)"
+51,32,1572821,1572821,FALSE,,"32 * (48 pages + 1 byte)"
+52,32,1605588,1605588,FALSE,,"32 * (49 pages = 1 file + 1 page)"
+53,16,3145684,3145684,FALSE,,"16 * (96 pages = 2 files exactly)"
+54,16,3145685,3145685,FALSE,,"16 * (96 pages + 1 byte)"
+55,16,3178452,3178452,FALSE,,"16 * (97 pages = 2 files + 1 page"
+56,8,6291412,6291412,FALSE,,"8 * (192 pages = 4 files exactly)"
+57,8,6291413,6291413,FALSE,,"8 * (192 pages + 1 byte)"
+58,8,6324180,6324180,FALSE,,"8 * (193 pages = 4 files + 1 page)"
+59,32,1572692,1572692,TRUE,,"32 * (48 pages including 1 sblk for deq record = 1 file exactly)"
+60,32,1572693,1572693,TRUE,,"32 * (48 pages including 1 sblk for deq record + 1 byte)"
+61,32,1605460,1605460,TRUE,,"32 * (49 pages including 1 sblk for deq record = 1 file + 1 page)"
+62,16,3145556,3145556,TRUE,,"16 * (96 pages including 1 sblk for deq record = 2 files exactly)"
+63,16,3145557,3145557,TRUE,,"16 * (96 pages including 1 sblk for deq record + 1 byte)"
+64,16,3178324,3178324,TRUE,,"16 * (97 pages including 1 sblk for deq record = 2 files + 1 page"
+65,8,6291284,6291284,TRUE,,"8 * (192 pages including 1 sblk for deq record = 4 files exactly)"
+66,8,6291285,6291285,TRUE,,"8 * (192 pages including 1 sblk for deq record + 1 byte)"
+67,8,6324052,6324052,TRUE,,"8 * (193 pages including 1 sblk for deq record = 4 files + 1 page)"
,,,,,,
"High volume tests of random message lengths - RHM_WRONLY req'd for auto-dequeue == FALSE",,,,,,
-68,5000000,0,92,FALSE,,"1 dblk max"
-69,3000000,0,348,FALSE,,"3 dblks max"
-70,1600000,0,1244,FALSE,,"10 dblks max"
-71,600000,0,3804,FALSE,,"30 dblks max"
-72,200000,0,12764,FALSE,,"100 dblks max"
-73,60000,0,38364,FALSE,,"300 dblks max"
-74,20000,0,127964,FALSE,,"1000 dblks max"
-75,2500000,0,92,TRUE,,"1 dblk max"
-76,1500000,0,348,TRUE,,"3 dblks max"
-77,800000,0,1244,TRUE,,"10 dblks max"
-78,300000,0,3804,TRUE,,"30 dblks max"
-79,100000,0,12768,TRUE,,"100 dblks max"
-80,30000,0,38364,TRUE,,"300 dblks max"
-81,10000,0,127964,TRUE,,"1000 dblks max"
+68,5000000,0,84,FALSE,,"1 dblk max"
+69,3000000,0,340,FALSE,,"3 dblks max"
+70,1600000,0,1236,FALSE,,"10 dblks max"
+71,600000,0,3796,FALSE,,"30 dblks max"
+72,200000,0,12756,FALSE,,"100 dblks max"
+73,60000,0,38356,FALSE,,"300 dblks max"
+74,20000,0,127956,FALSE,,"1000 dblks max"
+75,2500000,0,84,TRUE,,"1 dblk max"
+76,1500000,0,340,TRUE,,"3 dblks max"
+77,800000,0,1236,TRUE,,"10 dblks max"
+78,300000,0,3796,TRUE,,"30 dblks max"
+79,100000,0,12756,TRUE,,"100 dblks max"
+80,30000,0,38356,TRUE,,"300 dblks max"
+81,10000,0,127956,TRUE,,"1000 dblks max"
,,,,,,
"STANDARD PERFORMANCE BENCHMARK: 10,000,000 writes, data=220b (2 dblks)",,,,,,
82,10000000,220,220,FALSE,,"2 dblks"
17 years, 5 months
- 1
- 0
rhmessaging commits: r946 - store/trunk/cpp/tests.
by rhmessaging-commits＠lists.jboss.org
Author: aconway
Date: 2007-09-25 15:25:41 -0400 (Tue, 25 Sep 2007)
New Revision: 946
Modified:
store/trunk/cpp/tests/OrderingTest.cpp
store/trunk/cpp/tests/SimpleTest.cpp
store/trunk/cpp/tests/TransactionalTest.cpp
store/trunk/cpp/tests/TwoPhaseCommitTest.cpp
Log:
Updated for qpid renaming: broker/BrokerQueue.cpp -> Queue.cpp
Modified: store/trunk/cpp/tests/OrderingTest.cpp
===================================================================
--- store/trunk/cpp/tests/OrderingTest.cpp 2007-09-25 16:39:31 UTC (rev 945)
+++ store/trunk/cpp/tests/OrderingTest.cpp 2007-09-25 19:25:41 UTC (rev 946)
@@ -26,7 +26,7 @@
#include <qpid/framing/AMQMethodBody.h>
#include <qpid/framing/AMQHeaderBody.h>
#include <qpid/broker/Message.h>
-#include <qpid/broker/BrokerQueue.h>
+#include <qpid/broker/Queue.h>
#include <qpid/broker/RecoveryManagerImpl.h>
#include "test_plugin.h"
#include <iostream>
Modified: store/trunk/cpp/tests/SimpleTest.cpp
===================================================================
--- store/trunk/cpp/tests/SimpleTest.cpp 2007-09-25 16:39:31 UTC (rev 945)
+++ store/trunk/cpp/tests/SimpleTest.cpp 2007-09-25 19:25:41 UTC (rev 946)
@@ -31,7 +31,7 @@
#include <qpid/framing/AMQMethodBody.h>
#include <qpid/framing/ChannelAdapter.h>
#include <qpid/broker/Message.h>
-#include <qpid/broker/BrokerQueue.h>
+#include <qpid/broker/Queue.h>
#include <qpid/broker/DirectExchange.h>
#include <qpid/broker/RecoveryManagerImpl.h>
#include <qpid/broker/QueuePolicy.h>
Modified: store/trunk/cpp/tests/TransactionalTest.cpp
===================================================================
--- store/trunk/cpp/tests/TransactionalTest.cpp 2007-09-25 16:39:31 UTC (rev 945)
+++ store/trunk/cpp/tests/TransactionalTest.cpp 2007-09-25 19:25:41 UTC (rev 946)
@@ -27,7 +27,7 @@
#include <qpid/framing/AMQHeaderBody.h>
#include <qpid/framing/AMQMethodBody.h>
#include <qpid/broker/Message.h>
-#include <qpid/broker/BrokerQueue.h>
+#include <qpid/broker/Queue.h>
#include <qpid/broker/RecoveryManagerImpl.h>
#include <iostream>
#include <memory>
Modified: store/trunk/cpp/tests/TwoPhaseCommitTest.cpp
===================================================================
--- store/trunk/cpp/tests/TwoPhaseCommitTest.cpp 2007-09-25 16:39:31 UTC (rev 945)
+++ store/trunk/cpp/tests/TwoPhaseCommitTest.cpp 2007-09-25 19:25:41 UTC (rev 946)
@@ -27,7 +27,7 @@
#include <qpid/framing/AMQHeaderBody.h>
#include <qpid/framing/AMQMethodBody.h>
#include <qpid/broker/Message.h>
-#include <qpid/broker/BrokerQueue.h>
+#include <qpid/broker/Queue.h>
#include <qpid/broker/RecoveryManagerImpl.h>
#include <iostream>
#include <memory>
17 years, 5 months
- 1
- 0
rhmessaging commits: r945 - in store/branches/java/M2: java/bdbstore and 2 other directories.
by rhmessaging-commits＠lists.jboss.org
Author: rupertlssmith
Date: 2007-09-25 12:39:31 -0400 (Tue, 25 Sep 2007)
New Revision: 945
Added:
store/branches/java/M2/java/bdbstore/mvn-repo/sleepycat/berkeleydb-je/3.2.42/
store/branches/java/M2/java/bdbstore/mvn-repo/sleepycat/berkeleydb-je/3.2.42/berkeleydb-je-3.2.42.jar
store/branches/java/M2/java/bdbstore/mvn-repo/sleepycat/berkeleydb-je/3.2.42/berkeleydb-je-3.2.42.pom
Removed:
store/branches/java/M2/java/bdbstore/mvn-repo/sleepycat/berkeleydb-je/3.2.42/berkeleydb-je-3.2.42.jar
store/branches/java/M2/java/bdbstore/mvn-repo/sleepycat/berkeleydb-je/3.2.42/berkeleydb-je-3.2.42.pom
Modified:
store/branches/java/M2/
store/branches/java/M2/java/bdbstore/pom.xml
Log:
Merged revisions 889 via svnmerge from
https://svn.jboss.org/repos/rhmessaging/store/branches/java/M2.1
........
r889 | rupertlssmith | 2007-08-23 11:56:22 +0100 (Thu, 23 Aug 2007) | 1 line
Trial update of BDB to 3.2.42 to fix slow-down problems.
........
Property changes on: store/branches/java/M2
___________________________________________________________________
Name: svnmerge-integrated
- /store/branches/java/M2.1:1-888,931-933
+ /store/branches/java/M2.1:1-889,931-933
Copied: store/branches/java/M2/java/bdbstore/mvn-repo/sleepycat/berkeleydb-je/3.2.42 (from rev 889, store/branches/java/M2.0.1/java/bdbstore/mvn-repo/sleepycat/berkeleydb-je/3.2.42)
Deleted: store/branches/java/M2/java/bdbstore/mvn-repo/sleepycat/berkeleydb-je/3.2.42/berkeleydb-je-3.2.42.jar
===================================================================
(Binary files differ)
Copied: store/branches/java/M2/java/bdbstore/mvn-repo/sleepycat/berkeleydb-je/3.2.42/berkeleydb-je-3.2.42.jar (from rev 889, store/branches/java/M2.0.1/java/bdbstore/mvn-repo/sleepycat/berkeleydb-je/3.2.42/berkeleydb-je-3.2.42.jar)
===================================================================
(Binary files differ)
Deleted: store/branches/java/M2/java/bdbstore/mvn-repo/sleepycat/berkeleydb-je/3.2.42/berkeleydb-je-3.2.42.pom
===================================================================
--- store/branches/java/M2.0.1/java/bdbstore/mvn-repo/sleepycat/berkeleydb-je/3.2.42/berkeleydb-je-3.2.42.pom 2007-08-23 10:56:22 UTC (rev 889)
+++ store/branches/java/M2/java/bdbstore/mvn-repo/sleepycat/berkeleydb-je/3.2.42/berkeleydb-je-3.2.42.pom 2007-09-25 16:39:31 UTC (rev 945)
@@ -1,9 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?><project>
- <modelVersion>4.0.0</modelVersion>
- <groupId>sleepycat</groupId>
- <artifactId>berkeleydb-je</artifactId>
- <version>3.2.42</version>
- <distributionManagement>
- <status>deployed</status>
- </distributionManagement>
-</project>
\ No newline at end of file
Copied: store/branches/java/M2/java/bdbstore/mvn-repo/sleepycat/berkeleydb-je/3.2.42/berkeleydb-je-3.2.42.pom (from rev 889, store/branches/java/M2.0.1/java/bdbstore/mvn-repo/sleepycat/berkeleydb-je/3.2.42/berkeleydb-je-3.2.42.pom)
===================================================================
--- store/branches/java/M2/java/bdbstore/mvn-repo/sleepycat/berkeleydb-je/3.2.42/berkeleydb-je-3.2.42.pom (rev 0)
+++ store/branches/java/M2/java/bdbstore/mvn-repo/sleepycat/berkeleydb-je/3.2.42/berkeleydb-je-3.2.42.pom 2007-09-25 16:39:31 UTC (rev 945)
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?><project>
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>sleepycat</groupId>
+ <artifactId>berkeleydb-je</artifactId>
+ <version>3.2.42</version>
+ <distributionManagement>
+ <status>deployed</status>
+ </distributionManagement>
+</project>
\ No newline at end of file
Modified: store/branches/java/M2/java/bdbstore/pom.xml
===================================================================
--- store/branches/java/M2/java/bdbstore/pom.xml 2007-09-25 13:00:25 UTC (rev 944)
+++ store/branches/java/M2/java/bdbstore/pom.xml 2007-09-25 16:39:31 UTC (rev 945)
@@ -61,7 +61,7 @@
<dependency>
<groupId>sleepycat</groupId>
<artifactId>berkeleydb-je</artifactId>
- <version>3.1.0</version>
+ <version>3.2.42</version>
</dependency>
<dependency>
<groupId>log4j</groupId>
17 years, 5 months
- 1
- 0