Author: kpvdr
Date: 2007-09-28 12:02:33 -0400 (Fri, 28 Sep 2007)
New Revision: 950
Added:
store/trunk/cpp/tests/jrnl/janalyze.py
Removed:
store/trunk/cpp/tests/jrnl/ftest.py
Modified:
store/trunk/cpp/lib/jrnl/jcntl.hpp
store/trunk/cpp/tests/jrnl/Makefile.am
store/trunk/cpp/tests/jrnl/rtest
store/trunk/cpp/tests/jrnl/tests.ods
store/trunk/cpp/tests/jrnl/wtests.csv
Log:
Updated python analysis tool to handle XID
Modified: store/trunk/cpp/lib/jrnl/jcntl.hpp
===================================================================
--- store/trunk/cpp/lib/jrnl/jcntl.hpp 2007-09-27 19:29:25 UTC (rev 949)
+++ store/trunk/cpp/lib/jrnl/jcntl.hpp 2007-09-28 16:02:33 UTC (rev 950)
@@ -295,8 +295,6 @@
* \param tot_data_len Total data length.
* \param this_data_len Amount to be written in this enqueue operation.
* \param dtokp Pointer to data token which contains the details of the enqueue
operation.
- * \param xid String containing xid. An empty string (i.e. length=0) will be
considered
- * non-transactional.
* \param transient Flag indicating transient persistence (ie, ignored on
recover).
*
* \exception TODO
@@ -304,6 +302,19 @@
const iores enqueue_data_record(const void* const data_buff, const size_t
tot_data_len,
const size_t this_data_len, data_tok* dtokp, const bool transient =
false)
throw (jexception);
+ /**
+ * \brief Enqueue data.
+ *
+ * \param data_buff Pointer to data to be enqueued for this enqueue operation.
+ * \param tot_data_len Total data length.
+ * \param this_data_len Amount to be written in this enqueue operation.
+ * \param dtokp Pointer to data token which contains the details of the enqueue
operation.
+ * \param xid String containing xid. An empty string (i.e. length=0) will be
considered
+ * non-transactional.
+ * \param transient Flag indicating transient persistence (ie, ignored on
recover).
+ *
+ * \exception TODO
+ */
const iores enqueue_data_record(const void* const data_buff, const size_t
tot_data_len,
const size_t this_data_len, data_tok* dtokp, const std::string& xid,
const bool transient = false) throw (jexception);
@@ -403,12 +414,27 @@
*
* \param dtokp Pointer to data_tok instance for this data, used to track state of
data
* through journal.
+ *
+ * \exception TODO
+ */
+ const iores dequeue_data_record(data_tok* const dtokp) throw (jexception);
+
+ /**
+ * \brief Dequeues (marks as no longer needed) data record in journal.
+ *
+ * Dequeues (marks as no longer needed) data record in journal as part of a
transaction.
+ * Note that it is possible to use the same data token instance used to enqueue
this data;
+ * it contains the RID needed to correctly mark this data as dequeued in the
journal.
+ * Otherwise the RID of the record to be dequeued and the write state of ENQ must
be
+ * manually set in a new or reset instance of data_tok.
+ *
+ * \param dtokp Pointer to data_tok instance for this data, used to track state of
data
+ * through journal.
* \param xid String containing xid. An empty string (i.e. length=0) will be
considered
* non-transactional.
*
* \exception TODO
*/
- const iores dequeue_data_record(data_tok* const dtokp) throw (jexception);
const iores dequeue_data_record(data_tok* const dtokp, const std::string&
xid)
throw (jexception);
Modified: store/trunk/cpp/tests/jrnl/Makefile.am
===================================================================
--- store/trunk/cpp/tests/jrnl/Makefile.am 2007-09-27 19:29:25 UTC (rev 949)
+++ store/trunk/cpp/tests/jrnl/Makefile.am 2007-09-28 16:02:33 UTC (rev 950)
@@ -50,4 +50,4 @@
libdlclose_noop_la_SOURCES = ../dlclose_noop.c
libdlclose_noop_la_LDFLAGS = -module -rpath $(abs_builddir)
-EXTRA_DIST = $(TESTS) rtest ftest.py tests.ods rtests.csv rwtests.csv wtests.csv
+EXTRA_DIST = $(TESTS) rtest janalyze.py tests.ods rtests.csv rwtests.csv wtests.csv
Deleted: store/trunk/cpp/tests/jrnl/ftest.py
===================================================================
--- store/trunk/cpp/tests/jrnl/ftest.py 2007-09-27 19:29:25 UTC (rev 949)
+++ store/trunk/cpp/tests/jrnl/ftest.py 2007-09-28 16:02:33 UTC (rev 950)
@@ -1,433 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright (C) 2007 Red Hat Inc.
-#
-# This file is part of Red Hat Messaging.
-#
-# Red Hat Messaging is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
-# USA
-#
-# The GNU Lesser General Public License is available in the file COPYING.
-
-import sys
-from struct import unpack, calcsize
-from time import gmtime, strftime
-
-dblk_size = 128
-sblk_size = 4 * dblk_size
-file_size = (3072 + 1) * sblk_size
-num_files = 8
-hdr_ver = 1
-
-def load(f, klass):
- args = load_args(f, klass)
- subclass = klass.descriminate(args)
- result = subclass(*args)
- if subclass != klass:
- result.init(f, *load_args(f, subclass))
- result.skip(f)
- return result;
-
-def load_args(f, klass):
- size = calcsize(klass.format)
- foffs = f.tell(),
- bin = f.read(size)
- if len(bin) != size:
- raise Exception("end of file")
- return foffs + unpack(klass.format, bin)
-
-def size_blks(size, blk_size):
- return (size + blk_size - 1)/blk_size
-
-def rem_in_blk(f, blk_size):
- foffs = f.tell()
- return (size_blks(f.tell(), blk_size) * blk_size) - foffs;
-
-def compl(n):
- return ~n & 0xffffffff
-
-def file_full(f):
- return f.tell() >= file_size
-
-
-class Sizeable:
-
- def size(self):
- classes = [self.__class__]
-
- size = 0
- while classes:
- cls = classes.pop()
- if hasattr(cls, "format"):
- size += calcsize(cls.format)
- classes.extend(cls.__bases__)
-
- return size
-
-class Hdr(Sizeable):
-
- format = '=4sBBHQ'
-
- @staticmethod
- def descriminate(args):
- return CLASSES.get(args[1][-1], Hdr)
-
- def __init__(self, foffs, magic, ver, end, flags, rid):
- self.foffs = foffs
- self.magic = magic
- self.ver = ver
- self.end = end
- self.flags = flags
- self.rid = rid
- if self.magic[-1] not in ['0x00', 'd', 'e', 'f',
'x']:
- error = 3
-
- def __str__(self):
- if self.empty():
- return '0x%08x: <empty>' % (self.foffs)
- if self.magic[-1] == 'x':
- return '0x%08x: \"%s\"' % (self.foffs, self.magic)
- if self.magic[-1] in ['d', 'e', 'f', 'x']:
- return '0x%08x: \"%s\" v=%d e=%d f=0x%04x rid=%d' %
(self.foffs, self.magic, self.ver, self.end, self.flags, self.rid)
- return '0x%08x: <error, unknown magic \"%s\" (possible overwrite
boundary?)>' % (self.foffs, self.magic)
-
- def empty(self):
- return self.magic == '\x00'*4
-
- def skip(self, f):
- f.read(rem_in_blk(f, dblk_size))
-
- def check(self):
- if self.empty() or self.magic[-1] not in ['d', 'e', 'f',
'x']:
- return True
- if self.ver != hdr_ver and self.magic[-1] != 'x':
- raise Exception('%s: Invalid header version: found %d, expected %d.'
% (self, self.ver, hdr_ver))
- return False
-
-
-
-class FileHdr(Hdr):
-
- format = '=I4x3Q'
-
- def init(self, f, foffs, fid, fro, time_sec, time_ns):
- self.fid = fid
- self.fro = fro
- self.time_sec = time_sec
- self.time_ns = time_ns
-
- def __str__(self):
- return '%s fid=%d fro=0x%08x t=%s' % (Hdr.__str__(self), self.fid,
self.fro, self.timestamp_str())
-
- def skip(self, f):
- f.read(rem_in_blk(f, sblk_size))
-
- def timestamp(self):
- return (self.time_sec, self.time_ns)
-
- def timestamp_str(self):
- ts = gmtime(self.time_sec)
- fstr = '%%a %%b %%d %%H:%%M:%%S.%d %%Y' % (self.time_ns)
- return strftime(fstr, ts)
-
-
-class DeqHdr(Hdr):
-
- format = '=QQ'
-
- def init(self, f, foffs, deq_rid, xid):
- self.deq_rid = deq_rid
- self.xid = xid
-
- def __str__(self):
- return '%s d=%d x=%d' % (Hdr.__str__(self), self.deq_rid, self.xid)
-
-
-class RecTail(Sizeable):
-
- format = '=IQ'
-
- def __init__(self, foffs, magic_inv, rid):
- self.foffs = foffs
- self.magic_inv = magic_inv
- self.rid = rid
-
- def __str__(self):
- return '0x%04x rid=%d' % (self.magic_inv, self.rid)
-
- def skip(self, f):
- f.read(rem_in_blk(f, dblk_size))
-
-
-class EnqRec(Hdr):
-
- format = '=QQ'
-
- def init(self, f, foffs, xid, dsize):
- self.xid = xid
- self.dsize = dsize
- self.data = None
- self.enq_tail = None
- self.data_complete = False
- self.tail_complete = False
- self.tail_bin = None
- self.tail_offs = 0
- self.load(f)
-
- def load(self, f):
- if not self.data_complete:
- self.load_data(f)
- if self.data_complete and not self.tail_complete:
- self.load_tail(f)
- return self.complete()
-
- def load_data(self, f):
- if self.data == None:
- loaded = 0
- else:
- loaded = len(self.data)
- foverflow = f.tell() + self.dsize - loaded > file_size
- if foverflow:
- rsize = file_size - f.tell()
- else:
- rsize = self.dsize - loaded
- bin = f.read(rsize)
- if self.data == None:
- self.data = unpack('%ds' % (rsize), bin)[0]
- else:
- self.data = self.data + unpack('%ds' % (rsize), bin)[0]
- self.data_complete = not foverflow
- return self.data_complete
-
- def load_tail(self, f):
- if self.tail_bin == None:
- self.tail_offs = f.tell()
- self.tail_bin = f.read(calcsize(RecTail.format))
- if len(self.tail_bin) != calcsize(RecTail.format):
- return
- else:
- self.tail_bin += f.read(calcsize(RecTail.format) - len(self.tail_bin))
- self.enq_tail = RecTail(self.tail_offs, *unpack(RecTail.format, self.tail_bin))
- magic_int = 0
- inv_magic_int_1 = 0
- for i in range(0,4):
- magic_int = (magic_int << 8) + ord(self.magic[3-i]) # Little endian
only
- if self.enq_tail.magic_inv != compl(magic_int) or self.enq_tail.rid != self.rid:
- print " > %s" % self
- raise Exception('Invalid enqueue record tail (magic=0x%08x; rid=%d) at
0x%08x' % (self.enq_tail.magic_inv, self.enq_tail.rid, self.enq_tail.foffs))
- self.enq_tail.skip(f)
- self.tail_complete = True
-
- def complete(self):
- return self.data_complete and self.tail_complete
-
- def __str__(self):
- if len(self.data) > 50:
- dstr = self.data[:20] + ' ... ' + self.data[-20:]
- else:
- dstr = self.data
- if self.enq_tail == None:
- return '%s s=%d data=\"%s\" [no tail]' %
(Hdr.__str__(self), self.dsize, dstr)
- return '%s x=%d s=%d data=\"%s\" %s' % (Hdr.__str__(self),
self.xid, self.dsize, dstr, self.enq_tail)
-
-
-class Main:
-
- def __init__(self, tfile, tnum):
- tparams = self.get_test(tfile, tnum)
- if tparams == None:
- raise Exception('Test %d not found in file %s' % (tnum, tfile))
- self.num_msgs = tparams['num_msgs']
- if tparams['min_size'] == tparams['max_size']:
- self.msg_len = tparams['max_size']
- else:
- self.msg_len = 0
- self.auto_deq = tparams['auto_deq']
- self.file_start = 0
- self.file_num = 0
- self.fro = 0x200
- self.enqueued = {}
- self.msg_cnt = 0
- self.fhdr = None
- self.f = None
- self.first_rec = False
- self.last_file = False
- self.last_rid = -1
-
-
- def run(self):
- for msg_num in range(self.num_msgs):
- start_info = self.analyze_files()
- stop = self.advance_file(*start_info)
- while not stop:
- warn = ''
- if file_full(self.f):
- stop = self.advance_file()
- if stop:
- break
- hdr = load(self.f, Hdr)
- if hdr.empty():
- stop = True;
- break
- if hdr.check():
- stop = True;
- if self.first_rec:
- if self.fhdr.fro != hdr.foffs:
- raise Exception('File header first record offset mismatch:
fro=0x%08x; rec_offs=0x%08x' % (self.fhdr.fro, hdr.foffs))
- else:
- print ' * fro ok: 0x%08x' % self.fhdr.fro
- self.first_rec = False
- if isinstance(hdr, EnqRec) and not stop:
- while not hdr.complete():
- stop = self.advance_file()
- if stop:
- break
- hdr.load(self.f)
- if self.msg_len > 0 and len(hdr.data) != self.msg_len:
- raise Exception('Message length (%d) incorrect: expected
%d' % (len(hdr.data), self.msg_len))
- stop = not self.check_rid(hdr)
- if stop:
- warn = ' (WARNING: rid out of order, last rid = %d - could be
overwrite boundary.)' % hdr.rid
- else:
- self.msg_cnt += 1
- if self.auto_deq:
- self.enqueued[hdr.rid] = hdr
- elif isinstance(hdr, DeqHdr) and not stop:
- if self.auto_deq:
- if hdr.deq_rid in self.enqueued:
- del self.enqueued[hdr.deq_rid]
- else:
- warn = ' (WARNING: dequeue rid %d not found in enqueued
records)' % hdr.deq_rid
- stop = not self.check_rid(hdr)
- if stop:
- warn = ' (WARNING: rid out of order, last rid = %d -
could be overwrite boundary.)' % hdr.rid
- else:
- warn = 'WARNING: Dequeue record rid=%d found in non-dequeue
test - ignoring.' % hdr.rid
- print ' > %s%s' % (hdr, warn)
- if not stop:
- stop = (self.last_file and hdr.check()) or hdr.empty() or
self.fhdr.empty() or msg_num >= self.num_msgs
- if stop:
- break
-
- def report(self):
- if self.msg_cnt != self.num_msgs:
- print 'WARNING: Found %d messages; %d expected.' % (self.msg_cnt,
self.num_msgs)
- if len(self.enqueued) > 0:
- print 'Remaining enqueued records: ', len(self.enqueued)
- for h in self.enqueued:
- print self.enqueued[h]
- print 'WARNING: Enqueue-Dequeue mismatch, %d enqueued records
remain.' % len(self.enqueued)
- print 'Test passed; %d records processed.' % self.msg_cnt
-
- def advance_file(self, *start_info):
- seek_flag = False
- if len(start_info) == 3:
- self.file_start = self.file_num = start_info[0]
- self.fro = start_info[2]
- seek_flag = True
- if self.f != None and file_full(self.f):
- self.file_num = self.incr_fnum()
- if self.file_num == self.file_start:
- return True
- if self.file_start == 0:
- self.last_file = self.file_num == num_files - 1
- else:
- self.last_file = self.file_num == self.file_start - 1
- if self.file_num < 0 or self.file_num >= num_files:
- raise Exception('Bad file number %d' % self.file_num)
- file_name = 'jdata/test.%04d.jdat' % self.file_num
- self.f = open(file_name)
- self.fhdr = load(self.f, Hdr)
- if seek_flag and self.f.tell() != self.fro:
- self.f.seek(self.fro)
- self.first_rec = True
- print file_name, ": ", self.fhdr
- return False
-
- def incr_fnum(self):
- self.file_num += 1
- if self.file_num >= num_files:
- self.file_num = 0;
- return self.file_num
-
- def check_rid(self, hdr):
- if self.last_rid != -1 and hdr.rid != self.last_rid + 1:
- return False
- self.last_rid = hdr.rid
- return True
-
- def get_test(self, filename, tnum):
- f=open(filename, 'r')
- for l in f:
- sl = l.strip().split(',')
- if len(sl[1]) > 0: #Comments are in col 0, remaining cols are empty
- try:
- if (int(sl[0]) == tnum):
- return { 'num_msgs':int(sl[1]),
- 'min_size':int(sl[2]),
- 'max_size':int(sl[3]),
- 'auto_deq':sl[4] != 'FALSE' }
- except Exception:
- pass
- return None
-
- def analyze_files(self):
- fname = ''
- fnum = -1
- rid = -1
- fro = -1
- tss = ''
- print 'Analyzing journal files:'
- for i in range(0, num_files):
- file_name = 'jdata/test.%04d.jdat' % i
- f = open(file_name)
- fhdr = load(f, Hdr)
- if fhdr.empty():
- break
- if (rid == -1 or fhdr.rid < rid) and fhdr.fro > 0:
- fname = file_name
- fnum = i
- rid = fhdr.rid
- fro = fhdr.fro
- tss = fhdr.timestamp_str()
- print ' %s: rid=%d, fro=0x%08x ts=%s' % (file_name, fhdr.rid,
fhdr.fro, fhdr.timestamp_str())
- if fnum < 0 or rid < 0 or fro < 0:
- raise Exception('All journal files empty')
- print 'Lowest rid found in file %s: rid=%d, fro=0x%08x ts=%s' % (fname,
rid, fro, tss)
- return (fnum, rid, fro)
-
-
-CLASSES = {
- "e": EnqRec,
- "d": DeqHdr,
- "f": FileHdr
-}
-
-if len(sys.argv) != 3:
- print 'Incorrect number of command-line arguments (expected 2, found %d).' %
(len(sys.argv) - 1)
- print 'Usage: ftest test-file test-number'
- raise Exception('Bad command-line arguments')
-if not sys.argv[2].isdigit():
- print 'Illegal argument (\"%s\"): test-number must be a number' %
a
- print 'Usage: ftest test-file test-number'
- raise Exception('Illegal argument value')
-
-tnum = int(sys.argv[2])
-print 'Test number: %d' % tnum
-print 'Test file: %s' %sys.argv[1]
-
-m = Main(sys.argv[1], tnum)
-m.run()
-m.report()
Copied: store/trunk/cpp/tests/jrnl/janalyze.py (from rev 947,
store/trunk/cpp/tests/jrnl/ftest.py)
===================================================================
--- store/trunk/cpp/tests/jrnl/janalyze.py (rev 0)
+++ store/trunk/cpp/tests/jrnl/janalyze.py 2007-09-28 16:02:33 UTC (rev 950)
@@ -0,0 +1,540 @@
+#!/usr/bin/env python
+
+# Copyright (C) 2007 Red Hat Inc.
+#
+# This file is part of Red Hat Messaging.
+#
+# Red Hat Messaging is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
+# USA
+#
+# The GNU Lesser General Public License is available in the file COPYING.
+
+import sys
+from struct import unpack, calcsize
+from time import gmtime, strftime
+
+dblk_size = 128
+sblk_size = 4 * dblk_size
+file_size = (3072 + 1) * sblk_size
+num_files = 8
+hdr_ver = 1
+
+def load(f, klass):
+ args = load_args(f, klass)
+ subclass = klass.descriminate(args)
+ result = subclass(*args)
+ if subclass != klass:
+ result.init(f, *load_args(f, subclass))
+ result.skip(f)
+ return result;
+
+def load_args(f, klass):
+ size = calcsize(klass.format)
+ foffs = f.tell(),
+ bin = f.read(size)
+ if len(bin) != size:
+ raise Exception("end of file")
+ return foffs + unpack(klass.format, bin)
+
+def size_blks(size, blk_size):
+ return (size + blk_size - 1)/blk_size
+
+def rem_in_blk(f, blk_size):
+ foffs = f.tell()
+ return (size_blks(f.tell(), blk_size) * blk_size) - foffs;
+
+def file_full(f):
+ return f.tell() >= file_size
+
+def print_xid(xidsize, xid):
+ if xid == None:
+ if xidsize > 0:
+ raise Exception('Inconsistent XID size: xidsize=%d, xid=None' %
xidsize)
+ return ''
+ if len(xid) > 25:
+ xidstr = xid[:10] + ' ... ' + xid[-10:]
+ else:
+ xidstr = xid
+ if xidsize != len(xid):
+ raise Exception('Inconsistent XID size: xidsize=%d,
xid(%d)=\"%s\"' % (xidsize, len(xid), xidstr))
+ return 'xid(%d)=\"%s\" ' % (xidsize, xidstr)
+
+def print_data(dsize, data):
+ if len(data) > 25:
+ datastr = data[:10] + ' ... ' + data[-10:]
+ else:
+ datastr = data
+ if dsize != len(data):
+ raise Exception('Inconsistent data size: dsize=%d,
data(%d)=\"%s\"' % (dsize, len(data), datastr))
+ return 'data(%d)=\"%s\" ' % (dsize, datastr)
+
+def inv_str(s):
+ si = ''
+ for i in range(0,len(s)):
+ si += chr(~ord(s[i]) & 0xff)
+ return si
+
+def load_file_data(f, size, data):
+ if size == 0:
+ return (data, True)
+ if data == None:
+ loaded = 0
+ else:
+ loaded = len(data)
+ foverflow = f.tell() + size - loaded > file_size
+ if foverflow:
+ rsize = file_size - f.tell()
+ else:
+ rsize = size - loaded
+ bin = f.read(rsize)
+ if data == None:
+ data = unpack('%ds' % (rsize), bin)[0]
+ else:
+ data = data + unpack('%ds' % (rsize), bin)[0]
+ return (data, not foverflow)
+
+def load_tail(self, f):
+ if self.tail_bin == None:
+ self.tail_offs = f.tell()
+ self.tail_bin = f.read(calcsize(RecTail.format))
+ if len(self.tail_bin) != calcsize(RecTail.format):
+ return
+ else:
+ self.tail_bin += f.read(calcsize(RecTail.format) - len(self.tail_bin))
+ self.enq_tail = RecTail(self.tail_offs, *unpack(RecTail.format, self.tail_bin))
+ if self.enq_tail.magic_inv != inv_str(self.magic) or self.enq_tail.rid != self.rid:
+ print " > %s" % self
+ raise Exception('Invalid enqueue record tail (magic=0x%08x; rid=%d) at
0x%08x' % (self.enq_tail.magic_inv, self.enq_tail.rid, self.enq_tail.foffs))
+ self.enq_tail.skip(f)
+ self.tail_complete = True
+
+
+class Sizeable:
+
+ def size(self):
+ classes = [self.__class__]
+
+ size = 0
+ while classes:
+ cls = classes.pop()
+ if hasattr(cls, "format"):
+ size += calcsize(cls.format)
+ classes.extend(cls.__bases__)
+
+ return size
+
+class Hdr(Sizeable):
+
+ format = '=4sBBHQ'
+
+ @staticmethod
+ def descriminate(args):
+ return CLASSES.get(args[1][-1], Hdr)
+
+ def __init__(self, foffs, magic, ver, end, flags, rid):
+ self.foffs = foffs
+ self.magic = magic
+ self.ver = ver
+ self.end = end
+ self.flags = flags
+ self.rid = rid
+ if self.magic[-1] not in ['0x00', 'a', 'c', 'd',
'e', 'f', 'x']:
+ error = 3
+
+ def __str__(self):
+ if self.empty():
+ return '0x%08x: <empty>' % (self.foffs)
+ if self.magic[-1] == 'x':
+ return '0x%08x: \"%s\"' % (self.foffs, self.magic)
+ if self.magic[-1] in ['a', 'c', 'd', 'e',
'f', 'x']:
+ return '0x%08x: \"%s\" v=%d e=%d f=0x%04x rid=%d' %
(self.foffs, self.magic, self.ver, self.end, self.flags, self.rid)
+ return '0x%08x: <error, unknown magic \"%s\" (possible overwrite
boundary?)>' % (self.foffs, self.magic)
+
+ def empty(self):
+ return self.magic == '\x00'*4
+
+ def skip(self, f):
+ f.read(rem_in_blk(f, dblk_size))
+
+ def check(self):
+ if self.empty() or self.magic[-1] not in ['a', 'c', 'd',
'e', 'f', 'x']:
+ return True
+ if self.ver != hdr_ver and self.magic[-1] != 'x':
+ raise Exception('%s: Invalid header version: found %d, expected %d.'
% (self, self.ver, hdr_ver))
+ return False
+
+
+
+class FileHdr(Hdr):
+
+ format = '=I4x3Q'
+
+ def init(self, f, foffs, fid, fro, time_sec, time_ns):
+ self.fid = fid
+ self.fro = fro
+ self.time_sec = time_sec
+ self.time_ns = time_ns
+
+ def __str__(self):
+ return '%s fid=%d fro=0x%08x t=%s' % (Hdr.__str__(self), self.fid,
self.fro, self.timestamp_str())
+
+ def skip(self, f):
+ f.read(rem_in_blk(f, sblk_size))
+
+ def timestamp(self):
+ return (self.time_sec, self.time_ns)
+
+ def timestamp_str(self):
+ ts = gmtime(self.time_sec)
+ fstr = '%%a %%b %%d %%H:%%M:%%S.%d %%Y' % (self.time_ns)
+ return strftime(fstr, ts)
+
+
+class DeqHdr(Hdr):
+
+ format = '=QQ'
+
+ def init(self, f, foffs, deq_rid, xidsize):
+ self.deq_rid = deq_rid
+ self.xidsize = xidsize
+ self.xid = None
+ self.deq_tail = None
+ self.xid_complete = False
+ self.tail_complete = False
+ self.load(f)
+
+ def load(self, f):
+ if self.xidsize == 0:
+ self.xid_complete = True
+ self.tail_complete = True
+ else:
+ if not self.xid_complete:
+ ret = load_file_data(f, self.xidsize, self.xid)
+ self.xid = ret[0]
+ self.xid_complete = ret[1]
+ if self.xid_complete and not self.tail_complete:
+ ret = load_file_data(f, calcsize(RecTail.format), self.tail_bin)
+ self.tail_bin = ret[0]
+ if ret[1]:
+ self.enq_tail = RecTail(self.tail_offs, *unpack(RecTail.format,
self.tail_bin))
+ if self.enq_tail.magic_inv != inv_str(self.magic) or
self.enq_tail.rid != self.rid:
+ print " > %s" % self
+ raise Exception('Invalid enqueue record tail (magic=0x%08x;
rid=%d) at 0x%08x' % (self.enq_tail.magic_inv, self.enq_tail.rid,
self.enq_tail.foffs))
+ self.enq_tail.skip(f)
+ self.tail_complete = ret[1]
+ return self.complete()
+
+ def complete(self):
+ return self.xid_complete and self.tail_complete
+
+ def __str__(self):
+ return '%s %sdrid=%d' % (Hdr.__str__(self), print_xid(self.xidsize,
self.xid), self.deq_rid)
+
+
+class TxHdr(Hdr):
+
+ format = '=Q'
+
+ def init(self, f, foffs, xidsize):
+ self.xidsize = xid
+ self.xid = None
+ self.tx_tail = None
+ self.xid_complete = False
+ self.tail_complete = False
+ self.load(f)
+
+ def load(self, f):
+ if not self.xid_complete:
+ ret = load_file_data(f, self.xidsize, self.xid)
+ self.xid = ret[0]
+ self.xid_complete = ret[1]
+ if self.xid_complete and not self.tail_complete:
+ ret = load_file_data(f, calcsize(RecTail.format), self.tail_bin)
+ self.tail_bin = ret[0]
+ if ret[1]:
+ self.enq_tail = RecTail(self.tail_offs, *unpack(RecTail.format,
self.tail_bin))
+ if self.enq_tail.magic_inv != inv_str(self.magic) or self.enq_tail.rid !=
self.rid:
+ print " > %s" % self
+ raise Exception('Invalid enqueue record tail (magic=0x%08x;
rid=%d) at 0x%08x' % (self.enq_tail.magic_inv, self.enq_tail.rid,
self.enq_tail.foffs))
+ self.enq_tail.skip(f)
+ self.tail_complete = ret[1]
+ return self.complete()
+
+ def complete(self):
+ return self.xid_complete and self.tail_complete
+
+ def __str__(self):
+ return '%s x=%d xid=\"%s\"' % (Hdr.__str__(self), self.xidsize,
self.xid)
+
+
+class RecTail(Sizeable):
+
+ format = '=4sQ'
+
+ def __init__(self, foffs, magic_inv, rid):
+ self.foffs = foffs
+ self.magic_inv = magic_inv
+ self.rid = rid
+
+ def __str__(self):
+ magic = inv_str(self.magic_inv)
+ return '[\"%s\" rid=%d]' % (magic, self.rid)
+
+ def skip(self, f):
+ f.read(rem_in_blk(f, dblk_size))
+
+
+class EnqRec(Hdr):
+
+ format = '=QQ'
+
+ def init(self, f, foffs, xidsize, dsize):
+ self.xidsize = xidsize
+ self.dsize = dsize
+ self.xid = None
+ self.data = None
+ self.enq_tail = None
+ self.xid_complete = False
+ self.data_complete = False
+ self.tail_complete = False
+ self.tail_bin = None
+ self.tail_offs = 0
+ self.load(f)
+
+ def load(self, f):
+ if not self.xid_complete:
+ ret = load_file_data(f, self.xidsize, self.xid)
+ self.xid = ret[0]
+ self.xid_complete = ret[1]
+ if self.xid_complete and not self.data_complete:
+ ret = load_file_data(f, self.dsize, self.data)
+ self.data = ret[0]
+ self.data_complete = ret[1]
+ if self.data_complete and not self.tail_complete:
+ ret = load_file_data(f, calcsize(RecTail.format), self.tail_bin)
+ self.tail_bin = ret[0]
+ if ret[1]:
+ self.enq_tail = RecTail(self.tail_offs, *unpack(RecTail.format,
self.tail_bin))
+ if self.enq_tail.magic_inv != inv_str(self.magic) or self.enq_tail.rid !=
self.rid:
+ print " > %s" % self
+ raise Exception('Invalid enqueue record tail (magic=0x%08x;
rid=%d) at 0x%08x' % (self.enq_tail.magic_inv, self.enq_tail.rid,
self.enq_tail.foffs))
+ self.enq_tail.skip(f)
+ self.tail_complete = ret[1]
+ return self.complete()
+
+ def complete(self):
+ return self.xid_complete and self.data_complete and self.tail_complete
+
+ def __str__(self):
+ if self.enq_tail == None:
+ return '%s %s [no tail]' % (Hdr.__str__(self), self.dsize, dstr)
+ return '%s %s%s %s' % (Hdr.__str__(self), print_xid(self.xidsize,
self.xid), print_data(self.dsize, self.data), self.enq_tail)
+
+
+class Main:
+
+ def __init__(self, tfile, tnum):
+ if tfile != None:
+ tparams = self.get_test(tfile, tnum)
+ if tparams == None:
+ raise Exception('Test %d not found in file %s' % (tnum, tfile))
+ self.num_msgs = tparams['num_msgs']
+ if tparams['min_size'] == tparams['max_size']:
+ self.msg_len = tparams['max_size']
+ else:
+ self.msg_len = 0
+ self.auto_deq = tparams['auto_deq']
+ else:
+ self.num_msgs = 0
+ self.msg_len = 0
+ self.auto_deq = False
+ self.file_start = 0
+ self.file_num = 0
+ self.fro = 0x200
+ self.enqueued = {}
+ self.msg_cnt = 0
+ self.fhdr = None
+ self.f = None
+ self.first_rec = False
+ self.last_file = False
+ self.last_rid = -1
+
+
+ def run(self):
+ start_info = self.analyze_files()
+ stop = self.advance_file(*start_info)
+ while not stop:
+ warn = ''
+ if file_full(self.f):
+ stop = self.advance_file()
+ if stop:
+ break
+ hdr = load(self.f, Hdr)
+ if hdr.empty():
+ stop = True;
+ break
+ if hdr.check():
+ stop = True;
+ if self.first_rec:
+ if self.fhdr.fro != hdr.foffs:
+ raise Exception('File header first record offset mismatch:
fro=0x%08x; rec_offs=0x%08x' % (self.fhdr.fro, hdr.foffs))
+ else:
+ print ' * fro ok: 0x%08x' % self.fhdr.fro
+ self.first_rec = False
+ if isinstance(hdr, EnqRec) and not stop:
+ while not hdr.complete():
+ stop = self.advance_file()
+ if stop:
+ break
+ hdr.load(self.f)
+ if self.msg_len > 0 and len(hdr.data) != self.msg_len:
+ raise Exception('Message length (%d) incorrect: expected %d'
% (len(hdr.data), self.msg_len))
+ stop = not self.check_rid(hdr)
+ if stop:
+ warn = ' (WARNING: rid out of order, last rid = %d - could be
overwrite boundary.)' % hdr.rid
+ else:
+ self.msg_cnt += 1
+ if self.auto_deq:
+ self.enqueued[hdr.rid] = hdr
+ elif isinstance(hdr, DeqHdr) and not stop:
+ if self.auto_deq:
+ if hdr.deq_rid in self.enqueued:
+ del self.enqueued[hdr.deq_rid]
+ else:
+ warn = ' (WARNING: dequeue rid %d not found in enqueued
records)' % hdr.deq_rid
+ stop = not self.check_rid(hdr)
+ if stop:
+ warn = ' (WARNING: rid out of order, last rid = %d - could be
overwrite boundary.)' % hdr.rid
+ else:
+ warn = 'WARNING: Dequeue record rid=%d found in non-dequeue test
- ignoring.' % hdr.rid
+ print ' > %s%s' % (hdr, warn)
+ if not stop:
+ stop = (self.last_file and hdr.check()) or hdr.empty() or
self.fhdr.empty()
+
+ def report(self):
+ if self.num_msgs > 0 and self.msg_cnt != self.num_msgs:
+ print 'WARNING: Found %d messages; %d expected.' % (self.msg_cnt,
self.num_msgs)
+ if len(self.enqueued) > 0:
+ print 'Remaining enqueued records: ', len(self.enqueued)
+ for h in self.enqueued:
+ print self.enqueued[h]
+ print 'WARNING: Enqueue-Dequeue mismatch, %d enqueued records
remain.' % len(self.enqueued)
+ print 'Test passed; %d records processed.' % self.msg_cnt
+
+ def advance_file(self, *start_info):
+ seek_flag = False
+ if len(start_info) == 3:
+ self.file_start = self.file_num = start_info[0]
+ self.fro = start_info[2]
+ seek_flag = True
+ if self.f != None and file_full(self.f):
+ self.file_num = self.incr_fnum()
+ if self.file_num == self.file_start:
+ return True
+ if self.file_start == 0:
+ self.last_file = self.file_num == num_files - 1
+ else:
+ self.last_file = self.file_num == self.file_start - 1
+ if self.file_num < 0 or self.file_num >= num_files:
+ raise Exception('Bad file number %d' % self.file_num)
+ file_name = 'jdata/test.%04d.jdat' % self.file_num
+ self.f = open(file_name)
+ self.fhdr = load(self.f, Hdr)
+ if seek_flag and self.f.tell() != self.fro:
+ self.f.seek(self.fro)
+ self.first_rec = True
+ print file_name, ": ", self.fhdr
+ return False
+
+ def incr_fnum(self):
+ self.file_num += 1
+ if self.file_num >= num_files:
+ self.file_num = 0;
+ return self.file_num
+
+ def check_rid(self, hdr):
+ if self.last_rid != -1 and hdr.rid != self.last_rid + 1:
+ return False
+ self.last_rid = hdr.rid
+ return True
+
+ def get_test(self, filename, tnum):
+ f=open(filename, 'r')
+ for l in f:
+ sl = l.strip().split(',')
+ if len(sl[1]) > 0: #Comments are in col 0, remaining cols are empty
+ try:
+ if (int(sl[0]) == tnum):
+ return { 'num_msgs':int(sl[1]),
+ 'min_size':int(sl[2]),
+ 'max_size':int(sl[3]),
+ 'auto_deq':sl[4] != 'FALSE' }
+ except Exception:
+ pass
+ return None
+
+ def analyze_files(self):
+ fname = ''
+ fnum = -1
+ rid = -1
+ fro = -1
+ tss = ''
+ print 'Analyzing journal files:'
+ for i in range(0, num_files):
+ file_name = 'jdata/test.%04d.jdat' % i
+ f = open(file_name)
+ fhdr = load(f, Hdr)
+ if fhdr.empty():
+ break
+ if (rid == -1 or fhdr.rid < rid) and fhdr.fro > 0:
+ fname = file_name
+ fnum = i
+ rid = fhdr.rid
+ fro = fhdr.fro
+ tss = fhdr.timestamp_str()
+ print ' %s: rid=%d, fro=0x%08x ts=%s' % (file_name, fhdr.rid,
fhdr.fro, fhdr.timestamp_str())
+ if fnum < 0 or rid < 0 or fro < 0:
+ raise Exception('All journal files empty')
+ print ' Lowest rid found in file %s: rid=%d, fro=0x%08x ts=%s' % (fname,
rid, fro, tss)
+ return (fnum, rid, fro)
+
+
+CLASSES = {
+ "a": TxHdr,
+ "c": TxHdr,
+ "d": DeqHdr,
+ "e": EnqRec,
+ "f": FileHdr
+}
+
+if len(sys.argv) != 1 and len(sys.argv) != 3:
+ print 'Incorrect number of command-line arguments (expected 0 or 2, found
%d).' % (len(sys.argv) - 1)
+ print 'Usage: ftest test-file test-number'
+ raise Exception('Bad command-line arguments')
+if len(sys.argv) == 3 and not sys.argv[2].isdigit():
+ print 'Illegal argument (\"%s\"): test-number must be a number' %
a
+ print 'Usage: ftest test-file test-number'
+ raise Exception('Illegal argument value')
+
+if len(sys.argv) != 0:
+ m = Main(None, None)
+ m.run()
+else:
+ tnum = int(sys.argv[2])
+ print 'Test number: %d' % tnum
+ print 'Test file: %s' %sys.argv[1]
+ m = Main(sys.argv[1], tnum)
+ m.run()
+ m.report()
Modified: store/trunk/cpp/tests/jrnl/rtest
===================================================================
--- store/trunk/cpp/tests/jrnl/rtest 2007-09-27 19:29:25 UTC (rev 949)
+++ store/trunk/cpp/tests/jrnl/rtest 2007-09-28 16:02:33 UTC (rev 950)
@@ -55,7 +55,7 @@
RM=rm
RM_DIR="${RM} -rf"
TEST_PROG="./jtest"
-CHK_PROG="./ftest.py"
+CHK_PROG="./janalyze.py"
VALGRIND="valgrind -q --track-fds=yes --leak-check=full --leak-resolution=high
--show-reachable=yes"
#VALGRIND="valgrind -q --track-fds=yes --leak-check=full --leak-resolution=high
--show-reachable=yes --suppressions=/usr/lib/valgrind/glibc-2.5.supp"
MAKE="make -f Makefile.rtest"
Modified: store/trunk/cpp/tests/jrnl/tests.ods
===================================================================
(Binary files differ)
Modified: store/trunk/cpp/tests/jrnl/wtests.csv
===================================================================
--- store/trunk/cpp/tests/jrnl/wtests.csv 2007-09-27 19:29:25 UTC (rev 949)
+++ store/trunk/cpp/tests/jrnl/wtests.csv 2007-09-28 16:02:33 UTC (rev 950)
@@ -102,6 +102,6 @@
80,30000,0,38356,TRUE,,"300 dblks max"
81,10000,0,127956,TRUE,,"1000 dblks max"
,,,,,,
-"STANDARD PERFORMANCE BENCHMARK: 10,000,000 writes, data=220b (2 dblks)",,,,,,
-82,10000000,220,220,FALSE,,"2 dblks"
-83,10000000,220,220,TRUE,,"2 dblks"
+"STANDARD PERFORMANCE BENCHMARK: 10,000,000 writes, data=212b (2 dblks)",,,,,,
+82,10000000,212,212,FALSE,,"2 dblks"
+83,10000000,212,212,TRUE,,"2 dblks"