[rhmessaging-commits] rhmessaging commits: r950 - in store/trunk/cpp: tests/jrnl and 1 other directory.

Author: kpvdr Date: 2007-09-28 12:02:33 -0400 (Fri, 28 Sep 2007) New Revision: 950 Added: store/trunk/cpp/tests/jrnl/janalyze.py Removed: store/trunk/cpp/tests/jrnl/ftest.py Modified: store/trunk/cpp/lib/jrnl/jcntl.hpp store/trunk/cpp/tests/jrnl/Makefile.am store/trunk/cpp/tests/jrnl/rtest store/trunk/cpp/tests/jrnl/tests.ods store/trunk/cpp/tests/jrnl/wtests.csv Log: Updated python analysis tool to handle XID Modified: store/trunk/cpp/lib/jrnl/jcntl.hpp =================================================================== --- store/trunk/cpp/lib/jrnl/jcntl.hpp 2007-09-27 19:29:25 UTC (rev 949) +++ store/trunk/cpp/lib/jrnl/jcntl.hpp 2007-09-28 16:02:33 UTC (rev 950) @@ -295,8 +295,6 @@ * \param tot_data_len Total data length. * \param this_data_len Amount to be written in this enqueue operation. * \param dtokp Pointer to data token which contains the details of the enqueue operation. - * \param xid String containing xid. An empty string (i.e. length=0) will be considered - * non-transactional. * \param transient Flag indicating transient persistence (ie, ignored on recover). * * \exception TODO @@ -304,6 +302,19 @@ const iores enqueue_data_record(const void* const data_buff, const size_t tot_data_len, const size_t this_data_len, data_tok* dtokp, const bool transient = false) throw (jexception); + /** + * \brief Enqueue data. + * + * \param data_buff Pointer to data to be enqueued for this enqueue operation. + * \param tot_data_len Total data length. + * \param this_data_len Amount to be written in this enqueue operation. + * \param dtokp Pointer to data token which contains the details of the enqueue operation. + * \param xid String containing xid. An empty string (i.e. length=0) will be considered + * non-transactional. + * \param transient Flag indicating transient persistence (ie, ignored on recover). + * + * \exception TODO + */ const iores enqueue_data_record(const void* const data_buff, const size_t tot_data_len, const size_t this_data_len, data_tok* dtokp, const std::string& xid, const bool transient = false) throw (jexception); @@ -403,12 +414,27 @@ * * \param dtokp Pointer to data_tok instance for this data, used to track state of data * through journal. + * + * \exception TODO + */ + const iores dequeue_data_record(data_tok* const dtokp) throw (jexception); + + /** + * \brief Dequeues (marks as no longer needed) data record in journal. + * + * Dequeues (marks as no longer needed) data record in journal as part of a transaction. + * Note that it is possible to use the same data token instance used to enqueue this data; + * it contains the RID needed to correctly mark this data as dequeued in the journal. + * Otherwise the RID of the record to be dequeued and the write state of ENQ must be + * manually set in a new or reset instance of data_tok. + * + * \param dtokp Pointer to data_tok instance for this data, used to track state of data + * through journal. * \param xid String containing xid. An empty string (i.e. length=0) will be considered * non-transactional. * * \exception TODO */ - const iores dequeue_data_record(data_tok* const dtokp) throw (jexception); const iores dequeue_data_record(data_tok* const dtokp, const std::string& xid) throw (jexception); Modified: store/trunk/cpp/tests/jrnl/Makefile.am =================================================================== --- store/trunk/cpp/tests/jrnl/Makefile.am 2007-09-27 19:29:25 UTC (rev 949) +++ store/trunk/cpp/tests/jrnl/Makefile.am 2007-09-28 16:02:33 UTC (rev 950) @@ -50,4 +50,4 @@ libdlclose_noop_la_SOURCES = ../dlclose_noop.c libdlclose_noop_la_LDFLAGS = -module -rpath $(abs_builddir) -EXTRA_DIST = $(TESTS) rtest ftest.py tests.ods rtests.csv rwtests.csv wtests.csv +EXTRA_DIST = $(TESTS) rtest janalyze.py tests.ods rtests.csv rwtests.csv wtests.csv Deleted: store/trunk/cpp/tests/jrnl/ftest.py =================================================================== --- store/trunk/cpp/tests/jrnl/ftest.py 2007-09-27 19:29:25 UTC (rev 949) +++ store/trunk/cpp/tests/jrnl/ftest.py 2007-09-28 16:02:33 UTC (rev 950) @@ -1,433 +0,0 @@ -#!/usr/bin/env python - -# Copyright (C) 2007 Red Hat Inc. -# -# This file is part of Red Hat Messaging. -# -# Red Hat Messaging is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 -# USA -# -# The GNU Lesser General Public License is available in the file COPYING. - -import sys -from struct import unpack, calcsize -from time import gmtime, strftime - -dblk_size = 128 -sblk_size = 4 * dblk_size -file_size = (3072 + 1) * sblk_size -num_files = 8 -hdr_ver = 1 - -def load(f, klass): - args = load_args(f, klass) - subclass = klass.descriminate(args) - result = subclass(*args) - if subclass != klass: - result.init(f, *load_args(f, subclass)) - result.skip(f) - return result; - -def load_args(f, klass): - size = calcsize(klass.format) - foffs = f.tell(), - bin = f.read(size) - if len(bin) != size: - raise Exception("end of file") - return foffs + unpack(klass.format, bin) - -def size_blks(size, blk_size): - return (size + blk_size - 1)/blk_size - -def rem_in_blk(f, blk_size): - foffs = f.tell() - return (size_blks(f.tell(), blk_size) * blk_size) - foffs; - -def compl(n): - return ~n & 0xffffffff - -def file_full(f): - return f.tell() >= file_size - - -class Sizeable: - - def size(self): - classes = [self.__class__] - - size = 0 - while classes: - cls = classes.pop() - if hasattr(cls, "format"): - size += calcsize(cls.format) - classes.extend(cls.__bases__) - - return size - -class Hdr(Sizeable): - - format = '=4sBBHQ' - - @staticmethod - def descriminate(args): - return CLASSES.get(args[1][-1], Hdr) - - def __init__(self, foffs, magic, ver, end, flags, rid): - self.foffs = foffs - self.magic = magic - self.ver = ver - self.end = end - self.flags = flags - self.rid = rid - if self.magic[-1] not in ['0x00', 'd', 'e', 'f', 'x']: - error = 3 - - def __str__(self): - if self.empty(): - return '0x%08x: <empty>' % (self.foffs) - if self.magic[-1] == 'x': - return '0x%08x: \"%s\"' % (self.foffs, self.magic) - if self.magic[-1] in ['d', 'e', 'f', 'x']: - return '0x%08x: \"%s\" v=%d e=%d f=0x%04x rid=%d' % (self.foffs, self.magic, self.ver, self.end, self.flags, self.rid) - return '0x%08x: <error, unknown magic \"%s\" (possible overwrite boundary?)>' % (self.foffs, self.magic) - - def empty(self): - return self.magic == '\x00'*4 - - def skip(self, f): - f.read(rem_in_blk(f, dblk_size)) - - def check(self): - if self.empty() or self.magic[-1] not in ['d', 'e', 'f', 'x']: - return True - if self.ver != hdr_ver and self.magic[-1] != 'x': - raise Exception('%s: Invalid header version: found %d, expected %d.' % (self, self.ver, hdr_ver)) - return False - - - -class FileHdr(Hdr): - - format = '=I4x3Q' - - def init(self, f, foffs, fid, fro, time_sec, time_ns): - self.fid = fid - self.fro = fro - self.time_sec = time_sec - self.time_ns = time_ns - - def __str__(self): - return '%s fid=%d fro=0x%08x t=%s' % (Hdr.__str__(self), self.fid, self.fro, self.timestamp_str()) - - def skip(self, f): - f.read(rem_in_blk(f, sblk_size)) - - def timestamp(self): - return (self.time_sec, self.time_ns) - - def timestamp_str(self): - ts = gmtime(self.time_sec) - fstr = '%%a %%b %%d %%H:%%M:%%S.%d %%Y' % (self.time_ns) - return strftime(fstr, ts) - - -class DeqHdr(Hdr): - - format = '=QQ' - - def init(self, f, foffs, deq_rid, xid): - self.deq_rid = deq_rid - self.xid = xid - - def __str__(self): - return '%s d=%d x=%d' % (Hdr.__str__(self), self.deq_rid, self.xid) - - -class RecTail(Sizeable): - - format = '=IQ' - - def __init__(self, foffs, magic_inv, rid): - self.foffs = foffs - self.magic_inv = magic_inv - self.rid = rid - - def __str__(self): - return '0x%04x rid=%d' % (self.magic_inv, self.rid) - - def skip(self, f): - f.read(rem_in_blk(f, dblk_size)) - - -class EnqRec(Hdr): - - format = '=QQ' - - def init(self, f, foffs, xid, dsize): - self.xid = xid - self.dsize = dsize - self.data = None - self.enq_tail = None - self.data_complete = False - self.tail_complete = False - self.tail_bin = None - self.tail_offs = 0 - self.load(f) - - def load(self, f): - if not self.data_complete: - self.load_data(f) - if self.data_complete and not self.tail_complete: - self.load_tail(f) - return self.complete() - - def load_data(self, f): - if self.data == None: - loaded = 0 - else: - loaded = len(self.data) - foverflow = f.tell() + self.dsize - loaded > file_size - if foverflow: - rsize = file_size - f.tell() - else: - rsize = self.dsize - loaded - bin = f.read(rsize) - if self.data == None: - self.data = unpack('%ds' % (rsize), bin)[0] - else: - self.data = self.data + unpack('%ds' % (rsize), bin)[0] - self.data_complete = not foverflow - return self.data_complete - - def load_tail(self, f): - if self.tail_bin == None: - self.tail_offs = f.tell() - self.tail_bin = f.read(calcsize(RecTail.format)) - if len(self.tail_bin) != calcsize(RecTail.format): - return - else: - self.tail_bin += f.read(calcsize(RecTail.format) - len(self.tail_bin)) - self.enq_tail = RecTail(self.tail_offs, *unpack(RecTail.format, self.tail_bin)) - magic_int = 0 - inv_magic_int_1 = 0 - for i in range(0,4): - magic_int = (magic_int << 8) + ord(self.magic[3-i]) # Little endian only - if self.enq_tail.magic_inv != compl(magic_int) or self.enq_tail.rid != self.rid: - print " > %s" % self - raise Exception('Invalid enqueue record tail (magic=0x%08x; rid=%d) at 0x%08x' % (self.enq_tail.magic_inv, self.enq_tail.rid, self.enq_tail.foffs)) - self.enq_tail.skip(f) - self.tail_complete = True - - def complete(self): - return self.data_complete and self.tail_complete - - def __str__(self): - if len(self.data) > 50: - dstr = self.data[:20] + ' ... ' + self.data[-20:] - else: - dstr = self.data - if self.enq_tail == None: - return '%s s=%d data=\"%s\" [no tail]' % (Hdr.__str__(self), self.dsize, dstr) - return '%s x=%d s=%d data=\"%s\" %s' % (Hdr.__str__(self), self.xid, self.dsize, dstr, self.enq_tail) - - -class Main: - - def __init__(self, tfile, tnum): - tparams = self.get_test(tfile, tnum) - if tparams == None: - raise Exception('Test %d not found in file %s' % (tnum, tfile)) - self.num_msgs = tparams['num_msgs'] - if tparams['min_size'] == tparams['max_size']: - self.msg_len = tparams['max_size'] - else: - self.msg_len = 0 - self.auto_deq = tparams['auto_deq'] - self.file_start = 0 - self.file_num = 0 - self.fro = 0x200 - self.enqueued = {} - self.msg_cnt = 0 - self.fhdr = None - self.f = None - self.first_rec = False - self.last_file = False - self.last_rid = -1 - - - def run(self): - for msg_num in range(self.num_msgs): - start_info = self.analyze_files() - stop = self.advance_file(*start_info) - while not stop: - warn = '' - if file_full(self.f): - stop = self.advance_file() - if stop: - break - hdr = load(self.f, Hdr) - if hdr.empty(): - stop = True; - break - if hdr.check(): - stop = True; - if self.first_rec: - if self.fhdr.fro != hdr.foffs: - raise Exception('File header first record offset mismatch: fro=0x%08x; rec_offs=0x%08x' % (self.fhdr.fro, hdr.foffs)) - else: - print ' * fro ok: 0x%08x' % self.fhdr.fro - self.first_rec = False - if isinstance(hdr, EnqRec) and not stop: - while not hdr.complete(): - stop = self.advance_file() - if stop: - break - hdr.load(self.f) - if self.msg_len > 0 and len(hdr.data) != self.msg_len: - raise Exception('Message length (%d) incorrect: expected %d' % (len(hdr.data), self.msg_len)) - stop = not self.check_rid(hdr) - if stop: - warn = ' (WARNING: rid out of order, last rid = %d - could be overwrite boundary.)' % hdr.rid - else: - self.msg_cnt += 1 - if self.auto_deq: - self.enqueued[hdr.rid] = hdr - elif isinstance(hdr, DeqHdr) and not stop: - if self.auto_deq: - if hdr.deq_rid in self.enqueued: - del self.enqueued[hdr.deq_rid] - else: - warn = ' (WARNING: dequeue rid %d not found in enqueued records)' % hdr.deq_rid - stop = not self.check_rid(hdr) - if stop: - warn = ' (WARNING: rid out of order, last rid = %d - could be overwrite boundary.)' % hdr.rid - else: - warn = 'WARNING: Dequeue record rid=%d found in non-dequeue test - ignoring.' % hdr.rid - print ' > %s%s' % (hdr, warn) - if not stop: - stop = (self.last_file and hdr.check()) or hdr.empty() or self.fhdr.empty() or msg_num >= self.num_msgs - if stop: - break - - def report(self): - if self.msg_cnt != self.num_msgs: - print 'WARNING: Found %d messages; %d expected.' % (self.msg_cnt, self.num_msgs) - if len(self.enqueued) > 0: - print 'Remaining enqueued records: ', len(self.enqueued) - for h in self.enqueued: - print self.enqueued[h] - print 'WARNING: Enqueue-Dequeue mismatch, %d enqueued records remain.' % len(self.enqueued) - print 'Test passed; %d records processed.' % self.msg_cnt - - def advance_file(self, *start_info): - seek_flag = False - if len(start_info) == 3: - self.file_start = self.file_num = start_info[0] - self.fro = start_info[2] - seek_flag = True - if self.f != None and file_full(self.f): - self.file_num = self.incr_fnum() - if self.file_num == self.file_start: - return True - if self.file_start == 0: - self.last_file = self.file_num == num_files - 1 - else: - self.last_file = self.file_num == self.file_start - 1 - if self.file_num < 0 or self.file_num >= num_files: - raise Exception('Bad file number %d' % self.file_num) - file_name = 'jdata/test.%04d.jdat' % self.file_num - self.f = open(file_name) - self.fhdr = load(self.f, Hdr) - if seek_flag and self.f.tell() != self.fro: - self.f.seek(self.fro) - self.first_rec = True - print file_name, ": ", self.fhdr - return False - - def incr_fnum(self): - self.file_num += 1 - if self.file_num >= num_files: - self.file_num = 0; - return self.file_num - - def check_rid(self, hdr): - if self.last_rid != -1 and hdr.rid != self.last_rid + 1: - return False - self.last_rid = hdr.rid - return True - - def get_test(self, filename, tnum): - f=open(filename, 'r') - for l in f: - sl = l.strip().split(',') - if len(sl[1]) > 0: #Comments are in col 0, remaining cols are empty - try: - if (int(sl[0]) == tnum): - return { 'num_msgs':int(sl[1]), - 'min_size':int(sl[2]), - 'max_size':int(sl[3]), - 'auto_deq':sl[4] != 'FALSE' } - except Exception: - pass - return None - - def analyze_files(self): - fname = '' - fnum = -1 - rid = -1 - fro = -1 - tss = '' - print 'Analyzing journal files:' - for i in range(0, num_files): - file_name = 'jdata/test.%04d.jdat' % i - f = open(file_name) - fhdr = load(f, Hdr) - if fhdr.empty(): - break - if (rid == -1 or fhdr.rid < rid) and fhdr.fro > 0: - fname = file_name - fnum = i - rid = fhdr.rid - fro = fhdr.fro - tss = fhdr.timestamp_str() - print ' %s: rid=%d, fro=0x%08x ts=%s' % (file_name, fhdr.rid, fhdr.fro, fhdr.timestamp_str()) - if fnum < 0 or rid < 0 or fro < 0: - raise Exception('All journal files empty') - print 'Lowest rid found in file %s: rid=%d, fro=0x%08x ts=%s' % (fname, rid, fro, tss) - return (fnum, rid, fro) - - -CLASSES = { - "e": EnqRec, - "d": DeqHdr, - "f": FileHdr -} - -if len(sys.argv) != 3: - print 'Incorrect number of command-line arguments (expected 2, found %d).' % (len(sys.argv) - 1) - print 'Usage: ftest test-file test-number' - raise Exception('Bad command-line arguments') -if not sys.argv[2].isdigit(): - print 'Illegal argument (\"%s\"): test-number must be a number' % a - print 'Usage: ftest test-file test-number' - raise Exception('Illegal argument value') - -tnum = int(sys.argv[2]) -print 'Test number: %d' % tnum -print 'Test file: %s' %sys.argv[1] - -m = Main(sys.argv[1], tnum) -m.run() -m.report() Copied: store/trunk/cpp/tests/jrnl/janalyze.py (from rev 947, store/trunk/cpp/tests/jrnl/ftest.py) =================================================================== --- store/trunk/cpp/tests/jrnl/janalyze.py (rev 0) +++ store/trunk/cpp/tests/jrnl/janalyze.py 2007-09-28 16:02:33 UTC (rev 950) @@ -0,0 +1,540 @@ +#!/usr/bin/env python + +# Copyright (C) 2007 Red Hat Inc. +# +# This file is part of Red Hat Messaging. +# +# Red Hat Messaging is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 +# USA +# +# The GNU Lesser General Public License is available in the file COPYING. + +import sys +from struct import unpack, calcsize +from time import gmtime, strftime + +dblk_size = 128 +sblk_size = 4 * dblk_size +file_size = (3072 + 1) * sblk_size +num_files = 8 +hdr_ver = 1 + +def load(f, klass): + args = load_args(f, klass) + subclass = klass.descriminate(args) + result = subclass(*args) + if subclass != klass: + result.init(f, *load_args(f, subclass)) + result.skip(f) + return result; + +def load_args(f, klass): + size = calcsize(klass.format) + foffs = f.tell(), + bin = f.read(size) + if len(bin) != size: + raise Exception("end of file") + return foffs + unpack(klass.format, bin) + +def size_blks(size, blk_size): + return (size + blk_size - 1)/blk_size + +def rem_in_blk(f, blk_size): + foffs = f.tell() + return (size_blks(f.tell(), blk_size) * blk_size) - foffs; + +def file_full(f): + return f.tell() >= file_size + +def print_xid(xidsize, xid): + if xid == None: + if xidsize > 0: + raise Exception('Inconsistent XID size: xidsize=%d, xid=None' % xidsize) + return '' + if len(xid) > 25: + xidstr = xid[:10] + ' ... ' + xid[-10:] + else: + xidstr = xid + if xidsize != len(xid): + raise Exception('Inconsistent XID size: xidsize=%d, xid(%d)=\"%s\"' % (xidsize, len(xid), xidstr)) + return 'xid(%d)=\"%s\" ' % (xidsize, xidstr) + +def print_data(dsize, data): + if len(data) > 25: + datastr = data[:10] + ' ... ' + data[-10:] + else: + datastr = data + if dsize != len(data): + raise Exception('Inconsistent data size: dsize=%d, data(%d)=\"%s\"' % (dsize, len(data), datastr)) + return 'data(%d)=\"%s\" ' % (dsize, datastr) + +def inv_str(s): + si = '' + for i in range(0,len(s)): + si += chr(~ord(s[i]) & 0xff) + return si + +def load_file_data(f, size, data): + if size == 0: + return (data, True) + if data == None: + loaded = 0 + else: + loaded = len(data) + foverflow = f.tell() + size - loaded > file_size + if foverflow: + rsize = file_size - f.tell() + else: + rsize = size - loaded + bin = f.read(rsize) + if data == None: + data = unpack('%ds' % (rsize), bin)[0] + else: + data = data + unpack('%ds' % (rsize), bin)[0] + return (data, not foverflow) + +def load_tail(self, f): + if self.tail_bin == None: + self.tail_offs = f.tell() + self.tail_bin = f.read(calcsize(RecTail.format)) + if len(self.tail_bin) != calcsize(RecTail.format): + return + else: + self.tail_bin += f.read(calcsize(RecTail.format) - len(self.tail_bin)) + self.enq_tail = RecTail(self.tail_offs, *unpack(RecTail.format, self.tail_bin)) + if self.enq_tail.magic_inv != inv_str(self.magic) or self.enq_tail.rid != self.rid: + print " > %s" % self + raise Exception('Invalid enqueue record tail (magic=0x%08x; rid=%d) at 0x%08x' % (self.enq_tail.magic_inv, self.enq_tail.rid, self.enq_tail.foffs)) + self.enq_tail.skip(f) + self.tail_complete = True + + +class Sizeable: + + def size(self): + classes = [self.__class__] + + size = 0 + while classes: + cls = classes.pop() + if hasattr(cls, "format"): + size += calcsize(cls.format) + classes.extend(cls.__bases__) + + return size + +class Hdr(Sizeable): + + format = '=4sBBHQ' + + @staticmethod + def descriminate(args): + return CLASSES.get(args[1][-1], Hdr) + + def __init__(self, foffs, magic, ver, end, flags, rid): + self.foffs = foffs + self.magic = magic + self.ver = ver + self.end = end + self.flags = flags + self.rid = rid + if self.magic[-1] not in ['0x00', 'a', 'c', 'd', 'e', 'f', 'x']: + error = 3 + + def __str__(self): + if self.empty(): + return '0x%08x: <empty>' % (self.foffs) + if self.magic[-1] == 'x': + return '0x%08x: \"%s\"' % (self.foffs, self.magic) + if self.magic[-1] in ['a', 'c', 'd', 'e', 'f', 'x']: + return '0x%08x: \"%s\" v=%d e=%d f=0x%04x rid=%d' % (self.foffs, self.magic, self.ver, self.end, self.flags, self.rid) + return '0x%08x: <error, unknown magic \"%s\" (possible overwrite boundary?)>' % (self.foffs, self.magic) + + def empty(self): + return self.magic == '\x00'*4 + + def skip(self, f): + f.read(rem_in_blk(f, dblk_size)) + + def check(self): + if self.empty() or self.magic[-1] not in ['a', 'c', 'd', 'e', 'f', 'x']: + return True + if self.ver != hdr_ver and self.magic[-1] != 'x': + raise Exception('%s: Invalid header version: found %d, expected %d.' % (self, self.ver, hdr_ver)) + return False + + + +class FileHdr(Hdr): + + format = '=I4x3Q' + + def init(self, f, foffs, fid, fro, time_sec, time_ns): + self.fid = fid + self.fro = fro + self.time_sec = time_sec + self.time_ns = time_ns + + def __str__(self): + return '%s fid=%d fro=0x%08x t=%s' % (Hdr.__str__(self), self.fid, self.fro, self.timestamp_str()) + + def skip(self, f): + f.read(rem_in_blk(f, sblk_size)) + + def timestamp(self): + return (self.time_sec, self.time_ns) + + def timestamp_str(self): + ts = gmtime(self.time_sec) + fstr = '%%a %%b %%d %%H:%%M:%%S.%d %%Y' % (self.time_ns) + return strftime(fstr, ts) + + +class DeqHdr(Hdr): + + format = '=QQ' + + def init(self, f, foffs, deq_rid, xidsize): + self.deq_rid = deq_rid + self.xidsize = xidsize + self.xid = None + self.deq_tail = None + self.xid_complete = False + self.tail_complete = False + self.load(f) + + def load(self, f): + if self.xidsize == 0: + self.xid_complete = True + self.tail_complete = True + else: + if not self.xid_complete: + ret = load_file_data(f, self.xidsize, self.xid) + self.xid = ret[0] + self.xid_complete = ret[1] + if self.xid_complete and not self.tail_complete: + ret = load_file_data(f, calcsize(RecTail.format), self.tail_bin) + self.tail_bin = ret[0] + if ret[1]: + self.enq_tail = RecTail(self.tail_offs, *unpack(RecTail.format, self.tail_bin)) + if self.enq_tail.magic_inv != inv_str(self.magic) or self.enq_tail.rid != self.rid: + print " > %s" % self + raise Exception('Invalid enqueue record tail (magic=0x%08x; rid=%d) at 0x%08x' % (self.enq_tail.magic_inv, self.enq_tail.rid, self.enq_tail.foffs)) + self.enq_tail.skip(f) + self.tail_complete = ret[1] + return self.complete() + + def complete(self): + return self.xid_complete and self.tail_complete + + def __str__(self): + return '%s %sdrid=%d' % (Hdr.__str__(self), print_xid(self.xidsize, self.xid), self.deq_rid) + + +class TxHdr(Hdr): + + format = '=Q' + + def init(self, f, foffs, xidsize): + self.xidsize = xid + self.xid = None + self.tx_tail = None + self.xid_complete = False + self.tail_complete = False + self.load(f) + + def load(self, f): + if not self.xid_complete: + ret = load_file_data(f, self.xidsize, self.xid) + self.xid = ret[0] + self.xid_complete = ret[1] + if self.xid_complete and not self.tail_complete: + ret = load_file_data(f, calcsize(RecTail.format), self.tail_bin) + self.tail_bin = ret[0] + if ret[1]: + self.enq_tail = RecTail(self.tail_offs, *unpack(RecTail.format, self.tail_bin)) + if self.enq_tail.magic_inv != inv_str(self.magic) or self.enq_tail.rid != self.rid: + print " > %s" % self + raise Exception('Invalid enqueue record tail (magic=0x%08x; rid=%d) at 0x%08x' % (self.enq_tail.magic_inv, self.enq_tail.rid, self.enq_tail.foffs)) + self.enq_tail.skip(f) + self.tail_complete = ret[1] + return self.complete() + + def complete(self): + return self.xid_complete and self.tail_complete + + def __str__(self): + return '%s x=%d xid=\"%s\"' % (Hdr.__str__(self), self.xidsize, self.xid) + + +class RecTail(Sizeable): + + format = '=4sQ' + + def __init__(self, foffs, magic_inv, rid): + self.foffs = foffs + self.magic_inv = magic_inv + self.rid = rid + + def __str__(self): + magic = inv_str(self.magic_inv) + return '[\"%s\" rid=%d]' % (magic, self.rid) + + def skip(self, f): + f.read(rem_in_blk(f, dblk_size)) + + +class EnqRec(Hdr): + + format = '=QQ' + + def init(self, f, foffs, xidsize, dsize): + self.xidsize = xidsize + self.dsize = dsize + self.xid = None + self.data = None + self.enq_tail = None + self.xid_complete = False + self.data_complete = False + self.tail_complete = False + self.tail_bin = None + self.tail_offs = 0 + self.load(f) + + def load(self, f): + if not self.xid_complete: + ret = load_file_data(f, self.xidsize, self.xid) + self.xid = ret[0] + self.xid_complete = ret[1] + if self.xid_complete and not self.data_complete: + ret = load_file_data(f, self.dsize, self.data) + self.data = ret[0] + self.data_complete = ret[1] + if self.data_complete and not self.tail_complete: + ret = load_file_data(f, calcsize(RecTail.format), self.tail_bin) + self.tail_bin = ret[0] + if ret[1]: + self.enq_tail = RecTail(self.tail_offs, *unpack(RecTail.format, self.tail_bin)) + if self.enq_tail.magic_inv != inv_str(self.magic) or self.enq_tail.rid != self.rid: + print " > %s" % self + raise Exception('Invalid enqueue record tail (magic=0x%08x; rid=%d) at 0x%08x' % (self.enq_tail.magic_inv, self.enq_tail.rid, self.enq_tail.foffs)) + self.enq_tail.skip(f) + self.tail_complete = ret[1] + return self.complete() + + def complete(self): + return self.xid_complete and self.data_complete and self.tail_complete + + def __str__(self): + if self.enq_tail == None: + return '%s %s [no tail]' % (Hdr.__str__(self), self.dsize, dstr) + return '%s %s%s %s' % (Hdr.__str__(self), print_xid(self.xidsize, self.xid), print_data(self.dsize, self.data), self.enq_tail) + + +class Main: + + def __init__(self, tfile, tnum): + if tfile != None: + tparams = self.get_test(tfile, tnum) + if tparams == None: + raise Exception('Test %d not found in file %s' % (tnum, tfile)) + self.num_msgs = tparams['num_msgs'] + if tparams['min_size'] == tparams['max_size']: + self.msg_len = tparams['max_size'] + else: + self.msg_len = 0 + self.auto_deq = tparams['auto_deq'] + else: + self.num_msgs = 0 + self.msg_len = 0 + self.auto_deq = False + self.file_start = 0 + self.file_num = 0 + self.fro = 0x200 + self.enqueued = {} + self.msg_cnt = 0 + self.fhdr = None + self.f = None + self.first_rec = False + self.last_file = False + self.last_rid = -1 + + + def run(self): + start_info = self.analyze_files() + stop = self.advance_file(*start_info) + while not stop: + warn = '' + if file_full(self.f): + stop = self.advance_file() + if stop: + break + hdr = load(self.f, Hdr) + if hdr.empty(): + stop = True; + break + if hdr.check(): + stop = True; + if self.first_rec: + if self.fhdr.fro != hdr.foffs: + raise Exception('File header first record offset mismatch: fro=0x%08x; rec_offs=0x%08x' % (self.fhdr.fro, hdr.foffs)) + else: + print ' * fro ok: 0x%08x' % self.fhdr.fro + self.first_rec = False + if isinstance(hdr, EnqRec) and not stop: + while not hdr.complete(): + stop = self.advance_file() + if stop: + break + hdr.load(self.f) + if self.msg_len > 0 and len(hdr.data) != self.msg_len: + raise Exception('Message length (%d) incorrect: expected %d' % (len(hdr.data), self.msg_len)) + stop = not self.check_rid(hdr) + if stop: + warn = ' (WARNING: rid out of order, last rid = %d - could be overwrite boundary.)' % hdr.rid + else: + self.msg_cnt += 1 + if self.auto_deq: + self.enqueued[hdr.rid] = hdr + elif isinstance(hdr, DeqHdr) and not stop: + if self.auto_deq: + if hdr.deq_rid in self.enqueued: + del self.enqueued[hdr.deq_rid] + else: + warn = ' (WARNING: dequeue rid %d not found in enqueued records)' % hdr.deq_rid + stop = not self.check_rid(hdr) + if stop: + warn = ' (WARNING: rid out of order, last rid = %d - could be overwrite boundary.)' % hdr.rid + else: + warn = 'WARNING: Dequeue record rid=%d found in non-dequeue test - ignoring.' % hdr.rid + print ' > %s%s' % (hdr, warn) + if not stop: + stop = (self.last_file and hdr.check()) or hdr.empty() or self.fhdr.empty() + + def report(self): + if self.num_msgs > 0 and self.msg_cnt != self.num_msgs: + print 'WARNING: Found %d messages; %d expected.' % (self.msg_cnt, self.num_msgs) + if len(self.enqueued) > 0: + print 'Remaining enqueued records: ', len(self.enqueued) + for h in self.enqueued: + print self.enqueued[h] + print 'WARNING: Enqueue-Dequeue mismatch, %d enqueued records remain.' % len(self.enqueued) + print 'Test passed; %d records processed.' % self.msg_cnt + + def advance_file(self, *start_info): + seek_flag = False + if len(start_info) == 3: + self.file_start = self.file_num = start_info[0] + self.fro = start_info[2] + seek_flag = True + if self.f != None and file_full(self.f): + self.file_num = self.incr_fnum() + if self.file_num == self.file_start: + return True + if self.file_start == 0: + self.last_file = self.file_num == num_files - 1 + else: + self.last_file = self.file_num == self.file_start - 1 + if self.file_num < 0 or self.file_num >= num_files: + raise Exception('Bad file number %d' % self.file_num) + file_name = 'jdata/test.%04d.jdat' % self.file_num + self.f = open(file_name) + self.fhdr = load(self.f, Hdr) + if seek_flag and self.f.tell() != self.fro: + self.f.seek(self.fro) + self.first_rec = True + print file_name, ": ", self.fhdr + return False + + def incr_fnum(self): + self.file_num += 1 + if self.file_num >= num_files: + self.file_num = 0; + return self.file_num + + def check_rid(self, hdr): + if self.last_rid != -1 and hdr.rid != self.last_rid + 1: + return False + self.last_rid = hdr.rid + return True + + def get_test(self, filename, tnum): + f=open(filename, 'r') + for l in f: + sl = l.strip().split(',') + if len(sl[1]) > 0: #Comments are in col 0, remaining cols are empty + try: + if (int(sl[0]) == tnum): + return { 'num_msgs':int(sl[1]), + 'min_size':int(sl[2]), + 'max_size':int(sl[3]), + 'auto_deq':sl[4] != 'FALSE' } + except Exception: + pass + return None + + def analyze_files(self): + fname = '' + fnum = -1 + rid = -1 + fro = -1 + tss = '' + print 'Analyzing journal files:' + for i in range(0, num_files): + file_name = 'jdata/test.%04d.jdat' % i + f = open(file_name) + fhdr = load(f, Hdr) + if fhdr.empty(): + break + if (rid == -1 or fhdr.rid < rid) and fhdr.fro > 0: + fname = file_name + fnum = i + rid = fhdr.rid + fro = fhdr.fro + tss = fhdr.timestamp_str() + print ' %s: rid=%d, fro=0x%08x ts=%s' % (file_name, fhdr.rid, fhdr.fro, fhdr.timestamp_str()) + if fnum < 0 or rid < 0 or fro < 0: + raise Exception('All journal files empty') + print ' Lowest rid found in file %s: rid=%d, fro=0x%08x ts=%s' % (fname, rid, fro, tss) + return (fnum, rid, fro) + + +CLASSES = { + "a": TxHdr, + "c": TxHdr, + "d": DeqHdr, + "e": EnqRec, + "f": FileHdr +} + +if len(sys.argv) != 1 and len(sys.argv) != 3: + print 'Incorrect number of command-line arguments (expected 0 or 2, found %d).' % (len(sys.argv) - 1) + print 'Usage: ftest test-file test-number' + raise Exception('Bad command-line arguments') +if len(sys.argv) == 3 and not sys.argv[2].isdigit(): + print 'Illegal argument (\"%s\"): test-number must be a number' % a + print 'Usage: ftest test-file test-number' + raise Exception('Illegal argument value') + +if len(sys.argv) != 0: + m = Main(None, None) + m.run() +else: + tnum = int(sys.argv[2]) + print 'Test number: %d' % tnum + print 'Test file: %s' %sys.argv[1] + m = Main(sys.argv[1], tnum) + m.run() + m.report() Modified: store/trunk/cpp/tests/jrnl/rtest =================================================================== --- store/trunk/cpp/tests/jrnl/rtest 2007-09-27 19:29:25 UTC (rev 949) +++ store/trunk/cpp/tests/jrnl/rtest 2007-09-28 16:02:33 UTC (rev 950) @@ -55,7 +55,7 @@ RM=rm RM_DIR="${RM} -rf" TEST_PROG="./jtest" -CHK_PROG="./ftest.py" +CHK_PROG="./janalyze.py" VALGRIND="valgrind -q --track-fds=yes --leak-check=full --leak-resolution=high --show-reachable=yes" #VALGRIND="valgrind -q --track-fds=yes --leak-check=full --leak-resolution=high --show-reachable=yes --suppressions=/usr/lib/valgrind/glibc-2.5.supp" MAKE="make -f Makefile.rtest" Modified: store/trunk/cpp/tests/jrnl/tests.ods =================================================================== (Binary files differ) Modified: store/trunk/cpp/tests/jrnl/wtests.csv =================================================================== --- store/trunk/cpp/tests/jrnl/wtests.csv 2007-09-27 19:29:25 UTC (rev 949) +++ store/trunk/cpp/tests/jrnl/wtests.csv 2007-09-28 16:02:33 UTC (rev 950) @@ -102,6 +102,6 @@ 80,30000,0,38356,TRUE,,"300 dblks max" 81,10000,0,127956,TRUE,,"1000 dblks max" ,,,,,, -"STANDARD PERFORMANCE BENCHMARK: 10,000,000 writes, data=220b (2 dblks)",,,,,, -82,10000000,220,220,FALSE,,"2 dblks" -83,10000000,220,220,TRUE,,"2 dblks" +"STANDARD PERFORMANCE BENCHMARK: 10,000,000 writes, data=212b (2 dblks)",,,,,, +82,10000000,212,212,FALSE,,"2 dblks" +83,10000000,212,212,TRUE,,"2 dblks"