Author: kpvdr Date: 2008-09-26 17:00:58 -0400 (Fri, 26 Sep 2008) New Revision: 2549 Modified: store/trunk/cpp/lib/jrnl/jcntl.cpp store/trunk/cpp/lib/jrnl/jinf.cpp store/trunk/cpp/lib/jrnl/jinf.hpp store/trunk/cpp/lib/jrnl/lf_map.cpp store/trunk/cpp/lib/jrnl/lf_map.hpp store/trunk/cpp/tests/jrnl/_ut_jinf.cpp Log: Completed new jinf class analyze function and comprehensive boost tests. This is in preparation for journal file insertion. Modified: store/trunk/cpp/lib/jrnl/jcntl.cpp =================================================================== --- store/trunk/cpp/lib/jrnl/jcntl.cpp 2008-09-26 19:25:40 UTC (rev 2548) +++ store/trunk/cpp/lib/jrnl/jcntl.cpp 2008-09-26 21:00:58 UTC (rev 2549) @@ -606,8 +606,8 @@ try { - rd._ffid = ji.get_start_file(); - rd._lfid = ji.get_end_file(); + rd._ffid = ji.get_first_fid(); + rd._lfid = ji.get_last_fid(); rd._owi = ji.get_initial_owi(); rd._frot = ji.get_frot(); rd._jempty = false; Modified: store/trunk/cpp/lib/jrnl/jinf.cpp =================================================================== --- store/trunk/cpp/lib/jrnl/jinf.cpp 2008-09-26 19:25:40 UTC (rev 2548) +++ store/trunk/cpp/lib/jrnl/jinf.cpp 2008-09-26 21:00:58 UTC (rev 2549) @@ -59,8 +59,6 @@ _tm_ptr(0), _valid_flag(false), _analyzed_flag(false), - _start_file(0), - _end_file(0), _initial_owi(false), _frot(false) { @@ -88,8 +86,6 @@ _tm_ptr(std::localtime(&ts.tv_sec)), _valid_flag(false), _analyzed_flag(false), - _start_file(0), - _end_file(0), _initial_owi(false) {} @@ -142,72 +138,21 @@ _valid_flag = true; } -u_int16_t +void jinf::analyze() { -// lf_map early_map; // map for all owi flags same as fid 0 -// lf_map late_map; // map for all owi flags opposite to fid 0 -// -// if (!_valid_flag) -// validate(); -// bool done = false; -// for (u_int16_t fid=0; fid<_num_jfiles && !done; fid++) -// { -// std::ostringstream oss; -// oss << _jdir << "/" << _base_filename << "."; -// oss << std::setw(4) << std::setfill('0') << std::hex << fid; -// oss << "." << JRNL_DATA_EXTENSION; -// std::ifstream jifs(oss.str().c_str()); -// if (!jifs.good()) -// throw jexception(jerrno::JERR__FILEIO, oss.str(), "jinf", "analyze"); -// file_hdr fhdr; -// jifs.read((char*)&fhdr, sizeof(fhdr)); -// if (fhdr._magic != RHM_JDAT_FILE_MAGIC) // No file header -// { -// if (!fid) -// throw jexception(jerrno::JERR_JINF_JDATEMPTY, "jinf", "analyze"); -// _frot = true; -// done = true; -// } -// else -// { -// assert(fid == fhdr._fid); -// if (fid == 0) -// { -// _initial_owi = fhdr.get_owi(); -// early_map.insert(fhdr._lid, fid); -// } -// else -// { -// if (_initial_owi == fhdr.get_owi()) -// early_map.insert(fhdr._lid, fid); -// else -// late_map.insert(fhdr._lid, fid); -// } -// } -// jifs.close(); -// } // for (fid) -// -// // If this is not the first rotation, all files should be in either early or late maps -// if (!_frot) assert(early_map.size() + late_map.size() == _num_jfiles); -// -// if (late_map.empty()) -// _start_file = 0; -// else -// _start_file = (*late_map.begin()).second; -// ------------------- - u_int16_t ffid = 0xffff; - bool owi = false; bool done = false; - - if (!_valid_flag) - validate(); - u_int16_t fnum=0; - while (!done && fnum < _num_jfiles) + lf_map early_map; // map for all owi flags same as fid 0 + lf_map late_map; // map for all owi flags opposite to fid 0 + + if (!_valid_flag) + validate(); + done = false; + for (u_int16_t fid=0; fid<_num_jfiles && !done; fid++) { std::ostringstream oss; oss << _jdir << "/" << _base_filename << "."; - oss << std::setw(4) << std::setfill('0') << std::hex << fnum; + oss << std::setw(4) << std::setfill('0') << std::hex << fid; oss << "." << JRNL_DATA_EXTENSION; std::ifstream jifs(oss.str().c_str()); if (!jifs.good()) @@ -216,30 +161,37 @@ jifs.read((char*)&fhdr, sizeof(fhdr)); if (fhdr._magic != RHM_JDAT_FILE_MAGIC) // No file header { - if (!fnum) + if (!fid) // fid 0 == lid 0 cannot be empty throw jexception(jerrno::JERR_JINF_JDATEMPTY, "jinf", "analyze"); _frot = true; done = true; } - else if (fnum == 0) // First file only + else { - owi = fhdr.get_owi(); - _initial_owi = owi; - ffid = 0; + assert(fid == fhdr._fid); + if (fid == 0) + { + _initial_owi = fhdr.get_owi(); + early_map.insert(fhdr._lid, fid); + } + else + { + if (_initial_owi == fhdr.get_owi()) + early_map.insert(fhdr._lid, fid); + else + late_map.insert(fhdr._lid, fid); + } } - else if (fhdr.get_owi() != owi) // Change in OWI - { - ffid = fnum; - done = true; - } - else - _end_file = fnum; jifs.close(); - fnum++; - } - _start_file = ffid; + } // for (fid) + + // If this is not the first rotation, all files should be in either early or late maps + if (!_frot) assert(early_map.size() + late_map.size() == _num_jfiles); + + _fidl.clear(); + late_map.get_fid_list(_fidl); + early_map.get_fid_list(_fidl); _analyzed_flag = true; - return _start_file; } void @@ -263,21 +215,29 @@ } u_int16_t -jinf::get_start_file() +jinf::get_first_fid() { if (!_analyzed_flag) analyze(); - return _start_file; + return *_fidl.begin(); } u_int16_t -jinf::get_end_file() +jinf::get_last_fid() { if (!_analyzed_flag) analyze(); - return _end_file; + return *_fidl.rbegin(); } +jinf::fid_list& +jinf::get_fid_list() +{ + if (!_analyzed_flag) + analyze(); + return _fidl; +} + bool jinf::get_initial_owi() { @@ -302,8 +262,7 @@ oss << "Journal ID \"" << _jid << "\" initialized " << (_tm_ptr->tm_year + 1900) << "/"; oss << std::setw(2) << (_tm_ptr->tm_mon + 1) << "/" << std::setw(2) << _tm_ptr->tm_mday << " "; oss << std::setw(2) << _tm_ptr->tm_hour << ":" << std::setw(2) << _tm_ptr->tm_min << ":"; - oss << std::setw(2) << _tm_ptr->tm_sec << "." << std::setw(9) << _ts.tv_nsec << ":" << - std::endl; + oss << std::setw(2) << _tm_ptr->tm_sec << "." << std::setw(9) << _ts.tv_nsec << ":" << std::endl; oss << " Journal directory: \"" << _jdir << "\"" << std::endl; oss << " Journal base filename: \"" << _base_filename << "\"" << std::endl; oss << " Journal version: " << (unsigned)_jver << std::endl; @@ -313,8 +272,7 @@ oss << " Datablock size (JRNL_DBLK_SIZE): " << _dblk_size << " bytes" << std::endl; oss << " Write page size: " << _wcache_pgsize_sblks << " sblks" << std::endl; oss << " Number of write pages: " << _wcache_num_pages << std::endl; - oss << " Read page size (JRNL_RMGR_PAGE_SIZE): " << _rcache_pgsize_sblks << " sblks" << - std::endl; + oss << " Read page size (JRNL_RMGR_PAGE_SIZE): " << _rcache_pgsize_sblks << " sblks" << std::endl; oss << " Number of read pages (JRNL_RMGR_PAGES): " << _rcache_num_pages << std::endl; return oss.str(); } @@ -322,9 +280,10 @@ const std::string jinf::xml_str() const { + // TODO: This is *not* an XML writer, rather for simplicity, it uses literals. I'm sure a more elegant way can be + // found to do this using the real thing... + std::ostringstream oss; - // TODO: I'm sure a more elegant way can be found to do this, but direct and simple - // seems like a good start! oss << std::setfill('0'); oss << "<?xml version=\"1.0\" ?>" << std::endl; oss << "<jrnl>" << std::endl; @@ -362,16 +321,16 @@ void jinf::read(const std::string& jinf_filename) { - // TODO: This is *not* an XML reader, rather for simplicity, it is a brute-force - // line reader which relies on string recognition. + // TODO: This is *not* an XML reader, rather for simplicity, it is a brute-force line reader which relies on string + // recognition. Can it be replaced cheaply by the real thing? - char buff[1024]; + char buff[1024]; // limit of line input length std::ifstream jinfs(jinf_filename.c_str()); if (!jinfs.good()) throw jexception(jerrno::JERR__FILEIO, jinf_filename.c_str(), "jinf", "read"); while (jinfs.good()) { - jinfs.getline(buff, 1024); + jinfs.getline(buff, 1023); if (std::strstr(buff, "journal_version")) _jver = u_int16_value(buff); else if(std::strstr(buff, "id_string")) Modified: store/trunk/cpp/lib/jrnl/jinf.hpp =================================================================== --- store/trunk/cpp/lib/jrnl/jinf.hpp 2008-09-26 19:25:40 UTC (rev 2548) +++ store/trunk/cpp/lib/jrnl/jinf.hpp 2008-09-26 21:00:58 UTC (rev 2549) @@ -35,6 +35,7 @@ #include <ctime> #include <string> #include <sys/types.h> +#include <vector> namespace mrg { @@ -46,6 +47,9 @@ */ class jinf { + public: + typedef std::vector<u_int16_t> fid_list; + private: u_int8_t _jver; std::string _jid; @@ -63,8 +67,7 @@ std::tm* _tm_ptr; bool _valid_flag; bool _analyzed_flag; - u_int16_t _start_file; - u_int16_t _end_file; + fid_list _fidl; bool _initial_owi; bool _frot; @@ -79,7 +82,7 @@ virtual ~jinf(); void validate(); - u_int16_t analyze(); + void analyze(); void write(); inline u_int8_t jver() const { return _jver; } @@ -97,8 +100,9 @@ inline u_int16_t wcache_num_pages() const { return _wcache_num_pages; } inline u_int32_t rcache_pgsize_sblks() const { return _rcache_pgsize_sblks; } inline u_int16_t rcache_num_pages() const { return _rcache_num_pages; } - u_int16_t get_start_file(); - u_int16_t get_end_file(); + u_int16_t get_first_fid(); + u_int16_t get_last_fid(); + fid_list& get_fid_list(); bool get_initial_owi(); bool get_frot(); Modified: store/trunk/cpp/lib/jrnl/lf_map.cpp =================================================================== --- store/trunk/cpp/lib/jrnl/lf_map.cpp 2008-09-26 19:25:40 UTC (rev 2548) +++ store/trunk/cpp/lib/jrnl/lf_map.cpp 2008-09-26 21:00:58 UTC (rev 2549) @@ -61,5 +61,20 @@ fid_list.push_back(i->second); } +// debug aid +std::string +lf_map::to_string() +{ + std::ostringstream oss; + oss << "{lid:fid "; + for (lfmap_citr i=_map.begin(); i!=_map.end(); i++) + { + if (i != _map.begin()) oss << ", "; + oss << (*i).first << ":" << (*i).second; + } + oss << "}"; + return oss.str(); +} + } // namespace journal } // namespace mrg Modified: store/trunk/cpp/lib/jrnl/lf_map.hpp =================================================================== --- store/trunk/cpp/lib/jrnl/lf_map.hpp 2008-09-26 19:25:40 UTC (rev 2548) +++ store/trunk/cpp/lib/jrnl/lf_map.hpp 2008-09-26 21:00:58 UTC (rev 2549) @@ -34,6 +34,7 @@ #define mrg_journal_lf_map_hpp #include <map> +#include <string> #include <sys/types.h> #include <vector> @@ -52,6 +53,7 @@ public: typedef std::map<u_int16_t, u_int16_t> lfmap; typedef lfmap::const_iterator lfmap_citr; + typedef lfmap::const_reverse_iterator lfmap_critr; private: typedef std::pair<u_int16_t, u_int16_t> lfpair; @@ -67,7 +69,12 @@ inline bool empty() const { return _map.empty(); } inline lfmap_citr begin() { return _map.begin(); } inline lfmap_citr end() { return _map.end(); } + inline lfmap_critr rbegin() { return _map.rbegin(); } + inline lfmap_critr rend() { return _map.rend(); } void get_fid_list(std::vector<u_int16_t>& fid_list); + + // debug aid + std::string to_string(); }; } // namespace journal Modified: store/trunk/cpp/tests/jrnl/_ut_jinf.cpp =================================================================== --- store/trunk/cpp/tests/jrnl/_ut_jinf.cpp 2008-09-26 19:25:40 UTC (rev 2548) +++ store/trunk/cpp/tests/jrnl/_ut_jinf.cpp 2008-09-26 21:00:58 UTC (rev 2549) @@ -35,9 +35,11 @@ #include <iostream> #include "jrnl/file_hdr.hpp" #include "jrnl/jcfg.hpp" +#include "jrnl/jdir.hpp" #include "jrnl/jerrno.hpp" #include "jrnl/jexception.hpp" #include "jrnl/jinf.hpp" +#include <map> #include <vector> #define NUM_JFILES 4 @@ -56,95 +58,456 @@ // === Helper functions === const string jid("test journal id"); -const string base_filename("test_base"); +const string base_filename("test_base_"); timespec ts; -enum rid_scheme { RID_NONE, RID_LINEAR }; -void create_journal_filenames(vector<string>& jfiles) +/* +* This test helper class is used by first setting up a map or "blueprint" of what the journal should +* look like for recovery, then creating it. The jinf object under test then reads and analyzes the +* created journal, and its analysis is checked against what is expected. +* +* General usage pattern: +* 1. create instance of lid_fid_map. +* 2. call lid_fid_map::initial_journal_create() to simulate initial journal creation. +* 3. (optional) call lid_fid_map::journal_append() one or more times to simulate the addition of journal files. +* 4. call lid_fid_map::create_journal() to create dummy journal files (files containing only file headers) +* 5. create and initialize the jinf object under test +* 6. call jinf::analyze() to determine the fid order - and thus also first and last lids +* 7. call lid_fid_map::check_analysis() to check the conclusions of the analysis +* 8. call lid_fid_map::destroy_journal() to delete the journal files and reset the lid_fid_map object. +* 9. (optional) back to step 2 for more tests +* +* See the individual methods below for more details. +*/ +class lid_fid_map { - for (int fnum=0; fnum<NUM_JFILES; fnum++) - { - stringstream fn; - fn << test_dir << "/" << base_filename << "."; - fn << setfill('0') << hex << setw(4) << fnum << "." << JRNL_DATA_EXTENSION; - jfiles.push_back(fn.str()); - } -} + public: + typedef pair<u_int16_t, file_hdr> lfpair; // Used for loading the map + typedef multimap<u_int16_t, file_hdr> lfmap; // Stores the journal "plan" before it is created on-disk + typedef lfmap::const_iterator lfmap_citr; // General purpose iterator + typedef pair<lfmap_citr, lfmap_citr> lfmap_range; // Range of values returned by multimap's equal_range() fn -void init_fhdr(file_hdr& fh, const u_int32_t fid, const u_int64_t rid, const bool owi, - const bool no_enq = false) -{ - fh._magic = RHM_JDAT_FILE_MAGIC; - fh._version = RHM_JDAT_VERSION; + private: + lfmap _map; // Stores the journal "blueprint" before it is created on-disk + u_int16_t _num_used_files; // number of files which contain jorunals + u_int16_t _oldest_lid; // lid where owi flips; always 0 if !_full + u_int16_t _last_fid; // last fid (ie file added) + + public: + lid_fid_map() : _num_used_files(0), _oldest_lid(0), _last_fid(0) {} + virtual ~lid_fid_map() {} + + // Mainly used for debugging + void print() + { + int cnt = 0; + for (lfmap_citr i=_map.begin(); i!=_map.end(); i++, cnt++) + { + const file_hdr fh = i->second; + cout << " " << cnt << ": owi=" << (fh.get_owi()?"t":"f") << hex << " frid=0x" << fh._rid; + cout << " fid=0x" << fh._fid << " lid=0x" << fh._lid << " fro=0x" << fh._fro << dec << endl; + } + } + + std::size_t size() + { + return _map.size(); + } + + /* + * Method initial_journal_create(): Used to simulate the initial creation of a journal before file insertions + * take place. + * + * num_jfiles: The initial journal file count. + * num_used_jfiles: If this number is less than num_jfiles, it indicates a clean journal that has not yet + * completed its first rotation, and some files are empty (ie all null). The first + * num_used_jfiles will contain file headers, the remainder will be blank. + * oldest_lid: The lid (==fid, see note 1 below) at which the owi flag flips. During normal operation, each + * time the journal rotates back to file 0, a flag (called the overwrite indicator or owi) is + * flipped. This flag is saved in the file header. During recovery, if scanning from logical + * file 0 upwards, the file at which this flag reverses from its value in file 0 is the file + * that was to have been overwritten next, and is thus the "oldest" file. Recovery analysis must + * start with this file. oldest_lid sets the file at which this flag will flip value for the + * simulated recovery analysis. Note that this will be ignored if num_used_jfiles < num_jfiles, + * as it is not possible for an overwrite to have occurred if not all the files have been used. + * first_owi: Sets the value of the owi flag in file 0. If set to false, then the flip will be found with + * a true flag (and visa versa). + * + * NOTES: + * 1. By definition, the lids and fids coincide for a journal containing no inserted files. Thus fid == lid for + * all journals created after using initial_journal_create() alone. + * 2. By definition, if a journal is not full (num_used_jfiles < num_jfiles), then all owi flags for those files + * that are used must be the same. It is not possible for an overwrite situation to arise if a journal is not + * full. + * 3. This function acts on map _map only, and does not create any test files. Call journal_create() to do that. + * 4. This function must be called on a clean test object or on one where the previous test data has been + * cleared by calling journal_destroy(). Running this function moret than once on existing data will + * result in invalid journals which cannot be recovered. + */ + void initial_journal_create(const u_int16_t num_jfiles, // Total number of files + const u_int16_t num_used_jfiles, // Number of used files, rest empty at end + const u_int16_t oldest_lid = 0, // Fid where owi reverses + const bool first_owi = false) // Value of first owi flag (ie fid=0) + { + const bool full = num_used_jfiles == num_jfiles; + bool owi = first_owi; + _oldest_lid = full ? oldest_lid : 0; + for (u_int16_t lid = 0; lid < num_jfiles; lid++) + { + const u_int16_t fid = lid; + file_hdr fh; + if (fid < num_used_jfiles) + { + _num_used_files = num_used_jfiles; + if (full && oldest_lid > 0 && lid == oldest_lid) + owi = ~owi; // Flip owi if all files in use and oldest_lid > 0 + const u_int64_t frid = u_int64_t(random()); + init_fhdr(fh, frid, fid, lid, owi); + } + _map.insert(lfpair(lid, fh)); + } + } + + /* + * Method journal_append(): Used to simulate the addition of journal files into an existing journal. When one or + * more files are inserted after a particular lid, the lids of the following files are incremented. The fids of + * the inserted files follow those of all existing files, thus leading to a lid-fid discreppancy. + * + * Example: Before insertion: + * Logical view: Physical view: + * +---+---+---+---+---+---+ +---+---+---+---+---+---+ + * lid --> | 0 | 1 | 2 | 3 | 4 | 5 | lid --> | 0 | 1 | 2 | 3 | 4 | 5 | + * +---+---+---+---+---+---+ +---+---+---+---+---+---+ + * fid --> 0 1 2 3 4 5 fid --> 0 1 2 3 4 5 + * + * After insertion of 2 files after lid 2 (marked with *s): + * Logical view: Physical view: + * +---+---+---+---+---+---+---+---+ +---+---+---+---+---+---+---+---+ + * lid --> | 0 | 1 | 2 |*3*|*4*| 5 | 6 | 7 | lid --> | 0 | 1 | 2 | 5 | 6 | 7 |*3*|*4*| + * +---+---+---+---+---+---+---+---+ +---+---+---+---+---+---+---+---+ + * fid --> 0 1 2 6 7 3 4 5 fid --> 0 1 2 3 4 5 6 7 + * + * Note that the journal only writes file headers (and hence the lids in those headers) as the files are + * overwritten. If the journal should fail after insertion but before the files following those inserted are + * overwritten, then duplicate lids will be present (though no duplicate fids are possible). The overwrite + * indicator (owi) flag and the fid numbers may be used to resolve the ambiguity and determine the logically + * earlier lid in this case as follows: + * + * Example: Before insertion: + * Logical view: Physical view: + * +---+---+---+---+---+---+ +---+---+---+---+---+---+ + * lid --> | 0 | 1 | 2 | 3 | 4 | 5 | lid --> | 0 | 1 | 2 | 3 | 4 | 5 | + * owi --> | t | t | t | f | f | f | owi --> | t | t | t | f | f | f | + * +---+---+---+---+---+---+ +---+---+---+---+---+---+ + * fid --> 0 1 2 3 4 5 fid --> 0 1 2 3 4 5 + * + * After insertion of 2 files after lid 2 (marked with *s) and failure: + * Logical view: Physical view: + * +---+---+---+---+---+---+---+---+ +---+---+---+---+---+---+---+---+ + * lid --> | 0 | 1 | 2 |*3*|*4*| 3 | 4 | 5 | lid --> | 0 | 1 | 2 | 3 | 4 | 5 |*3*|*4*| + * owi --> | t | t | t | t | t | f | f | f | owi --> | t | t | t | f | f | f | t | t | + * +---+---+---+---+---+---+---+---+ +---+---+---+---+---+---+---+---+ + * fid --> 0 1 2 6 7 3 4 5 fid --> 0 1 2 3 4 5 6 7 + * + * Test 1: Correct lid has owi flag that matches that of fid/lid 0 + * Test 2: Correct lid has higher fids + * + * after_lid: The logical id (lid) after which the new file is to be inserted. + * num_files: The number of files to be inserted. + * adjust_lids: Flag indicating that the lids of files _following_ the inserted files are to be adjusted upwards + * by the number of inserted files. Not doing so simulates a recovery immediatly after insertion + * but before the following files are overwritten with their new lids. If this is set false, then: + * a) after_lid MUST be the most recent file (_oldest_lid-1 ie last lid before owi changes). + * b) This call must be the last insert call. + * + * NOTES: + * 1. It is not possible to insert before lid/fid 0; thus these are always coincident. This operation is + * logically equivilent to inserting after the last lid, which is possible. + * 2. It is not possible to append to a journal that is not full. Doing so will result in an unrecoverable + * journal (one that is logically inconsistent that can never occur in reality). + * 3. If a journal is stopped/interrupted immediately after a file insertion, there could be duplicate lids in + * play at recovery, as the following file lids in their headers are only overwritten when the file is + * eventually written to during normal operation. The owi flags, however, are used to determine which of the + * ambiguous lids are the inserted files. + */ + void journal_append(const u_int16_t after_lid, // Insert files after this lid + const u_int16_t num_files = 1, // Number of files to insert + const bool adjust_lids = true) // Adjust lids following inserted files + { + _num_used_files += num_files; + const u_int16_t num_jfiles_before_append = _map.size(); + lfmap_citr i = _map.find(after_lid); + if (i == _map.end()) BOOST_FAIL("Unable to find lid=" << after_lid << " in map."); + const file_hdr fh_before = (*i).second; + + // Move overlapping lids (if req'd) + if (adjust_lids && after_lid < num_jfiles_before_append - 1) + { + for (u_int16_t lid = num_jfiles_before_append - 1; lid > after_lid; lid--) + { + lfmap_citr itr = _map.find(lid); + if (itr == _map.end()) BOOST_FAIL("Unable to find lid=" << after_lid << " in map."); + file_hdr fh = itr->second; + _map.erase(lid); + fh._lid += num_files; + if (lid == _oldest_lid) + _oldest_lid += num_files; + _map.insert(lfpair(fh._lid, fh)); + } + } + + // Add new file headers + u_int16_t fid = num_jfiles_before_append; + u_int16_t lid = after_lid + 1; + while (fid < num_jfiles_before_append + num_files) + { + const u_int64_t frid = u_int64_t(random()); + const size_t fro = 0x200; + const file_hdr fh(RHM_JDAT_FILE_MAGIC, RHM_JDAT_VERSION, frid, fid, lid, fro, fh_before.get_owi(), true); + _map.insert(lfpair(lid, fh)); + _last_fid = fid; + fid++; + lid++; + } + } + + /* + * Method check_analysis(): Used to check the result of the test jinf object analysis by comparing the fid order + * array it produces against the internal map. + * + * ji: A ref to the jinf object under test. + */ + void check_analysis(jinf& ji) // jinf object under test after analyze() has been called + { + BOOST_CHECK_EQUAL(ji.get_first_fid(), get_first_fid()); + BOOST_CHECK_EQUAL(ji.get_last_fid(), get_last_fid()); + + jinf::fid_list& fidl = ji.get_fid_list(); + const u_int16_t num_jfiles = _map.size(); + const bool all_used = _num_used_files == num_jfiles; + BOOST_CHECK_EQUAL(fidl.size(), _num_used_files); + + const u_int16_t lid_start = all_used ? _oldest_lid : 0; + // Because a simulated failure would leave lid dups in map and last_fid would not exist in map in this + // case, we must find lid_stop via fid instead. Search for fid == num_files. + lfmap_citr itr = _map.begin(); + while (itr != _map.end() && itr->second._fid != _num_used_files - 1) itr++; + if (itr == _map.end()) + BOOST_FAIL("check(): Unable to find fid=" << (_num_used_files - 1) << " in map."); + const u_int16_t lid_stop = itr->second._lid; + + std::size_t fidl_index = 0; + for (u_int16_t lid_cnt = lid_start; lid_cnt < lid_stop; lid_cnt++, fidl_index++) + { + const u_int16_t lid = lid_cnt % num_jfiles; + lfmap_citr itr = _map.find(lid); + if (itr == _map.end()) + BOOST_FAIL("check(): Unable to find lid=" << lid << " in map."); + BOOST_CHECK_EQUAL(itr->second._fid, fidl[fidl_index]); + } + } + + /* + * Method get_fid(): Look up a fid from a known lid. + */ + u_int16_t get_fid(const u_int16_t lid, const bool initial_owi = false) + { + switch (_map.count(lid)) + { + case 1: + return _map.find(lid)->second._fid; + case 2: + for (lfmap_citr itr = _map.lower_bound(lid); itr != _map.upper_bound(lid); itr++) + { + if (itr->second.get_owi() != initial_owi) + return itr->second._fid; + } + default:; + } + BOOST_FAIL("get_fid(): lid=" << lid << " not found in map."); + return 0xffff; + } + + /* + * Method get_first_fid(): Look up the first (oldest, or next-to-be-overwritten) fid in the analysis sequence. + */ + u_int16_t get_first_fid() + { + return get_fid(_oldest_lid); + } + + /* + * Method get_last_fid(): Look up the last (newest, or most recently written) fid in the analysis sequence. + */ + u_int16_t get_last_fid() + { + u_int16_t flid = 0; + if (_num_used_files == _map.size()) // journal full? + { + if (_oldest_lid) + { + // if failed insert, cycle past duplicate lids + while (_map.count(_oldest_lid) == 2) + _oldest_lid++; + while (_map.find(_oldest_lid) != _map.end() && _map.find(_oldest_lid)->second.get_owi() == false) + _oldest_lid++; + flid = _oldest_lid - 1; + } + else + flid = _map.size() - 1; + } + else + flid = _num_used_files - 1; + return get_fid(flid, true); + } + + /* + * Method create_journal(): Used to create the dummy journal files from the built-up map created by calling + * initial_journal_create() and optionally journal_append() one or more times. Since the jinf object reads the + * jinf file and the file headers only, the create object creates a dummy journal file containing only a file + * header (512 bytes each) and a single jinf file which contains the journal metadata required for recovery + * analysis. + */ + void create_journal() + { + create_jinf(); + u_int16_t fid = 0; + for (lfmap_citr itr = _map.begin(); itr != _map.end(); itr++, fid++) + { + if (itr->second._fid == 0 && itr->second._magic == 0) // empty header, use fid counter instaed + create_journal_file(fid, itr->second); + else + create_journal_file(itr->second._fid, itr->second); + } + } + + /* + * Method destroy_journal(): Destroy the files created by create_journal() and reset the lid_fid_map test object. + * A new test may be started using the same lid_fid_map test object once this call has been made. + */ + void destroy_journal() + { + for (u_int16_t fid = 0; fid < _map.size(); fid++) + { + string fn = create_journal_filename(fid); + BOOST_WARN_MESSAGE(::unlink(fn.c_str()) == 0, "destroy_journal(): Failed to remove file " << fn); + } + clean_journal_info_file(); + _map.clear(); + _num_used_files = 0; + _oldest_lid = 0; + _last_fid = 0; + } + + /* + * Method create_new_jinf(): This static call creates a default jinf file only. This is used to test the read + * constructor of a jinf test object which reads a jinf file at instantiation. + */ + static void create_new_jinf() + { + if (jdir::exists(test_dir)) + jdir::delete_dir(test_dir); + create_jinf(NUM_JFILES); + } + + /* + * Method clean_journal_info_file(): This static method deletes only a jinf file without harming any other + * journal file or its directory. This is used to clear those tests which rely only on the existance of a + * jinf file. + */ + static void clean_journal_info_file() + { + stringstream fn; + fn << test_dir << "/" << base_filename << "." << JRNL_INFO_EXTENSION; + BOOST_WARN_MESSAGE(::unlink(fn.str().c_str()) == 0, "clean_journal_info_file(): Failed to remove file " << fn.str()); + } + + private: + static void init_fhdr(file_hdr& fh, + const u_int64_t frid, + const u_int16_t fid, + const u_int16_t lid, + const bool owi, + const bool no_enq = false) + { + fh._magic = RHM_JDAT_FILE_MAGIC; + fh._version = RHM_JDAT_VERSION; #if defined(JRNL_BIG_ENDIAN) - fh._eflag = RHM_BENDIAN_FLAG; + fh._eflag = RHM_BENDIAN_FLAG; #else - fh._eflag = RHM_LENDIAN_FLAG; + fh._eflag = RHM_LENDIAN_FLAG; #endif - fh._uflag = owi ? rec_hdr::HDR_OVERWRITE_INDICATOR_MASK : 0; - fh._rid = rid; - fh._fid = fid; - fh._fro = no_enq ? 0 : 0x200; - timespec ts; - ::clock_gettime(CLOCK_REALTIME, &ts); - fh._ts_sec = ts.tv_sec; - fh._ts_nsec = ts.tv_nsec; -} + fh._uflag = owi ? rec_hdr::HDR_OVERWRITE_INDICATOR_MASK : 0; + fh._rid = frid; + fh._fid = fid; + fh._lid = lid; + fh._fro = no_enq ? 0 : 0x200; + timespec ts; + ::clock_gettime(CLOCK_REALTIME, &ts); + fh._ts_sec = ts.tv_sec; + fh._ts_nsec = ts.tv_nsec; + } -void create_journal_files(vector<string>& jfiles, rid_scheme scheme, - u_int32_t min_fid_offs = 0, u_int64_t rid_offs = 0) -{ - const u_int64_t rid_incr = 0x10; - file_hdr fh; - vector<string>::iterator itr; - u_int32_t fid = 0; - u_int64_t rid = rid_offs + ((NUM_JFILES - min_fid_offs) % NUM_JFILES) * rid_incr; - for (itr=jfiles.begin(); itr<jfiles.end(); itr++) - { - - ofstream of(itr->c_str(), ofstream::out | ofstream::trunc); - if (!of.good()) - BOOST_FAIL("Unable to open test journal file " << *itr << " for writing."); - - // prepare file_hdr - int cnt = sizeof(file_hdr); - if (scheme == RID_NONE) // create file containing 0s - std::memset(&fh, 0, cnt); - else - init_fhdr(fh, fid, rid, fid >= min_fid_offs); - - // write file header - of.write((const char*)&fh, cnt); - - // fill remaining sblk with 0s - while (cnt++ < JRNL_DBLK_SIZE * JRNL_SBLK_SIZE) - of.put(0); - of.close(); - - if (++fid == min_fid_offs) - rid -= rid_incr * (NUM_JFILES - 1); - else - rid += rid_incr; - } -} + void create_jinf() + { + if (jdir::exists(test_dir)) + jdir::delete_dir(test_dir); + create_jinf(_map.size()); + } -void clean_journal_info_file() -{ - stringstream fn; - fn << test_dir << "/" << base_filename << "." << JRNL_INFO_EXTENSION; - BOOST_WARN_MESSAGE(::unlink(fn.str().c_str()) == 0, "Failed to remove file " << fn.str()); -} + static void create_jinf(u_int16_t num_files) + { + jdir::create_dir(test_dir); // Check test dir exists; create it if not + ::clock_gettime(CLOCK_REALTIME, &ts); + jinf ji(jid, test_dir, base_filename, num_files, JFSIZE_SBLKS, JRNL_WMGR_DEF_PAGE_SIZE, JRNL_WMGR_DEF_PAGES, + ts); + ji.write(); + } -void clean_journal_files(vector<string>& jfiles) -{ - for (vector<string>::iterator itr=jfiles.begin(); itr != jfiles.end(); itr++) - BOOST_WARN_MESSAGE(::unlink(itr->c_str()) == 0, "Failed to remove file " << *itr); - jfiles.clear(); -} + static void create_journal_file(const u_int16_t fid, const file_hdr& fh) + { + write_file_header(fh, create_journal_filename(fid)); + } + static void write_file_header(const file_hdr& fh, + const string filename) + { + ofstream of(filename.c_str(), ofstream::out | ofstream::trunc); + if (!of.good()) + BOOST_FAIL("Unable to open test journal file \"" << filename << "\" for writing."); + + // write file header + u_int32_t cnt = sizeof(file_hdr); + of.write((const char*)&fh, cnt); + if (of.fail() || of.bad()) + BOOST_FAIL("Error writing file header to test journal file \"" << filename << "\"."); + + // fill remaining sblk with 0s + while (cnt++ < JRNL_DBLK_SIZE * JRNL_SBLK_SIZE) + { + of.put(0); + if (of.fail() || of.bad()) + BOOST_FAIL("Error writing filler to test journal file \"" << filename << "\"."); + } + + of.close(); + if (of.fail() || of.bad()) + BOOST_FAIL("Error closing test journal file \"" << filename << "\"."); + } + + static string create_journal_filename(const u_int16_t fid) + { + stringstream fn; + fn << test_dir << "/" << base_filename << "."; + fn << setfill('0') << hex << setw(4) << fid << "." << JRNL_DATA_EXTENSION; + return fn.str(); + } +}; + QPID_AUTO_TEST_CASE(write_constructor) { cout << test_filename << ".write_constructor: " << flush; + jdir::create_dir(test_dir); // Check test dir exists; create it if not ::clock_gettime(CLOCK_REALTIME, &ts); jinf ji(jid, test_dir, base_filename, NUM_JFILES, JFSIZE_SBLKS, JRNL_WMGR_DEF_PAGE_SIZE, JRNL_WMGR_DEF_PAGES, ts); @@ -152,20 +515,26 @@ BOOST_CHECK(ji.jid().compare(jid) == 0); BOOST_CHECK(ji.jdir().compare(test_dir) == 0); BOOST_CHECK(ji.base_filename().compare(base_filename) == 0); - timespec this_ts = ji.ts(); + const timespec this_ts = ji.ts(); BOOST_CHECK_EQUAL(this_ts.tv_sec, ts.tv_sec); BOOST_CHECK_EQUAL(this_ts.tv_nsec, ts.tv_nsec); - BOOST_CHECK_EQUAL(ji.num_jfiles(), (u_int16_t)NUM_JFILES); - BOOST_CHECK_EQUAL(ji.jfsize_sblks(), (u_int32_t)JFSIZE_SBLKS); - BOOST_CHECK_EQUAL(ji.sblk_size_dblks(), (u_int16_t)JRNL_SBLK_SIZE); - BOOST_CHECK_EQUAL(ji.dblk_size(), (u_int32_t)JRNL_DBLK_SIZE); + BOOST_CHECK_EQUAL(ji.num_jfiles(), u_int16_t(NUM_JFILES)); + BOOST_CHECK_EQUAL(ji.jfsize_sblks(), u_int32_t(JFSIZE_SBLKS)); + BOOST_CHECK_EQUAL(ji.sblk_size_dblks(), u_int16_t(JRNL_SBLK_SIZE)); + BOOST_CHECK_EQUAL(ji.dblk_size(), u_int32_t(JRNL_DBLK_SIZE)); + BOOST_CHECK_EQUAL(ji.wcache_pgsize_sblks(), u_int32_t(JRNL_WMGR_DEF_PAGE_SIZE)); + BOOST_CHECK_EQUAL(ji.wcache_num_pages(), u_int16_t(JRNL_WMGR_DEF_PAGES)); + BOOST_CHECK_EQUAL(ji.rcache_pgsize_sblks(), u_int32_t(JRNL_RMGR_PAGE_SIZE)); + BOOST_CHECK_EQUAL(ji.rcache_num_pages(), u_int16_t(JRNL_RMGR_PAGES)); ji.write(); - cout << "ok" << endl; + cout << "done" << endl; } QPID_AUTO_TEST_CASE(read_constructor) { cout << test_filename << ".read_constructor: " << flush; + lid_fid_map::create_new_jinf(); + stringstream fn; fn << test_dir << "/" << base_filename << "." << JRNL_INFO_EXTENSION; jinf ji(fn.str(), false); @@ -176,31 +545,63 @@ const timespec this_ts = ji.ts(); BOOST_CHECK_EQUAL(this_ts.tv_sec, ts.tv_sec); BOOST_CHECK_EQUAL(this_ts.tv_nsec, ts.tv_nsec); - BOOST_CHECK_EQUAL(ji.num_jfiles(), (u_int16_t)NUM_JFILES); - BOOST_CHECK_EQUAL(ji.jfsize_sblks(), (u_int32_t)JFSIZE_SBLKS); - BOOST_CHECK_EQUAL(ji.sblk_size_dblks(), (u_int16_t)JRNL_SBLK_SIZE); - BOOST_CHECK_EQUAL(ji.dblk_size(), (u_int32_t)JRNL_DBLK_SIZE); - cout << "ok" << endl; + BOOST_CHECK_EQUAL(ji.num_jfiles(), u_int16_t(NUM_JFILES)); + BOOST_CHECK_EQUAL(ji.jfsize_sblks(), u_int32_t(JFSIZE_SBLKS)); + BOOST_CHECK_EQUAL(ji.sblk_size_dblks(), u_int16_t(JRNL_SBLK_SIZE)); + BOOST_CHECK_EQUAL(ji.dblk_size(), u_int32_t(JRNL_DBLK_SIZE)); + BOOST_CHECK_EQUAL(ji.wcache_pgsize_sblks(), u_int32_t(JRNL_WMGR_DEF_PAGE_SIZE)); + BOOST_CHECK_EQUAL(ji.wcache_num_pages(), u_int16_t(JRNL_WMGR_DEF_PAGES)); + BOOST_CHECK_EQUAL(ji.rcache_pgsize_sblks(), u_int32_t(JRNL_RMGR_PAGE_SIZE)); + BOOST_CHECK_EQUAL(ji.rcache_num_pages(), u_int16_t(JRNL_RMGR_PAGES)); + + cout << "done" << endl; } +QPID_AUTO_TEST_CASE(set_functions) +{ + cout << test_filename << ".set_functions: " << flush; + lid_fid_map::create_new_jinf(); + + stringstream fn; + fn << test_dir << "/" << base_filename << "." << JRNL_INFO_EXTENSION; + jinf ji(fn.str(), false); + + ji.set_jdir("abc123"); + BOOST_CHECK(ji.jdir().compare("abc123") == 0); + ji.set_jdir(test_dir); + BOOST_CHECK(ji.jdir().compare(test_dir) == 0); + ji.incr_num_jfiles(); + BOOST_CHECK_EQUAL(ji.num_jfiles(), u_int16_t(NUM_JFILES+1)); + ji.incr_num_jfiles(); + BOOST_CHECK_EQUAL(ji.num_jfiles(), u_int16_t(NUM_JFILES+2)); + + lid_fid_map::clean_journal_info_file(); + cout << "done" << endl; +} + QPID_AUTO_TEST_CASE(validate) { cout << test_filename << ".validate: " << flush; + lid_fid_map::create_new_jinf(); + stringstream fn; fn << test_dir << "/" << base_filename << "." << JRNL_INFO_EXTENSION; jinf ji(fn.str(), true); // TODO: Check validation picks up conflict, but need to be friend to jinf to do it - cout << "ok" << endl; + + lid_fid_map::clean_journal_info_file(); + cout << "done" << endl; } - + QPID_AUTO_TEST_CASE(analyze_empty_journal) { cout << test_filename << ".analyze_empty_journal: " << flush; - vector<string> jfiles; - create_journal_filenames(jfiles); - - create_journal_files(jfiles, RID_NONE); + jdir::create_dir(test_dir); // Check test dir exists; create it if not + lid_fid_map m; + m.initial_journal_create(NUM_JFILES, 0, 0); + m.create_journal(); + stringstream fn; fn << test_dir << "/" << base_filename << "." << JRNL_INFO_EXTENSION; jinf ji(fn.str(), false); @@ -211,31 +612,140 @@ BOOST_ERROR("Failed to throw expected exception jerrno::JERR_JINF_JDATEMPTY"); } - clean_journal_files(jfiles); - cout << "ok" << endl; + m.destroy_journal(); + cout << "done" << endl; } -QPID_AUTO_TEST_CASE(analyze_linear_journal) +QPID_AUTO_TEST_CASE(analyze_part_full_journal) { - cout << test_filename << ".analyze_linear_journal: " << flush; - vector<string> jfiles; - for (int i=0; i<NUM_JFILES; i++) + cout << test_filename << ".analyze_part_full_journal: " << flush; + lid_fid_map m; + for (u_int16_t num_files = 1; num_files < NUM_JFILES; num_files++) { - create_journal_filenames(jfiles); - - create_journal_files(jfiles, RID_LINEAR, i, 0x12340000); + m.initial_journal_create(NUM_JFILES, num_files, 0); + m.create_journal(); stringstream fn; fn << test_dir << "/" << base_filename << "." << JRNL_INFO_EXTENSION; jinf ji(fn.str(), false); - BOOST_CHECK_EQUAL(ji.analyze(), i); + ji.analyze(); + m.check_analysis(ji); - clean_journal_files(jfiles); + m.destroy_journal(); } - - // last test cleans up jinf file - clean_journal_info_file(); - cout << "ok" << endl; + cout << "done" << endl; } +QPID_AUTO_TEST_CASE(analyze_full_journal) +{ + cout << test_filename << ".analyze_full_journal: " << flush; + lid_fid_map m; + for (u_int16_t file_num = 0; file_num < NUM_JFILES; file_num++) + { + m.initial_journal_create(NUM_JFILES, NUM_JFILES, file_num); + m.create_journal(); + + stringstream fn; + fn << test_dir << "/" << base_filename << "." << JRNL_INFO_EXTENSION; + jinf ji(fn.str(), false); + ji.analyze(); + m.check_analysis(ji); + + m.destroy_journal(); + } + cout << "done" << endl; +} + +QPID_AUTO_TEST_CASE(analyze_single_appended_journal) +{ + cout << test_filename << ".analyze_single_appended_journal: " << flush; + lid_fid_map m; + for (u_int16_t oldest_lid = 0; oldest_lid < NUM_JFILES; oldest_lid++) + for (u_int16_t after_lid = 0; after_lid < NUM_JFILES; after_lid++) + for (u_int16_t num_files = 1; num_files <= 5; num_files++) + { + m.initial_journal_create(NUM_JFILES, NUM_JFILES, oldest_lid); + m.journal_append(after_lid, num_files); + m.create_journal(); + + stringstream fn; + fn << test_dir << "/" << base_filename << "." << JRNL_INFO_EXTENSION; + jinf ji(fn.str(), false); + ji.analyze(); + m.check_analysis(ji); + + m.destroy_journal(); + } + cout << "done" << endl; +} + +QPID_AUTO_TEST_CASE(analyze_multi_appended_journal) +{ + cout << test_filename << ".analyze_multi_appended_journal: " << flush; + lid_fid_map m; + ::srand48(1); + + for (u_int16_t num_appends = 1; num_appends <= 2*NUM_JFILES; num_appends++) + { + const u_int16_t oldest_lid = u_int16_t(NUM_JFILES * ::drand48()); + m.initial_journal_create(NUM_JFILES, NUM_JFILES, oldest_lid); + for (u_int16_t a = 0; a < num_appends; a++) + { + const u_int16_t num_files = u_int16_t(1 + (NUM_JFILES * ::drand48())); + const u_int16_t after_lid = u_int16_t(m.size() * ::drand48()); + m.journal_append(after_lid, num_files); + } + m.create_journal(); + + stringstream fn; + fn << test_dir << "/" << base_filename << "." << JRNL_INFO_EXTENSION; + jinf ji(fn.str(), false); + ji.analyze(); + m.check_analysis(ji); + + m.destroy_journal(); + } + + cout << "done" << endl; +} + +QPID_AUTO_TEST_CASE(analyze_multi_appended_then_failed_journal) +{ + cout << test_filename << ".analyze_multi_appended_then_failed_journal: " << flush; + lid_fid_map m; + ::srand48(1); + + // As this test relies on repeatable but random sequences, use many ierations for coverage + for (int c = 1; c <= 100; c++) + { + for (u_int16_t num_appends = 1; num_appends <= 2*NUM_JFILES; num_appends++) + { + u_int16_t oldest_lid = u_int16_t(NUM_JFILES * ::drand48()); + m.initial_journal_create(NUM_JFILES, NUM_JFILES, oldest_lid); + for (u_int16_t a = 0; a < num_appends-1; a++) + { + const u_int16_t num_files = u_int16_t(1 + (NUM_JFILES * ::drand48())); + const u_int16_t after_lid = u_int16_t(m.size() * ::drand48()); + m.journal_append(after_lid, num_files); + if (after_lid < oldest_lid) + oldest_lid += num_files; + } + const u_int16_t num_files = u_int16_t(1 + (NUM_JFILES * ::drand48())); + const u_int16_t after_lid = oldest_lid == 0 ? m.size() - 1 : oldest_lid - 1; + m.journal_append(after_lid, num_files, false); + m.create_journal(); + + stringstream fn; + fn << test_dir << "/" << base_filename << "." << JRNL_INFO_EXTENSION; + jinf ji(fn.str(), false); + ji.analyze(); + m.check_analysis(ji); + + m.destroy_journal(); + } + } + + cout << "done" << endl; +} + QPID_AUTO_TEST_SUITE_END()