Author: kpvdr Date: 2008-10-17 13:51:24 -0400 (Fri, 17 Oct 2008) New Revision: 2649 Modified: store/trunk/cpp/lib/JournalImpl.cpp store/trunk/cpp/lib/jrnl/jcntl.cpp store/trunk/cpp/lib/jrnl/jcntl.hpp store/trunk/cpp/lib/jrnl/jerrno.cpp store/trunk/cpp/lib/jrnl/jerrno.hpp store/trunk/cpp/lib/jrnl/jinf.cpp store/trunk/cpp/lib/jrnl/jinf.hpp store/trunk/cpp/lib/jrnl/lfmgr.cpp store/trunk/cpp/lib/jrnl/lfmgr.hpp store/trunk/cpp/lib/jrnl/rcvdat.hpp store/trunk/cpp/lib/jrnl/rrfc.cpp store/trunk/cpp/lib/jrnl/wrfc.cpp store/trunk/cpp/tests/jrnl/_st_helper_fns.hpp store/trunk/cpp/tests/jrnl/_ut_jdir.cpp store/trunk/cpp/tests/jrnl/_ut_jinf.cpp store/trunk/cpp/tests/jrnl/_ut_lfmgr.cpp Log: Added auto-expand management functionality to class lfmgr. Added tests for all auto-expand functionality into _ut_lfmgr. Modified: store/trunk/cpp/lib/JournalImpl.cpp =================================================================== --- store/trunk/cpp/lib/JournalImpl.cpp 2008-10-17 17:28:47 UTC (rev 2648) +++ store/trunk/cpp/lib/JournalImpl.cpp 2008-10-17 17:51:24 UTC (rev 2649) @@ -144,9 +144,9 @@ if (_mgmtObject != 0) { - _mgmtObject->set_initialFileCount(_lfmgr.size()); + _mgmtObject->set_initialFileCount(_lfmgr.num_jfiles()); _mgmtObject->set_dataFileSize(_jfsize_sblks * JRNL_SBLK_SIZE * JRNL_DBLK_SIZE); - _mgmtObject->set_currentFileCount(_lfmgr.size()); + _mgmtObject->set_currentFileCount(_lfmgr.num_jfiles()); _mgmtObject->set_writePageSize(wcache_pgsize_sblks * JRNL_SBLK_SIZE * JRNL_DBLK_SIZE); _mgmtObject->set_writePages(wcache_num_pages); } @@ -172,9 +172,9 @@ if (_mgmtObject != 0) { - _mgmtObject->set_initialFileCount(_lfmgr.size()); + _mgmtObject->set_initialFileCount(_lfmgr.num_jfiles()); _mgmtObject->set_dataFileSize(_jfsize_sblks * JRNL_SBLK_SIZE * JRNL_DBLK_SIZE); - _mgmtObject->set_currentFileCount(_lfmgr.size()); + _mgmtObject->set_currentFileCount(_lfmgr.num_jfiles()); _mgmtObject->set_writePageSize(wcache_pgsize_sblks * JRNL_SBLK_SIZE * JRNL_DBLK_SIZE); _mgmtObject->set_writePages(wcache_num_pages); } Modified: store/trunk/cpp/lib/jrnl/jcntl.cpp =================================================================== --- store/trunk/cpp/lib/jrnl/jcntl.cpp 2008-10-17 17:28:47 UTC (rev 2648) +++ store/trunk/cpp/lib/jrnl/jcntl.cpp 2008-10-17 17:51:24 UTC (rev 2649) @@ -112,7 +112,7 @@ // Clear any existing journal files _jdir.clear_dir(); - _lfmgr.initialize(num_jfiles, this, &new_fcntl); + _lfmgr.initialize(num_jfiles, false, 0, this, &new_fcntl); _wrfc.initialize(_jfsize_sblks); _rrfc.initialize(); @@ -154,7 +154,7 @@ // Verify journal dir and journal files _jdir.verify_dir(); - _rcvdat.reset(num_jfiles); + _rcvdat.reset(num_jfiles, false, 0); rcvr_janalyze(_rcvdat, prep_txn_list_ptr); highest_rid = _rcvdat._h_rid; @@ -180,7 +180,7 @@ { if (!_readonly_flag) throw jexception(jerrno::JERR_JCNTL_NOTRECOVERED, "jcntl", "recover_complete"); - for (u_int16_t i=0; i<_lfmgr.size(); i++) + for (u_int16_t i=0; i<_lfmgr.num_jfiles(); i++) _lfmgr.get_fcntlp(i)->reset(&_rcvdat); _wrfc.initialize(_jfsize_sblks, &_rcvdat); _rrfc.initialize(); @@ -387,7 +387,7 @@ u_int16_t fid = _wrfc.index(); while ( _emap.get_enq_cnt(ffid) == 0 && _tmap.get_txn_fid_cnt(ffid) == 0 && ffid != fid) { - if (++ffid >= _lfmgr.size()) + if (++ffid >= _lfmgr.num_jfiles()) ffid = 0; } if (!_rrfc.is_active()) @@ -435,7 +435,7 @@ } fcntl* -jcntl::new_fcntl(jcntl* const jcp, const std::size_t lid, const std::size_t fid, const rcvdat* const rdp) +jcntl::new_fcntl(jcntl* const jcp, const u_int16_t lid, const u_int16_t fid, const rcvdat* const rdp) { if (!jcp) return 0; std::ostringstream oss; @@ -475,8 +475,8 @@ oss << FORMAT_SYSERR(errno); throw jexception(jerrno::JERR__RTCLOCK, oss.str(), "jcntl", "write_infofile"); } - jinf ji(_jid, _jdir.dirname(), _base_filename, _lfmgr.size(), _jfsize_sblks, - _wmgr.cache_pgsize_sblks(), _wmgr.cache_num_pages(), ts); + jinf ji(_jid, _jdir.dirname(), _base_filename, _lfmgr.num_jfiles(), _lfmgr.is_ae(), _lfmgr.ae_max_jfiles(), + _jfsize_sblks, _wmgr.cache_pgsize_sblks(), _wmgr.cache_num_pages(), ts); ji.write(); } @@ -550,14 +550,14 @@ // If the number of files does not tie up with the jinf file from the journal being recovered, // use the jinf data. - if (rd._num_jfiles != ji.num_jfiles()) + if (rd._njf != ji.num_jfiles()) { std::ostringstream oss; oss << "Recovery found " << ji.num_jfiles() << - " files (different from --num-jfiles value of " << rd._num_jfiles << ")."; + " files (different from --num-jfiles value of " << rd._njf << ")."; this->log(LOG_WARN, oss.str()); - rd._num_jfiles = ji.num_jfiles(); - _rcvdat._enq_cnt_list.resize(rd._num_jfiles); + rd._njf = ji.num_jfiles(); + _rcvdat._enq_cnt_list.resize(rd._njf); } if (_jfsize_sblks != ji.jfsize_sblks()) { @@ -627,7 +627,7 @@ rd._lffull = rd._eo == (1 + _jfsize_sblks) * JRNL_SBLK_SIZE * JRNL_DBLK_SIZE; // Check for journal full condition - u_int16_t next_wr_fid = (rd._lfid + 1) % rd._num_jfiles; + u_int16_t next_wr_fid = (rd._lfid + 1) % rd._njf; rd._jfull = rd._ffid == next_wr_fid && rd._enq_cnt_list[next_wr_fid]; } } @@ -851,7 +851,7 @@ rd._eo = ifsp->tellg(); // remember file offset before closing assert(rd._eo != std::numeric_limits<std::size_t>::max()); // Check for error code -1 ifsp->close(); - if (++fid >= rd._num_jfiles) + if (++fid >= rd._njf) { fid = 0; lowi = !lowi; // Flip local owi @@ -896,7 +896,7 @@ { if (rd._ffid ? h.get_owi() == lowi : h.get_owi() != lowi) // Overwrite indicator changed { - u_int16_t expected_fid = rd._ffid ? rd._ffid - 1 : rd._num_jfiles - 1; + u_int16_t expected_fid = rd._ffid ? rd._ffid - 1 : rd._njf - 1; if (fid == expected_fid) { check_journal_alignment(fid, file_pos); Modified: store/trunk/cpp/lib/jrnl/jcntl.hpp =================================================================== --- store/trunk/cpp/lib/jrnl/jcntl.hpp 2008-10-17 17:28:47 UTC (rev 2648) +++ store/trunk/cpp/lib/jrnl/jcntl.hpp 2008-10-17 17:51:24 UTC (rev 2649) @@ -616,7 +616,7 @@ */ inline const std::string& base_filename() const { return _base_filename; } - inline u_int16_t num_jfiles() const { return _lfmgr.size(); } + inline u_int16_t num_jfiles() const { return _lfmgr.num_jfiles(); } inline fcntl* get_fcntlp(const u_int16_t fid) const { return _lfmgr.get_fcntlp(fid); } @@ -637,8 +637,7 @@ /** * /brief Static function for creating new fcntl objects for use with obj_arr. */ - static fcntl* new_fcntl(jcntl* const jcp, const std::size_t lid, const std::size_t fid, - const rcvdat* const rdp); + static fcntl* new_fcntl(jcntl* const jcp, const u_int16_t lid, const u_int16_t fid, const rcvdat* const rdp); protected: /** Modified: store/trunk/cpp/lib/jrnl/jerrno.cpp =================================================================== --- store/trunk/cpp/lib/jrnl/jerrno.cpp 2008-10-17 17:28:47 UTC (rev 2648) +++ store/trunk/cpp/lib/jrnl/jerrno.cpp 2008-10-17 17:51:24 UTC (rev 2649) @@ -82,12 +82,17 @@ const u_int32_t jerrno::JERR_FCNTL_CMPLOFFSOVFL = 0x0404; const u_int32_t jerrno::JERR_FCNTL_RDOFFSOVFL = 0x0405; +// class lfmgr +const u_int32_t jerrno::JERR_LFMGR_BADAEFNUMLIM = 0x0500; +const u_int32_t jerrno::JERR_LFMGR_AEFNUMLIMIT = 0x0501; +const u_int32_t jerrno::JERR_LFMGR_AEDISABLED = 0x0502; + // class rrfc -const u_int32_t jerrno::JERR_RRFC_OPENRD = 0x0500; +const u_int32_t jerrno::JERR_RRFC_OPENRD = 0x0600; // class jrec, enq_rec, deq_rec, txn_rec -const u_int32_t jerrno::JERR_JREC_BADRECHDR = 0x0600; -const u_int32_t jerrno::JERR_JREC_BADRECTAIL = 0x0601; +const u_int32_t jerrno::JERR_JREC_BADRECHDR = 0x0700; +const u_int32_t jerrno::JERR_JREC_BADRECTAIL = 0x0701; // class wmgr const u_int32_t jerrno::JERR_WMGR_BADPGSTATE = 0x0801; @@ -136,17 +141,12 @@ // class jcntl _err_map[JERR_JCNTL_STOPPED] = "JERR_JCNTL_STOPPED: Operation on stopped journal."; - _err_map[JERR_JCNTL_READONLY] = "JERR_JCNTL_READONLY: " - "Write operation on read-only journal (during recovery)."; - _err_map[JERR_JCNTL_AIOCMPLWAIT] = "JERR_JCNTL_AIOCMPLWAIT: " - "Timeout waiting for AIOs to complete."; + _err_map[JERR_JCNTL_READONLY] = "JERR_JCNTL_READONLY: Write operation on read-only journal (during recovery)."; + _err_map[JERR_JCNTL_AIOCMPLWAIT] = "JERR_JCNTL_AIOCMPLWAIT: Timeout waiting for AIOs to complete."; _err_map[JERR_JCNTL_UNKNOWNMAGIC] = "JERR_JCNTL_UNKNOWNMAGIC: Found record with unknown magic."; - _err_map[JERR_JCNTL_NOTRECOVERED] = "JERR_JCNTL_NOTRECOVERED: " - "Operation requires recover() to be run first."; - _err_map[JERR_JCNTL_RECOVERJFULL] = "JERR_JCNTL_RECOVERJFULL: " - "Journal data files full, cannot write."; - _err_map[JERR_JCNTL_OWIMISMATCH] = "JERR_JCNTL_OWIMISMATCH: " - "Overwrite Indecator (OWI) change found in unexpected location."; + _err_map[JERR_JCNTL_NOTRECOVERED] = "JERR_JCNTL_NOTRECOVERED: Operation requires recover() to be run first."; + _err_map[JERR_JCNTL_RECOVERJFULL] = "JERR_JCNTL_RECOVERJFULL: Journal data files full, cannot write."; + _err_map[JERR_JCNTL_OWIMISMATCH] = "JERR_JCNTL_OWIMISMATCH: Overwrite Indecator (OWI) change found in unexpected location."; // class jdir _err_map[JERR_JDIR_NOTDIR] = "JERR_JDIR_NOTDIR: Directory name exists but is not a directory."; @@ -165,13 +165,14 @@ _err_map[JERR_FCNTL_OPENWR] = "JERR_FCNTL_OPENWR: Unable to open file for write."; _err_map[JERR_FCNTL_WRITE] = "JERR_FCNTL_WRITE: Unable to write to file."; _err_map[JERR_FCNTL_CLOSE] = "JERR_FCNTL_CLOSE: File close failed."; - _err_map[JERR_FCNTL_FILEOFFSOVFL] = "JERR_FCNTL_FILEOFFSOVFL: " - "Attempted increase file offset past file size."; - _err_map[JERR_FCNTL_CMPLOFFSOVFL] = "JERR_FCNTL_CMPLOFFSOVFL: " - "Attempted increase completed file offset past submitted offset."; - _err_map[JERR_FCNTL_RDOFFSOVFL] = "JERR_FCNTL_RDOFFSOVFL: " - "Attempted increase read offset past write offset."; + _err_map[JERR_FCNTL_FILEOFFSOVFL] = "JERR_FCNTL_FILEOFFSOVFL: Attempted increase file offset past file size."; + _err_map[JERR_FCNTL_CMPLOFFSOVFL] = "JERR_FCNTL_CMPLOFFSOVFL: Attempted increase completed file offset past submitted offset."; + _err_map[JERR_FCNTL_RDOFFSOVFL] = "JERR_FCNTL_RDOFFSOVFL: Attempted increase read offset past write offset."; + // class lfmgr + _err_map[JERR_LFMGR_BADAEFNUMLIM] = "JERR_LFMGR_BADAEFNUMLIM: Bad auto-expand file number limit."; + _err_map[JERR_LFMGR_AEFNUMLIMIT] = "JERR_LFMGR_AEFNUMLIMIT: Exceeded auto-expand file number limit."; + _err_map[JERR_LFMGR_AEDISABLED] = "JERR_LFMGR_AEDISABLED: Attempted to expand with auto-expand disabled."; // class rrfc _err_map[JERR_RRFC_OPENRD] = "JERR_RRFC_OPENRD: Unable to open file for read."; @@ -181,43 +182,32 @@ _err_map[JERR_JREC_BADRECTAIL] = "JERR_JREC_BADRECTAIL: Invalid data record tail."; // class wmgr - _err_map[JERR_WMGR_BADPGSTATE] = "JERR_WMGR_BADPGSTATE: " - "Page buffer in illegal state for operation."; - _err_map[JERR_WMGR_BADDTOKSTATE] = "JERR_WMGR_BADDTOKSTATE: " - "Data token in illegal state for operation."; - _err_map[JERR_WMGR_ENQDISCONT] = "JERR_WMGR_ENQDISCONT: " - "Enqueued new dtok when previous enqueue returned partly completed (state ENQ_PART)."; - _err_map[JERR_WMGR_DEQDISCONT] = "JERR_WMGR_DEQDISCONT: " - "Dequeued new dtok when previous dequeue returned partly completed (state DEQ_PART)."; + _err_map[JERR_WMGR_BADPGSTATE] = "JERR_WMGR_BADPGSTATE: Page buffer in illegal state for operation."; + _err_map[JERR_WMGR_BADDTOKSTATE] = "JERR_WMGR_BADDTOKSTATE: Data token in illegal state for operation."; + _err_map[JERR_WMGR_ENQDISCONT] = "JERR_WMGR_ENQDISCONT: Enqueued new dtok when previous enqueue returned partly completed (state ENQ_PART)."; + _err_map[JERR_WMGR_DEQDISCONT] = "JERR_WMGR_DEQDISCONT: Dequeued new dtok when previous dequeue returned partly completed (state DEQ_PART)."; _err_map[JERR_WMGR_DEQRIDNOTENQ] = "JERR_WMGR_DEQRIDNOTENQ: Dequeue rid is not enqueued."; // class rmgr _err_map[JERR_RMGR_UNKNOWNMAGIC] = "JERR_RMGR_UNKNOWNMAGIC: Found record with unknown magic."; - _err_map[JERR_RMGR_RIDMISMATCH] = "JERR_RMGR_RIDMISMATCH: " - "RID mismatch between current record and dtok RID"; + _err_map[JERR_RMGR_RIDMISMATCH] = "JERR_RMGR_RIDMISMATCH: RID mismatch between current record and dtok RID"; //_err_map[JERR_RMGR_FIDMISMATCH] = "JERR_RMGR_FIDMISMATCH: FID mismatch between emap and rrfc"; - _err_map[JERR_RMGR_ENQSTATE] = "JERR_RMGR_ENQSTATE: " - "Attempted read when data token wstate was not ENQ"; - _err_map[JERR_RMGR_BADRECTYPE] = "JERR_RMGR_BADRECTYPE: " - "Attempted operation on inappropriate record type"; + _err_map[JERR_RMGR_ENQSTATE] = "JERR_RMGR_ENQSTATE: Attempted read when data token wstate was not ENQ"; + _err_map[JERR_RMGR_BADRECTYPE] = "JERR_RMGR_BADRECTYPE: Attempted operation on inappropriate record type"; // class data_tok - _err_map[JERR_DTOK_ILLEGALSTATE] = "JERR_MTOK_ILLEGALSTATE: " - "Attempted to change to illegal state."; + _err_map[JERR_DTOK_ILLEGALSTATE] = "JERR_MTOK_ILLEGALSTATE: Attempted to change to illegal state."; //_err_map[JERR_DTOK_RIDNOTSET] = "JERR_DTOK_RIDNOTSET: Record ID not set."; // class enq_map, txn_map - _err_map[JERR_MAP_DUPLICATE] = "JERR_MAP_DUPLICATE: " - "Attempted to insert record into map using duplicate key."; + _err_map[JERR_MAP_DUPLICATE] = "JERR_MAP_DUPLICATE: Attempted to insert record into map using duplicate key."; _err_map[JERR_MAP_NOTFOUND] = "JERR_MAP_NOTFOUND: Key not found in map."; _err_map[JERR_MAP_LOCKED] = "JERR_MAP_LOCKED: Record ID locked by a pending transaction."; // class jinf - _err_map[JERR_JINF_CVALIDFAIL] = "JERR_JINF_CVALIDFAIL: " - "Journal compatibility validation failure."; + _err_map[JERR_JINF_CVALIDFAIL] = "JERR_JINF_CVALIDFAIL: Journal compatibility validation failure."; _err_map[JERR_JINF_NOVALUESTR] = "JERR_JINF_NOVALUESTR: No value attribute found in jinf file."; - _err_map[JERR_JINF_BADVALUESTR] = "JERR_JINF_BADVALUESTR: " - "Bad format for value attribute in jinf file"; + _err_map[JERR_JINF_BADVALUESTR] = "JERR_JINF_BADVALUESTR: Bad format for value attribute in jinf file"; _err_map[JERR_JINF_JDATEMPTY] = "JERR_JINF_JDATEMPTY: Journal data files empty."; _err_map[JERR_JINF_TOOMANYFILES] = "JERR_JINF_TOOMANYFILES: Too many journal data files."; Modified: store/trunk/cpp/lib/jrnl/jerrno.hpp =================================================================== --- store/trunk/cpp/lib/jrnl/jerrno.hpp 2008-10-17 17:28:47 UTC (rev 2648) +++ store/trunk/cpp/lib/jrnl/jerrno.hpp 2008-10-17 17:51:24 UTC (rev 2649) @@ -100,6 +100,11 @@ static const u_int32_t JERR_FCNTL_CMPLOFFSOVFL; ///< Increased cmpl offs past subm offs static const u_int32_t JERR_FCNTL_RDOFFSOVFL; ///< Increased read offs past write offs + // class lfmgr + static const u_int32_t JERR_LFMGR_BADAEFNUMLIM; ///< Bad auto-expand file number limit + static const u_int32_t JERR_LFMGR_AEFNUMLIMIT; ///< Exceeded auto-expand file number limit + static const u_int32_t JERR_LFMGR_AEDISABLED; ///< Attempted to expand with auto-expand disabled + // class rrfc static const u_int32_t JERR_RRFC_OPENRD; ///< Unable to open file for read Modified: store/trunk/cpp/lib/jrnl/jinf.cpp =================================================================== --- store/trunk/cpp/lib/jrnl/jinf.cpp 2008-10-17 17:28:47 UTC (rev 2648) +++ store/trunk/cpp/lib/jrnl/jinf.cpp 2008-10-17 17:51:24 UTC (rev 2649) @@ -49,6 +49,8 @@ jinf::jinf(const std::string& jinf_filename, bool validate_flag): _jver(0), _num_jfiles(0), + _ae(false), + _ae_max_jfiles(0), _jfsize_sblks(0), _sblk_size_dblks(0), _dblk_size(0), @@ -67,8 +69,8 @@ validate(); } -jinf::jinf(const std::string& jid, const std::string& jdir, const std::string& base_filename, - const u_int16_t num_jfiles, const u_int32_t jfsize_sblks, +jinf::jinf(const std::string& jid, const std::string& jdir, const std::string& base_filename, const u_int16_t num_jfiles, + const bool auto_expand, const u_int16_t ae_max_jfiles, const u_int32_t jfsize_sblks, const u_int32_t wcache_pgsize_sblks, const u_int16_t wcache_num_pages, const timespec& ts): _jver(RHM_JDAT_VERSION), _jid(jid), @@ -76,6 +78,8 @@ _base_filename(base_filename), _ts(ts), _num_jfiles(num_jfiles), + _ae(auto_expand), + _ae_max_jfiles(ae_max_jfiles), _jfsize_sblks(jfsize_sblks), _sblk_size_dblks(JRNL_SBLK_SIZE), _dblk_size(JRNL_DBLK_SIZE), @@ -115,6 +119,21 @@ oss << "; maximum=" << JRNL_MAX_NUM_FILES << std::endl; err = true; } + if (_ae) + { + if (_ae_max_jfiles > _num_jfiles) + { + oss << "Number of journal files exceeds auto-expansion limit: found=" << _num_jfiles; + oss << "; maximum=" << _ae_max_jfiles; + err = true; + } + if (_ae_max_jfiles > JRNL_MAX_NUM_FILES) + { + oss << "Auto-expansion file limit too large: found=" << _ae_max_jfiles; + oss << "; maximum=" << JRNL_MAX_NUM_FILES; + err = true; + } + } if (_jfsize_sblks < JRNL_MIN_FILE_SIZE) { oss << "Journal file size too small: found=" << _jfsize_sblks; @@ -271,7 +290,7 @@ return _frot; } -const std::string +std::string jinf::to_string() const { std::ostringstream oss; @@ -284,6 +303,8 @@ oss << " Journal base filename: \"" << _base_filename << "\"" << std::endl; oss << " Journal version: " << (unsigned)_jver << std::endl; oss << " Number of journal files: " << _num_jfiles << std::endl; + oss << " Auto-expand mode: " << (_ae ? "enabled" : "disabled") << std::endl; + if (_ae) oss << " Max. number of journal files (in auto-expand mode): " << _ae_max_jfiles << std::endl; oss << " Journal file size: " << _jfsize_sblks << " sblks" << std::endl; oss << " Softblock size (JRNL_SBLK_SIZE): " << _sblk_size_dblks << " dblks" << std::endl; oss << " Datablock size (JRNL_DBLK_SIZE): " << _dblk_size << " bytes" << std::endl; @@ -294,7 +315,7 @@ return oss.str(); } -const std::string +std::string jinf::xml_str() const { // TODO: This is *not* an XML writer, rather for simplicity, it uses literals. I'm sure a more elegant way can be @@ -321,6 +342,8 @@ oss << " </creation_time>" << std::endl; oss << " <journal_file_geometry>" << std::endl; oss << " <number_jrnl_files value=\"" << _num_jfiles << "\" />" << std::endl; + oss << " <auto_expand value=\"" << (_ae ? "true" : "false") << "\" />" << std::endl; + if (_ae) oss << " <auto_expand_max_number_jrnl_files value=\"" << _ae_max_jfiles << "\" />" << std::endl; oss << " <jrnl_file_size_sblks value=\"" << _jfsize_sblks << "\" />" << std::endl; oss << " <JRNL_SBLK_SIZE value=\"" << _sblk_size_dblks << "\" />" << std::endl; oss << " <JRNL_DBLK_SIZE value=\"" << _dblk_size << "\" />" << std::endl; @@ -339,7 +362,8 @@ jinf::read(const std::string& jinf_filename) { // TODO: This is *not* an XML reader, rather for simplicity, it is a brute-force line reader which relies on string - // recognition. Can it be replaced cheaply by the real thing? + // recognition. It relies on the format of xml_str() above; it will not handle a XML restructuring. + // *** Can it be replaced cheaply by a real XML reader? Should it be, or is this sufficient? *** char buff[1024]; // limit of line input length std::ifstream jinfs(jinf_filename.c_str()); @@ -358,6 +382,10 @@ string_value(_base_filename, buff); else if(std::strstr(buff, "number_jrnl_files")) _num_jfiles = u_int16_value(buff); + else if(std::strstr(buff, "auto_expand")) + _ae = bool_value(buff); + else if(std::strstr(buff, "auto_expand_max_number_jrnl_files")) + _ae_max_jfiles = u_int16_value(buff); else if(std::strstr(buff, "jrnl_file_size_sblks")) _jfsize_sblks = u_int32_value(buff); else if(std::strstr(buff, "JRNL_SBLK_SIZE")) @@ -383,6 +411,12 @@ jinfs.close(); } +bool +jinf::bool_value(char* line) const +{ + return std::strcmp(find_value(line), "true") == 0; +} + u_int16_t jinf::u_int16_value(char* line) const { @@ -395,14 +429,14 @@ return std::atol(find_value(line)); } -const std::string& +std::string& jinf::string_value(std::string& str, char* line) const { str.assign(find_value(line)); return str; } -const char* +char* jinf::find_value(char* line) const { const char* target1_str = "value=\""; Modified: store/trunk/cpp/lib/jrnl/jinf.hpp =================================================================== --- store/trunk/cpp/lib/jrnl/jinf.hpp 2008-10-17 17:28:47 UTC (rev 2648) +++ store/trunk/cpp/lib/jrnl/jinf.hpp 2008-10-17 17:51:24 UTC (rev 2649) @@ -58,6 +58,8 @@ std::string _base_filename; timespec _ts; u_int16_t _num_jfiles; + bool _ae; + u_int32_t _ae_max_jfiles; u_int32_t _jfsize_sblks; u_int16_t _sblk_size_dblks; u_int32_t _dblk_size; @@ -77,8 +79,8 @@ jinf(const std::string& jinf_filename, bool validate_flag); // constructor for writing jinf file jinf(const std::string& jid, const std::string& jdir, const std::string& base_filename, - const u_int16_t num_jfiles, const u_int32_t jfsize_sblks, - const u_int32_t wcache_pgsize_sblks, const u_int16_t wcache_num_pages, + const u_int16_t num_jfiles, const bool auto_expand, const u_int16_t ae_max_jfiles, + const u_int32_t jfsize_sblks, const u_int32_t wcache_pgsize_sblks, const u_int16_t wcache_num_pages, const timespec& ts); virtual ~jinf(); @@ -94,6 +96,8 @@ inline const timespec& ts() const { return _ts; } inline u_int16_t num_jfiles() const { return _num_jfiles; } u_int16_t incr_num_jfiles(); + inline bool is_ae() const { return _ae; } + inline u_int16_t ae_max_jfiles() const { return _ae_max_jfiles; } inline u_int32_t jfsize_sblks() const { return _jfsize_sblks; } inline u_int16_t sblk_size_dblks() const { return _sblk_size_dblks; } inline u_int32_t dblk_size() const { return _dblk_size; } @@ -108,15 +112,16 @@ bool get_initial_owi(); bool get_frot(); - const std::string to_string() const; - const std::string xml_str() const; + std::string to_string() const; + std::string xml_str() const; private: void read(const std::string& jinf_filename); + bool bool_value(char* line) const; u_int16_t u_int16_value(char* line) const; u_int32_t u_int32_value(char* line) const; - const std::string& string_value(std::string& str, char* line) const; - const char* find_value(char* line) const; + std::string& string_value(std::string& str, char* line) const; + char* find_value(char* line) const; }; } // namespace journal Modified: store/trunk/cpp/lib/jrnl/lfmgr.cpp =================================================================== --- store/trunk/cpp/lib/jrnl/lfmgr.cpp 2008-10-17 17:28:47 UTC (rev 2648) +++ store/trunk/cpp/lib/jrnl/lfmgr.cpp 2008-10-17 17:51:24 UTC (rev 2649) @@ -32,13 +32,15 @@ #include "jrnl/lfmgr.hpp" #include <cassert> +#include <jrnl/jerrno.hpp> +#include <jrnl/jexception.hpp> namespace mrg { namespace journal { -lfmgr::lfmgr() +lfmgr::lfmgr() : _ae(false), _ae_max_jfiles(0) {} lfmgr::~lfmgr() @@ -47,85 +49,140 @@ } void -lfmgr::initialize(const std::size_t num_jfiles, +lfmgr::initialize(const u_int16_t num_jfiles, + const bool ae, + const u_int16_t ae_max_jfiles, jcntl* const jcp, - fcntl* (new_obj_fn)(jcntl* const jcp, - const std::size_t lid, - const std::size_t fid, - const rcvdat* const rdp)) + new_obj_fn_ptr fp) { + assert(jcp != 0); finalize(); - if (num_jfiles > 0) + + // Validate params + if (ae && ae_max_jfiles > 0 && ae_max_jfiles <= num_jfiles) { - _fcntl_arr.reserve(2 * num_jfiles); // reserve some extra for possible add_obj() operations - append(jcp, new_obj_fn, num_jfiles); + std::ostringstream oss; + oss << "_ae_max_jfiles=" << _ae_max_jfiles << "; num_jfiles=" << num_jfiles; + throw jexception(jerrno::JERR_LFMGR_BADAEFNUMLIM, oss.str(), "lfmgr", "initialize"); } + _ae = ae; + _ae_max_jfiles = ae_max_jfiles; + + const std::size_t num_res_files = ae + ? (ae_max_jfiles ? ae_max_jfiles : JRNL_MAX_NUM_FILES) + : num_jfiles; + _fcntl_arr.reserve(num_res_files); + append(jcp, fp, num_jfiles); } void lfmgr::recover(const rcvdat& rd, jcntl* const jcp, - fcntl* (new_obj_fn)(jcntl* const jcp, - const std::size_t lid, - const std::size_t fid, - const rcvdat* const rdp)) + new_obj_fn_ptr fp) { + assert(jcp != 0); finalize(); - _fcntl_arr.reserve(2 * rd._num_jfiles); // reserve some extra for possible add_obj() operations - _fcntl_arr.assign(rd._num_jfiles, 0); + + // Validate rd params + if (rd._aemjf > 0 && rd._aemjf <= rd._njf) + { + std::ostringstream oss; + oss << "_ae_max_jfiles=" << rd._aemjf << "; num_jfiles=" <<rd._njf ; + throw jexception(jerrno::JERR_LFMGR_BADAEFNUMLIM, oss.str(), "lfmgr", "recover"); + } + _ae = rd._ae; + _ae_max_jfiles = rd._aemjf; + + const std::size_t num_res_files = rd._ae + ? (rd._aemjf ? rd._aemjf : JRNL_MAX_NUM_FILES) + : rd._njf; + _fcntl_arr.reserve(num_res_files); + _fcntl_arr.assign(rd._njf, 0); std::vector<u_int16_t> lid_list(rd._fid_list.size(), 0); for (std::size_t lid = 0; lid < rd._fid_list.size(); lid++) lid_list[rd._fid_list[lid]] = lid; - // NOTE: rd._fid_list may be smaller than rd._num_jfiles (journal may be empty or not yet file-cycled) - for (std::size_t fid = 0; fid < rd._num_jfiles; fid++) + // NOTE: rd._fid_list may be smaller than rd._njf (journal may be empty or not yet file-cycled) + for (std::size_t fid = 0; fid < rd._njf; fid++) if (fid < rd._fid_list.size()) - _fcntl_arr[lid_list[fid]] = new_obj_fn(jcp, lid_list[fid], fid, &rd); + _fcntl_arr[lid_list[fid]] = fp(jcp, lid_list[fid], fid, &rd); else - _fcntl_arr[fid] = new_obj_fn(jcp, fid, fid, &rd); + _fcntl_arr[fid] = fp(jcp, fid, fid, &rd); } void -lfmgr::append(jcntl* const jcp, - fcntl* (new_obj_fn)(jcntl* const jcp, - const std::size_t lid, - const std::size_t fid, - const rcvdat* const rdp), - const std::size_t num_jfiles) -{ - std::size_t s = _fcntl_arr.size(); - for (std::size_t i = s; i < s + num_jfiles; i++) - _fcntl_arr.push_back(new_obj_fn(jcp, i, i, 0)); -} - -void -lfmgr::insert(const std::size_t after_index, +lfmgr::insert(const u_int16_t after_index, jcntl* const jcp, - fcntl* (new_obj_fn)(jcntl* const jcp, - const std::size_t lid, - const std::size_t fid, - const rcvdat* const rdp), - const std::size_t num_jfiles) + new_obj_fn_ptr fp, + const u_int16_t num_jfiles) { + assert(jcp != 0); assert(after_index < _fcntl_arr.size()); + if (!_ae) throw jexception(jerrno::JERR_LFMGR_AEDISABLED, "lfmgr", "insert"); + if (num_jfiles == 0) return; std::size_t fid = _fcntl_arr.size(); + const u_int16_t eff_ae_max_jfiles = _ae_max_jfiles ? _ae_max_jfiles : JRNL_MAX_NUM_FILES; + if (fid + num_jfiles > eff_ae_max_jfiles) + { + std::ostringstream oss; + oss << "num_files=" << fid << " incr=" << num_jfiles << " limit=" << _ae_max_jfiles; + throw jexception(jerrno::JERR_LFMGR_AEFNUMLIMIT, oss.str(), "lfmgr", "insert"); + } for (std::size_t lid = after_index + 1; lid <= after_index + num_jfiles; lid++, fid++) - _fcntl_arr.insert(_fcntl_arr.begin() + lid, new_obj_fn(jcp, lid, fid, 0)); + _fcntl_arr.insert(_fcntl_arr.begin() + lid, fp(jcp, lid, fid, 0)); for (std::size_t lid = after_index + num_jfiles + 1; lid < _fcntl_arr.size(); lid++) { - fcntl* fp = _fcntl_arr[lid]; - assert(fp != 0); - fp->set_lid(fp->lid() + num_jfiles); + fcntl* p = _fcntl_arr[lid]; + assert(p != 0); + p->set_lid(p->lid() + num_jfiles); } } void lfmgr::finalize() { - for (u_int32_t i=0; i<_fcntl_arr.size(); i++) + for (u_int32_t i = 0; i < _fcntl_arr.size(); i++) delete _fcntl_arr[i]; _fcntl_arr.clear(); + _ae = false; + _ae_max_jfiles = 0; } +void +lfmgr::set_ae(const bool ae) +{ + if (ae && _ae_max_jfiles > 0 && _ae_max_jfiles <= _fcntl_arr.size()) + { + std::ostringstream oss; + oss << "_ae_max_jfiles=" << _ae_max_jfiles << "; _fcntl_arr.size()=" << _fcntl_arr.size(); + throw jexception(jerrno::JERR_LFMGR_BADAEFNUMLIM, oss.str(), "lfmgr", "set_ae"); + } + if (ae && _fcntl_arr.max_size() < _ae_max_jfiles) + _fcntl_arr.reserve(_ae_max_jfiles ? _ae_max_jfiles : JRNL_MAX_NUM_FILES); + _ae = ae; +} + +void +lfmgr::set_ae_max_jfiles(const u_int16_t ae_max_jfiles) +{ + if (_ae && ae_max_jfiles > 0 && ae_max_jfiles <= _fcntl_arr.size()) + { + std::ostringstream oss; + oss << "_ae_max_jfiles=" << _ae_max_jfiles << "; _fcntl_arr.size()=" << _fcntl_arr.size(); + throw jexception(jerrno::JERR_LFMGR_BADAEFNUMLIM, oss.str(), "lfmgr", "set_ae_max_jfiles"); + } + if (_ae && _fcntl_arr.max_size() < ae_max_jfiles) + _fcntl_arr.reserve(ae_max_jfiles ? ae_max_jfiles : JRNL_MAX_NUM_FILES); + _ae_max_jfiles = ae_max_jfiles; +} + +u_int16_t +lfmgr::ae_jfiles_rem() const +{ + if (_ae_max_jfiles > _fcntl_arr.size()) return _ae_max_jfiles - _fcntl_arr.size(); + if (_ae_max_jfiles == 0) return JRNL_MAX_NUM_FILES - _fcntl_arr.size(); + return 0; +} + // Testing functions void @@ -145,5 +202,23 @@ lid_list[_fcntl_arr[i]->fid()] = i; } +// === protected fns === + +void +lfmgr::append(jcntl* const jcp, + new_obj_fn_ptr fp, + const u_int16_t num_jfiles) +{ + std::size_t s = _fcntl_arr.size(); + if (_ae_max_jfiles && s + num_jfiles > _ae_max_jfiles) + { + std::ostringstream oss; + oss << "num_files=" << s << " incr=" << num_jfiles << " limit=" << _ae_max_jfiles; + throw jexception(jerrno::JERR_LFMGR_AEFNUMLIMIT, oss.str(), "lfmgr", "append"); + } + for (std::size_t i = s; i < s + num_jfiles; i++) + _fcntl_arr.push_back(fp(jcp, i, i, 0)); +} + } // namespace journal } // namespace mrg Modified: store/trunk/cpp/lib/jrnl/lfmgr.hpp =================================================================== --- store/trunk/cpp/lib/jrnl/lfmgr.hpp 2008-10-17 17:28:47 UTC (rev 2648) +++ store/trunk/cpp/lib/jrnl/lfmgr.hpp 2008-10-17 17:51:24 UTC (rev 2649) @@ -113,16 +113,35 @@ * +---+---+---+---+---+---+---+---+ +---+---+---+---+---+---+---+---+ * </pre> * - * There are two independent tests that may be made to resolve duplicat lids during recovery in such cases: + * There are two independent tests that may be made to resolve duplicate lids during recovery in such cases: * <ol> * <li>The correct lid has owi flag that matches that of fid/lid 0</li> * <li>The most recently inserted (hence correct) lid has fids that are higher than the duplicat that was not * overwritten</li> * </ol> + * + * NOTE: NOT THREAD SAFE. Provide external thread protection if used in multi-threaded environments. */ class lfmgr { - protected: + public: + /** + * \brief Function pointer to function that will create a new fcntl object and return its pointer. + * + * \param jcp Pointer to jcntl instance from which journal file details will be obtained. + * \param lid Logical ID for new fcntl instance. + * \param fid Physical file ID for file associated with new fcntl instance. + * \param rdp Pointer to rcvdat instance which conatins recovery information for new fcntl instance when + * recovering an existing file, or null if a new file is to be created. + */ + typedef fcntl* (new_obj_fn_ptr)(jcntl* const jcp, + const u_int16_t lid, + const u_int16_t fid, + const rcvdat* const rdp); + + private: + bool _ae; ///< Auto-expand mode + u_int16_t _ae_max_jfiles; ///< Max file count for auto-expansion; 0 = no limit std::vector<fcntl*> _fcntl_arr; ///< Array of pointers to objects public: @@ -135,17 +154,20 @@ * * \param num_jfiles Number of files to be created, and consequently the number of fcntl objects in array * _fcntl_arr. + * \param ae If true, allows auto-expansion; if false, disables auto-expansion. + * \param ae_max_jfiles The maximum number of files allowed for auto-expansion. Cannot be lower than the current + * number of files. However, a zero value disables the limit checks, and allows unlimited + * expansion. * \param jcp Pointer to jcntl instance. This is used to find the file path and base filename so that * new files may be created. - * \param new_obj_fn Pointer to function which creates and returns a pointer to a new fcntl object (and hence + * \param fp Pointer to function which creates and returns a pointer to a new fcntl object (and hence * causes a new %journal file to be created). */ - void initialize(const std::size_t num_jfiles, + void initialize(const u_int16_t num_jfiles, + const bool ae, + const u_int16_t ae_max_jfiles, jcntl* const jcp, - fcntl* (new_obj_fn)(jcntl* const jcp, - const std::size_t lid, - const std::size_t fid, - const rcvdat* const rdp)); + new_obj_fn_ptr fp); /** * \brief Initialize from a known lid-fid map fid_list, which is usually obtained from a recover. The index of @@ -154,34 +176,14 @@ * \param rd Ref to rcvdat struct which contains recovery data. * \param jcp Pointer to jcntl instance. This is used to find the file path and base filename so that * new files may be created. - * \param new_obj_fn Pointer to function which creates and returns a pointer to a new fcntl object (and hence + * \param fp Pointer to function which creates and returns a pointer to a new fcntl object (and hence * causes a new %journal file to be created). */ void recover(const rcvdat& rd, jcntl* const jcp, - fcntl* (new_obj_fn)(jcntl* const jcp, - const std::size_t lid, - const std::size_t fid, - const rcvdat* const rdp)); + new_obj_fn_ptr fp); /** - * \brief Append num_jfiles files to the end of the logical and file id sequence. This is similar to extending - * the from-scratch initialization. - * - * \param jcp Pointer to jcntl instance. This is used to find the file path and base filename so that - * new files may be created. - * \param new_obj_fn Pointer to function which creates and returns a pointer to a new fcntl object (and hence - * causes a new %journal file to be created). - * \param num_jfiles The number of files by which to increase. - */ - void append(jcntl* const jcp, - fcntl* (new_obj_fn)(jcntl* const jcp, - const std::size_t lid, - const std::size_t fid, - const rcvdat* const rdp), - const std::size_t num_jfiles = 1); - - /** * \brief Insert num_jfiles files after lid index after_index. This causes all lids after after_index to be * increased by num_jfiles. * @@ -191,17 +193,14 @@ * \param after_index Lid index after which to insert file(s). * \param jcp Pointer to jcntl instance. This is used to find the file path and base filename so that * new files may be created. - * \param new_obj_fn Pointer to function which creates and returns a pointer to a new fcntl object (and hence + * \param fp Pointer to function which creates and returns a pointer to a new fcntl object (and hence * causes a new %journal file to be created). * \param num_jfiles The number of files by which to increase. */ - void insert(const std::size_t after_index, + void insert(const u_int16_t after_index, jcntl* const jcp, - fcntl* (new_obj_fn)(jcntl* const jcp, - const std::size_t lid, - const std::size_t fid, - const rcvdat* const rdp), - const std::size_t num_jfiles = 1); + new_obj_fn_ptr fp, + const u_int16_t num_jfiles = 1); /** * \brief Clears _fcntl_arr and deletes all fcntl instances. @@ -213,23 +212,62 @@ * either initialize() is called; thereafter true until finalize() is called, whereupon it will * return false again. * - * \return True if initialized; False otherwise. + * \return True if initialized; false otherwise. */ inline bool is_init() const { return _fcntl_arr.size() > 0; } /** + * \brief Returns true if auto-expand mode is enabled; false if not. + * + * \return True if auto-expand mode is enabled; false if not. + */ + inline bool is_ae() const { return _ae; } + + /** + * \brief Sets the auto-expand mode to enabled if ae is true, to disabled otherwise. The value of _ae_max_jfiles + * must be valid to succeed (i.e. _ae_max_jfiles must be greater than the current number of files or be zero). + * + * \param ae If true will enable auto-expand mode; if false will disable it. + */ + void set_ae(const bool ae); + + /** * \brief Returns the number of %journal files, including any that were appended or inserted since * initialization. * * \return Number of %journal files if initialized; 0 otherwise. */ - inline std::size_t size() const { return _fcntl_arr.size(); } + inline u_int16_t num_jfiles() const { return static_cast<u_int16_t>(_fcntl_arr.size()); } /** + * \brief Returns the maximum number of files allowed for auto-expansion. + * + * \return Maximum number of files allowed for auto-expansion. A zero value represents a disabled limit + * - i.e. unlimited expansion. + */ + inline u_int16_t ae_max_jfiles() const { return _ae_max_jfiles; } + + /** + * \brief Sets the maximum number of files allowed for auto-expansion. A zero value disables the limit. + * + * \param ae_max_jfiles The maximum number of files allowed for auto-expansion. Cannot be lower than the current + * number of files. However, a zero value disables the limit checks, and allows unlimited + * expansion. + */ + void set_ae_max_jfiles(const u_int16_t ae_max_jfiles); + + /** + * \brief Calculates the number of future files available for auto-expansion. + * + * \return The number of future files available for auto-expansion. + */ + u_int16_t ae_jfiles_rem() const; + + /** * \brief Get a pointer to fcntl instance for a given lid. * - * \return Pointer to fcntl object corresponding to logical id lid, or 0 if lid is out of range (greater than - * number of files in use). + * \return Pointer to fcntl object corresponding to logical id lid, or 0 if lid is out of range + * (greater than number of files in use). */ inline fcntl* get_fcntlp(const u_int16_t lid) const { if (lid >= _fcntl_arr.size()) return 0; return _fcntl_arr[lid]; } @@ -237,6 +275,23 @@ // Testing functions void get_fid_list(std::vector<u_int16_t>& fid_list) const; void get_lid_list(std::vector<u_int16_t>& lid_list) const; + + private: + + /** + * \brief Append num_jfiles files to the end of the logical and file id sequence. This is similar to extending + * the from-scratch initialization. + * + * \param jcp Pointer to jcntl instance. This is used to find the file path and base filename so that + * new files may be created. + * \param fp Pointer to function which creates and returns a pointer to a new fcntl object (and hence + * causes a new %journal file to be created). + * \param num_jfiles The number of files by which to increase. + */ + void append(jcntl* const jcp, + new_obj_fn_ptr fp, + const u_int16_t num_jfiles = 1); + }; } // namespace journal Modified: store/trunk/cpp/lib/jrnl/rcvdat.hpp =================================================================== --- store/trunk/cpp/lib/jrnl/rcvdat.hpp 2008-10-17 17:28:47 UTC (rev 2648) +++ store/trunk/cpp/lib/jrnl/rcvdat.hpp 2008-10-17 17:51:24 UTC (rev 2649) @@ -47,22 +47,26 @@ struct rcvdat { - u_int16_t _num_jfiles; ///< Number of journal files - bool _owi; ///< Overwrite indicator - bool _frot; ///< First rotation flag - bool _jempty; ///< Journal data files empty - u_int16_t _ffid; ///< First file id - std::size_t _fro; ///< First record offset in ffid - u_int16_t _lfid; ///< Last file id - std::size_t _eo; ///< End offset (first byte past last record) - u_int64_t _h_rid; ///< Highest rid found - bool _lffull; ///< Last file is full - bool _jfull; ///< Journal is full + u_int16_t _njf; ///< Number of journal files + bool _ae; ///< Auto-expand mode + u_int16_t _aemjf; ///< Auto-expand mode max journal files + bool _owi; ///< Overwrite indicator + bool _frot; ///< First rotation flag + bool _jempty; ///< Journal data files empty + u_int16_t _ffid; ///< First file id + std::size_t _fro; ///< First record offset in ffid + u_int16_t _lfid; ///< Last file id + std::size_t _eo; ///< End offset (first byte past last record) + u_int64_t _h_rid; ///< Highest rid found + bool _lffull; ///< Last file is full + bool _jfull; ///< Journal is full std::vector<u_int16_t> _fid_list; ///< Fid-lid mapping - list of fids in order of lid std::vector<u_int32_t> _enq_cnt_list; ///< Number enqueued records found for each file rcvdat(): - _num_jfiles(0), + _njf(0), + _ae(false), + _aemjf(0), _owi(false), _frot(false), _jempty(true), @@ -77,9 +81,11 @@ _enq_cnt_list() {} - void reset(u_int16_t num_jfiles) + void reset(const u_int16_t num_jfiles, const bool auto_expand, const u_int16_t ae_max_jfiles) { - _num_jfiles = num_jfiles; + _njf = num_jfiles; + _ae = auto_expand; + _aemjf = ae_max_jfiles; _owi=false; _frot = false; _jempty=true; @@ -101,17 +107,19 @@ u_int16_t index = _ffid; while (index != _lfid && _enq_cnt_list[index] == 0) { - if (++index >= _num_jfiles) + if (++index >= _njf) index = 0; } return index; } - std::string to_string(std::string& jid) + std::string to_string(const std::string& jid) { std::ostringstream oss; oss << "Recover file analysis (jid=\"" << jid << "\"):" << std::endl; - oss << " Number of journal files (_num_jfiles) = " << _num_jfiles << std::endl; + oss << " Number of journal files (_njf) = " << _njf << std::endl; + oss << " Auto-expand mode (_ae) = " << (_ae ? "TRUE" : "FALSE") << std::endl; + if (_ae) oss << " Auto-expand mode max journal files (_aemjf) = " << _aemjf << std::endl; oss << " Overwrite indicator (_owi) = " << (_owi ? "TRUE" : "FALSE") << std::endl; oss << " First rotation (_frot) = " << (_frot ? "TRUE" : "FALSE") << std::endl; oss << " Journal empty (_jempty) = " << (_jempty ? "TRUE" : "FALSE") << std::endl; @@ -138,12 +146,14 @@ return oss.str(); } - std::string to_log(std::string& jid) + std::string to_log(const std::string& jid) { std::ostringstream oss; oss << "Recover file analysis (jid=\"" << jid << "\"):"; - oss << " njf=" << _num_jfiles; - oss << " owi=" << (_owi ? "T" : "F"); + oss << " njf=" << _njf; + oss << " ae=" << (_owi ? "T" : "F"); + oss << " aemjf=" << _aemjf; + oss << " owi=" << (_ae ? "T" : "F"); oss << " frot=" << (_frot ? "T" : "F"); oss << " jempty=" << (_jempty ? "T" : "F"); oss << " ffid=" << _ffid; Modified: store/trunk/cpp/lib/jrnl/rrfc.cpp =================================================================== --- store/trunk/cpp/lib/jrnl/rrfc.cpp 2008-10-17 17:28:47 UTC (rev 2648) +++ store/trunk/cpp/lib/jrnl/rrfc.cpp 2008-10-17 17:51:24 UTC (rev 2649) @@ -74,10 +74,10 @@ iores rrfc::rotate() { - if (!_lfmp->size()) + if (!_lfmp->num_jfiles()) throw jexception(jerrno::JERR__NINIT, "rrfc", "rotate"); u_int16_t next_fc_index = _fc_index + 1; - if (next_fc_index == _lfmp->size()) + if (next_fc_index == _lfmp->num_jfiles()) next_fc_index = 0; fcntl* next_fc = _lfmp->get_fcntlp(next_fc_index); _fc_index = next_fc_index; Modified: store/trunk/cpp/lib/jrnl/wrfc.cpp =================================================================== --- store/trunk/cpp/lib/jrnl/wrfc.cpp 2008-10-17 17:28:47 UTC (rev 2648) +++ store/trunk/cpp/lib/jrnl/wrfc.cpp 2008-10-17 17:51:24 UTC (rev 2649) @@ -77,7 +77,7 @@ } _fsize_sblks = fsize_sblks; _fsize_dblks = fsize_sblks * JRNL_SBLK_SIZE; - _enq_cap_offs_dblks = (u_int32_t)std::ceil(_fsize_dblks * _lfmp->size() * (100.0 - JRNL_ENQ_THRESHOLD) / 100); + _enq_cap_offs_dblks = (u_int32_t)std::ceil(_fsize_dblks * _lfmp->num_jfiles() * (100.0 - JRNL_ENQ_THRESHOLD) / 100); // Check the offset is at least one file; if not, make it so if (_enq_cap_offs_dblks < _fsize_dblks) _enq_cap_offs_dblks = _fsize_dblks; @@ -85,10 +85,10 @@ iores wrfc::rotate() { - if (!_lfmp->size()) + if (!_lfmp->num_jfiles()) throw jexception(jerrno::JERR__NINIT, "wrfc", "rotate"); _fc_index++; - if (_fc_index == _lfmp->size()) + if (_fc_index == _lfmp->num_jfiles()) { _fc_index = 0; _owi = !_owi; @@ -107,7 +107,7 @@ if (_frot) return 0; u_int16_t next_index = _fc_index + 1; - if (next_index >= _lfmp->size()) + if (next_index >= _lfmp->num_jfiles()) next_index = 0; return next_index; } @@ -129,7 +129,7 @@ fwd_dblks -= fwd_dblks > _fsize_dblks ? _fsize_dblks : fwd_dblks; if (fwd_dblks) { - if (++findex == _lfmp->size()) + if (++findex == _lfmp->num_jfiles()) findex = 0; fcp = _lfmp->get_fcntlp(findex); } Modified: store/trunk/cpp/tests/jrnl/_st_helper_fns.hpp =================================================================== --- store/trunk/cpp/tests/jrnl/_st_helper_fns.hpp 2008-10-17 17:28:47 UTC (rev 2648) +++ store/trunk/cpp/tests/jrnl/_st_helper_fns.hpp 2008-10-17 17:51:24 UTC (rev 2649) @@ -68,8 +68,10 @@ jcntl(jid, jdir, base_filename) {} virtual ~test_jrnl() {} void initialize(const u_int16_t num_jfiles, const u_int32_t jfsize_sblks) - { jcntl::initialize(num_jfiles, jfsize_sblks, JRNL_WMGR_DEF_PAGES, JRNL_WMGR_DEF_PAGE_SIZE, - 0, &aio_wr_callback); } + { + jcntl::initialize(num_jfiles, jfsize_sblks, JRNL_WMGR_DEF_PAGES, JRNL_WMGR_DEF_PAGE_SIZE, 0, &aio_wr_callback); + _jdir.create_dir(); + } void recover(const u_int16_t num_jfiles, const u_int32_t jfsize_sblks, vector<string>* txn_list, u_int64_t& highest_rid) { jcntl::recover(num_jfiles, jfsize_sblks, JRNL_WMGR_DEF_PAGES, JRNL_WMGR_DEF_PAGE_SIZE, 0, @@ -226,6 +228,7 @@ const u_int16_t num_files = 1, // Number of files to insert const bool adjust_lids = true) // Adjust lids following inserted files { + if (num_files == 0) return; _num_used_files += num_files; const u_int16_t num_jfiles_before_append = _map.size(); lfmap_citr i = _map.find(after_lid); @@ -375,9 +378,9 @@ * header (512 bytes each) and a single jinf file which contains the journal metadata required for recovery * analysis. */ - void write_journal() + void write_journal(const bool ae, const u_int16_t ae_max_jfiles) { - create_jinf(); + create_jinf(ae, ae_max_jfiles); u_int16_t fid = 0; for (lfmap_citr itr = _map.begin(); itr != _map.end(); itr++, fid++) { @@ -410,11 +413,11 @@ * Method create_new_jinf(): This static call creates a default jinf file only. This is used to test the read * constructor of a jinf test object which reads a jinf file at instantiation. */ - static void create_new_jinf(const string jid, const string base_filename) + static void create_new_jinf(const string jid, const string base_filename, const bool ae) { if (jdir::exists(test_dir)) jdir::delete_dir(test_dir); - create_jinf(NUM_JFILES, jid, base_filename); + create_jinf(NUM_JFILES, ae, (ae ? 5 * NUM_JFILES : 0), jid, base_filename); } /* @@ -455,20 +458,21 @@ fh._ts_nsec = ts.tv_nsec; } - void create_jinf() + void create_jinf(const bool ae, const u_int16_t ae_max_jfiles) { if (jdir::exists(test_dir)) jdir::delete_dir(test_dir); - create_jinf(_map.size(), _jid, _base_filename); + create_jinf(_map.size(), ae, ae_max_jfiles, _jid, _base_filename); } - static void create_jinf(u_int16_t num_files, const string jid, const string base_filename) + static void create_jinf(u_int16_t num_files, const bool ae, const u_int16_t ae_max_jfiles, const string jid, + const string base_filename) { jdir::create_dir(test_dir); // Check test dir exists; create it if not timespec ts; ::clock_gettime(CLOCK_REALTIME, &ts); - jinf ji(jid, test_dir, base_filename, num_files, JFSIZE_SBLKS, JRNL_WMGR_DEF_PAGE_SIZE, JRNL_WMGR_DEF_PAGES, - ts); + jinf ji(jid, test_dir, base_filename, num_files, ae, ae_max_jfiles, JFSIZE_SBLKS, JRNL_WMGR_DEF_PAGE_SIZE, + JRNL_WMGR_DEF_PAGES, ts); ji.write(); } Modified: store/trunk/cpp/tests/jrnl/_ut_jdir.cpp =================================================================== --- store/trunk/cpp/tests/jrnl/_ut_jdir.cpp 2008-10-17 17:28:47 UTC (rev 2648) +++ store/trunk/cpp/tests/jrnl/_ut_jdir.cpp 2008-10-17 17:51:24 UTC (rev 2649) @@ -93,7 +93,7 @@ { timespec ts; ::clock_gettime(CLOCK_REALTIME, &ts); - jinf ji("test journal id", dirname, base_filename, NUM_JFILES, JFSIZE_SBLKS, + jinf ji("test journal id", dirname, base_filename, NUM_JFILES, false, 0, JFSIZE_SBLKS, JRNL_WMGR_DEF_PAGE_SIZE, JRNL_WMGR_DEF_PAGES, ts); ji.write(); } Modified: store/trunk/cpp/tests/jrnl/_ut_jinf.cpp =================================================================== --- store/trunk/cpp/tests/jrnl/_ut_jinf.cpp 2008-10-17 17:28:47 UTC (rev 2648) +++ store/trunk/cpp/tests/jrnl/_ut_jinf.cpp 2008-10-17 17:51:24 UTC (rev 2649) @@ -52,7 +52,7 @@ const string base_filename = test_name + "_bfn"; jdir::create_dir(test_dir); // Check test dir exists; create it if not ::clock_gettime(CLOCK_REALTIME, &ts); - jinf ji(jid, test_dir, base_filename, NUM_JFILES, JFSIZE_SBLKS, JRNL_WMGR_DEF_PAGE_SIZE, JRNL_WMGR_DEF_PAGES, ts); + jinf ji(jid, test_dir, base_filename, NUM_JFILES, false, 0, JFSIZE_SBLKS, JRNL_WMGR_DEF_PAGE_SIZE, JRNL_WMGR_DEF_PAGES, ts); BOOST_CHECK_EQUAL(ji.jver(), RHM_JDAT_VERSION); BOOST_CHECK(ji.jid().compare(jid) == 0); BOOST_CHECK(ji.jdir().compare(test_dir) == 0); @@ -61,6 +61,8 @@ BOOST_CHECK_EQUAL(this_ts.tv_sec, ts.tv_sec); BOOST_CHECK_EQUAL(this_ts.tv_nsec, ts.tv_nsec); BOOST_CHECK_EQUAL(ji.num_jfiles(), u_int16_t(NUM_JFILES)); + BOOST_CHECK_EQUAL(ji.is_ae(), false); + BOOST_CHECK_EQUAL(ji.ae_max_jfiles(), u_int16_t(0)); BOOST_CHECK_EQUAL(ji.jfsize_sblks(), u_int32_t(JFSIZE_SBLKS)); BOOST_CHECK_EQUAL(ji.sblk_size_dblks(), u_int16_t(JRNL_SBLK_SIZE)); BOOST_CHECK_EQUAL(ji.dblk_size(), u_int32_t(JRNL_DBLK_SIZE)); @@ -77,7 +79,7 @@ string test_name = get_test_name(test_filename, "read_constructor"); const string jid = test_name + "_jid"; const string base_filename = test_name + "_bfn"; - lid_fid_map::create_new_jinf(jid, base_filename); + lid_fid_map::create_new_jinf(jid, base_filename, false); stringstream fn; fn << test_dir << "/" <<base_filename << "." << JRNL_INFO_EXTENSION; @@ -90,6 +92,8 @@ // BOOST_CHECK_EQUAL(this_ts.tv_sec, ts.tv_sec); // BOOST_CHECK_EQUAL(this_ts.tv_nsec, ts.tv_nsec); BOOST_CHECK_EQUAL(ji.num_jfiles(), u_int16_t(NUM_JFILES)); + BOOST_CHECK_EQUAL(ji.is_ae(), false); + BOOST_CHECK_EQUAL(ji.ae_max_jfiles(), u_int16_t(0)); BOOST_CHECK_EQUAL(ji.jfsize_sblks(), u_int32_t(JFSIZE_SBLKS)); BOOST_CHECK_EQUAL(ji.sblk_size_dblks(), u_int16_t(JRNL_SBLK_SIZE)); BOOST_CHECK_EQUAL(ji.dblk_size(), u_int32_t(JRNL_DBLK_SIZE)); @@ -106,7 +110,7 @@ string test_name = get_test_name(test_filename, "set_functions"); const string jid = test_name + "_jid"; const string base_filename = test_name + "_bfn"; - lid_fid_map::create_new_jinf(jid, base_filename); + lid_fid_map::create_new_jinf(jid, base_filename, false); stringstream fn; fn << test_dir << "/" << base_filename << "." << JRNL_INFO_EXTENSION; @@ -130,7 +134,7 @@ string test_name = get_test_name(test_filename, "validate"); const string jid = test_name + "_jid"; const string base_filename = test_name + "_bfn"; - lid_fid_map::create_new_jinf(jid, base_filename); + lid_fid_map::create_new_jinf(jid, base_filename, false); stringstream fn; fn << test_dir << "/" << base_filename << "." << JRNL_INFO_EXTENSION; @@ -150,7 +154,7 @@ lid_fid_map m(jid, base_filename); m.journal_create(NUM_JFILES, 0, 0); - m.write_journal(); + m.write_journal(false, 0); stringstream fn; fn << test_dir << "/" << base_filename << "." << JRNL_INFO_EXTENSION; @@ -175,7 +179,7 @@ for (u_int16_t num_files = 1; num_files < NUM_JFILES; num_files++) { m.journal_create(NUM_JFILES, num_files, 0); - m.write_journal(); + m.write_journal(false, 0); stringstream fn; fn << test_dir << "/" << base_filename << "." << JRNL_INFO_EXTENSION; @@ -197,7 +201,7 @@ for (u_int16_t file_num = 0; file_num < NUM_JFILES; file_num++) { m.journal_create(NUM_JFILES, NUM_JFILES, file_num); - m.write_journal(); + m.write_journal(false, 0); stringstream fn; fn << test_dir << "/" << base_filename << "." << JRNL_INFO_EXTENSION; @@ -222,7 +226,7 @@ { m.journal_create(NUM_JFILES, NUM_JFILES, oldest_lid); m.journal_insert(after_lid, num_files); - m.write_journal(); + m.write_journal(false, 0); stringstream fn; fn << test_dir << "/" << base_filename << "." << JRNL_INFO_EXTENSION; @@ -253,7 +257,7 @@ const u_int16_t after_lid = u_int16_t(m.size() * ::drand48()); m.journal_insert(after_lid, num_files); } - m.write_journal(); + m.write_journal(false, 0); stringstream fn; fn << test_dir << "/" << base_filename << "." << JRNL_INFO_EXTENSION; @@ -293,7 +297,7 @@ const u_int16_t num_files = u_int16_t(1 + (NUM_JFILES * ::drand48())); const u_int16_t after_lid = oldest_lid == 0 ? m.size() - 1 : oldest_lid - 1; m.journal_insert(after_lid, num_files, false); - m.write_journal(); + m.write_journal(false, 0); stringstream fn; fn << test_dir << "/" << base_filename << "." << JRNL_INFO_EXTENSION; Modified: store/trunk/cpp/tests/jrnl/_ut_lfmgr.cpp =================================================================== --- store/trunk/cpp/tests/jrnl/_ut_lfmgr.cpp 2008-10-17 17:28:47 UTC (rev 2648) +++ store/trunk/cpp/tests/jrnl/_ut_lfmgr.cpp 2008-10-17 17:51:24 UTC (rev 2649) @@ -82,25 +82,271 @@ linear_vectors_equal(lm, fid_lid_size, res, false); } - static void rcvdat_init(rcvdat& rd, const u_int16_t num_jfiles, const u_int16_t fids[]) + static void rcvdat_init(rcvdat& rd, const u_int16_t num_jfiles, const bool ae, const u_int16_t ae_max_jfiles, + const u_int16_t fids[]) { - rd.reset(num_jfiles); + rd.reset(num_jfiles, ae, ae_max_jfiles); load_vector(fids, num_jfiles, rd._fid_list); rd._jempty = false; rd._lfid = fids[num_jfiles - 1]; rd._eo = 100 * JRNL_DBLK_SIZE * JRNL_SBLK_SIZE; } - static void rcvdat_init(rcvdat& rd, const flist& fidl) + static void rcvdat_init(rcvdat& rd, const flist& fidl, const bool ae, const u_int16_t ae_max_jfiles) { const u_int16_t num_jfiles = fidl.size(); - rd.reset(num_jfiles); + rd.reset(num_jfiles, ae, ae_max_jfiles); load_vector(fidl, rd._fid_list); rd._jempty = false; rd._lfid = fidl[num_jfiles - 1]; rd._eo = 100 * JRNL_DBLK_SIZE * JRNL_SBLK_SIZE; } + static void initialize(lfmgr& lm, test_jrnl& jc, const u_int16_t num_jfiles, const bool ae, + const u_int16_t ae_max_jfiles) + { + lm.initialize(num_jfiles, ae, ae_max_jfiles, &jc, &jc.new_fcntl); + BOOST_CHECK_EQUAL(lm.is_init(), true); + BOOST_CHECK_EQUAL(lm.is_ae(), ae); + BOOST_CHECK_EQUAL(lm.ae_max_jfiles(), ae_max_jfiles); + BOOST_CHECK_EQUAL(lm.num_jfiles(), num_jfiles); + if (num_jfiles) + check_linear_fids_lids(lm, num_jfiles); + else + BOOST_CHECK_EQUAL(lm.get_fcntlp(0), (void*)0); + } + + // version which sets up the lid_fid_map for later manipulation by insert tests + static void initialize(lid_fid_map& lfm, lfmgr& lm, test_jrnl& jc, const u_int16_t num_jfiles, const bool ae, + const u_int16_t ae_max_jfiles) + { + lfm.journal_create(num_jfiles, num_jfiles); + initialize(lm, jc, num_jfiles, ae, ae_max_jfiles); + } + + static void prepare_recover(lid_fid_map& lfm, const u_int16_t size) + { + if (size < 4) BOOST_FAIL("prepare_recover(): size parameter (" << size << ") too small."); + lfm.journal_create(4, 4); // initial journal of size 4 + u_int16_t s = 4; // cumulative size + while (s < size) + { + const u_int16_t ins_posn = u_int16_t(s * ::drand48()); // this insert posn + if (3.0 * ::drand48() > 1.0 || size - s < 2) // 2:1 chance of single insert when >= 2 still to insert + { + lfm.journal_insert(ins_posn); // single insert + s++; + } + else + { + // multiple insert, either 2 - 5 + const u_int16_t max_ins_size = size - s >5 ? 5 : size - s; + const u_int16_t ins_size = 2 + u_int16_t((max_ins_size - 2) * ::drand48()); // this insert size + lfm.journal_insert(ins_posn, ins_size); + s += ins_size; + } + } + } + + static void recover(lid_fid_map& lfm, lfmgr& lm, test_jrnl& jc, const bool ae, const u_int16_t ae_max_jfiles) + { + flist fidl; + flist lidl; + rcvdat rd; + const u_int16_t num_jfiles = lfm.size(); + + lfm.get_fid_list(fidl); + lfm.get_lid_list(lidl); + lfm.write_journal(ae, ae_max_jfiles); + + lfmgr_test_helper::rcvdat_init(rd, fidl, ae, ae_max_jfiles); + lm.recover(rd, &jc, &jc.new_fcntl); + BOOST_CHECK_EQUAL(lm.is_init(), true); + BOOST_CHECK_EQUAL(lm.is_ae(), ae); + BOOST_CHECK_EQUAL(lm.ae_max_jfiles(), ae_max_jfiles); + BOOST_CHECK_EQUAL(lm.num_jfiles(), num_jfiles); + if (num_jfiles) + check_fids_lids(lm, fidl, lidl); + else + BOOST_CHECK_EQUAL(lm.get_fcntlp(0), (void*)0); + } + + static void finalize(lfmgr& lm) + { + lm.finalize(); + BOOST_CHECK_EQUAL(lm.is_init(), false); + BOOST_CHECK_EQUAL(lm.is_ae(), false); + BOOST_CHECK_EQUAL(lm.ae_max_jfiles(), u_int16_t(0)); + BOOST_CHECK_EQUAL(lm.num_jfiles(), u_int16_t(0)); + BOOST_CHECK_EQUAL(lm.get_fcntlp(0), (void*)0); + vector<u_int16_t> res; + lm.get_fid_list(res); + BOOST_CHECK_EQUAL(res.size(), u_int16_t(0)); + lm.get_lid_list(res); + BOOST_CHECK_EQUAL(res.size(), u_int16_t(0)); + } + + static void insert(lid_fid_map& lfm, lfmgr& lm, test_jrnl& jc, const u_int16_t after_lid, const u_int16_t incr = 1) + { + flist fidl; + flist lidl; + const u_int16_t num_jfiles = lm.num_jfiles(); + lfm.journal_insert(after_lid, incr); + lfm.get_fid_list(fidl); + lfm.get_lid_list(lidl); + lm.insert(after_lid, &jc, &jc.new_fcntl, incr); + BOOST_CHECK_EQUAL(lm.num_jfiles(), num_jfiles + incr); + lfmgr_test_helper::check_fids_lids(lm, fidl, lidl); + } + + static void check_ae_max_jfiles(lfmgr& lm, const u_int16_t num_jfiles, const u_int16_t ae_max_jfiles) + { + bool legal = ae_max_jfiles > num_jfiles || ae_max_jfiles == 0; + + lm.set_ae(false); + BOOST_CHECK(!lm.is_ae()); + if (legal) + { + lm.set_ae_max_jfiles(ae_max_jfiles); + BOOST_CHECK_EQUAL(lm.ae_max_jfiles(), ae_max_jfiles); + lm.set_ae(true); + BOOST_CHECK(lm.is_ae()); + BOOST_CHECK_EQUAL(lm.ae_jfiles_rem(), ae_max_jfiles + ? ae_max_jfiles - num_jfiles + : JRNL_MAX_NUM_FILES - num_jfiles); + } + else + { + lm.set_ae_max_jfiles(ae_max_jfiles); + BOOST_CHECK_EQUAL(lm.ae_max_jfiles(), ae_max_jfiles); + try + { + lm.set_ae(true); // should raise exception + BOOST_ERROR("Auto-expand enabled with out-of-range ae_max_jfiles"); + } + catch (const jexception& e) { BOOST_CHECK_EQUAL(e.err_code(), jerrno::JERR_LFMGR_BADAEFNUMLIM); } + BOOST_CHECK(!lm.is_ae()); + BOOST_CHECK_EQUAL(lm.ae_jfiles_rem(), 0); + } + BOOST_CHECK_EQUAL(lm.ae_max_jfiles(), ae_max_jfiles); + } + + static void check_multiple_initialization_recover(lid_fid_map& lfm, test_jrnl& jc, + const u_int16_t num_jfiles_arr[][2], const bool init_flag_0, const bool finalize_flag, + const bool init_flag_1) + { + unsigned i_njf = 0; + while (num_jfiles_arr[i_njf][0] && num_jfiles_arr[i_njf][1]) + { + for (unsigned i1_njf = 0; i1_njf <= 1; i1_njf++) + { + const u_int16_t num_jfiles_0 = num_jfiles_arr[i_njf][i1_njf == 0]; + const u_int16_t num_jfiles_1 = num_jfiles_arr[i_njf][i1_njf != 0]; + + for (unsigned i_ae = 0; i_ae < 4; i_ae++) + { + const bool ae_0 = i_ae & 0x1; + const bool ae_1 = i_ae & 0x2; + for (unsigned i_aemjf = 0; i_aemjf < 4; i_aemjf++) + { + const u_int16_t ae_max_jfiles_0 = i_aemjf & 0x1 ? 3 * num_jfiles_0 : 0; + const u_int16_t ae_max_jfiles_1 = i_aemjf & 0x2 ? 4 * num_jfiles_1 : 0; + + lfmgr lm; + + if (init_flag_0) + initialize(lm, jc, num_jfiles_0, ae_0, ae_max_jfiles_0); + else + { + prepare_recover(lfm, num_jfiles_0); + recover(lfm, lm, jc, ae_1, ae_max_jfiles_0); + lfm.destroy_journal(); + } + + if (finalize_flag) finalize(lm); + + if (init_flag_1) + initialize(lm, jc, num_jfiles_1, ae_1, ae_max_jfiles_1); + else + { + prepare_recover(lfm, num_jfiles_1); + recover(lfm, lm, jc, ae_1, ae_max_jfiles_1); + lfm.destroy_journal(); + } + } + } + } + i_njf++; + } + } + + static void check_insert(lid_fid_map& lfm, lfmgr& lm, test_jrnl& jc, const u_int16_t after_lid, + const u_int16_t incr = 1) + { + const u_int16_t num_jfiles = lm.num_jfiles(); + const u_int16_t ae_max_jfiles = lm.ae_max_jfiles(); + const u_int16_t effective_ae_max_jfiles = ae_max_jfiles ? ae_max_jfiles : JRNL_MAX_NUM_FILES; + BOOST_CHECK_EQUAL(lm.ae_jfiles_rem(), effective_ae_max_jfiles - num_jfiles); + bool legal = lm.is_ae() && num_jfiles + incr <= effective_ae_max_jfiles; + if (legal) + { + insert(lfm, lm, jc, after_lid, incr); + BOOST_CHECK_EQUAL(lm.num_jfiles(), num_jfiles + incr); + BOOST_CHECK_EQUAL(lm.ae_jfiles_rem(), effective_ae_max_jfiles - num_jfiles - incr); + } + else + { + try + { + insert(lfm, lm, jc, after_lid, incr); + if (lm.is_ae()) + BOOST_ERROR("lfmgr::insert() succeeded and exceeded limit"); + else + BOOST_ERROR("lfmgr::insert() succeeded with auto-expand disabled"); + } + catch (const jexception& e) + { + if (lm.is_ae()) + BOOST_CHECK_EQUAL(e.err_code(), jerrno::JERR_LFMGR_AEFNUMLIMIT); + else + BOOST_CHECK_EQUAL(e.err_code(), jerrno::JERR_LFMGR_AEDISABLED); + } + BOOST_CHECK_EQUAL(lm.num_jfiles(), num_jfiles); + BOOST_CHECK_EQUAL(lm.ae_jfiles_rem(), effective_ae_max_jfiles - num_jfiles); + } + } + + static void check_limit(lid_fid_map& lfm, test_jrnl& jc, const bool ae, const u_int16_t num_jfiles, + const u_int16_t ae_max_jfiles) + { + lfmgr lm; + + for (unsigned i = 0; i < 2; i++) + { + if (i) + initialize(lfm, lm, jc, num_jfiles, ae, ae_max_jfiles); + else + { + prepare_recover(lfm, num_jfiles); + recover(lfm, lm, jc, ae, ae_max_jfiles); + } + + // use up all available files + unsigned j = ae_max_jfiles ? ae_max_jfiles : JRNL_MAX_NUM_FILES; + while (ae && j > num_jfiles) + { + const u_int16_t posn = (lm.num_jfiles() - 1) * ::drand48(); + const u_int16_t incr = 1 + (lm.ae_jfiles_rem() > 4 ? 3 : lm.ae_jfiles_rem() - 1) * ::drand48(); + check_insert(lfm, lm, jc, posn, incr); + j -= incr; + } + // these should be over the limit or illegal + check_insert(lfm, lm, jc, 0); + check_insert(lfm, lm, jc, 2, 2); + lfm.destroy_journal(); + } + } + private: static void load_vector(const u_int16_t a[], const size_t n, flist& v) { @@ -164,7 +410,9 @@ { lfmgr lm; BOOST_CHECK_EQUAL(lm.is_init(), false); - BOOST_CHECK_EQUAL(lm.size(), size_t(0)); + BOOST_CHECK_EQUAL(lm.is_ae(), false); + BOOST_CHECK_EQUAL(lm.ae_max_jfiles(), u_int16_t(0)); + BOOST_CHECK_EQUAL(lm.num_jfiles(), u_int16_t(0)); BOOST_CHECK_EQUAL(lm.get_fcntlp(0), (void*)0); } catch(const exception& e) { BOOST_FAIL(e.what()); } @@ -177,15 +425,23 @@ QPID_AUTO_TEST_CASE(initialize) { string test_name = get_test_name(test_filename, "initialize"); - u_int16_t num_jfiles = 8; + const u_int16_t num_jfiles = 8; try { + jdir::create_dir(test_dir); // Check test dir exists; create it if not test_jrnl jc(test_name, test_dir, test_name); - lfmgr lm; - - lm.initialize(num_jfiles, &jc, &jc.new_fcntl); - BOOST_CHECK_EQUAL(lm.is_init(), true); - lfmgr_test_helper::check_linear_fids_lids(lm, num_jfiles); + { + lfmgr lm; + lfmgr_test_helper::initialize(lm, jc, num_jfiles, false, 0); + } + { + lfmgr lm; + lfmgr_test_helper::initialize(lm, jc, num_jfiles, true, 0); + } + { + lfmgr lm; + lfmgr_test_helper::initialize(lm, jc, num_jfiles, true, 5 * num_jfiles); + } } catch(const exception& e) { BOOST_FAIL(e.what()); } cout << "done" << endl; @@ -197,26 +453,30 @@ QPID_AUTO_TEST_CASE(recover) { string test_name = get_test_name(test_filename, "recover"); - lid_fid_map lfm(test_name, test_name); - flist fidl; - flist lidl; - rcvdat rd; + ::srand48(1); // init random gen for repeatable tests when using lfmgr_test_helper::prepare_recover() try { test_jrnl jc(test_name, test_dir, test_name); - lfmgr lm; + lid_fid_map lfm(test_name, test_name); - lfm.journal_create(4, 4); - lfm.journal_insert(0); - lfm.journal_insert(2, 2); - lfm.journal_insert(6); - lfm.get_fid_list(fidl); - lfm.get_lid_list(lidl); - lfm.write_journal(); - lfmgr_test_helper::rcvdat_init(rd, fidl); - lm.recover(rd, &jc, &jc.new_fcntl); - BOOST_CHECK_EQUAL(lm.is_init(), true); - lfmgr_test_helper::check_fids_lids(lm, fidl, lidl); + { + lfmgr lm; + lfmgr_test_helper::prepare_recover(lfm, 8); + lfmgr_test_helper::recover(lfm, lm, jc, false, 0); + lfm.destroy_journal(); + } + { + lfmgr lm; + lfmgr_test_helper::prepare_recover(lfm, 8); + lfmgr_test_helper::recover(lfm, lm, jc, true, 0); + lfm.destroy_journal(); + } + { + lfmgr lm; + lfmgr_test_helper::prepare_recover(lfm, 8); + lfmgr_test_helper::recover(lfm, lm, jc, true, 5 * lfm.size()); + lfm.destroy_journal(); + } } catch(const exception& e) { BOOST_FAIL(e.what()); } cout << "done" << endl; @@ -228,24 +488,26 @@ QPID_AUTO_TEST_CASE(initialize_finalize) { string test_name = get_test_name(test_filename, "initialize_finalize"); - u_int16_t num_jfiles = 8; + const u_int16_t num_jfiles = 8; try { + jdir::create_dir(test_dir); // Check test dir exists; create it if not test_jrnl jc(test_name, test_dir, test_name); - lfmgr lm; - lm.initialize(num_jfiles, &jc, &jc.new_fcntl); - BOOST_CHECK_EQUAL(lm.is_init(), true); - lfmgr_test_helper::check_linear_fids_lids(lm, num_jfiles); - - lm.finalize(); - BOOST_CHECK_EQUAL(lm.is_init(), false); - BOOST_CHECK_EQUAL(lm.size(), size_t(0)); - BOOST_CHECK_EQUAL(lm.get_fcntlp(0), (void*)0); - vector<u_int16_t> res; - lm.get_fid_list(res); - BOOST_CHECK_EQUAL(res.size(), size_t(0)); - lm.get_lid_list(res); - BOOST_CHECK_EQUAL(res.size(), size_t(0)); + { + lfmgr lm; + lfmgr_test_helper::initialize(lm, jc, num_jfiles, false, 0); + lfmgr_test_helper::finalize(lm); + } + { + lfmgr lm; + lfmgr_test_helper::initialize(lm, jc, num_jfiles, true, 0); + lfmgr_test_helper::finalize(lm); + } + { + lfmgr lm; + lfmgr_test_helper::initialize(lm, jc, num_jfiles, true, 5 * num_jfiles); + lfmgr_test_helper::finalize(lm); + } } catch(const exception& e) { BOOST_FAIL(e.what()); } cout << "done" << endl; @@ -257,517 +519,246 @@ QPID_AUTO_TEST_CASE(recover_finalize) { string test_name = get_test_name(test_filename, "recover_finalize"); - lid_fid_map lfm(test_name, test_name); - flist fidl; - flist lidl; - rcvdat rd; + const u_int16_t num_jfiles = 8; + ::srand48(1); // init random gen for repeatable tests when using lfmgr_test_helper::prepare_recover() try { test_jrnl jc(test_name, test_dir, test_name); - lfmgr lm; + lid_fid_map lfm(test_name, test_name); - lfm.journal_create(4, 4); - lfm.journal_insert(0); - lfm.journal_insert(2, 2); - lfm.journal_insert(6); - lfm.get_fid_list(fidl); - lfm.get_lid_list(lidl); - lfm.write_journal(); - lfmgr_test_helper::rcvdat_init(rd, fidl); - lm.recover(rd, &jc, &jc.new_fcntl); - BOOST_CHECK_EQUAL(lm.is_init(), true); - lfmgr_test_helper::check_fids_lids(lm, fidl, lidl); - - lm.finalize(); - BOOST_CHECK_EQUAL(lm.is_init(), false); - BOOST_CHECK_EQUAL(lm.size(), size_t(0)); - BOOST_CHECK_EQUAL(lm.get_fcntlp(0), (void*)0); - vector<u_int16_t> res; - lm.get_fid_list(res); - BOOST_CHECK_EQUAL(res.size(), size_t(0)); - lm.get_lid_list(res); - BOOST_CHECK_EQUAL(res.size(), size_t(0)); + { + lfmgr lm; + lfmgr_test_helper::prepare_recover(lfm, num_jfiles); + lfmgr_test_helper::recover(lfm, lm, jc, false, 0); + lfmgr_test_helper::finalize(lm); + lfm.destroy_journal(); + } + { + lfmgr lm; + lfmgr_test_helper::prepare_recover(lfm, num_jfiles); + lfmgr_test_helper::recover(lfm, lm, jc, true, 0); + lfmgr_test_helper::finalize(lm); + lfm.destroy_journal(); + } + { + lfmgr lm; + lfmgr_test_helper::prepare_recover(lfm, num_jfiles); + lfmgr_test_helper::recover(lfm, lm, jc, true, 5 * lfm.size()); + lfmgr_test_helper::finalize(lm); + lfm.destroy_journal(); + } } catch(const exception& e) { BOOST_FAIL(e.what()); } cout << "done" << endl; } /* -* Check that initialize() correctly creates an ordered fid array _fcntl_arr after the lfmgr instance has been -* initialized and finalized. +* Check that 0 and/or null and other extreme/boundary parameters behave as expected. */ -QPID_AUTO_TEST_CASE(initialize_finalize_initialize) +QPID_AUTO_TEST_CASE(zero_null_params) { - string test_name = get_test_name(test_filename, "initialize_finalize_initialize"); - u_int16_t num_jfiles0 = 8; - u_int16_t num_jfiles1 = 12; + string test_name = get_test_name(test_filename, "zero_null_params"); + const u_int16_t num_jfiles = 8; try { test_jrnl jc(test_name, test_dir, test_name); + lid_fid_map lfm(test_name, test_name); lfmgr lm; + lfmgr_test_helper::initialize(lfm, lm, jc, num_jfiles, true, 0); - lm.initialize(num_jfiles0, &jc, &jc.new_fcntl); - BOOST_CHECK_EQUAL(lm.is_init(), true); - lfmgr_test_helper::check_linear_fids_lids(lm, num_jfiles0); - - lm.finalize(); - BOOST_CHECK_EQUAL(lm.is_init(), false); - - lm.initialize(num_jfiles1, &jc, &jc.new_fcntl); - BOOST_CHECK_EQUAL(lm.is_init(), true); - lfmgr_test_helper::check_linear_fids_lids(lm, num_jfiles1); + // Check that inserting 0 files works ok + lfmgr_test_helper::insert(lfm, lm, jc, 0, 0); + lfmgr_test_helper::insert(lfm, lm, jc, 2, 0); + lfmgr_test_helper::insert(lfm, lm, jc, num_jfiles - 1, 0); } catch(const exception& e) { BOOST_FAIL(e.what()); } cout << "done" << endl; } /* -* Check that recover() correctly sets up the specified fid list order after the lfmgr instance has been initialized -* and finalized. +* Check that initialize()/recover() works correctly after a previous initialize()/recover() with/without an intervening +* finalize(). */ -QPID_AUTO_TEST_CASE(initialize_finalize_recover) +QPID_AUTO_TEST_CASE(multiple_initialization_recover) { - string test_name = get_test_name(test_filename, "initialize_finalize_recover"); - u_int16_t num_jfiles = 8; - lid_fid_map lfm(test_name, test_name); - flist fidl; - flist lidl; - rcvdat rd; - try - { - test_jrnl jc(test_name, test_dir, test_name); - lfmgr lm; + string test_name = get_test_name(test_filename, "multiple_initialization_recover"); + ::srand48(1); // init random gen for repeatable tests when using lfmgr_test_helper::prepare_recover() - lm.initialize(num_jfiles, &jc, &jc.new_fcntl); - BOOST_CHECK_EQUAL(lm.is_init(), true); - lfmgr_test_helper::check_linear_fids_lids(lm, num_jfiles); - - lm.finalize(); - BOOST_CHECK_EQUAL(lm.is_init(), false); - - lfm.journal_create(4, 4); - lfm.journal_insert(0); - lfm.journal_insert(2, 2); - lfm.journal_insert(6); - lfm.journal_insert(7); - lfm.journal_insert(8, 3); - lfm.get_fid_list(fidl); - lfm.get_lid_list(lidl); - lfm.write_journal(); - lfmgr_test_helper::rcvdat_init(rd, fidl); - lm.recover(rd, &jc, &jc.new_fcntl); - BOOST_CHECK_EQUAL(lm.is_init(), true); - lfmgr_test_helper::check_fids_lids(lm, fidl, lidl); - } - catch(const exception& e) { BOOST_FAIL(e.what()); } - cout << "done" << endl; -} - -/* -* Check that initialize() correctly creates an ordered fid array _fcntl_arr after the lfmgr instance has been recovered -* and finalized. -*/ -QPID_AUTO_TEST_CASE(recover_finalize_initialize) -{ - string test_name = get_test_name(test_filename, "recover_finalize_initialize"); - lid_fid_map lfm(test_name, test_name); - flist fidl; - flist lidl; - rcvdat rd; - u_int16_t num_jfiles = 12; + // Set combinations of value pairs to be used for number of journal files in first and second init + u_int16_t num_jfiles_arr[][2] = {{8, 12}, {4, 7}, {0, 0}}; // end with zeros try { test_jrnl jc(test_name, test_dir, test_name); - lfmgr lm; - - lfm.journal_create(4, 4); - lfm.journal_insert(0); - lfm.journal_insert(2, 2); - lfm.journal_insert(6); - lfm.get_fid_list(fidl); - lfm.get_lid_list(lidl); - lfm.write_journal(); - lfmgr_test_helper::rcvdat_init(rd, fidl); - lm.recover(rd, &jc, &jc.new_fcntl); - BOOST_CHECK_EQUAL(lm.is_init(), true); - lfmgr_test_helper::check_fids_lids(lm, fidl, lidl); - - lm.finalize(); - BOOST_CHECK_EQUAL(lm.is_init(), false); - - lm.initialize(num_jfiles, &jc, &jc.new_fcntl); - BOOST_CHECK_EQUAL(lm.is_init(), true); - lfmgr_test_helper::check_linear_fids_lids(lm, num_jfiles); + lid_fid_map lfm(test_name, test_name); + for (unsigned p = 0; p < 8; p++) + { + const bool i_0 = p & 0x01; // first bit + const bool i_1 = p & 0x02; // second bit + const bool f = p & 0x04; // third bit + lfmgr_test_helper::check_multiple_initialization_recover(lfm, jc, num_jfiles_arr, i_0,f, i_1); + } } catch(const exception& e) { BOOST_FAIL(e.what()); } cout << "done" << endl; } /* -* Check that recover() correctly sets up the specified fid list order after the lfmgr instance has been recovered and -* finalized. +* Check that insert() works correctly after initialize() and shifts the fid sequence beyond the insert point correctly: +* +* The following sequence is tested: +* initialize 4 fids=[0,1,2,3] lids=[0,1,2,3] +* insert 1 after lid 0 fids=[0,4,1,2,3] lids=[0,2,3,4,1] +* insert 2 after lid 2 fids=[0,4,1,5,6,2,3] lids=[0,2,5,6,1,3,4] +* insert 1 after lid 6 fids=[0,4,1,5,6,2,3,7] lids=[0,2,5,6,1,3,4,7] +* issert 1 after lid 3 fids=[0,4,1,5,8,6,2,3,7] lids=[0,2,6,7,1,3,5,8,4] */ -QPID_AUTO_TEST_CASE(recover_finalize_recover) +QPID_AUTO_TEST_CASE(initialize_insert) { - string test_name = get_test_name(test_filename, "recover_finalize_recover"); - lid_fid_map lfm(test_name, test_name); - flist fidl; - flist lidl; - rcvdat rd; + string test_name = get_test_name(test_filename, "initialize_insert"); + const u_int16_t initial_num_jfiles = 8; try { + jdir::create_dir(test_dir); // Check test dir exists; create it if not test_jrnl jc(test_name, test_dir, test_name); + lid_fid_map lfm(test_name, test_name); lfmgr lm; + lfmgr_test_helper::initialize(lfm, lm, jc, initial_num_jfiles, true, 0); - lfm.journal_create(4, 4); - lfm.journal_insert(0); - lfm.journal_insert(2, 2); - lfm.journal_insert(6); - lfm.get_fid_list(fidl); - lfm.get_lid_list(lidl); - lfm.write_journal(); - lfmgr_test_helper::rcvdat_init(rd, fidl); - lm.recover(rd, &jc, &jc.new_fcntl); - BOOST_CHECK_EQUAL(lm.is_init(), true); - lfmgr_test_helper::check_fids_lids(lm, fidl, lidl); - - lm.finalize(); - BOOST_CHECK_EQUAL(lm.is_init(), false); - - lfm.journal_insert(3); - lfm.journal_insert(8, 3); - lfm.get_fid_list(fidl); - lfm.get_lid_list(lidl); - lfm.write_journal(); - lfmgr_test_helper::rcvdat_init(rd, fidl); - - lm.recover(rd, &jc, &jc.new_fcntl); - BOOST_CHECK_EQUAL(lm.is_init(), true); - lfmgr_test_helper::check_fids_lids(lm, fidl, lidl); + lfmgr_test_helper::insert(lfm, lm, jc, 0); + lfmgr_test_helper::insert(lfm, lm, jc, 2, 2); + lfmgr_test_helper::insert(lfm, lm, jc, 6); + lfmgr_test_helper::insert(lfm, lm, jc, 3); } catch(const exception& e) { BOOST_FAIL(e.what()); } cout << "done" << endl; } /* -* Check that a second call to initialize() without calling finilize() correctly clears the first initialize and -* correctly creates an ordered fid array of the correct size _fcntl_arr. +* Check that insert() works correctly after recover() and shifts the fid sequence beyond the insert point correctly: +* +* The following sequence is tested: +* recover 4 fids=[0,2,3,1] lids=[0,3,1,2] +* insert 1 after lid 0 fids=[0,4,2,3,1] lids=[0,4,2,3,1] +* insert 2 after lid 2 fids=[0,4,2,5,6,3,1] lids=[0,6,2,5,1,3,4] +* insert 1 after lid 6 fids=[0,4,2,5,6,3,1,7] lids=[0,6,2,5,1,3,4,7] +* issert 1 after lid 3 fids=[0,4,2,5,8,6,3,1,7] lids=[0,7,2,6,1,3,5,8,4] */ -QPID_AUTO_TEST_CASE(initialize_initialize) +QPID_AUTO_TEST_CASE(recover_insert) { - string test_name = get_test_name(test_filename, "initialize_initialize"); - u_int16_t num_jfiles0 = 8; - u_int16_t num_jfiles1 = 12; + string test_name = get_test_name(test_filename, "recover_insert"); + const u_int16_t initial_num_jfiles = 4; + ::srand48(1); // init random gen for repeatable tests when using lfmgr_test_helper::prepare_recover() try { + jdir::create_dir(test_dir); // Check test dir exists; create it if not test_jrnl jc(test_name, test_dir, test_name); + lid_fid_map lfm(test_name, test_name); lfmgr lm; + lfmgr_test_helper::prepare_recover(lfm, initial_num_jfiles); + lfmgr_test_helper::recover(lfm, lm, jc, true, 0); - lm.initialize(num_jfiles0, &jc, &jc.new_fcntl); - BOOST_CHECK_EQUAL(lm.is_init(), true); - lfmgr_test_helper::check_linear_fids_lids(lm, num_jfiles0); - - lm.initialize(num_jfiles1, &jc, &jc.new_fcntl); - BOOST_CHECK_EQUAL(lm.is_init(), true); - lfmgr_test_helper::check_linear_fids_lids(lm, num_jfiles1); + lfmgr_test_helper::insert(lfm, lm, jc, 0); + lfmgr_test_helper::insert(lfm, lm, jc, 2, 2); + lfmgr_test_helper::insert(lfm, lm, jc, 6); + lfmgr_test_helper::insert(lfm, lm, jc, 3); } catch(const exception& e) { BOOST_FAIL(e.what()); } cout << "done" << endl; } /* -* Check that a call to recover() after initialize() without calling finilize() correctly clears the first initialize and -* correctly sets up the specified fid list order. +* Check that illegal ae parameter combinations are caught and result in an exception being thrown. */ -QPID_AUTO_TEST_CASE(initialize_recover) +QPID_AUTO_TEST_CASE(ae_parameters) { - string test_name = get_test_name(test_filename, "initialize_recover"); - u_int16_t num_jfiles = 8; - lid_fid_map lfm(test_name, test_name); - flist fidl; - flist lidl; - rcvdat rd; + string test_name = get_test_name(test_filename, "ae_parameters"); + ::srand48(1); // init random gen for repeatable tests when using lfmgr_test_helper::prepare_recover() try { + jdir::create_dir(test_dir); // Check test dir exists; create it if not test_jrnl jc(test_name, test_dir, test_name); + lid_fid_map lfm(test_name, test_name); + const u_int16_t num_jfiles = 8; lfmgr lm; + + for (unsigned i = 0; i < 2; i++) + { + if (i) + lfmgr_test_helper::initialize(lfm, lm, jc, num_jfiles, false, 0); + else + { + lfmgr_test_helper::prepare_recover(lfm, num_jfiles); + lfmgr_test_helper::recover(lfm, lm, jc, false, 0); + } - lm.initialize(num_jfiles, &jc, &jc.new_fcntl); - BOOST_CHECK_EQUAL(lm.is_init(), true); - lfmgr_test_helper::check_linear_fids_lids(lm, num_jfiles); - - lfm.journal_create(4, 4); - lfm.journal_insert(0); - lfm.journal_insert(2, 2); - lfm.journal_insert(6); - lfm.journal_insert(7); - lfm.journal_insert(8, 3); - lfm.get_fid_list(fidl); - lfm.get_lid_list(lidl); - lfm.write_journal(); - lfmgr_test_helper::rcvdat_init(rd, fidl); - lm.recover(rd, &jc, &jc.new_fcntl); - BOOST_CHECK_EQUAL(lm.is_init(), true); - lfmgr_test_helper::check_fids_lids(lm, fidl, lidl); + lfmgr_test_helper::check_ae_max_jfiles(lm, num_jfiles, num_jfiles - 2); + lfmgr_test_helper::check_ae_max_jfiles(lm, num_jfiles, 0); + lfmgr_test_helper::check_ae_max_jfiles(lm, num_jfiles, 2 * num_jfiles); + lfmgr_test_helper::check_ae_max_jfiles(lm, num_jfiles, num_jfiles); + lfm.destroy_journal(); + } } catch(const exception& e) { BOOST_FAIL(e.what()); } cout << "done" << endl; } /* -* Check that a call to initialize() after recover() without calling finilize() correctly clears the first recover and -* correctly creates an ordered fid array of the correct size _fcntl_arr. +* Check that initialized or recovered journals with auto-expand disabled will not allow either inserts or appends. */ -QPID_AUTO_TEST_CASE(recover_initialize) +QPID_AUTO_TEST_CASE(ae_disabled) { - string test_name = get_test_name(test_filename, "recover_initialize"); - lid_fid_map lfm(test_name, test_name); - flist fidl; - flist lidl; - rcvdat rd; - u_int16_t num_jfiles = 12; + string test_name = get_test_name(test_filename, "ae_disabled"); + ::srand48(1); // init random gen for repeatable tests when using lfmgr_test_helper::prepare_recover() try { + jdir::create_dir(test_dir); // Check test dir exists; create it if not test_jrnl jc(test_name, test_dir, test_name); - lfmgr lm; - - lfm.journal_create(4, 4); - lfm.journal_insert(0); - lfm.journal_insert(2, 2); - lfm.journal_insert(6); - lfm.get_fid_list(fidl); - lfm.get_lid_list(lidl); - lfm.write_journal(); - lfmgr_test_helper::rcvdat_init(rd, fidl); - lm.recover(rd, &jc, &jc.new_fcntl); - BOOST_CHECK_EQUAL(lm.is_init(), true); - lfmgr_test_helper::check_fids_lids(lm, fidl, lidl); - - lm.initialize(num_jfiles, &jc, &jc.new_fcntl); - BOOST_CHECK_EQUAL(lm.is_init(), true); - lfmgr_test_helper::check_linear_fids_lids(lm, num_jfiles); + lid_fid_map lfm(test_name, test_name); + lfmgr_test_helper::check_limit(lfm, jc, false, 8, 0); } catch(const exception& e) { BOOST_FAIL(e.what()); } cout << "done" << endl; } /* -* Check that a second call to recover() without calling finilize() correctly clears the first recover and correctly -* sets up the specified fid list order. +* Check that initialized or recovered journals with auto-expand enabled and a file limit set will enforce the correct +* limits on inserts and appends. */ -QPID_AUTO_TEST_CASE(recover_recover) +QPID_AUTO_TEST_CASE(ae_enabled_limit) { - string test_name = get_test_name(test_filename, "recover_recover"); - lid_fid_map lfm(test_name, test_name); - flist fidl; - flist lidl; - rcvdat rd; + string test_name = get_test_name(test_filename, "ae_enabled_limit"); + ::srand48(1); // init random gen for repeatable tests when using lfmgr_test_helper::prepare_recover() try { + jdir::create_dir(test_dir); // Check test dir exists; create it if not test_jrnl jc(test_name, test_dir, test_name); - lfmgr lm; - - lfm.journal_create(4, 4); - lfm.journal_insert(0); - lfm.journal_insert(2, 2); - lfm.journal_insert(6); - lfm.get_fid_list(fidl); - lfm.get_lid_list(lidl); - lfm.write_journal(); - lfmgr_test_helper::rcvdat_init(rd, fidl); - lm.recover(rd, &jc, &jc.new_fcntl); - BOOST_CHECK_EQUAL(lm.is_init(), true); - lfmgr_test_helper::check_fids_lids(lm, fidl, lidl); - - lfm.journal_insert(3); - lfm.journal_insert(8, 3); - lfm.get_fid_list(fidl); - lfm.get_lid_list(lidl); - lfm.write_journal(); - lfmgr_test_helper::rcvdat_init(rd, fidl); - - lm.recover(rd, &jc, &jc.new_fcntl); - BOOST_CHECK_EQUAL(lm.is_init(), true); - lfmgr_test_helper::check_fids_lids(lm, fidl, lidl); + lid_fid_map lfm(test_name, test_name); + lfmgr_test_helper::check_limit(lfm, jc, true, 8, 32); } catch(const exception& e) { BOOST_FAIL(e.what()); } cout << "done" << endl; } /* -* Check that append correctly adds to the end of the array _fcntl_arr after initialize(). +* Check that initialized or recovered journals with auto-expand enabled and no file limit set (0) will allow inserts and +* appends up to the file limit JRNL_MAX_NUM_FILES. */ -QPID_AUTO_TEST_CASE(initialize_append) +QPID_AUTO_TEST_CASE(ae_enabled_unlimited) { - string test_name = get_test_name(test_filename, "initialize_append"); - u_int16_t num_jfiles = 8; - u_int16_t num_append_jfiles = 3; + string test_name = get_test_name(test_filename, "ae_enabled_unlimited"); + ::srand48(1); // init random gen for repeatable tests when using lfmgr_test_helper::prepare_recover() try { + jdir::create_dir(test_dir); // Check test dir exists; create it if not test_jrnl jc(test_name, test_dir, test_name); - lfmgr lm; - lm.initialize(num_jfiles, &jc, &jc.new_fcntl); - lfmgr_test_helper::check_linear_fids_lids(lm, num_jfiles); - - // Append single file - lm.append(&jc, &jc.new_fcntl); - lfmgr_test_helper::check_linear_fids_lids(lm, num_jfiles + 1); - - // Append 3 more files - lm.append(&jc, &jc.new_fcntl, num_append_jfiles); - lfmgr_test_helper::check_linear_fids_lids(lm, num_jfiles + 1 + num_append_jfiles); + lid_fid_map lfm(test_name, test_name); + lfmgr_test_helper::check_limit(lfm, jc, true, 8, 0); } catch(const exception& e) { BOOST_FAIL(e.what()); } cout << "done" << endl; } /* -* Check that append correctly adds to the end of the array _fcntl_arr after recover(). -*/ -QPID_AUTO_TEST_CASE(recover_append) -{ - string test_name = get_test_name(test_filename, "recover_append"); - lid_fid_map lfm(test_name, test_name); - flist fidl; - flist lidl; - rcvdat rd; - try - { - test_jrnl jc(test_name, test_dir, test_name); - lfmgr lm; - - lfm.journal_create(4, 4); - lfm.journal_insert(0); - lfm.journal_insert(2, 2); - lfm.journal_insert(6); - lfm.get_fid_list(fidl); - lfm.get_lid_list(lidl); - lfm.write_journal(); - lfmgr_test_helper::rcvdat_init(rd, fidl); - lm.recover(rd, &jc, &jc.new_fcntl); - lfmgr_test_helper::check_fids_lids(lm, fidl, lidl); - - // Append single file - lfm.journal_insert(7); - lfm.get_fid_list(fidl); - lfm.get_lid_list(lidl); - lm.append(&jc, &jc.new_fcntl); - lfmgr_test_helper::check_fids_lids(lm, fidl, lidl); - - // Append 3 more files - lfm.journal_insert(8, 3); - lfm.get_fid_list(fidl); - lfm.get_lid_list(lidl); - lm.append(&jc, &jc.new_fcntl, 3); - lfmgr_test_helper::check_fids_lids(lm, fidl, lidl); - } - catch(const exception& e) { BOOST_FAIL(e.what()); } - cout << "done" << endl; -} - -/* -* Check that insert() works correctly after initialize() and shifts the fid sequence beyond the insert point correctly: -* -* The following sequence is tested: -* initialize 4 fids=[0,1,2,3] lids=[0,1,2,3] -* insert 1 after lid 0 fids=[0,4,1,2,3] lids=[0,2,3,4,1] -* insert 2 after lid 2 fids=[0,4,1,5,6,2,3] lids=[0,2,5,6,1,3,4] -* insert 1 after lid 6 fids=[0,4,1,5,6,2,3,7] lids=[0,2,5,6,1,3,4,7] -* issert 1 after lid 3 fids=[0,4,1,5,8,6,2,3,7] lids=[0,2,6,7,1,3,5,8,4] -*/ -QPID_AUTO_TEST_CASE(initialize_insert) -{ - string test_name = get_test_name(test_filename, "initialize_insert"); - u_int16_t num_jfiles = 4; - u_int16_t fids1[] = {0, 4, 1, 2, 3}; - u_int16_t lids1[] = {0, 2, 3, 4, 1}; - u_int16_t fids2[] = {0, 4, 1, 5, 6, 2, 3}; - u_int16_t lids2[] = {0, 2, 5, 6, 1, 3, 4}; - u_int16_t fids3[] = {0, 4, 1, 5, 6, 2, 3, 7}; - u_int16_t lids3[] = {0, 2, 5, 6, 1, 3, 4, 7}; - u_int16_t fids4[] = {0, 4, 1, 5, 8, 6, 2, 3, 7}; - u_int16_t lids4[] = {0, 2, 6, 7, 1, 3, 5, 8, 4}; - try - { - test_jrnl jc(test_name, test_dir, test_name); - lfmgr lm; - lm.initialize(num_jfiles, &jc, &jc.new_fcntl); - lfmgr_test_helper::check_linear_fids_lids(lm, num_jfiles); - - lm.insert(0, &jc, &jc.new_fcntl); - lfmgr_test_helper::check_fids_lids(lm, fids1, lids1, num_jfiles + 1); - - lm.insert(2, &jc, &jc.new_fcntl, 2); - lfmgr_test_helper::check_fids_lids(lm, fids2, lids2, num_jfiles + 3); - - lm.insert(6, &jc, &jc.new_fcntl); - lfmgr_test_helper::check_fids_lids(lm, fids3, lids3, num_jfiles + 4); - - lm.insert(3, &jc, &jc.new_fcntl); - lfmgr_test_helper::check_fids_lids(lm, fids4, lids4, num_jfiles + 5); - } - catch(const exception& e) { BOOST_FAIL(e.what()); } - cout << "done" << endl; -} - -/* -* Check that insert() works correctly after recover() and shifts the fid sequence beyond the insert point correctly: -* -* The following sequence is tested: -* recover 4 fids=[0,2,3,1] lids=[0,3,1,2] -* insert 1 after lid 0 fids=[0,4,2,3,1] lids=[0,4,2,3,1] -* insert 2 after lid 2 fids=[0,4,2,5,6,3,1] lids=[0,6,2,5,1,3,4] -* insert 1 after lid 6 fids=[0,4,2,5,6,3,1,7] lids=[0,6,2,5,1,3,4,7] -* issert 1 after lid 3 fids=[0,4,2,5,8,6,3,1,7] lids=[0,7,2,6,1,3,5,8,4] -*/ -QPID_AUTO_TEST_CASE(recover_insert) -{ - string test_name = get_test_name(test_filename, "recover_insert"); - lid_fid_map lfm(test_name, test_name); - flist fidl; - flist lidl; - rcvdat rd; - try - { - lfm.journal_create(4, 4); - lfm.get_fid_list(fidl); - lfm.get_lid_list(lidl); - lfm.write_journal(); - lfmgr_test_helper::rcvdat_init(rd, fidl); - test_jrnl jc(test_name, test_dir, test_name); - lfmgr lm; - lm.recover(rd, &jc, &jc.new_fcntl); - lfmgr_test_helper::check_fids_lids(lm, fidl, lidl); - - lfm.journal_insert(0); - lfm.get_fid_list(fidl); - lfm.get_lid_list(lidl); - lm.insert(0, &jc, &jc.new_fcntl); - lfmgr_test_helper::check_fids_lids(lm, fidl, lidl); - - lfm.journal_insert(2, 2); - lfm.get_fid_list(fidl); - lfm.get_lid_list(lidl); - lm.insert(2, &jc, &jc.new_fcntl, 2); - lfmgr_test_helper::check_fids_lids(lm, fidl, lidl); - - lfm.journal_insert(6); - lfm.get_fid_list(fidl); - lfm.get_lid_list(lidl); - lm.insert(6, &jc, &jc.new_fcntl); - lfmgr_test_helper::check_fids_lids(lm, fidl, lidl); - - lfm.journal_insert(3); - lfm.get_fid_list(fidl); - lfm.get_lid_list(lidl); - lm.insert(3, &jc, &jc.new_fcntl); - lfmgr_test_helper::check_fids_lids(lm, fidl, lidl); - } - catch(const exception& e) { BOOST_FAIL(e.what()); } - cout << "done" << endl; -} - -/* * Tests randomized combinations of initialization/recovery, initial size, number, size and location of inserts. * * To reproduce a specific test, comment out the get_seed() statement and uncomment the literal below, adjusting the seed @@ -777,7 +768,7 @@ { string test_name = get_test_name(test_filename, "randomized_tests"); const long seed = get_seed(); - // const long seed = 0xd94c29b3; + // const long seed = 0x2d9b69d32; cout << "seed=0x" << hex << seed << dec << " " << flush; ::srand48(seed); @@ -785,20 +776,24 @@ flist fidl; flist lidl; rcvdat rd; + u_int16_t curr_ae_max_jfiles = 0; + jdir::create_dir(test_dir); // Check test dir exists; create it if not for (int test_num = 0; test_num < 100; test_num++) { test_jrnl jc(test_name, test_dir, test_name); lfmgr lm; - // 50% chance of recovery except first run and if >= 800 journal files - const bool recover_flag = test_num > 0 && lfm.size() < 800 ? 2.0 * ::drand48() < 1.0 : false; + // 50% chance of recovery except first run and if there is still ae space left + const bool recover_flag = test_num > 0 && + curr_ae_max_jfiles > lfm.size() && + 2.0 * ::drand48() < 1.0; if (recover_flag) { // Recover from previous iteration lfm.get_fid_list(fidl); lfm.get_lid_list(lidl); - lfm.write_journal(); - lfmgr_test_helper::rcvdat_init(rd, fidl); + lfm.write_journal(true, curr_ae_max_jfiles); + lfmgr_test_helper::rcvdat_init(rd, fidl, true, curr_ae_max_jfiles); lm.recover(rd, &jc, &jc.new_fcntl); lfmgr_test_helper::check_fids_lids(lm, fidl, lidl); } @@ -806,31 +801,53 @@ { // Initialize from scratch const u_int16_t num_jfiles = 4 + u_int16_t(21.0 * ::drand48()); // size: 4 - 25 files + curr_ae_max_jfiles = u_int16_t(4 * num_jfiles * ::drand48()); // size: 0 - 100 files + if (curr_ae_max_jfiles > JRNL_MAX_NUM_FILES) curr_ae_max_jfiles = JRNL_MAX_NUM_FILES; + else if (curr_ae_max_jfiles <= num_jfiles) curr_ae_max_jfiles = 0; lfm.destroy_journal(); lfm.journal_create(num_jfiles, num_jfiles); lfm.get_fid_list(fidl); lfm.get_lid_list(lidl); - lm.initialize(num_jfiles, &jc, &jc.new_fcntl); + lm.initialize(num_jfiles, true, curr_ae_max_jfiles, &jc, &jc.new_fcntl); lfmgr_test_helper::check_linear_fids_lids(lm, num_jfiles); } // Loop to insert fids - const int num_inserts = 1 + int(lfm.size()/10.0 * ::drand48()); + const int num_inserts = 1 + int(lfm.size() * ::drand48()); for (int i = 0; i < num_inserts; i++) { - const u_int16_t size = lm.size(); + const u_int16_t size = lm.num_jfiles(); const u_int16_t after_lid = u_int16_t(1.0 * size * ::drand48()); - const u_int16_t num_files = 1 + u_int16_t(4.0 * ::drand48()); - lfm.journal_insert(after_lid, num_files); - lfm.get_fid_list(fidl); - lfm.get_lid_list(lidl); + const u_int16_t num_jfiles = 1 + u_int16_t(4.0 * ::drand48()); + const bool legal = lm.ae_max_jfiles() + ? size + num_jfiles <= lm.ae_max_jfiles() + : size + num_jfiles <= JRNL_MAX_NUM_FILES; + if (legal) + { + lfm.journal_insert(after_lid, num_jfiles); + lfm.get_fid_list(fidl); + lfm.get_lid_list(lidl); - lm.insert(after_lid, &jc, &jc.new_fcntl, num_files); - lfmgr_test_helper::check_fids_lids(lm, fidl, lidl); + lm.insert(after_lid, &jc, &jc.new_fcntl, num_jfiles); + lfmgr_test_helper::check_fids_lids(lm, fidl, lidl); + } + else + { + try + { + lm.insert(after_lid, &jc, &jc.new_fcntl, num_jfiles); + BOOST_FAIL("lfmgr::insert() succeeded and exceeded limit"); + } + catch (const jexception& e) + { + BOOST_CHECK_EQUAL(e.err_code(), jerrno::JERR_LFMGR_AEFNUMLIMIT); + break; // no more inserts... + } + } } lm.finalize(); BOOST_CHECK_EQUAL(lm.is_init(), false); - BOOST_CHECK_EQUAL(lm.size(), size_t(0)); + BOOST_CHECK_EQUAL(lm.num_jfiles(), u_int16_t(0)); BOOST_CHECK_EQUAL(lm.get_fcntlp(0), (void*)0); } cout << "done" << endl;