Author: kpvdr Date: 2010-09-22 14:16:18 -0400 (Wed, 22 Sep 2010) New Revision: 4330 Modified: store/trunk/cpp/tests/jrnl/_st_helper_fns.hpp store/trunk/cpp/tests/jrnl/_ut_jinf.cpp Log: Added two additional tests for the new broker recovery checks - inconsistent jdat file sizes and inconsistent OWI flags (where possible). Some misc spelling corrections in the comments. Modified: store/trunk/cpp/tests/jrnl/_st_helper_fns.hpp =================================================================== --- store/trunk/cpp/tests/jrnl/_st_helper_fns.hpp 2010-09-22 15:40:39 UTC (rev 4329) +++ store/trunk/cpp/tests/jrnl/_st_helper_fns.hpp 2010-09-22 18:16:18 UTC (rev 4330) @@ -104,7 +104,7 @@ * General usage pattern: * 1. Create instance of lfid_pfid_map. * 2. Call lfid_pfid_map::journal_create() to simulate initial journal creation. -* 3. (optional) Call lfid_pfid_map::journal_append() one or more times to simulate the addition of journal files. +* 3. (optional) Call lfid_pfid_map::journal_insert() one or more times to simulate the addition of journal files. * 4. Call lfid_pfid_map::write_journal() to create dummy journal files (files containing only file headers) * 5. Create and initialize the jinf object under test * 6. Call jinf::analyze() to determine the pfid order - and thus also first and last lids @@ -178,14 +178,16 @@ * 2. By definition, if a journal is not full (num_used_jfiles < num_jfiles), then all owi flags for those files * that are used must be the same. It is not possible for an overwrite situation to arise if a journal is not * full. - * 3. This function acts on map _map only, and does not create any test files. Call journal_create() to do that. + * 3. This function acts on map _map only, and does not create any test files. Call write_journal() to do that. * 4. This function must be called on a clean test object or on one where the previous test data has been - * cleared by calling journal_destroy(). Running this function moret than once on existing data will + * cleared by calling journal_destroy(). Running this function more than once on existing data will * result in invalid journals which cannot be recovered. */ void journal_create(const u_int16_t num_jfiles, // Total number of files const u_int16_t num_used_jfiles, // Number of used files, rest empty at end const u_int16_t oldest_lfid = 0, // Fid where owi reverses + const u_int16_t bad_lfid = 0, // Fid where owi reverses again (must be > oldest_lifd), + // used for testing bad owi detection const bool first_owi = false) // Value of first owi flag (ie pfid=0) { const bool full = num_used_jfiles == num_jfiles; @@ -198,8 +200,22 @@ if (pfid < num_used_jfiles) { _num_used_files = num_used_jfiles; - if (full && oldest_lfid > 0 && lfid == oldest_lfid) - owi = ~owi; // Flip owi if all files in use and oldest_lfid > 0 + /* + * Invert the owi flag from its current value (initially given by first_owi param) only if: + * 1. The journal is full (ie all files are used) + * AND + * 2. oldest_lfid param is non-zero (this is default, but lfid 0 being inverted is logically + * inconsistent with first_owi parameter being present) + * AND + * 3. Either: + * * current lfid == oldest_lfid (ie we are preparing the oldest lfid) + * OR + * * current lfid == bad_lfid AND bad_lfid > oldest (ie we are past the oldest and preparing the + * bad lfid) + */ + if (owi_invert_flag = full && oldest_lfid > 0 && + (lfid == oldest_lfid || (bad_lfid > oldest_lfid && lfid == bad_lfid))) + owi = !owi; const u_int64_t frid = u_int64_t(random()); init_fhdr(fh, frid, pfid, lfid, owi); } @@ -208,27 +224,28 @@ } /* - * Method journal_append(): Used to simulate the insertion of journal files into an existing journal. + * Method journal_insert(): Used to simulate the insertion of journal files into an existing journal. * * after_lfid: The logical file id (lfid) after which the new file is to be inserted. * num_files: The number of files to be inserted. * adjust_lids: Flag indicating that the lids of files _following_ the inserted files are to be adjusted upwards - * by the number of inserted files. Not doing so simulates a recovery immediatly after insertion + * by the number of inserted files. Not doing so simulates a recovery immediately after insertion * but before the following files are overwritten with their new lids. If this is set false, then: * a) after_lfid MUST be the most recent file (_oldest_lfid-1 ie last lfid before owi changes). * b) This call must be the last insert call. * * NOTES: - * 1. It is not possible to insert before lfid/pfid 0; thus these are always coincident. This operation is - * logically equivilent to inserting after the last lfid, which is possible. - * 2. It is not possible to append to a journal that is not full. Doing so will result in an unrecoverable + * 1. It is not possible to insert before lfid/pfid 0; thus these are always coincidental. This operation is + * logically equivalent to inserting after the last lfid, which is possible. + * 2. It is not possible to insert into a journal that is not full. Doing so will result in an unrecoverable * journal (one that is logically inconsistent that can never occur in reality). * 3. If a journal is stopped/interrupted immediately after a file insertion, there could be duplicate lids in * play at recovery, as the following file lids in their headers are only overwritten when the file is * eventually written to during normal operation. The owi flags, however, are used to determine which of the * ambiguous lids are the inserted files. + * 4. This function acts on map _map only, and does not create any test files. Call write_journal() to do that. */ - void journal_insert(const u_int16_t after_lfid, // Insert files after this lfid + void journal_insert(const u_int16_t after_lfid, // Insert files after this lfid const u_int16_t num_files = 1, // Number of files to insert const bool adjust_lids = true) // Adjust lids following inserted files { @@ -271,6 +288,11 @@ } } + /* + * Get the list of pfids in the map in order of lfid. The pfids are appended to the supplied vector. Only + * as many headers as are in the map are appended. + * NOTE: will clear any contents from supplied vector before appending pfid list. + */ void get_pfid_list(vector<u_int16_t>& pfid_list) { pfid_list.clear(); @@ -278,6 +300,11 @@ pfid_list.push_back(i->second._pfid); } + /* + * Get the list of lfids in the map. The lfids are appended to the supplied vector in the order they appear + * in the map (which is not necessarily the natural or sorted order). + * NOTE: will clear any contents from supplied vector before appending lfid list. + */ void get_lfid_list(vector<u_int16_t>& lfid_list) { lfid_list.clear(); @@ -427,7 +454,7 @@ /* * Method clean_journal_info_file(): This static method deletes only a jinf file without harming any other - * journal file or its directory. This is used to clear those tests which rely only on the existance of a + * journal file or its directory. This is used to clear those tests which rely only on the existence of a * jinf file. */ static void clean_journal_info_file(const string base_filename) @@ -437,6 +464,14 @@ BOOST_WARN_MESSAGE(::unlink(fn.str().c_str()) == 0, "clean_journal_info_file(): Failed to remove file " << fn.str()); } + static string create_journal_filename(const u_int16_t pfid, const string base_filename) + { + stringstream fn; + fn << test_dir << "/" << base_filename << "."; + fn << setfill('0') << hex << setw(4) << pfid << "." << JRNL_DATA_EXTENSION; + return fn.str(); + } + private: static void init_fhdr(file_hdr& fh, const u_int64_t frid, @@ -527,14 +562,6 @@ of.write(sblk_buffer.data(), JRNL_DBLK_SIZE * JRNL_SBLK_SIZE); } } - - static string create_journal_filename(const u_int16_t pfid, const string base_filename) - { - stringstream fn; - fn << test_dir << "/" << base_filename << "."; - fn << setfill('0') << hex << setw(4) << pfid << "." << JRNL_DATA_EXTENSION; - return fn.str(); - } }; const string Modified: store/trunk/cpp/tests/jrnl/_ut_jinf.cpp =================================================================== --- store/trunk/cpp/tests/jrnl/_ut_jinf.cpp 2010-09-22 15:40:39 UTC (rev 4329) +++ store/trunk/cpp/tests/jrnl/_ut_jinf.cpp 2010-09-22 18:16:18 UTC (rev 4330) @@ -273,7 +273,7 @@ lfid_pfid_map m(jid, base_filename); ::srand48(1); - // As this test relies on repeatable but random sequences, use many ierations for coverage + // As this test relies on repeatable but random sequences, use many iterations for coverage for (int c = 1; c <= 100; c++) { for (u_int16_t num_appends = 1; num_appends <= 2*NUM_JFILES; num_appends++) @@ -306,4 +306,99 @@ cout << "done" << endl; } +QPID_AUTO_TEST_CASE(analyze_inconsistent_jdat_file_size_in_journal) +{ + string test_name = get_test_name(test_filename, "analyze_inconsistent_jdat_file_size_in_journal"); + const string jid = test_name + "_jid"; + const string base_filename = test_name + "_bfn"; + lfid_pfid_map m(jid, base_filename); + ::srand48(1); + + for (u_int16_t pfid = 1; pfid < NUM_JFILES; pfid++) + { + m.journal_create(NUM_JFILES, NUM_JFILES, 0); + m.write_journal(false, 0); + + const std::string filename = m.create_journal_filename(pfid, base_filename); + std::ofstream of(filename.c_str(), ofstream::out | ofstream::app); + if (!of.good()) + BOOST_FAIL("Unable to open test journal file \"" << filename << "\" for writing."); + std::size_t expand_size = std::size_t(10 * JRNL_DBLK_SIZE * JRNL_SBLK_SIZE * ::drand48()); + std::vector<char> sblk_buffer(expand_size, 0); + of.write(sblk_buffer.data(), expand_size); + of.close(); + + stringstream fn; + fn << test_dir << "/" << base_filename << "." << JRNL_INFO_EXTENSION; + jinf ji(fn.str(), false); + try + { + ji.analyze(); + BOOST_FAIL("Failed to detect irregular journal file size in file \"" << filename << "\""); + } + catch (const jexception& e) {} // ignore - expected + + m.destroy_journal(); + } + cout << "done" << endl; +} + +QPID_AUTO_TEST_CASE(analyze_owi_in_non_ae_journal) +{ + string test_name = get_test_name(test_filename, "analyze_owi_in_non_ae_journal"); + const string jid = test_name + "_jid"; + const string base_filename = test_name + "_bfn"; + lfid_pfid_map m(jid, base_filename); + for (u_int16_t oldest_file = 1; oldest_file < NUM_DEFAULT_JFILES-1; oldest_file++) + { + for (u_int16_t bad_owi_file = oldest_file + 1; bad_owi_file < NUM_DEFAULT_JFILES; bad_owi_file++) + { + m.journal_create(NUM_DEFAULT_JFILES, NUM_DEFAULT_JFILES, oldest_file, bad_owi_file); + m.write_journal(false, 0); + + stringstream fn; + fn << test_dir << "/" << base_filename << "." << JRNL_INFO_EXTENSION; + jinf ji(fn.str(), false); + try + { + ji.analyze(); + BOOST_FAIL("Failed to detect irregular OWI flag in non-ae journal file \"" << fn << "\""); + } + catch (const jexception& e) {} // ignore - expected + + m.destroy_journal(); + } + } + cout << "done" << endl; +} + +QPID_AUTO_TEST_CASE(analyze_owi_in_ae_min_size_journal) +{ + string test_name = get_test_name(test_filename, "analyze_owi_in_ae_min_size_journal"); + const string jid = test_name + "_jid"; + const string base_filename = test_name + "_bfn"; + lfid_pfid_map m(jid, base_filename); + for (u_int16_t oldest_file = 1; oldest_file < NUM_JFILES-1; oldest_file++) + { + for (u_int16_t bad_owi_file = oldest_file + 1; bad_owi_file < NUM_JFILES; bad_owi_file++) + { + m.journal_create(NUM_JFILES, NUM_JFILES, oldest_file, bad_owi_file); + m.write_journal(true, 16); + + stringstream fn; + fn << test_dir << "/" << base_filename << "." << JRNL_INFO_EXTENSION; + jinf ji(fn.str(), false); + try + { + ji.analyze(); + BOOST_FAIL("Failed to detect irregular OWI flag in min-sized ae journal file \"" << fn << "\""); + } + catch (const jexception& e) {} // ignore - expected + + m.destroy_journal(); + } + } + cout << "done" << endl; +} + QPID_AUTO_TEST_SUITE_END()