From ce2116fccd0e787dd6a6d402cabaaf1eafd2ae06 Mon Sep 17 00:00:00 2001 From: Ichthyostega Date: Mon, 11 Nov 2024 16:31:43 +0100 Subject: [PATCH] Library: option to provide an explicit random seed for tests * add new option to the commandline option parser * pass this as std::optional to the test-suite constructor * use this value optionally to inject a fixed value on re-seeding * provide diagnostic output to show the actual seed value used --- src/lib/test/suite.cpp | 42 +++++++------ src/lib/test/suite.hpp | 4 +- src/lib/test/testoption.cpp | 12 ++++ src/lib/test/testoption.hpp | 9 ++- tests/00support.tests | 81 +++++++++++++++---------- tests/library/random-test.cpp | 6 +- tests/library/test/test-option-test.cpp | 28 +++++++-- tests/testrunner.cpp | 2 +- wiki/thinkPad.ichthyo.mm | 21 ++++--- 9 files changed, 138 insertions(+), 67 deletions(-) diff --git a/src/lib/test/suite.cpp b/src/lib/test/suite.cpp index 4b1fc402d..539032e41 100644 --- a/src/lib/test/suite.cpp +++ b/src/lib/test/suite.cpp @@ -28,6 +28,7 @@ #include "lib/error.hpp" +#include "lib/symbol.hpp" #include "lib/format-cout.hpp" #include "lib/test/suite.hpp" #include "lib/test/run.hpp" @@ -50,10 +51,12 @@ namespace test { using std::vector; using std::optional; using std::shared_ptr; + using std::string_literals::operator ""s; using boost::algorithm::trim; using util::isnil; using util::contains; + using util::toString; using util::typeStr; using lib::SeedNucleus; using lib::Random; @@ -103,13 +106,17 @@ namespace test { class SuiteSeedNucleus : public SeedNucleus { - optional fixedSeed_; + public: + /** optionally a fixed random seed to inject in each invoked test */ + opt_uint64 fixedSeed; uint64_t getSeed() override { - return fixedSeed_? *fixedSeed_ - : lib::entropyGen.u64(); + auto seed = fixedSeed? *fixedSeed : lib::entropyGen.u64(); + auto kind = fixedSeed? "!fix" : "rand"; + NOTICE (test, " ++>>> SEED(%s) <<<: %s", kind, toString(seed).c_str()); + return seed; } }; @@ -128,9 +135,8 @@ namespace test { * @param test the Launcher object used to run this test * @param testID unique ID to refer to this test (will be used as std::map key) * @param groups List of group-IDs selected by whitespace - * */ - void + void Suite::enrol (Launcher* test, string testID, string groups) { REQUIRE( test ); @@ -154,12 +160,11 @@ namespace test { - /** create a suite comprised of all the testcases - * previously @link #enrol() registered @endlink with this - * this group. - * @see #run() running tests in a Suite + /** create a suite comprised of all the testcases + * previously @link #enrol() registered @endlink with this this group. + * @see #run() running tests in a Suite */ - Suite::Suite(string groupID) + Suite::Suite (string groupID, opt_uint64 optSeed) : groupID_(groupID) , exitCode_(0) { @@ -169,6 +174,8 @@ namespace test { // Seed random number generator std::srand (std::time (nullptr)); + suiteSeed.fixedSeed = optSeed; + if (!testcases.getGroup(groupID)) throw lumiera::error::Invalid ("empty testsuite"); } @@ -181,7 +188,7 @@ namespace test { } - + #define IS_VALID(test,testID) \ ASSERT ((test), "NULL testcase launcher for test '%s' found in testsuite '%s'", groupID_.c_str(),testID.c_str()); @@ -191,8 +198,7 @@ namespace test { int invokeTestCase (Test& theTest, Arg cmdline) { - try - { + try { INFO (test, "++------------------- invoking TEST: %s", cStr(typeStr (theTest))); theTest.run (cmdline); return Suite::TEST_OK; @@ -227,13 +233,13 @@ namespace test { /** run all testcases contained in this Suite. - * The first argument in the commandline, if present, + * The first argument in the commandline, if present, * will select one single testcase with a matching ID. * In case of invoking a single testcase, the given cmdline * will be forwarded to the testcase, after removing the - * testcaseID from cmdline[0]. Otherwise, every testcase + * testcaseID from cmdline[0]. Otherwise, every testcase * in this suite is invoked with a empty cmdline vector. - * @param cmdline ref to the vector of commandline tokens + * @param cmdline ref to the vector of commandline tokens */ bool Suite::run (Arg cmdline) @@ -248,7 +254,7 @@ namespace test { trim(testID); if ( contains (*tests, testID)) { - // first cmdline argument denotes a valid testcase registered in + // first cmdline argument denotes a valid testcase registered in // this group: invoke just this test with the remaining cmdline Launcher* test = (*tests)[testID]; IS_VALID (test,testID); @@ -303,7 +309,7 @@ namespace test { { test->makeInstance()->run(noCmdline); // run it to insert test generated output } - catch (...) + catch (...) { cout << "PLANNED ============= " << lumiera_error() << "\n"; } diff --git a/src/lib/test/suite.hpp b/src/lib/test/suite.hpp index 6e88d4a2b..1a49ec514 100644 --- a/src/lib/test/suite.hpp +++ b/src/lib/test/suite.hpp @@ -42,12 +42,14 @@ #include #include +#include namespace test { using std::string; + using opt_uint64 = std::optional; // Forward declarations for run.hpp class Test; @@ -69,7 +71,7 @@ namespace test { int exitCode_; public: - Suite (string groupID); + Suite (string groupID, opt_uint64 seed); bool run (Arg cmdline); void describe (); int getExitCode () const; diff --git a/src/lib/test/testoption.cpp b/src/lib/test/testoption.cpp index 9f6f3718f..706e71cef 100644 --- a/src/lib/test/testoption.cpp +++ b/src/lib/test/testoption.cpp @@ -39,6 +39,7 @@ typedef boost::program_options::variables_map VarMap; namespace op = boost::program_options; using lib::VectS; +using std::optional; namespace test { @@ -61,6 +62,8 @@ namespace test { "the group (selection) of testcases to execute") ("describe", op::bool_switch(), "enumerate all testcases in this Suite in a format usable with ./test.sh.") + ("seed", op::value(), + "the group (selection) of testcases to execute") ("id", op::value(), "an individual testcase to be called.\nIf not specified, run all.") ; @@ -107,6 +110,15 @@ namespace test { return string (); } + optional + TestOption::optSeed() + { + if (parameters.count ("seed")) + return parameters["seed"].as(); + else + return std::nullopt; + } + /** @return \c true if --describe switch was given */ bool TestOption::shouldDescribe () diff --git a/src/lib/test/testoption.hpp b/src/lib/test/testoption.hpp index cb6427378..804424e15 100644 --- a/src/lib/test/testoption.hpp +++ b/src/lib/test/testoption.hpp @@ -34,6 +34,7 @@ #include #include +#include #include #include @@ -61,10 +62,12 @@ namespace test { { public: TestOption (lib::Cmdline& cmdline); - const string getTestgroup (); - const string getTestID (); + const string getTestgroup(); + const string getTestID(); bool handleHelpRequest(); - bool shouldDescribe (); + bool shouldDescribe(); + + std::optional optSeed(); private: boost::program_options::options_description syntax; diff --git a/tests/00support.tests b/tests/00support.tests index 2d55fb421..3e8fbc752 100644 --- a/tests/00support.tests +++ b/tests/00support.tests @@ -65,38 +65,55 @@ END TEST "Testsuite option handling" TestOption_test < Testgroup=ALL -out: --> Test-ID =--missing-- -out: --> remaining= -out: Testing invocation with cmdline: --help... -out: --> Testgroup=ALL -out: --> Test-ID =--missing-- -out: --> remaining= -out: Testing invocation with cmdline: --group TestGroupID... -out: --> Testgroup=TestGroupID -out: --> Test-ID =--missing-- -out: --> remaining= -out: Testing invocation with cmdline: SingleTestID... -out: --> Testgroup=ALL -out: --> Test-ID =SingleTestID -out: --> remaining=SingleTestID -out: Testing invocation with cmdline: SingleTestID --group TestGroupID... -out: --> Testgroup=TestGroupID -out: --> Test-ID =SingleTestID -out: --> remaining=SingleTestID -out: Testing invocation with cmdline: --group TestGroupID SingleTestID ... -out: --> Testgroup=TestGroupID -out: --> Test-ID =SingleTestID -out: --> remaining=SingleTestID -out: Testing invocation with cmdline: --group TestGroupID SingleTestID spam eggs... -out: --> Testgroup=TestGroupID -out: --> Test-ID =SingleTestID -out: --> remaining=SingleTestID spam eggs -out: Testing invocation with cmdline: SingleTestID spam --group TestGroupID --eggs... -out: --> Testgroup=TestGroupID -out: --> Test-ID =SingleTestID -out: --> remaining=SingleTestID spam --eggs +out-lit: Testing invocation with cmdline: ... +out-lit: --> Testgroup=ALL +out-lit: --> Test-ID =--missing-- +out-lit: --> remaining= +out-lit: Testing invocation with cmdline: --help... +out-lit: --> Testgroup=ALL +out-lit: --> Test-ID =--missing-- +out-lit: --> remaining= +out-lit: Testing invocation with cmdline: --group TestGroupID... +out-lit: --> Testgroup=TestGroupID +out-lit: --> Test-ID =--missing-- +out-lit: --> remaining= +out-lit: Testing invocation with cmdline: SingleTestID... +out-lit: --> Testgroup=ALL +out-lit: --> Test-ID =SingleTestID +out-lit: --> remaining=SingleTestID +out-lit: Testing invocation with cmdline: SingleTestID --group TestGroupID... +out-lit: --> Testgroup=TestGroupID +out-lit: --> Test-ID =SingleTestID +out-lit: --> remaining=SingleTestID +out-lit: Testing invocation with cmdline: --group TestGroupID SingleTestID ... +out-lit: --> Testgroup=TestGroupID +out-lit: --> Test-ID =SingleTestID +out-lit: --> remaining=SingleTestID +out-lit: Testing invocation with cmdline: --group TestGroupID SingleTestID spam eggs... +out-lit: --> Testgroup=TestGroupID +out-lit: --> Test-ID =SingleTestID +out-lit: --> remaining=SingleTestID spam eggs +out: Testing invocation with cmdline: .+SingleTestID spam --group TestGroupID.+--eggs... +out-lit: --> Testgroup=TestGroupID +out-lit: --> Test-ID =SingleTestID +out-lit: --> remaining=SingleTestID spam --eggs +out-lit: Testing invocation with cmdline: ham --group spam... +out-lit: --> Testgroup=spam +out-lit: --> Test-ID =ham +out-lit: --> remaining=ham +out-lit: Testing invocation with cmdline: ham --seed 7 spam... +out-lit: --> Testgroup=ALL +out-lit: --> Test-ID =ham +out-lit: --> remaining=ham spam +out-lit: Testing invocation with cmdline: ham --seed 0 spam... +out-lit: --> Testgroup=ALL +out-lit: --> Test-ID =ham +out-lit: --> remaining=ham spam +out-lit: Testing invocation with cmdline: ham --seed spam... +out-lit: Testing invocation with cmdline: --seed=-1... +out-lit: --> Testgroup=ALL +out-lit: --> Test-ID =--missing-- +out-lit: --> remaining= return: 0 END diff --git a/tests/library/random-test.cpp b/tests/library/random-test.cpp index 260d687d7..ae18a18ad 100644 --- a/tests/library/random-test.cpp +++ b/tests/library/random-test.cpp @@ -49,10 +49,14 @@ namespace test { } - /** @test demonstrate usage of default random number generators */ + /** @test demonstrate usage of default random number generators. + * @note should [draw a seed](\ref Test::seedRand()) once per Test instance + */ void simpleUsage() { + seedRand(); + int r1 = rani(); CHECK (0 <= r1 and r1 < RAND_MAX); diff --git a/tests/library/test/test-option-test.cpp b/tests/library/test/test-option-test.cpp index dfb47bfeb..0518e4e35 100644 --- a/tests/library/test/test-option-test.cpp +++ b/tests/library/test/test-option-test.cpp @@ -26,11 +26,14 @@ #include "lib/test/run.hpp" +#include "lib/test/test-helper.hpp" #include "lib/test/testoption.hpp" #include "lib/format-cout.hpp" #include "lib/util.hpp" +#include +using std::make_unique; using lib::Cmdline; using util::isnil; @@ -55,20 +58,23 @@ namespace test { groupFilter2(); additionalCmd(); additionalCmd2(); + verify_seed(); } /** @test performs the actual test for the option parser test::TestOption */ - void doIt (const string cmdline) + auto + doIt (const string cmdline) { cout << "Testing invocation with cmdline: " << cmdline << "..." << endl; Cmdline args(cmdline); - TestOption optparser (args); - const string testID = optparser.getTestID(); - cout << "--> Testgroup=" << optparser.getTestgroup() << endl; + auto optparser = make_unique(args); + const string testID = optparser->getTestID(); + cout << "--> Testgroup=" << optparser->getTestgroup() << endl; cout << "--> Test-ID =" << (isnil(testID)? "--missing--" : testID ) << endl; cout << "--> remaining=" << args << endl; + return std::move (optparser); } void noOptions() { doIt (""); } @@ -80,6 +86,20 @@ namespace test { void additionalCmd() { doIt (" --group TestGroupID SingleTestID spam eggs"); } void additionalCmd2() { doIt ("\t\tSingleTestID spam --group TestGroupID \t --eggs"); } + void + verify_seed() + { + CHECK ( not doIt("ham --group spam")->optSeed()); + CHECK (7 == * doIt("ham --seed 7 spam")->optSeed()); + CHECK (0 == * doIt("ham --seed 0 spam")->optSeed()); + + VERIFY_FAIL("argument (\'spam\') for option \'--seed\' is invalid" + , doIt("ham --seed spam")); + + // yet negative numbers are parsed and force-converted + CHECK (std::numeric_limits::max() + == * doIt("--seed=-1")->optSeed()); + } }; LAUNCHER (TestOption_test, "function common"); diff --git a/tests/testrunner.cpp b/tests/testrunner.cpp index 475dd3c0d..2cf899b40 100644 --- a/tests/testrunner.cpp +++ b/tests/testrunner.cpp @@ -48,7 +48,7 @@ int main (int argc, const char* argv[]) { lib::Cmdline args (argc,argv); test::TestOption optparser (args); - test::Suite suite (optparser.getTestgroup()); + test::Suite suite (optparser.getTestgroup(), optparser.optSeed()); LifecycleHook::trigger (ON_GLOBAL_INIT); if (optparser.shouldDescribe()) diff --git a/wiki/thinkPad.ichthyo.mm b/wiki/thinkPad.ichthyo.mm index 3375642ed..c7464d4c3 100644 --- a/wiki/thinkPad.ichthyo.mm +++ b/wiki/thinkPad.ichthyo.mm @@ -57376,14 +57376,16 @@ - + - + - - + + + + @@ -57394,11 +57396,13 @@ das ist der klassische Fall, in dem eine »saubere« Repräsentation in der einzelnen Testklasse zu massiver Speicherverschwendung führt, weil jede Testklasse einen redundanten Pointer auf den gleichen globalen Kontext bekommt

- - + + + + - + @@ -57418,6 +57422,9 @@ + + +