ichthyo/LUMIERA.clone
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Library: option to provide an explicit random seed for tests

Browse source 
* add new option to the commandline option parser
 * pass this as std::optional to the test-suite constructor
 * use this value optionally to inject a fixed value on re-seeding
 * provide diagnostic output to show the actual seed value used
This commit is contained in:
Fischlurch 2024-11-11 16:31:43 +01:00
parent cffa2cd6c2
commit ce2116fccd
9 changed files with 138 additions and 67 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
src/lib/test/suite.cpp
View file

@ -28,6 +28,7 @@
#include "lib/error.hpp"
#include "lib/symbol.hpp"
#include "lib/format-cout.hpp"
#include "lib/test/suite.hpp"
#include "lib/test/run.hpp"
@ -50,10 +51,12 @@ namespace test {
using std::vector;
using std::optional;
using std::shared_ptr;
using std::string_literals::operator ""s;
using boost::algorithm::trim;
using util::isnil;
using util::contains;
using util::toString;
using util::typeStr;
using lib::SeedNucleus;
using lib::Random;
@ -103,13 +106,17 @@ namespace test {
class SuiteSeedNucleus
: public SeedNucleus
{
optional<uint64_t> fixedSeed_;
public:
/** optionally a fixed random seed to inject in each invoked test */
opt_uint64 fixedSeed;
uint64_t
getSeed() override
{
return fixedSeed_? *fixedSeed_
: lib::entropyGen.u64();
auto seed = fixedSeed? *fixedSeed : lib::entropyGen.u64();
auto kind = fixedSeed? "!fix" : "rand";
NOTICE (test, " ++>>> SEED(%s) <<<: %s", kind, toString(seed).c_str());
return seed;
}
};
@ -128,7 +135,6 @@ namespace test {
* @param test the Launcher object used to run this test
* @param testID unique ID to refer to this test (will be used as std::map key)
* @param groups List of group-IDs selected by whitespace
*
*/
void
Suite::enrol (Launcher* test, string testID, string groups)
@ -155,11 +161,10 @@ namespace test {
/** create a suite comprised of all the testcases
* previously @link #enrol() registered @endlink with this
* this group.
* previously @link #enrol() registered @endlink with this this group.
* @see #run() running tests in a Suite
*/
Suite::Suite(string groupID)
Suite::Suite (string groupID, opt_uint64 optSeed)
: groupID_(groupID)
, exitCode_(0)
{
@ -169,6 +174,8 @@ namespace test {
// Seed random number generator
std::srand (std::time (nullptr));
suiteSeed.fixedSeed = optSeed;
if (!testcases.getGroup(groupID))
throw lumiera::error::Invalid ("empty testsuite");
}
@ -191,8 +198,7 @@ namespace test {
int
invokeTestCase (Test& theTest, Arg cmdline)
{
try
{
try {
INFO (test, "++------------------- invoking TEST: %s", cStr(typeStr (theTest)));
theTest.run (cmdline);
return Suite::TEST_OK;

4
src/lib/test/suite.hpp
View file

@ -42,12 +42,14 @@
#include <vector>
#include <string>
#include <optional>
namespace test {
using std::string;
using opt_uint64 = std::optional<uint64_t>;
// Forward declarations for run.hpp
class Test;
@ -69,7 +71,7 @@ namespace test {
int exitCode_;
public:
Suite (string groupID);
Suite (string groupID, opt_uint64 seed);
bool run (Arg cmdline);
void describe ();
int getExitCode () const;

12
src/lib/test/testoption.cpp
View file

@ -39,6 +39,7 @@ typedef boost::program_options::variables_map VarMap;
namespace op = boost::program_options;
using lib::VectS;
using std::optional;
namespace test {
@ -61,6 +62,8 @@ namespace test {
"the group (selection) of testcases to execute")
("describe", op::bool_switch(),
"enumerate all testcases in this Suite in a format usable with ./test.sh.")
("seed", op::value<uint64_t>(),
"the group (selection) of testcases to execute")
("id", op::value<VectS>(),
"an individual testcase to be called.\nIf not specified, run all.")
;
@ -107,6 +110,15 @@ namespace test {
return string ();
}
optional<uint64_t>
TestOption::optSeed()
{
if (parameters.count ("seed"))
return parameters["seed"].as<uint64_t>();
else
return std::nullopt;
}
/** @return \c true if --describe switch was given */
bool
TestOption::shouldDescribe ()

3
src/lib/test/testoption.hpp
View file

@ -34,6 +34,7 @@
#include <string>
#include <iostream>
#include <optional>
#include <boost/program_options.hpp>
#include <boost/utility.hpp>
@ -66,6 +67,8 @@ namespace test {
bool handleHelpRequest();
bool shouldDescribe();
std::optional<uint64_t> optSeed();
private:
boost::program_options::options_description syntax;
boost::program_options::variables_map parameters;

81
tests/00support.tests
View file

@ -65,38 +65,55 @@ END
TEST "Testsuite option handling" TestOption_test <<END
out: Testing invocation with cmdline: ...
out: --> Testgroup=ALL
out: --> Test-ID =--missing--
out: --> remaining=
out: Testing invocation with cmdline: --help...
out: --> Testgroup=ALL
out: --> Test-ID =--missing--
out: --> remaining=
out: Testing invocation with cmdline: --group TestGroupID...
out: --> Testgroup=TestGroupID
out: --> Test-ID =--missing--
out: --> remaining=
out: Testing invocation with cmdline: SingleTestID...
out: --> Testgroup=ALL
out: --> Test-ID =SingleTestID
out: --> remaining=SingleTestID
out: Testing invocation with cmdline: SingleTestID --group TestGroupID...
out: --> Testgroup=TestGroupID
out: --> Test-ID =SingleTestID
out: --> remaining=SingleTestID
out: Testing invocation with cmdline: --group TestGroupID SingleTestID ...
out: --> Testgroup=TestGroupID
out: --> Test-ID =SingleTestID
out: --> remaining=SingleTestID
out: Testing invocation with cmdline: --group TestGroupID SingleTestID spam eggs...
out: --> Testgroup=TestGroupID
out: --> Test-ID =SingleTestID
out: --> remaining=SingleTestID spam eggs
out: Testing invocation with cmdline: SingleTestID spam --group TestGroupID --eggs...
out: --> Testgroup=TestGroupID
out: --> Test-ID =SingleTestID
out: --> remaining=SingleTestID spam --eggs
out-lit: Testing invocation with cmdline: ...
out-lit: --> Testgroup=ALL
out-lit: --> Test-ID =--missing--
out-lit: --> remaining=
out-lit: Testing invocation with cmdline: --help...
out-lit: --> Testgroup=ALL
out-lit: --> Test-ID =--missing--
out-lit: --> remaining=
out-lit: Testing invocation with cmdline: --group TestGroupID...
out-lit: --> Testgroup=TestGroupID
out-lit: --> Test-ID =--missing--
out-lit: --> remaining=
out-lit: Testing invocation with cmdline: SingleTestID...
out-lit: --> Testgroup=ALL
out-lit: --> Test-ID =SingleTestID
out-lit: --> remaining=SingleTestID
out-lit: Testing invocation with cmdline: SingleTestID --group TestGroupID...
out-lit: --> Testgroup=TestGroupID
out-lit: --> Test-ID =SingleTestID
out-lit: --> remaining=SingleTestID
out-lit: Testing invocation with cmdline: --group TestGroupID SingleTestID ...
out-lit: --> Testgroup=TestGroupID
out-lit: --> Test-ID =SingleTestID
out-lit: --> remaining=SingleTestID
out-lit: Testing invocation with cmdline: --group TestGroupID SingleTestID spam eggs...
out-lit: --> Testgroup=TestGroupID
out-lit: --> Test-ID =SingleTestID
out-lit: --> remaining=SingleTestID spam eggs
out: Testing invocation with cmdline: .+SingleTestID spam --group TestGroupID.+--eggs...
out-lit: --> Testgroup=TestGroupID
out-lit: --> Test-ID =SingleTestID
out-lit: --> remaining=SingleTestID spam --eggs
out-lit: Testing invocation with cmdline: ham --group spam...
out-lit: --> Testgroup=spam
out-lit: --> Test-ID =ham
out-lit: --> remaining=ham
out-lit: Testing invocation with cmdline: ham --seed 7 spam...
out-lit: --> Testgroup=ALL
out-lit: --> Test-ID =ham
out-lit: --> remaining=ham spam
out-lit: Testing invocation with cmdline: ham --seed 0 spam...
out-lit: --> Testgroup=ALL
out-lit: --> Test-ID =ham
out-lit: --> remaining=ham spam
out-lit: Testing invocation with cmdline: ham --seed spam...
out-lit: Testing invocation with cmdline: --seed=-1...
out-lit: --> Testgroup=ALL
out-lit: --> Test-ID =--missing--
out-lit: --> remaining=
return: 0
END

6
tests/library/random-test.cpp
View file

@ -49,10 +49,14 @@ namespace test {
}
/** @test demonstrate usage of default random number generators */
/** @test demonstrate usage of default random number generators.
* @note should [draw a seed](\ref Test::seedRand()) once per Test instance
*/
void
simpleUsage()
{
seedRand();
int r1 = rani();
CHECK (0 <= r1 and r1 < RAND_MAX);

28
tests/library/test/test-option-test.cpp
View file

@ -26,11 +26,14 @@
#include "lib/test/run.hpp"
#include "lib/test/test-helper.hpp"
#include "lib/test/testoption.hpp"
#include "lib/format-cout.hpp"
#include "lib/util.hpp"
#include <memory>
using std::make_unique;
using lib::Cmdline;
using util::isnil;
@ -55,20 +58,23 @@ namespace test {
groupFilter2();
additionalCmd();
additionalCmd2();
verify_seed();
}
/** @test performs the actual test for the option parser test::TestOption */
void doIt (const string cmdline)
auto
doIt (const string cmdline)
{
cout << "Testing invocation with cmdline: " << cmdline << "..." << endl;
Cmdline args(cmdline);
TestOption optparser (args);
const string testID = optparser.getTestID();
cout << "--> Testgroup=" << optparser.getTestgroup() << endl;
auto optparser = make_unique<TestOption>(args);
const string testID = optparser->getTestID();
cout << "--> Testgroup=" << optparser->getTestgroup() << endl;
cout << "--> Test-ID =" << (isnil(testID)? "--missing--" : testID ) << endl;
cout << "--> remaining=" << args << endl;
return std::move (optparser);
}
void noOptions() { doIt (""); }
@ -80,6 +86,20 @@ namespace test {
void additionalCmd() { doIt (" --group TestGroupID SingleTestID spam eggs"); }
void additionalCmd2() { doIt ("\t\tSingleTestID spam --group TestGroupID \t --eggs"); }
void
verify_seed()
{
CHECK ( not doIt("ham --group spam")->optSeed());
CHECK (7 == * doIt("ham --seed 7 spam")->optSeed());
CHECK (0 == * doIt("ham --seed 0 spam")->optSeed());
VERIFY_FAIL("argument (\'spam\') for option \'--seed\' is invalid"
, doIt("ham --seed spam"));
// yet negative numbers are parsed and force-converted
CHECK (std::numeric_limits<uint64_t>::max()
== * doIt("--seed=-1")->optSeed());
}
};
LAUNCHER (TestOption_test, "function common");

2
tests/testrunner.cpp
View file

@ -48,7 +48,7 @@ int main (int argc, const char* argv[])
{
lib::Cmdline args (argc,argv);
test::TestOption optparser (args);
test::Suite suite (optparser.getTestgroup());
test::Suite suite (optparser.getTestgroup(), optparser.optSeed());
LifecycleHook::trigger (ON_GLOBAL_INIT);
if (optparser.shouldDescribe())

21
wiki/thinkPad.ichthyo.mm
View file

@ -57376,14 +57376,16 @@
</node>
<node BACKGROUND_COLOR="#eef0c5" COLOR="#990000" CREATED="1731208110582" ID="ID_991761175" MODIFIED="1731208117297" TEXT="Implementierung">
<icon BUILTIN="pencil"/>
<node BACKGROUND_COLOR="#f0d5c5" COLOR="#990033" CREATED="1731208120755" ID="ID_1347870232" MODIFIED="1731208132196" TEXT="wo wird der SeedNucleus angesiedelt?">
<node COLOR="#435e98" CREATED="1731208120755" ID="ID_1347870232" MODIFIED="1731338725711" TEXT="wo wird der SeedNucleus angesiedelt?">
<icon BUILTIN="help"/>
<node CREATED="1731208158296" ID="ID_1282528205" MODIFIED="1731208169202" TEXT="von der Struktur her geh&#xf6;rt der in die Suite"/>
<node BACKGROUND_COLOR="#f8f1cb" COLOR="#a50125" CREATED="1731208171087" ID="ID_913749404" MODIFIED="1731208197128" TEXT="zur Ausf&#xfc;hrungszeit ist aber der Options-Parser schon nicht mehr zug&#xe4;nglich">
<node BACKGROUND_COLOR="#e0ceaa" COLOR="#690f14" CREATED="1731208171087" ID="ID_913749404" MODIFIED="1731338684522" TEXT="zur Ausf&#xfc;hrungszeit ist aber der Options-Parser schon nicht mehr zug&#xe4;nglich">
<icon BUILTIN="messagebox_warning"/>
</node>
<node CREATED="1731208198122" ID="ID_556038851" MODIFIED="1731208233810" TEXT="also w&#xe4;re das wohl als Konstruktor-Parametrisierung explizit zu setzen"/>
<node BACKGROUND_COLOR="#fdfdcf" COLOR="#ff0000" CREATED="1731208300347" ID="ID_157287510" MODIFIED="1731208332129" TEXT="tats&#xe4;chlich aber brauche ich den Nucleus in der einzelnen Test-Subklasse">
<node COLOR="#435e98" CREATED="1731208198122" ID="ID_556038851" MODIFIED="1731338693385" TEXT="also w&#xe4;re das wohl als Konstruktor-Parametrisierung explizit zu setzen">
<icon BUILTIN="yes"/>
</node>
<node COLOR="#435e98" CREATED="1731208300347" ID="ID_157287510" MODIFIED="1731338680995" TEXT="tats&#xe4;chlich aber brauche ich den Nucleus in der einzelnen Test-Subklasse">
<icon BUILTIN="yes"/>
<node CREATED="1731209451725" ID="ID_774812395" MODIFIED="1731209472619" TEXT="die Suite ist mit dem Nucleus parametrisiert"/>
<node CREATED="1731247770927" ID="ID_886572495" MODIFIED="1731247837299" TEXT="sieht nach Dependency-Injection aus und ist effektiv global">
@ -57394,11 +57396,13 @@
das ist der klassische Fall, in dem eine &#187;saubere&#171; Repr&#228;sentation in der einzelnen Testklasse zu massiver Speicherverschwendung f&#252;hrt, weil jede Testklasse einen redundanten Pointer auf den gleichen globalen Kontext bekommt
</p>
</body>
</html>
</richcontent>
</html></richcontent>
</node>
<node COLOR="#338800" CREATED="1731272249906" ID="ID_1748350472" MODIFIED="1731272278660" TEXT="verwende eine lokal f&#xfc;r die Suite zug&#xe4;ngliche SeedNucleus-Instanz">
<icon BUILTIN="button_ok"/>
</node>
</node>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1731251266883" ID="ID_326274878" MODIFIED="1731251304272" TEXT="au&#xdf;erdem brauchen wir einen Seed-Wrapper um einen bestehenden Generator">
<node COLOR="#435e98" CREATED="1731251266883" ID="ID_326274878" MODIFIED="1731338677029" TEXT="au&#xdf;erdem brauchen wir einen Seed-Wrapper um einen bestehenden Generator">
<icon BUILTIN="yes"/>
<node CREATED="1731251305887" ID="ID_889654920" MODIFIED="1731251314393" TEXT="und das ist ein &#xe4;hnlich gelagertes Problem"/>
<node CREATED="1731251315053" ID="ID_667235767" MODIFIED="1731251348353" TEXT="das Design mit dem SeedNucleus als virtuelles Interface hat seine Schw&#xe4;chen">
@ -57418,6 +57422,9 @@
</node>
</node>
</node>
<node COLOR="#338800" CREATED="1731338700848" ID="ID_1793720955" MODIFIED="1731338719820" TEXT="Log-Ausgabe beim &#xbb;ziehen&#xab; des Seed erzeugen">
<icon BUILTIN="button_ok"/>
</node>
</node>
</node>
</node>