Library: option to provide an explicit random seed for tests

* add new option to the commandline option parser
 * pass this as std::optional to the test-suite constructor
 * use this value optionally to inject a fixed value on re-seeding
 * provide diagnostic output to show the actual seed value used

src/lib/test/suite.cpp

@@ -28,6 +28,7 @@
 
 
 #include "lib/error.hpp"
+#include "lib/symbol.hpp"
 #include "lib/format-cout.hpp"
 #include "lib/test/suite.hpp"
 #include "lib/test/run.hpp"
@@ -50,10 +51,12 @@ namespace test {
   using std::vector;
   using std::optional;
   using std::shared_ptr;
+  using std::string_literals::operator ""s;
   using boost::algorithm::trim;
   
   using util::isnil;
   using util::contains;
+  using util::toString;
   using util::typeStr;
   using lib::SeedNucleus;
   using lib::Random;
@@ -103,13 +106,17 @@ namespace test {
     class SuiteSeedNucleus
       : public SeedNucleus
       {
-        optional<uint64_t> fixedSeed_;
+      public:
+        /** optionally a fixed random seed to inject in each invoked test */
+        opt_uint64 fixedSeed;
         
         uint64_t
         getSeed()  override
           {
-            return fixedSeed_? *fixedSeed_
+            auto seed = fixedSeed? *fixedSeed : lib::entropyGen.u64();
-                             : lib::entropyGen.u64();
+            auto kind = fixedSeed? "!fix" : "rand";
+            NOTICE (test, "     ++>>> SEED(%s) <<<: %s", kind, toString(seed).c_str());
+            return seed;
           }
       };
     
@@ -128,9 +135,8 @@ namespace test {
    *  @param test the Launcher object used to run this test
    *  @param testID unique ID to refer to this test (will be used as std::map key)
    *  @param groups List of group-IDs selected by whitespace
-   * 
    */
-  void 
+  void
   Suite::enrol (Launcher* test, string testID, string groups)
   {
     REQUIRE( test );
@@ -154,12 +160,11 @@ namespace test {
 
   
   
-  /** create a suite comprised of all the testcases 
+  /** create a suite comprised of all the testcases
-   *  previously @link #enrol() registered @endlink with this
+   *  previously @link #enrol() registered @endlink with this this group.
-   *  this group. 
+   *  @see #run() running tests in a Suite
-   *  @see #run() running tests in a Suite 
    */
-  Suite::Suite(string groupID) 
+  Suite::Suite (string groupID, opt_uint64 optSeed)
     : groupID_(groupID)
     , exitCode_(0)
   {
@@ -169,6 +174,8 @@ namespace test {
     // Seed random number generator
     std::srand (std::time (nullptr));
     
+    suiteSeed.fixedSeed = optSeed;
+    
     if (!testcases.getGroup(groupID))
       throw lumiera::error::Invalid ("empty testsuite");
   }
@@ -181,7 +188,7 @@ namespace test {
     }
   
   
-    
+  
 #define IS_VALID(test,testID) \
   ASSERT ((test), "NULL testcase launcher for test '%s' found in testsuite '%s'", groupID_.c_str(),testID.c_str());
   
@@ -191,8 +198,7 @@ namespace test {
     int
     invokeTestCase (Test& theTest, Arg cmdline)
     {
-      try 
+      try {
-        {
           INFO (test, "++------------------- invoking TEST: %s", cStr(typeStr (theTest)));
           theTest.run (cmdline);
           return Suite::TEST_OK;
@@ -227,13 +233,13 @@ namespace test {
   
   
   /** run all testcases contained in this Suite.
-   *  The first argument in the commandline, if present, 
+   *  The first argument in the commandline, if present,
    *  will select one single testcase with a matching ID.
    *  In case of invoking a single testcase, the given cmdline
    *  will be forwarded to the testcase, after removing the
-   *  testcaseID from cmdline[0]. Otherwise, every testcase 
+   *  testcaseID from cmdline[0]. Otherwise, every testcase
    *  in this suite is invoked with a empty cmdline vector.
-   *  @param cmdline ref to the vector of commandline tokens  
+   *  @param cmdline ref to the vector of commandline tokens
    */
   bool
   Suite::run (Arg cmdline)
@@ -248,7 +254,7 @@ namespace test {
         trim(testID);
         if ( contains (*tests, testID))
           {
-            // first cmdline argument denotes a valid testcase registered in 
+            // first cmdline argument denotes a valid testcase registered in
             // this group: invoke just this test with the remaining cmdline
             Launcher* test = (*tests)[testID];
             IS_VALID (test,testID);
@@ -303,7 +309,7 @@ namespace test {
           {
             test->makeInstance()->run(noCmdline); // run it to insert test generated output
           }
-        catch (...) 
+        catch (...)
           {
             cout << "PLANNED ============= " << lumiera_error() << "\n";
           }

src/lib/test/suite.hpp

@@ -42,12 +42,14 @@
 
 #include <vector>
 #include <string>
+#include <optional>
 
 
 
 namespace test {
   
   using std::string;
+  using opt_uint64 = std::optional<uint64_t>;
   
   // Forward declarations for run.hpp
   class Test;
@@ -69,7 +71,7 @@ namespace test {
       int exitCode_;
       
     public:
-      Suite (string groupID);
+      Suite (string groupID, opt_uint64 seed);
       bool run (Arg cmdline);
       void describe ();
       int getExitCode ()  const;

src/lib/test/testoption.cpp

@@ -39,6 +39,7 @@ typedef boost::program_options::variables_map VarMap;
 namespace op = boost::program_options;
 
 using lib::VectS;
+using std::optional;
 
 namespace test {
   
@@ -61,6 +62,8 @@ namespace test {
                         "the group (selection) of testcases to execute")
         ("describe",    op::bool_switch(),
                         "enumerate all testcases in this Suite in a format usable with ./test.sh.")
+        ("seed",        op::value<uint64_t>(),
+                        "the group (selection) of testcases to execute")
         ("id",          op::value<VectS>(),
                         "an individual testcase to be called.\nIf not specified, run all.")
         ;
@@ -107,6 +110,15 @@ namespace test {
       return string ();
   }
   
+  optional<uint64_t>
+  TestOption::optSeed()
+  {
+    if (parameters.count ("seed"))
+      return parameters["seed"].as<uint64_t>();
+    else
+      return std::nullopt;
+  }
+  
   /** @return \c true if --describe switch was given */
   bool 
   TestOption::shouldDescribe ()

src/lib/test/testoption.hpp

@@ -34,6 +34,7 @@
 
 #include <string>
 #include <iostream>
+#include <optional>
 #include <boost/program_options.hpp>
 #include <boost/utility.hpp>
 
@@ -61,10 +62,12 @@ namespace test {
     {
     public:
       TestOption (lib::Cmdline& cmdline);
-      const string getTestgroup ();
+      const string getTestgroup();
-      const string getTestID ();
+      const string getTestID();
       bool handleHelpRequest();
-      bool shouldDescribe ();
+      bool shouldDescribe();
+      
+      std::optional<uint64_t> optSeed();
       
     private:
       boost::program_options::options_description syntax;

tests/00support.tests

@@ -65,38 +65,55 @@ END
 
 
 TEST "Testsuite option handling" TestOption_test <<END
-out: Testing invocation with cmdline: ...
+out-lit: Testing invocation with cmdline: ...
-out: --> Testgroup=ALL
+out-lit: --> Testgroup=ALL
-out: --> Test-ID  =--missing--
+out-lit: --> Test-ID  =--missing--
-out: --> remaining=
+out-lit: --> remaining=
-out: Testing invocation with cmdline: --help...
+out-lit: Testing invocation with cmdline: --help...
-out: --> Testgroup=ALL
+out-lit: --> Testgroup=ALL
-out: --> Test-ID  =--missing--
+out-lit: --> Test-ID  =--missing--
-out: --> remaining=
+out-lit: --> remaining=
-out: Testing invocation with cmdline: --group TestGroupID...
+out-lit: Testing invocation with cmdline: --group TestGroupID...
-out: --> Testgroup=TestGroupID
+out-lit: --> Testgroup=TestGroupID
-out: --> Test-ID  =--missing--
+out-lit: --> Test-ID  =--missing--
-out: --> remaining=
+out-lit: --> remaining=
-out: Testing invocation with cmdline:  SingleTestID...
+out-lit: Testing invocation with cmdline:  SingleTestID...
-out: --> Testgroup=ALL
+out-lit: --> Testgroup=ALL
-out: --> Test-ID  =SingleTestID
+out-lit: --> Test-ID  =SingleTestID
-out: --> remaining=SingleTestID
+out-lit: --> remaining=SingleTestID
-out: Testing invocation with cmdline:  SingleTestID --group TestGroupID...
+out-lit: Testing invocation with cmdline:  SingleTestID --group TestGroupID...
-out: --> Testgroup=TestGroupID
+out-lit: --> Testgroup=TestGroupID
-out: --> Test-ID  =SingleTestID
+out-lit: --> Test-ID  =SingleTestID
-out: --> remaining=SingleTestID
+out-lit: --> remaining=SingleTestID
-out: Testing invocation with cmdline:  --group TestGroupID SingleTestID ...
+out-lit: Testing invocation with cmdline:  --group TestGroupID SingleTestID ...
-out: --> Testgroup=TestGroupID
+out-lit: --> Testgroup=TestGroupID
-out: --> Test-ID  =SingleTestID
+out-lit: --> Test-ID  =SingleTestID
-out: --> remaining=SingleTestID
+out-lit: --> remaining=SingleTestID
-out: Testing invocation with cmdline:  --group TestGroupID SingleTestID spam eggs...
+out-lit: Testing invocation with cmdline:  --group TestGroupID SingleTestID spam eggs...
-out: --> Testgroup=TestGroupID
+out-lit: --> Testgroup=TestGroupID
-out: --> Test-ID  =SingleTestID
+out-lit: --> Test-ID  =SingleTestID
-out: --> remaining=SingleTestID spam eggs
+out-lit: --> remaining=SingleTestID spam eggs
-out: Testing invocation with cmdline: 		SingleTestID spam --group TestGroupID    	 --eggs...
+out: Testing invocation with cmdline: .+SingleTestID spam --group TestGroupID.+--eggs...
-out: --> Testgroup=TestGroupID
+out-lit: --> Testgroup=TestGroupID
-out: --> Test-ID  =SingleTestID
+out-lit: --> Test-ID  =SingleTestID
-out: --> remaining=SingleTestID spam --eggs
+out-lit: --> remaining=SingleTestID spam --eggs
+out-lit: Testing invocation with cmdline: ham --group spam...
+out-lit: --> Testgroup=spam
+out-lit: --> Test-ID  =ham
+out-lit: --> remaining=ham
+out-lit: Testing invocation with cmdline: ham --seed 7 spam...
+out-lit: --> Testgroup=ALL
+out-lit: --> Test-ID  =ham
+out-lit: --> remaining=ham spam
+out-lit: Testing invocation with cmdline: ham --seed 0 spam...
+out-lit: --> Testgroup=ALL
+out-lit: --> Test-ID  =ham
+out-lit: --> remaining=ham spam
+out-lit: Testing invocation with cmdline: ham --seed spam...
+out-lit: Testing invocation with cmdline: --seed=-1...
+out-lit: --> Testgroup=ALL
+out-lit: --> Test-ID  =--missing--
+out-lit: --> remaining=
 return: 0
 END
 

tests/library/random-test.cpp

@@ -49,10 +49,14 @@ namespace test {
         }
       
       
-      /** @test demonstrate usage of default random number generators */
+      /** @test demonstrate usage of default random number generators.
+       *  @note should [draw a seed](\ref Test::seedRand()) once per Test instance
+       */
       void
       simpleUsage()
         {
+          seedRand();
+          
           int r1 = rani();
           CHECK (0 <= r1 and r1 < RAND_MAX);
           

tests/library/test/test-option-test.cpp

@@ -26,11 +26,14 @@
 
 
 #include "lib/test/run.hpp"
+#include "lib/test/test-helper.hpp"
 #include "lib/test/testoption.hpp"
 #include "lib/format-cout.hpp"
 #include "lib/util.hpp"
 
+#include <memory>
 
+using std::make_unique;
 using lib::Cmdline;
 using util::isnil;
 
@@ -55,20 +58,23 @@ namespace test {
           groupFilter2();
           additionalCmd();
           additionalCmd2();
+          verify_seed();
         }
       
       
       /** @test performs the actual test for the option parser test::TestOption */
-      void doIt (const string cmdline)
+      auto
+      doIt (const string cmdline)
         {
           cout << "Testing invocation with cmdline: " << cmdline << "..." << endl;
           
           Cmdline args(cmdline);
-          TestOption optparser (args);
+          auto optparser = make_unique<TestOption>(args);
-          const string testID = optparser.getTestID();
+          const string testID = optparser->getTestID();
-          cout << "--> Testgroup=" << optparser.getTestgroup() << endl;
+          cout << "--> Testgroup=" << optparser->getTestgroup() << endl;
           cout << "--> Test-ID  =" << (isnil(testID)? "--missing--" : testID ) << endl;
           cout << "--> remaining=" << args << endl;
+          return std::move (optparser);
         }
       
       void noOptions()      { doIt (""); }
@@ -80,6 +86,20 @@ namespace test {
       void additionalCmd()  { doIt (" --group TestGroupID SingleTestID spam eggs"); }
       void additionalCmd2() { doIt ("\t\tSingleTestID spam --group TestGroupID    \t --eggs"); }
       
+      void
+      verify_seed()
+        {
+          CHECK ( not   doIt("ham --group spam")->optSeed());
+          CHECK (7 == * doIt("ham --seed 7 spam")->optSeed());
+          CHECK (0 == * doIt("ham --seed 0 spam")->optSeed());
+          
+          VERIFY_FAIL("argument (\'spam\') for option \'--seed\' is invalid"
+                     , doIt("ham --seed spam"));
+          
+          // yet negative numbers are parsed and force-converted
+          CHECK (std::numeric_limits<uint64_t>::max()
+                   == * doIt("--seed=-1")->optSeed());
+        }
     };
   
   LAUNCHER (TestOption_test, "function common");    

tests/testrunner.cpp

@@ -48,7 +48,7 @@ int main (int argc, const char* argv[])
 {
   lib::Cmdline args (argc,argv);
   test::TestOption optparser (args);
-  test::Suite suite (optparser.getTestgroup());
+  test::Suite suite (optparser.getTestgroup(), optparser.optSeed());
   LifecycleHook::trigger (ON_GLOBAL_INIT);
   
   if (optparser.shouldDescribe())

wiki/thinkPad.ichthyo.mm

@@ -57376,14 +57376,16 @@
 </node>
 <node BACKGROUND_COLOR="#eef0c5" COLOR="#990000" CREATED="1731208110582" ID="ID_991761175" MODIFIED="1731208117297" TEXT="Implementierung">
 <icon BUILTIN="pencil"/>
-<node BACKGROUND_COLOR="#f0d5c5" COLOR="#990033" CREATED="1731208120755" ID="ID_1347870232" MODIFIED="1731208132196" TEXT="wo wird der SeedNucleus angesiedelt?">
+<node COLOR="#435e98" CREATED="1731208120755" ID="ID_1347870232" MODIFIED="1731338725711" TEXT="wo wird der SeedNucleus angesiedelt?">
 <icon BUILTIN="help"/>
 <node CREATED="1731208158296" ID="ID_1282528205" MODIFIED="1731208169202" TEXT="von der Struktur her geh&#xf6;rt der in die Suite"/>
-<node BACKGROUND_COLOR="#f8f1cb" COLOR="#a50125" CREATED="1731208171087" ID="ID_913749404" MODIFIED="1731208197128" TEXT="zur Ausf&#xfc;hrungszeit ist aber der Options-Parser schon nicht mehr zug&#xe4;nglich">
+<node BACKGROUND_COLOR="#e0ceaa" COLOR="#690f14" CREATED="1731208171087" ID="ID_913749404" MODIFIED="1731338684522" TEXT="zur Ausf&#xfc;hrungszeit ist aber der Options-Parser schon nicht mehr zug&#xe4;nglich">
 <icon BUILTIN="messagebox_warning"/>
 </node>
-<node CREATED="1731208198122" ID="ID_556038851" MODIFIED="1731208233810" TEXT="also w&#xe4;re das wohl als Konstruktor-Parametrisierung explizit zu setzen"/>
+<node COLOR="#435e98" CREATED="1731208198122" ID="ID_556038851" MODIFIED="1731338693385" TEXT="also w&#xe4;re das wohl als Konstruktor-Parametrisierung explizit zu setzen">
-<node BACKGROUND_COLOR="#fdfdcf" COLOR="#ff0000" CREATED="1731208300347" ID="ID_157287510" MODIFIED="1731208332129" TEXT="tats&#xe4;chlich aber brauche ich den Nucleus in der einzelnen Test-Subklasse">
+<icon BUILTIN="yes"/>
+</node>
+<node COLOR="#435e98" CREATED="1731208300347" ID="ID_157287510" MODIFIED="1731338680995" TEXT="tats&#xe4;chlich aber brauche ich den Nucleus in der einzelnen Test-Subklasse">
 <icon BUILTIN="yes"/>
 <node CREATED="1731209451725" ID="ID_774812395" MODIFIED="1731209472619" TEXT="die Suite ist mit dem Nucleus parametrisiert"/>
 <node CREATED="1731247770927" ID="ID_886572495" MODIFIED="1731247837299" TEXT="sieht nach Dependency-Injection aus und ist effektiv global">
@@ -57394,11 +57396,13 @@
       das ist der klassische Fall, in dem eine &#187;saubere&#171; Repr&#228;sentation in der einzelnen Testklasse zu massiver Speicherverschwendung f&#252;hrt, weil jede Testklasse einen redundanten Pointer auf den gleichen globalen Kontext bekommt
     </p>
   </body>
-</html>
+</html></richcontent>
-</richcontent>
+</node>
+<node COLOR="#338800" CREATED="1731272249906" ID="ID_1748350472" MODIFIED="1731272278660" TEXT="verwende eine lokal f&#xfc;r die Suite zug&#xe4;ngliche SeedNucleus-Instanz">
+<icon BUILTIN="button_ok"/>
 </node>
 </node>
-<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1731251266883" ID="ID_326274878" MODIFIED="1731251304272" TEXT="au&#xdf;erdem brauchen wir einen Seed-Wrapper um einen bestehenden Generator">
+<node COLOR="#435e98" CREATED="1731251266883" ID="ID_326274878" MODIFIED="1731338677029" TEXT="au&#xdf;erdem brauchen wir einen Seed-Wrapper um einen bestehenden Generator">
 <icon BUILTIN="yes"/>
 <node CREATED="1731251305887" ID="ID_889654920" MODIFIED="1731251314393" TEXT="und das ist ein &#xe4;hnlich gelagertes Problem"/>
 <node CREATED="1731251315053" ID="ID_667235767" MODIFIED="1731251348353" TEXT="das Design mit dem SeedNucleus als virtuelles Interface hat seine Schw&#xe4;chen">
@@ -57418,6 +57422,9 @@
 </node>
 </node>
 </node>
+<node COLOR="#338800" CREATED="1731338700848" ID="ID_1793720955" MODIFIED="1731338719820" TEXT="Log-Ausgabe beim &#xbb;ziehen&#xab; des Seed erzeugen">
+<icon BUILTIN="button_ok"/>
+</node>
 </node>
 </node>
 </node>