From fda34a42ca8b37e68063b0bffd21e906863325d2 Mon Sep 17 00:00:00 2001
From: Ichthyostega <prg@ichthyostega.de>
Date: Wed, 3 Jan 2024 16:27:07 +0100
Subject: [PATCH] Scheduler-test: incorporate statistics computation

adapt the code written yesterday explicitly for the test case
into the new framework for performing a stress-test run.
Notable difference: times converted to millisecond immediately
---
 tests/vault/gear/stress-test-rig.hpp | 86 ++++++++++++++++++++++++++--
 wiki/thinkPad.ichthyo.mm             | 30 ++++++++--
 2 files changed, 105 insertions(+), 11 deletions(-)

diff --git a/tests/vault/gear/stress-test-rig.hpp b/tests/vault/gear/stress-test-rig.hpp
index 1f055f829..e22c7cc46 100644
--- a/tests/vault/gear/stress-test-rig.hpp
+++ b/tests/vault/gear/stress-test-rig.hpp
@@ -79,7 +79,7 @@
 //#include <string>
 //#include <vector>
 #include <tuple>
-//#include <array>
+#include <array>
 
 
 namespace vault{
@@ -136,6 +136,7 @@ namespace test {
             double stdDev{0};
             double avgDelta{0};
             double avgTime{0};
+            double expTime{0};
           };
         
         /** prepare the ScheduleCtx for a specifically parametrised test series */
@@ -148,10 +149,32 @@ namespace test {
         
         /** perform a repetition of test runs and compute statistics */
         Res
-        runProbes (TestSetup& testSetup)
+        runProbes (TestSetup& testSetup, double stressFac)
           {
-            UNIMPLEMENTED ("test loop and statistics computation");
-            Res res{};
+            auto sqr = [](auto n){ return n*n; };
+            Res res;
+            auto& [sf,pf,sdev,avgD,avgT,expT] = res;
+            sf = stressFac;
+            expT = testSetup.getExpectedEndTime() / 1000;
+            std::array<double, CONF::REPETITIONS> runTime;
+            for (uint i=0; i<CONF::REPETITIONS; ++i)
+              {
+                runTime[i] = testSetup.launch_and_wait() / 1000;
+                avgT += runTime[i];
+              }
+            avgT /= CONF::REPETITIONS;
+            avgD = fabs (avgT-expT);
+            for (uint i=0; i<CONF::REPETITIONS; ++i)
+              {
+                sdev += sqr (runTime[i] - avgT);
+                double delta = fabs (runTime[i] - expT);
+                bool fail = (delta > CONF::FAIL_LIMIT);
+                if (fail)
+                  ++ pf;
+                showRun(i, delta, runTime[i], runTime[i] > avgT, fail);
+              }
+            sdev = sqrt (sdev/CONF::REPETITIONS);
+            showStep(res);
             return res;
           }
         
@@ -173,6 +196,49 @@ namespace test {
             UNIMPLEMENTED ("invoke a library implementation of binary search");
           }
         
+        _Fmt fmtRun_ {"....·%-2d:  Δ=%4.1f         t=%4.1f %s %s"};                      //      i % Δ % t % t>avg?  % fail?
+        _Fmt fmtStep_{ "%4.2f|  : ∅Δ=%4.1f±%-4.2f  ∅t=%4.1f %%%3.1f -- expect:%4.1fms"}; // stress % ∅Δ % σ % ∅t % fail % t-expect
+        _Fmt fmtResVal_{"%9s: %5.2f%s"};
+        _Fmt fmtResSDv_{"%9s= %5.2f ±%4.2f%s"};
+        
+        void
+        showRun(uint i, double delta, double t, bool over, bool fail)
+          {
+            if (CONF::showRuns)
+              cout << fmtRun_ % i % delta % t % (over? "+":"-") % (fail? "●":"○")
+                   << endl;
+          }
+        
+        void
+        showStep(Res& res)
+          {
+            if (CONF::showStep)
+              cout << fmtStep_ % res.stressFac % res.avgDelta % res.stdDev % res.avgTime % res.percentOff % res.expTime
+                   << endl;
+          }
+        
+        void
+        showRes(Res& res)
+          {
+            if (CONF::showRes)
+              {
+                cout << fmtResVal_ % "stresFac" % res.stressFac             % ""  <<endl;
+                cout << fmtResVal_ %     "fail" %(res.percentOff * 100)     % '%' <<endl;
+                cout << fmtResSDv_ %       "∅Δ" % res.avgDelta % res.stdDev % "ms"<<endl;
+                cout << fmtResVal_ %  "runTime" % res.avgTime               % "ms"<<endl;
+                cout << fmtResVal_ % "expected" % res.expTime               % "ms"<<endl;
+              }
+          }
+        
+        void
+        showRef(TestLoad testLoad)
+          {
+            if (CONF::showRef)
+              cout << fmtResVal_ % "refTime"
+                                 % (testLoad.calcRuntimeReference(CONF::LOAD_BASE) /1000)
+                                 % "ms" << endl;
+          }
+        
         
       public:
         /**
@@ -191,7 +257,7 @@ namespace test {
             auto performEvaluation = [&](double stressFac)
                                         {
                                           configureTest (testSetup, stressFac);
-                                          auto res = runProbes (testSetup);
+                                          auto res = runProbes (testSetup, stressFac);
                                           return make_tuple (decideBreakPoint(res), res);
                                         };
             
@@ -213,6 +279,16 @@ namespace test {
       
       usec LOAD_BASE = 500us;
       uint CONCURRENCY = work::Config::getDefaultComputationCapacity();
+      double FAIL_LIMIT = 2.0;          ///< delta-limit when to count a run as failure
+      double TRIGGER_FAIL = 0.55;       ///< %-fact: criterion-1 failures above this rate
+      double TRIGGER_SDEV = FAIL_LIMIT; ///< in ms : criterion-2 standard derivation
+      double TRIGGER_DELTA = 4.0;       ///< in ms : criterion-3 delta above this limit
+      bool showRuns = false;    ///< print a line for each individual run
+      bool showStep = true;     ///< print a line for each binary search step
+      bool showRes  = true;     ///< print result data
+      bool showRef  = true;     ///< calculate single threaded reference time
+      
+      static uint constexpr REPETITIONS{30};
 
       BlockFlowAlloc bFlow{};
       EngineObserver watch{};
diff --git a/wiki/thinkPad.ichthyo.mm b/wiki/thinkPad.ichthyo.mm
index 455754a20..44647f39d 100644
--- a/wiki/thinkPad.ichthyo.mm
+++ b/wiki/thinkPad.ichthyo.mm
@@ -110441,6 +110441,13 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
 </node>
 <node BACKGROUND_COLOR="#d2beaf" COLOR="#5c4d6e" CREATED="1703798050746" ID="ID_339326725" MODIFIED="1703798059114" TEXT="Scheduler-Stre&#xdf;tests">
 <icon BUILTIN="hourglass"/>
+<node BACKGROUND_COLOR="#ccb59b" COLOR="#6e2a38" CREATED="1704288791820" ID="ID_1307311744" MODIFIED="1704288853776" TEXT="Achtung Fokus">
+<linktarget COLOR="#903858" DESTINATION="ID_1307311744" ENDARROW="Default" ENDINCLINATION="-395;563;" ID="Arrow_ID_1663621363" SOURCE="ID_755885227" STARTARROW="None" STARTINCLINATION="-424;24;"/>
+<font ITALIC="true" NAME="SansSerif" SIZE="14"/>
+<icon BUILTIN="yes"/>
+<node BACKGROUND_COLOR="#e0ceaa" COLOR="#690f14" CREATED="1704288869866" ID="ID_1475817699" MODIFIED="1704288911093" TEXT="Aufgabe: Belegen der grunds&#xe4;tzlichen Tauglichkeit"/>
+<node BACKGROUND_COLOR="#e0ceaa" COLOR="#690f14" CREATED="1704288897590" ID="ID_1251840457" MODIFIED="1704288911094" TEXT="Parameter + Grenzen f&#xfc;r eine Standard-Last herausfinden"/>
+</node>
 </node>
 <node CREATED="1702954431054" ID="ID_669378388" MODIFIED="1702954436305" TEXT="offen/aufzukl&#xe4;ren">
 <node BACKGROUND_COLOR="#f8f1cb" COLOR="#a50125" CREATED="1702954476683" ID="ID_1030796415" MODIFIED="1702954846789" TEXT="Verhalten von Workern unter concurrency">
@@ -110535,7 +110542,7 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
 </html></richcontent>
 <linktarget COLOR="#9198a2" DESTINATION="ID_1389677902" ENDARROW="Default" ENDINCLINATION="-297;15;" ID="Arrow_ID_16649799" SOURCE="ID_118653070" STARTARROW="None" STARTINCLINATION="209;9;"/>
 </node>
-<node CREATED="1698936996449" ID="ID_1112572068" MODIFIED="1698937052690" TEXT="wie aufwendig ist die kontinuierliche Job-Erzeugung?">
+<node CREATED="1698936996449" ID="ID_1112572068" MODIFIED="1704288806299" TEXT="wie aufwendig ist die kontinuierliche Job-Erzeugung?">
 <linktarget COLOR="#9098b3" DESTINATION="ID_1112572068" ENDARROW="Default" ENDINCLINATION="-298;17;" ID="Arrow_ID_292608667" SOURCE="ID_41803972" STARTARROW="None" STARTINCLINATION="506;32;"/>
 </node>
 </node>
@@ -110680,6 +110687,17 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
 <node CREATED="1698201366557" ID="ID_627077813" MODIFIED="1698201388021" TEXT="kombiniert mit Anforderungs-getriebenem top-down-Ansatz"/>
 <node CREATED="1698201429620" ID="ID_1635005942" MODIFIED="1698201443678" TEXT="und dem Vermeiden vorgreifender Festlegungen"/>
 </node>
+<node BACKGROUND_COLOR="#ccb59b" COLOR="#6e2a38" CREATED="1704287909203" ID="ID_1190802048" MODIFIED="1704288012513" TEXT="Ziele? Es geht um die grunds&#xe4;tzliche Tauglichkeit des Schedulers">
+<font ITALIC="true" NAME="SansSerif" SIZE="14"/>
+<icon BUILTIN="yes"/>
+<node CREATED="1704287928376" ID="ID_981420173" MODIFIED="1704287931644" TEXT="nicht um mehr"/>
+<node CREATED="1704287932886" ID="ID_1671307096" MODIFIED="1704287950009" TEXT="der Scheduler mu&#xdf; eine typische Last zuverl&#xe4;ssig abarbeiten k&#xf6;nnen"/>
+<node CREATED="1704287950525" ID="ID_777964760" MODIFIED="1704287959768" TEXT="es darf keinen exzessiven Overhead geben"/>
+<node BACKGROUND_COLOR="#e0ceaa" COLOR="#690f14" CREATED="1704287960268" ID="ID_755885227" MODIFIED="1704288861878" TEXT="derzeit(Jan.24) ist dieses Ziel nicht klar im Blick">
+<arrowlink COLOR="#903858" DESTINATION="ID_1307311744" ENDARROW="Default" ENDINCLINATION="-395;563;" ID="Arrow_ID_1663621363" STARTARROW="None" STARTINCLINATION="-424;24;"/>
+<icon BUILTIN="messagebox_warning"/>
+</node>
+</node>
 </node>
 <node BACKGROUND_COLOR="#f0d5c5" COLOR="#990033" CREATED="1699571704643" ID="ID_1753880750" MODIFIED="1699571808810" TEXT="offengebliebene Fragen...?">
 <linktarget COLOR="#93236a" DESTINATION="ID_1753880750" ENDARROW="Default" ENDINCLINATION="-499;-50;" ID="Arrow_ID_1018529334" SOURCE="ID_1657236591" STARTARROW="None" STARTINCLINATION="-219;1274;"/>
@@ -110891,7 +110909,7 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
 </node>
 <node BACKGROUND_COLOR="#d2beaf" COLOR="#5c4d6e" CREATED="1698936950769" ID="ID_499900060" MODIFIED="1698936959061" TEXT="Einordnen der Gesamtperformance">
 <icon BUILTIN="hourglass"/>
-<node BACKGROUND_COLOR="#d2beaf" COLOR="#5c4d6e" CREATED="1698936822873" ID="ID_41803972" MODIFIED="1698937090405" TEXT="Performance f&#xfc;r Job-Erstellung &#xfc;berpr&#xfc;fen">
+<node BACKGROUND_COLOR="#d2beaf" COLOR="#5c4d6e" CREATED="1698936822873" ID="ID_41803972" MODIFIED="1704288806299" TEXT="Performance f&#xfc;r Job-Erstellung &#xfc;berpr&#xfc;fen">
 <arrowlink COLOR="#997b81" DESTINATION="ID_1985217972" ENDARROW="Default" ENDINCLINATION="863;46;" ID="Arrow_ID_1068642654" STARTARROW="None" STARTINCLINATION="793;49;"/>
 <arrowlink COLOR="#9098b3" DESTINATION="ID_1112572068" ENDARROW="Default" ENDINCLINATION="-298;17;" ID="Arrow_ID_292608667" STARTARROW="None" STARTINCLINATION="506;32;"/>
 <icon BUILTIN="hourglass"/>
@@ -111051,11 +111069,11 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
 <node CREATED="1702416877517" ID="ID_1100109564" MODIFIED="1702416887240" TEXT="Schema Kapazit&#xe4;tsverteilung"/>
 <node CREATED="1702416892102" ID="ID_305734802" MODIFIED="1702416898645" TEXT="Konzept zum Umgang mit Priorit&#xe4;ten"/>
 </node>
-<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1702417929236" ID="ID_1002777321" MODIFIED="1702417950197" TEXT="SchedulerProcessing">
-<icon BUILTIN="flag-yellow"/>
+<node BACKGROUND_COLOR="#eef0c5" COLOR="#990000" CREATED="1702417929236" ID="ID_1002777321" MODIFIED="1704287870161" TEXT="SchedulerProcessing">
+<icon BUILTIN="pencil"/>
 </node>
-<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1702417945750" ID="ID_219061374" MODIFIED="1702417950198" TEXT="SchedulerTest">
-<icon BUILTIN="flag-yellow"/>
+<node BACKGROUND_COLOR="#eef0c5" COLOR="#990000" CREATED="1702417945750" ID="ID_219061374" MODIFIED="1704287870162" TEXT="SchedulerTest">
+<icon BUILTIN="pencil"/>
 </node>
 </node>
 </node>