benn/lumiera_
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Scheduler-test: incorporate statistics computation

Browse source 
adapt the code written yesterday explicitly for the test case
into the new framework for performing a stress-test run.
Notable difference: times converted to millisecond immediately
This commit is contained in:
Fischlurch 2024-01-03 16:27:07 +01:00
parent e704f4aae0
commit fda34a42ca
2 changed files with 105 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

86
tests/vault/gear/stress-test-rig.hpp
View file

@ -79,7 +79,7 @@
//#include <string>
//#include <vector>
#include <tuple>
//#include <array>
#include <array>
namespace vault{
@ -136,6 +136,7 @@ namespace test {
double stdDev{0};
double avgDelta{0};
double avgTime{0};
double expTime{0};
};
/** prepare the ScheduleCtx for a specifically parametrised test series */
@ -148,10 +149,32 @@ namespace test {
/** perform a repetition of test runs and compute statistics */
Res
runProbes (TestSetup& testSetup)
runProbes (TestSetup& testSetup, double stressFac)
{
UNIMPLEMENTED ("test loop and statistics computation");
Res res{};
auto sqr = [](auto n){ return n*n; };
Res res;
auto& [sf,pf,sdev,avgD,avgT,expT] = res;
sf = stressFac;
expT = testSetup.getExpectedEndTime() / 1000;
std::array<double, CONF::REPETITIONS> runTime;
for (uint i=0; i<CONF::REPETITIONS; ++i)
{
runTime[i] = testSetup.launch_and_wait() / 1000;
avgT += runTime[i];
}
avgT /= CONF::REPETITIONS;
avgD = fabs (avgT-expT);
for (uint i=0; i<CONF::REPETITIONS; ++i)
{
sdev += sqr (runTime[i] - avgT);
double delta = fabs (runTime[i] - expT);
bool fail = (delta > CONF::FAIL_LIMIT);
if (fail)
++ pf;
showRun(i, delta, runTime[i], runTime[i] > avgT, fail);
}
sdev = sqrt (sdev/CONF::REPETITIONS);
showStep(res);
return res;
}
@ -173,6 +196,49 @@ namespace test {
UNIMPLEMENTED ("invoke a library implementation of binary search");
}
_Fmt fmtRun_ {"....·%-2d: Δ=%4.1f t=%4.1f %s %s"}; // i % Δ % t % t>avg? % fail?
_Fmt fmtStep_{ "%4.2f| : ∅Δ=%4.1f±%-4.2f ∅t=%4.1f %%%3.1f -- expect:%4.1fms"}; // stress % ∅Δ % σ % ∅t % fail % t-expect
_Fmt fmtResVal_{"%9s: %5.2f%s"};
_Fmt fmtResSDv_{"%9s= %5.2f ±%4.2f%s"};
void
showRun(uint i, double delta, double t, bool over, bool fail)
{
if (CONF::showRuns)
cout << fmtRun_ % i % delta % t % (over? "+":"-") % (fail? "●":"○")
<< endl;
}
void
showStep(Res& res)
{
if (CONF::showStep)
cout << fmtStep_ % res.stressFac % res.avgDelta % res.stdDev % res.avgTime % res.percentOff % res.expTime
<< endl;
}
void
showRes(Res& res)
{
if (CONF::showRes)
{
cout << fmtResVal_ % "stresFac" % res.stressFac % "" <<endl;
cout << fmtResVal_ % "fail" %(res.percentOff * 100) % '%' <<endl;
cout << fmtResSDv_ % "∅Δ" % res.avgDelta % res.stdDev % "ms"<<endl;
cout << fmtResVal_ % "runTime" % res.avgTime % "ms"<<endl;
cout << fmtResVal_ % "expected" % res.expTime % "ms"<<endl;
}
}
void
showRef(TestLoad testLoad)
{
if (CONF::showRef)
cout << fmtResVal_ % "refTime"
% (testLoad.calcRuntimeReference(CONF::LOAD_BASE) /1000)
% "ms" << endl;
}
public:
/**
@ -191,7 +257,7 @@ namespace test {
auto performEvaluation = [&](double stressFac)
{
configureTest (testSetup, stressFac);
auto res = runProbes (testSetup);
auto res = runProbes (testSetup, stressFac);
return make_tuple (decideBreakPoint(res), res);
};
@ -213,6 +279,16 @@ namespace test {
usec LOAD_BASE = 500us;
uint CONCURRENCY = work::Config::getDefaultComputationCapacity();
double FAIL_LIMIT = 2.0; ///< delta-limit when to count a run as failure
double TRIGGER_FAIL = 0.55; ///< %-fact: criterion-1 failures above this rate
double TRIGGER_SDEV = FAIL_LIMIT; ///< in ms : criterion-2 standard derivation
double TRIGGER_DELTA = 4.0; ///< in ms : criterion-3 delta above this limit
bool showRuns = false; ///< print a line for each individual run
bool showStep = true; ///< print a line for each binary search step
bool showRes = true; ///< print result data
bool showRef = true; ///< calculate single threaded reference time
static uint constexpr REPETITIONS{30};
BlockFlowAlloc bFlow{};
EngineObserver watch{};

30
wiki/thinkPad.ichthyo.mm
View file

@ -110441,6 +110441,13 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
</node>
<node BACKGROUND_COLOR="#d2beaf" COLOR="#5c4d6e" CREATED="1703798050746" ID="ID_339326725" MODIFIED="1703798059114" TEXT="Scheduler-Stre&#xdf;tests">
<icon BUILTIN="hourglass"/>
<node BACKGROUND_COLOR="#ccb59b" COLOR="#6e2a38" CREATED="1704288791820" ID="ID_1307311744" MODIFIED="1704288853776" TEXT="Achtung Fokus">
<linktarget COLOR="#903858" DESTINATION="ID_1307311744" ENDARROW="Default" ENDINCLINATION="-395;563;" ID="Arrow_ID_1663621363" SOURCE="ID_755885227" STARTARROW="None" STARTINCLINATION="-424;24;"/>
<font ITALIC="true" NAME="SansSerif" SIZE="14"/>
<icon BUILTIN="yes"/>
<node BACKGROUND_COLOR="#e0ceaa" COLOR="#690f14" CREATED="1704288869866" ID="ID_1475817699" MODIFIED="1704288911093" TEXT="Aufgabe: Belegen der grunds&#xe4;tzlichen Tauglichkeit"/>
<node BACKGROUND_COLOR="#e0ceaa" COLOR="#690f14" CREATED="1704288897590" ID="ID_1251840457" MODIFIED="1704288911094" TEXT="Parameter + Grenzen f&#xfc;r eine Standard-Last herausfinden"/>
</node>
</node>
<node CREATED="1702954431054" ID="ID_669378388" MODIFIED="1702954436305" TEXT="offen/aufzukl&#xe4;ren">
<node BACKGROUND_COLOR="#f8f1cb" COLOR="#a50125" CREATED="1702954476683" ID="ID_1030796415" MODIFIED="1702954846789" TEXT="Verhalten von Workern unter concurrency">
@ -110535,7 +110542,7 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
</html></richcontent>
<linktarget COLOR="#9198a2" DESTINATION="ID_1389677902" ENDARROW="Default" ENDINCLINATION="-297;15;" ID="Arrow_ID_16649799" SOURCE="ID_118653070" STARTARROW="None" STARTINCLINATION="209;9;"/>
</node>
<node CREATED="1698936996449" ID="ID_1112572068" MODIFIED="1698937052690" TEXT="wie aufwendig ist die kontinuierliche Job-Erzeugung?">
<node CREATED="1698936996449" ID="ID_1112572068" MODIFIED="1704288806299" TEXT="wie aufwendig ist die kontinuierliche Job-Erzeugung?">
<linktarget COLOR="#9098b3" DESTINATION="ID_1112572068" ENDARROW="Default" ENDINCLINATION="-298;17;" ID="Arrow_ID_292608667" SOURCE="ID_41803972" STARTARROW="None" STARTINCLINATION="506;32;"/>
</node>
</node>
@ -110680,6 +110687,17 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
<node CREATED="1698201366557" ID="ID_627077813" MODIFIED="1698201388021" TEXT="kombiniert mit Anforderungs-getriebenem top-down-Ansatz"/>
<node CREATED="1698201429620" ID="ID_1635005942" MODIFIED="1698201443678" TEXT="und dem Vermeiden vorgreifender Festlegungen"/>
</node>
<node BACKGROUND_COLOR="#ccb59b" COLOR="#6e2a38" CREATED="1704287909203" ID="ID_1190802048" MODIFIED="1704288012513" TEXT="Ziele? Es geht um die grunds&#xe4;tzliche Tauglichkeit des Schedulers">
<font ITALIC="true" NAME="SansSerif" SIZE="14"/>
<icon BUILTIN="yes"/>
<node CREATED="1704287928376" ID="ID_981420173" MODIFIED="1704287931644" TEXT="nicht um mehr"/>
<node CREATED="1704287932886" ID="ID_1671307096" MODIFIED="1704287950009" TEXT="der Scheduler mu&#xdf; eine typische Last zuverl&#xe4;ssig abarbeiten k&#xf6;nnen"/>
<node CREATED="1704287950525" ID="ID_777964760" MODIFIED="1704287959768" TEXT="es darf keinen exzessiven Overhead geben"/>
<node BACKGROUND_COLOR="#e0ceaa" COLOR="#690f14" CREATED="1704287960268" ID="ID_755885227" MODIFIED="1704288861878" TEXT="derzeit(Jan.24) ist dieses Ziel nicht klar im Blick">
<arrowlink COLOR="#903858" DESTINATION="ID_1307311744" ENDARROW="Default" ENDINCLINATION="-395;563;" ID="Arrow_ID_1663621363" STARTARROW="None" STARTINCLINATION="-424;24;"/>
<icon BUILTIN="messagebox_warning"/>
</node>
</node>
</node>
<node BACKGROUND_COLOR="#f0d5c5" COLOR="#990033" CREATED="1699571704643" ID="ID_1753880750" MODIFIED="1699571808810" TEXT="offengebliebene Fragen...?">
<linktarget COLOR="#93236a" DESTINATION="ID_1753880750" ENDARROW="Default" ENDINCLINATION="-499;-50;" ID="Arrow_ID_1018529334" SOURCE="ID_1657236591" STARTARROW="None" STARTINCLINATION="-219;1274;"/>
@ -110891,7 +110909,7 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
</node>
<node BACKGROUND_COLOR="#d2beaf" COLOR="#5c4d6e" CREATED="1698936950769" ID="ID_499900060" MODIFIED="1698936959061" TEXT="Einordnen der Gesamtperformance">
<icon BUILTIN="hourglass"/>
<node BACKGROUND_COLOR="#d2beaf" COLOR="#5c4d6e" CREATED="1698936822873" ID="ID_41803972" MODIFIED="1698937090405" TEXT="Performance f&#xfc;r Job-Erstellung &#xfc;berpr&#xfc;fen">
<node BACKGROUND_COLOR="#d2beaf" COLOR="#5c4d6e" CREATED="1698936822873" ID="ID_41803972" MODIFIED="1704288806299" TEXT="Performance f&#xfc;r Job-Erstellung &#xfc;berpr&#xfc;fen">
<arrowlink COLOR="#997b81" DESTINATION="ID_1985217972" ENDARROW="Default" ENDINCLINATION="863;46;" ID="Arrow_ID_1068642654" STARTARROW="None" STARTINCLINATION="793;49;"/>
<arrowlink COLOR="#9098b3" DESTINATION="ID_1112572068" ENDARROW="Default" ENDINCLINATION="-298;17;" ID="Arrow_ID_292608667" STARTARROW="None" STARTINCLINATION="506;32;"/>
<icon BUILTIN="hourglass"/>
@ -111051,11 +111069,11 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
<node CREATED="1702416877517" ID="ID_1100109564" MODIFIED="1702416887240" TEXT="Schema Kapazit&#xe4;tsverteilung"/>
<node CREATED="1702416892102" ID="ID_305734802" MODIFIED="1702416898645" TEXT="Konzept zum Umgang mit Priorit&#xe4;ten"/>
</node>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1702417929236" ID="ID_1002777321" MODIFIED="1702417950197" TEXT="SchedulerProcessing">
<icon BUILTIN="flag-yellow"/>
<node BACKGROUND_COLOR="#eef0c5" COLOR="#990000" CREATED="1702417929236" ID="ID_1002777321" MODIFIED="1704287870161" TEXT="SchedulerProcessing">
<icon BUILTIN="pencil"/>
</node>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1702417945750" ID="ID_219061374" MODIFIED="1702417950198" TEXT="SchedulerTest">
<icon BUILTIN="flag-yellow"/>
<node BACKGROUND_COLOR="#eef0c5" COLOR="#990000" CREATED="1702417945750" ID="ID_219061374" MODIFIED="1704287870162" TEXT="SchedulerTest">
<icon BUILTIN="pencil"/>
</node>
</node>
</node>