benn/lumiera_
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Scheduler-test: rework ParameterRange tool for data visualisation

Browse source 
Rework the existing tool to capture the measurement series
into the newly integrated CSV-based data storage, allowing
to turn the results into a Gnuplot-visualisation.
This commit is contained in:
Fischlurch 2024-04-04 00:44:11 +02:00
parent 55f8f229f1
commit 0d3dc91584
5 changed files with 178 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
src/lib/incidence-count.hpp
View file

@ -188,11 +188,11 @@ namespace lib {
return idx < data.size()? data[idx]
: VAL{};
}
size_t cntCase (size_t id) { return access (caseCntr, id); }
size_t cntThread(size_t id) { return access (thrdCntr, id); }
double timeCase (size_t id) { return access (caseTime, id); }
double timeThread(size_t id) { return access (thrdTime, id); }
double timeAtConc(size_t id) { return access (concTime, id); }
size_t cntCase (size_t id) const { return access (caseCntr, id); }
size_t cntThread(size_t id) const { return access (thrdCntr, id); }
double timeCase (size_t id) const { return access (caseTime, id); }
double timeThread(size_t id) const { return access (thrdTime, id); }
double timeAtConc(size_t id) const { return access (concTime, id); }
};
Statistic evaluate();

4
src/lib/stat/data.hpp
View file

@ -128,7 +128,7 @@ namespace stat{
*/
template<typename VAL>
struct Column
: util::NonCopyable
: util::MoveOnly
{
string header;
vector<VAL> data;
@ -193,7 +193,7 @@ namespace stat{
template<class TAB>
class DataFile
: public TAB
, util::NonCopyable
, util::MoveOnly
{
fs::path filename_;

30
tests/vault/gear/scheduler-stress-test.cpp
View file

@ -32,6 +32,7 @@
#include "lib/time/timevalue.hpp"
#include "lib/format-string.hpp"
#include "lib/format-cout.hpp"
#include "lib/gnuplot-gen.hpp"
#include "lib/test/diagnostic-output.hpp"//////////////////////////TODO work in distress
//#include "lib/format-string.hpp"
#include "lib/test/transiently.hpp"
@ -394,24 +395,33 @@ namespace test {
uint CONCURRENCY = 4;
bool showRuns = true;
auto testLoad(size_t nodes)
using Param = size_t;
using Table = bench::DataTable<Param>;
auto
testLoad(Param nodes)
{
TestChainLoad testLoad{nodes};
// return testLoad.seedingRule(testLoad.rule().probability(0.6).minVal(2))
// .pruningRule(testLoad.rule().probability(0.44))
// .weightRule(testLoad.value(1))
// .setSeed(55);
// return testLoad.setWeight(1);
return testLoad.configure_isolated_nodes();
}
void
collectResult(Table& data, double millis, bench::IncidenceStat const& stat)
{
data.time = stat.coveredTime / 1000;
data.conc = stat.avgConcurrency;
data.jobtime = stat.activeTime/stat.activationCnt;
data.overhead = stat.timeAtConc(1) / stat.activationCnt; ////OOO not really clear if sensible
}
};
auto results = StressRig::with<Setup>()
.perform<bench::ParameterRange> (2,64);
cout<<"\"len\";\"dur\""<<endl;
for (auto val : results)
cout<<val.first<<";"<<val.second<<endl;
auto csv = results.renderCSV();
cout << csv <<endl;
cout << "───═══───═══───═══───═══───═══───═══───═══───═══───═══───═══───"<<endl;
cout << lib::gnuplot_gen::scatterRegression(csv);
}

77
tests/vault/gear/stress-test-rig.hpp
View file

@ -93,6 +93,7 @@
#include "lib/meta/function.hpp"
#include "lib/format-string.hpp"
#include "lib/format-cout.hpp"//////////////////////////TODO RLY?
#include "lib/stat/data.hpp"
#include "lib/util.hpp"
//#include <functional>
@ -421,6 +422,34 @@ namespace test {
using lib::stat::Column;
using lib::stat::DataFile;
using lib::stat::CSVData;
using IncidenceStat = lib::IncidenceCount::Statistic;
template<typename PAR>
struct DataRow
{
Column<PAR> param {"test param"}; // independent variable / control parameter
Column<double> time {"result time"};
Column<double> conc {"concurrency"};
Column<double> jobtime {"avg jobtime"};
Column<double> overhead{"overhead"};
auto allColumns()
{ return std::tie(param
,time
,conc
,jobtime
,overhead
);
}
};
template<typename PAR>
using DataTable = DataFile<DataRow<PAR>>;
/**************************************************//**
* Specific test scheme to perform a Scheduler setup
@ -431,18 +460,17 @@ namespace test {
class ParameterRange
: public CONF
{
using TestLoad = decltype(declval<ParameterRange>().testLoad(1));
using Table = typename CONF::Table;
using Param = typename CONF::Param;
using TestLoad = decltype(declval<ParameterRange>().testLoad(Param()));
using TestSetup = decltype(declval<ParameterRange>().testSetup (declval<TestLoad&>()));
template<typename PAR>
using Point = std::pair<PAR, double>;
template<typename PAR>
void
runTest (Point<PAR>& point)
runTest (Table& data)
{
PAR param = point.first;
Param param = data.param;
double stressFac = 1.0;
TestLoad testLoad = CONF::testLoad(param).buildTopology();
TestSetup testSetup = CONF::testSetup (testLoad)
@ -452,10 +480,9 @@ namespace test {
.withSchedDepends(CONF::SCHED_DEPENDS)
.withAdaptedSchedule(stressFac, CONF::CONCURRENCY)
.withInstrumentation();
double testMillis = testSetup.launch_and_wait() / 1000;
double millis = testSetup.launch_and_wait() / 1000;
auto stat = testSetup.getInvocationStatistic();
point.second = stat.coveredTime / 1000;
cout << "x="<<point.first<<"\t y="<<point.second<<"\t e2e="<<testMillis<<"\t conc:"<<stat.avgConcurrency<<" ∅t="<<stat.activeTime/stat.activationCnt<<" ("<<stat.activationCnt<<")"<<endl;
CONF::collectResult (data, millis, stat);
}
public:
@ -464,31 +491,35 @@ cout << "x="<<point.first<<"\t y="<<point.second<<"\t e2e="<<testMillis<<"\t con
* varying parameter value to investigate (x,y) correlations.
* @return ////TODO a tuple `[stress-factor, ∅delta, ∅run-time]`
*/
template<typename PAR>
auto
perform (PAR lower, PAR upper)
Table
perform (Param lower, Param upper)
{
TRANSIENTLY(work::Config::COMPUTATION_CAPACITY) = CONF::CONCURRENCY;
PAR dist = upper - lower;
Param dist = upper - lower;
uint cnt = CONF::REPETITIONS;
vector<Point<PAR>> results(cnt);
PAR minP{upper}, maxP{lower};
vector<Param> points;
points.reserve (cnt);
Param minP{upper}, maxP{lower};
for (uint i=0; i<cnt; ++i)
{
auto random = double(rand())/RAND_MAX;
PAR pos = lower + PAR(floor (random*dist + 0.5));
results[i].first = pos;
Param pos = lower + Param(floor (random*dist + 0.5));
points.push_back(pos);
minP = min (pos, minP);
maxP = max (pos, maxP);
}
// ensure the bounds participate in test
if (maxP < upper) results[cnt-2].first = upper;
if (minP > lower) results[cnt-1].first = lower;
for (auto& point : results)
runTest (point);
if (maxP < upper) points[cnt-2] = upper;
if (minP > lower) points[cnt-1] = lower;
Table results;
for (Param& point : points)
{
results.newRow();
results.param = point;
runTest (results);
}
return results;
}
};

106
wiki/thinkPad.ichthyo.mm
View file

@ -111374,7 +111374,7 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
<icon BUILTIN="button_ok"/>
</node>
</node>
<node COLOR="#338800" CREATED="1707755248647" ID="ID_1760925660" MODIFIED="1708035237496" TEXT="Einsatz informell verifizieren">
<node COLOR="#338800" CREATED="1707755248647" FOLDED="true" ID="ID_1760925660" MODIFIED="1708035237496" TEXT="Einsatz informell verifizieren">
<icon BUILTIN="button_ok"/>
<node BACKGROUND_COLOR="#c8c0b6" CREATED="1707960794133" ID="ID_711451755" MODIFIED="1707960809107" TEXT="neuer Testfall in SchedulerStress_test">
<icon BUILTIN="info"/>
@ -111664,7 +111664,7 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
<node CREATED="1708268487001" ID="ID_1962998809" MODIFIED="1708268499603" TEXT="wird erst w&#xe4;hrend der Messung entdeckt"/>
<node CREATED="1708268571343" ID="ID_1673801414" MODIFIED="1708268584520" TEXT="verschiebt dann das Koordinatensystem der Proze&#xdf;steuerung"/>
</node>
<node COLOR="#338800" CREATED="1708268586535" ID="ID_551006374" MODIFIED="1708295680878" TEXT="trotzdem mal versuchen....">
<node COLOR="#338800" CREATED="1708268586535" FOLDED="true" ID="ID_551006374" MODIFIED="1708295680878" TEXT="trotzdem mal versuchen....">
<icon BUILTIN="button_ok"/>
<node CREATED="1708268601529" ID="ID_202306919" MODIFIED="1708268614652" TEXT="das &#xbb;adaptedSchedule&#xab; wendet die concurrency als Faktor an"/>
<node CREATED="1708268615552" ID="ID_632780223" MODIFIED="1708271692425" TEXT="das lie&#xdf;e sich durchaus auf einen double-Factor erweitern"/>
@ -111760,7 +111760,7 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
<node CREATED="1708275070790" ID="ID_989990379" MODIFIED="1708275106286" TEXT="Konsequenz: Anpassung des Schedule sollte graduell konvergieren"/>
</node>
</node>
<node BACKGROUND_COLOR="#c8c0b6" COLOR="#435e98" CREATED="1708295682057" ID="ID_1692984155" MODIFIED="1708650309261" TEXT="Verhalten im Test">
<node BACKGROUND_COLOR="#c8c0b6" COLOR="#435e98" CREATED="1708295682057" FOLDED="true" ID="ID_1692984155" MODIFIED="1708650309261" TEXT="Verhalten im Test">
<icon BUILTIN="list"/>
<node COLOR="#435e98" CREATED="1708295760082" ID="ID_1240131502" MODIFIED="1708295777613" TEXT="kumulierte Statistik nicht ber&#xfc;cksichtigt">
<icon BUILTIN="broken-line"/>
@ -111883,6 +111883,7 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
<icon BUILTIN="yes"/>
</node>
</node>
</node>
<node COLOR="#338800" CREATED="1708359949377" ID="ID_3066237" MODIFIED="1708359973277" TEXT="damit kommt ein Stre&#xdf;-Faktor ganz nah an 1 heraus!!">
<linktarget COLOR="#3895bf" DESTINATION="ID_3066237" ENDARROW="Default" ENDINCLINATION="-1136;-1018;" ID="Arrow_ID_1263603979" SOURCE="ID_582492163" STARTARROW="None" STARTINCLINATION="1149;47;"/>
<icon BUILTIN="ksmiletris"/>
@ -111894,7 +111895,6 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
</node>
</node>
</node>
</node>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1708651353088" ID="ID_9850721" MODIFIED="1708651367397" TEXT="erg&#xe4;nzende Me&#xdf;methode : lineare Regression">
<icon BUILTIN="flag-yellow"/>
<node CREATED="1708651381229" ID="ID_34422716" MODIFIED="1708651432845" TEXT="Ansatz: Verhaltesmuster durch anderen Me&#xdf;-Ansatz best&#xe4;tigen">
@ -111981,7 +111981,7 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
<node CREATED="1708653266575" ID="ID_974267734" MODIFIED="1708653290547" TEXT="sowohl f&#xfc;r den Parameter, alsauch f&#xfc;r den Beobachtungswert"/>
<node COLOR="#338800" CREATED="1708704390490" ID="ID_1955311779" MODIFIED="1708738756134" TEXT="erscheint l&#xf6;sbar">
<icon BUILTIN="button_ok"/>
<node CREATED="1708704485480" ID="ID_1418959649" MODIFIED="1708704532222" TEXT="mu&#xdf; &#x201e;lediglich&#x201c; die Builder-Notation auf das konrete TOOL parametrisieren"/>
<node CREATED="1708704485480" ID="ID_1418959649" MODIFIED="1708704532222" TEXT="mu&#xdf; &#x201e;lediglich&#x201c; die Builder-Notation auf das konkrete TOOL parametrisieren"/>
<node BACKGROUND_COLOR="#e0ceaa" COLOR="#0f1269" CREATED="1708704444553" ID="ID_765627090" MODIFIED="1708738903548" STYLE="bubble">
<richcontent TYPE="NODE"><html>
<head/>
@ -112386,7 +112386,7 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
</node>
</node>
</node>
<node COLOR="#338800" CREATED="1710079820663" ID="ID_346209836" MODIFIED="1712165155677" TEXT="ein Gnuplot-Skript generieren">
<node COLOR="#338800" CREATED="1710079820663" FOLDED="true" ID="ID_346209836" MODIFIED="1712165155677" TEXT="ein Gnuplot-Skript generieren">
<icon BUILTIN="button_ok"/>
<node BACKGROUND_COLOR="#b6c3c8" COLOR="#2a44ae" CREATED="1710633755833" ID="ID_395770848" MODIFIED="1712165207008" TEXT="Grundlagen Gnuplot lernen">
<arrowlink COLOR="#425fc8" DESTINATION="ID_1898552649" ENDARROW="Default" ENDINCLINATION="-1042;110;" ID="Arrow_ID_1130805562" STARTARROW="None" STARTINCLINATION="-1030;94;"/>
@ -114466,7 +114466,7 @@ std::cout &lt;&lt; tmpl.render({&quot;what&quot;, &quot;World&quot;}) &lt;&lt; s
<node CREATED="1711900361616" ID="ID_1245877883" MODIFIED="1711900371643" TEXT="dann gibt es zus&#xe4;tzliche Builder, die schon Teile vorbereiten"/>
</node>
</node>
<node COLOR="#338800" CREATED="1711820846814" ID="ID_131434294" MODIFIED="1712165063590" TEXT="Template-Bausteine">
<node COLOR="#338800" CREATED="1711820846814" FOLDED="true" ID="ID_131434294" MODIFIED="1712165063590" TEXT="Template-Bausteine">
<icon BUILTIN="button_ok"/>
<node COLOR="#338800" CREATED="1711820852404" ID="ID_1098550176" MODIFIED="1711904777126" TEXT="einfacher Daten-Plot">
<icon BUILTIN="button_ok"/>
@ -114677,8 +114677,96 @@ std::cout &lt;&lt; tmpl.render({&quot;what&quot;, &quot;World&quot;}) &lt;&lt; s
</node>
</node>
</node>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1712165247528" ID="ID_1095964212" MODIFIED="1712165254925" TEXT="Regression berechnen und visualisieren">
<icon BUILTIN="flag-yellow"/>
<node BACKGROUND_COLOR="#eef0c5" COLOR="#990000" CREATED="1712165247528" ID="ID_1095964212" MODIFIED="1712190796632" TEXT="Regression berechnen und visualisieren">
<icon BUILTIN="pencil"/>
<node COLOR="#338800" CREATED="1712180140911" ID="ID_1277334109" MODIFIED="1712190777261" TEXT="mu&#xdf; Parameter-&#xdc;beragbe und Daten-R&#xfc;ckgabe &#xfc;berarbeiten">
<icon BUILTIN="button_ok"/>
<node CREATED="1712180159277" ID="ID_1547350046" MODIFIED="1712180172746" TEXT="bisher: std::pair&lt;PAR, zeit&gt;"/>
<node CREATED="1712180185563" ID="ID_1446764491" MODIFIED="1712180246389" TEXT="gew&#xfc;nscht: Daten-Tupel(PAR, Zeit, Concurrency, JobTime)"/>
<node CREATED="1712183741715" ID="ID_986844768" MODIFIED="1712183751741" TEXT="relativ einfach realisierbar">
<node CREATED="1712183753046" ID="ID_1287892532" MODIFIED="1712183793104" TEXT="DataFile als RVO-Parameter einf&#xfc;hren"/>
<node CREATED="1712183793908" ID="ID_1299689687" MODIFIED="1712183813221" TEXT="Referenz darauf anstatt dem Tupel in die runTest(data) geben"/>
</node>
<node CREATED="1712183881848" ID="ID_277756370" MODIFIED="1712190772873" TEXT="Probleme dabei...">
<icon BUILTIN="messagebox_warning"/>
<node CREATED="1712183893055" ID="ID_1518548729" MODIFIED="1712183902505" TEXT="Table-Def mu&#xdf; &#xf6;ffentlich werden"/>
<node CREATED="1712183903573" ID="ID_1522610699" MODIFIED="1712183933315" TEXT="der &#xbb;offen gelassene&#xab; Parameter-Typ PAR macht das alles etwas kniffelig"/>
<node CREATED="1712184089624" ID="ID_1503052479" MODIFIED="1712184103952" TEXT="eigentlich ist der Parameter PAR redundant">
<icon BUILTIN="idea"/>
<node CREATED="1712184136782" ID="ID_670493985" MODIFIED="1712184148599" TEXT="es ist n&#xe4;mlich CONF::testLoad(PAR)"/>
<node CREATED="1712184184363" ID="ID_222136690" MODIFIED="1712184225643" TEXT="ich wollte n&#xe4;mlich die unabh&#xe4;ngige Variable &#xbb;offen&#xab; lassen">
<richcontent TYPE="NOTE"><html>
<head/>
<body>
<p>
damit man verschiende Messungen mit dem geleichen Tool-Code machen kann
</p>
</body>
</html></richcontent>
</node>
<node CREATED="1712184403250" ID="ID_1616254097" MODIFIED="1712184496760" TEXT="das hat so lala funktioniert....">
<richcontent TYPE="NOTE"><html>
<head/>
<body>
<p>
der Parameter bleibt in der Tat &#8222;offen&#8220; &#8212; aber allzuviel kann man damit nicht anstellen, denn
</p>
<ul>
<li>
der Parameter mu&#223; numerisch (und total geordnet) sein
</li>
<li>
er mu&#223; auf double konvertierbar sein und aus double erstellbar
</li>
<li>
und die tats&#228;chlich vorzunehmenden Messungen sind auch fix
</li>
</ul>
</body>
</html></richcontent>
</node>
<node CREATED="1712184312086" ID="ID_829995870" MODIFIED="1712190730825" TEXT="wenn schon, denn schon!">
<arrowlink COLOR="#502c93" DESTINATION="ID_577782644" ENDARROW="Default" ENDINCLINATION="12;-53;" ID="Arrow_ID_682783" STARTARROW="None" STARTINCLINATION="-133;13;"/>
<icon BUILTIN="yes"/>
</node>
</node>
<node COLOR="#338800" CREATED="1712184322989" ID="ID_577782644" MODIFIED="1712190768568" TEXT="Me&#xdf;daten-Erhebung per CONF parametrisieren">
<linktarget COLOR="#502c93" DESTINATION="ID_577782644" ENDARROW="Default" ENDINCLINATION="12;-53;" ID="Arrow_ID_682783" SOURCE="ID_829995870" STARTARROW="None" STARTINCLINATION="-133;13;"/>
<icon BUILTIN="button_ok"/>
<node CREATED="1712184635916" ID="ID_1082061089" MODIFIED="1712184646838" TEXT="stellt Typen und Bindings bereit">
<node CREATED="1712184647946" ID="ID_1912596603" MODIFIED="1712184652269" TEXT="Typ Table">
<node CREATED="1712184719760" ID="ID_1405042081" MODIFIED="1712184724092" TEXT="ist ein DataFile"/>
</node>
<node CREATED="1712184688185" ID="ID_765432091" MODIFIED="1712184692160" TEXT="Typ Param"/>
<node CREATED="1712184696043" ID="ID_366607622" MODIFIED="1712184711493" TEXT="Konvention: Param steht in Table.param (Spalte)"/>
<node CREATED="1712184732838" ID="ID_33213641" MODIFIED="1712184818671" TEXT="Methode: collectResult(Table, millis, stat&amp;)"/>
</node>
<node BACKGROUND_COLOR="#e0ceaa" COLOR="#690f14" CREATED="1712184912878" ID="ID_214292021" MODIFIED="1712190738184" TEXT="das wird aber viel Definitions-Aufwand">
<icon BUILTIN="messagebox_warning"/>
<node CREATED="1712184924845" ID="ID_1900197884" MODIFIED="1712184933964" TEXT="vordefiniertes Schema"/>
<node COLOR="#435e98" CREATED="1712184956801" ID="ID_194118436" MODIFIED="1712190764406" TEXT="vordefinieren DataTable&lt;PAR&gt;">
<icon BUILTIN="idea"/>
<node CREATED="1712184973777" ID="ID_1489071309" MODIFIED="1712184977746" TEXT="param"/>
<node CREATED="1712184979213" ID="ID_741554251" MODIFIED="1712185063979" TEXT="time"/>
<node CREATED="1712185065618" ID="ID_1263120478" MODIFIED="1712189789392" TEXT="conc"/>
<node CREATED="1712185072985" ID="ID_1200962225" MODIFIED="1712185075613" TEXT="jobTime"/>
<node CREATED="1712185076777" ID="ID_1271427599" MODIFIED="1712185113759" TEXT="overhead"/>
</node>
</node>
<node BACKGROUND_COLOR="#f0d5c5" COLOR="#990033" CREATED="1712190824923" ID="ID_1753510801" MODIFIED="1712190852825" TEXT="unklar: &#xbb;overhead&#xab;-Messung">
<icon BUILTIN="help"/>
<node CREATED="1712190864143" ID="ID_438624459" MODIFIED="1712190868082" TEXT="Aussagekraft unklar"/>
<node CREATED="1712190868742" ID="ID_1042265591" MODIFIED="1712190880728" TEXT="pa&#xdf;t nicht in das gleiche Koordinatensystem"/>
<node CREATED="1712190889155" ID="ID_841914264" MODIFIED="1712190916249" TEXT="messe jetzt: durchschnittliche Single-Thread-Zeit pro Job">
<icon BUILTIN="info"/>
</node>
</node>
</node>
</node>
</node>
<node COLOR="#338800" CREATED="1712190783314" ID="ID_561451802" MODIFIED="1712190793652" TEXT="Gnuplot-Skript generieren">
<icon BUILTIN="button_ok"/>
</node>
</node>
</node>
</node>