benn/lumiera_
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Scheduler-test: experiment with extended load and run time

Browse source 
Various experiments to watch Scheduler behaviour under extended load.
Notably the example committed here makes the Scheduler run for 1.2 sec
and process 800 jobs with 10ms each, thereby putting the system into
100% load on all CPUs
This commit is contained in:
Fischlurch 2024-02-16 03:45:27 +01:00
parent 27b34c4ed6
commit 0e7bdcc5b5
3 changed files with 217 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
tests/vault/gear/scheduler-stress-test.cpp
View file

@ -295,8 +295,8 @@ namespace test {
verify_instrumentation()
{
MARK_TEST_FUN
TestChainLoad testLoad{20};
testLoad
TestChainLoad testLoad{800};
testLoad.setWeight(1)
.printTopologyDOT()
.printTopologyStatistics()
;
@ -304,23 +304,25 @@ namespace test {
BlockFlowAlloc bFlow;
EngineObserver watch;
Scheduler scheduler{bFlow, watch};
auto LOAD_BASE = 500us;
auto LOAD_BASE = 10ms;
// auto stressFac = 1.0;
// auto concurrency = 8;
//
ComputationalLoad cpuLoad;
cpuLoad.timeBase = LOAD_BASE;
cpuLoad.timeBase = 200us;
cpuLoad.calibrate();
//
double loadMicros = cpuLoad.invoke();
double refTime = testLoad.calcRuntimeReference(LOAD_BASE);
// double refTime = testLoad.calcRuntimeReference(LOAD_BASE);
SHOW_EXPR(loadMicros)
auto testSetup =
testLoad.setupSchedule(scheduler)
.withLoadTimeBase(LOAD_BASE)
.withJobDeadline(50ms)
.withUpfrontPlanning()
.withLevelDuration(10us)
.withJobDeadline(2s)
// .withBaseExpense(500us)
// .withUpfrontPlanning()
.withInstrumentation()
// .withAdaptedSchedule (stressFac, concurrency)
;
@ -328,7 +330,7 @@ SHOW_EXPR(loadMicros)
double expected = testSetup.getExpectedEndTime();
SHOW_EXPR(runTime)
SHOW_EXPR(expected)
SHOW_EXPR(refTime)
//SHOW_EXPR(refTime)
auto stat = testSetup.getInvocationStatistic();
SHOW_EXPR(stat.cumulatedTime);
SHOW_EXPR(stat.activeTime);

17
tests/vault/gear/test-chain-load.hpp
View file

@ -69,6 +69,10 @@
** [retrieved](\ref TestChainLoad::getHash). The node structure can then be traversed or
** [scheduled as Render Jobs](\ref TestChainLoad::scheduleJobs).
**
** A special use case is _not to build any topology,_ rather only #setWeight().
** All nodes ** will then be at level-0 and scheduled at t=0, causing the scheduler
** to process best effort in arbitrary order.
**
** ## Test support
** A tool for generating roughly calibrated [computational load](\ref ComputationalLoad) is included,
** to be controlled by the Node::weight stepping. Load can either be generated by arithmetic (hash)
@ -688,6 +692,19 @@ namespace test {
}
/**
* Set a fixed weight for all nodes
* @note no change to topology (works even without any topology)
*/
TestChainLoad&&
setWeight (size_t fixedNodeWeight =1)
{
for (Node& n : allNodes())
n.weight = fixedNodeWeight;
return move(*this);
}
/**
* Recalculate all node hashes and propagate seed value.
*/

204
wiki/thinkPad.ichthyo.mm
View file

@ -111126,23 +111126,25 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
<icon BUILTIN="button_ok"/>
</node>
</node>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1707755113190" ID="ID_320712272" MODIFIED="1707755150362" TEXT="IncidenceCount bereitstellen">
<icon BUILTIN="flag-yellow"/>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1707755183413" ID="ID_1998051706" MODIFIED="1707755247237" TEXT="Init">
<icon BUILTIN="flag-yellow"/>
<node COLOR="#338800" CREATED="1707755113190" ID="ID_320712272" MODIFIED="1708034209989" TEXT="IncidenceCount bereitstellen">
<icon BUILTIN="button_ok"/>
<node COLOR="#338800" CREATED="1707755183413" ID="ID_1998051706" MODIFIED="1708034181770" TEXT="Init">
<icon BUILTIN="button_ok"/>
</node>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1707755186021" ID="ID_1513713092" MODIFIED="1707755247238" TEXT="auto-Reset vor jedem Lauf">
<icon BUILTIN="flag-yellow"/>
<node COLOR="#5b280f" CREATED="1707755186021" ID="ID_1513713092" MODIFIED="1708034174422" TEXT="auto-Reset vor jedem Lauf">
<icon BUILTIN="button_cancel"/>
</node>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1707755215777" ID="ID_1011450460" MODIFIED="1707755247238" TEXT="Heuristik zur Dimensionierung">
<icon BUILTIN="flag-yellow"/>
<node COLOR="#338800" CREATED="1707755215777" ID="ID_1011450460" MODIFIED="1708034185433" TEXT="Heuristik zur Dimensionierung">
<icon BUILTIN="button_ok"/>
<node CREATED="1708034186458" ID="ID_1592987408" MODIFIED="1708034198381" TEXT="ganz banal: nimm Hardware-Concurrency"/>
<node CREATED="1708034198776" ID="ID_211063778" MODIFIED="1708034206811" TEXT="plus Maximalzahl an Nodes"/>
</node>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1707755225871" ID="ID_940749173" MODIFIED="1707755247238" TEXT="Statistik abrufen">
<icon BUILTIN="flag-yellow"/>
<node COLOR="#338800" CREATED="1707755225871" ID="ID_940749173" MODIFIED="1708034180623" TEXT="Statistik abrufen">
<icon BUILTIN="button_ok"/>
</node>
</node>
<node BACKGROUND_COLOR="#eef0c5" COLOR="#990000" CREATED="1707755248647" ID="ID_1760925660" MODIFIED="1707960777624" TEXT="Einsatz informell verifizieren">
<icon BUILTIN="pencil"/>
<node COLOR="#338800" CREATED="1707755248647" ID="ID_1760925660" MODIFIED="1708035237496" TEXT="Einsatz informell verifizieren">
<icon BUILTIN="button_ok"/>
<node BACKGROUND_COLOR="#c8c0b6" CREATED="1707960794133" ID="ID_711451755" MODIFIED="1707960809107" TEXT="neuer Testfall in SchedulerStress_test">
<icon BUILTIN="info"/>
</node>
@ -111255,7 +111257,7 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
<node CREATED="1707960988842" ID="ID_1122286519" MODIFIED="1707960994318" TEXT="erste Ergebnisse sehen gut aus"/>
<node CREATED="1707960994969" ID="ID_835639287" MODIFIED="1707961013083" TEXT="kein Graph &#x27f9; keine Node-weights &#x27f9; lauter leere Jobs"/>
<node CREATED="1707961014519" ID="ID_8171946" MODIFIED="1707961026267" TEXT="ist in &lt; 3ms fertig"/>
<node BACKGROUND_COLOR="#fdfdcf" COLOR="#ff0000" CREATED="1707961027284" ID="ID_995544734" MODIFIED="1707961133751" TEXT="Instrumentierung registiert nur Zeitspanne von 245&#xb5;s">
<node BACKGROUND_COLOR="#c8c0b6" COLOR="#690f14" CREATED="1707961027284" ID="ID_995544734" MODIFIED="1708029818811" TEXT="Instrumentierung registiert nur Zeitspanne von 245&#xb5;s">
<icon BUILTIN="messagebox_warning"/>
<node CREATED="1707961224035" ID="ID_47823213" MODIFIED="1707961230985" TEXT="Werte">
<icon BUILTIN="info"/>
@ -111263,11 +111265,170 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
<node CREATED="1707961179169" ID="ID_1852802808" MODIFIED="1707961198210" TEXT="sowie 34&#xb5;s single-threaded"/>
<node CREATED="1707961198678" ID="ID_1071621043" MODIFIED="1707961218591" TEXT="und 0.5&#xb5;s concurrency &#x2261; 2"/>
</node>
<node BACKGROUND_COLOR="#f0d5c5" COLOR="#990033" CREATED="1707961249356" ID="ID_1961702882" MODIFIED="1707961258458" TEXT="ist das plausibel? kann das sein?">
<node COLOR="#435e98" CREATED="1707961249356" ID="ID_1961702882" MODIFIED="1708029567421" TEXT="ist das plausibel? kann das sein?">
<icon BUILTIN="help"/>
<node BACKGROUND_COLOR="#c8c0b6" CREATED="1708028991888" ID="ID_1540292040" MODIFIED="1708029554200" TEXT="mit DUMP beobachtet">
<icon BUILTIN="idea"/>
<node CREATED="1708029003759" ID="ID_1572671669" MODIFIED="1708029016554" TEXT="Laufzeit w&#xe4;chst datdurch gleich um +1ms an"/>
<node CREATED="1708029017125" ID="ID_734606017" MODIFIED="1708029027832" TEXT="au&#xdf;erdem haben wir nun mehr Concurrency (bis 3)"/>
<node COLOR="#435e98" CREATED="1708029030612" ID="ID_111310837" MODIFIED="1708029043941" TEXT="Ablauf sieht plausibel aus">
<icon BUILTIN="yes"/>
</node>
</node>
<node CREATED="1708029051425" ID="ID_1274163927" MODIFIED="1708029061003" TEXT="Beobachtungen">
<node CREATED="1708029062111" ID="ID_735324014" MODIFIED="1708029080838" TEXT="es gibt einen erheblichen Vorlauf f&#xfc;r die Planung">
<node CREATED="1708029124935" ID="ID_1396915650" MODIFIED="1708029138698" TEXT="jeder Job braucht 100&#xb5;s f&#xfc;r Planung+Scheduling"/>
<node CREATED="1708029139349" ID="ID_208485056" MODIFIED="1708029150439" TEXT="jede Dependency auf den wake-Up braucht nochmal 50&#xb5;s"/>
</node>
<node CREATED="1708029160570" ID="ID_1497780046" MODIFIED="1708029185075" TEXT="es erst mal nur ein Worker zur Ausf&#xfc;hrung da"/>
<node CREATED="1708029185863" ID="ID_1413198418" MODIFIED="1708029198257" TEXT="der bekommt Vorfahrt f&#xfc;r den &#xbb;jeweils n&#xe4;chsten Job&#xab;"/>
<node CREATED="1708029198901" ID="ID_245244859" MODIFIED="1708029207128" TEXT="dadurch gehen weitere Worker in contention-wait"/>
<node CREATED="1708029212411" ID="ID_568904534" MODIFIED="1708029232542" TEXT="Abarbeitung (mit DUMP) braucht auch ca 100&#xb5;s / Job"/>
<node CREATED="1708029285946" ID="ID_278507918" MODIFIED="1708029411133" TEXT="das beinhaltet auch POST eines Notification follow-up"/>
<node CREATED="1708029425159" ID="ID_1329294264" MODIFIED="1708029440393" TEXT="Nachlauf: nachdem alle Jobs abgearbeitet sind...">
<node CREATED="1708029441269" ID="ID_188822401" MODIFIED="1708029458039" TEXT="werden alle Notification-Jobs aktivierbar"/>
<node CREATED="1708029458939" ID="ID_1665382552" MODIFIED="1708029498911" TEXT="(sie liegen dann schon in der Vergangenheit)">
<richcontent TYPE="NOTE"><html>
<head>
</head>
<body>
<p>
...waren aber geblockt, solange noch die eigentlichen Jobs mit t=0 in der Queue stehen
</p>
</body>
</html>
</richcontent>
</node>
<node CREATED="1708029502085" ID="ID_43040367" MODIFIED="1708029516159" TEXT="auch hierf&#xfc;r kommt nur ein Worker zum Zug"/>
<node CREATED="1708029516987" ID="ID_650283156" MODIFIED="1708029523286" TEXT="einige Contention w&#xe4;hrenddessen"/>
<node CREATED="1708029523738" ID="ID_738784561" MODIFIED="1708029536084" TEXT="&#x27f9; dauert insges. 1ms"/>
</node>
</node>
</node>
<node COLOR="#435e98" CREATED="1708029539754" ID="ID_958664378" MODIFIED="1708029564923" TEXT="Fazit: plausibel">
<font BOLD="true" NAME="SansSerif" SIZE="12"/>
<icon BUILTIN="forward"/>
<node CREATED="1708029614214" ID="ID_1728298948" MODIFIED="1708029626907" TEXT="es sind die bekannten Selbst-Regulierungsmechanismen"/>
<node CREATED="1708029738565" ID="ID_1641558536" MODIFIED="1708029756350" TEXT="w&#xe4;hrend der Organisations-Arbeit blockt das GroomingToken"/>
<node CREATED="1708029627922" ID="ID_242925751" MODIFIED="1708029790131" TEXT="der Scheduler ist nicht auf Contention ausgelegt">
<richcontent TYPE="NOTE"><html>
<head>
</head>
<body>
<p>
...und die Selbstreguliertung sorgt schntell daf&#252;r, da&#223; der gr&#246;&#223;te Teil der WorkForce in contention-Wait geht
</p>
</body>
</html>
</richcontent>
</node>
<node CREATED="1708029641698" ID="ID_1689204888" MODIFIED="1708029653045" TEXT="trotzdem findet noch eine gewisse Parallelisierung statt"/>
</node>
</node>
</node>
</node>
<node CREATED="1708029837603" ID="ID_641612793" MODIFIED="1708033285779">
<richcontent TYPE="NODE"><html>
<head>
</head>
<body>
<p>
zweiter Versuch: <i>unconnected Nodes with weight</i><font size="2">&#160;(500&#181;s)</font>
</p>
</body>
</html>
</richcontent>
<node COLOR="#338800" CREATED="1708033299088" ID="ID_1415838317" MODIFIED="1708033320164" TEXT="hierf&#xfc;r neue setWeight()">
<icon BUILTIN="button_ok"/>
<node CREATED="1708033322701" ID="ID_1244159152" MODIFIED="1708033352875" TEXT="infolge der neuen Einsicht, da&#xdf; keine Topologie notwendig ist"/>
<node CREATED="1708033353665" ID="ID_105904667" MODIFIED="1708033383612" TEXT="setzt lediglich f&#xfc;r alle Nodes den weight-factor"/>
</node>
<node CREATED="1708033391293" ID="ID_1964212800" MODIFIED="1708033396485" TEXT="Beobachtungen">
<icon BUILTIN="forward"/>
<node CREATED="1708033405521" ID="ID_1489319899" MODIFIED="1708033419148" TEXT="gemessene Zeit deutet auf 1ms / Node hin"/>
<node CREATED="1708033421848" ID="ID_1525501888" MODIFIED="1708033451278" TEXT="das hei&#xdf;t: wir verbringen doppelt so viel Zeit wie geplant"/>
<node CREATED="1708033477217" ID="ID_1969991325" MODIFIED="1708033482723" TEXT="Concurrency geht auf 4"/>
<node CREATED="1708033499822" ID="ID_1170331389" MODIFIED="1708033509675" TEXT="Laufzeit 8.4ms"/>
<node CREATED="1708033541160" ID="ID_716582623" MODIFIED="1708033567799" TEXT="coveredTime 5.118ms"/>
<node CREATED="1708033575180" ID="ID_1360284834" MODIFIED="1708033591128">
<richcontent TYPE="NODE"><html>
<head>
</head>
<body>
<p>
das entspricht<b>&#160;exakt</b>&#160;der Concurrency
</p>
</body>
</html>
</richcontent>
</node>
<node CREATED="1708033977406" ID="ID_1893515841" MODIFIED="1708033994569" TEXT="Planung abgeschlossen bei t=2.8ms"/>
<node CREATED="1708034237043" ID="ID_858054398" MODIFIED="1708034251356" TEXT="binnen 1.3ms sind alle 8 Worker da und haben einen Job &#xfc;bernommen"/>
<node CREATED="1708034482178" ID="ID_395297387" MODIFIED="1708034491600" TEXT="dann kommen aber bereits die ersten Worker zur&#xfc;ck"/>
<node BACKGROUND_COLOR="#f8f1cb" COLOR="#a50125" CREATED="1708034492457" ID="ID_1613332485" MODIFIED="1708034529957" TEXT="w&#xe4;hrend der follow-up-POST ist der Eingang f&#xfc;r 400&#xb5;s blockiert">
<icon BUILTIN="messagebox_warning"/>
</node>
<node CREATED="1708034779427" ID="ID_526130451" MODIFIED="1708034798903" TEXT="dann wirkt auch die Dominanz der R&#xfc;ckkehrer (diese kommen vorrangig zum Zug)"/>
<node CREATED="1708034799848" ID="ID_1922696787" MODIFIED="1708034809851" TEXT="ein Worker wird wegen Contention inaktiv"/>
<node CREATED="1708034833323" ID="ID_1828062709" MODIFIED="1708034842774" TEXT="eigentliche Arbeits-Phase: 2.8ms ... 5ms">
<node CREATED="1708035002864" ID="ID_755338221" MODIFIED="1708035019443" TEXT="das sind (nur) 3.2ms">
<font NAME="SansSerif" SIZE="14"/>
<icon BUILTIN="forward"/>
</node>
<node CREATED="1708034908809" ID="ID_1836121465" MODIFIED="1708034936555">
<richcontent TYPE="NODE"><html>
<head>
</head>
<body>
<p>
das ist effektiv <b>concurrency &#8801; 6</b>
</p>
</body>
</html>
</richcontent>
<icon BUILTIN="idea"/>
</node>
</node>
<node CREATED="1708034970145" ID="ID_647295832" MODIFIED="1708034986779" TEXT="es folgen 2ms zum Abarbeiten der dependency-Trigger auf den wake-up"/>
</node>
<node BACKGROUND_COLOR="#f0d5c5" COLOR="#990033" CREATED="1708033592825" ID="ID_790589282" MODIFIED="1708033600955" TEXT="Fragen / Schlu&#xdf;folgerungen">
<icon BUILTIN="help"/>
<node CREATED="1708033603304" ID="ID_1901761517" MODIFIED="1708034157042" TEXT="warum sch&#xf6;pfen wir die m&#xf6;gliche Concurrency nicht aus?">
<node CREATED="1708035045595" ID="ID_362743911" MODIFIED="1708035065348" TEXT="weil die Planung in die Contention l&#xe4;uft und viel zu lange braucht"/>
<node CREATED="1708035066068" ID="ID_838732543" MODIFIED="1708035089525" TEXT="weil der wakeup-dependency-Trigger ebenfalls erheblich Ressourcen fri&#xdf;t"/>
<node CREATED="1708035110148" ID="ID_1711271553" MODIFIED="1708035118265" TEXT="weil R&#xfc;ckkehrer bevorzugt werden"/>
</node>
<node CREATED="1708033630684" ID="ID_1325587760" MODIFIED="1708033656045" TEXT="abgesehen davon: ab Job-Start ist die Ausf&#xfc;hrung effizient"/>
</node>
</node>
<node CREATED="1708050404196" ID="ID_951934036" MODIFIED="1708050408863" TEXT="weitere Experimente">
<node CREATED="1708050412751" ID="ID_638536593" MODIFIED="1708050422134" TEXT="l&#xe4;ngere Node-Zeiten">
<node CREATED="1708050425513" ID="ID_567927785" MODIFIED="1708050453145" TEXT="5ms &#x27fc; real &#x2205; 6ms"/>
<node CREATED="1708050458589" ID="ID_1695942732" MODIFIED="1708050467855" TEXT="10ms &#x27fc; real &#x2205; 11ms"/>
<node CREATED="1708050481797" ID="ID_1497974818" MODIFIED="1708050641215">
<richcontent TYPE="NODE"><html>
<head>
</head>
<body>
<p>
mit 400 Nodes&#160;&#160;&#10236; real &#8709; 12ms und <b>Concurrency &#8801; 7.9</b>
</p>
</body>
</html>
</richcontent>
<font NAME="SansSerif" SIZE="12"/>
</node>
<node CREATED="1708051327255" ID="ID_695348528" MODIFIED="1708051345087" TEXT="mit 800 Nodes (und l&#xe4;ngerer Dealine): Scheuler l&#xe4;uft f&#xfc;r 1.2 sec"/>
</node>
</node>
<node COLOR="#338800" CREATED="1708035211793" ID="ID_1358819222" MODIFIED="1708035233542" TEXT="&#x27f9; funkioniert und f&#xfc;hrt zu plausiblen Aussagen">
<icon BUILTIN="button_ok"/>
</node>
</node>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1707755273281" ID="ID_787410697" MODIFIED="1707755280011" TEXT="geeignet Dokumentieren (Test)">
@ -111674,6 +111835,21 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1702955389111" ID="ID_265829575" LINK="#ID_1120118362" MODIFIED="1702955418119" TEXT="simpleUsage in SchedulerLoadControl_test">
<icon BUILTIN="flag-yellow"/>
</node>
<node BACKGROUND_COLOR="#f0d5c5" COLOR="#990033" CREATED="1708044890924" ID="ID_1806382832" MODIFIED="1708045069169" TEXT="warum braucht das Planen des wake-up-Job so extrem lang (&gt;3ms)?">
<richcontent TYPE="NOTE"><html>
<head>
</head>
<body>
<p>
das beobachte ich mit dem DUMP-Logging immer wieder. Und zwar ohne eine ausgepr&#228;gte Contention-Spitze. Nach dem Absetzen des letzten regul&#228;ren Node-Jobs sieht man das Log vom Continuation-Aufruf, dann lang nichts, dann die Sequenz der dependencies.
</p>
</body>
</html>
</richcontent>
<icon BUILTIN="help"/>
<node CREATED="1708047710359" ID="ID_1233103402" MODIFIED="1708047732392" TEXT="Beobachtung: davon gehen ~2ms in buildTerm()"/>
</node>
</node>
</node>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1697980200719" ID="ID_1502196172" MODIFIED="1697982166711" TEXT="Lastverhalten empirisch feststellen">