benn/lumiera_
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Scheduler: automatically step up capacity on new task

Browse source 
WorkForce scales down automatically after 2 seconds when
workers fall idle; thus we need to step up automatically
with each new task.

Later we'll also add some capacity management to both the
LoadController and the Job-Planning, but for now this rather
crude approach should suffice.

NOTE: most of the cases in SchedulerService_test verify parts
of the component integration and thus need to bypass this
automatism, because the test code wants to invoke the
work-Function directly (without any interference
from running workers)
This commit is contained in:
Fischlurch 2023-11-07 17:00:24 +01:00
parent 3c3d31dd40
commit 0ed7dba641
3 changed files with 75 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
src/vault/gear/scheduler.hpp
View file

@ -319,6 +319,7 @@ namespace gear {
private:
void handleDutyCycle (Time now);
void handleWorkerTermination (bool isFailure);
void maybeScaleWorkForce();
void triggerEmergency();
@ -500,6 +501,7 @@ namespace gear {
inline void
Scheduler::postChain (ActivationEvent actEvent)
{
maybeScaleWorkForce();
ExecutionCtx ctx{*this, actEvent};
layer2_.postDispatch (actEvent, ctx, layer1_);
}
@ -673,6 +675,23 @@ namespace gear {
loadControl_.markWorkerExit();
}
/**
* Hook invoked whenever a new task is passed in.
* Ensures that the Scheduler is in running state and
* possibly steps up the WorkForce if not yet running at
* full computation power.
* @note the capacity scales down automatically when some
* workers fall idle for extended time (> 2sec).
*/
inline void
Scheduler::maybeScaleWorkForce()
{
if (empty())
ignite();
else
workForce_.incScale();
}
/**
* Trip the emergency brake and unwind processing while retaining all state.
*/

29
tests/vault/gear/scheduler-service-test.cpp
View file

@ -80,10 +80,10 @@ namespace test {
virtual void
run (Arg)
{
// simpleUsage();
// verify_StartStop();
// verify_LoadFactor();
// invokeWorkFunction();
simpleUsage();
verify_StartStop();
verify_LoadFactor();
invokeWorkFunction();
scheduleRenderJob();
walkingDeadline();
}
@ -101,6 +101,21 @@ namespace test {
}
/**
* @internal helper to inject a new task into the Scheduler,
* without also activating WorkForce and load control.
* @remark this class is declared friend by the Scheduler to grant
* this kind of »implementation backdoor« access; the function
* defined there does essentially the same than Scheduler::postChain()
*/
void
postNewTask (Scheduler& scheduler, Activity& chain, Time start)
{
ActivationEvent actEvent{chain, start};
Scheduler::ExecutionCtx ctx{scheduler, actEvent};
scheduler.layer2_.postDispatch (actEvent, ctx, scheduler.layer1_);
}
/** @test get the scheduler into running state
* @todo WIP 10/23 ✔ define ⟶ ✔ implement
@ -114,7 +129,7 @@ namespace test {
CHECK (isnil (scheduler));
Activity dummy{Activity::FEED};
auto postIt = [&] { scheduler.postChain (ActivationEvent{dummy, RealClock::now()+t200us}); };
auto postIt = [&] { postNewTask (scheduler, dummy, RealClock::now()+t200us); };
scheduler.ignite();
CHECK (isnil (scheduler)); // no start without any post()
@ -169,7 +184,7 @@ namespace test {
auto createLoad = [&](Offset start, uint cnt)
{ // use internal API (this test is declared as friend)
for (uint i=0; i<cnt; ++i) // flood the queue
scheduler.postChain (ActivationEvent{dummy, anchor + start + TimeValue{i}});
postNewTask (scheduler, dummy, anchor + start + TimeValue{i});
};
@ -316,7 +331,7 @@ namespace test {
auto post = [&](Time start)
{ // this test class is declared friend to get a backdoor into Scheduler internals...
scheduler.layer2_.acquireGoomingToken();
scheduler.postChain(ActivationEvent{probe, start});
postNewTask (scheduler, probe, start);
};
auto pullWork = [&] {

70
wiki/thinkPad.ichthyo.mm
View file

@ -82118,17 +82118,27 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
<linktarget COLOR="#a9b4c1" DESTINATION="ID_1964298619" ENDARROW="Default" ENDINCLINATION="27;865;" ID="Arrow_ID_900145073" SOURCE="ID_760916314" STARTARROW="Default" STARTINCLINATION="-554;-36;"/>
<icon BUILTIN="button_ok"/>
</node>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1699325172901" ID="ID_1435123962" MODIFIED="1699325566562" TEXT="mu&#xdf; Kapazit&#xe4;t pr&#xfc;fen / Hochskalieren">
<node COLOR="#338800" CREATED="1699325172901" ID="ID_1435123962" MODIFIED="1699372410733" TEXT="mu&#xdf; Kapazit&#xe4;t pr&#xfc;fen / Hochskalieren">
<arrowlink COLOR="#57496c" DESTINATION="ID_745204926" ENDARROW="Default" ENDINCLINATION="249;-377;" ID="Arrow_ID_55492946" STARTARROW="None" STARTINCLINATION="668;346;"/>
<linktarget COLOR="#662136" DESTINATION="ID_1435123962" ENDARROW="Default" ENDINCLINATION="304;54;" ID="Arrow_ID_928822414" SOURCE="ID_1792556883" STARTARROW="None" STARTINCLINATION="-206;16;"/>
<icon BUILTIN="flag-yellow"/>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1699329348003" ID="ID_396553511" MODIFIED="1699368192260" TEXT="Parameter zur WorkForce-St&#xe4;rke ber&#xfc;cksichtigen">
<linktarget COLOR="#7f4745" DESTINATION="ID_396553511" ENDARROW="Default" ENDINCLINATION="486;26;" ID="Arrow_ID_329496296" SOURCE="ID_651840656" STARTARROW="None" STARTINCLINATION="728;32;"/>
<icon BUILTIN="flag-yellow"/>
<linktarget COLOR="#373a76" DESTINATION="ID_1435123962" ENDARROW="Default" ENDINCLINATION="304;54;" ID="Arrow_ID_928822414" SOURCE="ID_1792556883" STARTARROW="None" STARTINCLINATION="-255;13;"/>
<icon BUILTIN="button_ok"/>
<node COLOR="#338800" CREATED="1699329348003" ID="ID_396553511" MODIFIED="1699372243603" TEXT="Parameter zur WorkForce-St&#xe4;rke wird ber&#xfc;cksichtigt">
<linktarget COLOR="#45647f" DESTINATION="ID_396553511" ENDARROW="Default" ENDINCLINATION="486;26;" ID="Arrow_ID_329496296" SOURCE="ID_651840656" STARTARROW="None" STARTINCLINATION="728;32;"/>
<icon BUILTIN="button_ok"/>
</node>
<node COLOR="#435e98" CREATED="1699329449911" ID="ID_1960065591" MODIFIED="1699372221033" TEXT="wenn....">
<node COLOR="#435e98" CREATED="1699329454534" ID="ID_258337881" MODIFIED="1699372224340" TEXT="Scheduler l&#xe4;uft nicht &#x27f9; ignite()"/>
<node COLOR="#435e98" CREATED="1699329466069" ID="ID_1149902545" MODIFIED="1699372224340" TEXT="nicht volle St&#xe4;rke &#x27f9; step"/>
</node>
<node COLOR="#338800" CREATED="1699372253576" ID="ID_869013268" MODIFIED="1699372262575" TEXT="Implementierung in postChain eingeh&#xe4;ngt">
<icon BUILTIN="button_ok"/>
<node COLOR="#435e98" CREATED="1699372263855" ID="ID_1329492790" MODIFIED="1699372374425" TEXT="damit stets wirksam, wenn etwas in den Scheduler geht">
<arrowlink COLOR="#5184bf" DESTINATION="ID_660894370" ENDARROW="Default" ENDINCLINATION="-622;-30;" ID="Arrow_ID_841520455" STARTARROW="None" STARTINCLINATION="1037;52;"/>
<icon BUILTIN="idea"/>
</node>
<node COLOR="#338800" CREATED="1699372349395" ID="ID_1120366594" MODIFIED="1699372369723" TEXT="Neben-Hintert&#xfc;r f&#xfc;r den SchedulerService_test geschaffen">
<icon BUILTIN="button_ok"/>
</node>
<node CREATED="1699329449911" ID="ID_1960065591" MODIFIED="1699329453611" TEXT="wenn....">
<node CREATED="1699329454534" ID="ID_258337881" MODIFIED="1699329465057" TEXT="Scheduler l&#xe4;uft nicht &#x27f9; ignite()"/>
<node CREATED="1699329466069" ID="ID_1149902545" MODIFIED="1699329477423" TEXT="nicht volle St&#xe4;rke &#x27f9; step"/>
</node>
</node>
<node BACKGROUND_COLOR="#eef0c5" COLOR="#990000" CREATED="1699245611846" ID="ID_788545423" MODIFIED="1699307283267" TEXT="Definitions-Struktur und Code-Anordnung kl&#xe4;ren">
@ -82694,21 +82704,18 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
</node>
<node CREATED="1699368212285" ID="ID_855305118" MODIFIED="1699368213601" TEXT="wird von WorkForce und LoadController als Referenz verwendet"/>
<node COLOR="#338800" CREATED="1699329306877" ID="ID_651840656" MODIFIED="1699368342608" TEXT="dieser Parameter ist nun explizit &#xe4;nderbar (nicht mehr const)">
<arrowlink COLOR="#7f4745" DESTINATION="ID_396553511" ENDARROW="Default" ENDINCLINATION="486;26;" ID="Arrow_ID_329496296" STARTARROW="None" STARTINCLINATION="728;32;"/>
<arrowlink COLOR="#45647f" DESTINATION="ID_396553511" ENDARROW="Default" ENDINCLINATION="486;26;" ID="Arrow_ID_329496296" STARTARROW="None" STARTINCLINATION="728;32;"/>
<icon BUILTIN="button_ok"/>
</node>
<node BACKGROUND_COLOR="#e0ceaa" COLOR="#690f14" CREATED="1699368246481" ID="ID_817072584" MODIFIED="1699368325222" TEXT="wird aber wie eine Konstante behandelt">
<richcontent TYPE="NOTE"><html>
<head>
</head>
<head/>
<body>
<p>
....also kein Locking etc; allerdings greift der LoadController jetzt per Lambda darauf zu.
</p>
</body>
</html>
</richcontent>
</html></richcontent>
</node>
<node CREATED="1699368298187" ID="ID_1401898380" MODIFIED="1699368321811" TEXT="sollte man also bei stehendem Scheduler &#xe4;ndern">
<icon BUILTIN="messagebox_warning"/>
@ -82948,6 +82955,7 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
</p>
</body>
</html></richcontent>
<linktarget COLOR="#5184bf" DESTINATION="ID_660894370" ENDARROW="Default" ENDINCLINATION="-622;-30;" ID="Arrow_ID_841520455" SOURCE="ID_1329492790" STARTARROW="None" STARTINCLINATION="1037;52;"/>
<font BOLD="true" NAME="SansSerif" SIZE="12"/>
<icon BUILTIN="idea"/>
<node COLOR="#594398" CREATED="1699068463527" ID="ID_82378192" MODIFIED="1699068489999" TEXT="gef&#xfc;hlsm&#xe4;&#xdf;ig... jetzt sortiert sich das Design doch noch">
@ -82974,16 +82982,13 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
<icon BUILTIN="clanbomber"/>
<node CREATED="1699314936837" ID="ID_1589318425" MODIFIED="1699315051503" TEXT="die pre-Dispatch-Kapazit&#xe4;tssteuerung will das (bisher) nicht machen">
<richcontent TYPE="NOTE"><html>
<head>
</head>
<head/>
<body>
<p>
...im Lastbetrieb m&#246;chte man das eigentlich auch nicht: ein ankommender Thread soll gem&#228;&#223; aktueller Situation klassifiziert werden, aber nicht erst mal um das Grooming-Token konkurrieren
</p>
</body>
</html>
</richcontent>
</html></richcontent>
</node>
<node CREATED="1699315054445" ID="ID_1061745270" MODIFIED="1699315073575" TEXT="diese Situation d&#xfc;rfte im Regelbetrieb nicht auftreten">
<node CREATED="1699315075175" ID="ID_1648796415" MODIFIED="1699315087836" TEXT="denn dort wird aus einem Managment-Job heraus weiter geplant"/>
@ -83005,19 +83010,17 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
<arrowlink COLOR="#751420" DESTINATION="ID_179251569" ENDARROW="Default" ENDINCLINATION="-787;-76;" ID="Arrow_ID_272593119" STARTARROW="None" STARTINCLINATION="1260;101;"/>
<icon BUILTIN="messagebox_warning"/>
</node>
<node CREATED="1699325057420" ID="ID_1792556883" MODIFIED="1699325504658" TEXT="Hochskalieren fehlt ebenfalls">
<node COLOR="#338800" CREATED="1699325057420" ID="ID_1792556883" MODIFIED="1699372417701" TEXT="Hochskalieren mu&#xdf; auch noch eingebaut werden">
<richcontent TYPE="NOTE"><html>
<head>
</head>
<head/>
<body>
<p>
....und das w&#252;rde in dem Fall das Problem durch den ersten &#187;Tick&#171; nebenbei beheben &#8212; aber nicht, falls der Scheduler bereits l&#228;uft und leer gefallen ist
</p>
</body>
</html>
</richcontent>
<arrowlink COLOR="#662136" DESTINATION="ID_1435123962" ENDARROW="Default" ENDINCLINATION="304;54;" ID="Arrow_ID_928822414" STARTARROW="None" STARTINCLINATION="-206;16;"/>
</html></richcontent>
<arrowlink COLOR="#373a76" DESTINATION="ID_1435123962" ENDARROW="Default" ENDINCLINATION="304;54;" ID="Arrow_ID_928822414" STARTARROW="None" STARTINCLINATION="-255;13;"/>
<icon BUILTIN="button_ok"/>
</node>
</node>
</node>
@ -84276,22 +84279,19 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
</node>
</node>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1699325295132" ID="ID_745204926" MODIFIED="1699325566562" TEXT="WorkForce hochfahren">
<linktarget COLOR="#533354" DESTINATION="ID_745204926" ENDARROW="Default" ENDINCLINATION="-41;67;" ID="Arrow_ID_204394164" SOURCE="ID_1772549848" STARTARROW="None" STARTINCLINATION="-125;-5;"/>
<linktarget COLOR="#57496c" DESTINATION="ID_745204926" ENDARROW="Default" ENDINCLINATION="249;-377;" ID="Arrow_ID_55492946" SOURCE="ID_1435123962" STARTARROW="None" STARTINCLINATION="668;346;"/>
<linktarget COLOR="#533354" DESTINATION="ID_745204926" ENDARROW="Default" ENDINCLINATION="-41;67;" ID="Arrow_ID_204394164" SOURCE="ID_1772549848" STARTARROW="None" STARTINCLINATION="-125;-5;"/>
<icon BUILTIN="flag-yellow"/>
<node CREATED="1699325589332" ID="ID_25430169" MODIFIED="1699325606390" TEXT="WorkForce regelt sich selber herunter bei Leerlauf"/>
<node CREATED="1699325607274" ID="ID_718145429" MODIFIED="1699325631062">
<richcontent TYPE="NODE"><html>
<head>
</head>
<head/>
<body>
<p>
daher mu&#223; f&#252;r einen neuen Job ein <i>step up</i>&#160;erfolgen
</p>
</body>
</html>
</richcontent>
</html></richcontent>
<icon BUILTIN="yes"/>
</node>
<node CREATED="1699325634014" ID="ID_1493688459" MODIFIED="1699325647951" TEXT="bei Bedarf: ignite()"/>
@ -92419,9 +92419,7 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
</node>
<node CREATED="1699328515692" ID="ID_649427536" MODIFIED="1699328747184" TEXT="das ist problematisch">
<richcontent TYPE="NOTE"><html>
<head>
</head>
<head/>
<body>
<p>
Aus Sicht des Testaufbaues w&#228;re es sehr w&#252;nschenswert. Aber da das Scheduler-API high-Level ist, sehe ich keine einfache M&#246;glichkeit, f&#252;r einen Test dazwischen zu gehen. Ein Mocking / Austauschen des Scheduler w&#228;re hier nicht zielf&#252;hrend (weil es genau auf die Interaktion mit der integrierten Scheduler-Implementierung ankommt).