ichthyo/LUMIERA.clone
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Chain-Load: switch planning chunk-size from level to node

Browse source 
This is a trick to get much better scheduling and timing guesses.
Instead of targeting a specific level, rather a fixed number of nodes
is processed in each chunk, yet still always processing complete levels.

The final level number to expect can be retrieved from the chain-load graph.

With this refactoring, we can now schedule a wake-up job precisely
after the expected completion of the last level
This commit is contained in:
Fischlurch 2023-12-08 23:52:57 +01:00
parent 34d6423660
commit 1df328cfc1
4 changed files with 83 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/vault/gear/scheduler.hpp
View file

@ -723,7 +723,7 @@ cout<<" ·‖ "+markThread()+": @ "+relT(now)+" HT:"+relT(layer1_.headTime())+
return activity::WAIT; // prompts to switch this thread into sleep mode
case LoadController::TENDNEXT:
doTendNextHead();
doTargetedSleep(); // let this thread wait until nest head time is due
doTargetedSleep(); // let this thread wait until next head time is due
return activity::SKIP;
default:
doTargetedSleep();

18
tests/vault/gear/test-chain-load-test.cpp
View file

@ -1018,6 +1018,7 @@ SHOW_EXPR(testLoad.getHash())
// Let the callbacks create a clone — which at the end should generate the same hash
array<Node,4> clone;
size_t lastTouched(-1);
size_t lastNode (-1);
size_t lastLevel(-1);
bool shallContinue{false};
auto getNodeIdx = [&](Node* n) { return n - &nodes[0]; };
@ -1038,8 +1039,9 @@ SHOW_EXPR(testLoad.getHash())
// replicate this relation into the clone array
clone[predIdx].addSucc(clone[succIdx]);
};
auto continuation = [&](size_t levelDone, bool work_left)
auto continuation = [&](size_t nodeDone, size_t levelDone, bool work_left)
{
lastNode =nodeDone;
lastLevel = levelDone;
shallContinue = work_left;
};
@ -1049,15 +1051,16 @@ SHOW_EXPR(testLoad.getHash())
,setDependency
,continuation};
Job jobP1{planJob
,InvocationInstanceID()
,planJob.encodeLevel(1)};
,planJob.encodeNodeID(1)
,Time::ANYTIME};
Job jobP2{planJob
,InvocationInstanceID()
,planJob.encodeLevel(3)};
,planJob.encodeNodeID(5)
,Time::ANYTIME};
jobP1.triggerJob();
CHECK (lastTouched = 2);
CHECK (lastLevel = 1);
CHECK (lastTouched = 2);
CHECK (lastTouched == lastNode);
Node* lastN = &clone[lastTouched];
CHECK (lastN->level == lastLevel);
CHECK ( isnil (lastN->succ));
@ -1065,8 +1068,9 @@ SHOW_EXPR(testLoad.getHash())
CHECK (shallContinue);
jobP2.triggerJob();
CHECK (lastTouched = 3);
CHECK (lastLevel = 3);
CHECK (lastTouched = 3);
CHECK (lastTouched == lastNode);
lastN = &clone[lastTouched];
CHECK (lastN->level == 2);
CHECK (lastN->level < lastLevel);

51
tests/vault/gear/test-chain-load.hpp
View file

@ -1159,9 +1159,9 @@ cout<<_Fmt{"\n!◆! %s: calc(i=%d, lev:%d)"} % markThread() % nodeIdx % level <<
{
using Node = typename TestChainLoad<maxFan>::Node;
function<void(size_t,size_t)> scheduleCalcJob_;
function<void(Node*,Node*)> markDependency_;
function<void(size_t,bool)> continuation_;
function<void(size_t,size_t)> scheduleCalcJob_;
function<void(Node*,Node*)> markDependency_;
function<void(size_t,size_t,bool)> continuation_;
size_t maxCnt_;
Node* nodes_;
@ -1188,19 +1188,24 @@ cout<<_Fmt{"\n!◆! %s: calc(i=%d, lev:%d)"} % markThread() % nodeIdx % level <<
void
invokeJobOperation (JobParameter param) override
{
size_t targetLevel = decodeLevel (TimeValue{param.nominalTime});
cout<<_Fmt{"\n!◆!plan...to:%d%19t|curr=%d (max:%d)"} % targetLevel % currIdx_ % maxCnt_<<endl;
size_t reachedLevel{0};
size_t targetNodeIDX = decodeNodeID (param.invoKey);
cout<<_Fmt{"\n!◆!plan...to:%d%19t|curr=%d (max:%d)"} % targetNodeIDX % currIdx_ % maxCnt_<<endl;
for ( ; currIdx_<maxCnt_; ++currIdx_)
{
Node* n = &nodes_[currIdx_];
cout<<_Fmt{"%16t|n.(%d,lev:%d)"} % currIdx_ % n->level <<endl;
if (n->level > targetLevel)
break;
if (currIdx_ <= targetNodeIDX)
reachedLevel = n->level;
else // continue until end of current level
if (n->level > reachedLevel)
break;
scheduleCalcJob_(currIdx_, n->level);
for (Node* pred: n->pred)
markDependency_(pred,n);
}
continuation_(targetLevel, currIdx_ < maxCnt_);
ENSURE (currIdx_ > 0);
continuation_(currIdx_-1, reachedLevel, currIdx_ < maxCnt_);
}
@ -1273,15 +1278,15 @@ cout <<_Fmt{"... dispose(i=%d,lev:%d) -> @%s"} % idx % level % relT(calcStartTim
/** continue planning: schedule follow-up planning job */
void
continuation (size_t levelDone, bool work_left)
continuation (size_t lastNodeIDX, size_t levelDone, bool work_left)
{
cout <<_Fmt{"+++ %s: Continuation(levelDone=%d, work_left:%s)"} % markThread() % levelDone % work_left <<endl;
cout <<_Fmt{"+++ %s: Continuation(lastNode=%d, levelDone=%d, work_left:%s)"} % markThread() % lastNodeIDX % levelDone % work_left <<endl;
if (work_left)
{
size_t nextChunkLevel = calcNextLevel (levelDone);
cout <<"--> reschedule to "<<nextChunkLevel<<endl;
scheduler_.continueMetaJob (calcPlanScheduleTime (nextChunkLevel)
,planningJob (nextChunkLevel)
size_t nextChunkEndNode = calcNextChunkEnd (lastNodeIDX);
cout <<"--> reschedule to ..."<<nextChunkEndNode<<endl;
scheduler_.continueMetaJob (calcPlanScheduleTime (nextChunkEndNode)
,planningJob (nextChunkEndNode)
,manID_);
}
else
@ -1295,10 +1300,11 @@ cout <<"--> reschedule to "<<nextChunkLevel<<endl;
{
auto finished = attachNewCompletionSignal();
size_t numNodes = chainLoad_.size();
size_t firstChunkEndNode = calcNextChunkEnd(0)-1;
cout <<"+++ "<<markThread()<<": seed(num:"<<numNodes<<")"<<endl;
schedule_.allocate (numNodes);
startTime_ = anchorStartTime();
scheduler_.seedCalcStream (planningJob(calcNextLevel(0)-1)
scheduler_.seedCalcStream (planningJob(firstChunkEndNode)
,manID_
,calcLoadHint());
return finished;
@ -1312,7 +1318,7 @@ cout <<"+++ "<<markThread()<<": seed(num:"<<numNodes<<")"<<endl;
, planFunctor_{new RandomChainPlanFunctor<maxFan>{chainLoad_.nodes_[0], chainLoad_.numNodes_
,[this](size_t i, size_t l){ disposeStep(i,l); }
,[this](auto* p, auto* s) { setDependency(p,s);}
,[this](size_t l, bool w) { continuation(l,w); }
,[this](size_t n,size_t l, bool w){ continuation(n,l,w); }
}}
{ }
@ -1352,11 +1358,11 @@ cout <<"+++ "<<markThread()<<": seed(num:"<<numNodes<<")"<<endl;
}
Job
planningJob (size_t level)
planningJob (size_t endNodeIDX)
{
return Job{*planFunctor_
, InvocationInstanceID()
, planFunctor_->encodeLevel(level)
, planFunctor_->encodeNodeID(endNodeIDX)
, Time::ANYTIME
};
}
@ -1389,9 +1395,9 @@ cout<<"ANCHOR="+relT(ank)+" preRoll="+util::toString(_raw(preRoll_))<<endl;
}
size_t
calcNextLevel (size_t levelDone)
calcNextChunkEnd (size_t lastNodeIDX)
{
return levelDone + chunkSize_;
return lastNodeIDX + chunkSize_;
}
Time
@ -1401,7 +1407,7 @@ cout<<"ANCHOR="+relT(ank)+" preRoll="+util::toString(_raw(preRoll_))<<endl;
}
Time
calcPlanScheduleTime (size_t nextChunkLevel)
calcPlanScheduleTime (size_t lastNodeIDX)
{/* must be at least 1 level ahead,
because dependencies are defined backwards;
the chain-load graph only defines dependencies over one level
@ -1409,6 +1415,7 @@ cout<<"ANCHOR="+relT(ank)+" preRoll="+util::toString(_raw(preRoll_))<<endl;
dependencies to the last row of the preceding chunk, implying that
those still need to be ahead of schedule.
*/
size_t nextChunkLevel = chainLoad_.nodes_[lastNodeIDX].level;
nextChunkLevel = nextChunkLevel>2? nextChunkLevel-2 : 0;
return calcStartTime(nextChunkLevel) - preRoll_;
}

52
wiki/thinkPad.ichthyo.mm
View file

@ -101282,9 +101282,37 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
<icon BUILTIN="button_ok"/>
</node>
</node>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1702012511964" ID="ID_788253401" MODIFIED="1702012564832" TEXT="Logik umprogrammieren auf vorgegebene Node-Zahl">
<linktarget COLOR="#91464a" DESTINATION="ID_788253401" ENDARROW="Default" ENDINCLINATION="-2;380;" ID="Arrow_ID_1856813245" SOURCE="ID_1304679964" STARTARROW="None" STARTINCLINATION="-149;0;"/>
<icon BUILTIN="flag-yellow"/>
<node COLOR="#338800" CREATED="1702012511964" ID="ID_788253401" MODIFIED="1702094456903" TEXT="Logik umprogrammieren auf vorgegebene Node-Zahl">
<linktarget COLOR="#466a91" DESTINATION="ID_788253401" ENDARROW="Default" ENDINCLINATION="-2;380;" ID="Arrow_ID_1856813245" SOURCE="ID_1304679964" STARTARROW="None" STARTINCLINATION="-149;0;"/>
<icon BUILTIN="button_ok"/>
<node CREATED="1702080463433" ID="ID_479353313" MODIFIED="1702080490992">
<richcontent TYPE="NODE"><html>
<head>
</head>
<body>
<p>
targetNode : bis einschie&#223;lich diese Node-ID soll <i>mindestens </i>behandelt werden
</p>
</body>
</html>
</richcontent>
</node>
<node CREATED="1702080491692" ID="ID_843468561" MODIFIED="1702080505366" TEXT="es wird aber stets auch noch der aktuelle Level komplett abgeschlossen"/>
<node CREATED="1702080506898" ID="ID_1648828975" MODIFIED="1702080525979" TEXT="f&#xfc;r den Level die bestehenden Logik weiterverwenden (sie funktioniert gut)"/>
<node CREATED="1702080526831" ID="ID_1161585807" MODIFIED="1702080590924" TEXT="aber: den tats&#xe4;chlich abgeschlossenen Level zur&#xfc;ckgeben">
<arrowlink COLOR="#6d7a8a" DESTINATION="ID_1932900692" ENDARROW="Default" ENDINCLINATION="-39;-66;" ID="Arrow_ID_943720507" STARTARROW="None" STARTINCLINATION="222;11;"/>
</node>
<node COLOR="#5b280f" CREATED="1702081218921" ID="ID_882772343" MODIFIED="1702081230349" TEXT="Zielkonflikt">
<icon BUILTIN="stop-sign"/>
<node CREATED="1702081236911" ID="ID_1708264859" MODIFIED="1702081244110" TEXT="Scheduling beruht auf der Level-Nr"/>
<node CREATED="1702081383395" ID="ID_1664629107" MODIFIED="1702081400949" TEXT="bei vorgegebener Level-Nr l&#xe4;&#xdf;t sich leicht die Deadline f&#xfc;r die Planung ableiten"/>
<node CREATED="1702081323517" ID="ID_1266244014" MODIFIED="1702081380220" TEXT="bei vorgegebenem Node-IDX kann die Deadline nicht festgelegt werden"/>
<node CREATED="1702081249437" ID="ID_350648836" MODIFIED="1702081321375" TEXT="der Zusammenhang Node-ID &#x27fc; Level ... ist aber bekannt">
<icon BUILTIN="idea"/>
</node>
</node>
<node CREATED="1702081414108" ID="ID_273894390" MODIFIED="1702081435384" TEXT="&#x27f9; also: Zugriff auf den Chain-Load-Grapen um die Deadline festzulegen"/>
</node>
</node>
<node BACKGROUND_COLOR="#eef0c5" COLOR="#990000" CREATED="1701493107387" ID="ID_1826192324" MODIFIED="1701814987380" TEXT="Aufruf-front-End">
@ -101308,9 +101336,9 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
<font NAME="SansSerif" SIZE="8"/>
</node>
</node>
<node BACKGROUND_COLOR="#eef0c5" COLOR="#990000" CREATED="1701913043245" ID="ID_445438847" MODIFIED="1701995114300" TEXT="bessere Steuerung des Completion-Callback">
<node COLOR="#338800" CREATED="1701913043245" ID="ID_445438847" MODIFIED="1702094487984" TEXT="bessere Steuerung des Completion-Callback">
<linktarget COLOR="#594b6a" DESTINATION="ID_445438847" ENDARROW="Default" ENDINCLINATION="-1207;75;" ID="Arrow_ID_1132943593" SOURCE="ID_1739832819" STARTARROW="None" STARTINCLINATION="-73;-483;"/>
<icon BUILTIN="pencil"/>
<icon BUILTIN="button_ok"/>
<node CREATED="1701995129106" ID="ID_1885461201" MODIFIED="1701995147012">
<richcontent TYPE="NODE"><html>
<head/>
@ -101368,9 +101396,13 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
</richcontent>
<icon BUILTIN="button_ok"/>
</node>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1702012060672" ID="ID_1038613808" MODIFIED="1702012610866" TEXT="Level-Berechnung pr&#xe4;ziser machen">
<node COLOR="#338800" CREATED="1702012060672" ID="ID_1038613808" MODIFIED="1702094473135" TEXT="Level-Berechnung pr&#xe4;ziser machen">
<arrowlink COLOR="#9e5576" DESTINATION="ID_53360642" ENDARROW="Default" ENDINCLINATION="396;-17;" ID="Arrow_ID_1980379788" STARTARROW="None" STARTINCLINATION="78;6;"/>
<icon BUILTIN="flag-yellow"/>
<icon BUILTIN="button_ok"/>
<node COLOR="#435e98" CREATED="1702080547549" ID="ID_1932900692" MODIFIED="1702094466919" TEXT="Continuation bekommt tats&#xe4;chlich abgeschlossenen Level">
<linktarget COLOR="#6d7a8a" DESTINATION="ID_1932900692" ENDARROW="Default" ENDINCLINATION="-39;-66;" ID="Arrow_ID_943720507" SOURCE="ID_1161585807" STARTARROW="None" STARTINCLINATION="222;11;"/>
<icon BUILTIN="idea"/>
</node>
</node>
</node>
</node>
@ -101425,9 +101457,9 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
<node CREATED="1701823711000" ID="ID_20712355" MODIFIED="1701823725194" TEXT="wird auch von wiederholungs-Chunks abgezogen"/>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1702012085806" ID="ID_1944577455" MODIFIED="1702012105308" TEXT="genauere Sch&#xe4;tzung, basierend auf der Zahl der Nodes/Chunk">
<icon BUILTIN="flag-yellow"/>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1702012494224" ID="ID_1304679964" MODIFIED="1702012564832" TEXT="erfordert Umprogrammieren des Planungs-Funktors">
<arrowlink COLOR="#91464a" DESTINATION="ID_788253401" ENDARROW="Default" ENDINCLINATION="-2;380;" ID="Arrow_ID_1856813245" STARTARROW="None" STARTINCLINATION="-149;0;"/>
<icon BUILTIN="flag-yellow"/>
<node COLOR="#338800" CREATED="1702012494224" ID="ID_1304679964" MODIFIED="1702094497913" TEXT="erfordert Umprogrammieren des Planungs-Funktors">
<arrowlink COLOR="#466a91" DESTINATION="ID_788253401" ENDARROW="Default" ENDINCLINATION="-2;380;" ID="Arrow_ID_1856813245" STARTARROW="None" STARTINCLINATION="-149;0;"/>
<icon BUILTIN="button_ok"/>
</node>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1702012570378" ID="ID_53360642" MODIFIED="1702012601899" TEXT="damit aber auch bessere Zeit-Steuerung m&#xf6;glich">
<linktarget COLOR="#9e5576" DESTINATION="ID_53360642" ENDARROW="Default" ENDINCLINATION="396;-17;" ID="Arrow_ID_1980379788" SOURCE="ID_1038613808" STARTARROW="None" STARTINCLINATION="78;6;"/>