ichthyo/LUMIERA.clone
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Dispatcher: rework loop control logic

Browse source 
- we got occasional hangups when waiting for disabled state
- the builder was not triggered properly, sometimes redundant, sometimes without timeout

As it turned out, the loop control logic is more like a state machine,
and the state variables need to be separated from the external influenced variables.

As a consequence, the inChange_ variable was not calculated properly when disabled in a race,
and then the loop went into infinite wait state, without propagating this to
the externally waiting client, which caused the deadlock
This commit is contained in:
Fischlurch 2018-11-23 21:19:21 +01:00
parent 48a829d544
commit 2ea89fcb54
4 changed files with 212 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

40
src/steam/control/looper.hpp
View file

@ -116,12 +116,13 @@ namespace control {
bool shutdown_ = false;
bool disabled_ = false;
bool inChange_ = false;
bool hasWork_ = false;
bool isDirty_ = false;
TimeVar gotDirty_ = Time::NEVER;
Predicate hasCommandsPending_;
uint dirty_ = 0;
TimeVar gotDirty_ = Time::NEVER;
public:
template<class FUN>
@ -136,8 +137,9 @@ namespace control {
bool isDying() const { return shutdown_; }
bool isDisabled() const { return disabled_ or isDying(); }
bool isWorking() const { return hasCommandsPending_() and not isDisabled(); }
bool idleBuild() const { return dirty_ and not hasCommandsPending_(); }
bool useTimeout() const { return isDirty_ and not isDisabled(); }
bool isWorking() const { return hasWork_ and not isDisabled(); }
bool idleBuild() const { return isDirty_ and not hasWork_; }
bool runBuild() const { return (idleBuild() or forceBuild()) and not isDisabled(); }
bool isIdle() const { return not (isWorking() or runBuild() or isDisabled()); }
@ -170,13 +172,12 @@ namespace control {
markStateProcessed()
{
inChange_ = false;
if (idleBuild() or forceBuild())
--dirty_;
ENSURE (dirty_ <= 2);
if (runBuild())
isDirty_ = false; // assume the builder has been triggered in the loop body
}
bool
hasPendingChanges() const
hasPendingChanges() const ///< "check point"
{
return inChange_;
}
@ -185,16 +186,17 @@ namespace control {
bool
requireAction()
{
inChange_ = true;
if (isWorking() and not dirty_)
{
dirty_ = 2;
hasWork_ = hasCommandsPending_();
bool proceedImmediately = isWorking() or forceBuild() or isDying();
inChange_ = proceedImmediately or useTimeout();
if (isWorking() and not isDirty_)
{ // schedule Builder run after timeout
startBuilderTimeout();
isDirty_ = true;
}
return isWorking()
or forceBuild()
or isDying();
return proceedImmediately;
}
/** state fusion to control looping */
@ -207,11 +209,11 @@ namespace control {
ulong /////////////////////////////////////////////TICKET #1056 : better return a std::chrono value here
getTimeout() const
{
if (isDisabled() or not dirty_)
if (not useTimeout())
return 0;
else
return wakeTimeout_ms()
* (dirty_ and not isWorking()? 1 : slowdownFactor());
* (isDirty_ and not isWorking()? 1 : slowdownFactor());
}
@ -262,7 +264,7 @@ namespace control {
{
static Duration maxBuildTimeout{Time(wakeTimeout_ms() * slowdownFactor(), 0)};
return dirty_
return isDirty_
and maxBuildTimeout < Offset(gotDirty_, RealClock::now());
} ///////////////////TICKET #1055 : likely to become more readable when Lumiera Time has std::chrono integration

10
src/steam/control/proc-dispatcher.cpp
View file

@ -191,7 +191,7 @@ namespace control {
void
awaitStateProcessed() const
{
Lock blockWaiting(unConst(this), &DispatcherLoop::stateIsSynched); ///////////////////////TICKET #1057 : const correctness on wait predicate
Lock blockWaiting(unConst(this), &DispatcherLoop::isStateSynched); ///////////////////////TICKET #1057 : const correctness on wait predicate
// wake-up typically by updateState()
}
@ -242,8 +242,8 @@ namespace control {
if (looper_.runBuild())
startBuilder();
else
if (looper_.isWorking())
processCommands();
if (looper_.isWorking())
processCommands();
updateState();
}
}
@ -277,7 +277,7 @@ namespace control {
}
bool
stateIsSynched()
isStateSynched()
{
if (this->invokedWithinThread())
throw error::Fatal("Possible Deadlock. "
@ -314,7 +314,7 @@ namespace control {
void
startBuilder()
{
TODO ("+++ start the Steam-Builder...");
INFO (builder, "+++ start the Steam-Builder...");
}
};

31
tests/core/steam/control/dispatcher-looper-test.cpp
View file

@ -172,7 +172,12 @@ namespace test {
setup.has_commands_in_queue = false;
looper.markStateProcessed(); // after command processing
looper.markStateProcessed(); // after builder run
CHECK (not looper.requireAction()); // stops immediate work state
CHECK ( looper.useTimeout()); // but still performs timeout
CHECK (not looper.isWorking());
CHECK (not looper.isIdle()); // still need to run the builder
looper.markStateProcessed(); // second round-trip, after builder run
CHECK (not looper.requireAction());
CHECK (not looper.isWorking());
@ -362,8 +367,7 @@ namespace test {
CHECK (not looper.runBuild()); // ...build still postponed
CHECK (not looper.isIdle());
sleep_for (800ms);
looper.markStateProcessed(); // let's assume we did command processing for a long time...
sleep_for (800ms); // let's assume we did command processing for a long time...
CHECK ( looper.requireAction());
CHECK (not looper.isDisabled());
@ -381,7 +385,6 @@ namespace test {
setup.has_commands_in_queue = false; // now emptied our queue
looper.markStateProcessed(); // at least one further command has been handled
CHECK (not looper.requireAction());
CHECK (not looper.isDisabled());
@ -458,17 +461,27 @@ namespace test {
looper.enableProcessing(true); // enable back
// NOTE special twist: it's unclear, if builder was triggered before the disabled state...
CHECK (isFast (looper.getTimeout())); // ...and thus we remain in dirty state
CHECK (not looper.requireAction());
CHECK (not looper.isDisabled());
CHECK (not looper.isWorking());
CHECK (not looper.runBuild()); // ...note: but now it becomes clear builder is not dirty
CHECK ( looper.isIdle());
CHECK ( looper.runBuild()); // so the builder will be triggered (possibly a second time) after a short timeout
CHECK (not looper.isIdle());
looper.markStateProcessed(); // and after one round-trip the builder was running and is now finished
CHECK (not looper.requireAction());
CHECK (not looper.isDisabled());
CHECK (not looper.isWorking());
CHECK (not looper.runBuild());
CHECK ( looper.isIdle()); // ...system is in idle state now and waits until triggered externally
CHECK (isDisabled (looper.getTimeout()));
setup.has_commands_in_queue = true; // more commands again
setup.has_commands_in_queue = true; // more commands again -> wake up
looper.markStateProcessed(); // ...and let's assume one command has already been processed
CHECK ( looper.requireAction());
@ -491,13 +504,15 @@ namespace test {
setup.has_commands_in_queue = false; // and even when done with all commands...
looper.markStateProcessed();
CHECK (isDisabled (looper.getTimeout()));
CHECK (not looper.shallLoop()); // we remain disabled and break out of the loop
CHECK ( looper.requireAction());
CHECK ( looper.isDisabled());
CHECK (not looper.isWorking());
CHECK (not looper.runBuild()); // ...note: still no need for builder run, since in shutdown
CHECK (not looper.isIdle());
CHECK (isDisabled (looper.getTimeout()));
}
};

166
wiki/thinkPad.ichthyo.mm
View file

@ -6706,7 +6706,7 @@
</node>
<node CREATED="1504833678189" ID="ID_173722644" MODIFIED="1518487921063" TEXT="Einstiegspunkt">
<arrowlink COLOR="#717686" DESTINATION="ID_65709251" ENDARROW="Default" ENDINCLINATION="-8;-209;" ID="Arrow_ID_1510990213" STARTARROW="None" STARTINCLINATION="92;95;"/>
<node CREATED="1504833683333" ID="ID_583036636" MODIFIED="1518487921063" TEXT="Component View">
<node CREATED="1504833683333" ID="ID_583036636" MODIFIED="1544329029063" TEXT="Component View">
<arrowlink COLOR="#92a9df" DESTINATION="ID_1717772756" ENDARROW="Default" ENDINCLINATION="-1346;-3359;" ID="Arrow_ID_1986148222" STARTARROW="None" STARTINCLINATION="385;845;"/>
<font ITALIC="true" NAME="SansSerif" SIZE="12"/>
</node>
@ -20207,7 +20207,7 @@
<node CREATED="1504833487359" ID="ID_1631525475" MODIFIED="1518487921084" TEXT="UI-Frame: Fenster"/>
<node CREATED="1504833498453" ID="ID_815439481" MODIFIED="1518487921084" TEXT="Perspektive"/>
<node CREATED="1504833508516" ID="ID_1973916831" MODIFIED="1518487921084" TEXT="einzelne Panel"/>
<node CREATED="1504833540720" ID="ID_1717772756" MODIFIED="1518487921084" TEXT="Component View">
<node CREATED="1504833540720" ID="ID_1717772756" MODIFIED="1544329029063" TEXT="Component View">
<linktarget COLOR="#92a9df" DESTINATION="ID_1717772756" ENDARROW="Default" ENDINCLINATION="-1346;-3359;" ID="Arrow_ID_1986148222" SOURCE="ID_583036636" STARTARROW="None" STARTINCLINATION="385;845;"/>
<linktarget COLOR="#929fdf" DESTINATION="ID_1717772756" ENDARROW="Default" ENDINCLINATION="-34;-71;" ID="Arrow_ID_1343685046" SOURCE="ID_349655067" STARTARROW="None" STARTINCLINATION="-79;76;"/>
<node CREATED="1504833565425" ID="ID_529173859" MODIFIED="1518487921084" TEXT="Timeline"/>
@ -44602,7 +44602,8 @@
<icon BUILTIN="flag-yellow"/>
</node>
</node>
<node COLOR="#435e98" CREATED="1544310988414" FOLDED="true" ID="ID_134908056" MODIFIED="1544320824720" TEXT="ProcDispatcher macht idle-Loop">
<node COLOR="#435e98" CREATED="1544310988414" FOLDED="true" ID="ID_134908056" MODIFIED="1544329969857" TEXT="ProcDispatcher macht idle-Loop">
<arrowlink COLOR="#ce3649" DESTINATION="ID_71855569" ENDARROW="Default" ENDINCLINATION="105;0;" ID="Arrow_ID_1871191743" STARTARROW="None" STARTINCLINATION="181;0;"/>
<icon BUILTIN="messagebox_warning"/>
<node CREATED="1544311004661" ID="ID_1064987345" MODIFIED="1544311012720" TEXT="Symptom: CPU-Last im Ruhezustand"/>
<node CREATED="1544311014140" ID="ID_392351205" MODIFIED="1544320819468" TEXT="Grund: f&#xe4;llt sofort wieder aus timed wait">
@ -44630,6 +44631,165 @@
</node>
</node>
</node>
<node COLOR="#435e98" CREATED="1544328974221" FOLDED="true" ID="ID_71855569" MODIFIED="1544386435837" TEXT="ProcDispatcher bleibt h&#xe4;ngen">
<linktarget COLOR="#ce3649" DESTINATION="ID_71855569" ENDARROW="Default" ENDINCLINATION="105;0;" ID="Arrow_ID_1871191743" SOURCE="ID_134908056" STARTARROW="None" STARTINCLINATION="181;0;"/>
<icon BUILTIN="messagebox_warning"/>
<node CREATED="1544329099419" ID="ID_1105978437" MODIFIED="1544329164023" TEXT="SessionCommandFunction_test">
<richcontent TYPE="NOTE"><html>
<head>
</head>
<body>
<p>
sporadich, nicht bei jedem Lauf, aber reproduzierbar.
</p>
<p>
Bleibt h&#228;ngen an der Stelle, wo der Test den Dispatcher vor&#252;bergehend deaktiviert,
</p>
<p>
und dann auf den Deaktiviert-Zustand <b>wartet</b>
</p>
</body>
</html>
</richcontent>
<icon BUILTIN="info"/>
</node>
<node CREATED="1544329167436" ID="ID_739717124" MODIFIED="1544329363509" TEXT="Problem mit der Zustands-Logik">
<icon BUILTIN="messagebox_warning"/>
<node CREATED="1544329192703" ID="ID_1013421079" MODIFIED="1544329202330" TEXT="deaktivert bedeutet requireAction() == false"/>
<node CREATED="1544329202903" ID="ID_1019916123" MODIFIED="1544329356857" TEXT="und dadurch warten wir ohne Timeout">
<icon BUILTIN="broken-line"/>
<node CREATED="1544329280619" ID="ID_1784122176" MODIFIED="1544329348875" TEXT="und durch den Bug in lib::Sync wurde das nicht wirksam">
<richcontent TYPE="NOTE"><html>
<head>
</head>
<body>
<p>
...weil der Objekt-Monitor nicht mehr bedingungslos gewartet hat,
</p>
<p>
nachdem <i>einmal</i>&#160;ein wait mit Timeout verwendet worden war
</p>
</body>
</html>
</richcontent>
<icon BUILTIN="ksmiletris"/>
</node>
</node>
<node CREATED="1544329216228" ID="ID_422426535" MODIFIED="1544329234043">
<richcontent TYPE="NODE"><html>
<head>
</head>
<body>
<p>
aber <b>inChange</b>&#160;bleibt <b>true</b>
</p>
</body>
</html>
</richcontent>
</node>
<node CREATED="1544329235934" ID="ID_605336452" MODIFIED="1544329273933" TEXT="...und darauf wartet die &#xe4;u&#xdf;ere H&#xfc;lle">
<icon BUILTIN="idea"/>
</node>
</node>
<node COLOR="#338800" CREATED="1544329367261" ID="ID_50498392" MODIFIED="1544386159034" TEXT="L&#xf6;sungen" VGAP="15">
<icon BUILTIN="button_ok"/>
<node CREATED="1544329727165" FOLDED="true" ID="ID_1043095728" MODIFIED="1544386423962" TEXT="requireAction() == true auch wenn disabled">
<icon BUILTIN="button_cancel"/>
<node CREATED="1544329753698" ID="ID_1588685067" MODIFIED="1544329760502" TEXT="geht nicht.">
<icon BUILTIN="stop-sign"/>
</node>
<node CREATED="1544329764640" ID="ID_706638200" MODIFIED="1544329781788" TEXT="denn dann kommen wir nicht mehr in den wait-state">
<icon BUILTIN="info"/>
</node>
<node CREATED="1544329793943" ID="ID_1668546000" MODIFIED="1544329822612" TEXT="requireAction wird ja als Condition f&#xfc;r den Wait verwendet">
<icon BUILTIN="idea"/>
</node>
</node>
<node CREATED="1544329372318" FOLDED="true" ID="ID_88425113" MODIFIED="1544386423070" TEXT="inChange nur setzen wenn requireAction() == true">
<icon BUILTIN="button_cancel"/>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1544329663574" ID="ID_565123980" MODIFIED="1544332799520" TEXT="Verdacht: kann dann trotzdem per Timeout aufwachen">
<icon BUILTIN="clanbomber"/>
</node>
<node CREATED="1544329684707" ID="ID_651175391" MODIFIED="1544329708068" TEXT="und dann w&#xe4;re inChange == false, obwohl wir arbeiten (Builder l&#xe4;uft)"/>
<node BACKGROUND_COLOR="#fdfdcf" COLOR="#ff0000" CREATED="1544332821150" ID="ID_484338481" MODIFIED="1544332874097" TEXT="fatale Konsequenzen....">
<richcontent TYPE="NOTE"><html>
<head>
</head>
<body>
<p>
...es k&#246;nnte dann n&#228;mlich die Session geschlossen und freigegeben werden,
</p>
<p>
obwohl noch der Builder l&#228;uft
</p>
</body>
</html>
</richcontent>
<icon BUILTIN="broken-line"/>
</node>
<node CREATED="1544332907967" ID="ID_760793562" MODIFIED="1544332932033" TEXT="Analyse: kann das passieren?">
<icon BUILTIN="help"/>
<node CREATED="1544332934619" ID="ID_558823055" MODIFIED="1544333038537" TEXT="Fall: disabled-setzen w&#xe4;hrend dem Timeout">
<node CREATED="1544333050044" ID="ID_318735401" MODIFIED="1544333066940" TEXT="Timeout eingetreten ==&gt; requireAction war false"/>
<node CREATED="1544333069881" ID="ID_555896336" MODIFIED="1544333084882" TEXT="nach ge&#xe4;nderter Logik w&#xe4;re dann auch inChange == false"/>
<node CREATED="1544333087958" ID="ID_1209619554" MODIFIED="1544333108407" TEXT="Folglich wird f&#xe4;lschlicherweise nicht gewartet"/>
</node>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1544333135104" ID="ID_549018822" MODIFIED="1544333155423" TEXT="Antwort: ja das kann passieren">
<icon BUILTIN="back"/>
</node>
</node>
<node CREATED="1544333313167" ID="ID_1694353778" MODIFIED="1544333323414" TEXT="damit ist diese L&#xf6;sung hinf&#xe4;llig">
<icon BUILTIN="stop-sign"/>
</node>
</node>
<node CREATED="1544333328341" FOLDED="true" ID="ID_349762653" MODIFIED="1544386421940" TEXT="inChange unabh&#xe4;ngig von requireAction() machen">
<icon BUILTIN="button_cancel"/>
<node CREATED="1544333344379" ID="ID_1647118494" MODIFIED="1544333369026" TEXT="eigene Methode, die aufgerufen wird, wenn wir aus dem wait() rauskommen"/>
<node CREATED="1544333456987" ID="ID_585691817" MODIFIED="1544333897248" TEXT="k&#xf6;nnte gehen, erscheint mir aber gef&#xe4;hrlich">
<icon BUILTIN="help"/>
<icon BUILTIN="yes"/>
<node CREATED="1544333510982" ID="ID_1431407168" MODIFIED="1544333516439" TEXT="was ist wenn....">
<node CREATED="1544333517379" ID="ID_689738064" MODIFIED="1544333533684" TEXT="disabled gesetzt wird, w&#xe4;hrend wir im wait() sind?">
<node CREATED="1544333550151" ID="ID_100143787" MODIFIED="1544333556410" TEXT="dann ist inChange == false"/>
<node CREATED="1544333610495" ID="ID_942332687" MODIFIED="1544333626888" TEXT="aber deactivateCommandProecssing() -&gt; notifyAll()"/>
<node CREATED="1544333630018" ID="ID_1936407440" MODIFIED="1544333636367" TEXT="noch w&#xe4;hrend er das Lock hat"/>
<node CREATED="1544333723957" ID="ID_1134715607" MODIFIED="1544333742040" TEXT="sobald er das Lock aufgibt, wacht der Loop-Thread auf"/>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1544333778756" ID="ID_705117923" MODIFIED="1544333804986" TEXT="gibt dann aber auch das Lock auf, bevor er inChange setzen kann">
<icon BUILTIN="broken-line"/>
</node>
</node>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1544333809035" ID="ID_1284134914" MODIFIED="1544333881455" TEXT="m&#xf6;glicher Race">
<linktarget COLOR="#a9b4c1" DESTINATION="ID_1284134914" ENDARROW="Default" ENDINCLINATION="-26;57;" ID="Arrow_ID_662106184" SOURCE="ID_151622227" STARTARROW="None" STARTINCLINATION="159;0;"/>
<icon BUILTIN="clanbomber"/>
</node>
</node>
</node>
<node CREATED="1544333819034" ID="ID_151622227" MODIFIED="1544333881455" TEXT="L&#xf6;sung hinf&#xe4;llig, wegen potentiellem Race">
<arrowlink DESTINATION="ID_1284134914" ENDARROW="Default" ENDINCLINATION="-26;57;" ID="Arrow_ID_662106184" STARTARROW="None" STARTINCLINATION="159;0;"/>
<icon BUILTIN="stop-sign"/>
</node>
</node>
<node CREATED="1544333909701" ID="ID_328876957" MODIFIED="1544386156765" STYLE="fork" TEXT="inChange in requireAction() setzen, aber logisch pr&#xe4;zise">
<icon BUILTIN="button_ok"/>
<node CREATED="1544333933346" ID="ID_593273191" MODIFIED="1544386155027" TEXT="d.h. nur wenn nicht warten und kein Timeout"/>
<node CREATED="1544381554267" ID="ID_193163875" MODIFIED="1544386155027" TEXT="Zustand nur einmal in die Flags &#xfc;bernhmen, zu Beginn"/>
<node CREATED="1544382733514" ID="ID_1386275063" MODIFIED="1544386155027" TEXT="Trick mit dem Countdown auf isDirty_ zur&#xfc;ckbauen"/>
</node>
</node>
<node CREATED="1544386177470" ID="ID_1764197819" MODIFIED="1544386413663" TEXT="erst jetzt sehe ich das erwartete Builder-Verhalten">
<icon BUILTIN="idea"/>
</node>
<node CREATED="1544386386938" ID="ID_1316391277" MODIFIED="1544386416510" TEXT="und der SessionCommandFunction_test bleibt nicht mehr h&#xe4;ngen">
<icon BUILTIN="idea"/>
</node>
<node COLOR="#338800" CREATED="1544386406767" ID="ID_789739914" MODIFIED="1544386410583" TEXT="problem solved">
<icon BUILTIN="button_ok"/>
</node>
</node>
</node>
</node>
</node>