From 1d5b8c3e9c3e860a392575ed52f86e68769a3a7f Mon Sep 17 00:00:00 2001 From: Ichthyostega Date: Tue, 24 Oct 2023 03:59:27 +0200 Subject: [PATCH] Scheduler: implement and verify random reshuffling of capacity ...using the current time itself as source for randomisation; the test indicates this yields a smooth and even distribution. --- src/vault/gear/load-controller.hpp | 54 ++++++-- .../gear/scheduler-load-control-test.cpp | 65 +++++---- wiki/renderengine.html | 12 +- wiki/thinkPad.ichthyo.mm | 128 ++++++++++++------ 4 files changed, 178 insertions(+), 81 deletions(-) diff --git a/src/vault/gear/load-controller.hpp b/src/vault/gear/load-controller.hpp index af646b8bc..d7958620e 100644 --- a/src/vault/gear/load-controller.hpp +++ b/src/vault/gear/load-controller.hpp @@ -111,16 +111,17 @@ namespace gear { return TimeValue{us.count()}; } - + Duration SLEEP_HORIZON{_uTicks (20ms)}; Duration WORK_HORIZON {_uTicks ( 5ms)}; Duration NOW_HORIZON {_uTicks (50us)}; } + /** * Controller to coordinate resource usage related to the Scheduler. - * @todo WIP-WIP 10/2023 just a placeholder for now + * @todo WIP-WIP 10/2023 gradually filling in functionality as needed * @see BlockFlow * @see Scheduler */ @@ -131,6 +132,7 @@ namespace gear { struct Wiring { size_t maxCapacity{2}; + ///////TODO add here functors to access performance indicators }; explicit @@ -146,8 +148,8 @@ namespace gear { const Wiring wiring_; TimeVar tendedHead_{Time::ANYTIME}; - public: + public: /** * did we already tend for the indicated next head time? * @note const and non-grooming @@ -163,6 +165,7 @@ namespace gear { * @remark while this is just implemented as simple state, * the meaning is that some free capacity has been directed * towards that time, and thus further capacity go elsewhere. + * @warning must hold the grooming-Token to use this mutation. */ void tendNext (Time nextHead) @@ -170,14 +173,16 @@ namespace gear { tendedHead_ = nextHead; } - enum - Capacity {DISPATCH ///< sent to work - ,TENDNEXT ///< reserved for next task - ,SPINTIME ///< awaiting imminent activities - ,NEARTIME ///< capacity for active processing required - ,WORKTIME ///< typical stable work task rhythm expected - ,IDLETIME ///< time to go to sleep - }; + + + /** Allocation of capacity to time horizon of expected work */ + enum Capacity {DISPATCH ///< sent to work + ,TENDNEXT ///< reserved for next task + ,SPINTIME ///< awaiting imminent activities + ,NEARTIME ///< capacity for active processing required + ,WORKTIME ///< typical stable work task rhythm expected + ,IDLETIME ///< time to go to sleep + }; /** classification of time horizon for scheduling */ static Capacity @@ -191,6 +196,8 @@ namespace gear { } + /** decide how this thread's capacity shall be used + * after it returned from being actively employed */ Capacity markOutgoingCapacity (Time head, Time now) { @@ -200,6 +207,8 @@ namespace gear { : horizon; } + /** decide how this thread's capacity shall be used + * when returning from idle wait and asking for work */ Capacity markIncomingCapacity (Time head, Time now) { @@ -209,13 +218,30 @@ namespace gear { } + + /** + * Generate a time offset to relocate currently unused capacity + * to a time range where it's likely to be needed. Assuming the + * classification is based on the current distance to the next + * Activity known to the scheduler (the next tended head time). + * - for capacity immediately to be dispatched this function + * will not be used, yet returns logically sound values. + * - after the next head time has been tended for, free capacity + * should be relocated into a time span behind that point + * - the closer the next head time, the more focused this relocation + * - but each individual delay is randomised within those time bounds, + * to produce an even »flow« of capacity on average. Randomisation + * relies on a hash (bit rotation) of current time, broken down + * to the desired time horizon. + */ Offset scatteredDelayTime (Time now, Capacity capacity) { auto scatter = [&](Duration horizon) { - size_t step = 1;////////////////////////////////////////////////////TODO implement randomisation - return Offset{_raw(horizon) * step / wiring_.maxCapacity}; + gavl_time_t wrap = hash_value(now) % _raw(horizon); + ENSURE (0 <= wrap and wrap < _raw(horizon)); + return TimeValue{wrap}; }; switch (capacity) { @@ -230,7 +256,7 @@ namespace gear { case WORKTIME: return Offset{tendedHead_-now + scatter(SLEEP_HORIZON)}; case IDLETIME: - return /*without start offset*/ scatter(SLEEP_HORIZON); + return Offset{/*no base offset*/ scatter(SLEEP_HORIZON)}; default: NOTREACHED ("uncovered work capacity classification."); } diff --git a/tests/vault/gear/scheduler-load-control-test.cpp b/tests/vault/gear/scheduler-load-control-test.cpp index f56bf86ce..cf799bb9f 100644 --- a/tests/vault/gear/scheduler-load-control-test.cpp +++ b/tests/vault/gear/scheduler-load-control-test.cpp @@ -31,6 +31,7 @@ //#include "lib/time/timevalue.hpp" //#include "lib/format-cout.hpp" //#include "lib/util.hpp" +#include "lib/test/diagnostic-output.hpp"/////////////////////TODO //#include @@ -70,8 +71,8 @@ namespace test { tendNextActivity(); classifyCapacity(); scatteredReCheck(); + walkingDeadline(); - setupLalup(); } @@ -82,6 +83,7 @@ namespace test { simpleUsage() { LoadController ctrl; + /////////////////////////TODO a simple usage example focusing on load diagnostics } @@ -233,30 +235,55 @@ namespace test { /** @test verify the re-distribution of free capacity by targeted delay - * @todo WIP 10/23 🔁 define ⟶ implement + * - the implementation uses the next-tended start time as anchor point + * - capacity classes which should be scheduled right away will actually + * never call this function — yet still a sensible value is returned here + * - capacity targeted at current work will be redistributed behind the + * next-tended time, and within a time span corresponding to the work realm + * - capacity targeted towards more future work will be distributed within + * the horizon defined by the sleep-cycle + * - especially for capacity sent to sleep, this redistribution works + * without being shifted behind the next-tended time, since in that case + * the goal is to produce a random distribution of the »sleeper« callbacks. + * - the offset is indeed randomised, using current time for randomisation + * @see LoadController::scatteredDelayTime() + * @todo WIP 10/23 ✔ define ⟶ ✔ implement */ void scatteredReCheck() { - Wiring setup; - setup.maxCapacity = 16; - LoadController lctrl{move(setup)}; - - auto isBetween = [](auto lo, auto hi, auto val) + auto is_between = [](auto lo, auto hi, auto val) { return lo <= val and val < hi; }; + LoadController lctrl; + TimeVar now = RealClock::now(); - Time next{now + FSecs(10)}; - lctrl.tendNext (next); + Offset ten{FSecs(10)}; + Time next{now + ten}; + lctrl.tendNext(next); + CHECK (Time::ZERO == lctrl.scatteredDelayTime (now, Capacity::DISPATCH) ); CHECK (Time::ZERO == lctrl.scatteredDelayTime (now, Capacity::SPINTIME) ); - CHECK ( next == lctrl.scatteredDelayTime (now, Capacity::TENDNEXT) ); - CHECK (isBetween ( next, next+WORK_HORIZON , lctrl.scatteredDelayTime (now, Capacity::NEARTIME))); - CHECK (isBetween ( next, next+SLEEP_HORIZON, lctrl.scatteredDelayTime (now, Capacity::WORKTIME))); - CHECK (isBetween (Time::ZERO, SLEEP_HORIZON , lctrl.scatteredDelayTime (now, Capacity::IDLETIME))); - } + CHECK ( ten == lctrl.scatteredDelayTime (now, Capacity::TENDNEXT) ); + CHECK (is_between ( ten, ten+ WORK_HORIZON, lctrl.scatteredDelayTime (now, Capacity::NEARTIME))); + CHECK (is_between ( ten, ten+SLEEP_HORIZON, lctrl.scatteredDelayTime (now, Capacity::WORKTIME))); + CHECK (is_between (Time::ZERO, SLEEP_HORIZON, lctrl.scatteredDelayTime (now, Capacity::IDLETIME))); + + // Offset is randomised based on the current time + // Verify this yields an even distribution + double avg{0}; + const size_t REPETITIONS = 1e6; + for (size_t i=0; i< REPETITIONS; ++i) + avg += _raw(lctrl.scatteredDelayTime (RealClock::now(), Capacity::IDLETIME)); + avg /= REPETITIONS; + + auto expect = _raw(SLEEP_HORIZON)/2; + auto error = fabs(avg/expect - 1); + CHECK (0.001 > error); // observing a quite stable skew ~ 0.4‰ on my system + } // let's see if this error bound triggers eventually... + /** @test TODO @@ -267,16 +294,6 @@ namespace test { { UNIMPLEMENTED ("walking Deadline"); } - - - - /** @test TODO - * @todo WIP 10/23 🔁 define ⟶ implement - */ - void - setupLalup() - { - } }; diff --git a/wiki/renderengine.html b/wiki/renderengine.html index f6fec7c74..0f2c2b0d0 100644 --- a/wiki/renderengine.html +++ b/wiki/renderengine.html @@ -7167,7 +7167,7 @@ Later on we expect a distinct __query subsystem__ to emerge, presumably embeddin &rarr; QuantiserImpl -
+
//Invoke and control the dependency and time based execution of  [[render jobs|RenderJob]]//
 The Scheduler acts as the central hub in the implementation of the RenderEngine and coordinates the //processing resources// of the application. Regarding architecture, the Scheduler is located in the Vault-Layer and //running// the Scheduler is equivalent to activating the »Vault Subsystem«. An EngineFaçade acts as entrance point, providing high-level render services to other parts of the application: [[render jobs|RenderJob]] can be activated under various timing and dependency constraints. Internally, the implementation is organised into two layers:
 ;Layer-2: Coordination
@@ -7184,6 +7184,9 @@ This leads to the observation that every render or playback process has to deal
 The time-based ordering and prioritisation of [[render activities|RenderActivity]] is thus used as a //generic medium and agent// to support and implement complex interwoven computational tasks. On the layer-1 mentioned above, a combination of a lock-free dispatch queue is used, feeding into a single threaded priority queue organised by temporal deadlines. Most render activities are lightweight and entail quick updates to some state flags, while certain activities are extremely long running -- and those are shifted into worker threads based on priority.
 &rarr; [[the Activity-Language|RenderActivity]]
 &rarr; [[implementing Activities|RenderOperationLogic]]
+!!!Managing the Load
+At large, these considerations hint at a diverse and unspecific range of usages -- necessitating to rely on organisational schemes able to adapt to a wide array of load patterns, including sudden and drastic changes. Moreover the act of scheduling involves operating on time scales spanning several orders of magnitude, from dispatching and cascading notifications working on the µs scale up to a planning horizon for render calculations reaching out into a dimension of several seconds. And while the basic construction of the scheduler can be made to //behave correct// and accommodate without failure, it seems way more challenging to establish an arrangement of functionality able to yield good performance on average. Certainly this goal can not be achieved in one step, and by a single layer of implementation. Rather a second layer of operational control may be necessary to keep the machinery within suitable limits, possibly even some degree of dynamic optimisation of working parameters.
+&rarr; [[Load management and operational control|SchedulerLoadControl]]
 
 !Usage pattern
 The [[Language of render activities|RenderActivity]] forms the interface to the scheduler -- new activities are defined as //terms// and handed over to the scheduler. This happens as part of the ongoing job planning activities -- and thus will be performed //from within jobs managed by the scheduler.// Thus the access to the scheduler happens almost entirely from within the scheduler's realm itself, and is governed by the usage scheme of the [[Workers|SchedulerWorker]].
@@ -7198,6 +7201,13 @@ The Scheduler is now considered an implementation-level facility with an interfa
 &rarr; [[Workers|SchedulerWorker]]
+
+
The scheduling mechanism //requires active control of work parameters to achieve good performance on average.//
+In a nutshell, the scheduler arranges planned [[render activities|RenderActivity]] onto a time axis -- and is complemented by an [[active »work force«|SchedulerWorker]] to //pull and retrieve// the most urgent next task when free processing capacity becomes available. This arrangement shifts focus from the //management of tasks// towards the //management of capacity// -- which seems more adequate, given that capacity is scarce while tasks are abundant, yet limited in size small and atomic.
+
+This change of perspective however implies that not the time-axis is in control; rather processing is driven by the workers, which happen to show up in a pattern assumed to be //essentially random.// This leads to the conclusion that capacity may not be available at the point where it's needed most, and some overarching yet slowly adapting scheme of redistribution is required to make the render engine run smoothly.
+
+
//The Scheduler uses an »Extent« based memory management scheme known as {{{BlockFlow}}}.//
 The organisation of rendering happens in terms of [[Activities|RenderActivity]], which may bound by //dependencies// and limited by //deadlines.// For the operational point of view this implies that a sequence of allocations must be able to „flow through the Scheduler“ -- in fact, only references to these {{{Activity}}}-records are passed, while the actual descriptors reside at fixed memory locations. This is essential to model the dependencies and conditional execution structures efficiently. At some point however, any {{{Activity}}}-record will either be //performed// or //obsoleted// -- and this leads to the idea of managing the allocations in //extents// of memory here termed as »Epochs«
diff --git a/wiki/thinkPad.ichthyo.mm b/wiki/thinkPad.ichthyo.mm
index 1feda0621..250fb51ef 100644
--- a/wiki/thinkPad.ichthyo.mm
+++ b/wiki/thinkPad.ichthyo.mm
@@ -81981,9 +81981,7 @@ Date:   Thu Apr 20 18:53:17 2023 +0200

 
 
 
-  
-    
-  
+  
   
     

       ...unterscheiden sich doch im Detail
@@ -82278,9 +82276,7 @@ Date:   Thu Apr 20 18:53:17 2023 +0200

 
 
 
-  
-    
-  
+  
   
     

       hier könnte man named arguments gebrauchen....
@@ -82292,9 +82288,7 @@ Date:   Thu Apr 20 18:53:17 2023 +0200

 
 
 
-  
-    
-  
+  
   
     

       Parametrisierung: maxCapacity = work::Config::COMPUTATION_CAPACITY
@@ -82310,9 +82304,7 @@ Date:   Thu Apr 20 18:53:17 2023 +0200

 
 
 
-  
-    
-  
+  
   
     

       schwierige Frage....
@@ -82725,10 +82717,10 @@ Date:   Thu Apr 20 18:53:17 2023 +0200

 
 
 
-
-
-
-
+
+
+
+
 
 
 
@@ -82753,36 +82745,30 @@ Date:   Thu Apr 20 18:53:17 2023 +0200

 
 
 
-  
-    
-  
+  
   
     

       Die Zeitangaben im std::chrono-Framework reichen bis in den Nano-Bereich, und es gibt einen high-precision-Timer
     

   
-
-
+
 
 
 
-  
-    
-  
+  
   
     

       ...andererseits weiß ich, daß man schon einfachste Scheduling-Delays ab mindestens 400ns mißt, und daß das Starten eines Thread auf meinem System mindestens 100µs braucht. Der aktuelle Scheduler unter Linux (CFS) verwendet keine festen Time-Slices mehr, aber man versucht definitiv die Kontext-Switches zu minimieren. Latenzen oder Scheduling-Zyklen für normale (nicht-realtime)-Prozesse liegen im Bereich von Millisekunden. Andererseits arbeitet die C++-Chrono-Funktion sleep_for nachweislich im Schnitt bis in den zweistelligen µs-Bereich genau
     

   
-
-
+
 
 
 
 
 
 
-
+
 
 
 
@@ -82794,41 +82780,36 @@ Date:   Thu Apr 20 18:53:17 2023 +0200

 
 
 
-  
-    
-  
+  
   
     

       die Funktion liefert eine Duration
     

   
-
-
+
 
 
 
 
 
 
-  
-    
-  
+  
   
     

       ...und spart einen zusätzlichen clamp-Schritt und den Absolutwert (den das OS sowiso machen wird)
     

   
-
-
+
 
 
 
 
 
 
-
-
-
+
+
+
+
 
 
 
@@ -82836,10 +82817,70 @@ Date:   Thu Apr 20 18:53:17 2023 +0200

 
 
 
-
-
+
+
+
+
+
+
+
+
+  
+  
+    

+      es ist nicht auszuschließen, daß gewisse End-Bits häufiger auftreten, weil die OS-Aufrufe / Scheduler-Aktivitäten eben doch mit einer gewissen Regelmäßigkeit stattfinden
+    

+  
+
 
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+  
+  
+    

+      Im Zusammenhang mit dem JobTicket (in der Tat, da braucht man einen ordentlichen hash-Key aus beliebiger Zeit)
+    

+    

+      Siehe TimeValue_test::checkTimeHash()
+    

+  
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 
 
 
@@ -90484,6 +90525,9 @@ Date:   Thu Apr 20 18:53:17 2023 +0200

 
 
 
+
+
+