ichthyo/LUMIERA.clone
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Chain-Load: implement translation into Scheduler invocations

Browse source 
... so this (finally) is the missing cornerstone
... traverse the calculation graph and generate render jobs
... provide a chunk-wise pre-planning of the next batch
... use a future to block the (test) thread until completed
This commit is contained in:
Fischlurch 2023-12-05 23:53:42 +01:00
parent 5e9b115283
commit 481e35a597
4 changed files with 451 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

109
src/lib/uninitialised-storage.hpp
View file

@ -126,5 +126,114 @@ namespace lib {
};
/**
* Managed uninitialised Heap-allocated storage with array like access.
* @tparam T the nominal type assumed to sit in each »slot«
*/
template<typename T>
class UninitialisedDynBlock
{
using _Arr = T[];
T* buff_{nullptr};
size_t size_{0};
public:
T*
allocate(size_t cnt)
{
if (buff_) discard();
size_ = cnt;
buff_ = cnt? std::aligned_alloc (std::alignment_of<T>(), cnt * sizeof(T))
: nullptr;
return buff_;
}
void
discard()
{
std::free (buff_);
buff_ = nullptr;
size_ = 0;
}
UninitialisedDynBlock() =default;
~UninitialisedDynBlock()
{
if (buff_)
discard();
}
explicit
UninitialisedDynBlock (size_t cnt)
{
if (cnt)
allocate(cnt);
}
UninitialisedDynBlock (UninitialisedDynBlock && rr)
{
if (this != &rr)
swap (*this, rr);
}
UninitialisedDynBlock (UninitialisedDynBlock const&) =delete;
UninitialisedDynBlock& operator= (UninitialisedDynBlock &&) =delete;
UninitialisedDynBlock& operator= (UninitialisedDynBlock const&) =delete;
friend void
swap (UninitialisedDynBlock& u1, UninitialisedDynBlock& u2)
{
std::swap (u1.size_, u2.size_);
std::swap (u1.buff_, u2.buff_);
}
explicit
operator bool() const
{
return bool(buff_);
}
size_t
size() const
{
return size_;
}
_Arr&
array()
{
return * std::launder (reinterpret_cast<_Arr* > (buff_));
}
_Arr const&
array() const
{
return * std::launder (reinterpret_cast<_Arr const*> (buff_));
}
T & operator[] (size_t idx) { return array()[idx]; }
T const& operator[] (size_t idx) const { return array()[idx]; }
template<typename...Args>
T&
createAt (size_t idx, Args&& ...args)
{
return *new(&operator[](idx)) T{std::forward<Args>(args)...};
}
void
destroyAt (size_t idx)
{
operator[](idx).~T();
}
};
} // namespace lib
#endif /*LIB_UNINITIALISED_STORAGE_H*/

14
src/vault/gear/scheduler.hpp
View file

@ -190,6 +190,20 @@ namespace gear {
/** build Activity chain and hand-over to the Scheduler. */
ScheduleSpec post();
ScheduleSpec
linkToSuccessor (ScheduleSpec& succSpec)
{
term_->appendNotificationTo (*succSpec.term_);
return move(*this);
}
ScheduleSpec
linkToPredecessor (ScheduleSpec& predSpec)
{
predSpec.term_->appendNotificationTo (*term_);
return move(*this);
}
};

212
tests/vault/gear/test-chain-load.hpp
View file

@ -85,8 +85,9 @@
//#include "lib/hash-value.h"
#include "vault/gear/job.h"
//#include "vault/gear/activity.hpp"
#include "vault/gear/scheduler.hpp"
//#include "vault/gear/nop-job-functor.hpp"
#include "lib/uninitialised-storage.hpp"
#include "lib/time/timevalue.hpp"
//#include "lib/meta/variadic-helper.hpp"
//#include "lib/meta/function.hpp"
@ -102,10 +103,10 @@
#include <boost/functional/hash.hpp>
#include <functional>
#include <utility>
#include <string>
#include <vector>
#include <future>
#include <memory>
#include <string>
#include <vector>
#include <tuple>
#include <array>
@ -119,6 +120,7 @@ namespace test {
using lib::time::Time;
using lib::time::TimeValue;
using lib::time::FrameRate;
using lib::time::Duration;
// using lib::time::FSecs;
// using lib::time::Offset;
// using lib::meta::RebindVariadic;
@ -138,12 +140,18 @@ namespace test {
using std::swap;
using std::move;
// using boost::hash_combine;
using std::chrono_literals::operator ""s;
namespace err = lumiera::error;
namespace dot = lib::dot_gen;
namespace { // Default definitions for topology generation
const size_t DEFAULT_FAN = 16;
const size_t DEFAULT_SIZ = 256;
const auto SAFETY_TIMEOUT = 5s;
const auto STANDARD_DEADLINE = 10ms;
const size_t DEFAULT_CHUNKSIZE = 64;
}
struct Statistic;
@ -606,7 +614,10 @@ namespace test {
Statistic computeGraphStatistics();
TestChainLoad&& printTopologyStatistics();
private:
class ScheduleCtx;
friend class ScheduleCtx; // accesses raw storage array
ScheduleCtx setupSchedule (Scheduler& scheduler);
};
@ -1186,5 +1197,198 @@ namespace test {
/**
* Setup and wiring for a test run to schedule a computation structure
* as defined by this TestChainLoad instance. This context is linked to
* a concrete TestChainLoad and Scheduler instance and holds a memory block
* with actual schedules, which are dispatched in batches into the Scheduler.
* It is *crucial* to keep this object *alive during the complete test run*,
* which is achieved by a blocking wait on the callback triggered after
* dispatching the last batch of calculation jobs. This process itself
* is meant for test usage and not thread-safe (while obviously the
* actual scheduling and processing happens in the worker threads).
* Yet the instance can be re-used to dispatch further test runs.
*/
template<size_t maxFan>
class TestChainLoad<maxFan>::ScheduleCtx
: util::MoveOnly
{
TestChainLoad& chainLoad_;
Scheduler& scheduler_;
lib::UninitialisedDynBlock<ScheduleSpec> schedule_;
FrameRate levelSpeed_{1, Duration{_uTicks(1ms)}};
uint loadFactor_{2};
size_t chunkSize_{DEFAULT_CHUNKSIZE};
TimeVar startTime_{Time::ANYTIME};
Duration preRoll_{_uTicks(200us)};
microseconds deadline_{STANDARD_DEADLINE};
ManifestationID manID_{};
std::promise<void> signalDone_{};
RandomChainCalcFunctor<maxFan> calcFunctor_;
RandomChainPlanFunctor<maxFan> planFunctor_;
/* ==== Callbacks from job planning ==== */
/** Callback: place a single job into the scheduler */
void
disposeStep (size_t idx, size_t level)
{
schedule_[idx] = scheduler_.defineSchedule(calcJob (idx,level))
.manifestation(manID_)
.startTime (calcStartTime(level))
.lifeWindow (deadline_)
.post();
}
/** Callback: define a dependency between scheduled jobs */
void
setDependency (Node* pred, Node* succ)
{
size_t predIdx = chainLoad_.nodeID (pred);
size_t succIdx = chainLoad_.nodeID (succ);
schedule_[predIdx].linkToSuccessor (schedule_[succIdx]);
}
/** continue planning: schedule follow-up planning job */
void
continuation (size_t levelDone, bool work_left)
{
if (work_left)
{
size_t nextChunkLevel = calcNextLevel (levelDone);
scheduler_.continueMetaJob (calcPlanScheduleTime (nextChunkLevel)
,planningJob (nextChunkLevel)
,manID_);
}
else
signalDone_.set_value();
}
std::future<void>
performRun()
{
size_t numNodes = chainLoad_.size();
schedule_.allocate (numNodes);
startTime_ = anchorStartTime();
scheduler_.seedCalcStream (planningJob(0)
,manID_
,calcLoadHint());
return attachNewCompletionSignal();
}
public:
ScheduleCtx (TestChainLoad& mother, Scheduler& scheduler)
: chainLoad_{mother}
, scheduler_{scheduler}
, calcFunctor_{chainLoad_.nodes_[0]}
, planFunctor_{chainLoad_.nodes_[0], chainLoad_.numNodes_
,[this](size_t i, size_t l){ disposeStep(i,l); }
,[this](auto* p, auto* s) { setDependency(p,s);}
,[this](size_t l, bool w) { continuation(l,w); }
}
{ }
ScheduleCtx
launch_and_wait()
{
awaitBlocking(
performRun());
return move(*this);
}
private:
/** push away any existing wait state and attach new clean state */
std::future<void>
attachNewCompletionSignal()
{
signalDone_.set_exception (std::make_exception_ptr(std::future_error (std::future_errc::broken_promise)));
std::promise<void> notYetTriggered;
signalDone_.swap (notYetTriggered);
return signalDone_.get_future();
}
void
awaitBlocking(std::future<void> signal)
{
if (std::future_status::timeout == signal.wait_for (SAFETY_TIMEOUT))
throw err::Fatal("Timeout on Scheduler test exceeded.");
}
Job
calcJob (size_t idx, size_t level)
{
return Job{calcFunctor_
,calcFunctor_.encodeNodeID(idx)
,calcFunctor_.encodeLevel(level)
};
}
Job
planningJob (size_t level)
{
return Job{planFunctor_
,InvocationInstanceID()
,planFunctor_.encodeLevel(level)
};
}
Time
anchorStartTime()
{
return RealClock::now() + preRoll_;
}
FrameRate
calcLoadHint()
{
return FrameRate{levelSpeed_ * loadFactor_};
}
size_t
calcNextLevel (size_t levelDone)
{
return levelDone + chunkSize_;
}
Time
calcStartTime(size_t level)
{
return startTime_ + level / levelSpeed_;
}
Time
calcPlanScheduleTime (size_t nextChunkLevel)
{/* must be at least 1 level ahead,
because dependencies are defined backwards;
the chain-load graph only defines dependencies over one level
thus the first level in the next chunk must still be able to attach
dependencies to the last row of the preceding chunk, implying that
those still need to be ahead of schedule.
*/
nextChunkLevel = nextChunkLevel>2? nextChunkLevel-2 : 0;
return calcStartTime(nextChunkLevel) - preRoll_;
}
};
/**
* establish and configure the context used for scheduling computations.
*/
template<size_t maxFan>
typename TestChainLoad<maxFan>::ScheduleCtx
TestChainLoad<maxFan>::setupSchedule(Scheduler& scheduler)
{
return ScheduleCtx{*this, scheduler};
}
}}} // namespace vault::gear::test
#endif /*VAULT_GEAR_TEST_TEST_CHAIN_LOAD_H*/

123
wiki/thinkPad.ichthyo.mm
View file

@ -100597,13 +100597,17 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
<node CREATED="1701492828257" ID="ID_1372403993" MODIFIED="1701492835845" TEXT="also kann es nur aus dem Meta-Job kommen"/>
<node CREATED="1701492899063" ID="ID_261543502" MODIFIED="1701492907570" TEXT="der k&#xf6;nnte eine Closure ausf&#xfc;hren oder eine Flag setzen"/>
</node>
<node CREATED="1701492932866" ID="ID_331874903" MODIFIED="1701548899190" TEXT="man k&#xf6;nnte hier fest ein wait()-notify() codieren">
<node COLOR="#5b280f" CREATED="1701492932866" ID="ID_331874903" MODIFIED="1701805809979" TEXT="man k&#xf6;nnte hier fest ein wait()-notify() codieren">
<icon BUILTIN="button_cancel"/>
<node CREATED="1701492951352" ID="ID_599266710" MODIFIED="1701492962514" TEXT="d.h. der Schedule()-Aufruf..."/>
<node CREATED="1701492963126" ID="ID_1362334556" MODIFIED="1701492969265" TEXT="w&#xfc;rde den Meta-Job absetzen"/>
<node CREATED="1701492969853" ID="ID_354060729" MODIFIED="1701492985848" TEXT="und danach in ein thread:;wait() gehen"/>
<node CREATED="1701492990299" ID="ID_999473580" MODIFIED="1701492995350" TEXT="ggfs sogar mit Timeout()!"/>
<node CREATED="1701493003512" ID="ID_436816851" MODIFIED="1701493015501" TEXT="die Closure aus dem Meta-Job weckt uns dann wieder auf"/>
</node>
<node CREATED="1701805794013" ID="ID_377567761" MODIFIED="1701805803843" TEXT="noch besser: ein Future">
<icon BUILTIN="forward"/>
</node>
</node>
</node>
</node>
@ -100644,8 +100648,40 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
<node CREATED="1701550947518" ID="ID_844132583" MODIFIED="1701550954585" TEXT="die waren als Builder gedacht"/>
<node CREATED="1701630020156" ID="ID_1451722632" MODIFIED="1701630029914" TEXT="und damit auf eine Einzelfall-Anwendung zugeschnitten"/>
<node CREATED="1701630030692" ID="ID_1766960147" MODIFIED="1701630048493" TEXT="nun haben wir es mit schematischer Massen-Verarbeitung zu tun"/>
<node CREATED="1701574628934" ID="ID_12588074" MODIFIED="1701574720900" TEXT="mit einem UninitialisedStorage-Array k&#xf6;nnte es besser gehen">
<node COLOR="#5b280f" CREATED="1701574628934" ID="ID_12588074" MODIFIED="1701784680952" TEXT="mit einem UninitialisedStorage-Array k&#xf6;nnte es besser gehen">
<arrowlink COLOR="#5b92bc" DESTINATION="ID_1331044905" ENDARROW="Default" ENDINCLINATION="-1107;79;" ID="Arrow_ID_1234105866" STARTARROW="None" STARTINCLINATION="-2546;116;"/>
<icon BUILTIN="button_cancel"/>
<node COLOR="#9c3b76" CREATED="1701784703807" HGAP="45" ID="ID_227978995" MODIFIED="1701784860107" TEXT="b&#xe4;&#xe4;&#xe4;h ... reingefallen" VSHIFT="4">
<richcontent TYPE="NOTE"><html>
<head>
</head>
<body>
<p>
Gut da&#223; wir dar&#252;ber geredet haben.
</p>
<p>
Da <i>offensichtlich</i>&#160;die L&#228;nge des Test-Graphen keine <i>Eigenschaft</i>&#160; der Teststruktur ist, landen wir bei einer rein dynamischen Allokation. Trotzdem war es eine gute Idee, das mit der <i>uninitialised storage</i>&#160; endlich mal zu codifizierren...
</p>
</body>
</html>
</richcontent>
<font NAME="SansSerif" SIZE="11"/>
</node>
</node>
<node COLOR="#338800" CREATED="1701814584902" ID="ID_1483667434" MODIFIED="1701814756796" TEXT="&#x27f9; weitere Komponente: UninitialisedDynBlock">
<icon BUILTIN="button_ok"/>
<node CREATED="1701814636846" ID="ID_1886070155" MODIFIED="1701814655345" TEXT="wie ein unique_ptr mit einem rohen Array"/>
<node CREATED="1701814656141" ID="ID_947012950" MODIFIED="1701814670398" TEXT="tricky... aligned heap storage">
<icon BUILTIN="messagebox_warning"/>
<node CREATED="1701814673346" ID="ID_1216474072" MODIFIED="1701814677940" TEXT="geht mit C++17"/>
<node CREATED="1701814678586" ID="ID_1765186733" MODIFIED="1701814701681" TEXT="aber besser std::alligned_alloc / free"/>
<node CREATED="1701814703981" ID="ID_1906895163" MODIFIED="1701814715759" TEXT="und auch kein unique_ptr"/>
<node CREATED="1701814716017" ID="ID_183435871" MODIFIED="1701814734472" TEXT="Immer Besser Manuell"/>
</node>
<node CREATED="1701814764605" ID="ID_1865767062" MODIFIED="1701814780196" TEXT="hat dann direkt einen operator[idx]">
<icon BUILTIN="idea"/>
</node>
</node>
</node>
<node CREATED="1701630108121" ID="ID_466850642" MODIFIED="1701630276535" TEXT="Vorteil (nur) hier: Prerequisites liegen stetes genau einen Level tiefer">
@ -100974,8 +101010,89 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
</node>
</node>
</node>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1701493107387" ID="ID_1826192324" MODIFIED="1701493117170" TEXT="Aufruf-front-End">
<node BACKGROUND_COLOR="#eef0c5" COLOR="#990000" CREATED="1701493107387" ID="ID_1826192324" MODIFIED="1701814987380" TEXT="Aufruf-front-End">
<icon BUILTIN="pencil"/>
<node COLOR="#435e98" CREATED="1701814975960" ID="ID_1515169294" MODIFIED="1701814982780" TEXT="move-only">
<icon BUILTIN="yes"/>
</node>
<node BACKGROUND_COLOR="#f8f1cb" COLOR="#a50125" CREATED="1701800889991" ID="ID_735638406" MODIFIED="1701800928579" TEXT="Gefahr: Rahmen mu&#xdf; am Leben bleiben bis zum Ende">
<icon BUILTIN="messagebox_warning"/>
<node CREATED="1701800940860" ID="ID_962597170" MODIFIED="1701801201157" TEXT="bedingt durch das Builder-API ist das nicht klar">
<icon BUILTIN="clanbomber"/>
</node>
<node CREATED="1701801057709" ID="ID_383743671" MODIFIED="1701801083509" TEXT="&#x27f9; der launch()-call mu&#xdf; blocken"/>
<node COLOR="#5b280f" CREATED="1701801100142" ID="ID_869440181" MODIFIED="1701801109118" TEXT="Deadlock? Timeout?">
<icon BUILTIN="button_cancel"/>
<node BACKGROUND_COLOR="#ccb59b" COLOR="#6e2a38" CREATED="1701801112720" ID="ID_676583243" MODIFIED="1701801179434" TEXT="es ist ein TE-EST !!!!">
<font ITALIC="true" NAME="SansSerif" SIZE="14"/>
<icon BUILTIN="yes"/>
</node>
<node COLOR="#435e98" CREATED="1701801150632" ID="ID_1979003433" MODIFIED="1701801163214" TEXT="(der funktioniert oder er wird repariert)">
<font NAME="SansSerif" SIZE="8"/>
</node>
</node>
</node>
<node COLOR="#338800" CREATED="1701814806243" ID="ID_1359675923" MODIFIED="1701823453699" TEXT="Job-Funktoren">
<icon BUILTIN="button_ok"/>
<node COLOR="#435e98" CREATED="1701814828932" ID="ID_1598907893" MODIFIED="1701814991598" TEXT="RandomChainCalcFunctor"/>
<node COLOR="#435e98" CREATED="1701814837569" ID="ID_1253997420" MODIFIED="1701814991599" TEXT="RandomChainPlanFunctor">
<node CREATED="1701814852805" ID="ID_437138180" MODIFIED="1701814876079" TEXT="die Callbacks &#x27f6; member functions"/>
</node>
<node COLOR="#338800" CREATED="1701814811675" ID="ID_1396437995" MODIFIED="1701823438336" STYLE="fork" TEXT="geleich im Ctor verdrahten">
<icon BUILTIN="yes"/>
</node>
</node>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1701823443375" ID="ID_122352824" MODIFIED="1701823451292" TEXT="Konfiguration">
<icon BUILTIN="flag-yellow"/>
<node CREATED="1701823497984" ID="ID_1899753971" MODIFIED="1701823497984" TEXT="levelSpeed">
<node CREATED="1701823532081" ID="ID_731746076" MODIFIED="1701823543538" TEXT="Scheduling erfolgt stets in ganzen Levels"/>
<node CREATED="1701823580099" ID="ID_1709081551" MODIFIED="1701823587419" TEXT="Framerate &#xbb;levels per second&#xab;"/>
</node>
<node CREATED="1701823497984" ID="ID_7329016" MODIFIED="1701823497984" TEXT="loadFactor">
<node CREATED="1701823590880" ID="ID_83421731" MODIFIED="1701823595968" TEXT="f&#xfc;r die Last-Ank&#xfc;ndigung"/>
<node CREATED="1701823596785" ID="ID_936842920" MODIFIED="1701823605026" TEXT="levelSpeed * loadFactor"/>
<node CREATED="1701823611337" ID="ID_1999427852" MODIFIED="1701823629887" TEXT="kann besonders &quot;dicke&quot; graphen ber&#xfc;cksichtigen"/>
<node CREATED="1701823630635" ID="ID_1043324643" MODIFIED="1701823633647" TEXT="default = 2"/>
</node>
<node CREATED="1701823497985" ID="ID_1273899536" MODIFIED="1701823497985" TEXT="chunkSize">
<node CREATED="1701823635949" ID="ID_1273047930" MODIFIED="1701823642301" TEXT="wieviele Level auf einmal geplant werden"/>
<node CREATED="1701823642889" ID="ID_55027963" MODIFIED="1701823648269" TEXT="default = 64">
<node CREATED="1701823649193" ID="ID_747457374" MODIFIED="1701823654972" TEXT="damit&apos;s glatt aufgeht"/>
<node CREATED="1701823656704" ID="ID_423929270" MODIFIED="1701823660616" TEXT="mu&#xdf; aber nicht..."/>
</node>
</node>
<node CREATED="1701823497985" ID="ID_444779522" MODIFIED="1701823497985" TEXT="startTime">
<node CREATED="1701823663531" ID="ID_794484348" MODIFIED="1701823675537" TEXT="wird beim Triggern aus RealClock::now() gebildet"/>
<node CREATED="1701823677717" ID="ID_1040577946" MODIFIED="1701823680608" TEXT="+ preRoll"/>
</node>
<node CREATED="1701823497985" ID="ID_378847515" MODIFIED="1701823497985" TEXT="preRoll">
<node CREATED="1701823682284" ID="ID_746474019" MODIFIED="1701823709396" TEXT="vorlauf zum Beginn des geplanten Schedule"/>
<node CREATED="1701823711000" ID="ID_20712355" MODIFIED="1701823725194" TEXT="wird auch von wiederholungs-Chunks abgezogen"/>
</node>
<node CREATED="1701823497985" ID="ID_158048244" MODIFIED="1701823727929" TEXT="deadline">
<node CREATED="1701823729483" ID="ID_334630071" MODIFIED="1701823749646" TEXT="Deadline-Fenster gegen&#xfc;ber der gegebenen Startzeit"/>
<node CREATED="1701823750307" ID="ID_1925894290" MODIFIED="1701823764478" TEXT="hat Einflu&#xdf; auf die Auslastung des BlockFlow">
<icon BUILTIN="messagebox_warning"/>
</node>
</node>
<node CREATED="1701823497985" ID="ID_959522068" MODIFIED="1701823778291" TEXT="ManifestationID">
<node CREATED="1701823779532" ID="ID_1454168570" MODIFIED="1701823785370" TEXT="per default leer"/>
<node CREATED="1701823806663" ID="ID_141955882" MODIFIED="1701823816646" TEXT="wird f&#xfc;r alle Jobs und Planungs-Jobs mit angegeben"/>
<node CREATED="1701823785942" ID="ID_1311722331" MODIFIED="1701823803144" TEXT="damit k&#xf6;nnte man laufende Tests abbrechen"/>
</node>
</node>
<node COLOR="#338800" CREATED="1701823833695" ID="ID_1531475388" MODIFIED="1701823843723" TEXT="Scheduling verdrahten">
<icon BUILTIN="button_ok"/>
<node COLOR="#338800" CREATED="1701823845406" ID="ID_1891718856" MODIFIED="1701823854911" TEXT="Platzieren der Einzel-Schedules">
<icon BUILTIN="button_ok"/>
</node>
<node COLOR="#338800" CREATED="1701823855757" ID="ID_953268597" MODIFIED="1701823863268" TEXT="Verschalten der Dependencies">
<icon BUILTIN="button_ok"/>
</node>
<node COLOR="#338800" CREATED="1701823864060" ID="ID_1910037102" MODIFIED="1701823874147" TEXT="Planen der Planungs-Chunks">
<icon BUILTIN="button_ok"/>
</node>
</node>
</node>
</node>
</node>