ichthyo/LUMIERA.clone
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Chain-Load: implement planning JobFunctor

Browse source 
- decided to abstract the scheduler invocations as λ
- so this functor contains the bare loop logic

Investigation regarding hash-framework:
It turns out that boost::hash uses a different hash_combine,
than what we have extracted/duplicated in lib/hash-value.hpp
(either this was a mistake, or boost::hash did use this weaker
 function at that time and supplied a dedicated 64bit implementation later)

Anyway, should use boost::hash for the time being
maybe also fix the duplicated impl in lib/hash-value.hpp
This commit is contained in:
Fischlurch 2023-12-04 16:29:57 +01:00
parent 2e6712e816
commit 29ca3a485f
5 changed files with 282 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
src/lib/hash-value.h
View file

@ -75,7 +75,36 @@ namespace lib {
+ (combinedHash<<6)
+ (combinedHash>>2);
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////TICKET #722 : Boost uses a stronger impl here on 64bit platforms
/// see: Boost 1.67 <include>/boost/container_has/hash.hpp
///
/*
// Don't define 64-bit hash combine on platforms without 64 bit integers,
// and also not for 32-bit gcc as it warns about the 64-bit constant.
#if !defined(BOOST_NO_INT64_T) && \
!(defined(__GNUC__) && ULONG_MAX == 0xffffffff)
inline void hash_combine_impl(boost::uint64_t& h,
boost::uint64_t k)
{
const boost::uint64_t m = UINT64_C(0xc6a4a7935bd1e995);
const int r = 47;
k *= m;
k ^= k >> r;
k *= m;
h ^= k;
h *= m;
// Completely arbitrary number, to prevent 0's
// from hashing to 0.
h += 0xe6546b64;
}
#endif // BOOST_NO_INT64_T
*/
//
// WIP more utils to come here....
}

3
src/vault/gear/job.h
View file

@ -72,7 +72,8 @@ enum JobKind
{
CALC_JOB, ///< calculating frame data, CPU bound
LOAD_JOB, ///< accessing prerequisites, IO bound
META_JOB ///< render process self organisation
META_JOB, ///< render process self organisation
TEST_JOB ///< test and diagnostic and research
};
enum JobPriority

5
src/vault/gear/nop-job-functor.hpp
View file

@ -51,9 +51,8 @@ namespace gear {
/**
* Stub implementation of the JobFunctor interface
* Stub/Test implementation of the JobFunctor interface
* for a render job _to do nothing at all_
* @todo WIP as of 4/2023
*/
class NopJobFunctor
: public JobClosure
@ -64,7 +63,7 @@ namespace gear {
JobKind
getJobKind() const
{
return META_JOB;
return TEST_JOB;
}
std::string

189
tests/vault/gear/test-chain-load.hpp
View file

@ -27,7 +27,7 @@
** characteristics of the implementation, a well-defined artificial computation load is
** necessary, comprised of the invocation of an extended number of Jobs, each configured
** to carry out a reproducible computation. Data dependencies between jobs can be established
** to verify handling of dependent jobs and job completion messages within the scheduler.
** to verify handling of dependent jobs and job completion messages within the scheduler.
**
** # Random computation structure
** A system of connected hash values is used as computation load, akin to a blockchain.
@ -83,9 +83,10 @@
#include "vault/common.hpp"
#include "lib/test/transiently.hpp"
//#include "lib/hash-value.h"
#include "vault/gear/job.h"
//#include "vault/gear/activity.hpp"
#include "vault/gear/nop-job-functor.hpp"
//#include "vault/gear/nop-job-functor.hpp"
#include "lib/time/timevalue.hpp"
//#include "lib/meta/variadic-helper.hpp"
//#include "lib/meta/function.hpp"
@ -114,9 +115,9 @@ namespace gear {
namespace test {
using std::string;
// using std::function;
// using lib::time::TimeValue;
using std::function;
using lib::time::Time;
using lib::time::TimeValue;
using lib::time::FrameRate;
// using lib::time::FSecs;
// using lib::time::Offset;
@ -132,11 +133,11 @@ namespace test {
using lib::meta::_FunRet;
using lib::test::Transiently;
// using std::forward;
// using std::string;
using std::forward;
using std::string;
using std::swap;
using std::move;
using boost::hash_combine;
// using boost::hash_combine;
namespace dot = lib::dot_gen;
@ -241,9 +242,8 @@ namespace test {
size_t
calculate()
{
for (Node*& entry: pred)
if (entry)
hash_combine (hash, entry->hash);
for (Node* entry: pred)
boost::hash_combine (hash, entry->hash);
return hash;
}
@ -735,7 +735,7 @@ namespace test {
namespace {
template<class NOD>
inline auto
prepareEvaluaions()
prepareEvaluations()
{
return std::array<std::function<uint(NOD&)>, CAT>
{ [](NOD& ){ return 1; }
@ -893,7 +893,7 @@ namespace test {
TestChainLoad<maxFan>::computeGraphStatistics()
{
auto totalLevels = uint(topLevel());
auto classify = prepareEvaluaions<Node>();
auto classify = prepareEvaluations<Node>();
Statistic stat(totalLevels);
LevelSums particulars{0};
size_t level{0},
@ -1009,46 +1009,46 @@ namespace test {
/* ========= Render Job generation and Scheduling ========= */
template<size_t maxFan>
class RandomChainCalcFunctor
: public NopJobFunctor
/**
* Baseclass: JobFunctor to invoke TestChainLoad
*/
class ChainFunctor
: public JobClosure
{
using Node = typename TestChainLoad<maxFan>::Node;
Node* startNode_;
static lib::time::Grid&
testGrid() ///< Meyer's Singleton : a fixed 1fps quantiser
testGrid() ///< Meyer's Singleton : a fixed 1-f/s quantiser
{
static lib::time::FixedFrameQuantiser gridOne{FrameRate::STEP};
return gridOne;
}
/* === JobFunctor Interface === */
string diagnostic() const =0;
void invokeJobOperation (JobParameter) =0;
JobKind
getJobKind() const
{
return TEST_JOB;
}
InvocationInstanceID
buildInstanceID (HashVal) const override
{
return InvocationInstanceID();
}
size_t
hashOfInstance (InvocationInstanceID invoKey) const override
{
std::hash<size_t> hashr;
HashVal res = hashr (invoKey.frameNumber);
return res;
}
public:
RandomChainCalcFunctor(Node& startNode)
: startNode_{&startNode}
{ }
/** rigged special implementation of job invocation
*/
void
invokeJobOperation (JobParameter param) override
{
size_t nodeIdx = decodeNodeID (param.invoKey);
size_t level = decodeLevel (TimeValue{param.nominalTime});
Node& target = startNode_[nodeIdx];
ASSERT (target.level == level);
// invoke the »media calculation«
target.calculate();
}
string diagnostic() const override
{
return _Fmt{"ChainCalc(w:%d)▶%s"}
% maxFan
% util::showAddr(startNode_);
}
/** package the node-index to invoke.
* @note per convention for this test, this info will be
@ -1083,5 +1083,108 @@ namespace test {
/**
* Render JobFunctor to invoke the _calculation_ of a single Node.
* The existing Node connectivity is used to retrieve the hash values
* from predecessors — so these are expected to be calculated beforehand.
* For setup, the start of the ChainLoad's Node array is required.
* @tparam maxFan controls expected Node memory layout
*/
template<size_t maxFan>
class RandomChainCalcFunctor
: public ChainFunctor
{
using Node = typename TestChainLoad<maxFan>::Node;
Node* startNode_;
public:
RandomChainCalcFunctor(Node& startNode)
: startNode_{&startNode}
{ }
/** render job invocation to trigger one Node recalculation */
void
invokeJobOperation (JobParameter param) override
{
size_t nodeIdx = decodeNodeID (param.invoKey);
size_t level = decodeLevel (TimeValue{param.nominalTime});
Node& target = startNode_[nodeIdx];
ASSERT (target.level == level);
// invoke the »media calculation«
target.calculate();
}
string diagnostic() const override
{
return _Fmt{"ChainCalc(w:%d)◀%s"}
% maxFan
% util::showAddr(startNode_);
}
};
/**
* Render JobFunctor to perform chunk wise planning of Node jobs
* to calculate a complete Chain-Load graph step by step.
*/
template<size_t maxFan>
class RandomChainPlanFunctor
: public ChainFunctor
{
using Node = typename TestChainLoad<maxFan>::Node;
function<void(size_t,size_t)> scheduleCalcJob_;
function<void(size_t,size_t)> markDependency_;
function<void(size_t,bool)> continuation_;
size_t chunkSize_;
size_t maxCnt_;
Node* nodes_;
size_t currIdx_{0};
public:
template<class CAL, class DEP, class CON>
RandomChainPlanFunctor(size_t chunkSize, size_t maxLevel,
Node& nodeArray,
CAL&& schedule, DEP&& markDepend,
CON&& continuation)
: scheduleCalcJob_{forward<CAL> (schedule)}
, markDependency_{forward<CAL> (markDepend)}
, continuation_{continuation}
, chunkSize_{chunkSize}
, maxCnt_{maxLevel}
, nodes_{&nodeArray}
{ }
/** render job invocation to trigger one Node recalculation */
void
invokeJobOperation (JobParameter param) override
{
size_t targetLevel = decodeLevel (TimeValue{param.nominalTime});
for ( ; currIdx_<maxCnt_; ++currIdx_)
{
Node* n = &nodes_[currIdx_];
if (n->level > targetLevel)
break;
scheduleCalcJob_(currIdx_, n->level);
for (Node* pred: n.pred)
markDependency_(pred,n);
}
continuation_(targetLevel, currIdx_ < maxCnt_);
}
string diagnostic() const override
{
return "ChainPlan";
}
};
}}} // namespace vault::gear::test
#endif /*VAULT_GEAR_TEST_TEST_CHAIN_LOAD_H*/

103
wiki/thinkPad.ichthyo.mm
View file

@ -57667,6 +57667,70 @@
</node>
</node>
</node>
<node BACKGROUND_COLOR="#d2beaf" COLOR="#5c4d6e" CREATED="1701698319632"
ID="ID_883682364" MODIFIED="1701699545121" TEXT="Hash-Berechnung">
<icon BUILTIN="hourglass"/>
<icon BUILTIN="clanbomber"/>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1701698589858" ID="ID_1039799906" MODIFIED="1701698595947" TEXT="#722 uniform uses of hash values">
<icon BUILTIN="flag-yellow"/>
</node>
<node BACKGROUND_COLOR="#e0ceaa" COLOR="#690f14" CREATED="1701698037531" ID="ID_167943021" MODIFIED="1701698050090" TEXT="verwirrender Misch-Zustand hier">
<icon BUILTIN="broken-line"/>
<node CREATED="1701698058280" ID="ID_1250091422" MODIFIED="1701698189464" TEXT="teilweise boost, teilweise STDLIB">
<richcontent TYPE="NOTE"><html>
<head/>
<body>
<p>
ein Grund ist: STDLIB ist umst&#228;ndlich und unvollst&#228;ndig...
</p>
<ul>
<li>
w&#228;hrend Boost-functional-hash direkt per free-function (hash_value(x)) erweitert werden kann, mu&#223; man f&#252;r die STDLIB-Variante immer den Namespace std aufmachen
</li>
<li>
Boost bietet automatisch bereits Hash-Spezialisierungen f&#252;r alle erdenklichen Container, Tupel und sonstiges
</li>
</ul>
</body>
</html></richcontent>
<node BACKGROUND_COLOR="#f8f1cb" COLOR="#a50125" CREATED="1701698621411" ID="ID_1690934543" MODIFIED="1701698648712" TEXT="Vorsicht: lib::hash::combine ist schwach">
<icon BUILTIN="messagebox_warning"/>
<node CREATED="1701698649934" ID="ID_1402802007" MODIFIED="1701698665460" TEXT="das ist die Fallback-Implementierung aus boost::hash"/>
<node CREATED="1701698666056" ID="ID_1059722984" MODIFIED="1701698723192" TEXT="boost::hash verwendet aber auf 64bit eine st&#xe4;rkere Impl">
<richcontent TYPE="NOTE"><html>
<head/>
<body>
<p>
12/2023&#160;&#160;per Debugger verifiziert &#10230; TestChainLoad
</p>
</body>
</html></richcontent>
</node>
</node>
</node>
<node CREATED="1701698191775" ID="ID_1333941621" MODIFIED="1701698305483" TEXT="wir haben unsere LUID, die ist 128bit">
<icon BUILTIN="bell"/>
<node CREATED="1701698271332" ID="ID_615812427" MODIFIED="1701698278751" TEXT="Performance-Implikationen nicht klar"/>
<node CREATED="1701698280027" ID="ID_786194248" MODIFIED="1701698299569" TEXT="es gibt hierf&#xfc;r keinerlei Framework (Hasher, compbiner, Container-Adapter)"/>
</node>
<node BACKGROUND_COLOR="#e0ceaa" COLOR="#690f14" CREATED="1701698215387" ID="ID_1138411401" MODIFIED="1701698269652" TEXT="Hash-Values sind plattformabh&#xe4;ngig (Problem: Tests)">
<richcontent TYPE="NOTE"><html>
<head/>
<body>
<p>
de-facto wird nur noch 64-bit entwickelt. Inzwischen habe ich einige Tests, bei denen ich Hash-Values direkt pr&#252;fe. Die w&#252;rden auf 32bit alle brechen
</p>
</body>
</html></richcontent>
<icon BUILTIN="messagebox_warning"/>
</node>
</node>
<node BACKGROUND_COLOR="#d2beaf" COLOR="#5c4d6e" CREATED="1701698319632" ID="ID_1255956298" MODIFIED="1701698326482" TEXT="Bedeutung der Hash-Values">
<icon BUILTIN="hourglass"/>
<node CREATED="1701698328124" ID="ID_420862396" MODIFIED="1701698341086" TEXT="ein Zweck ist nat&#xfc;rlich: Hashtables"/>
<node CREATED="1701698342243" ID="ID_2662858" MODIFIED="1701698366532" TEXT="Invocation-Instance-ID &#x27fc; Cache-Keys"/>
</node>
</node>
</node>
<node CREATED="1667336713909" ID="ID_259014029" MODIFIED="1667336726184" TEXT="Framework">
<font NAME="SansSerif" SIZE="14"/>
@ -100825,10 +100889,29 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
</node>
<node COLOR="#338800" CREATED="1701661755565" ID="ID_91855144" MODIFIED="1701661758314" TEXT="RandomChainCalcFunctor">
<icon BUILTIN="button_ok"/>
<node CREATED="1701692351228" ID="ID_971527823" MODIFIED="1701692368970" TEXT="TPar: maxFan &#x27f6; Node-Layout"/>
<node CREATED="1701692370541" ID="ID_1427040653" MODIFIED="1701692389312" TEXT="Node-Array &#x27f9; raw pointer arrithmetics"/>
</node>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1701662028749" ID="ID_1982023018" MODIFIED="1701662069143" TEXT="RandomChainPlanFunctor">
<arrowlink COLOR="#546bbc" DESTINATION="ID_114421309" ENDARROW="Default" ENDINCLINATION="-34;-130;" ID="Arrow_ID_234763232" STARTARROW="None" STARTINCLINATION="-98;10;"/>
<icon BUILTIN="flag-yellow"/>
<node CREATED="1701692393563" ID="ID_1979090741" MODIFIED="1701702795082" TEXT="parametirisert mit 3 Funktoren">
<node CREATED="1701702805338" ID="ID_598071439" MODIFIED="1701702815217" TEXT="halte damit das Scheduler-API noch raus"/>
<node CREATED="1701702816068" ID="ID_1652334733" MODIFIED="1701702833077" TEXT="hoffe, so die Schleifen-Logik testen zu k&#xf6;nnen"/>
</node>
<node CREATED="1701692404354" ID="ID_410999909" MODIFIED="1701692419283" TEXT="damit rein f&#xfc;r die Schleifen-Logik zust&#xe4;ndig"/>
</node>
<node CREATED="1701693653847" ID="ID_735944505" MODIFIED="1701693663905" TEXT="gemeinsame Basisklasse">
<node CREATED="1701693665213" ID="ID_874266292" MODIFIED="1701693686411" TEXT="ChainFunctor (abstrakt)"/>
<node CREATED="1701693696854" ID="ID_1473365249" MODIFIED="1701693708309" TEXT="Handhabung der InvocationInstanceID"/>
<node CREATED="1701693708846" ID="ID_1233561389" MODIFIED="1701693713042" TEXT="Parameter encode/decode"/>
<node CREATED="1701693716219" ID="ID_935448923" MODIFIED="1701693724668" TEXT="gemeinsames Frame-Grid"/>
</node>
</node>
<node BACKGROUND_COLOR="#e0ceaa" COLOR="#690f14" CREATED="1701699491154" ID="ID_689678610" LINK="#ID_1690934543" MODIFIED="1701699590214" TEXT="wieder das Wirrwarr mit den Hash-Implementierungen &#xd83e;&#xdc46; #722">
<icon BUILTIN="messagebox_warning"/>
<node CREATED="1701699592274" ID="ID_1063618089" MODIFIED="1701699610879" TEXT="vorerst: verwende Boost!">
<icon BUILTIN="yes"/>
</node>
</node>
</node>
@ -100852,10 +100935,28 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
<icon BUILTIN="button_ok"/>
</node>
</node>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1701493075903" ID="ID_114421309" MODIFIED="1701662062918" TEXT="Meta-Job implementieren">
<node BACKGROUND_COLOR="#eef0c5" COLOR="#990000" CREATED="1701493075903" ID="ID_114421309" MODIFIED="1701702784104" TEXT="Meta-Job implementieren">
<linktarget COLOR="#546bbc" DESTINATION="ID_114421309" ENDARROW="Default" ENDINCLINATION="-34;-130;" ID="Arrow_ID_234763232" SOURCE="ID_1982023018" STARTARROW="None" STARTINCLINATION="-98;10;"/>
<icon BUILTIN="pencil"/>
<node COLOR="#338800" CREATED="1701692565258" ID="ID_706254089" MODIFIED="1701702780297" TEXT="nun doch die Operationen als &#x3bb;">
<icon BUILTIN="button_ok"/>
</node>
<node COLOR="#338800" CREATED="1701702725087" ID="ID_27314773" MODIFIED="1701702780298" TEXT="braucht ebenfalls Zugang zum Node-Array und damit den Node-Formfaktor(template-Param)">
<icon BUILTIN="button_ok"/>
</node>
<node COLOR="#338800" CREATED="1701702694691" ID="ID_425732900" MODIFIED="1701702780298" TEXT="Schleifenlogik: currIdx, maxCnt">
<icon BUILTIN="button_ok"/>
</node>
<node COLOR="#338800" CREATED="1701702751249" ID="ID_473909242" MODIFIED="1701702780299" TEXT="targetLevel wird per Argument vorgegeben">
<icon BUILTIN="button_ok"/>
</node>
<node COLOR="#338800" CREATED="1701702762388" ID="ID_1493951597" MODIFIED="1701702780299" TEXT="ruft eine Continuation auf und signalisiert in-bound">
<icon BUILTIN="button_ok"/>
</node>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1701702840464" ID="ID_211506547" MODIFIED="1701702848927" TEXT="reine Schleifen-Logik testen">
<icon BUILTIN="flag-yellow"/>
</node>
</node>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1701493107387" ID="ID_1826192324" MODIFIED="1701493117170" TEXT="Aufruf-front-End">
<icon BUILTIN="flag-yellow"/>
</node>