ichthyo/LUMIERA.clone
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Scheduler-test: extract a generic grouping iterator

Browse source 
...so IterExplorer got yet another processing layer,
which uses the grouping mechanics developed yesterday,
but is freely configurable through λ-Functions.

At actual usage sit in TestChainLoad, now only the actual
aggregation computation must be supplied, and follow-up computations
can now be chained up easily as further transformation layers.
This commit is contained in:
Fischlurch 2023-12-30 23:40:13 +01:00
parent fec117039e
commit 409a60238a
5 changed files with 266 additions and 77 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

146
src/lib/iter-explorer.hpp
View file

@ -117,6 +117,7 @@
#include "lib/util.hpp"
#include <functional>
#include <optional>
#include <utility>
@ -420,6 +421,7 @@ namespace lib {
using Sig = typename FunDetector<FUN>::Sig;
using Arg = typename _Fun<Sig>::Args::List::Head; // assuming function with a single argument
using Res = typename _Fun<Sig>::Ret;
static_assert (meta::is_UnaryFun<Sig>());
@ -971,6 +973,107 @@ namespace lib {
/**
* @internal Decorator for IterExplorer to group consecutive elements controlled by some
* grouping value \a GRP and compute an aggregate value \a AGG for each such group as
* iterator yield. The first group is consumed eagerly, each further group on iteration;
* thus when the aggregate for the last group appears as result, the source iterator has
* already been exhausted. The aggregate is default-initialised at start of each group
* and then the computation functor \a FAGG is invoked for each consecutive element marked
* with the same _grouping value_ — and this grouping value itself is obtained by invoking
* the functor \a FGRP on each source value. No capturing or even reordering of the source
* elements takes place, rather groups are formed based on the changes of the grouping value
* over the source iterator's result sequence.
* @tparam AGG data type to collect the aggregate; must be default constructible and assignable
* @tparam GRP value type to indicate a group
* @note while `groupFun` is adapted, the `aggFun` is _not adapted_ to the source iterator,
* but expected always to take the _value type_ of the preceding iterator, i.e. `*srcIter`.
* This limitation was deemed acceptable (adapting a function with several arguments would
* require quite some nasty technicalities). The first argument of this `aggFun` refers
* to the accumulator by value, and thereby also implititly defines the aggregate result type.
*/
template<class SRC, typename AGG, class GRP>
class GroupAggregator
: public SRC
{
static_assert(can_IterForEach<SRC>::value, "Lumiera Iterator required as source");
protected:
using SrcValue = typename meta::ValueTypeBinding<SRC>::value_type;
using Grouping = function<GRP(SRC&)>;
using Aggregator = function<void(AGG&, SrcValue&)>;
std::optional<AGG> agg_{};
Grouping grouping_;
Aggregator aggregate_;
public:
using value_type = typename meta::RefTraits<AGG>::Value;
using reference = typename meta::RefTraits<AGG>::Reference;
using pointer = typename meta::RefTraits<AGG>::Pointer;
GroupAggregator() =default;
// inherited default copy operations
template<class FGRP, class FAGG>
GroupAggregator (SRC&& dataSrc, FGRP&& groupFun, FAGG&& aggFun)
: SRC{move (dataSrc)}
, grouping_{_FunTraits<FGRP,SRC>::adaptFunctor (forward<FGRP> (groupFun))}
, aggregate_{forward<FAGG> (aggFun)}
{
pullGroup(); // initially pull to establish the invariant
}
public: /* === Iteration control API for IterableDecorator === */
bool
checkPoint() const
{
return bool(agg_);
}
reference
yield() const
{
return *unConst(this)->agg_;
}
void
iterNext()
{
if (srcIter())
pullGroup();
else
agg_ = std::nullopt;
}
protected:
SRC&
srcIter() const
{
return unConst(*this);
}
/** @note establishes the invariant:
* source has been consumed up to the beginning of next group */
void
pullGroup()
{
GRP group = grouping_(srcIter());
agg_ = AGG{};
do{
aggregate_(*agg_, *srcIter());
++ srcIter();
}
while (srcIter() and group == grouping_(srcIter()));
}
};
/**
* @internal Decorator for IterExplorer to filter elements based on a predicate.
* Similar to the Transformer, the given functor is adapted as appropriate. However,
@ -1581,7 +1684,7 @@ namespace lib {
}
/** adapt this IterExplorer group result elements into fixed size chunks, packaged as std::array.
/** adapt this IterExplorer to group result elements into fixed size chunks, packaged as std::array.
* The first group of elements is pulled eagerly at construction, while further groups are formed
* on consecutive iteration. Iteration ends when no further full group can be formed; this may
* leave out some leftover elements, which can then be retrieved by iteration through the
@ -1601,6 +1704,47 @@ namespace lib {
}
/** adapt this IterExplorer to group elements by a custom criterium and aggregate the group members.
* The first group of elements is pulled eagerly at construction, further groups are formed on each
* iteration. Aggregation is done by a custom functor, which takes an _aggregator value_ as first
* argument and the current element (or iterator) as second argument. Downstream, the aggregator
* value computed for each group is yielded on iteration.
* @param groupFun a functor to derive a grouping criterium from the source sequence; consecutive elements
* yielding the same grouping value will be combined / aggregated
* @param aggFun a functor to compute contribution to the aggregate value. Signature `void(AGG&, Val&)`,
* where the type AGG implicitly also defines the _value_ to use for accumulation and the result value,
* while Val must be assignable from the _source value_ provided by the preceding iterator in the pipeline.
*/
template<class FGRP, class FAGG>
auto
groupedBy (FGRP&& groupFun, FAGG&& aggFun)
{
using GroupVal = typename iter_explorer::_FunTraits<FGRP,SRC>::Res;
static_assert (meta::is_BinaryFun<FAGG>());
using ArgType1 = typename _Fun<FAGG>::Args::List::Head;
using Aggregate = typename meta::RefTraits<ArgType1>::Value;
using ResCore = iter_explorer::GroupAggregator<SRC, Aggregate, GroupVal>;
using ResIter = typename _DecoratorTraits<ResCore>::SrcIter;
return IterExplorer<ResIter> (ResCore {move(*this)
,forward<FGRP> (groupFun)
,forward<FAGG> (aggFun)});
}
/** simplified grouping to sum / combine all values in a group */
template<class FGRP>
auto
groupedBy (FGRP&& groupFun)
{
using Value = typename meta::ValueTypeBinding<SRC>::value_type;
return groupedBy (forward<FGRP> (groupFun)
,[](Value& agg, Value const& val){ agg += val; }
);
}
/** adapt this IterExplorer to iterate only as long as a condition holds true.
* @return processing pipeline with attached [stop condition](\ref iter_explorer::StopTrigger)
*/

44
tests/library/iter-explorer-test.cpp
View file

@ -67,6 +67,7 @@
#include <limits>
#include <string>
#include <tuple>
#include <cmath>
namespace lib {
@ -279,6 +280,7 @@ namespace test{
verify_expand_rootCurrent();
verify_transformOperation();
verify_elementGroupingOperation();
verify_aggregatingGroupItration();
verify_combinedExpandTransform();
verify_customProcessingLayer();
verify_scheduledExpansion();
@ -668,6 +670,7 @@ namespace test{
}
/** @test package elements from the source pipeline into fixed-sized groups.
* These groups are implemented as std::array and initialised with the values
* yielded consecutively from the underlying source pipeline. The main iterator
@ -729,6 +732,47 @@ namespace test{
}
/** @test another form of grouping, where groups are formed by a derived property
* thereby passing each element in the group to an aggregator function, working on
* an accumulator per group. Downstream, the resulting, accumulated value is exposed
* for each group, while consuming all source values belonging to this group.
* - in the simple form, all members of a group are "added" together
* - the elaborate form allows to provide a custom aggregation function, which takes
* the »accumulator« as first argument by reference; the type of this argument
* implicitly defines what is instantiated for each group and yielded as result.
*/
void
verify_aggregatingGroupItration()
{
CHECK (materialise (
explore(CountDown{10})
.groupedBy(std::ilogbf)
)
== "27-22-5-1"_expect); // 10+9+8|7+6+5+4|3+2|1
CHECK (materialise (
explore(CountDown{10})
.transform(util::toString<uint>)
.groupedBy([](auto& it) { return std::ilogbf (it.p); }) // note trickery: takes not the value, rather the iterator and
) // accesses internals of CountDown, bypassing the transform layer above
== "1098-7654-32-1"_expect); // `+` does string concatenation
auto showGroup = [](auto it){ return "["+util::join(*it)+"]"; };
// elaborate form with custom aggregation...
CHECK (materialise (
explore(CountDown{10})
.groupedBy(std::ilogbf
,[](vector<uint>& accum, uint val)
{
accum.push_back (val);
})
.transform(showGroup)
)
== "[10, 9, 8]-[7, 6, 5, 4]-[3, 2]-[1]"_expect);
}
/** @test combine the recursion into children with a tail mapping operation.
* Wile basically this is just the layering structure of IterExplorer put into action,
* you should note one specific twist: the iter_explorer::Expander::expandChildren() call

4
tests/vault/gear/scheduler-stress-test.cpp
View file

@ -161,7 +161,7 @@ SHOW_EXPR(micros);
CHECK (micros < 550);
CHECK (micros > 450);
std::vector<TestChainLoad<>::Agg> levelWeights = testLoad.allNodes().processingLayer<TestChainLoad<>::WeightAggregator>().effuse();
std::vector<LevelWeight> levelWeights = testLoad.allLevelWeights().effuse();
SHOW_EXPR(levelWeights.size())
for (auto& w : levelWeights)
{
@ -175,7 +175,7 @@ SHOW_EXPR(levelWeights.size())
.effuse();
for (auto& n : nodeData)
cout<<"level:"<<n.first<<" w="<<n.second<<endl;
auto getWeight = [&](uint level) { return std::get<2> (levelWeights[level]); };
auto getWeight = [&](uint level) { return levelWeights[level].weight; };
CHECK (nodeData[22].first == 16);
CHECK (nodeData[23].first == 17); CHECK (nodeData[23].second == 3); CHECK (getWeight(17) == 3);

79
tests/vault/gear/test-chain-load.hpp
View file

@ -170,6 +170,12 @@ namespace test {
const Duration SCHEDULE_PLAN_STEP{_uTicks(100us)}; ///< time budget to reserve for each node to be planned and scheduled
}
struct LevelWeight
{
size_t level{0};
size_t nodes{0};
size_t weight{0};
};
struct Statistic;
@ -748,74 +754,19 @@ namespace test {
}
using Agg = std::tuple<size_t,size_t,size_t>;
template<class IT>
class WeightAggregator
: public IT
{
std::optional<Agg> agg_{};
IT&
srcIter() const
{
return unConst(*this);
}
static void
account (Agg& agg, IT const& item)
{
auto& [level,nodes,weight] = agg;
auto& n = *item;
level = n.level;
weight += n.weight;
++nodes;
}
void
pullGroup()
{
size_t group = srcIter()->level;
agg_ = Agg{};
do{
account (*agg_, srcIter());
++ srcIter();
}
while (srcIter() and group == srcIter()->level);
}
public:
WeightAggregator() =default;
WeightAggregator (IT&& src)
: IT{move (src)}
{
if (srcIter())
pullGroup();
}
bool
checkPoint() const
{
return bool(agg_);
}
Agg&
yield() const
{
return *unConst(this)->agg_;
}
void
iterNext()
{
if (srcIter())
pullGroup();
else
agg_ = std::nullopt;
}
};
/** calculate node weights aggregated per level */
auto
allLevelWeights()
{
// return allNodes().processingLayer<WeightAggregator>();
return allNodes()
.groupedBy([](Node& n){ return n.level; }
,[](LevelWeight& lw, Node const& n)
{
lw.level = n.level;
lw.weight += n.weight;
++lw.nodes;
}
);
}

70
wiki/thinkPad.ichthyo.mm
View file

@ -107419,7 +107419,8 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
<node CREATED="1703797597576" ID="ID_496053162" MODIFIED="1703797638501" TEXT="liefert f&#xfc;r jeden Level: (levelNr, &#x3a3;w, &#x3a3;n)"/>
<node BACKGROUND_COLOR="#eef0c5" COLOR="#990000" CREATED="1703801391911" ID="ID_1010028454" MODIFIED="1703807940482" TEXT="Implementierung">
<icon BUILTIN="pencil"/>
<node COLOR="#5b280f" CREATED="1703801398285" ID="ID_1449254847" MODIFIED="1703804125235" TEXT="Iter-Explorer mit gruppierendem Processing-Layer">
<node COLOR="#5b280f" CREATED="1703801398285" FOLDED="true" ID="ID_1449254847" MODIFIED="1703979248329" TEXT="Iter-Explorer mit gruppierendem Processing-Layer">
<arrowlink COLOR="#5874c0" DESTINATION="ID_1740211471" ENDARROW="Default" ENDINCLINATION="-136;-9;" ID="Arrow_ID_443078033" STARTARROW="None" STARTINCLINATION="53;91;"/>
<linktarget COLOR="#f53565" DESTINATION="ID_1449254847" ENDARROW="Default" ENDINCLINATION="-19;118;" ID="Arrow_ID_669204155" SOURCE="ID_1512834256" STARTARROW="None" STARTINCLINATION="-120;-17;"/>
<icon BUILTIN="button_cancel"/>
<node CREATED="1703801415915" ID="ID_1892218840" MODIFIED="1703801420014" TEXT="Invariante">
@ -107489,7 +107490,8 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
</node>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1703882997602" ID="ID_1587426873" MODIFIED="1703883014897" TEXT="nachfolgend w&#xe4;re aber eine Pipeline doch nicht schlecht">
<icon BUILTIN="idea"/>
<node BACKGROUND_COLOR="#f0d5c5" COLOR="#990033" CREATED="1703883016888" ID="ID_1740211471" MODIFIED="1703895721626" TEXT="also einen zweiten Anlauf versuchen">
<node COLOR="#435e98" CREATED="1703883016888" ID="ID_1740211471" MODIFIED="1703979248329" TEXT="also einen zweiten Anlauf versuchen">
<linktarget COLOR="#5874c0" DESTINATION="ID_1740211471" ENDARROW="Default" ENDINCLINATION="-136;-9;" ID="Arrow_ID_443078033" SOURCE="ID_1449254847" STARTARROW="None" STARTINCLINATION="53;91;"/>
<icon BUILTIN="yes"/>
<node CREATED="1703895722656" ID="ID_527352372" MODIFIED="1703895731221" TEXT="sschwere Geburt">
<icon BUILTIN="smiley-angry"/>
@ -107518,16 +107520,64 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
<icon BUILTIN="button_ok"/>
</node>
</node>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1703895976686" ID="ID_233747473" MODIFIED="1703895999350" TEXT="dann diese L&#xf6;sung extrahieren und verallgemeinern">
<icon BUILTIN="flag-yellow"/>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1703896008082" ID="ID_77845528" MODIFIED="1703896027228" TEXT="erst mal in TestChainLoad mit Lambdas konfigurieren">
<icon BUILTIN="flag-yellow"/>
<node COLOR="#338800" CREATED="1703895976686" ID="ID_233747473" MODIFIED="1703979202514" TEXT="dann diese L&#xf6;sung extrahieren und verallgemeinern">
<icon BUILTIN="button_ok"/>
<node COLOR="#5b280f" CREATED="1703896008082" ID="ID_77845528" MODIFIED="1703974649167" TEXT="erst mal in TestChainLoad mit Lambdas konfigurieren">
<icon BUILTIN="button_cancel"/>
<node CREATED="1703974651311" ID="ID_1612095932" MODIFIED="1703974659788" TEXT="geht so nicht...."/>
<node CREATED="1703974660365" ID="ID_515087640" MODIFIED="1703974706210" TEXT="das Interface f&#xfc;r IterExplorer::processingLayer() ist festgelegt auf ein Un&#xe4;res Template"/>
<node CREATED="1703974710699" ID="ID_1069454746" MODIFIED="1703974722984" TEXT="daher sofort den Sprung auf einen generischen Aggregator machen"/>
</node>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1703896028383" ID="ID_373477842" MODIFIED="1703896035228" TEXT="dann in den IterExplorer &#xfc;bernehmen">
<icon BUILTIN="flag-yellow"/>
<node COLOR="#338800" CREATED="1703896028383" ID="ID_373477842" MODIFIED="1703979137992" TEXT="dann f&#xfc;r den IterExplorer verallgemeinern">
<icon BUILTIN="button_ok"/>
<node CREATED="1703974886502" ID="ID_1266426996" MODIFIED="1703974905655" TEXT="Adaptierbarkeit f&#xfc;r Src-Iterator-Argument"/>
<node BACKGROUND_COLOR="#e0ceaa" COLOR="#690f14" CREATED="1703974906379" FOLDED="true" ID="ID_1063698965" MODIFIED="1703979193326" TEXT="Problem: das Framework stellt auf Un&#xe4;re Funktionen ab">
<icon BUILTIN="messagebox_warning"/>
<icon BUILTIN="stop-sign"/>
<node CREATED="1703974924785" ID="ID_694048108" MODIFIED="1703974936830" TEXT="war bisher immer der Fall (Transformer, Filter,...)"/>
<node CREATED="1703974943335" ID="ID_1332119101" MODIFIED="1703974954521" TEXT="nun m&#xfc;&#xdf;te aber das 2.Argument einer bin&#xe4;ren Funktion adaptiert werden"/>
<node CREATED="1703974985226" ID="ID_1314783596" MODIFIED="1703974993803" TEXT="Curry bzw. partial application...">
<node CREATED="1703975001782" ID="ID_669651191" MODIFIED="1703975022711" TEXT="als Vorbereitung an den Akkumulator binden"/>
<node COLOR="#5b280f" CREATED="1703975058314" ID="ID_1623063408" MODIFIED="1703975149057" TEXT="Binden per Referenz widerspricht IterExplorer Nutzungs-Muster">
<icon BUILTIN="closed"/>
<node CREATED="1703975150947" ID="ID_1414201479" MODIFIED="1703975162108" TEXT="IterExploer wird als Builder aufgebaut"/>
<node CREATED="1703975162649" ID="ID_1096493459" MODIFIED="1703975169196" TEXT="und jeweils in das Ziel geschoben"/>
<node CREATED="1703975170072" ID="ID_981130825" MODIFIED="1703975193265" TEXT="&#x27f9; this &#xe4;ndert sich unterwegs"/>
</node>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1703896035966" ID="ID_707528864" MODIFIED="1703896039984" TEXT="Test daf&#xfc;r schreiben">
<icon BUILTIN="flag-yellow"/>
</node>
<node COLOR="#5b280f" CREATED="1703975817543" ID="ID_323717828" MODIFIED="1703976031049" TEXT="andere Auswege?">
<icon BUILTIN="help"/>
<icon BUILTIN="button_cancel"/>
<node CREATED="1703975869658" ID="ID_990729285" MODIFIED="1703975904370" TEXT="die _FunTraits::adaptFunctor k&#xf6;nnte man verallgemeinern...">
<node CREATED="1703975905557" ID="ID_93452190" MODIFIED="1703975907289" TEXT="zOMG"/>
<node CREATED="1703975907709" ID="ID_399985710" MODIFIED="1703975918872" TEXT="wer will sich das antun..!?"/>
</node>
<node CREATED="1703976194479" ID="ID_435449867" MODIFIED="1703976231726" TEXT="man k&#xf6;nnte die Funktor-Tools ausbauen, und compose auf beliebige Argumente erm&#xf6;glichen">
<node CREATED="1703976233813" ID="ID_783321602" MODIFIED="1703976246676" TEXT="hiermit g&#xe4;be es also den ersten Anwendungsfall"/>
<node CREATED="1703976247383" ID="ID_1631613099" MODIFIED="1703976255602" TEXT="...den im Moment noch niemand braucht"/>
<node CREATED="1703976256262" ID="ID_1431849596" MODIFIED="1703976263095" TEXT="gut da&#xdf; wir dar&#xfc;ber geredet haben">
<icon BUILTIN="ksmiletris"/>
</node>
</node>
<node CREATED="1703975970540" ID="ID_768475065" MODIFIED="1703975994429" TEXT="man k&#xf6;nnte die Anpassung hier &#x201e;zu fu&#xdf;&#x201c; nochmal programmieren"/>
</node>
<node BACKGROUND_COLOR="#ccb59b" COLOR="#6e2a38" CREATED="1703976043723" ID="ID_1128153890" MODIFIED="1703976061431" TEXT="also erst mal nicht anpassbar machen">
<font ITALIC="true" NAME="SansSerif" SIZE="14"/>
<icon BUILTIN="yes"/>
<node CREATED="1703976063609" ID="ID_1594454385" MODIFIED="1703976067252" TEXT="YAGNI"/>
<node CREATED="1703976067872" ID="ID_174383133" MODIFIED="1703976079554" TEXT="ist im Moment nur &#x201e;nice-to-have&#x201c;"/>
<node CREATED="1703976093596" ID="ID_1279220583" MODIFIED="1703976114709" TEXT="und ich werde wieder ein St&#xfc;ck komplexen Code los..."/>
</node>
</node>
<node COLOR="#338800" CREATED="1703979125116" ID="ID_381984711" MODIFIED="1703979137009" TEXT="im &#xfc;brigen... tut wie erwartet">
<icon BUILTIN="button_ok"/>
</node>
</node>
<node COLOR="#338800" CREATED="1703896035966" ID="ID_707528864" MODIFIED="1703979141744" TEXT="Test daf&#xfc;r schreiben">
<icon BUILTIN="button_ok"/>
<node COLOR="#435e98" CREATED="1703979142762" ID="ID_901668854" MODIFIED="1703979183660" TEXT="einfache Summation"/>
<node COLOR="#435e98" CREATED="1703979146985" ID="ID_1998259803" MODIFIED="1703979183661" TEXT="Summation mit String-Konkatenation und Durchgriff auf die Iterator-Core"/>
<node COLOR="#435e98" CREATED="1703979167940" ID="ID_117779918" MODIFIED="1703979183661" TEXT="komplexe Gruppierung mit Custom-Allokator (Beispiel: ein vector)"/>
</node>
</node>
</node>