From fec117039ece46cf7b163f4ec26e3fba6c2baa7b Mon Sep 17 00:00:00 2001
From: Ichthyostega <prg@ichthyostega.de>
Date: Fri, 29 Dec 2023 23:45:12 +0100
Subject: [PATCH] Scheduler-test: need this group aggregation as pipeline
 rather

Yesterday I've written a simple loop-based implementation of
a grouping aggregation to count the node weights per level.

Unfortunately it turns out we'll use several flavours of this
and we'd have to chain up postprocessing -- thus from a usage perspective
it would be better to have the same functionality packaged as interator pipeline.
This turns out to be surprisingly tricky and there is no suitable library
function available, which means I'll have to write one myself.

This changeset is the first step into this direction: reformulate
the simple for-loop into a demand-driven grouping iterator
---
 tests/vault/gear/scheduler-stress-test.cpp | 18 +++--
 tests/vault/gear/test-chain-load.hpp       | 77 ++++++++++++++++++----
 wiki/thinkPad.ichthyo.mm                   | 46 ++++++++++++-
 3 files changed, 120 insertions(+), 21 deletions(-)

diff --git a/tests/vault/gear/scheduler-stress-test.cpp b/tests/vault/gear/scheduler-stress-test.cpp
index e0faf6a09..00119394f 100644
--- a/tests/vault/gear/scheduler-stress-test.cpp
+++ b/tests/vault/gear/scheduler-stress-test.cpp
@@ -161,10 +161,13 @@ SHOW_EXPR(micros);
           CHECK (micros < 550);
           CHECK (micros > 450);
           
-          auto levelWeights = testLoad.allLevelWeights();
+          std::vector<TestChainLoad<>::Agg> levelWeights = testLoad.allNodes().processingLayer<TestChainLoad<>::WeightAggregator>().effuse();
 SHOW_EXPR(levelWeights.size())
           for (auto& w : levelWeights)
-            cout<<"level:"<<w.first<<" ∑="<<w.second<<endl;
+            {
+              auto& [level,nodes,weight] = w;
+              cout<<"level:"<<level<<" nodes:"<<nodes<<" ∑="<<weight<<endl;
+            }
           
           using Node = TestChainLoad<>::Node;
           auto nodeData = testLoad.allNodes()
@@ -172,27 +175,28 @@ SHOW_EXPR(levelWeights.size())
                                   .effuse();
           for (auto& n : nodeData)
             cout<<"level:"<<n.first<<" w="<<n.second<<endl;
+          auto getWeight = [&](uint level) { return std::get<2> (levelWeights[level]); };
           CHECK (nodeData[22].first == 16);
-          CHECK (nodeData[23].first == 17);  CHECK (nodeData[23].second == 3);  CHECK (levelWeights[17].second == 3);
+          CHECK (nodeData[23].first == 17);  CHECK (nodeData[23].second == 3);  CHECK (getWeight(17) == 3);
           
           CHECK (nodeData[24].first == 18);  CHECK (nodeData[24].second == 0);
           CHECK (nodeData[25].first == 18);  CHECK (nodeData[25].second == 0);
           CHECK (nodeData[26].first == 18);  CHECK (nodeData[26].second == 0);
           CHECK (nodeData[27].first == 18);  CHECK (nodeData[27].second == 0);
-          CHECK (nodeData[28].first == 18);  CHECK (nodeData[28].second == 0);  CHECK (levelWeights[18].second == 0);
+          CHECK (nodeData[28].first == 18);  CHECK (nodeData[28].second == 0);  CHECK (getWeight(18) == 0);
           
           CHECK (nodeData[29].first == 19);  CHECK (nodeData[29].second == 2);
           CHECK (nodeData[30].first == 19);  CHECK (nodeData[30].second == 2);
           CHECK (nodeData[31].first == 19);  CHECK (nodeData[31].second == 2);
           CHECK (nodeData[32].first == 19);  CHECK (nodeData[32].second == 2);
-          CHECK (nodeData[33].first == 19);  CHECK (nodeData[33].second == 2);  CHECK (levelWeights[19].second == 10);
+          CHECK (nodeData[33].first == 19);  CHECK (nodeData[33].second == 2);  CHECK (getWeight(19) == 10);
           
           CHECK (nodeData[34].first == 20);  CHECK (nodeData[34].second == 3);
-          CHECK (nodeData[35].first == 20);  CHECK (nodeData[35].second == 2);  CHECK (levelWeights[20].second == 5);
+          CHECK (nodeData[35].first == 20);  CHECK (nodeData[35].second == 2);  CHECK (getWeight(20) == 5);
           
           CHECK (nodeData[36].first == 21);  CHECK (nodeData[36].second == 1);
           CHECK (nodeData[37].first == 21);  CHECK (nodeData[37].second == 1);
-          CHECK (nodeData[38].first == 21);  CHECK (nodeData[38].second == 3);  CHECK (levelWeights[21].second == 5);
+          CHECK (nodeData[38].first == 21);  CHECK (nodeData[38].second == 3);  CHECK (getWeight(21) == 5);
         }
       
       
diff --git a/tests/vault/gear/test-chain-load.hpp b/tests/vault/gear/test-chain-load.hpp
index be90c638a..c43abf05e 100644
--- a/tests/vault/gear/test-chain-load.hpp
+++ b/tests/vault/gear/test-chain-load.hpp
@@ -748,23 +748,74 @@ namespace test {
         }
       
       
+      using Agg = std::tuple<size_t,size_t,size_t>;
+      
+      template<class IT>
+      class WeightAggregator
+        : public IT
+        {
+          std::optional<Agg> agg_{};
+          
+          IT&
+          srcIter()  const
+            {
+              return unConst(*this);
+            }
+          
+          static void
+          account (Agg& agg, IT const& item)
+            {
+              auto& [level,nodes,weight] = agg;
+              auto& n = *item;
+              level = n.level;
+              weight += n.weight;
+              ++nodes;
+            }
+          void
+          pullGroup()
+            {
+              size_t group = srcIter()->level;
+              agg_ = Agg{};
+              do{
+                  account (*agg_, srcIter());
+                  ++ srcIter();
+                }
+              while (srcIter() and group == srcIter()->level);
+            }
+        public:
+          WeightAggregator() =default;
+          WeightAggregator (IT&& src)
+            : IT{move (src)}
+            {
+              if (srcIter())
+                pullGroup();
+            }
+          bool
+          checkPoint()  const
+            {
+              return bool(agg_);
+            }
+          
+          Agg&
+          yield()  const
+            {
+              return *unConst(this)->agg_;
+            }
+          
+          void
+          iterNext()
+            {
+              if (srcIter())
+                pullGroup();
+              else
+                agg_ = std::nullopt;
+            }
+        };
       /** calculate node weights aggregated per level */
       auto
       allLevelWeights()
         {
-          std::vector<std::pair<size_t,size_t>> weightTab;
-          weightTab.reserve (topLevel()+1);
-          auto currLevel = [&]()-> decltype(auto) { return weightTab.back();           };
-          auto nextLevel = [&](size_t levelID)    { weightTab.emplace_back(levelID,0); };
-          nextLevel (0);
-          for (Node& n : allNodes())
-            {
-              if (n.level > currLevel().first)
-                nextLevel(n.level);
-               // aggregate weights within level
-              currLevel().second += n.weight;
-            }
-          return weightTab;
+//          return allNodes().processingLayer<WeightAggregator>();
         }
       
       
diff --git a/wiki/thinkPad.ichthyo.mm b/wiki/thinkPad.ichthyo.mm
index 48b709a25..2aad2cb4d 100644
--- a/wiki/thinkPad.ichthyo.mm
+++ b/wiki/thinkPad.ichthyo.mm
@@ -107404,8 +107404,8 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
 <icon BUILTIN="button_ok"/>
 </node>
 <node BACKGROUND_COLOR="#eef0c5" COLOR="#990000" CREATED="1702944889122" ID="ID_816186443" MODIFIED="1703807949690" TEXT="Berechnung der systematisch-erwartbaren Ausf&#xfc;hrungszeit">
-<linktarget COLOR="#5d567f" DESTINATION="ID_816186443" ENDARROW="Default" ENDINCLINATION="-25;77;" ID="Arrow_ID_280715198" SOURCE="ID_1983964457" STARTARROW="None" STARTINCLINATION="-621;-17;"/>
 <linktarget COLOR="#618eb4" DESTINATION="ID_816186443" ENDARROW="Default" ENDINCLINATION="-871;-40;" ID="Arrow_ID_888227312" SOURCE="ID_1434946452" STARTARROW="None" STARTINCLINATION="-203;20;"/>
+<linktarget COLOR="#5d567f" DESTINATION="ID_816186443" ENDARROW="Default" ENDINCLINATION="-25;77;" ID="Arrow_ID_280715198" SOURCE="ID_1983964457" STARTARROW="None" STARTINCLINATION="-621;-17;"/>
 <icon BUILTIN="pencil"/>
 <node CREATED="1702944906320" ID="ID_1870513553" MODIFIED="1702944934194" TEXT="per Chain-Load Graph-processing">
 <icon BUILTIN="info"/>
@@ -107487,6 +107487,50 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
 <node CREATED="1703806119013" ID="ID_848627487" MODIFIED="1703806130911" TEXT="For-Schleife mit schleppendem Aggregat"/>
 <node CREATED="1703807871832" ID="ID_414635297" MODIFIED="1703807880606" TEXT="Gewichte pro Level aufsummieren"/>
 </node>
+<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1703882997602" ID="ID_1587426873" MODIFIED="1703883014897" TEXT="nachfolgend w&#xe4;re aber eine Pipeline doch nicht schlecht">
+<icon BUILTIN="idea"/>
+<node BACKGROUND_COLOR="#f0d5c5" COLOR="#990033" CREATED="1703883016888" ID="ID_1740211471" MODIFIED="1703895721626" TEXT="also einen zweiten Anlauf versuchen">
+<icon BUILTIN="yes"/>
+<node CREATED="1703895722656" ID="ID_527352372" MODIFIED="1703895731221" TEXT="sschwere Geburt">
+<icon BUILTIN="smiley-angry"/>
+<node CREATED="1703895737462" ID="ID_998389333" MODIFIED="1703895770446" TEXT="ich bin an der Idee &#xbb;schleppende Iteration&#xab; h&#xe4;ngen geblieben"/>
+<node CREATED="1703895773857" ID="ID_91740622" MODIFIED="1703895796842" TEXT="dem entsprechend wolle ich das &#xbb;pull&#xab; immerfort lazy / on demand machen"/>
+</node>
+<node CREATED="1703895798194" ID="ID_21629149" MODIFIED="1703895969078" TEXT="entscheidende Einsichten">
+<icon BUILTIN="idea"/>
+<node CREATED="1703895835761" ID="ID_1930618506" MODIFIED="1703895838540" TEXT="man mu&#xdf; eine Gruppe voraus pullen"/>
+<node CREATED="1703895939371" ID="ID_1543713695" MODIFIED="1703895964277" TEXT="Invariante: steht hinter einem Gruppenwechsel"/>
+<node CREATED="1703895844842" ID="ID_1108667761" MODIFIED="1703895937211" TEXT="man braucht zwingend eine zus&#xe4;tzliche State-Variable wegen Iterationsende">
+<richcontent TYPE="NOTE"><html>
+  <head>
+    
+  </head>
+  <body>
+    <p>
+      weil die Invariante eben auch <i>erf&#252;llt</i>&#160;ist, wenn man hinter dem Ende steht; insofern ist die Auswertung ja tats&#228;chlich &#187;schleppend&#171;, d.h. man gibt immer die <i>vorhergehende</i>&#160;Gruppe aus
+    </p>
+  </body>
+</html>
+</richcontent>
+</node>
+</node>
+<node COLOR="#338800" CREATED="1703896048196" ID="ID_92615600" MODIFIED="1703898484617" TEXT="bestehende Funktionalit&#xe4;t damit reproduziert">
+<icon BUILTIN="button_ok"/>
+</node>
+</node>
+<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1703895976686" ID="ID_233747473" MODIFIED="1703895999350" TEXT="dann diese L&#xf6;sung extrahieren und verallgemeinern">
+<icon BUILTIN="flag-yellow"/>
+<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1703896008082" ID="ID_77845528" MODIFIED="1703896027228" TEXT="erst mal in TestChainLoad mit Lambdas konfigurieren">
+<icon BUILTIN="flag-yellow"/>
+</node>
+<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1703896028383" ID="ID_373477842" MODIFIED="1703896035228" TEXT="dann in den IterExplorer &#xfc;bernehmen">
+<icon BUILTIN="flag-yellow"/>
+</node>
+<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1703896035966" ID="ID_707528864" MODIFIED="1703896039984" TEXT="Test daf&#xfc;r schreiben">
+<icon BUILTIN="flag-yellow"/>
+</node>
+</node>
+</node>
 </node>
 </node>
 <node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1702944922179" ID="ID_424913055" MODIFIED="1702944931701" TEXT="Fall2: concurrent-X">