ichthyo/LUMIERA.clone
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Scheduler-test: build configurable measurement setup

Browse source 
Elaborate the draft to include all the elements used directly in the test case thus far;
the goal is to introduce some structuring and leave room for flexible confguration,
while implementing the actual binary search as library function over Lambdas.

My expectation is to write a series of individual test instances with varying parameters;
while it seems possible to add further performance test variations into that scheme later on.
This commit is contained in:
Fischlurch 2024-01-02 23:51:47 +01:00
parent 6f4bd150fd
commit e704f4aae0
6 changed files with 151 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
tests/vault/gear/scheduler-service-test.cpp
View file

@ -548,7 +548,7 @@ namespace test {
MARK_TEST_FUN
TestChainLoad<16> testLoad{64};
testLoad.configureShape_chain_loadBursts()
.buildToplolgy();
.buildTopology();
auto stats = testLoad.computeGraphStatistics();
cout << _Fmt{"Test-Load: Nodes: %d Levels: %d ∅Node/Level: %3.1f Forks: %d Joins: %d"}

6
tests/vault/gear/scheduler-stress-test.cpp
View file

@ -97,7 +97,7 @@ namespace test {
MARK_TEST_FUN
TestChainLoad testLoad{512};
testLoad.configureShape_chain_loadBursts()
.buildToplolgy()
.buildTopology()
// .printTopologyDOT()
;
@ -153,7 +153,7 @@ namespace test {
{
TestChainLoad testLoad{64};
testLoad.configureShape_chain_loadBursts()
.buildToplolgy()
.buildTopology()
// .printTopologyDOT()
// .printTopologyStatistics()
;
@ -293,7 +293,7 @@ SHOW_EXPR(micros);
MARK_TEST_FUN
TestChainLoad testLoad{64};
testLoad.configureShape_chain_loadBursts()
.buildToplolgy()
.buildTopology()
// .printTopologyDOT()
// .printTopologyStatistics()
;

116
tests/vault/gear/stress-test-rig.hpp
View file

@ -74,7 +74,7 @@
//#include "lib/util.hpp"
//#include <functional>
//#include <utility>
#include <utility>
//#include <memory>
//#include <string>
//#include <vector>
@ -108,7 +108,7 @@ namespace test {
// using std::forward;
// using std::string;
// using std::swap;
// using std::move;
using std::move;
namespace err = lumiera::error; //////////////////////////TODO RLY?
@ -118,21 +118,92 @@ namespace test {
namespace stress_test_rig {
/**
* Specific stress test scheme to determine the
* »breaking point« where the Scheduler starts to slip
*/
template<class CONF>
class BreakingPointBench
: CONF
{
using TestLoad = decltype(std::declval<CONF>().testLoad());
using TestSetup = decltype(std::declval<CONF>().testSetup (std::declval<TestLoad&>()));
struct Res
{
double stressFac{0};
double percentOff{0};
double stdDev{0};
double avgDelta{0};
double avgTime{0};
};
/** prepare the ScheduleCtx for a specifically parametrised test series */
void
configureTest (TestSetup& testSetup, double stressFac)
{
testSetup.withLoadTimeBase (CONF::LOAD_BASE)
.withAdaptedSchedule(stressFac, CONF::CONCURRENCY);
}
/** perform a repetition of test runs and compute statistics */
Res
runProbes (TestSetup& testSetup)
{
UNIMPLEMENTED ("test loop and statistics computation");
Res res{};
return res;
}
/** criterion to decide if this test series constitutes a slipped schedule */
bool
decideBreakPoint (Res& res)
{
return true; //////TODO booooo
}
/**
* invoke a binary search to produce a sequence of test series
* with the goal to narrow down the stressFact where the Schedule slips away.
*/
template<class FUN>
Res
conductBinarySearch (FUN&& runTestCase)
{
UNIMPLEMENTED ("invoke a library implementation of binary search");
}
public:
/**
* Launch a measurement sequence to determine the »breaking point«
* for the configured test load and parametrisation of the Scheduler.
* @return a tuple `[stress-factor, ∅delta, ∅run-time]`
*/
auto
searchBreakingPoint()
{
double finalStress{0};
double avgDelta{0};
double avgTime{0};
return make_tuple (finalStress, avgDelta, avgTime);
TRANSIENTLY(work::Config::COMPUTATION_CAPACITY) = CONF::CONCURRENCY;
TestLoad testLoad = CONF::testLoad().buildTopology();
TestSetup testSetup = CONF::testSetup (testLoad);
auto performEvaluation = [&](double stressFac)
{
configureTest (testSetup, stressFac);
auto res = runProbes (testSetup);
return make_tuple (decideBreakPoint(res), res);
};
Res res = conductBinarySearch (move (performEvaluation));
return make_tuple (res.stressFac, res.avgDelta, res.avgTime);
}
};
}
}//namespace stress_test_rig
/** configurable template for running Scheduler Stress tests */
class StressRig
: util::NonCopyable
{
@ -140,6 +211,37 @@ namespace test {
public:
using usec = std::chrono::microseconds;
usec LOAD_BASE = 500us;
uint CONCURRENCY = work::Config::getDefaultComputationCapacity();
BlockFlowAlloc bFlow{};
EngineObserver watch{};
Scheduler scheduler{bFlow, watch};
/** Extension point: build the computation topology for this test */
auto
testLoad()
{
return TestChainLoad<>{64};
}
/** (optional) extension point: base configuration of the test ScheduleCtx */
template<class TL>
auto
testSetup (TL& testLoad)
{
return testLoad.setupSchedule(scheduler)
.withJobDeadline(100ms)
.withUpfrontPlanning();
}
/**
* Entrance Point: build a stress test measurement setup
* to determine the »breaking point« where the Scheduler is unable
* to keep up with the defined schedule.
* @tparam CONF specialised subclass of StressRig with customisation
* @return a builder to configure and then launch the actual test
*/
template<class CONF>
static auto
with()

62
tests/vault/gear/test-chain-load-test.cpp
View file

@ -100,7 +100,7 @@ namespace test {
auto testLoad =
TestChainLoad{64}
.configureShape_short_segments3_interleaved()
.buildToplolgy();
.buildTopology();
// while building the graph, node hashes are computed
CHECK (testLoad.getHash() == 0xD2F292D864CF8086);
@ -210,7 +210,7 @@ namespace test {
verify_Topology()
{
auto graph = ChainLoad16{32}
.buildToplolgy();
.buildTopology();
CHECK (graph.topLevel() == 31);
CHECK (graph.getSeed() == 0);
@ -273,7 +273,7 @@ namespace test {
// moderate symmetrical expansion with 40% probability and maximal +2 links
graph.expansionRule(graph.rule().probability(0.4).maxVal(2))
.buildToplolgy()
.buildTopology()
// .printTopologyDOT()
// .printTopologyStatistics()
;
@ -292,7 +292,7 @@ namespace test {
// with additional re-shuffling, probability acts independent in each branch
// leading to more chances to draw a »fork«, leading to a faster expanding graph
graph.expansionRule(graph.rule().probability(0.4).maxVal(2).shuffle(23))
.buildToplolgy()
.buildTopology()
// .printTopologyDOT()
// .printTopologyStatistics()
;
@ -311,7 +311,7 @@ namespace test {
// while more constrained in width...
TestChainLoad<8> gra_2{256};
gra_2.expansionRule(gra_2.rule().probability(0.4).maxVal(2).shuffle(23))
.buildToplolgy()
.buildTopology()
// .printTopologyDOT()
// .printTopologyStatistics()
;
@ -343,7 +343,7 @@ namespace test {
// expand immediately at start and then gradually reduce / join chains
graph.expansionRule(graph.rule_atStart(8))
.reductionRule(graph.rule().probability(0.2).maxVal(3).shuffle(555))
.buildToplolgy()
.buildTopology()
// .printTopologyDOT()
// .printTopologyStatistics()
;
@ -360,7 +360,7 @@ namespace test {
// expansion and reduction can counterbalance each other
graph.expansionRule(graph.rule().probability(0.2).maxVal(3).shuffle(555))
.reductionRule(graph.rule().probability(0.2).maxVal(3).shuffle(555))
.buildToplolgy()
.buildTopology()
// .printTopologyDOT()
// .printTopologyStatistics()
;
@ -378,7 +378,7 @@ namespace test {
// expansion bursts can be balanced with a heightened reduction intensity
graph.expansionRule(graph.rule().probability(0.3).maxVal(4).shuffle(555))
.reductionRule(graph.rule().probability(0.9).maxVal(2).shuffle(555))
.buildToplolgy()
.buildTopology()
// .printTopologyDOT()
// .printTopologyStatistics()
;
@ -411,7 +411,7 @@ namespace test {
// randomly start new chains, to be carried-on linearly
graph.seedingRule(graph.rule().probability(0.2).maxVal(3).shuffle())
.buildToplolgy()
.buildTopology()
// .printTopologyDOT()
// .printTopologyStatistics()
;
@ -433,7 +433,7 @@ namespace test {
// with side-chaines successively joined into a single common result
graph.seedingRule(graph.rule().probability(0.2).maxVal(3).shuffle())
.reductionRule(graph.rule().probability(0.9).maxVal(2))
.buildToplolgy()
.buildTopology()
// .printTopologyDOT()
// .printTopologyStatistics()
;
@ -468,7 +468,7 @@ namespace test {
// terminate chains randomly
graph.pruningRule(graph.rule().probability(0.2))
.buildToplolgy()
.buildTopology()
// .printTopologyDOT()
// .printTopologyStatistics()
;
@ -488,7 +488,7 @@ namespace test {
graph.pruningRule(graph.rule().probability(0.2))
.expansionRule(graph.rule().probability(0.6))
.setSeed(10101)
.buildToplolgy()
.buildTopology()
// .printTopologyDOT()
// .printTopologyStatistics()
;
@ -513,7 +513,7 @@ namespace test {
graph.pruningRule(graph.rule().probability(0.2).shuffle(5))
.expansionRule(graph.rule().probability(0.6))
.setSeed(10101)
.buildToplolgy()
.buildTopology()
// .printTopologyDOT()
// .printTopologyStatistics()
;
@ -536,7 +536,7 @@ namespace test {
graph.seedingRule(graph.rule_atStart(1))
.pruningRule(graph.rule().probability(0.2))
.setSeed(10101)
.buildToplolgy()
.buildTopology()
// .printTopologyDOT()
// .printTopologyStatistics()
;
@ -566,7 +566,7 @@ namespace test {
.pruningRule(graph.rule().probability(0.5))
.reductionRule(graph.rule().probability(0.8).maxVal(4))
.setSeed(10101)
.buildToplolgy()
.buildTopology()
// .printTopologyDOT()
// .printTopologyStatistics()
;
@ -613,7 +613,7 @@ namespace test {
.pruningRule(graph.rule().probability(0.4))
.reductionRule(graph.rule().probability(0.6).maxVal(5).minVal(2))
.setSeed(23)
.buildToplolgy()
.buildTopology()
// .printTopologyDOT()
// .printTopologyStatistics()
;
@ -642,7 +642,7 @@ namespace test {
.pruningRule(graph.rule().probability(0.5))
.reductionRule(graph.rule().probability(0.6).maxVal(5).minVal(2))
.setSeed(23)
.buildToplolgy()
.buildTopology()
// .printTopologyDOT()
// .printTopologyStatistics()
;
@ -667,7 +667,7 @@ namespace test {
.reductionRule(graph.rule().probability(0.75).maxVal(3))
.pruningRule(graph.rule_atJoin(1))
.setSeed(47)
.buildToplolgy()
.buildTopology()
// .printTopologyDOT()
// .printTopologyStatistics()
;
@ -690,7 +690,7 @@ namespace test {
.reductionRule(graph.rule().probability(0.75).maxVal(3))
.pruningRule(graph.rule_atJoin(1))
.setSeed(0)
.buildToplolgy()
.buildTopology()
// .printTopologyDOT()
// .printTopologyStatistics()
;
@ -711,7 +711,7 @@ namespace test {
.reductionRule(graph.rule().probability(0.75).maxVal(3).shuffle())
.pruningRule(graph.rule_atJoin(1))
.setSeed(0)
.buildToplolgy()
.buildTopology()
// .printTopologyDOT()
// .printTopologyStatistics()
;
@ -736,7 +736,7 @@ namespace test {
.reductionRule(graph.rule().probability(0.75).maxVal(3))
.pruningRule(graph.rule().probability(0.55))
.setSeed(55) // ◁───────────────────────────────────────────── use 31 for width limited to 8 nodes
.buildToplolgy()
.buildTopology()
// .printTopologyDOT()
// .printTopologyStatistics()
;
@ -760,7 +760,7 @@ namespace test {
.reductionRule(graph.rule().probability(0.6).maxVal(5).minVal(2))
.pruningRule(graph.rule().probability(0.4))
.setSeed(42)
.buildToplolgy()
.buildTopology()
// .printTopologyDOT()
// .printTopologyStatistics()
;
@ -782,7 +782,7 @@ namespace test {
.reductionRule(graph.rule().probability(0.6).maxVal(5).minVal(2))
.pruningRule(graph.rule().probability(0.42))
.setSeed(23)
.buildToplolgy()
.buildTopology()
// .printTopologyDOT()
// .printTopologyStatistics()
;
@ -803,7 +803,7 @@ namespace test {
.reductionRule(graph.rule().probability(0.6).maxVal(5).minVal(2))
.pruningRule(graph.rule().probability(0.42))
.setSeed(23)
.buildToplolgy()
.buildTopology()
// .printTopologyDOT()
// .printTopologyStatistics()
;
@ -835,7 +835,7 @@ namespace test {
.seedingRule(graph.rule())
.pruningRule(graph.rule())
.setSeed(62)
.buildToplolgy()
.buildTopology()
// .printTopologyDOT()
// .printTopologyStatistics()
;
@ -937,7 +937,7 @@ namespace test {
graph.expansionRule(graph.rule().probability(0.8).maxVal(1))
.pruningRule(graph.rule().probability(0.6))
.weightRule((graph.rule().probability(0.5)))
.buildToplolgy();
.buildTopology();
CHECK (8 == graph.allNodes().filter(isStartNode).count());
CHECK (15 == graph.allNodes().filter(isExitNode).count());
@ -1039,19 +1039,19 @@ namespace test {
double t1 =
TestChainLoad{64}
.configureShape_short_segments3_interleaved()
.buildToplolgy()
.buildTopology()
.calcRuntimeReference();
double t2 =
TestChainLoad{64}
.configureShape_short_segments3_interleaved()
.buildToplolgy()
.buildTopology()
.calcRuntimeReference(1ms);
double t3 =
TestChainLoad{256}
.configureShape_short_segments3_interleaved()
.buildToplolgy()
.buildTopology()
.calcRuntimeReference();
auto isWithin10Percent = [](double t, double r)
@ -1071,7 +1071,7 @@ namespace test {
auto graph =
TestChainLoad{64}
.configureShape_short_segments3_interleaved()
.buildToplolgy();
.buildTopology();
CHECK (graph.getHash() == 0xD2F292D864CF8086);
@ -1102,7 +1102,7 @@ namespace test {
{
TestChainLoad testLoad{64};
testLoad.configureShape_chain_loadBursts()
.buildToplolgy()
.buildTopology()
// .printTopologyDOT()
;

2
tests/vault/gear/test-chain-load.hpp
View file

@ -571,7 +571,7 @@ namespace test {
* evaluating the _drawing rules_ on the current node, computing its hash.
*/
TestChainLoad&&
buildToplolgy()
buildTopology()
{
NodeTab a,b, // working data for generation
*curr{&a}, // the current set of nodes to carry on

6
wiki/thinkPad.ichthyo.mm
View file

@ -107830,6 +107830,12 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
<node CREATED="1704237555914" ID="ID_1779084715" MODIFIED="1704237568213" TEXT="funktionale Konfigurations-Hooks"/>
<node CREATED="1704237569280" ID="ID_227955379" MODIFIED="1704237574469" TEXT="Basis-Setup einmischen"/>
</node>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1704243593362" ID="ID_1513298833" MODIFIED="1704243606440" TEXT="Statistik-Berechnung integrieren">
<icon BUILTIN="flag-yellow"/>
</node>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1704243607600" ID="ID_1968188887" MODIFIED="1704243618662" TEXT="Lib-Implementierung: bin&#xe4;re Suche">
<icon BUILTIN="flag-yellow"/>
</node>
</node>
</node>
</node>