ichthyo/LUMIERA.clone
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Scheduler-test: extract search algo into lib

Browse source
This commit is contained in:
Fischlurch 2024-01-04 02:03:05 +01:00
parent e52aed0b3c
commit 032e4f6db5
3 changed files with 91 additions and 415 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

409
src/lib/binary-search.hpp
View file

@ -22,8 +22,19 @@
/** @file binary-search.hpp
** Textbook implementation of the classical binary search over continuous domain.
** The domain is given by its lower and upper end points. Within this domain,
** a _breaking point_ is located, where the result of a _probe predicate_
** flips from `false` to `true`. For the core search, the _invariant_
** is assumed, implying that the `predicate(lower) ≡ false` and
** `predicate(upper) ≡ true`.
**
** @see TestChainLoad_test
** For good convergence, it is advisable to enter the search with rather tight
** bounds. For the case that it's not clear if the invariant holds for both ends,
** two alternative entrance points are provided, which check the condition on the
** interval ends and possibly shift and expand the search domain in case the
** assumption is broken.
**
** @see stress-test-rig.hpp
** @see SchedulerStress_test
*/
@ -32,355 +43,77 @@
#define LIB_BINARY_SEARCH_H
#include "vault/common.hpp"
//#include "test-chain-load.hpp"
//#include "lib/test/transiently.hpp"
#include "vault/gear/scheduler.hpp"
#include "lib/time/timevalue.hpp"
//#include "lib/iter-explorer.hpp"
#include "lib/meta/function.hpp"
#include "lib/format-string.hpp"
#include "lib/format-cout.hpp"//////////////////////////TODO RLY?
//#include "lib/util.hpp"
//#include <functional>
#include <utility>
//#include <memory>
//#include <string>
#include <vector>
#include <tuple>
#include <array>
namespace lib {
using util::_Fmt;
using util::min;
using util::max;
// using util::isnil;
// using util::limited;
// using util::unConst;
// using util::toString;
// using util::isLimited;
// using lib::time::Time;
// using lib::time::TimeValue;
// using lib::time::FrameRate;
// using lib::time::Duration;
// using lib::test::Transiently;
// using lib::meta::_FunRet;
// using std::string;
// using std::function;
using std::make_pair;
using std::make_tuple;
// using std::forward;
// using std::string;
// using std::swap;
using std::vector;
using std::move;
using std::forward;
namespace err = lumiera::error; //////////////////////////TODO RLY?
namespace { // Default definitions ....
/** binary search: actual search loop
* - search until (upper-lower) < epsilon
* - the \a FUN performs the actual test
* - the goal is to narrow down the breaking point
* @note `fun(lower)` must be `false` and
* `fun(upper)` must be `true`
*/
template<class FUN, typename PAR>
inline auto
binarySearch_inner (FUN&& fun, PAR lower, PAR upper, PAR epsilon)
{
ASSERT_VALID_SIGNATURE (FUN, bool(PAR) );
REQUIRE (lower <= upper);
while ((upper-lower) >= epsilon)
{
PAR div = (lower+upper) / 2;
bool hit = fun(div);
if (hit)
upper = div;
else
lower = div;
}
return (lower+upper)/2;
}
namespace stress_test_rig {
template<class FUN, typename PAR>
inline auto
binarySearch_inner (FUN&& fun, PAR lower, PAR upper, PAR epsilon)
{
ASSERT_VALID_SIGNATURE (FUN, bool(PAR) );
REQUIRE (lower <= upper);
while ((upper-lower) >= epsilon)
{
PAR div = (lower+upper) / 2;
bool hit = fun(div);
if (hit)
upper = div;
else
lower = div;
}
return (lower+upper)/2;
}
template<class FUN, typename PAR>
inline auto
binarySearch_upper (FUN&& fun, PAR lower, PAR upper, PAR epsilon)
{
REQUIRE (lower <= upper);
bool hit = fun(upper);
if (not hit)
{// the upper end breaks contract => search above
PAR len = (upper-lower);
lower = upper - len/10;
upper = lower + 14*len/10;
}
return binarySearch_inner (forward<FUN> (fun), lower,upper,epsilon);
}
template<class FUN, typename PAR>
inline auto
binarySearch (FUN&& fun, PAR lower, PAR upper, PAR epsilon)
{
REQUIRE (lower <= upper);
bool hit = fun(lower);
if (hit)
{// the lower end breaks contract => search below
PAR len = (upper-lower);
upper = lower + len/10;
lower = upper - 14*len/10;
}
return binarySearch_upper (forward<FUN> (fun), lower,upper,epsilon);
}
/**
* Specific stress test scheme to determine the
* »breaking point« where the Scheduler starts to slip
*/
template<class CONF>
class BreakingPointBench
: CONF
{
using TestLoad = decltype(std::declval<CONF>().testLoad());
using TestSetup = decltype(std::declval<CONF>().testSetup (std::declval<TestLoad&>()));
struct Res
{
double stressFac{0};
double percentOff{0};
double stdDev{0};
double avgDelta{0};
double avgTime{0};
double expTime{0};
};
/** prepare the ScheduleCtx for a specifically parametrised test series */
void
configureTest (TestSetup& testSetup, double stressFac)
{
testSetup.withLoadTimeBase (CONF::LOAD_BASE)
.withAdaptedSchedule(stressFac, CONF::CONCURRENCY);
}
/** perform a repetition of test runs and compute statistics */
Res
runProbes (TestSetup& testSetup, double stressFac)
{
auto sqr = [](auto n){ return n*n; };
Res res;
auto& [sf,pf,sdev,avgD,avgT,expT] = res;
sf = stressFac;
expT = testSetup.getExpectedEndTime() / 1000;
std::array<double, CONF::REPETITIONS> runTime;
for (uint i=0; i<CONF::REPETITIONS; ++i)
{
runTime[i] = testSetup.launch_and_wait() / 1000;
avgT += runTime[i];
}
avgT /= CONF::REPETITIONS;
avgD = fabs (avgT-expT);
for (uint i=0; i<CONF::REPETITIONS; ++i)
{
sdev += sqr (runTime[i] - avgT);
double delta = fabs (runTime[i] - expT);
bool fail = (delta > CONF::FAIL_LIMIT);
if (fail)
++ pf;
showRun(i, delta, runTime[i], runTime[i] > avgT, fail);
}
pf /= CONF::REPETITIONS;
sdev = sqrt (sdev/CONF::REPETITIONS);
showStep(res);
return res;
}
/** criterion to decide if this test series constitutes a slipped schedule */
bool
decideBreakPoint (Res& res)
{
return res.percentOff > CONF::TRIGGER_FAIL
and res.stdDev > CONF::TRIGGER_SDEV
and res.avgDelta > CONF::TRIGGER_DELTA;
}
/**
* invoke a binary search to produce a sequence of test series
* with the goal to narrow down the stressFact where the Schedule slips away.
*/
template<class FUN>
Res
conductBinarySearch (FUN&& runTestCase, vector<Res> const& results)
{
double breakPoint = binarySearch_upper (forward<FUN> (runTestCase), 0.0, CONF::UPPER_STRESS, CONF::EPSILON);
uint s = results.size();
ENSURE (s >= 2);
Res res;
auto& [sf,pf,sdev,avgD,avgT,expT] = res;
// average data over the last three steps investigated for smoothing
uint points = min (results.size(), 3u);
for (uint i=results.size()-points; i<results.size(); ++i)
{
Res const& resx = results[i];
pf += resx.percentOff;
sdev += resx.stdDev;
avgD += resx.avgDelta;
avgT += resx.avgTime;
expT += resx.expTime;
}
pf /= points;
sdev /= points;
avgD /= points;
avgT /= points;
expT /= points;
sf = breakPoint;
return res;
}
_Fmt fmtRun_ {"....·%-2d: Δ=%4.1f t=%4.1f %s %s"}; // i % Δ % t % t>avg? % fail?
_Fmt fmtStep_{ "%4.2f| : ∅Δ=%4.1f±%-4.2f ∅t=%4.1f %s %%%3.1f -- expect:%4.1fms"}; // stress % ∅Δ % σ % ∅t % fail % pecentOff % t-expect
_Fmt fmtResSDv_{"%9s= %5.2f ±%4.2f%s"};
_Fmt fmtResVal_{"%9s: %5.2f%s"};
void
showRun(uint i, double delta, double t, bool over, bool fail)
{
if (CONF::showRuns)
cout << fmtRun_ % i % delta % t % (over? "+":"-") % (fail? "●":"○")
<< endl;
}
void
showStep(Res& res)
{
if (CONF::showStep)
cout << fmtStep_ % res.stressFac % res.avgDelta % res.stdDev % res.avgTime
% (decideBreakPoint(res)? "—◆—":"—◇—")
% res.percentOff % res.expTime
<< endl;
}
void
showRes(Res& res)
{
if (CONF::showRes)
{
cout << fmtResVal_ % "stresFac" % res.stressFac % "" <<endl;
cout << fmtResVal_ % "fail" %(res.percentOff * 100) % '%' <<endl;
cout << fmtResSDv_ % "delta" % res.avgDelta % res.stdDev % "ms"<<endl;
cout << fmtResVal_ % "runTime" % res.avgTime % "ms"<<endl;
cout << fmtResVal_ % "expected" % res.expTime % "ms"<<endl;
}
}
void
showRef(TestLoad& testLoad)
{
if (CONF::showRef)
cout << fmtResVal_ % "refTime"
% (testLoad.calcRuntimeReference(CONF::LOAD_BASE) /1000)
% "ms" << endl;
}
public:
/**
* Launch a measurement sequence to determine the »breaking point«
* for the configured test load and parametrisation of the Scheduler.
* @return a tuple `[stress-factor, ∅delta, ∅run-time]`
*/
auto
searchBreakingPoint()
{
TRANSIENTLY(work::Config::COMPUTATION_CAPACITY) = CONF::CONCURRENCY;
TestLoad testLoad = CONF::testLoad().buildTopology();
TestSetup testSetup = CONF::testSetup (testLoad);
vector<Res> observations;
auto performEvaluation = [&](double stressFac)
{
configureTest (testSetup, stressFac);
auto res = runProbes (testSetup, stressFac);
observations.push_back (res);
return decideBreakPoint(res);
};
Res res = conductBinarySearch (move(performEvaluation), observations);
showRes (res);
showRef (testLoad);
return make_tuple (res.stressFac, res.avgDelta, res.avgTime);
}
};
}//namespace stress_test_rig
/** entrance point to binary search to ensure the upper point
* indeed fulfils the test. If this is not the case, the search domain
* is shifted up, but also expanded so that the given upper point is
* still located within, but close to the lower end.
* @note `fun(lower)` must be `false`
*/
template<class FUN, typename PAR>
inline auto
binarySearch_upper (FUN&& fun, PAR lower, PAR upper, PAR epsilon)
{
REQUIRE (lower <= upper);
bool hit = fun(upper);
if (not hit)
{// the upper end breaks contract => search above
PAR len = (upper-lower);
lower = upper - len/10;
upper = lower + 14*len/10;
}
return binarySearch_inner (forward<FUN> (fun), lower,upper,epsilon);
}
/** configurable template for running Scheduler Stress tests */
class StressRig
: util::NonCopyable
{
public:
using usec = std::chrono::microseconds;
usec LOAD_BASE = 500us;
uint CONCURRENCY = work::Config::getDefaultComputationCapacity();
double EPSILON = 0.01; ///< error bound to abort binary search
double UPPER_STRESS = 0.6; ///< starting point for the upper limit, likely to fail
double FAIL_LIMIT = 2.0; ///< delta-limit when to count a run as failure
double TRIGGER_FAIL = 0.55; ///< %-fact: criterion-1 failures above this rate
double TRIGGER_SDEV = FAIL_LIMIT; ///< in ms : criterion-2 standard derivation
double TRIGGER_DELTA = 2*FAIL_LIMIT; ///< in ms : criterion-3 delta above this limit
bool showRuns = false; ///< print a line for each individual run
bool showStep = true; ///< print a line for each binary search step
bool showRes = true; ///< print result data
bool showRef = true; ///< calculate single threaded reference time
static uint constexpr REPETITIONS{20};
BlockFlowAlloc bFlow{};
EngineObserver watch{};
Scheduler scheduler{bFlow, watch};
/** Extension point: build the computation topology for this test */
auto
testLoad()
{
return TestChainLoad<>{64};
}
/** (optional) extension point: base configuration of the test ScheduleCtx */
template<class TL>
auto
testSetup (TL& testLoad)
{
return testLoad.setupSchedule(scheduler)
.withJobDeadline(100ms)
.withUpfrontPlanning();
}
/**
* Entrance Point: build a stress test measurement setup
* to determine the »breaking point« where the Scheduler is unable
* to keep up with the defined schedule.
* @tparam CONF specialised subclass of StressRig with customisation
* @return a builder to configure and then launch the actual test
*/
template<class CONF>
static auto
with()
{
return stress_test_rig::BreakingPointBench<CONF>{};
}
};
template<class FUN, typename PAR>
inline auto
binarySearch (FUN&& fun, PAR lower, PAR upper, PAR epsilon)
{
REQUIRE (lower <= upper);
bool hit = fun(lower);
if (hit)
{// the lower end breaks contract => search below
PAR len = (upper-lower);
upper = lower + len/10;
lower = upper - 14*len/10;
}
return binarySearch_upper (forward<FUN> (fun), lower,upper,epsilon);
}
} // namespace lib

58
tests/vault/gear/stress-test-rig.hpp
View file

@ -67,6 +67,7 @@
**
** @see TestChainLoad_test
** @see SchedulerStress_test
** @see binary-search.hpp
*/
@ -75,6 +76,7 @@
#include "vault/common.hpp"
#include "lib/binary-search.hpp"
//#include "test-chain-load.hpp"
//#include "lib/test/transiently.hpp"
@ -132,58 +134,6 @@ namespace test {
namespace stress_test_rig {
template<class FUN, typename PAR>
inline auto
binarySearch_inner (FUN&& fun, PAR lower, PAR upper, PAR epsilon)
{
ASSERT_VALID_SIGNATURE (FUN, bool(PAR) );
REQUIRE (lower <= upper);
while ((upper-lower) >= epsilon)
{
PAR div = (lower+upper) / 2;
bool hit = fun(div);
if (hit)
upper = div;
else
lower = div;
}
return (lower+upper)/2;
}
template<class FUN, typename PAR>
inline auto
binarySearch_upper (FUN&& fun, PAR lower, PAR upper, PAR epsilon)
{
REQUIRE (lower <= upper);
bool hit = fun(upper);
if (not hit)
{// the upper end breaks contract => search above
PAR len = (upper-lower);
lower = upper - len/10;
upper = lower + 14*len/10;
}
return binarySearch_inner (forward<FUN> (fun), lower,upper,epsilon);
}
template<class FUN, typename PAR>
inline auto
binarySearch (FUN&& fun, PAR lower, PAR upper, PAR epsilon)
{
REQUIRE (lower <= upper);
bool hit = fun(lower);
if (hit)
{// the lower end breaks contract => search below
PAR len = (upper-lower);
upper = lower + len/10;
lower = upper - 14*len/10;
}
return binarySearch_upper (forward<FUN> (fun), lower,upper,epsilon);
}
/**
* Specific stress test scheme to determine the
* »breaking point« where the Scheduler starts to slip
@ -262,7 +212,9 @@ namespace test {
Res
conductBinarySearch (FUN&& runTestCase, vector<Res> const& results)
{
double breakPoint = binarySearch_upper (forward<FUN> (runTestCase), 0.0, CONF::UPPER_STRESS, CONF::EPSILON);
double breakPoint = lib::binarySearch_upper (forward<FUN> (runTestCase)
, 0.0, CONF::UPPER_STRESS
, CONF::EPSILON);
uint s = results.size();
ENSURE (s >= 2);
Res res;

39
wiki/thinkPad.ichthyo.mm
View file

@ -107505,16 +107505,13 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
<node CREATED="1703895939371" ID="ID_1543713695" MODIFIED="1703895964277" TEXT="Invariante: steht hinter einem Gruppenwechsel"/>
<node CREATED="1703895844842" ID="ID_1108667761" MODIFIED="1703895937211" TEXT="man braucht zwingend eine zus&#xe4;tzliche State-Variable wegen Iterationsende">
<richcontent TYPE="NOTE"><html>
<head>
</head>
<head/>
<body>
<p>
weil die Invariante eben auch <i>erf&#252;llt</i>&#160;ist, wenn man hinter dem Ende steht; insofern ist die Auswertung ja tats&#228;chlich &#187;schleppend&#171;, d.h. man gibt immer die <i>vorhergehende</i>&#160;Gruppe aus
</p>
</body>
</html>
</richcontent>
</html></richcontent>
</node>
</node>
<node COLOR="#338800" CREATED="1703896048196" ID="ID_92615600" MODIFIED="1703898484617" TEXT="bestehende Funktionalit&#xe4;t damit reproduziert">
@ -107706,16 +107703,13 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
</node>
<node CREATED="1704143931324" ID="ID_577773022" MODIFIED="1704143962808">
<richcontent TYPE="NODE"><html>
<head>
</head>
<head/>
<body>
<p>
und withAdaptedSchedule <i>mu&#223; man explizit </i>aufrufen (wegen Stress-Faktor)
</p>
</body>
</html>
</richcontent>
</html></richcontent>
<icon BUILTIN="idea"/>
</node>
</node>
@ -107803,23 +107797,20 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
</node>
<node CREATED="1704214422581" ID="ID_1633300286" MODIFIED="1704214486089" TEXT="nahe am Kipp-Punkt erreicht die &#x2205;Laufzeit ein flaches Minimum">
<richcontent TYPE="NOTE"><html>
<head>
</head>
<head/>
<body>
<p>
das ist aber nicht klar genug ausgepr&#228;gt, um daraus ein Kriterium zu machen
</p>
</body>
</html>
</richcontent>
</html></richcontent>
</node>
<node CREATED="1704215339258" ID="ID_1504282419" MODIFIED="1704215358766" TEXT="diese Kriterien bew&#xe4;hren sich auch bei gr&#xf6;&#xdf;erer Statistik">
<icon BUILTIN="idea"/>
</node>
</node>
<node BACKGROUND_COLOR="#eef0c5" COLOR="#990000" CREATED="1704217480508" ID="ID_653759226" MODIFIED="1704237587427" TEXT="diese Methode in eine Test-Komponente &#xfc;berf&#xfc;hren">
<icon BUILTIN="pencil"/>
<node COLOR="#338800" CREATED="1704217480508" ID="ID_653759226" MODIFIED="1704331915490" TEXT="diese Methode in eine Test-Komponente &#xfc;berf&#xfc;hren">
<icon BUILTIN="button_ok"/>
<node CREATED="1704232960613" ID="ID_689181948" MODIFIED="1704232994486" STYLE="bubble" TEXT="&#xbb;StressTestRig&#xab;">
<font BOLD="true" NAME="SansSerif" SIZE="14"/>
</node>
@ -107833,8 +107824,8 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
<node COLOR="#338800" CREATED="1704243593362" ID="ID_1513298833" MODIFIED="1704316330671" TEXT="Statistik-Berechnung integrieren">
<icon BUILTIN="button_ok"/>
</node>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1704243607600" ID="ID_1968188887" MODIFIED="1704243618662" TEXT="Lib-Implementierung: bin&#xe4;re Suche">
<icon BUILTIN="flag-yellow"/>
<node COLOR="#338800" CREATED="1704243607600" ID="ID_1968188887" MODIFIED="1704331913127" TEXT="Lib-Implementierung: bin&#xe4;re Suche">
<icon BUILTIN="button_ok"/>
<node COLOR="#338800" CREATED="1704316333600" ID="ID_198671243" MODIFIED="1704322903614" TEXT="Basis-Algo">
<icon BUILTIN="button_ok"/>
<node CREATED="1704316348241" ID="ID_118215771" MODIFIED="1704316368237" TEXT="Parameter-Range [lower, upper]"/>
@ -107855,16 +107846,13 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
<node CREATED="1704317076874" ID="ID_699422154" MODIFIED="1704317104362" TEXT="Intervall teilen: m"/>
<node CREATED="1704317083038" ID="ID_1260760369" MODIFIED="1704317115617">
<richcontent TYPE="NODE"><html>
<head>
</head>
<head/>
<body>
<p>
Mittelpunkt <b>auswerten</b>: test(m)
</p>
</body>
</html>
</richcontent>
</html></richcontent>
<node CREATED="1704317129562" ID="ID_160740709" MODIFIED="1704317166218" TEXT="true &#x27fc; [lower, m]"/>
<node CREATED="1704317167008" ID="ID_227786504" MODIFIED="1704317178064" TEXT="false &#x27fc; [m, upper]"/>
</node>
@ -107896,6 +107884,9 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
</node>
</node>
</node>
<node COLOR="#435e98" CREATED="1704331920936" ID="ID_538109369" MODIFIED="1704331940521" TEXT="Lib-Implementierung &#xd83e;&#xdc32; binary-search.hpp">
<icon BUILTIN="info"/>
</node>
</node>
</node>
</node>