From 8e33194882fcb47d16dfa865630c876f4d872a34 Mon Sep 17 00:00:00 2001 From: Ichthyostega Date: Sun, 7 Apr 2024 23:52:56 +0200 Subject: [PATCH] Scheduler-test: settle definition of specific test setup and data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After a lot of further tinkering, seemingly arriving at a somewhat satisfactory solution for the layout and arrangement of test definitions and especially the table for measurement series. While the complete setup remains fragile indeed, and complexity is more hidden than reduced — the pragmatic compromise established yesterday at least allows to reduce the amount of boilerplate in the test or measurement setup to make the actual specifics stand out clearly. ---- As an aside, the usage of the `DataFile` type imported from Yoshimi-test recently was re-shaped more towards a generic handling of tabular data with CSV storage option; thus renaming the type now into `DataTable`. Persistent storage is now just one option, while another usage pattern compounds observation data into table rows, which are then directly rendered into a CSV string, e.g. for visualisation as Gnuplot graph. --- src/lib/stat/csv.hpp | 2 +- src/lib/stat/data.hpp | 14 +- tests/library/stat/data-csv-test.cpp | 4 +- tests/vault/gear/scheduler-stress-test.cpp | 17 +- tests/vault/gear/stress-test-rig.hpp | 220 ++++++++++++++------- wiki/thinkPad.ichthyo.mm | 129 +++++++++++- 6 files changed, 278 insertions(+), 108 deletions(-) diff --git a/src/lib/stat/csv.hpp b/src/lib/stat/csv.hpp index bc6ef477b..770ddd15e 100644 --- a/src/lib/stat/csv.hpp +++ b/src/lib/stat/csv.hpp @@ -39,7 +39,7 @@ ** - only quoted fields may contain whitespace or comma ** - no escaping of quotes, i.e. no quotes within quotes ** [RFC 4180]: https://datatracker.ietf.org/doc/html/rfc4180 - ** @see lib::stat::DataFile + ** @see lib::stat::DataTable ** */ diff --git a/src/lib/stat/data.hpp b/src/lib/stat/data.hpp index c00254bb0..29b24e2a1 100644 --- a/src/lib/stat/data.hpp +++ b/src/lib/stat/data.hpp @@ -47,7 +47,7 @@ ** the most recent row of data can be accessed directly through these sub-components. ** ** # Usage - ** Create an actual instantiation of the DataFile template, passing a structure + ** Create an actual instantiation of the DataTable template, passing a structure ** with util::Column descriptors. You may then directly access the values of the ** _actual column_ or save/load from a persistent CSV file. ** @note mandatory to define a method `allColumns()` @@ -62,7 +62,7 @@ ** auto allColumns(){ return std::tie(name,n,x,y); } ** }; ** - ** using Dataz = lib::stat::DataFile; + ** using Dataz = lib::stat::DataTable; ** ** Dataz daz("filename.csv"); ** @@ -74,7 +74,7 @@ ** \par Variations ** The standard case is to have a table backed by persistent file storage, ** which can be initially empty. Under some conditions, especially for tests - ** - the DataFile can be created without filename + ** - the DataTable can be created without filename ** - it can be created from a CSVData, which is a `std::vector` of CSV-strings ** - it can be [rendered into CSV strings](\ref #renderCSV) ** - a (new) storage file name can be [given later](\ref saveAs) @@ -122,7 +122,7 @@ namespace stat{ /** - * Descriptor and Accessor for a data column within a DataFile table. + * Descriptor and Accessor for a data column within a DataTable table. * @tparam VAL type of values contained within this column; * this type must be _default constructible_ and _copyable._ */ @@ -191,20 +191,20 @@ namespace stat{ * within the table and persistent CSV storage. */ template - class DataFile + class DataTable : public TAB , util::MoveOnly { fs::path filename_; public: - DataFile(fs::path csvFile ="") + DataTable(fs::path csvFile ="") : filename_{fs::consolidated (csvFile)} { loadData(); } - DataFile (CSVData const& csv) + DataTable (CSVData const& csv) : filename_{} { appendFrom (csv); diff --git a/tests/library/stat/data-csv-test.cpp b/tests/library/stat/data-csv-test.cpp index 1e192686b..eb869c3d4 100644 --- a/tests/library/stat/data-csv-test.cpp +++ b/tests/library/stat/data-csv-test.cpp @@ -68,7 +68,7 @@ namespace test{ } }; - using TestTab = DataFile; + using TestTab = DataTable; }//(End)Test setup @@ -221,7 +221,7 @@ namespace test{ } - /** @test validate the simple CSV conversion functions used by DataFile */ + /** @test validate the simple CSV conversion functions used by DataTable */ void verify_CSV_Format() { diff --git a/tests/vault/gear/scheduler-stress-test.cpp b/tests/vault/gear/scheduler-stress-test.cpp index 48c6af04a..5789d6f67 100644 --- a/tests/vault/gear/scheduler-stress-test.cpp +++ b/tests/vault/gear/scheduler-stress-test.cpp @@ -397,13 +397,11 @@ namespace test { .printTopologyDOT() .printTopologyStatistics(); - struct Setup : StressRig + struct Setup + : StressRig, bench::LoadPeak_ParamRange_Evaluation { uint CONCURRENCY = 4; - using Param = size_t; - using Table = bench::DataTable; - auto testLoad(Param nodes) { TestLoad testLoad{nodes}; @@ -415,17 +413,6 @@ namespace test { return StressRig::testSetup(testLoad) .withLoadTimeBase(500us); } - - void - collectResult(Table& data, Param param, double millis, bench::IncidenceStat const& stat) - { - data.newRow(); - data.param = param; - data.time = stat.coveredTime / 1000; - data.conc = stat.avgConcurrency; - data.jobtime = stat.activeTime/stat.activationCnt; - data.overhead = stat.timeAtConc(1) / stat.activationCnt; ////OOO not really clear if sensible - } }; auto results = StressRig::with() diff --git a/tests/vault/gear/stress-test-rig.hpp b/tests/vault/gear/stress-test-rig.hpp index 599205692..fc76fc6da 100644 --- a/tests/vault/gear/stress-test-rig.hpp +++ b/tests/vault/gear/stress-test-rig.hpp @@ -45,15 +45,68 @@ ** indicated by an increasing variance of the overall runtime, and a departure from ** the nominal runtime of the executed schedule. ** + ** Another, complimentary observation method is to inject a defined and homogeneous + ** load peak into the scheduler and then watch the time it takes to process, the + ** processing overhead and achieved degree of concurrency. The actual observation + ** using this measurement setup attempts to establish a single _control parameter_ + ** as free variable, allowing to look for correlations and to build a linear + ** regression model to characterise a supposed functional dependency. Simply put, + ** given a number of fixed sizes jobs (not further correlated) as input, this + ** approach yields a »number of jobs per time unit« and »socked overhead« — + ** thereby distilling a _behaviour model_ to describe the actual stochastic data. + ** ** ## Setup ** To perform this test scheme, an operational Scheduler is required, and an instance ** of the TestChainLoad must be provided, configured with desired load properties. - ** The _stressFactor_ of the corresponding generated schedule will be the active parameter - ** of this test, performing a binary search for the _breaking point._ The Measurement - ** attempts to narrow down to the point of massive failure, when the ability to somehow - ** cope with the schedule completely break down. Based on watching the Scheduler in - ** operation, the detection was linked to three conditions, which typically will - ** be triggered together, and within a narrow and reproducible parameter range: + ** Moreover, the actual measurement setup requires to perform several test executions, + ** controlling some parameters in accordance to the observation scheme. The control + ** parameters and the specifics of the actual setup should be clearly visible, while + ** hiding the complexities of measurement execution. + ** + ** This can be achieved by a »Toolbench«, which is a framework with building blocks, + ** providing a pre-arranged _measurement rig_ for the various kinds of measurement setup. + ** The implementation code is arranged as a »sandwich« structure... + ** - StressTestRig, which is also the framework class, acts as _bottom layer_ to + ** provide an anchor point, some common definitions implying an invocation scheme + ** ** first a TestChainLoad topology is constructed, based on test parameters + ** ** this is used to create a TestChainLoad::SchedulerCtx, which is then + ** outfitted specifically for each test run + ** - the _middle layer_ is a custom `Setup` class, which inherits from the bottom + ** layer and fills in the actual topology and configuration for the desired test + ** - the test performance is then initiated by layering a specific _test tool_ on + ** top of the compound, which in turn picks up the parametrisation from the Setup + ** and base configuration, visible as base class (template param) \a CONF + ** Together, this leads to the following code scheme, which aims to simplify experimentation: + ** \code + ** using StressRig = StressTestRig<16>; + ** + ** struct Setup : StressRig + ** { + ** uint CONCURRENCY = 4; + ** //// more definitions + ** + ** auto testLoad() + ** {....define a Test-Chain-Load topology....} + ** + ** auto testSetup (TestLoad& testLoad) + ** { return StressRig::testSetup(testLoad) + ** .withLoadTimeBase(500us) + ** // ....more customisation here + ** } + ** }; + ** + ** auto result = StressRig::with() + ** .perform(); + ** \endcode + ** + ** ## Breaking Point search + ** The bench::BreakingPoint tool typically uses a complex interwoven job plan, which is + ** tightened until the timing breaks. The _stressFactor_ of the generated schedule will be + ** the active parameter of this test, performing a _binary search_ for the _breaking point._ + ** The Measurement attempts to narrow down to the point of massive failure, when the ability + ** to somehow cope with the schedule completely break down. Based on watching the Scheduler + ** in operation, the detection was linked to three conditions, which typically will be + ** triggered together, and within a narrow and reproducible parameter range: ** - an individual run counts as _accidentally failed_ when the execution slips ** away by more than 2ms with respect to the defined overall schedule. When more ** than 55% of all observed runs are considered as failed, the first condition is met @@ -63,18 +116,24 @@ ** - the third condition is that the ''averaged delta'' has surpassed 4ms, ** which is 2 times the basic failure indicator. ** - ** ## Observation tools - ** As a complement to the bench::BreakingPoint tool, another tool is provided to - ** run a specific Scheduler setup while varying a single control parameter within - ** defined limits. This produces a set of (x,y) data, which can be used to search - ** for correlations or build a linear regression model to describe the Scheduler's - ** behaviour as function of the control parameter. The typical use case would be - ** to use the input length (number of Jobs) as control parameter, leading to a - ** model for the Scheduler's expense. + ** ## Parameter Correlation + ** As a complement, the bench::ParameterRange tool is provided to run a specific Scheduler setup + ** while varying a single control parameter within defined limits. This produces a set of (x,y) data, + ** which can be used to search for correlations or build a linear regression model to describe the + ** Scheduler's behaviour as function of the control parameter. The typical use case would be to use + ** the input length (number of Jobs) as control parameter, leading to a model for Scheduling expense. ** + ** ## Observation tools + ** The TestChainLoad, together with its helpers and framework, already offers some tools to visualise + ** the generated topology and to calculate statistics, and to watch an performance with instrumentation. + ** In addition, the individual tools provide some debugging output to watch the measurement scheme. + ** Result data is either a tuple of values (in case of bench::BreakingPoint), or a table of result + ** data as function of the control parameter (for bench::ParameterRange). Result data, when converted + ** to CSV, can be visualised as Gnuplot diagram. ** @see TestChainLoad_test ** @see SchedulerStress_test ** @see binary-search.hpp + ** @see gnuplot-gen.hpp */ @@ -109,40 +168,18 @@ namespace vault{ namespace gear { namespace test { - using util::_Fmt; - using util::min; - using util::max; -// using util::isnil; -// using util::limited; -// using util::unConst; -// using util::toString; -// using util::isLimited; -// using lib::time::Time; -// using lib::time::TimeValue; -// using lib::time::FrameRate; -// using lib::time::Duration; -// using lib::test::Transiently; -// using lib::meta::_FunRet; - -// using std::string; -// using std::function; - using std::make_pair; using std::make_tuple; -// using std::forward; -// using std::string; -// using std::swap; - using std::vector; - using std::move; - - namespace err = lumiera::error; //////////////////////////TODO RLY? - - namespace { // Default definitions .... - - } + using std::forward; - - /** configurable template framework for running Scheduler Stress tests */ + /** + * Configurable template framework for running Scheduler Stress tests + * Use to build a custom setup class, which is then [injected](\ref StressTestRig::with) + * to [perform](\ref StressTestRig::Launcher::perform) a _specific measurement tool._ + * Several tools and detailed customisations are available in `namespace bench` + * - bench::BreakingPoint conducts a binary search to _break a schedule_ + * - bench::ParameterRange performs a randomised series of parametrised test runs + */ template class StressTestRig : util::NonCopyable @@ -226,6 +263,10 @@ namespace test { namespace bench { ///< Specialised tools to investigate scheduler performance + using util::_Fmt; + using util::min; + using util::max; + using std::vector; using std::declval; @@ -417,33 +458,6 @@ namespace test { - using lib::stat::Column; - using lib::stat::DataFile; - using lib::stat::CSVData; - using IncidenceStat = lib::IncidenceCount::Statistic; - - template - struct DataRow - { - Column param {"test param"}; // independent variable / control parameter - Column time {"result time"}; - Column conc {"concurrency"}; - Column jobtime {"avg jobtime"}; - Column overhead{"overhead"}; - - auto allColumns() - { return std::tie(param - ,time - ,conc - ,jobtime - ,overhead - ); - } - }; - - template - using DataTable = DataFile>; - /**************************************************//** @@ -455,12 +469,13 @@ namespace test { class ParameterRange : public CONF { - using Table = typename CONF::Table; - using Param = typename CONF::Param; - using TestLoad = typename CONF::TestLoad; using TestSetup = typename TestLoad::ScheduleCtx; + // Type binding for data evaluation + using Param = typename CONF::Param; + using Table = typename CONF::Table; + void runTest (Param param, Table& data) @@ -507,6 +522,61 @@ namespace test { return results; } }; + + + + /* ====== Preconfigured ParamRange-Evaluations ====== */ + + using lib::stat::Column; + using lib::stat::DataTable; + using lib::stat::CSVData; + using IncidenceStat = lib::IncidenceCount::Statistic; + + /** + * Mix-in for setup of a #ParameterRange evaluation to watch + * the processing of a single load peak, using the number of + * added job as independent parameter. + * @remark inject this definition (by inheritance) into the + * Setup, which should then also define a TestChainLoad + * graph with an overall size controlled by the #Param + * @see SchedulerStress_test#watch_expenseFunction() + */ + struct LoadPeak_ParamRange_Evaluation + { + using Param = size_t; + + struct DataRow + { + Column param {"load size"}; // independent variable / control parameter + Column time {"result time"}; + Column conc {"concurrency"}; + Column jobtime {"avg jobtime"}; + Column overhead{"overhead"}; + + auto allColumns() + { return std::tie(param + ,time + ,conc + ,jobtime + ,overhead + ); + } + }; + + using Table = DataTable; + + void + collectResult(Table& data, Param param, double millis, bench::IncidenceStat const& stat) + { + (void)millis; + data.newRow(); + data.param = param; + data.time = stat.coveredTime / 1000; + data.conc = stat.avgConcurrency; + data.jobtime = stat.activeTime/stat.activationCnt; + data.overhead = stat.timeAtConc(1) / stat.activationCnt; ////OOO not really clear if sensible + } + }; // }// namespace bench }}}// namespace vault::gear::test diff --git a/wiki/thinkPad.ichthyo.mm b/wiki/thinkPad.ichthyo.mm index 77e54d886..a82d92e35 100644 --- a/wiki/thinkPad.ichthyo.mm +++ b/wiki/thinkPad.ichthyo.mm @@ -114732,7 +114732,7 @@ std::cout << tmpl.render({"what", "World"}) << s - + @@ -114935,7 +114935,7 @@ std::cout << tmpl.render({"what", "World"}) << s - + @@ -114947,7 +114947,8 @@ std::cout << tmpl.render({"what", "World"}) << s - + + @@ -114958,23 +114959,135 @@ std::cout << tmpl.render({"what", "World"}) << s - - - + + + + + + + + +

+ ...man würde also bereits ganz von Grund auf einen Hüllen-Container bzw. eine virtuelle Schnittstelle brauchen — und ich wollte hier exakt den entgegengesetzten Weg gehen, mit lauter konkreten, weitgehend lokal definierten Typen und daher auch Template-Parametern für die Breite des Graphen +

+ + + + + + + + + + +
    +
  • + kann nicht einfach ausführbaren Code in einen Klassenrumpf schreiben (wie in Scala oder Python) +
    • +
  • + kann keine Parameter eines Basis-Typs dynamisch / late-binding modifizieren +
    • +
  • + kann keine virtuelle Funktion über generische Parameter arbeiten lassen +
    • +
  • + Typen aus Template-Basisklassen werden erst durch explizite Typedefs sichtbar ⟹ schwer lesbar +
    • +
  • + Template-Parameter einer Basis / Framework-Klasse müssen ganz zu Beginn schon feststehen, und können sich nicht aus konkreten Spezialisierungen ergeben +
    • +
  • + es sei denn... man würde ein sehr komplexes Framework mit Wrapper-Typen bauen, und darüber einen »Deckel« setzen, der dann alles zusammenlinkt; ein solcher Ansatz wäre vielleicht machbar (sicher Wochen an Arbeit) — dann aber wohl so komplex, daß selbst C++ -  Experten nicht mehr auf Anhieb sehen, was gespielt wird. +
    • +
+ + + + + + + + + + + + + + + + + + + + + + + + + +

+ direkt im Test: using StressRig = StressTestRig<16>; +

+ + + + + + + + + + +

+ Fazit: knapp am Abgrund vorbei +

+ + + + + + + + + + + +

+ Tests sehen einfach aus... +

+

+ und sind leicht anpassbar +

+ + + + + + + + + + + + - - + + + + + +