From 693ba32c8ea09cf3a571bf3a4e8710698f147df5 Mon Sep 17 00:00:00 2001 From: Ichthyostega Date: Sat, 16 Nov 2024 19:34:37 +0100 Subject: [PATCH] Library: sharpen criteria for detecting glitches A deeper investigation revealed that we can show the result of glitches for each relevant situation, simply by scrutinising the produced distribution. Even the 64-bit-Variant shows a skewed distribuion, in spite of all numbers being within definition range. So the conclusion is: we can expect tilted results, but in many cases this might not be an issue, if the result range is properly wrapped / clipped. Notably this is the case if we just want to inject a randomised sleep into a multithreaded test setup Build a self-contained test case to document these findings. --- tests/library/random-concurrent-test.cpp | 141 ++++++++++++++++------- wiki/thinkPad.ichthyo.mm | 90 ++++++++++++++- 2 files changed, 183 insertions(+), 48 deletions(-) diff --git a/tests/library/random-concurrent-test.cpp b/tests/library/random-concurrent-test.cpp index 1be684fc7..82102efbf 100644 --- a/tests/library/random-concurrent-test.cpp +++ b/tests/library/random-concurrent-test.cpp @@ -34,6 +34,8 @@ #include "lib/util.hpp" #include "lib/scoped-collection.hpp" #include "lib/test/microbenchmark.hpp" +#include "lib/format-string.hpp" +#include "lib/format-cout.hpp" #include "lib/test/diagnostic-output.hpp" #include @@ -43,18 +45,18 @@ //using std::array; using std::tuple; using std::deque; +using util::_Fmt; namespace lib { namespace test { namespace { - const uint NUM_THREADS = 8; - const uint NUM_REPEATS = 10; - const uint NUM_INVOKES = 1'000'000; - - + const uint NUM_THREADS = 8; ///< for concurrent probes + const uint NUM_SAMPLES = 80; ///< overall number measurement runs + const uint NUM_INVOKES = 1'000'000; ///< invocations of the target per measurment } + /******************************************************************//** * @test demonstrate simple access to random number generation, * as well as the setup of controlled random number sequences. @@ -71,52 +73,103 @@ namespace test { } + template + struct Experiment + : Sync<> + { + deque> results; + + void + recordRun (double err, uint fails) + { + Lock sync(this); + results.emplace_back (err, fails); + } + + + GEN generator; + + Experiment(GEN&& fun) + : generator{move (fun)} + { } + + const uint N = NUM_INVOKES; + const uint REPEATS = NUM_SAMPLES / threads; + using ResVal = typename GEN::result_type; + ResVal expect = (GEN::max() - GEN::min()) / 2; + + /* === Measurement Results === */ + double percentGlitches{0.0}; + double percentTilted {0.0}; + bool isFailure {false}; + + + void + perform() + { + auto drawRandom = [&]() + { + uint fail{0}; + double avg{0.0}; + for (uint i=0; i GEN::max()) + ++fail; + avg += 1.0/N * r;//(r % Engine::max()); + } + auto error = avg/expect - 1; + recordRun (error, fail); + }; + + threadBenchmark (drawRandom, REPEATS); + + uint cases{0}, lows{0}, glitches{0}; + _Fmt resultLine{"%6.3f ‰ : %d %s"}; + for (auto [err,fails] : results) + { + bool isGlitch = fails or fabs(err) >0.003; + cout << resultLine % (err*1000) + % fails + % (fails? "FAIL": isGlitch? " !! ":"") << endl; + ++cases; + if (err < 0) ++lows; + if (isGlitch) ++glitches; + } + // assess overall results...... + percentGlitches = 100.0 * glitches/cases; + percentTilted = 100.0 * fabs(double(lows)/cases - 0.5)*2; + isFailure = glitches or percentTilted > 30; + cout << _Fmt{"++-------------++ %s\n" + " Glitches: %5.1f %%\n" + " Tilted: %5.1f %%\n" + "++-------------++\n"} + % (isFailure? "FAIL": "(ok)") + % percentGlitches + % percentTilted + << endl; + } + }; + + /** @test examine behaviour of PRNG under concurrency stress */ void investigate_concurrentAccess() { - struct Results - : deque> - , Sync<> - { - void - post (double err, uint fails) - { - Lock sync(this); - emplace_back (err, fails); - } - }; - - Results results; + using Mersenne32 = std::mt19937; + using Mersenne64 = std::mt19937_64; - using Engine = std::mt19937; -// using Engine = std::mt19937_64; + Experiment single32{Mersenne32(defaultGen.uni())}; + Experiment concurr32{Mersenne32(defaultGen.uni())}; + Experiment concurr64{Mersenne64(defaultGen.uni())}; - Engine ranGen{defaultGen.u64()}; + single32.perform(); + concurr32.perform(); + concurr64.perform(); - const uint N = NUM_INVOKES; - auto expect = (Engine::max() - Engine::min()) / 2; - - auto drawRandom = [&]() - { - uint fail{0}; - double avg{0.0}; - for (uint i=0; i Engine::max()) - ++fail; - avg += 1.0/N * (r % Engine::max()); - } - auto error = avg/expect - 1; - results.post (error, fail); - }; - - auto [dur,sum] = threadBenchmark (drawRandom, NUM_REPEATS); - for (auto res : results) - SHOW_EXPR(res); - SHOW_EXPR(sum) - SHOW_EXPR(dur/NUM_INVOKES) + CHECK (not single32.isFailure, "ALARM : single-threaded Mersenne-Twister 32bit produces skewed distribution"); + CHECK ( single32.isFailure, "SURPRISE : Mersenne-Twister 32bit encountered NO glitches under concurrent pressure"); + CHECK ( single64.isFailure, "SURPRISE : Mersenne-Twister 64bit encountered NO glitches under concurrent pressure"); } }; diff --git a/wiki/thinkPad.ichthyo.mm b/wiki/thinkPad.ichthyo.mm index 5f7490168..656556800 100644 --- a/wiki/thinkPad.ichthyo.mm +++ b/wiki/thinkPad.ichthyo.mm @@ -58080,11 +58080,11 @@ - - + + - - + + @@ -58099,6 +58099,88 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

+ schon kleine Änderungen in der Payload-Funktion können die Inzidenz der Probleme drastisch ändern; beispielsweise hat das Hinzufügen einer Begrenzung per Modulo für einen Generator die Inzidenz drastisch erhöht, für einen anderen die Probleme nahezu zum Verschwinden gebracht +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +