From 0e7bdcc5b5df25f5a94d752541360fd25f0ae7ff Mon Sep 17 00:00:00 2001 From: Ichthyostega Date: Fri, 16 Feb 2024 03:45:27 +0100 Subject: [PATCH] Scheduler-test: experiment with extended load and run time Various experiments to watch Scheduler behaviour under extended load. Notably the example committed here makes the Scheduler run for 1.2 sec and process 800 jobs with 10ms each, thereby putting the system into 100% load on all CPUs --- tests/vault/gear/scheduler-stress-test.cpp | 18 +- tests/vault/gear/test-chain-load.hpp | 17 ++ wiki/thinkPad.ichthyo.mm | 204 +++++++++++++++++++-- 3 files changed, 217 insertions(+), 22 deletions(-) diff --git a/tests/vault/gear/scheduler-stress-test.cpp b/tests/vault/gear/scheduler-stress-test.cpp index 46f15f4a0..5dd6bc550 100644 --- a/tests/vault/gear/scheduler-stress-test.cpp +++ b/tests/vault/gear/scheduler-stress-test.cpp @@ -295,8 +295,8 @@ namespace test { verify_instrumentation() { MARK_TEST_FUN - TestChainLoad testLoad{20}; - testLoad + TestChainLoad testLoad{800}; + testLoad.setWeight(1) .printTopologyDOT() .printTopologyStatistics() ; @@ -304,23 +304,25 @@ namespace test { BlockFlowAlloc bFlow; EngineObserver watch; Scheduler scheduler{bFlow, watch}; - auto LOAD_BASE = 500us; + auto LOAD_BASE = 10ms; // auto stressFac = 1.0; // auto concurrency = 8; // ComputationalLoad cpuLoad; - cpuLoad.timeBase = LOAD_BASE; + cpuLoad.timeBase = 200us; cpuLoad.calibrate(); // double loadMicros = cpuLoad.invoke(); - double refTime = testLoad.calcRuntimeReference(LOAD_BASE); +// double refTime = testLoad.calcRuntimeReference(LOAD_BASE); SHOW_EXPR(loadMicros) auto testSetup = testLoad.setupSchedule(scheduler) .withLoadTimeBase(LOAD_BASE) - .withJobDeadline(50ms) - .withUpfrontPlanning() + .withLevelDuration(10us) + .withJobDeadline(2s) +// .withBaseExpense(500us) +// .withUpfrontPlanning() .withInstrumentation() // .withAdaptedSchedule (stressFac, concurrency) ; @@ -328,7 +330,7 @@ SHOW_EXPR(loadMicros) double expected = testSetup.getExpectedEndTime(); SHOW_EXPR(runTime) SHOW_EXPR(expected) -SHOW_EXPR(refTime) +//SHOW_EXPR(refTime) auto stat = testSetup.getInvocationStatistic(); SHOW_EXPR(stat.cumulatedTime); SHOW_EXPR(stat.activeTime); diff --git a/tests/vault/gear/test-chain-load.hpp b/tests/vault/gear/test-chain-load.hpp index 75dda3c55..60b5b0711 100644 --- a/tests/vault/gear/test-chain-load.hpp +++ b/tests/vault/gear/test-chain-load.hpp @@ -69,6 +69,10 @@ ** [retrieved](\ref TestChainLoad::getHash). The node structure can then be traversed or ** [scheduled as Render Jobs](\ref TestChainLoad::scheduleJobs). ** + ** A special use case is _not to build any topology,_ rather only #setWeight(). + ** All nodes ** will then be at level-0 and scheduled at t=0, causing the scheduler + ** to process best effort in arbitrary order. + ** ** ## Test support ** A tool for generating roughly calibrated [computational load](\ref ComputationalLoad) is included, ** to be controlled by the Node::weight stepping. Load can either be generated by arithmetic (hash) @@ -688,6 +692,19 @@ namespace test { } + /** + * Set a fixed weight for all nodes + * @note no change to topology (works even without any topology) + */ + TestChainLoad&& + setWeight (size_t fixedNodeWeight =1) + { + for (Node& n : allNodes()) + n.weight = fixedNodeWeight; + return move(*this); + } + + /** * Recalculate all node hashes and propagate seed value. */ diff --git a/wiki/thinkPad.ichthyo.mm b/wiki/thinkPad.ichthyo.mm index 872973f0e..be65a3050 100644 --- a/wiki/thinkPad.ichthyo.mm +++ b/wiki/thinkPad.ichthyo.mm @@ -111126,23 +111126,25 @@ Date:   Thu Apr 20 18:53:17 2023 +0200
- - - - + + + + - - + + - - + + + + - - + + - - + + @@ -111255,7 +111257,7 @@ Date:   Thu Apr 20 18:53:17 2023 +0200
- + @@ -111263,11 +111265,170 @@ Date:   Thu Apr 20 18:53:17 2023 +0200
- + + + + + + + + + + + + + + + + + + + + + + + + + + + +

+ ...waren aber geblockt, solange noch die eigentlichen Jobs mit t=0 in der Queue stehen +

+ + + + + + + + + + + + + + + + + + +

+ ...und die Selbstreguliertung sorgt schntell dafür, daß der größte Teil der WorkForce in contention-Wait geht +

+ + + + + + + + + + + + + + + +

+ zweiter Versuch: unconnected Nodes with weight (500µs) +

+ + + + + + + + + + + + + + + + + + + + + +

+ das entspricht exakt der Concurrency +

+ + + + + + + + + + + + + + + + + + + + + + + +

+ das ist effektiv concurrency ≡ 6 +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +

+ mit 400 Nodes  ⟼ real ∅ 12ms und Concurrency ≡ 7.9 +

+ + + + + + + + + + @@ -111674,6 +111835,21 @@ Date:   Thu Apr 20 18:53:17 2023 +0200
+ + + + + + +

+ das beobachte ich mit dem DUMP-Logging immer wieder. Und zwar ohne eine ausgeprägte Contention-Spitze. Nach dem Absetzen des letzten regulären Node-Jobs sieht man das Log vom Continuation-Aufruf, dann lang nichts, dann die Sequenz der dependencies. +

+ + + + + +