diff --git a/src/vault/gear/activity-lang.hpp b/src/vault/gear/activity-lang.hpp index 342a8bed5..9506dc64f 100644 --- a/src/vault/gear/activity-lang.hpp +++ b/src/vault/gear/activity-lang.hpp @@ -28,9 +28,9 @@ ** can be passed to the scheduler's messaging queue to cause the described ** activities to be performed in the context of the Lumiera render engine. ** - ** @note this is a wild fantasy - ** @see ////TODO_test usage example - ** @see scheduler.cpp implementation + ** @warning mostly this is planned functionality + ** @see SchedulerActivity_test + ** @see activity.hpp definition of verbs ** ** @todo WIP-WIP-WIP 6/2023 »Playback Vertical Slice« ** @@ -42,6 +42,7 @@ #include "vault/gear/activity.hpp" +#include "vault/gear/block-flow.hpp" //#include "lib/symbol.hpp" //#include "lib/util.hpp" @@ -56,19 +57,19 @@ namespace gear { /** - * Basic (abstracted) view of... + * TODO write type comment... * - * @see SomeSystem - * @see NA_test + * @see Activity + * @see SchedulerActivity_test */ class ActivityLang { - string nothing_; + BlockFlow& mem_; public: // explicit - ActivityLang () - : nothing_(b) + ActivityLang (BlockFlow& memManager) + : mem_{memManager} { } // using default copy/assignment diff --git a/src/vault/gear/block-flow.hpp b/src/vault/gear/block-flow.hpp index caf05636b..d7c95e8a3 100644 --- a/src/vault/gear/block-flow.hpp +++ b/src/vault/gear/block-flow.hpp @@ -38,8 +38,9 @@ ** ** @note currently this rather marks the intended memory management pattern, ** while the actual allocations are still performed on the heap. - ** @see ////TODO_test usage example - ** @see scheduler.cpp implementation + ** @see BlockFlow_test + ** @see SchedulerUsage_test + ** @see extent-family.hpp underlying allocation scheme ** ** @todo WIP-WIP-WIP 6/2023 »Playback Vertical Slice« ** @@ -68,8 +69,8 @@ namespace gear { /** * Basic (abstracted) view of... * - * @see SomeSystem - * @see NA_test + * @see SchedulerCommutator + * @see BlockFlow_test */ class BlockFlow : util::NonCopyable diff --git a/src/vault/gear/scheduler-commutator.hpp b/src/vault/gear/scheduler-commutator.hpp index eb5738c5b..51eb22810 100644 --- a/src/vault/gear/scheduler-commutator.hpp +++ b/src/vault/gear/scheduler-commutator.hpp @@ -25,8 +25,8 @@ ** Layer-2 of the Scheduler: coordination and interaction of activities. ** This is the upper layer of the implementation and provides high-level functionality. ** - ** @see ////TODO_test usage example - ** @see scheduler.cpp implementation + ** @see SchedulerCommutator_test + ** @see scheduler.hpp usage ** ** @todo WIP-WIP-WIP 6/2023 »Playback Vertical Slice« ** @@ -56,8 +56,8 @@ namespace gear { /** * Scheduler Layer-2 : coordination. * - * @see SomeSystem - * @see NA_test + * @see SchedulerInvocation (Layer-1) + * @see SchedulerCommutator_test */ class SchedulerCommutator : util::NonCopyable diff --git a/src/vault/gear/scheduler-frontend.hpp b/src/vault/gear/scheduler-frontend.hpp index 6cabbfe27..7f5528c53 100644 --- a/src/vault/gear/scheduler-frontend.hpp +++ b/src/vault/gear/scheduler-frontend.hpp @@ -25,6 +25,7 @@ ** Scheduler service access point for higher layers. ** @todo WIP unfinished since 9/2013 ** @warning as of 4/2023 Render-Engine integration work is underway ////////////////////////////////////////TICKET #1280 + ** @deprecated as of 7/2023 the scheduler API will likely draw upon the ActivityLang ** */ diff --git a/src/vault/gear/scheduler-invocation.hpp b/src/vault/gear/scheduler-invocation.hpp index 373168206..6971adce4 100644 --- a/src/vault/gear/scheduler-invocation.hpp +++ b/src/vault/gear/scheduler-invocation.hpp @@ -25,9 +25,9 @@ ** Layer-1 of the Scheduler: dispatch and invocation of activities. ** This is the lower layer of the implementation and provides low-level functionality. ** - ** @note as of X/2023 this is complete bs - ** @see ////TODO_test usage example - ** @see scheduler.cpp implementation + ** @see SchedulerInvocation_test + ** @see SchedulerUsage_test integrated usage + ** @see scheduler.cpp implementation details ** ** @todo WIP-WIP-WIP 6/2023 »Playback Vertical Slice« ** diff --git a/src/vault/gear/scheduler.hpp b/src/vault/gear/scheduler.hpp index 795cc0edd..5740d2c4e 100644 --- a/src/vault/gear/scheduler.hpp +++ b/src/vault/gear/scheduler.hpp @@ -28,8 +28,10 @@ ** - Layer-1 allows to enqueue and prioritise render activity records ** - Layer-2 connects and coordinates activities to conduct complex calculations ** - ** @see ////TODO_test usage example - ** @see scheduler.cpp implementation + ** @see SchedulerUsage_test Component integration test + ** @see scheduler.cpp implementation details + ** @see SchedulerInvocation Layer-1 + ** @see SchedulerCommutator Layer-2 ** ** @todo WIP-WIP-WIP 6/2023 »Playback Vertical Slice« ** @@ -61,8 +63,8 @@ namespace gear { /** * Schedule and coordinate render activities. * @todo WIP-WIP 6/2023 - * @see SomeSystem - * @see NA_test + * @see BlockFlow + * @see SchedulerUsage_test */ class Scheduler { diff --git a/src/vault/mem/extent-family.hpp b/src/vault/mem/extent-family.hpp new file mode 100644 index 000000000..e8f779aa2 --- /dev/null +++ b/src/vault/mem/extent-family.hpp @@ -0,0 +1,80 @@ +/* + EXTENT-FAMILY.hpp - maintain a sequence of memory extents used cyclically + + Copyright (C) Lumiera.org + 2023, Hermann Vosseler + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of + the License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + +*/ + + +/** @file extent-family.hpp + ** Memory management scheme for cyclically used memory extents. + ** In this context, _Extent_ denotes a larger block of memory to hold a + ** cluster of smaller records, which are closely interrelated and will be managed + ** and de-allocated together. The typical usage involves a constant demand for further + ** memory, with is satisfied by putting further unused extents into use; older extents + ** will fall out of use eventually, and can then be placed back into a buffer of free + ** extents. In accordance to overall demand, this reserve buffer can be scaled up + ** and down to avoid holding larger amounts of unused memory, while the availability + ** of a baseline amount of memory can be enforced. + ** + ** @see ////TODO_test usage example + ** @see gear::BlockFlow usage example + ** + ** @todo WIP-WIP-WIP 7/2023 »Playback Vertical Slice« + ** + */ + + +#ifndef SRC_VAULT_MEM_EXTENT_FAMILY_H_ +#define SRC_VAULT_MEM_EXTENT_FAMILY_H_ + + +#include "vault/common.hpp" +//#include "lib/symbol.hpp" +#include "lib/nocopy.hpp" +//#include "lib/util.hpp" + +//#include + + +namespace vault{ +namespace mem { + +// using util::isnil; +// using std::string; + + + /** + * Memory manager to provide a sequence of Extents for cyclic usage. + * @todo WIP-WIP 7/2023 + * @see NA_test + */ + class ExtentFamily + : util::NonCopyable + { + + public: + explicit + ExtentFamily() + { } + }; + + + +}} // namespace vault::mem +#endif /*SRC_VAULT_MEM_EXTENT_FAMILY_H_*/ diff --git a/tests/11vault-memory.tests b/tests/11vault-memory.tests new file mode 100644 index 000000000..e24544c4e --- /dev/null +++ b/tests/11vault-memory.tests @@ -0,0 +1,7 @@ +TESTING "Low-level Test Suite: Memory Management" ./test-suite --group=memory + + + +PLANNED "Cyclic Extent Sequence" ExtentFamily_test <high" - - diff --git a/tests/32scheduler.tests b/tests/32scheduler.tests new file mode 100644 index 000000000..adc7af79b --- /dev/null +++ b/tests/32scheduler.tests @@ -0,0 +1,31 @@ +TESTING "Component Test Suite: Scheduler" ./test-suite --group=engine + + + +PLANNED "BlockFlow memory management scheme" BlockFlow_test < + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of + the License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + +* *****************************************************/ + +/** @file block-flow-test.cpp + ** unit test \ref BlockFlow_test + */ + + +#include "lib/test/run.hpp" +#include "vault/gear/block-flow.hpp" +//#include "lib/time/timevalue.hpp" +//#include "lib/format-cout.hpp" +//#include "lib/util.hpp" + +//#include + +using test::Test; +//using std::move; +//using util::isSameObject; + + +namespace vault{ +namespace mem { +namespace test { + +// using lib::time::FrameRate; +// using lib::time::Offset; +// using lib::time::Time; + + + + + + /*****************************************************************//** + * @test document the memory management scheme used by the Scheduler. + * @see SchedulerActivity_test + * @see SchedulerUsage_test + */ + class BlockFlow_test : public Test + { + + virtual void + run (Arg) + { + simpleUsage(); + calculateDeadline(); + setupLalup(); + } + + + /** @test TODO demonstrate a simple usage scenario + */ + void + simpleUsage() + { + } + + + + /** @test TODO + */ + void + calculateDeadline() + { + } + + + + /** @test TODO + */ + void + setupLalup() + { + } + }; + + + /** Register this test class... */ + LAUNCHER (BlockFlow_test, "unit engine"); + + + +}}} // namespace vault::mem::test diff --git a/tests/vault/gear/scheduler-activity-test.cpp b/tests/vault/gear/scheduler-activity-test.cpp new file mode 100644 index 000000000..ab3a3edd2 --- /dev/null +++ b/tests/vault/gear/scheduler-activity-test.cpp @@ -0,0 +1,102 @@ +/* + SchedulerActivity(Test) - verify activities processed in the scheduler + + Copyright (C) Lumiera.org + 2023, Hermann Vosseler + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of + the License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + +* *****************************************************/ + +/** @file scheduler-activity-test.cpp + ** unit test \ref SchedulerActivity_test + */ + + +#include "lib/test/run.hpp" +#include "vault/gear/activity-lang.hpp" +//#include "lib/time/timevalue.hpp" +//#include "lib/format-cout.hpp" +//#include "lib/util.hpp" + +//#include + +using test::Test; +//using std::move; +//using util::isSameObject; + + +namespace vault{ +namespace mem { +namespace test { + +// using lib::time::FrameRate; +// using lib::time::Offset; +// using lib::time::Time; + + + + + + /*****************************************************************//** + * @test verify behaviour of the Scheduler _Activity Language._ + * @see SchedulerCommutator_test + * @see SchedulerUsage_test + */ + class SchedulerActivity_test : public Test + { + + virtual void + run (Arg) + { + simpleUsage(); + walkingDeadline(); + setupLalup(); + } + + + /** @test TODO demonstrate a simple usage scenario + */ + void + simpleUsage() + { + } + + + + /** @test TODO + */ + void + walkingDeadline() + { + } + + + + /** @test TODO + */ + void + setupLalup() + { + } + }; + + + /** Register this test class... */ + LAUNCHER (SchedulerActivity_test, "unit engine"); + + + +}}} // namespace vault::mem::test diff --git a/tests/vault/gear/scheduler-commutator-test.cpp b/tests/vault/gear/scheduler-commutator-test.cpp new file mode 100644 index 000000000..cd947388c --- /dev/null +++ b/tests/vault/gear/scheduler-commutator-test.cpp @@ -0,0 +1,102 @@ +/* + SchedulerCommutator(Test) - verify dependent activity processing in the scheduler + + Copyright (C) Lumiera.org + 2023, Hermann Vosseler + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of + the License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + +* *****************************************************/ + +/** @file scheduler-commutator-test.cpp + ** unit test \ref SchedulerCommutator_test + */ + + +#include "lib/test/run.hpp" +#include "vault/gear/scheduler-commutator.hpp" +//#include "lib/time/timevalue.hpp" +//#include "lib/format-cout.hpp" +//#include "lib/util.hpp" + +//#include + +using test::Test; +//using std::move; +//using util::isSameObject; + + +namespace vault{ +namespace mem { +namespace test { + +// using lib::time::FrameRate; +// using lib::time::Offset; +// using lib::time::Time; + + + + + + /******************************************************************//** + * @test Scheduler Layer-2: dependency notification and triggering. + * @see SchedulerActivity_test + * @see SchedulerUsage_test + */ + class SchedulerCommutator_test : public Test + { + + virtual void + run (Arg) + { + simpleUsage(); + walkingDeadline(); + setupLalup(); + } + + + /** @test TODO demonstrate a simple usage scenario + */ + void + simpleUsage() + { + } + + + + /** @test TODO + */ + void + walkingDeadline() + { + } + + + + /** @test TODO + */ + void + setupLalup() + { + } + }; + + + /** Register this test class... */ + LAUNCHER (SchedulerCommutator_test, "unit engine"); + + + +}}} // namespace vault::mem::test diff --git a/tests/vault/gear/scheduler-invocation-test.cpp b/tests/vault/gear/scheduler-invocation-test.cpp new file mode 100644 index 000000000..a2431601d --- /dev/null +++ b/tests/vault/gear/scheduler-invocation-test.cpp @@ -0,0 +1,102 @@ +/* + SchedulerInvocation(Test) - verify queue processing in the scheduler + + Copyright (C) Lumiera.org + 2023, Hermann Vosseler + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of + the License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + +* *****************************************************/ + +/** @file scheduler-invocation-test.cpp + ** unit test \ref SchedulerInvocation_test + */ + + +#include "lib/test/run.hpp" +#include "vault/gear/scheduler-invocation.hpp" +//#include "lib/time/timevalue.hpp" +//#include "lib/format-cout.hpp" +//#include "lib/util.hpp" + +//#include + +using test::Test; +//using std::move; +//using util::isSameObject; + + +namespace vault{ +namespace mem { +namespace test { + +// using lib::time::FrameRate; +// using lib::time::Offset; +// using lib::time::Time; + + + + + + /********************************************************************//** + * @test Scheduler Layer-1: queue processing and invocation by priority. + * @see SchedulerCommutator_test + * @see SchedulerUsage_test + */ + class SchedulerInvocation_test : public Test + { + + virtual void + run (Arg) + { + simpleUsage(); + walkingDeadline(); + setupLalup(); + } + + + /** @test TODO demonstrate a simple usage scenario + */ + void + simpleUsage() + { + } + + + + /** @test TODO + */ + void + walkingDeadline() + { + } + + + + /** @test TODO + */ + void + setupLalup() + { + } + }; + + + /** Register this test class... */ + LAUNCHER (SchedulerInvocation_test, "unit engine"); + + + +}}} // namespace vault::mem::test diff --git a/tests/vault/gear/scheduler-usage-test.cpp b/tests/vault/gear/scheduler-usage-test.cpp new file mode 100644 index 000000000..8249c8cfd --- /dev/null +++ b/tests/vault/gear/scheduler-usage-test.cpp @@ -0,0 +1,103 @@ +/* + SchedulerUsage(Test) - component integration test for the scheduler + + Copyright (C) Lumiera.org + 2023, Hermann Vosseler + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of + the License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + +* *****************************************************/ + +/** @file scheduler-usage-test.cpp + ** unit test \ref SchedulerUsage_test + */ + + +#include "lib/test/run.hpp" +#include "vault/gear/scheduler.hpp" +//#include "lib/time/timevalue.hpp" +//#include "lib/format-cout.hpp" +//#include "lib/util.hpp" + +//#include + +using test::Test; +//using std::move; +//using util::isSameObject; + + +namespace vault{ +namespace mem { +namespace test { + +// using lib::time::FrameRate; +// using lib::time::Offset; +// using lib::time::Time; + + + + + + /*************************************************************************//** + * @test Scheduler component integration test: add and process dependent jobs. + * @see SchedulerActivity_test + * @see SchedulerInvocation_test + * @see SchedulerCommutator_test + */ + class SchedulerUsage_test : public Test + { + + virtual void + run (Arg) + { + simpleUsage(); + walkingDeadline(); + setupLalup(); + } + + + /** @test TODO demonstrate a simple usage scenario + */ + void + simpleUsage() + { + } + + + + /** @test TODO + */ + void + walkingDeadline() + { + } + + + + /** @test TODO + */ + void + setupLalup() + { + } + }; + + + /** Register this test class... */ + LAUNCHER (SchedulerUsage_test, "unit engine"); + + + +}}} // namespace vault::mem::test diff --git a/tests/vault/mem/extent-family-test.cpp b/tests/vault/mem/extent-family-test.cpp new file mode 100644 index 000000000..e90202995 --- /dev/null +++ b/tests/vault/mem/extent-family-test.cpp @@ -0,0 +1,105 @@ +/* + ExtentFamily(Test) - verify cyclic extents allocation scheme + + Copyright (C) Lumiera.org + 2023, Hermann Vosseler + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of + the License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + +* *****************************************************/ + +/** @file extent-family-test.cpp + ** unit test \ref ExtentFamily_test + */ + + +#include "lib/test/run.hpp" +#include "vault/mem/extent-family.hpp" +//#include "lib/time/timevalue.hpp" +//#include "lib/format-cout.hpp" +//#include "lib/util.hpp" + +//#include + +using test::Test; +//using std::move; +//using util::isSameObject; + + +namespace vault{ +namespace mem { +namespace test { + +// using lib::time::FrameRate; +// using lib::time::Offset; +// using lib::time::Time; + + + + + + /***************************************************************//** + * @test document and verify a memory management scheme to maintain + * a flexible set of _»memory extents«_ for cyclic usage. + * @see BlockFlow_test + */ + class ExtentFamily_test : public Test + { + + virtual void + run (Arg) + { + simpleUsage(); + calculateDeadline(); + setupDependentJob(); + } + + + /** @test demonstrate a simple usage scenario + */ + void + simpleUsage() + { + ExtentFamily extents; + } + + + + /** @test verify the timing calculations to establish + * the scheduling deadline of a simple render job + */ + void + calculateDeadline() + { + } + + + + /** @test verify the setup of a prerequisite job in relation + * to the master job depending on this prerequisite + */ + void + setupDependentJob() + { + } + }; + + + /** Register this test class... */ + LAUNCHER (ExtentFamily_test, "unit memory"); + + + +}}} // namespace vault::mem::test diff --git a/wiki/renderengine.html b/wiki/renderengine.html index 9d4841b89..36cf79869 100644 --- a/wiki/renderengine.html +++ b/wiki/renderengine.html @@ -6183,7 +6183,7 @@ This is the core service provided by the player subsystem. The purpose is to cre :any details of this processing remain opaque for the clients; even the player subsystem just accesses the EngineFaçade -
+
//Integration effort to promote the development of rendering, playback and video display in the GUI//
 This IntegrationSlice was started in {{red{2023}}} as [[Ticket #1221|https://issues.lumiera.org/ticket/1221]] to coordinate the completion and integration of various implementation facilities, planned, drafted and built during the last years; this effort marks the return of development focus to the lower layers (after years of focussed UI development) and will implement the asynchronous and time-bound rendering coordinated by the [[Scheduler]] in the [[Vault|Vault-Layer]]
 
@@ -6204,7 +6204,16 @@ __May.23__: taking a //prototyping approach// now, since further development was
 * ✔ build a {{{MockSegmentation}}} to hold onto ~JobTickets, which can be created as Mock
 * ✔define a simple specification language (based on the existing {{{GenNode}}}-DSL to define segments, tickets and prerequisite jobs
 * ✔ implement a »~Split-Splice« algorithm for &rarr; SegmentationChange, rigged accordingly to generate a mocked Segementation for now
-* 🗘 create a testbed to assemble a JobPlanningPipeline step by step (&rarr; [[#920|https://issues.lumiera.org/ticket/920]] and [[#1275|https://issues.lumiera.org/ticket/1275|]])
+* ✔ create a testbed to assemble a JobPlanningPipeline step by step (&rarr; [[#920|https://issues.lumiera.org/ticket/920]] and [[#1275|https://issues.lumiera.org/ticket/1275|]])
+
+__June23__: building upon this prototyping approach, the dispatcher pipeline could be rearranged in the form of a pipeline builder, allowing to retract the originally used implementation scheme based on »Monads«. The implementation of the Dispatcher is complete, yet the build up of the [[»Render Drive« #1301|https://issues.lumiera.org/ticket/1301]] could not reasonably be completed, due to lack of a clear-shaped ''Scheduler interface''.
+
+__July23__: this leads to a shift of work focus towards implementing the [[Scheduler]] itself.
+The Scheduler will be structured into two Layers, where the lower layer is implemented as //priority queue// (using the STL). So the most tricky part to solve is the representation of //dependencies// between jobs, with the possible extension to handling IO operations asynchronously. Analysis and planning of the implementation indicate that the [[scheduler memory managment|SchedulerMemory]] can be based on //Extents//, which are interpreted as »Epochs« with a deadline. These considerations imply the next steps for building up the Scheduler functionality
+* 🗘 build a first working draft for the {{{BlockFlow}}} allocation scheme [[#1311|https://issues.lumiera.org/ticket/1311]]
+* ⌛ define and cover the basic [[Activities|RenderActivity]] necessary to implement a plain-simple-Job (without dependencies)
+* ⌛ pass such an Activity through the two layers of the Scheduler
+* ⌛ adapt the [[job planning pipeline|JobPlanningPipeline]] implemented thus far to produce the appropriate {{{Activity}}} records for the scheduler
 
 !Decisions
 ;Scheduler
@@ -6212,6 +6221,7 @@ __May.23__: taking a //prototyping approach// now, since further development was
 :* shall support concerns of process- and memory management
 :* thus needs to //understand Job dependencies//
 :* will be decomposed into several implementation layers
+:SchedulerMemory will be managed by an //Extent scheme.//
@@ -7044,7 +7054,7 @@ Later on we expect a distinct __query subsystem__ to emerge, presumably embeddin &rarr; QuantiserImpl
-
+
//Invoke and control the dependency and time based execution of  [[render jobs|RenderJob]]//
 The Scheduler acts as the central hub in the implementation of the RenderEngine and coordinates the //processing resources// of the application. Regarding architecture, the Scheduler is located in the Vault-Layer and //running// the Scheduler is equivalent to activating the »Vault Subsystem«. An EngineFaçade acts as entrance point, providing high-level render services to other parts of the application: [[render jobs|RenderJob]] can be activated under various timing and dependency constraints. Internally, the implementation is segregated into two layers
 ;Layer-2: Control
@@ -7059,6 +7069,29 @@ Time bound delivery of media data is an important aspect of editing and playback
 This leads to the observation that every render or playback process has to deal with rather incompatible processing patterns and trends: for one, processing has to start as soon as the event of an completed I/O-operation is published, yet on the other hand, limited computational resources must be distributed and prioritised in a way suitable to deliver the completed data as close as possible to a pre-established timing deadline, under the constraint of limited in-memory buffer capacity. The //control structure// of such a render engine is thus not only a time based computation plan -- first and foremost it should be conceived as an asynchronous messaging system, with the ability however to prioritise some messages based on urgency or approaching deadlines.
 
 The time-based ordering and prioritisation of [[render activities|RenderActivity]] is thus used as a //generic medium and agent// to support and implement complex interwoven computational tasks. On the layer-1 mentioned above, a combination of a lock-free dispatch queue is used, feeding into a single threaded priority queue organised by temporal deadlines. Most render activities are lightweight and entail quick updates to some state flags, while certain activities are extremely long running -- and those are shifted into worker threads based on priority.
+
+!Usage pattern
+The [[Language of render activities|RenderActivity]] forms the interface to the scheduler -- new activities are defined as //terms// and handed over to the scheduler. This happens as part of the ongoing job planning activities -- and thus will be performed //from within jobs managed by the scheduler.// Thus the access to the scheduler happens almost entirely from within the scheduler's realm itself, and is governed by the usage scheme of the [[Workers|SchedulerWorker]].
+
+These ''Worker Threads'' will perform actual render activities most of the time (or be idle). However -- idle workers contend for new work, and for doing so, they //also perform the internal scheduler management activities.// As a consequence, all Scheduler coordination and [[memory management|SchedulerMemory]] is ''performed non-concurrent'': only a single Worker can acquire the {{{GroomingToken}}} and will then perform managment work until the next render activity is encountered.
+
+
+
+
//The Scheduler uses an »Extent« based memory management scheme known as ''BlockFlow''.//
+The organisation of rendering happens in terms of [[Activities|RenderActivity]], which may bound by //dependencies// and limited by //deadlines.// For the operational point of view this implies that a sequence of allocations must be maintained to „flow through the Scheduler“ -- in fact, only references to these {{{Activity}}} records are passed, while the actual descriptors reside at fixed memory locations. This is essential to model dependencies and conditional execution structures efficiently. At some point however, any {{{Activity}}} record will either be //performed// or //obsoleted// -- and this leads to the idea of managing the allocations in memory extents termed as »Epochs«
+* a new Activity is planted into a suitable //Epoch,// based on its deadline
+* it is guaranteed to sit at a fixed memory location while it potentially can be activated
+* based on the deadlines, at some point a complete strike of activities can be reasoned to be //obsolete.//
+* this allows to discard a complete Extent without any further checks and processing (trivial destructors!)
+
+!Safeguards
+This is a rather fragile composition and chosen here for performance reasons; while activities are interconnected, there memory locations are adjacent, improving cache coherence. Moreover, most of the dependency processing and managing of activities happens single-threaded, while some [[worker|SchedulerWorker]] holds the {{{GroomingToken}}}; so most of the processing is local and does not require memory barriers.
+
+Unfortunately this also implies that most safety barriers of the C++ language are removed or circumvented. A strict processing regime must be established, with clear rules as to when activities may be accessed.
+* each »Epoch« gets an associated //deadline//
+* when the next [[job|RenderJob]] processed by a worker starts //after this Epoch's deadline//, the worker //has left the Epoch.//
+* when all workers have left an Epoch, only ''pending async IO tasks'' need to be considered, since such IO task can always be delayed for an extended period of time. For an IO task, buffers need to be prepared, and those buffers are indirectly tied to the job depending on them.
+* ⟹ thus a count of pending IO activities must be maintained //for each Epoch//  -- implemented by the same mechanism also employed for dependencies between render jobs, namely a notification leading to decreasing a local counter
diff --git a/wiki/thinkPad.ichthyo.mm b/wiki/thinkPad.ichthyo.mm index e50b33a5d..61e5b1f1b 100644 --- a/wiki/thinkPad.ichthyo.mm +++ b/wiki/thinkPad.ichthyo.mm @@ -77839,7 +77839,29 @@ Date:   Thu Apr 20 18:53:17 2023 +0200
- + + + + + + + + + + + +

+ Activity-Terme sind sowohl semantsich, als auch operational polymorph; das heißt es gibt einen gemeinsamen Kontrakt und eine flexible Ausdifferenzierung im Verhalten. ABER aus Performance-Gründen können wir uns im Scheduler keine unnötigen Indirektionen und variablen Allokationen leisten; stattdessen wird die gesamte Storage per Mehrfachbelegung auf einen einzigen Datenblock abgebildet, und das Verhalten wird in ein Switch-on-Selector-Field übersetzt +

+ + + + + + + + + @@ -77960,7 +77982,7 @@ Date:   Thu Apr 20 18:53:17 2023 +0200
- + @@ -78108,6 +78130,23 @@ Date:   Thu Apr 20 18:53:17 2023 +0200
+ + + + + + + + + + + + + + + + + @@ -78169,16 +78208,356 @@ Date:   Thu Apr 20 18:53:17 2023 +0200
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

+ d.h. wird stehts von einem Worker ausgeführt, der aktuell das GroomingToken hält +

+ + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

+ Dieses Schema hat grundsätzlich die gleichen Charakteristiken wie ein flacher Kachel-Pool, aber die Cluster-Größe wirkt als Hebel:  zusammengehörige Elemente liegen im Cluster und sind damit cache-freundlicher. Fragementierung findet zwar statt, aber auf Cluster-Ebene; deshalb braucht man auch weiterhin eine Free-List, aber auch diese wird nur einmal pro Cluster aktualisiert. +

+ + + + + + + + + +

+ Am Ende einer Kette von Activities muß eine Spezialbehandlung liegen, die die Länge der Kette weiß; zusätzlich kompliziert wird es, wenn diese Kette nicht in einen Cluster paßt (dann braucht es mehrere Notifications). Im Commutator ist spezielle Logik notwendig, die die Ausführung dieser Freigabe-Benachrichtigungen sicherstellt, damit diese dann ein gemeinsames Cluster-Gate dekrementieren, das in der ersten Allokation des Clusters liegt (und insofern Platz verschwendet) +

+ + + + + + + + + + + + + + +

+ Komplexe zeitliche Struktur, die auch zwingend zu einem gewissen Leerlauf führt, man braucht also reichlich Speicher. Außerdem ist nicht von Vornherein klar, wie viel Speicher bis zu einer Deadline noch gebraucht wird; man steht dadurch vor der Wahl, entweder Fragmentierungin Kauf zu nehmen, oder eben überschüssige Allokationen der nächsten Deadline zuzuschlagen, wodurch sie dann unnötig lange geblockt gehalten werden müssen +

+ + + + + + + + + +

+ Das führt zu einer Komplikation, da es solche Activities geben wird (Render-to-File, background activities); es muß dann eine pseudo-Deadline eingeführt werde, bei deren Überschreitung ein re-Scheduling in einen neuen Pool erfolgt. Ebenso stellen Deadlines weit in der Zukunft ein Problem dar +

+ + + + + + + + + +

+ man muß für jeden Worker mitverfolgen, ob er die Epoche verlassen hat; das ist so nicht ohne Weiteres möglich, da es Activities geben könnte, die später gestartet werden aber doch eine knappere Deadline haben (und damit noch in eine frühere Epoche fallen würden) — man müßte also warten, bis die Startzeitpunkte hinter der Deadline der Epoche liegen +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +

+ Allokationen erfolgen single-threaded, unter GroomingToken +

+ + + + + + + + + + + + + + + + + + + +

+ ein externer Trigger kann einen »post« auslösen; da ein solcher Trigger auch noch verspätet auftreten kann (nach Überschreiten der Deadline), muß nochmal explizit geprüft werden, ob die Epoche „noch lebt“, bevor man in ihr eine Activity auslöst. Das bedingt aber zumindest noch einen beweglichen Parameter für einen generischen Funktor, der irgendwo gespeichert werden muß (und nicht in der Storage der Epoche selber liegen kann). +

+ + + + + + + + + +

+ ...vielmehr muß man den Callback selber „erwischen“ und unschädlich machen, bevor er bereits wiederverwendete Storage anspricht +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

+ da Jobs nach Zeit geordnet aktiviert werden, kann man für jeden Worker eine Markierung der aktuellen Epoche erzeugen; sie wäre jeweils zu aktualisieren, wenn ein Worker das GroomingToken abgibt... (und damit in einen Render-Job einsteigt) — mithin ließe sich feststellen, wenn alle Worker eine Epoche verlassen haben. +

+ + + + + + + + + +

+ Es handelt sich hierbei um ein grundsätzliches Problem. Es liegt in der Natur von IO, daß eine solche Operation eine unbestimmte Zeit dauern kann; und diese Zeit kann ganz erheblich sein, wenn das IO-Subsystem überlastet wird. Es gibt keine Möglichkeit, eine IO-Operation abzubrechen; vielmehr kommen die Daten irgendwann an, und landen dann in dem dafür vorgesehenen Buffer. Und solange das nicht passiert ist, muß der Buffer und der Callback im Speicher bereitliegen. Ich sehe keine andere Möglichkeit, als für jede Epoche einen Zähler aller schwebenden IO-Operationen mitzuführen. Mithilfe der »post«, »notify« und »guard«-Activities ließe sich das jedoch single-threaded verwirklichen — Synchronisations-Effekte treten daher nur für Threads, die grade eine IO-Operation abgeschlossen haben, sowie den Thread, der das GroomingToken hält +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -78189,9 +78568,29 @@ Date:   Thu Apr 20 18:53:17 2023 +0200
- + + + + + + + + + + + + + + + + + + + + + @@ -78226,6 +78625,19 @@ Date:   Thu Apr 20 18:53:17 2023 +0200
 + + + + + + + + + + + + + @@ -84304,6 +84716,78 @@ class Something + + + + + + + + + + + + + + + +

+ man ruft explizit eine Service-Loop-Funktion auf — io_context::run() +

+ + + + + + + + + + + + + + + + + + + + +

+ Teil der Vision / wir wollen das +

+ + + + + + + + +

+ /** @file mmap.h +

+

+  ** MMap objects cover a memory mapped range in a file. +

+

+  ** They are managed through a global mmap registry/cache. +

+

+  */ +

+ + + + + + + + + + +