diff --git a/tests/46node.tests b/tests/46node.tests index 3e5143d68..7c58e7192 100644 --- a/tests/46node.tests +++ b/tests/46node.tests @@ -6,13 +6,21 @@ PLANNED "Proc Node basics" NodeBasic_test < + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of + the License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + +* *****************************************************/ + +/** @file node-devel-test.cpp + ** unit test \ref NodeDevel_test + */ + + +#include "lib/test/run.hpp" +//#include "lib/util.hpp" + + +//using std::string; + + +namespace steam { +namespace engine{ +namespace test { + + + + + /***************************************************************//** + * @test check render node operation modes and collaboration. + */ + class NodeDevel_test : public Test + { + virtual void run(Arg) + { + UNIMPLEMENTED ("operate some render nodes as linked together"); + } + }; + + + /** Register this test class... */ + LAUNCHER (NodeDevel_test, "unit node"); + + + +}}} // namespace steam::engine::test diff --git a/tests/core/steam/engine/node-fabrication-test.cpp b/tests/core/steam/engine/node-factory-test.cpp similarity index 86% rename from tests/core/steam/engine/node-fabrication-test.cpp rename to tests/core/steam/engine/node-factory-test.cpp index 63fb94e79..32f77a619 100644 --- a/tests/core/steam/engine/node-fabrication-test.cpp +++ b/tests/core/steam/engine/node-factory-test.cpp @@ -1,5 +1,5 @@ /* - NodeFabrication(Test) - building render nodes + NodeFactory(Test) - building render nodes Copyright (C) Lumiera.org 2009, Hermann Vosseler @@ -20,8 +20,8 @@ * *****************************************************/ -/** @file node-fabrication-test.cpp - ** unit test \ref NodeFabrication_test +/** @file node-factory-test.cpp + ** unit test \ref NodeFactory_test */ @@ -42,7 +42,7 @@ namespace test { /***************************************************************//** * @test creating and wiring various kinds of render nodes. */ - class NodeFabrication_test : public Test + class NodeFactory_test : public Test { virtual void run(Arg) { @@ -52,7 +52,7 @@ namespace test { /** Register this test class... */ - LAUNCHER (NodeFabrication_test, "unit node"); + LAUNCHER (NodeFactory_test, "unit node"); diff --git a/tests/core/steam/engine/node-source-test.cpp b/tests/core/steam/engine/node-input-test.cpp similarity index 86% rename from tests/core/steam/engine/node-source-test.cpp rename to tests/core/steam/engine/node-input-test.cpp index 4e3f20a23..bee705175 100644 --- a/tests/core/steam/engine/node-source-test.cpp +++ b/tests/core/steam/engine/node-input-test.cpp @@ -1,5 +1,5 @@ /* - NodeSource(Test) - unit test of the source reading render node + NodeInput(Test) - unit test of the source reading render node Copyright (C) Lumiera.org 2008, Hermann Vosseler @@ -20,8 +20,8 @@ * *****************************************************/ -/** @file node-source-test.cpp - ** unit test \ref NodeSource_test +/** @file node-input-test.cpp + ** unit test \ref NodeInput_test */ @@ -42,7 +42,7 @@ namespace test { /***************************************************************//** * @test the source reading render node. */ - class NodeSource_test : public Test + class NodeInput_test : public Test { virtual void run(Arg) { @@ -52,7 +52,7 @@ namespace test { /** Register this test class... */ - LAUNCHER (NodeSource_test, "unit node"); + LAUNCHER (NodeInput_test, "unit node"); diff --git a/tests/core/steam/engine/node-linkage-test.cpp b/tests/core/steam/engine/node-linkage-test.cpp new file mode 100644 index 000000000..971f9822e --- /dev/null +++ b/tests/core/steam/engine/node-linkage-test.cpp @@ -0,0 +1,59 @@ +/* + NodeLinkage(Test) - verify proper render node operation and calldown + + Copyright (C) Lumiera.org + 2009, Hermann Vosseler + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 2 of + the License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + +* *****************************************************/ + +/** @file node-linkage-test.cpp + ** unit test \ref NodeLinkage_test + */ + + +#include "lib/test/run.hpp" +//#include "lib/util.hpp" + + +//using std::string; + + +namespace steam { +namespace engine{ +namespace test { + + + + + /***************************************************************//** + * @test check render node operation modes and collaboration. + */ + class NodeLinkage_test : public Test + { + virtual void run(Arg) + { + UNIMPLEMENTED ("operate some render nodes as linked together"); + } + }; + + + /** Register this test class... */ + LAUNCHER (NodeLinkage_test, "unit node"); + + + +}}} // namespace steam::engine::test diff --git a/wiki/renderengine.html b/wiki/renderengine.html index 04e69c427..dbb906736 100644 --- a/wiki/renderengine.html +++ b/wiki/renderengine.html @@ -6183,7 +6183,7 @@ This is the core service provided by the player subsystem. The purpose is to cre :any details of this processing remain opaque for the clients; even the player subsystem just accesses the EngineFaçade -
+
//Integration effort to promote the development of rendering, playback and video display in the GUI//
 This IntegrationSlice was started in {{red{2023}}} as [[Ticket #1221|https://issues.lumiera.org/ticket/1221]] to coordinate the completion and integration of various implementation facilities, planned, drafted and built during the last years; this effort marks the return of development focus to the lower layers (after years of focussed UI development) and will implement the asynchronous and time-bound rendering coordinated by the [[Scheduler]] in the [[Vault|Vault-Layer]]
 
@@ -6206,17 +6206,22 @@ __May.23__: taking a //prototyping approach// now, since further development was
 * ✔ implement a »~Split-Splice« algorithm for &rarr; SegmentationChange, rigged accordingly to generate a mocked Segementation for now
 * ✔ create a testbed to assemble a JobPlanningPipeline step by step (&rarr; [[#920|https://issues.lumiera.org/ticket/920]] and [[#1275|https://issues.lumiera.org/ticket/1275|]])
 
-__June23__: building upon this prototyping approach, the dispatcher pipeline could be rearranged in the form of a pipeline builder, allowing to retract the originally used implementation scheme based on »Monads«. The implementation of the Dispatcher is complete, yet the build up of the [[»Render Drive« #1301|https://issues.lumiera.org/ticket/1301]] could not reasonably be completed, due to lack of a clear-shaped ''Scheduler interface''.
+__June.23__: building upon this prototyping approach, the dispatcher pipeline could be rearranged in the form of a pipeline builder, allowing to retract the originally used implementation scheme based on »Monads«. The implementation of the Dispatcher is complete, yet the build up of the [[»Render Drive« #1301|https://issues.lumiera.org/ticket/1301]] could not reasonably be completed, due to lack of a clear-shaped ''Scheduler interface''.
 
-__July23__: this leads to a shift of work focus towards implementing the [[Scheduler]] itself.
+__July.23__: this leads to a shift of work focus towards implementing the [[Scheduler]] itself.
 The Scheduler will be structured into two Layers, where the lower layer is implemented as //priority queue// (using the STL). So the most tricky part to solve is the representation of //dependencies// between jobs, with the possible extension to handling IO operations asynchronously. Analysis and planning of the implementation indicate that the [[scheduler memory managment|SchedulerMemory]] can be based on //Extents//, which are interpreted as »Epochs« with a deadline. These considerations imply what steps to take next for building up Scheduler functionality and memory management required for processing a simple job
 * ✔ build a first working draft for the {{{BlockFlow}}} allocation scheme [[#1311|https://issues.lumiera.org/ticket/1311]]
 * ✔ define and cover the basic [[Activities|RenderActivity]] necessary to implement a plain-simple-Job (without dependencies)
 * ✔ pass such an Activity through the two layers of the Scheduler
-* 🗘 establish a baseline for //performance measurements//
+* ✔ establish a baseline for //performance measurements//
 * ⌛ adapt the [[job planning pipeline|JobPlanningPipeline]] implemented thus far to produce the appropriate {{{Activity}}} records for the scheduler
 
-__December23__: building the Scheduler required some time and dedication, including some related topics like a [[dedicated memory management scheme|SchedulerMemory]], rework and modernisation of the [[#1279 thread handling framework|https://issues.lumiera.org/ticket/1279]], using a [[worker pool|SchedulerWorker]] and developing the [[foundation for load control|SchedulerLoadControl]]. This amounts to the creation of a considerable body of new code; some &rarr;[[load- and stress testing|SchedulerTest]] helps to establish &rarr;[[performance characteristics and traits|SchedulerBehaviour]].
+__December.23__: building the Scheduler required time and dedication, including some related topics like a [[suitable memory management scheme|SchedulerMemory]], rework and modernisation of the [[#1279 thread handling framework|https://issues.lumiera.org/ticket/1279]], using a [[worker pool|SchedulerWorker]] and developing the [[foundation for load control|SchedulerLoadControl]]. This amounts to the creation of a considerable body of new code; some &rarr;[[load- and stress testing|SchedulerTest]] helps to establish &rarr;[[performance characteristics and traits|SchedulerBehaviour]].
+
+__April.24__: after completing an extended round of performance tests for the new Scheduler, development focus is shifted now shifted upwards to the [[Render Node Network|ProcNode]], where Engine activity is carried out. This part was addressed at the very start of the project, and later again -- yet could never be completed, due to a lack of clear reference points and technical requirements. Hope to achieve a breakthrough rests on this integration effort now.
+* 🗘 establish a test setup for developing render node functionality
+* ⌛ introduce a middle layer for linking the JobTicket to the actual invocation
+* ⌛ rework and complete the existing node invocation code
 
 !Decisions
 ;Scheduler
@@ -7382,19 +7387,21 @@ A special operational condition mentioned above is the ''Scheduler Emergency''.
 &rarr; [[Rendering]]
-
+
The [[Scheduler]] is responsible for geting the individual [[render jobs|RenderJob]] to run. The basic idea is that individual render jobs //should never block// -- and thus the calculation of a single frame might be split into several jobs, including resource fetching. This, together with the data exchange protocol defined for the OutputSlot, and the requirements of storage management (especially releasing of superseded render nodes &rarr; FixtureStorage), leads to certain requirements to be ensured by the scheduler:
 ;ordering of jobs
 :the scheduler has to ensure all prerequisites of a given job are met
 ;job time window
-:when it's not possible to run a job within the defined target time window, it must be marked as failure
-;failure propagation
-:when a job fails, either due to an job internal error, or by timing glitch, any dependent jobs need to receive that failure state
+:when it's not possible to run a job within the defined target time window, it should be silently dropped
+;event propagation
+:when some job is done, or due to an external event, a //notification// must be propagated to dependent jobs
 ;guaranteed execution
-:some jobs are marked as "ensure run". These need to run reliable, even when prerequisite jobs fail -- and this failure state needs to be propagated
+:some jobs are marked as »compulsory«. These need to run reliable, even when prerequisite jobs fail -- and this failure state needs to be propagated
+;superseding groups of jobs
+:jobs can be marked as belonging to some group or stratum (called the »manifestation«); such a group can be masked out at a central registry, causing all the affected jobs to be //silently dropped//.
 
 !detecting termination
-The way other parts of the system are built, requires us to obtain a guaranteed knowledge of some job's termination. It is possible to obtain that knowledge with some limited delay, but it nees to be absoultely reliable (violations leading to segfault). The requirements stated above assume this can be achieved through //jobs with guaranteed execution.// Alternatively we could consider installing specific callbacks -- in this case the scheduler itself has to guarantee the invocation of these callbacks, even if the corresponding job fails or is never invoked. It doesn't seem there is any other option.
+The way other parts of the system are built, requires us to obtain a guaranteed knowledge of some job's termination. It is possible to obtain that knowledge with some limited delay, but propagation of this information needs to be absolutely reliable (violations leading either to segfault, or memory leaks). The requirements stated above assume this can be achieved through a combination of //compulsory jobs// and //notifications.// Alternatively we could consider using a separate messaging system within the engine (the »EngineObserver«), which shall also be used for timing observations, higher level capacity management and for signaling exceptional operational failures.
diff --git a/wiki/thinkPad.ichthyo.mm b/wiki/thinkPad.ichthyo.mm index 3e8ff5fd6..7d9bcab76 100644 --- a/wiki/thinkPad.ichthyo.mm +++ b/wiki/thinkPad.ichthyo.mm @@ -79513,12 +79513,12 @@ Date:   Thu Apr 20 18:53:17 2023 +0200
- +

- ...damit sie das Grooming-Token hat und damit Zugriff auf die Scheduler-Ressourcen. Außerdem ist damit ein konsitenter Rahmen sichergestellt: geplante Startzeiten sollten frühestend 20ms danach beginnen + ...damit sie das Grooming-Token hat und damit Zugriff auf die Scheduler-Ressourcen. Außerdem ist damit ein konsitenter Rahmen sichergestellt: geplante Startzeiten sollten frühestens 20ms danach beginnen und können (einschließlich Deadline) höchstens ein Fenster von ~20 Sekunden überspannen

@@ -79978,12 +79978,329 @@ Date:   Thu Apr 20 18:53:17 2023 +0200
 + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + +

+ ...mit der bin ich nämlich heute überhaupt nicht mehr d'accord... +

+ + + + + + + + + + + + +

+ insofern wichtig für einen 2.Anlauf +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

+ Das erscheint mir als eine wichtige neue Einsicht: wir haben keine Zweiteilung (in Planung vs. Ausführung), sondern eine dreistufige Organisation: +

+
    +
  • + Builder ⟼ Render-Node network +
    • +
  • + Job-Planning ⟼ JobTicket + Closure +
    • +
  • + Invocation ⟼ pull-linkage +
    • +
+ + + + + + + +

+ beinhaltet all die Verbindungen, die bis zuletzt flexibel bleiben müssen und für jeden einzelnen Aufruf durchgegangen werden, um die korrekten Verbindungen herzustellen; beispielsweise Verbindungen, die noch von einem Schalter oder von Automation abhängen, oder Verzweigung infolge des Cache-Zustands +

+ + + + + + + + + + + + + +

+ ...denn das wäre im direkten Widerspruch zu dem Builder-Ansatz; denn klassischerweise versteht man unter »pull-processing« eine lazy evaluation, eine late discovery — diese zielt im Kern darauf, eben genau keine erschöpfende Planung vorneweg zu machen. Das »pull principle« ist ein eleganter Implementierungs-Ansatz, der erstaunlich komplexe Funktionalität abbilden kann, ohne eigentlich vorher eine strukturierte Analyse und Vorbereitung zu machen; jedes Teil definiert nur seine Vorraussetzungen, und auf geheimnisvolle Weise findet die Berechnung ihren Weg, wenn man nur lange genung „saugt“. Tatsächlich ist es mir sogar im Gegenteil ein Anliegen, überhaupt nichts mehr zur Laufzeit „herauszufinden“. Ich ziele vielmehr auf eine komplette Durchdringung in einem vorgelagerten Compilations-Prozeß. Die genauere Analyse hat mich dorthin gebracht: ein klassisches »pull-processing« führt nämlich zu einem chaotischen Suchverhalten und erfordert zumindest dynamische Allokation, besser noch sogar einen Garbage-Collector. Und — was für mich noch viel wichtiger geworden ist — es verteilt die Entscheidungs-Logik über den Ausführungs-Pfad, und zwar in Form von cleveren Tricks. Kaum etwas verachte ich mehr, als einen solchen Programmierstil. +

+

+ Allerdings — zwei Aspekte bleiben von diesem Ansatz dennoch erhalten: +

+
    +
  • + der rekursive depth-first-Aufruf, und damit der Verzicht auf ein vorausgeplantes Daten-Management +
    • +
  • + die Analyse-Richtung rückwärts vom Ergebnis zu den Vorraussetzungen (wenngleich auch dies bereits im Builder passiert) +
    • +
+ + + + + + + + + +

+ ...so im Sinn eines »Node-Editors« — eine Node die eigentlich ein plug-in ist, und „alles machen“ kann; das würde nämlich voraussetzen, daß alle Nodes irgendwie miteinander kompatibel sind oder automatisch aneinander angepaßt werden — das wäre etwas, was wir hier ganz explizit nicht wollen, sondern bereits im Builder abhandeln +

+ + + + + + + + +

+ ...der erste Entwurf war viel zu sehr von der Idee einer „geschickten“ Implementierung geprägt; daher war vorgesehen, direkt auf eine »high-level«-Logik durchzugreifen — im Besonderen auch im Hinblick auf die Automation. Von diesem Ansatz habe ich mich inzwischen komplett gelöst; vielmehr setze ich auf symbolische Repräsentation, die dann die nächsten Übersetzungs-Schritte ansteuert, und so schrittweise in eine vollständige Parametrisierung und Konfiguration des Rechenvorgangs mündet +

+ + + + + + + + +

+ ...im ersten Entwurf dachte ich, daß der Kern der Berechnung aus dem direkten Aufruf von C-Funktionen bestehen muß, weil dadurch die Berechnung maximal performant wird; daraus ergab sich auch, daß der »pull-call« direkt ein Array von Buffer(pointern) weitergibt. Das hat sich auf mehreren Ebenen als naiv herausgestellt. Zunächst einmal, die Berechnung soll stets von einem externen plug-in oder einer Library übernommen werden; daher ist klar, daß jeder Aufruf zwangsläufig durch einen Adapter-Layer geht. Demzufolge wird es im »pull-call« stets darum gehen, gewisse Handles oder andere symbolische Repräsentationen weiterzuleiten, und die vorgeblich besonders performante low-Leveligkeit ist nichts als Augenwischerei, denn die Laufzeit wird ohnehin in der Berechnungsfunktion verbracht. Viel wichtiger ist es, explizite Auswertungen zur Laufzeit zu vermeiden, sofern diese bereits vorneweg für ein gesamtes Segment gemacht werden können. +

+ + + + + + + + + + + + + + +

+ es besteht (weiterhin) ein bedenkliches Henne-oder-Ei-Problem +

+ + + + + + + + + + + + + + + + +

+ ...weil es so stringent ist und man direkt bei der Compilation eine Rückmeldung bekommt; aufgrund der fehlenden Außen-Verankerung habe ich lange gar nicht gemerkt, daß die compiletime vs runtime division  eine Hürde darstellt, die die verlockende Idee praktisch unmöglich macht. Andererseits habe ich hier erstmals die sehr fruchbare Idee des compiletime-Layering angewendet, konnte damals aber nicht richtig damit umgehen. Inzwischen habe ich die Möglichkeiten und Grenzen dieser Idee verstanden und erfolgreich angewandt (für die GenNode, das Diff-System, die Event-Log-Suche und den IterExplorer) +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -118119,6 +118436,7 @@ std::cout << tmpl.render({"what", "World"}) << s + @@ -118177,6 +118495,41 @@ std::cout << tmpl.render({"what", "World"}) << s + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +