From c05a73d68aae49ec28bc1c9e4b431de439422fae Mon Sep 17 00:00:00 2001 From: Ichthyostega Date: Sat, 18 Apr 2026 23:49:12 +0200 Subject: [PATCH] BufferProvider: draft mechanism for buffer metadata synchronisation Since the intention is to run several thread-local instances of `BufferMetadata` in a concurrent setup, we need a way to publish and exchange chains of registered type-keys and metadata entries. The first step is to develop a mechanic for data exchange, and to define the rules when such synchronisation is (not) allowed. --- src/vault/mem/buffer-metadata.hpp | 81 ++++++-- src/vault/mem/local-buffer-stage.hpp | 141 ++++++++++++++ tests/47engine.tests | 5 + tests/vault/mem/local-buffer-stage-test.cpp | 195 ++++++++++++++++++++ wiki/thinkPad.ichthyo.mm | 116 +++++++++++- 5 files changed, 519 insertions(+), 19 deletions(-) create mode 100644 src/vault/mem/local-buffer-stage.hpp create mode 100644 tests/vault/mem/local-buffer-stage-test.cpp diff --git a/src/vault/mem/buffer-metadata.hpp b/src/vault/mem/buffer-metadata.hpp index def1c6d70..e825a4c75 100644 --- a/src/vault/mem/buffer-metadata.hpp +++ b/src/vault/mem/buffer-metadata.hpp @@ -58,12 +58,12 @@ #include "lib/error.hpp" #include "lib/hash-value.h" -#include "lib/util-foreach.hpp" #include "include/logging.h" #include "vault/mem/buffhandle.hpp" #include "vault/mem/type-handler.hpp" #include "vault/mem/buffer-local-tag.hpp" #include "lib/nocopy.hpp" +#include "lib/util.hpp" #include @@ -72,7 +72,7 @@ namespace vault{ namespace mem { using lib::HashVal; - using util::for_each; + using util::unConst; namespace err = lumiera::error; @@ -98,6 +98,18 @@ namespace mem { BLOCKED ///< allocated buffer blocked by protocol failure }; + inline bool + isValidBufferTransition (BufferState oldState, BufferState newState) + { + return ( (oldState == FREE and newState == LOCKED) + or (oldState == LOCKED and newState == EMITTED) + or (oldState == LOCKED and newState == BLOCKED) + or (oldState == LOCKED and newState == FREE) + or (oldState == EMITTED and newState == BLOCKED) + or (oldState == EMITTED and newState == FREE) + or (oldState == BLOCKED and newState == FREE) + ); + } @@ -357,15 +369,8 @@ namespace mem { { __must_not_be_NIL(); - if ( (state_ == FREE and newState == LOCKED) - or (state_ == LOCKED and newState == EMITTED) - or (state_ == LOCKED and newState == BLOCKED) - or (state_ == LOCKED and newState == FREE) - or (state_ == EMITTED and newState == BLOCKED) - or (state_ == EMITTED and newState == FREE) - or (state_ == BLOCKED and newState == FREE)) - { - // allowed transition + if (isValidBufferTransition (state_, newState)) + { // perform transition if (newState == FREE) invokeEmbeddedDtor_and_clear(); if (newState == LOCKED) @@ -534,7 +539,8 @@ namespace mem { verify_all_buffers_freed() try { - for_each (entries_, verify_is_free); + for (auto const& entry : entries_) + verify_is_free(entry); } ERROR_LOG_AND_IGNORE (engine,"Shutdown of BufferProvider metadata store") @@ -671,6 +677,12 @@ namespace mem { return *entry; } + Entry const& + get (HashVal hashID) const + { + return unConst(this)->get (hashID); + } + bool isKnown (HashVal key) const { @@ -766,6 +778,22 @@ namespace mem { } + /** Synchronise metadata, including the key path up to root + * @param key start point of data exchange + * @param srcReg another BufferMetadata registry from which + * metadata shall be retrieved and imported + */ + void + import (HashVal key, BufferMetadata const& srcReg) + { + if (not srcReg.isKnown (key)) + return; + Entry const& entry{srcReg.get(key)}; + doSynchronise (entry); + import (entry.parentKey(), srcReg); + } + + private: /** Core low-level operation to access or create a buffer metadata entry. * The hashID of the entry in question is built, based on the parentKey, @@ -861,6 +889,35 @@ namespace mem { } return newEntry; } + + + /** import data from another metadata entry, + * without violating sanity rules + * @remark usually this is used to import new type-keys; + * an active buffer entry can only be handled if it is new, + * or implies a valid state transition. + * @note if the buffer is not NULL, it must be the same address, + * otherwise the keys would differ, since they are generated + * with \ref Key::forEntry() + */ + void + doSynchronise (Entry const& otherEntry) + { + Entry* entry = table_.fetch (otherEntry); + if (not entry) + table_.store (otherEntry); + else + if (not otherEntry.isTypeKey() + and otherEntry.state() != entry->state()) + { + ASSERT (not entry->isTypeKey(), "Metadata State-Engine corrupted"); + if ( otherEntry.state() == FREE + or otherEntry.state() == LOCKED) + throw err::Logic ("Metadata synchronisation can not imply allocation/deallocation."); + else + entry->mark(otherEntry.state()); + } + } }; diff --git a/src/vault/mem/local-buffer-stage.hpp b/src/vault/mem/local-buffer-stage.hpp new file mode 100644 index 000000000..854fe63f5 --- /dev/null +++ b/src/vault/mem/local-buffer-stage.hpp @@ -0,0 +1,141 @@ +/* + SIMPLE-BUFFER-STATE-REGISTRY.hpp - buffer state management with thread-local sub-services + + Copyright (C) + 2026 Hermann Vosseler + +  **Lumiera** is free software; you can redistribute it and/or modify it +  under the terms of the GNU General Public License as published by the +  Free Software Foundation; either version 2 of the License, or (at your +  option) any later version. See the file COPYING for further details. + +*/ + +/** @file local-buffer-stage.hpp + ** Production-grade implementation of engine::BufferProvider state coordination. + ** Requests to the BufferProvider will be issued from all threads concurrently, + ** which requires to distinguish between known common state and particular state + ** relevant only for the current worker. Whenever a _buffer type_ is registered, + ** this information could become relevant later, when actually a buffer is locked + ** with this type, and such a request for an actual buffer can be encountered + ** in any thread. While, on the other hand, the state transitions for actual + ** buffer usage will happen within a single job invocation, and can thus be + ** handled locally within the corresponding worker thread. An allocated buffer + ** that is passed into the frame cache is yet another story, and will be handled + ** by a dedicated _cache provider_. + ** + ** Together these observations lead to a structure with locally replicated copies + ** of the buffer metadata store, where additional type registration information + ** is requested from the central metadata hub on-demand. Yet the front-end for + ** accessing the \ref BufferProvider is a common service of the Render Engine, + ** and thus the switch to the thread-local metadata management has to happen + ** somewhere in-between. This can be achieved by connecting the BufferMetadata + ** as a \ref lib::LocalSlice as part of the \ref BufferProvider::BufferStage + ** implementation — as provided by this class, LocalBufferStage. + ** + ** @see buffer-provider.hpp + ** @see buffer-provider-protocol-test.cpp + ** @see simple-buffer-state-registry.hpp + */ + +#ifndef VAULT_MEM_LOCAL_BUFFER_STAGE_H +#define VAULT_MEM_LOCAL_BUFFER_STAGE_H + + +#include "vault/mem/buffer-metadata.hpp" +#include "vault/mem/buffer-provider-setup.hpp" + +#include + + +namespace vault { +namespace mem { + + + + /** + * Simple Buffer type and state tracking registry, for test and demonstration. + * Relies on a central hashtable, without considering any concurrency concerns. + */ + class LocalBufferStage + : public BufferProviderSetup::Stage + { + HashVal familyID_; + BufferMetadata metadata_; + + /* === BufferStage interface === */ + + ID + lookup (HashVal key) override + { + return metadata_.isKnown(key)? metadata_.get (key) + : metadata::Key::INVALID; + } + + bool + isAllotted (HashVal stateKey) const override + { + return metadata_.isLocked (stateKey); + } + + bool + isAccessible (HashVal stateKey) const override + { + return metadata_.isAccessible (stateKey); + } + + ID + defineBufferType (size_t buffSiz, TypeHandler handlerFunctions, LocalTag localTag) + { + return lookup (metadata_.key (familyID_, buffSiz, move (handlerFunctions), localTag)); + } // deliberately: maybe create storage, and return reference to it + + ID + mark_locked (ID typeKey, BuffAlloc alloc) override + { + auto& [storage,size,implMark] = alloc; + return metadata_.markLocked (typeKey, storage, size, implMark); + } + + ID + mark_emitted (HashVal stateKey) override + { + metadata::Entry& metaEntry = metadata_.get (stateKey); + metaEntry.mark(EMITTED); + return metaEntry; // contains also the key + } + + ID + mark_released (HashVal stateKey) override + { + metadata::Entry& metaEntry = metadata_.get (stateKey); + metaEntry.mark(FREE); // might invoke embedded dtor function + return metaEntry; + } + + ID + abandon (HashVal stateKey, bool invokeDtor) override + { + metadata::Entry& metaEntry = metadata_.get (stateKey); + metaEntry.invalidate (invokeDtor); + return metaEntry; + } + + void + discard (HashVal stateKey) override + { + metadata_.release (stateKey); + } + + + public: + LocalBufferStage (Literal implementationID) + : familyID_{hash_value (implementationID)} + , metadata_{} + { } + }; + + + +}} // namespace vault::mem +#endif /*VAULT_MEM_LOCAL_BUFFER_STAGE_H*/ diff --git a/tests/47engine.tests b/tests/47engine.tests index 4629fbe60..5cac7d571 100644 --- a/tests/47engine.tests +++ b/tests/47engine.tests @@ -72,6 +72,11 @@ return: 0 END +TEST "Local buffer state management" LocalBufferStage_test < + +  **Lumiera** is free software; you can redistribute it and/or modify it +  under the terms of the GNU General Public License as published by the +  Free Software Foundation; either version 2 of the License, or (at your +  option) any later version. See the file COPYING for further details. + +* *****************************************************************/ + +/** @file local-buffer-stage-test.cpp + ** unit test \ref LocalBufferStage_test + */ + + +#include "lib/error.hpp" +#include "test/run.hpp" +#include "test/test-frame.hpp" +#include "test/test-helper.hpp" +#include "vault/mem/buffhandle.hpp" +#include "vault/mem/buffer-metadata.hpp" +#include "vault/mem/local-buffer-stage.hpp" +#include "lib/symbol.hpp" +#include "lib/util.hpp" + +#include + + +using std::strncpy; +using std::unique_ptr; +using test::TestFrame; +using test::testData; +using util::isSameObject; +using util::isnil; +using lib::randStr; +using lib::Literal; + +namespace vault { +namespace mem { +namespace test { + + using LERR_(LOGIC); +// using LERR_(FATAL); +// using LERR_(INVALID); + using LERR_(LIFECYCLE); + + + namespace { // Test fixture + + template + TY& + accessAs (metadata::Entry& entry) + { + TY* ptr = reinterpret_cast (entry.access()); + ASSERT (ptr); + return *ptr; + } + + template + Buff* + mark_as_Buffer(X& something) + { + return reinterpret_cast (std::addressof(something)); + } + + + const size_t TEST_MAX_SIZE = 1024 * 1024; + + HashVal JUST_SOMETHING = 123; + auto SOME_POINTER = mark_as_Buffer(JUST_SOMETHING); + + }//(End) Test fixture and helpers + + + + + + + /***************************************************************//** + * @test verify mechanism for a thread-local BufferStage implementation + * - verify data synchronisation between several BufferMetadata registries + * - verify typical usage situations, where a thread-local registry + * has to synchronise itself with a central metadata hub. + */ + class LocalBufferStage_test : public Test + { + + virtual void + run (Arg) + { + seedRand(); + verify_DataTransfer(); + verify_publishNewKey(); + verify_fetchNewKeyOnUse(); + } + + + /** @test key-chains can be exported and imported */ + void + verify_DataTransfer() + { + BufferMetadata meta1, meta2; + + const HashVal FAM = ranHash(); + const size_t SIZ_D = sizeof(double); + const double RAND1 = 1.0 + defaultGen.uni(); + + // create one type-key in Registry-1 + metadata::Key keyD = meta1.key (FAM,SIZ_D); + CHECK (meta1.isKnown (keyD)); + CHECK (not meta2.isKnown (keyD)); + CHECK (meta1.get(keyD).isTypeKey()); + + // create another type-key in Registry-2, this time with a constructor + metadata::Key keyDD = meta2.key (FAM,SIZ_D, TypeHandler::create (RAND1)); + CHECK (meta2.isKnown (keyDD)); + CHECK (not meta1.isKnown (keyDD)); + CHECK (meta2.get(keyDD).isTypeKey()); + + // Keys are systematic => KeyD happens to be parent of KeyDD + CHECK (keyD.storageSize() == keyDD.storageSize()); + CHECK (keyDD.parentKey() == HashVal(keyD)); + + // synchronise a sub-type + meta1.import (keyDD, meta2); + CHECK (meta1.isKnown (keyDD)); + + // now Registry-1 is able to use this subtype as well + double testBuffer{0}; + metadata::Entry& eDD = meta1.markLocked (keyDD, mark_as_Buffer(testBuffer)); + CHECK (eDD.state() == LOCKED); + CHECK (eDD.parentKey() == HashVal(keyDD)); + // and the »constructor« was indeed invoked... + CHECK (testBuffer == RAND1); + testBuffer += 1.1; // add marker... + + // within the confines of allowed state transitions, + // it is even possible to sync the state of active buffer entries + CHECK (meta1.isKnown (eDD)); + CHECK (not meta2.isKnown (eDD)); + meta2.import (eDD, meta1); + CHECK (meta2.isKnown (eDD)); + CHECK (meta2.get(eDD).state() == LOCKED); + + // importing the state record did not re-invoke the constructor + CHECK (testBuffer == RAND1 + 1.1); + + meta2.get(eDD).mark(EMITTED); + CHECK (meta2.get(eDD).state() == EMITTED); + CHECK (meta1.get(eDD).state() == LOCKED); + meta1.import (eDD, meta2); + CHECK (meta1.get(eDD).state() == EMITTED); + + // now mark the buffer as released in Registry-1 + CHECK (eDD.state() == EMITTED); + eDD.mark(FREE); + CHECK (eDD.state() == FREE); + CHECK (meta2.get(eDD).state() == EMITTED); + + // it is not allowed to mark de-allocated through synchronisation + VERIFY_ERROR (LOGIC, meta2.import (eDD, meta1) ); + + VERIFY_ERROR (LIFECYCLE, meta2.release(HashVal(eDD)) ); + CHECK (meta2.get(eDD).state() == EMITTED); + meta2.get(eDD).invalidate (false); + CHECK (meta2.get(eDD).state() == FREE); + meta2.release(eDD); + meta1.release(eDD); + } + + + /** @test simulate a standard buffer provider usage cycle + */ + void + verify_publishNewKey() + { + } + + + void + verify_fetchNewKeyOnUse() + { + } + }; + + + /** Register this test class... */ + LAUNCHER (LocalBufferStage_test, "unit engine"); + + + +}}} // namespace vault::mem::test diff --git a/wiki/thinkPad.ichthyo.mm b/wiki/thinkPad.ichthyo.mm index b17a58fa2..93e028432 100644 --- a/wiki/thinkPad.ichthyo.mm +++ b/wiki/thinkPad.ichthyo.mm @@ -144223,13 +144223,35 @@ StM_bind(Builder<R1> b1, Extension<R1,R2> extension) - + + + + + + +

+ Das erfordert Vorsicht, weil man dadurch die Eigenschaft einer geschlossenen Service-Instanz verliert; es ist also leider kein drop-in-replacement, mit dem man einfach eine tief eingebettete Koponente thread-local macht; theoretisch würde letzteres zwar gehen, aber dann auf einen zweistufigen Dispatch hinauslaufen, d.h. nach dem thread-local-Zugriff müßte noch ein Dispatch über eine Map per umschließender Service-Instanz erfolgen. +

+

+ +

+

+ In dem Fall beschränke ich mich aber letztlich auf unsere Anforderungen hier, und da brauchen wir tatsähchlich nur eine globale Instanz — ich hatte mir ja sogar schon überlegt, ob man nicht sogar die BufferMetadata koplett als separaten Service herauslösen müßte; das bekommen wir hier nun als Nebeneffekt geschenkt +

+ + + + + + + - + + @@ -144324,8 +144346,7 @@ StM_bind(Builder<R1> b1, Extension<R1,R2> extension) man muß den aktuellen Namespace schließen, in lib gehen, dort die explizite Spezialisierung definieren...

- - + @@ -144341,8 +144362,7 @@ StM_bind(Builder<R1> b1, Extension<R1,R2> extension) direkt bei LocalSlice definiert; erzeugt eine default-konstruierte Instanz vom Service

- - + @@ -144391,7 +144411,89 @@ StM_bind(Builder<R1> b1, Extension<R1,R2> extension) - + + + + + + + + + + + + + + + + + + +

+ Das bedeutet, die hier aufgestellten Regeln/Verbote sind so konstruiert, daß sie bei einem sinnvollen Abgleich niemals auftreten können.  Der Sinn ergibt sich dabei aus den vorgesehenen Zustandsübergängen; es sollten nur Typ-Informationen angereichert werden, und konkrete Einträge sollten eigentlich gar nicht synchronisiert werden; hilfsweise sehe ich die Möglichkeit vor, daß konkrete Einträge sich gemäß State-Machine vorwärts bewegen (falls wir tatsächlich konkrete Einträge zentral ablegen, wenn sie in den Cache wandern — ich kann diese Möglichkeit derzeit nicht ausschließen) +

+ + + + + + + + + + + + + + +

+ es sollte nur alloziert werden, wenn das auch notwendig ist; man könnte entweder einen Buffer vorsehen, oder direkt von Quelle zu Ziel arbeiten. +

+ + + + + + + + + + +

+ es müssen keine Komponenten gegeneinander isoliert werden; die Schnittstelle dient nur dazu, den vorgang leicht beherrschbar zu machen +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +