benn/lumiera_documentation
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

BufferProvider: expand BuffStore API to provide buffer size info

Browse source 
This is an important step to prepare a split of the existing implementation APIs
into a type-management part and a memory-allocation part; generally speaking,
a memory allocator needs to know the allocation size, preferably also for
releasing memory. Up to now, this information could be retrieved locally,
with a metadata lookup through the hash-ID; now we ensure that the
size is always passed alongside through the implementation API.

Incidentally, it looks like we might be able to form a common allocation-descriptor,
to represent this internal API in a more concise way. Note further that
all these changes are considered not relevant for the performance,
since work buffers are large and media computation dominates.
This commit is contained in:
Fischlurch 2026-02-12 22:25:32 +01:00
parent cd415c05aa
commit 925ee0f27e
5 changed files with 109 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
src/steam/engine/buffer-provider.cpp
View file

@ -107,9 +107,10 @@ namespace engine {
* @note the returned count may differ from the requested count. * @note the returned count may differ from the requested count.
*/ */
uint uint
BufferProvider::announce (uint count, BuffDescr const& type) BufferProvider::announce (uint cnt, BuffDescr const& type)
{ {
uint actually_possible = bufferStore_->prepareBuffers (count, type); size_t buffSiz = getBufferSize (type);
uint actually_possible = bufferStore_->prepareBuffers (cnt, buffSiz, type);
if (!actually_possible) if (!actually_possible)
throw error::State ("unable to fulfil request for buffers" throw error::State ("unable to fulfil request for buffers"
,LUMIERA_ERROR_BUFFER_MANAGEMENT); ,LUMIERA_ERROR_BUFFER_MANAGEMENT);
@ -134,8 +135,8 @@ namespace engine {
BufferProvider::lockBuffer (BuffDescr const& type) BufferProvider::lockBuffer (BuffDescr const& type)
{ {
REQUIRE (was_created_by_this_provider (type)); REQUIRE (was_created_by_this_provider (type));
size_t buffSiz = getBufferSize (type);
return bufferStore_->provideLockedBuffer (type); return bufferStore_->provideBuffer (buffSiz, type);
} // is expected to call buildHandle() --> state transition } // is expected to call buildHandle() --> state transition
@ -155,13 +156,14 @@ namespace engine {
BufferProvider::emitBuffer (BuffHandle const& handle) BufferProvider::emitBuffer (BuffHandle const& handle)
{ {
metadata::Entry& metaEntry = bufferStage_->get (handle.entryID()); metadata::Entry& metaEntry = bufferStage_->get (handle.entryID());
bufferStore_->mark_emitted (metaEntry.parentKey(), metaEntry.localTag()); size_t buffSiz = metaEntry.storageSize();
bufferStore_->mark_emitted (buffSiz, metaEntry.parentKey(), metaEntry.localTag());
metaEntry.mark(EMITTED); metaEntry.mark(EMITTED);
} }
/** BufferProvider API: declare done and detach. /** BufferProvider API: declare done and detach.
* Client code is required to release \em each previously locked buffer eventually. * Client code is required to release _each previously locked buffer_ eventually.
* @warning invalidates the BuffHandle, clients mustn't access the buffer anymore. * @warning invalidates the BuffHandle, clients mustn't access the buffer anymore.
* Right after releasing, an access through the handle will throw; * Right after releasing, an access through the handle will throw;
* yet the buffer might be re-used and the handle become valid * yet the buffer might be re-used and the handle become valid
@ -172,8 +174,9 @@ namespace engine {
BufferProvider::releaseBuffer (BuffHandle const& handle) BufferProvider::releaseBuffer (BuffHandle const& handle)
try { try {
metadata::Entry& metaEntry = bufferStage_->get (handle.entryID()); metadata::Entry& metaEntry = bufferStage_->get (handle.entryID());
size_t buffSiz = metaEntry.storageSize();
metaEntry.mark(FREE); // might invoke embedded dtor function metaEntry.mark(FREE); // might invoke embedded dtor function
bufferStore_->detachBuffer (metaEntry.parentKey(), metaEntry.localTag(), *handle); bufferStore_->detachBuffer (buffSiz, metaEntry.parentKey(), metaEntry.localTag(), *handle);
bufferStage_->release (metaEntry); bufferStage_->release (metaEntry);
} }
ERROR_LOG_AND_IGNORE (engine, "releasing a buffer from BufferProvider") ERROR_LOG_AND_IGNORE (engine, "releasing a buffer from BufferProvider")
@ -216,8 +219,9 @@ namespace engine {
BufferProvider::emergencyCleanup (BuffHandle const& target, bool invokeDtor) BufferProvider::emergencyCleanup (BuffHandle const& target, bool invokeDtor)
try { try {
metadata::Entry& metaEntry = bufferStage_->get (target.entryID()); metadata::Entry& metaEntry = bufferStage_->get (target.entryID());
size_t buffSiz = metaEntry.storageSize();
metaEntry.invalidate (invokeDtor); metaEntry.invalidate (invokeDtor);
bufferStore_->detachBuffer (metaEntry.parentKey(), metaEntry.localTag(), *target); bufferStore_->detachBuffer (buffSiz, metaEntry.parentKey(), metaEntry.localTag(), *target);
bufferStage_->release (metaEntry); bufferStage_->release (metaEntry);
} }
ERROR_LOG_AND_IGNORE (engine, "cleanup of buffer metadata while handling an error") ERROR_LOG_AND_IGNORE (engine, "cleanup of buffer metadata while handling an error")

8
src/steam/engine/buffer-provider.hpp
View file

@ -203,10 +203,10 @@ namespace engine {
public: public:
virtual ~BufferStore() { } ///< this is an interface virtual ~BufferStore() { } ///< this is an interface
virtual uint prepareBuffers (uint count, HashVal typeID) =0; virtual uint prepareBuffers (uint cnt, size_t,HashVal typeID) =0;
virtual BuffHandle provideLockedBuffer (HashVal typeID) =0; virtual BuffHandle provideBuffer (size_t,HashVal typeID) =0;
virtual void mark_emitted (HashVal, LocalTag const&) =0; virtual void mark_emitted (size_t,HashVal, LocalTag const&) =0;
virtual void detachBuffer (HashVal, LocalTag const&, Buff&) =0; virtual void detachBuffer (size_t,HashVal, LocalTag const&, Buff&) =0;
}; };
unique_ptr<BufferStage> bufferStage_; ///////////////////////////////////////////////////////////TICKET #1410 : must be turned into an internal interface unique_ptr<BufferStage> bufferStage_; ///////////////////////////////////////////////////////////TICKET #1410 : must be turned into an internal interface

31
src/steam/engine/heap-mem-buffer-store.cpp
View file

@ -281,17 +281,17 @@ namespace engine {
/* ==== Implementation of the BufferProvider interface ==== */ /* ==== Implementation of the BufferProvider interface ==== */
uint uint
HeapMemProvider::prepareBuffers(uint requestedAmount, HashVal typeID) HeapMemProvider::prepareBuffers (uint numBuffers, size_t buffSiz, HashVal typeID)
{ {
BlockPool& responsiblePool = getBlockPoolFor (typeID); BlockPool& responsiblePool = getBlockPoolFor (buffSiz, typeID);
return responsiblePool.prepare_for (requestedAmount); return responsiblePool.prepare_for (numBuffers);
} }
BuffHandle BuffHandle
HeapMemProvider::provideLockedBuffer(HashVal typeID) HeapMemProvider::provideBuffer (size_t buffSiz, HashVal typeID)
{ {
BlockPool& blocks = getBlockPoolFor (typeID); BlockPool& blocks = getBlockPoolFor (buffSiz, typeID);
Block& newBlock = blocks.createBlock(); Block& newBlock = blocks.createBlock();
UNIMPLEMENTED ("Separate tasks into two distinct APIs"); UNIMPLEMENTED ("Separate tasks into two distinct APIs");
// return buildHandle (typeID, asBuffer(newBlock.accessMemory()), &newBlock); /////////////////////////////OOO instead return a tuple (buffer, localTag) // return buildHandle (typeID, asBuffer(newBlock.accessMemory()), &newBlock); /////////////////////////////OOO instead return a tuple (buffer, localTag)
@ -299,13 +299,13 @@ namespace engine {
void void
HeapMemProvider::mark_emitted (HashVal typeID, LocalTag const& specifics) HeapMemProvider::mark_emitted (size_t buffSiz, HashVal typeID, LocalTag const& specifics)
{ {
Block* block4buffer = locateBlock (typeID, specifics); Block* block4buffer = locateBlock (buffSiz, typeID, specifics);
if (!block4buffer) if (!block4buffer)
throw error::Logic ("Attempt to emit a buffer not known to this BufferProvider" throw error::Logic ("Attempt to emit a buffer not known to this BufferProvider"
, LUMIERA_ERROR_BUFFER_MANAGEMENT); , LUMIERA_ERROR_BUFFER_MANAGEMENT);
BlockPool& pool = getBlockPoolFor (typeID); BlockPool& pool = getBlockPoolFor (buffSiz, typeID);
Block* active = pool.transferResponsibility (block4buffer); Block* active = pool.transferResponsibility (block4buffer);
if (active) if (active)
outSeq_.manage (active); outSeq_.manage (active);
@ -320,9 +320,9 @@ namespace engine {
/** mark a buffer as officially discarded */ /** mark a buffer as officially discarded */
void void
HeapMemProvider::detachBuffer (HashVal typeID, LocalTag const& specifics, Buff& storage) HeapMemProvider::detachBuffer (size_t buffSiz, HashVal typeID, LocalTag const& specifics, Buff& storage)
{ {
Block* block4buffer = locateBlock (typeID, specifics); Block* block4buffer = locateBlock (buffSiz, typeID, specifics);
REQUIRE (block4buffer, "releasing a buffer not allocated through this provider"); REQUIRE (block4buffer, "releasing a buffer not allocated through this provider");
REQUIRE (util::isSameAdr (storage, block4buffer->accessMemory())); REQUIRE (util::isSameAdr (storage, block4buffer->accessMemory()));
block4buffer->markReleased(); block4buffer->markReleased();
@ -364,19 +364,18 @@ namespace engine {
} }
HeapMemProvider::BlockPool& HeapMemProvider::BlockPool&
HeapMemProvider::getBlockPoolFor (HashVal typeID) HeapMemProvider::getBlockPoolFor (size_t buffSiz, HashVal typeID)
{ {
BlockPool& pool = (*pool_)[typeID]; BlockPool& pool = (*pool_)[typeID];
UNIMPLEMENTED ("Separate tasks into two distinct APIs"); if (not pool)
// if (!pool) pool.initialise (buffSiz);
// pool.initialise(getBufferSize(typeID)); //////////////////////////OOO getBufferSize is a helper function in the BufferProvider protected API. Need to be "here"
return pool; return pool;
} }
HeapMemProvider::Block* HeapMemProvider::Block*
HeapMemProvider::locateBlock (HashVal typeID, void* storage) HeapMemProvider::locateBlock (size_t buffSiz, HashVal typeID, void* storage)
{ {
BlockPool& pool = getBlockPoolFor (typeID); BlockPool& pool = getBlockPoolFor (buffSiz, typeID);
Block* block4buffer = pool.find (storage); ////////////////////////////////TICKET #856 Block* block4buffer = pool.find (storage); ////////////////////////////////TICKET #856
return block4buffer? block4buffer return block4buffer? block4buffer
: searchInOutSeqeuence (storage); : searchInOutSeqeuence (storage);

12
src/steam/engine/heap-mem-buffer-store.hpp
View file

@ -83,10 +83,10 @@ namespace engine {
public: public:
/* === BufferStore interface === */ /* === BufferStore interface === */
virtual uint prepareBuffers (uint count, HashVal typeID) override; virtual uint prepareBuffers (uint cnt, size_t,HashVal typeID) override;
virtual BuffHandle provideLockedBuffer (HashVal typeID) override; virtual BuffHandle provideBuffer (size_t,HashVal typeID) override;
virtual void mark_emitted (HashVal, LocalTag const&) override; virtual void mark_emitted (size_t,HashVal, LocalTag const&) override;
virtual void detachBuffer (HashVal, LocalTag const&, Buff&) override; virtual void detachBuffer (size_t,HashVal, LocalTag const&, Buff&) override;
public: public:
~HeapMemProvider(); ~HeapMemProvider();
@ -103,8 +103,8 @@ namespace engine {
private: private:
bool withinOutputSequence (uint bufferID) const; bool withinOutputSequence (uint bufferID) const;
BlockPool& getBlockPoolFor (HashVal typeID); BlockPool& getBlockPoolFor (size_t,HashVal typeID);
Block* locateBlock (HashVal typeID, void*); Block* locateBlock (size_t,HashVal typeID, void*);
Block* searchInOutSeqeuence (void* storage); Block* searchInOutSeqeuence (void* storage);
}; };

73
wiki/thinkPad.ichthyo.mm
View file

@ -111662,7 +111662,8 @@ StM_bind(Builder&lt;R1&gt; b1, Extension&lt;R1,R2&gt; extension)
<icon BUILTIN="broken-line"/> <icon BUILTIN="broken-line"/>
</node> </node>
</node> </node>
<node BACKGROUND_COLOR="#f8f1cb" COLOR="#a50125" CREATED="1770771226721" ID="ID_1487577652" MODIFIED="1770771248247" TEXT="auch der DiagnosticBufferProvider verwendet ein Basis_API: getBufferSize()"> <node BACKGROUND_COLOR="#f8f1cb" COLOR="#a50125" CREATED="1770771226721" ID="ID_1487577652" MODIFIED="1770929553915" TEXT="die Provider-Implementierungen verwendet ein Basis_API: getBufferSize()">
<linktarget COLOR="#fd583f" DESTINATION="ID_1487577652" ENDARROW="Default" ENDINCLINATION="-484;20;" ID="Arrow_ID_1256483946" SOURCE="ID_831737885" STARTARROW="None" STARTINCLINATION="-70;-57;"/>
<icon BUILTIN="messagebox_warning"/> <icon BUILTIN="messagebox_warning"/>
</node> </node>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1770770126934" ID="ID_850948251" MODIFIED="1770770161242" TEXT="&#x27f9; API-Anpassung notwendig"> <node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1770770126934" ID="ID_850948251" MODIFIED="1770770161242" TEXT="&#x27f9; API-Anpassung notwendig">
@ -111687,6 +111688,14 @@ StM_bind(Builder&lt;R1&gt; b1, Extension&lt;R1,R2&gt; extension)
<node BACKGROUND_COLOR="#fdfdcf" COLOR="#ff0000" CREATED="1770770350662" ID="ID_1511113078" MODIFIED="1770835721461" TEXT="L&#xf6;sung ist offensichtlich: auseinander-f&#xe4;deln"> <node BACKGROUND_COLOR="#fdfdcf" COLOR="#ff0000" CREATED="1770770350662" ID="ID_1511113078" MODIFIED="1770835721461" TEXT="L&#xf6;sung ist offensichtlich: auseinander-f&#xe4;deln">
<linktarget COLOR="#fe3e3a" DESTINATION="ID_1511113078" ENDARROW="Default" ENDINCLINATION="243;15;" ID="Arrow_ID_1668459979" SOURCE="ID_661880079" STARTARROW="None" STARTINCLINATION="636;0;"/> <linktarget COLOR="#fe3e3a" DESTINATION="ID_1511113078" ENDARROW="Default" ENDINCLINATION="243;15;" ID="Arrow_ID_1668459979" SOURCE="ID_661880079" STARTARROW="None" STARTINCLINATION="636;0;"/>
<icon BUILTIN="yes"/> <icon BUILTIN="yes"/>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1770929471826" ID="ID_472477187" MODIFIED="1770934001172" TEXT="Buffer-Store rein auf die Allokationen fokussieren">
<arrowlink COLOR="#616d7c" DESTINATION="ID_1572913456" ENDARROW="Default" ENDINCLINATION="433;33;" ID="Arrow_ID_1937407092" STARTARROW="None" STARTINCLINATION="1162;357;"/>
<linktarget COLOR="#692bc7" DESTINATION="ID_472477187" ENDARROW="Default" ENDINCLINATION="304;203;" ID="Arrow_ID_396413173" SOURCE="ID_756237025" STARTARROW="None" STARTINCLINATION="347;19;"/>
<icon BUILTIN="flag-yellow"/>
<node CREATED="1770932643675" ID="ID_1799354299" MODIFIED="1770932643675" TEXT="provideBuffer (size_t,HashVal typeID)"/>
<node CREATED="1770933709677" MODIFIED="1770933709677" TEXT="prepareBuffers (uint cnt, size_t,HashVal typeID)"/>
<node CREATED="1770933720241" MODIFIED="1770933720241" TEXT="detachBuffer (size_t,HashVal, LocalTag const&amp;, Buff&amp;)"/>
</node>
</node> </node>
</node> </node>
</node> </node>
@ -111836,6 +111845,56 @@ StM_bind(Builder&lt;R1&gt; b1, Extension&lt;R1,R2&gt; extension)
<arrowlink COLOR="#fe3e3a" DESTINATION="ID_1511113078" ENDARROW="Default" ENDINCLINATION="243;15;" ID="Arrow_ID_1668459979" STARTARROW="None" STARTINCLINATION="636;0;"/> <arrowlink COLOR="#fe3e3a" DESTINATION="ID_1511113078" ENDARROW="Default" ENDINCLINATION="243;15;" ID="Arrow_ID_1668459979" STARTARROW="None" STARTINCLINATION="636;0;"/>
<icon BUILTIN="flag-pink"/> <icon BUILTIN="flag-pink"/>
</node> </node>
<node BACKGROUND_COLOR="#fdfdcf" COLOR="#ff0000" CREATED="1770926260075" ID="ID_831737885" MODIFIED="1770929560653" TEXT="interner Zugriff auf buffer-size">
<arrowlink COLOR="#fd583f" DESTINATION="ID_1487577652" ENDARROW="Default" ENDINCLINATION="-484;20;" ID="Arrow_ID_1256483946" STARTARROW="None" STARTINCLINATION="-70;-57;"/>
<icon BUILTIN="flag-pink"/>
<node BACKGROUND_COLOR="#f0d5c5" COLOR="#990033" CREATED="1770926385056" ID="ID_1327594403" MODIFIED="1770926436770" TEXT="verwendet von getBlockPoolFor(HashVal) &#x27f5; in sehr vielen Aufrufen der erste Schritt">
<icon BUILTIN="clanbomber"/>
</node>
<node BACKGROUND_COLOR="#f8f1cb" COLOR="#a50125" CREATED="1770926280226" ID="ID_1629789719" MODIFIED="1770926305170" TEXT="bisher &#xfc;ber Hilfsfunktion auf der BufferProvider Basis-impl">
<icon BUILTIN="messagebox_warning"/>
<node CREATED="1770926311965" ID="ID_1256083121" MODIFIED="1770926337047" TEXT="dies verletzt jetzt die Verschachtelung"/>
<node CREATED="1770926338187" ID="ID_788003404" MODIFIED="1770926343457" TEXT="und die Trennung der Themen"/>
<node CREATED="1770926346025" ID="ID_817787716" MODIFIED="1770926361362" TEXT="ist n&#xe4;mlich implementiert durch Zugriff auf die Type-Registry"/>
</node>
<node CREATED="1770929023368" ID="ID_448168342" MODIFIED="1770929048141" TEXT="Einsch&#xe4;tzung: mu&#xdf; ohnehin ge&#xe4;ndert werden">
<icon BUILTIN="yes"/>
<node BACKGROUND_COLOR="#e0ceaa" COLOR="#690f14" CREATED="1770929095659" ID="ID_374106644" MODIFIED="1770929259555" TEXT="wir vermerken die Size nur einmal: im Typ-descriptor">
<richcontent TYPE="NOTE"><html>
<head>
</head>
<body>
<p>
...hier liegt ein potentiell gef&#228;hrlicher Hebel &#8212; wenngleich auch im Fall von Buffern vermutlich nicht besonders relevant (da Buffer gro&#223; sind)
</p>
</body>
</html>
</richcontent>
<icon BUILTIN="messagebox_warning"/>
</node>
<node CREATED="1770929050952" ID="ID_1844454362" MODIFIED="1770929066192" TEXT="ein typischer Allokator mu&#xdf; die Gr&#xf6;&#xdf;e wissen">
<node CREATED="1770929067315" ID="ID_1319809000" MODIFIED="1770929074957" TEXT="vor allem auch zur de-Allokation"/>
<node CREATED="1770929216403" ID="ID_1022871277" MODIFIED="1770929358425" TEXT="ein bekanntes Problem mit vielen Workarounds und Kniffen">
<richcontent TYPE="NOTE"><html>
<head>
</head>
<body>
<p>
F&#252;r small-objects-Allokatoren kann dieses Problem einen <i>drastischen</i>&#160;Einflu&#223; auf die Performance haben. Im konkreten Fall ist die Situation so komplex, da&#223; ich tunlichst jedwede Art von Absch&#228;tzung und Optimierung vermeiden sollte.
</p>
</body>
</html>
</richcontent>
</node>
</node>
</node>
<node CREATED="1770929365917" ID="ID_756237025" MODIFIED="1770929600076" TEXT="Konsequenz &#x27f9; API in dieser Hinsicht sofort anpassen">
<arrowlink COLOR="#692bc7" DESTINATION="ID_472477187" ENDARROW="Default" ENDINCLINATION="304;203;" ID="Arrow_ID_396413173" STARTARROW="None" STARTINCLINATION="347;19;"/>
</node>
<node CREATED="1770934037162" ID="ID_1977306902" MODIFIED="1770934049026" TEXT="getBlockPoolFor(size,HashVal)"/>
</node>
</node> </node>
</node> </node>
<node BACKGROUND_COLOR="#eef0c5" COLOR="#990000" CREATED="1770762458610" ID="ID_875010759" MODIFIED="1770762816021" TEXT="DiagnosticBufferProvider anpassen"> <node BACKGROUND_COLOR="#eef0c5" COLOR="#990000" CREATED="1770762458610" ID="ID_875010759" MODIFIED="1770762816021" TEXT="DiagnosticBufferProvider anpassen">
@ -112563,6 +112622,18 @@ StM_bind(Builder&lt;R1&gt; b1, Extension&lt;R1,R2&gt; extension)
</node> </node>
</node> </node>
</node> </node>
<node BACKGROUND_COLOR="#e0ceaa" COLOR="#690f14" CREATED="1770933762544" ID="ID_389070880" MODIFIED="1770933775068" TEXT="Tendenzen zeichnen sich ab...">
<icon BUILTIN="idea"/>
<node CREATED="1770933776885" ID="ID_1667378564" MODIFIED="1770933789787" TEXT="f&#xfc;r das Store-Interface">
<node CREATED="1770933791395" ID="ID_1572913456" MODIFIED="1770934001172" TEXT="...wird &#xfc;berall die buffSize mitgegeben">
<linktarget COLOR="#616d7c" DESTINATION="ID_1572913456" ENDARROW="Default" ENDINCLINATION="433;33;" ID="Arrow_ID_1937407092" SOURCE="ID_472477187" STARTARROW="None" STARTINCLINATION="1162;357;"/>
</node>
<node CREATED="1770933799954" ID="ID_1059437311" MODIFIED="1770933811425" TEXT="denkbar auch, das LocalTag &#xfc;berall durchzuf&#xe4;deln"/>
<node CREATED="1770933813057" ID="ID_1110032982" MODIFIED="1770933835165" TEXT="man k&#xf6;nnte das in eine BuffInfo packen">
<icon BUILTIN="idea"/>
</node>
</node>
</node>
</node> </node>
</node> </node>
</node> </node>