From d24e025de2cb589fd0c20716a9bf7631fe108ae3 Mon Sep 17 00:00:00 2001 From: Ichthyostega Date: Thu, 27 Nov 2025 02:08:50 +0100 Subject: [PATCH] Build: chase down and solve spurious SVG icon rebuilds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Many years ago, I integrated the IconSvgRenderer (written by Joel Holdsworth) directly into the SCons build by means of a custom builder with an ''emitter function''. The build as such works fine and automatically determines which icons can be generated from a given SVG source. However, those SVG icons tend to be rebuilt very frequently, even while none of the SVG sources has changed. Basically this was more of an annoyance, since we have now about 15 icons and the rendering is really fast ... it is just ugly, especially on incremental builds (and it could become a problem once we have a massive amount of graphics to process. So I took the opportunity to take close look now, since I am doing uninspiring clean-up work since several weeks now. But that problem turned out to be quite insidious and hard to spot. First, AI set me off into a completely wrong angle, since it is ''not'' caused by a missing string representation of the custom Action class. However, from there I went to watching the target checks with the debugger, and this eventually got me to realise, that SCons mistakenly detects a change in the Executor / the Action class. The root cause is, that we invoke Python code from an external Module, IconSvgRenderer.py, and this is »materialised« by SCons automatically into a string representation, which includes memory addresses of functions in that module. And that yields a signature, that is, quite obviously, not stable, even while you end up sometimes with loading the code to the same memory location. As a solution / workaround, we now subclass the standard implementation from SCons and override the signature function; instead of fingerprinting the binary code, we just compute a MD5 over the python source file, which can be easily achieved with the help of the File-Node from SCons. Essential resources: https://scons.org/doc/4.8.0/HTML/scons-user.html#chap-builders-writing ...and the Reference / Manpage of SCons https://scons.org/doc/4.8.0/HTML/scons-man.html#action_objects SCons/Action.py SCons/Node/__init__.py --- admin/scons/LumieraEnvironment.py | 66 ++- admin/scons/Setup.py | 2 +- wiki/thinkPad.ichthyo.mm | 820 +++++++++++++++++++++++++++++- 3 files changed, 866 insertions(+), 22 deletions(-) diff --git a/admin/scons/LumieraEnvironment.py b/admin/scons/LumieraEnvironment.py index 19c07c4b5..dd8f8a6fe 100644 --- a/admin/scons/LumieraEnvironment.py +++ b/admin/scons/LumieraEnvironment.py @@ -16,7 +16,8 @@ from os import path import SCons.SConf -from SCons.Action import Action +from SCons.Action import Action, FunctionAction +from SCons.Script import File as SConsFile from SCons.Environment import Environment from Buildhelper import * @@ -45,7 +46,8 @@ class LumieraEnvironment(Environment): self.Tool("BuilderDoxygen") self.Tool("ToolDistCC") self.Tool("ToolCCache") - register_LumieraResourceBuilder(self) + register_LumieraIconBuilder(self) + register_LumieraResourceBuilders(self) register_LumieraCustomBuilders(self) def _anchor_relative(self, key): @@ -124,12 +126,13 @@ class LumieraConfigContext(ConfigBase): + ############################################################################### ####### Lumiera custom tools and builders ##################################### -def register_LumieraResourceBuilder(env): - """ Registers Custom Builders for generating and installing Icons. +def register_LumieraIconBuilder(env): + """ Registers a custom Builder for generating and installing Icons from SVG. Additionally you need to build the tool (rsvg-convert.c) used to generate png from the svg source using librsvg. """ @@ -138,13 +141,39 @@ def register_LumieraResourceBuilder(env): renderer.rsvgPath = env.subst("$TARGDIR/rsvg-convert").removeprefix('#') # # the prefix '#' is a SCons specific convention, # # which the external tool can not handle + # + # MD5 signature for this specific python source code... + thisCodeSignature = SConsFile(__file__).get_csig() + SConsFile(renderer.__file__).get_csig() + thisCodeSignature = bytearray(thisCodeSignature, 'utf-8') - def invokeRenderer(target, source, env): - source = str(source[0]) - targetdir = env.subst(env.path.buildIcon).removeprefix('#') - renderer.main([source,targetdir]) - return 0 + + class IconRenderAction(FunctionAction): + """ SCons Action subclass to provide a controlled cache signature. + @note: usually it would be sufficient to pass just a callable to the Builder, + however, our implementation calls into an external Python module and thus + the default signature from SCons would not be stable, since it relies + on a code representation including memory addresses. Without this, + the icons would be frequently rebuilt unnecessarily. + """ + def __init__(self): + FunctionAction.__init__(self, IconRenderAction.invokeRenderer + , {'cmdstr' : "rendering Icon: $SOURCE --> $TARGETS"} + ) + + def get_contents(self, target, source, env): + """ a stable signature based on the source code """ + return thisCodeSignature + + @staticmethod + def invokeRenderer(target, source, env): + """ render the SVG icon with libRSVG """ + source = str(source[0]) + targetdir = env.subst(env.path.buildIcon).removeprefix('#') + renderer.main([source,targetdir]) + return 0 + + def createIconTargets(target,source,env): """ parse the SVG to get the target file names """ source = str(source[0]) @@ -162,6 +191,20 @@ def register_LumieraResourceBuilder(env): return (generateTargets, source) + + buildIcon = env.Builder( action = IconRenderAction() + , single_source = True + , emitter = createIconTargets + ) + env.Append(BUILDERS = {'IconRender' : buildIcon}) + + + + +def register_LumieraResourceBuilders(env): + """ Registers further Custom Methods for installing various Resources. + """ + def IconResource(env, source): """ copy icon pixmap to corresponding icon dir. """ subdir = getDirname(str(source)) @@ -227,11 +270,6 @@ def register_LumieraResourceBuilder(env): return env.InstallAs(toInstall, source) # this renames at target - buildIcon = env.Builder( action = Action(invokeRenderer, "rendering Icon: $SOURCE --> $TARGETS") - , single_source = True - , emitter = createIconTargets - ) - env.Append(BUILDERS = {'IconRender' : buildIcon}) env.AddMethod(IconResource) env.AddMethod(GuiResource) env.AddMethod(ConfigData) diff --git a/admin/scons/Setup.py b/admin/scons/Setup.py index 5070f0254..ce97167af 100644 --- a/admin/scons/Setup.py +++ b/admin/scons/Setup.py @@ -59,7 +59,7 @@ def defineBuildEnvironment(): """ EnsureSConsVersion(2,0) EnsurePythonVersion(2,6) - Decider('MD5-timestamp') # detect changed files by timestamp, then do a MD5 + Decider('content-timestamp') # detect changed files by timestamp, then do a MD5 buildVars = Variables([OPTCACHE, CUSTOPTFILE]) Options.defineCmdlineVariables(buildVars) diff --git a/wiki/thinkPad.ichthyo.mm b/wiki/thinkPad.ichthyo.mm index c99e90a7f..9fe5ec78c 100644 --- a/wiki/thinkPad.ichthyo.mm +++ b/wiki/thinkPad.ichthyo.mm @@ -159345,6 +159345,39 @@ unsigned int ThreadIdAsInt = *static_cast<unsigned int*>(static_cast<vo + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -168298,6 +168331,9 @@ that situation will improve in forseeable future. + + + @@ -170257,14 +170293,54 @@ that situation will improve in forseeable future. + + + + +

+ technical/build/LumieraDebianPackage.html +

+ + + + + + +

+ enthält auch Beschreibung des Installation-Bundle +

+ + + + + +

+ die ist gut und auch nützlich dort; könnte aber auch übernommen werden in die Beschreibun des Buildsystems +

+ + + - - + + + + - - - - + + + + + + + + + + + + + + + @@ -170301,6 +170377,25 @@ that situation will improve in forseeable future. + + + + + + + + + + + + + + + + + + + @@ -171942,7 +172037,7 @@ Since then others have made contributions, see the log for the history. - + @@ -178941,6 +179036,717 @@ env.Chmod(installed[0], 0o644) + + + + + + +

+ ....und jetzt wird's mal Zeit, das aufzuräumen, da ich nun sowiso schon so viel Aufwand in Clean-up gesteckt habe!!! +

+ + + + + + + + + + + + + + + + + + + +

+ wenn Files als Seiteneffekt erzeugt werden, kann es helfen, explizit ein Manifest-File als HIlfs-Target zu erzeugen +

+ + + + + + + + + + + + + +

+ ...und wir fügen das erzeugte Objekt per env.Append(BUILDERS=) hinzu. Genau wie in der Doku immer noch dargestellt +

+ + + + + + + + + + + +

+ Anmerkung: das war alles eine falsche Fährte +

+ + + + + + + + + + + +

+ Parsing icons/svg/track-unlocked.svg +

+

+ scons: rebuilding `target/gui/icons/24x24/track-unlocked.png' because: +

+

+            `data/icons/svg/track-unlocked.svg' changed +

+

+            `target/rsvg-convert' changed +

+

+ rendering Icon: data/icons/svg/track-unlocked.svg --> target/gui/icons/24x24/track-unlocked.png target/gui/icons/22x22/track-unlocked.png target/gui/icons/16x16/track-unlocked.png +

+

+ Parsing data/icons/svg/track-unlocked.svg +

+ + + + + + + + + +

+ in tool/SConscript (letzte Zeile) +

+
+
+

+ # Rendering the SVG Icons depends on rsvg-convert +

+

+ env.Depends(icons, rsvg) +

+

+ +

+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

+ daß er nämlich wenig systematisch aufgebaut ist, und darauf angewiesen, daß alle Daten korrekt normalisiert sind, und die Aufrufe jeweils richtig erfolgen: +

+
    +
  • + aufgerufen werden muß auf der Dependency +
    • +
  • + aber das 'node'-Argument muß die Node des davon abhängigen Targets liefern +
    • +
  • + denn es werden die Dependencies des Targets mit der Build-Information der Source verglichen +
    • +
+ + + + + + + + + + + +

+ ...um all die Komplexität von unserem SCons-Build auszuschalten; also praktisch das beispiel für einen Builder mit Emitter aus der Doku nachbauen +

+ + + + + + + + + + + + + + +
+
+

+ from SCons.Environment import  Environment +

+

+ from SCons.Builder import Builder +

+

+ from SCons.Script import Decider +

+

+
+ +

+

+
+ +

+

+ Decider('content-timestamp') +

+

+
+ +

+

+ env = Environment() +

+

+ bld = Builder(action='(echo -n "FOO `date -Isecond` :"; cat) < $SOURCE > $TARGET') +

+

+ env.Append(BUILDERS={'Foo': bld}) +

+

+
+ +

+

+
+ +

+

+ env.Foo('file.foo', 'file.input') +

+

+ env.Program('hello.c') +

+

+
+ +

+

+ +

+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

+ sondern enthält genau einen Eintrag, nämlich die SOURCE +

+ + + + + + + + + + + + + + + + + + + + + + + + + + +

+ Die Action ist hier ein Python-Objekt +

+ + + + + + + + + + + +

+ ...und hier wird self (=die Target-Node) als 4.Parameter mitgegeben, die Prüfung erfolgt auf dem child, also der Source-Node +

+ + + + + + + + + +

+ was korrekt wit, da auch explain() auf der Source-Node aufgerufen wird +

+ + + + + + + +

+ scons: rebuilding `target/gui/icons/24x24/track-unlocked.png' because the contents of the build action changed +

+ + + + + + + + + + + + + + + + +

+ Short answer +

+

+ Make the Action expose a stable, deterministic signature by providing an explicit string function (strfunction) and/or an explicit variable list (varlist) when creating the Action, and avoid putting non-deterministic values (timestamps, randoms, VM-specific paths) into that signature. +

+

+ Why +

+

+ SCons decides whether to rebuild partly from the Action's signature  (a string representing the action) and from node signatures. A Python Action built without an explicit strfunction can produce unstable or overly-broad signatures and cause unnecessary rebuilds. +

+

+ How (concise example) +

+
    +
  • + Import SCons Action and make the Python function do the work. +
    • +
  • + Provide an explicit strfunction that returns a deterministic string (or a short label). +
    • +
  • + Optionally provide varlist to include relevant environment variables in the signature. +
    • +
+

+ Example: +

+ 

+    

+      

+        python
+      

+    

+    
from SCons.Action import Action
+
+def my_build(target, source, env):
+    # do deterministic build steps
+    with open(str(target[0]), "wb") as out:
+        out.write(open(str(source[0]), "rb").read())
+    return None
+
+# deterministic signature string; keep it short and stable
+def my_strfunc(act, target, source, env):
+    return "my_build: %s -> %s" % (",".join([s.path for s in source]), ",".join([t.path for t in target]))
+
+# optionally include env variables that should affect rebuilds
+my_action = Action(my_build, my_strfunc, varlist=['MYFLAG', 'OTHER_VAR'])
+
+env.Command('out.bin', 'in.bin', my_action)

+    

+      Practical rules
+    

+    
    
+      
  • 
+        Provide strfunction (or cmdstr) so the action signature is explicit and stable.
+      
    • 
+      
  • 
+        Use varlist to include only environment variables that legitimately change build output.
+      
    • 
+      
  • 
+        Avoid including timestamps, absolute temp paths, random values, or local-only data in the signature.
+      
    • 
+      
  • 
+        Prefer content-based decider (SCons default) for file changes; only make action signature sensitive to what truly affects outputs.
+      
    • 
+    

+    

+      Tips & Alternatives
+    

+    
    
+      
  • 
+        To debug why SCons rebuilds, run SCons with --debug=explain to see which signature or node change triggered the rebuild.
+      
    • 
+      
  • 
+        If you need fully content-driven rebuilds, ensure the decider is set appropriately and rely on file content signatures rather than variable-heavy action signatures.
+      
    • 
+    

+  
+
+
+
+
+
+
+
+  
+  
+    

+      aber ist komplett falsch und irreführend
+    

+  
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  
+  
+    

+      Muß schon sagen, nach einiger Zeit Debugging bin ich schon wieder am Kotzen. Dieser Stil!!!!
+    

+    

+      Man akzeptiert irgendwas und geht dann durch eine zigfach verschachtelte Kette von Adaptern, solange bis es irgendwann.... wenn ... dann ... eben doch irgendwie paßt
+    

+  
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  
+  
+    

+      die Builder-Funktion in Lumiera ruft ein externes Python-Modul auf
+    

+  
+
+
+
+
+
+  
+  
+    

+      copyMergeDirectory=<function copyMergeDirectory at 0x7f0eeb296ac0>
+    

+  
+
+
+
+
+  
+  
+    

+      copyMergeDirectory=<function copyMergeDirectory at 0x7f5e85912ac0>
+    

+  
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  
+  
+    

+      r-grep über das ganze SCons-Paket gemacht....
+    

+    

+      Die Klasse erbt von der 'ABC' - Basisklasse (Python-3-Konstrukt). Aber die Argumente von gc(...) sprechen eigentlich dafür, daß das zu SCons gehört
+    

+  
+
+
+
+
+
+
+  
+  
+    

+      Diese Mentalität der Leute macht mich wütend.
+    

+    

+      Kann man mal sein Hirn einschalten, bevor man loshackt??
+    

+    

+      Wenn jemand eine eigene Implementierung liefert, dann hat er Gründe dafür und man kann erwarten, daß dann auch der Kontrakt erfüllt wird. Woher wollen die denn wissen, ob eine custom-Implementierung überhaupteine »Varlist« eingeschlossen haben möchte???!!  Zumal die ABC (ActionBase) gar kein Attribut 'self.varilist' hat... das kommt erst im nächsten Layer dazu.
+    

+  
+
+
+
+
+
+
+
+
+
+  
+  
+    

+      es teht ja nur darum, re-Builds der Icons zu vermeiden
+    

+  
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  
+  
+    

+      ...die Standard-Implementierung dieser get_contents()-Methode vewendet ein Rendering der involvierten Code-Objekte, inklusive der Variablen. Hier würde der IconSvgRenderer auftauchen. Stattdessen setzen wir eine Prüfsumme auf den Python-Quellcode; das Executable rsvog-convert ist sowiso auch noch eine Dependency, und auch Änderungen daran würden erkannt. Und natürlich Änderungen am SVG-Quellcode.
+    

+  
+
+
+
+
+  
+  
+    

+      ...und das lassen wir SCons machen, das kann das ja sehr gut...
+    

+  
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+