benn/lumiera_documentation
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'master' of git://git.lumiera.org/LUMIERA

Browse source
This commit is contained in:
Benny 2012-12-30 19:20:24 +01:00
commit 75e5ce2327
968 changed files with 49076 additions and 61223 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored
View file

@ -7,6 +7,8 @@
*.gch
,valgrind.log*
*.pyc
/.sconf_temp
/.settings
optcache
Makefile.in
build/*

2
DIR_INFO
View file

@ -1 +1 @@
The root dir for the Lumiera project
Root of the main Lumiera project tree

10
README
View file

@ -42,7 +42,7 @@ The Application can be installed and started to bring up an GTK GUI outline,
but the GUI is very preliminary and not connected to core functionality.
The video processing pipeline is still not complete.
See the http://issues.lumiera.org/roadmap[Project roadmap]
See http://issues.lumiera.org/roadmap[Project roadmap]
Build Requirements
@ -66,8 +66,8 @@ or the local Copy of this page in the file INSTALL
Debian Package
--------------
[verse]
Hermann Vosseler (aka Ichthyo) maintains a *Debian* packaging of the source tree,
which can be pulled from +git://git.lumiera.org/lumiera/debian+
It can be built by +git-buildpackage+
Hermann Vosseler (aka Ichthyo) maintains a *Debian* packaging of the source tree
- the package definition can be pulled from +git://git.lumiera.org/lumiera/debian+
- the package can be built by +git-buildpackage+

437
SConstruct
View file

@ -1,6 +1,6 @@
# -*- python -*-
##
## SConstruct - SCons based build-sytem for Lumiera
## SConstruct - SCons based build-system for Lumiera
##
# Copyright (C) Lumiera.org
@ -23,420 +23,63 @@
# NOTE: scons -h for help.
# This script /defines/ the components and how they fit together.
# SCons will derive dependencies and the necessary build steps.
# Read more about the SCons build system at: http://www.scons.org
# Basically, this script just /defines/ the components and how they
# fit together. SCons will derive the necessary build steps.
#-----------------------------------Configuration
TARGDIR = 'target'
VERSION = '0.pre.01'
TOOLDIR = './admin/scons' # SCons plugins
SCRIPTDIR = './admin'
OPTCACHE = 'optcache'
CUSTOPTFILE = 'custom-options'
# these are accessible via env.path.xxxx
srcIcon = 'icons'
srcConf = 'data/config'
buildExe = '#$TARGDIR'
buildLib = '#$TARGDIR/modules'
buildPlug = '#$TARGDIR/modules'
buildIcon = '#$TARGDIR/gui/icons'
buildUIRes = '#$TARGDIR/'
buildConf = '#$TARGDIR/config'
installExe = '#$DESTDIR/lib/lumiera'
installLib = '#$DESTDIR/lib/lumiera/modules'
installPlug = '#$DESTDIR/lib/lumiera/modules'
installIcon = '#$DESTDIR/share/lumiera/icons'
installUIRes = '#$DESTDIR/share/lumiera/'
installConf = '#$DESTDIR/lib/lumiera/config'
localDefinitions = locals()
#-----------------------------------Configuration
import os
# SCons plugins and extension modules
#------------------------------------------------
import sys
sys.path.append('./admin/scons')
#------------------------------------------------
sys.path.append(TOOLDIR)
sys.path.append(SCRIPTDIR)
import Setup
import Options
import Platform
from Buildhelper import *
from LumieraEnvironment import *
#####################################################################
def setupBasicEnvironment(localDefinitions):
""" define cmdline options, build type decisions
"""
EnsurePythonVersion(2,4)
EnsureSConsVersion(1,0)
Decider('MD5-timestamp') # detect changed files by timestamp, then do a MD5
vars = defineCmdlineVariables()
env = LumieraEnvironment(variables=vars
,toolpath = [TOOLDIR]
,pathConfig = extract_localPathDefs(localDefinitions) # e.g. buildExe -> env.path.buildExe
,TARGDIR = TARGDIR
,DESTDIR = '$INSTALLDIR/$PREFIX'
,VERSION = VERSION
)
handleVerboseMessages(env)
env.Append ( CCCOM=' -std=gnu99')
env.Append ( SHCCCOM=' -std=gnu99') # workaround for a bug: CCCOM currently doesn't honour CFLAGS, only CCFLAGS
env.Replace( CPPPATH =["#src"] # used to find includes, "#" means always absolute to build-root
, CPPDEFINES=['LUMIERA_VERSION='+VERSION ] # note: it's a list to append further defines
, CCFLAGS='-Wall -Wextra '
, CFLAGS='-std=gnu99'
)
handleNoBugSwitches(env)
env.Append(CPPDEFINES = '_GNU_SOURCE')
appendCppDefine(env,'DEBUG','DEBUG', 'NDEBUG')
# appendCppDefine(env,'OPENGL','USE_OPENGL')
appendVal(env,'ARCHFLAGS','CCFLAGS') # for both C and C++
appendVal(env,'OPTIMIZE', 'CCFLAGS', val=' -O3')
appendVal(env,'DEBUG', 'CCFLAGS', val=' -ggdb')
# setup search path for Lumiera plugins
appendCppDefine(env,'PKGLIBDIR','LUMIERA_PLUGIN_PATH=\\"$PKGLIBDIR/:ORIGIN/modules\\"'
,'LUMIERA_PLUGIN_PATH=\\"ORIGIN/modules\\"')
appendCppDefine(env,'PKGDATADIR','LUMIERA_CONFIG_PATH=\\"$PKGLIBDIR/:.\\"'
,'LUMIERA_CONFIG_PATH=\\"$DESTDIR/share/lumiera/:.\\"')
prepareOptionsHelp(vars,env)
vars.Save(OPTCACHE, env)
return env
def appendCppDefine(env,var,cppVar, elseVal=''):
if env[var]:
env.Append(CPPDEFINES = env.subst(cppVar) )
elif elseVal:
env.Append(CPPDEFINES = env.subst(elseVal))
def appendVal(env,var,targetVar,val=None):
if env[var]:
env.Append( **{targetVar: env.subst(val) or env[var]})
def handleNoBugSwitches(env):
""" set the build level for NoBug.
Release builds imply no DEBUG
whereas ALPHA and BETA require DEBUG
"""
level = env['BUILDLEVEL']
if level in ['ALPHA', 'BETA']:
if not env['DEBUG']:
print 'Warning: NoBug ALPHA or BETA builds requires DEBUG=yes, switching DEBUG on!'
env.Replace( DEBUG = 1 )
env.Append(CPPDEFINES = 'EBUG_'+level)
elif level == 'RELEASE':
env.Replace( DEBUG = 0 )
def handleVerboseMessages(env):
""" toggle verbose build output """
if not env['VERBOSE']:
# SetOption('silent', True)
env['CCCOMSTR'] = env['SHCCCOMSTR'] = " Compiling $SOURCE"
env['CXXCOMSTR'] = env['SHCXXCOMSTR'] = " Compiling++ $SOURCE"
env['LINKCOMSTR'] = " Linking --> $TARGET"
env['LDMODULECOMSTR'] = " creating module [ $TARGET ]"
def defineCmdlineVariables():
""" several toggles and configuration variables can be set on the commandline,
current settings will be persisted in a options cache file.
you may define custom variable settings in a separate file.
Commandline will override both.
"""
vars = Variables([OPTCACHE, CUSTOPTFILE])
vars.AddVariables(
('ARCHFLAGS', 'Set architecture-specific compilation flags (passed literally to gcc)','')
,('CC', 'Set the C compiler to use.', 'gcc')
,('CXX', 'Set the C++ compiler to use.', 'g++')
,PathVariable('CCACHE', 'Integrate with CCache', '', PathVariable.PathAccept)
,PathVariable('DISTCC', 'Invoke C/C++ compiler commands through DistCC', '', PathVariable.PathAccept)
,EnumVariable('BUILDLEVEL', 'NoBug build level for debugging', 'ALPHA', allowed_values=('ALPHA', 'BETA', 'RELEASE'))
,BoolVariable('DEBUG', 'Build with debugging information and no optimisations', False)
,BoolVariable('OPTIMIZE', 'Build with strong optimisation (-O3)', False)
,BoolVariable('VALGRIND', 'Run Testsuite under valgrind control', True)
,BoolVariable('VERBOSE', 'Print full build commands', False)
,('TESTSUITES', 'Run only Testsuites matching the given pattern', '')
# ,BoolVariable('OPENGL', 'Include support for OpenGL preview rendering', False)
# ,EnumVariable('DIST_TARGET', 'Build target architecture', 'auto',
# allowed_values=('auto', 'i386', 'i686', 'x86_64' ), ignorecase=2)
,PathVariable('PREFIX', 'Installation dir prefix', 'usr/local', PathVariable.PathAccept)
,PathVariable('INSTALLDIR', 'Root output directory for install. Final installation will happen in INSTALLDIR/PREFIX/... ', '/', PathVariable.PathIsDir)
,PathVariable('PKGLIBDIR', 'Installation dir for plugins, defaults to PREFIX/lib/lumiera/modules', '',PathVariable.PathAccept)
,PathVariable('PKGDATADIR', 'Installation dir for default config, usually PREFIX/share/lumiera', '',PathVariable.PathAccept)
)
return vars
def prepareOptionsHelp(vars,env):
prelude = """
USAGE: scons [-c] [OPTS] [key=val [key=val...]] [TARGETS]
Build and optionally install Lumiera.
Without specifying any target, just the (re)build target will run.
Add -c to the commandline to clean up anything a given target would produce
Special Targets:
build : just compile and link
testcode: additionally compile the Testsuite
check : build and run the Testsuite
doc : generate documentation (Doxygen)
all : build and testcode and doc
install : install created artifacts at PREFIX
Configuration Options:
"""
Help(prelude + vars.GenerateHelpText(env))
def configurePlatform(env):
""" locate required libs.
setup platform specific options.
Abort build in case of failure.
"""
conf = env.Configure()
# run all configuration checks in the given env
# Perform checks for prerequisites --------------------------------------------
problems = []
if not conf.TryAction('pkg-config --version > $TARGET')[0]:
problems.append('We need pkg-config for including library configurations, exiting.')
if not conf.CheckLibWithHeader('m', 'math.h','C'):
problems.append('Did not find math.h / libm.')
if not conf.CheckLibWithHeader('dl', 'dlfcn.h', 'C'):
problems.append('Functions for runtime dynamic loading not available.')
if not conf.CheckLibWithHeader('pthread', 'pthread.h', 'C'):
problems.append('Did not find the pthread lib or pthread.h.')
else:
conf.env.Append(CPPFLAGS = ' -DHAVE_PTHREAD')
conf.env.Append(CCFLAGS = ' -pthread')
if conf.CheckCHeader('execinfo.h'):
conf.env.Append(CPPFLAGS = ' -DHAVE_EXECINFO_H')
if conf.CheckCHeader('valgrind/valgrind.h'):
conf.env.Append(CPPFLAGS = ' -DHAVE_VALGRIND_H')
else:
print 'Valgrind not found. The use of Valgrind is optional; building without.'
if not conf.CheckPkgConfig('nobugmt', 201006.1):
problems.append('Did not find NoBug [http://www.lumiera.org/nobug_manual.html].')
else:
conf.env.mergeConf('nobugmt')
if not conf.CheckCXXHeader('tr1/memory'):
problems.append('We rely on the std::tr1 standard C++ extension for shared_ptr.')
if not conf.CheckCXXHeader('boost/config.hpp'):
problems.append('We need the C++ boost-libraries.')
else:
if not conf.CheckCXXHeader('boost/scoped_ptr.hpp'):
problems.append('We need boost::scoped_ptr (scoped_ptr.hpp).')
if not conf.CheckCXXHeader('boost/format.hpp'):
problems.append('We need boost::format (header).')
if not conf.CheckLibWithHeader('boost_program_options-mt','boost/program_options.hpp','C++'):
problems.append('We need boost::program_options (including binary lib for linking).')
if not conf.CheckLibWithHeader('boost_filesystem-mt','boost/filesystem.hpp','C++'):
problems.append('We need the boost::filesystem (including binary lib for linking).')
if not conf.CheckLibWithHeader('boost_regex-mt','boost/regex.hpp','C++'):
problems.append('We need the boost regular expression lib (incl. binary lib for linking).')
if conf.CheckLib(symbol='clock_gettime'):
print 'Using function clock_gettime() as defined in the C-lib...'
else:
if not conf.CheckLib(symbol='clock_gettime', library='rt'):
problems.append('No library known to provide the clock_gettime() function.')
if not conf.CheckPkgConfig('gavl', 1.0):
problems.append('Did not find Gmerlin Audio Video Lib [http://gmerlin.sourceforge.net/gavl.html].')
else:
conf.env.mergeConf('gavl')
if not conf.CheckPkgConfig('gtkmm-2.4', 2.8):
problems.append('Unable to configure GTK--')
if not conf.CheckPkgConfig('glibmm-2.4', '2.16'):
problems.append('Unable to configure Lib glib--')
if not conf.CheckPkgConfig('gthread-2.0', '2.12.4'):
problems.append('Need gthread support lib for glib-- based thread handling.')
if not conf.CheckPkgConfig('cairomm-1.0', 0.6):
problems.append('Unable to configure Cairo--')
verGDL = '2.27.1'
if not conf.CheckPkgConfig('gdl-lum', verGDL, alias='gdl'):
print 'Custom package "gdl-lum" not found. Trying official GDL release >=%s...' % verGDL
if not conf.CheckPkgConfig('gdl-1.0', verGDL, alias='gdl'):
problems.append('GNOME Docking Library not found. We either need a very recent GDL '
'version (>=%s), or the custom package "gdl-lum".' % verGDL)
if not conf.CheckPkgConfig('librsvg-2.0', '2.18.1'):
problems.append('Need rsvg Library for rendering icons.')
if not conf.CheckCHeader(['X11/Xutil.h', 'X11/Xlib.h'],'<>'):
problems.append('Xlib.h and Xutil.h required. Please install libx11-dev.')
if not conf.CheckPkgConfig('xv') : problems.append('Need libXv...')
if not conf.CheckPkgConfig('xext'): problems.append('Need libXext.')
# report missing dependencies
if problems:
print "*** unable to build due to the following problems:"
for isue in problems:
print " * %s" % isue
print
print "build aborted."
Exit(1)
print "** Gathered Library Info: %s" % conf.env.libInfo.keys()
# create new env containing the finished configuration
return conf.Finish()
def defineSetupTargets(env, artifacts):
""" build operations and targets to be done /before/ compiling.
things like creating a source tarball or preparing a version header.
"""
pass ## currently none
def defineBuildTargets(env, artifacts):
""" define the source file/dirs comprising each artifact to be built.
setup sub-environments with special build options if necessary.
We use a custom function to declare a whole tree of srcfiles.
"""
# use PCH to speed up building // disabled for now due to strange failures
# env['GCH'] = ( env.PrecompiledHeader('src/pre.hpp')
# + env.PrecompiledHeader('src/pre_a.hpp')
# )
lLib = env.SharedLibrary('lumiera', srcSubtree(env,'src/lib'), install=True)
lApp = env.SharedLibrary('lumieracommon', srcSubtree(env,'src/common'), install=True, LIBS=lLib)
lBack = env.SharedLibrary('lumierabackend', srcSubtree(env,'src/backend'),install=True)
lProc = env.SharedLibrary('lumieraproc', srcSubtree(env,'src/proc'), install=True)
core = lLib+lApp+lBack+lProc
artifacts['corelib'] = core
artifacts['support'] = lLib
artifacts['lumiera'] = ( env.Program('lumiera', ['src/lumiera/main.cpp'], LIBS=core, install=True)
+ env.ConfigData(env.path.srcConf+'setup.ini', targetDir='$ORIGIN')
+ env.ConfigData(env.path.srcConf+'dummy_lumiera.ini')
)
# building Lumiera Plugins
artifacts['plugins'] = [] # currently none
# render and install Icons
vector_icon_dir = env.path.srcIcon+'svg'
prerendered_icon_dir = env.path.srcIcon+'prerendered'
artifacts['icons'] = ( [env.IconRender(f) for f in scanSubtree(vector_icon_dir, ['*.svg'])]
+ [env.IconResource(f) for f in scanSubtree(prerendered_icon_dir, ['*.png'])]
)
# the Lumiera GTK GUI
envGtk = env.Clone()
envGtk.mergeConf(['gtkmm-2.4','gthread-2.0','cairomm-1.0','gdl','xv','xext','sm'])
envGtk.Append(LIBS=core)
objgui = srcSubtree(envGtk,'src/gui', appendCPP='LUMIERA_PLUGIN')
guimodule = envGtk.LumieraPlugin('gtk_gui', objgui, install=True)
artifacts['gui'] = ( guimodule
+ [env.GuiResource(f) for f in env.Glob('src/gui/*.rc')]
+ artifacts['icons']
)
# call subdir SConscript(s) for independent components
SConscript(dirs=['src/tool'], exports='env artifacts core')
SConscript(dirs=['tests'], exports='env artifacts core')
def definePostBuildTargets(env, artifacts):
""" define further actions after the core build (e.g. Documentaion).
define alias targets to trigger the installing.
"""
build = env.Alias('build', ( artifacts['lumiera']
+ artifacts['plugins']
+ artifacts['tools']
+ artifacts['gui']
))
# additional files to be cleaned when cleaning 'build'
env.Clean ('build', [ 'scache.conf', '.sconf_temp', '.sconsign.dblite', 'config.log' ])
env.Clean ('build', [ 'src/pre.gch' ])
doxydoc = artifacts['doxydoc'] = env.Doxygen('doc/devel/Doxyfile')
env.Alias ('doc', doxydoc)
env.Clean ('doc', doxydoc + ['doc/devel/,doxylog','doc/devel/warnings.txt'])
env.Alias ('all', build+artifacts['testsuite']+doxydoc)
env.Default('build')
# SCons default target
def defineInstallTargets(env, artifacts):
""" define additional artifacts to be installed into target locations.
@note: we use customised SCons builders defining install targets
for all executables automatically. see LumieraEnvironment.py
"""
env.SymLink('$DESTDIR/bin/lumiera',env.path.installExe+'lumiera','../lib/lumiera/lumiera')
# env.Install(dir = '$DESTDIR/share/doc/lumiera$VERSION/devel', source=artifacts['doxydoc'])
env.Alias('install', artifacts['gui'])
env.Alias('install', '$DESTDIR')
#####################################################################
env = Setup.defineBuildEnvironment() # dirs & compiler flags
env = Platform.configure(env) # library dependencies
### === MAIN BUILD === ##############################################
# call subdir SConscript(s) to define the actual build targets...
SConscript(dirs=['data','src','src/tool','research','tests','doc'], exports='env')
### === MAIN === ####################################################
# additional files to be cleaned when cleaning 'build'
env.Clean ('build', [ 'scache.conf', '.sconf_temp', '.sconsign.dblite', 'config.log' ])
env.Clean ('build', [ 'src/pre.gch' ])
env = setupBasicEnvironment(localDefinitions)
if not (isCleanupOperation(env) or isHelpRequest()):
env = configurePlatform(env)
artifacts = {}
# the various things we build.
# Each entry actually is a SCons-Node list.
# Passing these entries to other builders defines dependencies.
# 'lumiera' : the App
# 'gui' : the GTK UI (plugin)
# 'plugins' : plugin shared lib
# 'tools' : small tool applications (e.g mpegtoc)
defineSetupTargets(env, artifacts)
defineBuildTargets(env, artifacts)
definePostBuildTargets(env, artifacts)
defineInstallTargets(env, artifacts)
### === Alias Targets === ###########################################
# pick up the targets defined by the sub SConscripts
Import('lumiera plugins tools gui testsuite doxydoc')
build = env.Alias('build', lumiera + plugins + tools + gui)
env.Default('build')
# SCons default target
env.Alias ('all', build + testsuite + doxydoc)
env.Alias ('doc', doxydoc)
env.Alias ('none')
env.Alias('install', gui)
env.Alias('install', '$DESTDIR')
#####################################################################

213
admin/render_icon.py
View file

@ -1,213 +0,0 @@
#!/usr/bin/python
#
# render_icons.py - Icon rendering utility script
#
# Copyright (C) Lumiera.org
# 2008, Joel Holdsworth <joel@airwebreathe.org.uk>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
import sys
import getopt
from xml.dom import minidom
import os
import shutil
#svgDir = "svg"
#prerenderedDir = "prerendered"
inkscapePath = "/usr/bin/inkscape"
rsvgPath = "./rsvg-convert"
artworkLayerPrefix = "artwork:"
#
# 2/2011 some notes by Ichthyo
# The purpose of this python script is
# - to parse a SVG
# - to invoke Inkscape to render this SVG into a raster image (icon)
#
# For the actual call into Incscape we rely on an executable 'rsvg-convert',
# which is built during the Lumiera build process.
#
# Judging from the code and the actual SVGs, this seems to work as follows:
# The SVG contains a design to be rendered into raster images of various sizes.
# These sizes are determined by special rectangles, which act as bounding box and
# are placed on a special 'plate' layer, which is a child layer of the main
# 'artwork:' layer. The grid of the SVG is setup such as to result in pixel sizes
# suitable for icon generation. The actual size of the generated icons are then
# parsed from the height and width attributes of the mentioned bounding box
# rectangles.
#
# The parser seems to be rather simplistic; the sizes and positions need to be
# integral numbers. In one instance we had a float number in the y coordinate,
# which resulted in an invalid, zero sized output icon
#
#
def createDirectory( name ):
try:
if os.path.isfile(name):
os.remove(name)
if not os.path.exists(name):
os.mkdir(name)
except:
print 'WARNING: createDirectory("%s") failed. Permission problems?' % name
def copyMergeDirectory( src, dst ):
listing = os.listdir(src)
for file_name in listing:
src_file_path = os.path.join(src, file_name)
dst_file_path = os.path.join(dst, file_name)
shutil.copyfile(src_file_path, dst_file_path)
def getDocumentSize( svg_element ):
width = float(svg_element.getAttribute("width"))
height = float(svg_element.getAttribute("height"))
return [width, height]
def findChildLayerElement( parent_element ):
for node in parent_element.childNodes:
if node.nodeType == minidom.Node.ELEMENT_NODE:
if node.tagName == "g":
if node.getAttribute("inkscape:groupmode") == "layer":
return node
return None
def parsePlateLayer( layer ):
rectangles = []
for node in layer.childNodes:
if node.nodeType == minidom.Node.ELEMENT_NODE:
if node.tagName == "rect":
x = float(node.getAttribute("x"))
y = float(node.getAttribute("y"))
width = float(node.getAttribute("width"))
height = float(node.getAttribute("height"))
rectangles.append([x, y, width, height])
return rectangles
def parseSVG( file_path ):
print "Parsing " + file_path
svgdoc = minidom.parse(file_path)
for root_node in svgdoc.childNodes:
if root_node.nodeType == minidom.Node.ELEMENT_NODE:
if root_node.tagName == "svg":
size = getDocumentSize( root_node )
layer = findChildLayerElement( root_node )
if layer != None:
layer_name = layer.getAttribute("inkscape:label")
if layer_name[:len(artworkLayerPrefix)] == artworkLayerPrefix:
artwork_name = layer_name[len(artworkLayerPrefix):]
plate = findChildLayerElement( layer )
if plate != None:
return artwork_name, size, parsePlateLayer( plate )
return None
def renderSvgInkscape(file_path, out_dir, artwork_name, rectangle, doc_size):
# Calculate the rendering rectangle
x1 = rectangle[0]
y1 = doc_size[1] - rectangle[1] - rectangle[3]
x2 = x1 + rectangle[2]
y2 = y1 + rectangle[3]
# Call Inkscape to do the render
os.spawnlp(os.P_WAIT, inkscapePath, inkscapePath,
file_path,
"-z",
"-a %g:%g:%g:%g" % (x1, y1, x2, y2),
"-w %g" % (rectangle[2]), "-h %g" % (rectangle[3]),
"--export-png=" + os.path.join(out_dir, "%gx%g/%s.png" % (rectangle[2], rectangle[3], artwork_name)))
def renderSvgRsvg(file_path, out_dir, artwork_name, rectangle, doc_size):
# Prepare a Cairo context
width = int(rectangle[2])
height = int(rectangle[3])
if not os.path.exists(rsvgPath):
print "Error: executable %s not found." % rsvgPath
os.spawnlp(os.P_WAIT, rsvgPath, rsvgPath,
"--source-rect=%g:%g:%g:%g" % (rectangle[0], rectangle[1], rectangle[2], rectangle[3]),
"--output=" + os.path.join(out_dir, "%gx%g/%s.png" % (rectangle[2], rectangle[3], artwork_name)),
file_path)
def renderSvgIcon(file_path, out_dir):
artwork_name, doc_size, rectangles = parseSVG(file_path)
for rectangle in rectangles:
renderSvgRsvg(file_path, out_dir, artwork_name, rectangle, doc_size)
def getTargetNames(file_path):
"""get a list of target names to be rendered from the given source SVG
usable to setup the build targets for SCons
"""
artwork_name, _ , rectangles = parseSVG(file_path)
return ["%gx%g/%s.png" % (rectangle[2], rectangle[3], artwork_name) for rectangle in rectangles ]
#def renderSvgIcons():
# listing = os.listdir(svgDir)
# for file_path in listing:
# [root, extension] = os.path.splitext(file_path)
# if extension.lower() == ".svg":
# renderSvgIcon(os.path.join(svgDir, file_path))
#def copyPrerenderedIcons():
# listing = os.listdir(prerenderedDir)
# for list_item in listing:
# src_dir = os.path.join(prerenderedDir, list_item)
# copyMergeDirectory(src_dir, list_item)
def printHelp():
print "render-icon.py SRCFILE.svg TARGETDIR"
print "An icon rendering utility script for lumiera"
def parseArguments(argv):
optlist, args = getopt.getopt(argv, "")
if len(args) == 2:
return args[0], args[1]
printHelp()
return None, None
def main(argv):
in_path, out_dir = parseArguments(argv)
if not (in_path and out_dir):
print "Missing arguments in_path and out_dir."
sys.exit(1)
if os.path.isfile(out_dir):
print "Unable to use '%s' as output directory, because it\'s a file." % out_dir
sys.exit(1)
if not os.path.isdir(out_dir):
print "Output directory '%s' not found." % out_dir
sys.exit(1)
# Create the icons folders
createDirectory(os.path.join(out_dir, "48x48"))
createDirectory(os.path.join(out_dir, "32x32"))
createDirectory(os.path.join(out_dir, "24x24"))
createDirectory(os.path.join(out_dir, "22x22"))
createDirectory(os.path.join(out_dir, "16x16"))
renderSvgIcon(in_path, out_dir)
# Copy in prerendered icons
#copyPrerenderedIcons()
if __name__=="__main__":
main(sys.argv[1:])

204
admin/rfc.sh
View file

@ -2,66 +2,90 @@
shopt -s extglob
#
# RFC's are created from ./doc/devel/template/new_rfc.sh and stored in
# ./doc/devel/rfc/ . There are directories for the various states
# ./doc/devel/rfc_final, ./doc/devel/rfc_pending, ./doc/devel/rfc_parked,
# ./doc/devel/rfc_dropped . Which contain symlinks back to ../rfc/
#
function usage()
{
less <<"EOF"
Script to maintain Lumiera RFC's
grep -v '^// ' <<"EOF" | less -F
Lumiera RFC maintenance script
==============================
// Note: the source of this documentation is maintained
// directly admin/rfc.sh in its usage() function
// edit it only there and then regenerate
// doc/devel/technical/infra/ with:
// ./admin/rfc.sh help >doc/technical/infra/rfcsh.txt
Usage
-----
usage:
./admin/rfc.sh <command> [options]
options:
title - Quoted string used as RFC title
rfc - Name of the RFC, smart matched, unique
rfcs - Name of the RFC, smart matched, multiple
regex - Regex matched against the content of a RFC
chapter - Heading of a section
Commands (with <mandatory> and [optional] parameters)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
commands (with <mandatory> and [optional] parameters):
find <rfcs> [regex] - List all matching RFC's (matching 'regex')
show <rfcs> [regex] - Read RFC's (matching 'regex')
create <title> - Create a new RFC
edit <rfc> [chapter] - Edit RFC at chapter
asciidoc <rfc> - pass the rfc.txt to 'asciidoc' command
comment <rfc> - Add a new comment to a RFC
draft <rfc> - Change RFC to Draft state
final <rfc> - Change RFC to Final state
park <rfc> - Change RFC to Parked state
drop <rfc> - Change RFC to Dropped state
supersede <rfc> <new> - Supersede RFC with a new RFC
discard <rfc> - Delete an RFC
help - Show this help
process - Do automatic maintenance work
wrap <rfcs> - canonical reformatting
find <rfcs> [regex]:: List all matching RFC's (matching `regex`)
show <rfcs> [regex]:: Read RFC's (matching `regex`)
create <title>:: Create a new RFC
edit <rfc> [chapter]:: Edit RFC at chapter
asciidoc <rfc>:: pass the rfc.txt to `asciidoc` command
comment <rfc>:: Add a new comment to a RFC
draft <rfc>:: Change RFC to Draft state
final <rfc>:: Change RFC to Final state
park <rfc>:: Change RFC to Parked state
drop <rfc>:: Change RFC to Dropped state
supersede <rfc> <new>:: Supersede RFC with a new RFC
discard <rfc>:: Delete an RFC
help:: Show this help
process:: Do automatic maintenance work
wrap <rfcs>:: canonical reformatting
Smart matching:
RFC names don't need to be given exactly, they use a globbing pattern.
This is:
- case insensitive
- whitespaces are ignored
- '*' stands for any number of parameters
- '?' is any single character
- when starting with '/' they are matched against the begin of the name
- some regex operators work too
'find' and 'show' can operate on many matches so the given rfc name doesn't
need to be unique. The other commands will complain when the RFC name given
doesn't resolve to one unique RFC.
Option types
~~~~~~~~~~~~
Notes:
When less presents multiple files one can go forth and back with the ':n'
and ':p' commands.
title:: Quoted string used as RFC title
rfc:: Name of the RFC, smart matched, unique
rfcs:: Name of the RFC, smart matched, multiple
regex:: Regex matched against the content of a RFC
chapter:: Heading of a section
The tile for 'create' should be a normal document title. Possibly quoted
since it may contain spaces and not too long. The filename is this title
in CamelCase with all spaces and special characters removed.
Chapter machching single lines containing this word, special asciidoc
comments in the form '//word:.*' and asciidoc block attributes '[word.*]'
on a single line. When a chapter pattern is not unique, the last one is
picked.
Smart matching
--------------
rfc.sh executes git add/rm/mv commands, but never commits. One should do a
commit as soon he finished editing.
RFC names don't need to be given exactly, they use a globbing pattern.
This is:
* case insensitive
* whitespaces are ignored
* `*` stands for any number of parameters
* `?` is any single character
* when starting with `/` they are matched against the begin of the name
* some regex operators work too
`find` and `show` can operate on many matches so the given rfc name doesn't
need to be unique. The other commands will complain when the RFC name given
doesn't resolve to one unique RFC.
When `less` presents multiple files one can go forth and back with the `:n`
and `:p` commands.
The tile for `create` should be a normal document title. Possibly quoted
since it may contain spaces and not too long. The filename is this title
in CamelCase with all spaces and special characters removed.
Chapter machching single lines containing this word, special asciidoc
comments in the form `//word:.*` and asciidoc block attributes `[word.*]`
on a single line. When a chapter pattern is not unique, the last one is
picked.
rfc.sh executes git add/rm/mv commands, but never commits. One should do a
commit as soon he finished editing.
EOF
}
@ -183,7 +207,7 @@ function find_rfc()
local globstate=$(shopt -p nocasematch)
shopt -s nocasematch
for file in $(find ./doc/devel/rfc* -name '*.txt');
for file in $(find ./doc/devel/rfc/ -name '*.txt');
do
local name="/${file##*/}"
if [[ "$name" =~ $match ]]; then
@ -208,29 +232,43 @@ function find_chapter()
function process()
function process_file()
{
local file="$1"
local path="${1%/*}"
local basename="${1##*/}"
local linkdest="$path"
local state=$(grep '^\*State\* *' "$file")
case "$state" in
*Final*)
if [[ "$path" != "./doc/devel/rfc" ]]; then
git mv "$file" "./doc/devel/rfc"
fi
linkdest="./doc/devel/rfc_final"
;;
*Idea*|*Draft*)
if [[ "$path" != "./doc/devel/rfc_pending" ]]; then
git mv "$file" "./doc/devel/rfc_pending"
fi
linkdest="./doc/devel/rfc_pending"
;;
*Parked*|*Dropped*)
if [[ "$path" != "./doc/devel/rfc_dropped" ]]; then
git mv "$file" "./doc/devel/rfc_dropped"
fi
*Parked*)
linkdest="./doc/devel/rfc_parked"
;;
*Dropped*)
linkdest="./doc/devel/rfc_dropped"
;;
*)
echo "Unknown State: '$state'" >&2
exit 1
esac
local oldpath
for oldpath in ./doc/devel/rfc_*/$basename; do :; done
if [[ -h "$oldpath" ]]; then
if [[ "$oldpath" != "$linkdest/$basename" ]]; then
git mv "$oldpath" "$linkdest/$basename"
fi
elif [[ ! -s "$linkdest/$basename" ]]; then
ln -s "../rfc/$basename" "$linkdest/"
git add "$linkdest/$basename"
fi
}
@ -273,7 +311,7 @@ function add_comment()
{
local name="$1"
local nl=$'\n'
local comment="//edit comment$nl $(date +%c) $(git config --get user.name) <$(git config --get user.email)>$nl"
local comment="//edit comment$nl$nl$(git config --get user.name):: '$(date +%c)' ~<$(git config --get user.email)>~$nl"
ed "$name" 2>/dev/null <<EOF
/endof_comments:/-1i
@ -316,10 +354,10 @@ function change_state()
local state="$2"
local nl=$'\n'
local comment=".State -> $state$nl//add reason$nl $(date +%c) $(git config --get user.name) <$(git config --get user.email)>$nl"
local comment=".State -> $state$nl//add reason$nl$nl$(git config --get user.name):: '$(date +%c)' ~<$(git config --get user.email)>~$nl"
edit_state "$name" "$state" "$comment"
edit "$name" -4 "endof_comments"
process "$name"
process_file "$name"
}
@ -329,13 +367,16 @@ shift
case "$command" in
process)
# for all rfc's
for file in $(find ./doc/devel/rfc* -name '*.txt');
for file in $(find ./doc/devel/rfc -name '*.txt');
do
echo "process $file"
process "$file"
process_file "$file"
done
:
;;
search)
grep -r -C3 -n "$1" ./doc/devel/rfc | less -F
;;
find|list|ls)
if [[ "$2" ]]; then
find_rfc "$1" | xargs grep -i -C3 -n "$2"
@ -353,22 +394,20 @@ show|less|more)
create)
TITLE="$@"
name=$(camel_case "$TITLE")
if [[ -f "./doc/devel/rfc/${name}.txt" ||
-f "./doc/devel/rfc_pending/${name}.txt" ||
-f "./doc/devel/rfc_dropped/${name}.txt" ]]; then
if [[ -f "./doc/devel/rfc/${name}.txt" ]]; then
echo "$name.txt exists already"
else
source ./doc/devel/template/new_rfc.sh >"./doc/devel/rfc_pending/${name}.txt"
edit "./doc/devel/rfc_pending/${name}.txt" 2 abstract
git add "./doc/devel/rfc_pending/${name}.txt"
process "./doc/devel/rfc_pending/${name}.txt"
source ./doc/devel/template/new_rfc.sh >"./doc/devel/rfc/${name}.txt"
edit "./doc/devel/rfc/${name}.txt" 2 abstract
git add "./doc/devel/rfc/${name}.txt"
process_file "./doc/devel/rfc/${name}.txt"
fi
;;
edit)
name=$(unique_name "$1")
if [[ "$name" ]]; then
edit "${name}" 2 "$2"
git add "$name"
process_file "$name"
fi
;;
asciidoc)
@ -381,21 +420,18 @@ draft)
name=$(unique_name "$1")
if [[ "$name" ]]; then
change_state "$name" Draft
git add "$name"
fi
;;
park)
name=$(unique_name "$1")
if [[ "$name" ]]; then
change_state "$name" Parked
git add "$name"
fi
;;
final)
name=$(unique_name "$1")
if [[ "$name" ]]; then
change_state "$name" Final
git add "$name"
fi
;;
supersede)
@ -405,32 +441,38 @@ supersede)
newname="${newname%.txt}"
if [[ "$name" && "$newname" ]]; then
change_state "$name" "Superseded by $newname"
git add "$name"
fi
;;
drop)
name=$(unique_name "$1")
if [[ "$name" ]]; then
change_state "$name" Dropped
git add "$name"
fi
;;
comment)
name=$(unique_name "$1")
if [[ "$name" ]]; then
add_comment "${name}"
git add "$name"
process_file "$name"
fi
;;
discard)
name=$(unique_name "$1")
if [[ "$name" ]]; then
git rm "${name}"
for link in ./doc/devel/rfc_*/${name##*/}; do :; done
if [[ -h "$link" ]]; then
git rm -f "${link}" || rm "${link}"
fi
git rm -f "${name}" || rm "${name}"
fi
;;
wrap)
find_rfc "$1" | while read file; do
smart_wrap "$file" 80
process_file "$file"
done
;;
smart_wrap)

67
admin/scons/Buildhelper.py
View file

@ -25,11 +25,8 @@ import os
import sys
import glob
import fnmatch
import re
import tarfile
from SCons import Util
from SCons.Action import Action
@ -49,20 +46,11 @@ def isHelpRequest():
def srcSubtree(env,tree,isShared=True,builder=None,appendCPP=None, **args):
""" convenience wrapper: scans the given subtree, which is
relative to the current SConscript, find all source files and
declare them as Static or SharedObjects for compilation
def srcSubtree(tree, **args):
""" convenience wrapper: scan the given subtree, which is relative
to the current SConscript, and find all source files.
"""
if appendCPP: env.Append(CPPDEFINES=appendCPP)
root = env.subst(tree) # expand Construction Vars
if not builder:
if isShared:
builder = lambda f: env.SharedObject(f, **args)
else:
builder = lambda f: env.Object(f, **args)
return [builder(f) for f in scanSubtree(root)]
return list(scanSubtree(tree, **args))
@ -74,12 +62,12 @@ def scanSubtree(roots, patterns=SRCPATTERNS):
(python generator function)
"""
for root in globRootdirs(roots):
for (dir,_,files) in os.walk(root):
if dir.startswith('./'):
dir = dir[2:]
for (d,_,files) in os.walk(root):
if d.startswith('./'):
d = d[2:]
for p in patterns:
for f in fnmatch.filter(files, p):
yield os.path.join(dir,f)
yield os.path.join(d,f)
@ -87,16 +75,16 @@ def globRootdirs(roots):
""" helper: expand shell wildcards and filter the resulting list,
so that it only contains existing directories
"""
filter = lambda f: os.path.isdir(f) and os.path.exists(f)
isDirectory = lambda f: os.path.isdir(f) and os.path.exists(f)
roots = glob.glob(roots)
return (dir for dir in roots if filter(dir) )
return (d for d in roots if isDirectory(d) )
def findSrcTrees(location, patterns=SRCPATTERNS):
""" find possible source tree roots, starting with the given location.
When delving down from the initial location(s), a source tree is defined
as a directory containing source files and possibly further sub directories.
as a directory containidsource files and possibly further sub directories.
After having initially expanded the given location with #globRootdirs, each
directory is examined depth first, until encountering a directory containing
source files, which then yields a result. Especially, this can be used to traverse
@ -104,11 +92,11 @@ def findSrcTrees(location, patterns=SRCPATTERNS):
to be built into packages, plugins, individual tool executables etc.
@return: the relative path names of all source root dirs found (generator function).
"""
for dir in globRootdirs(location):
if isSrcDir(dir,patterns):
yield dir
for directory in globRootdirs(location):
if isSrcDir (directory,patterns):
yield directory
else:
for result in findSrcTrees(str(dir)+'/*'):
for result in findSrcTrees (str(directory)+'/*'):
yield result
@ -118,7 +106,7 @@ def isSrcDir(path, patterns=SRCPATTERNS):
@return: True if it's a directory containing any source file
"""
if not os.path.isdir(path):
return False
return False
else:
for p in patterns:
if glob.glob(path+'/'+p):
@ -133,38 +121,39 @@ def filterNodes(nlist, removeName=None):
if removeName:
predicate = lambda n : not fnmatch.fnmatch(os.path.basename(str(n[0])), removeName)
else:
predicate = lambda n : True;
predicate = lambda n : True
return filter(predicate, nlist)
def getDirname(dir, basePrefix=None):
def getDirname (d, basePrefix=None):
""" extract directory name without leading path,
or without the explicitly given basePrefix
"""
dir = os.path.realpath(dir)
if not os.path.isdir(dir):
dir,_ = os.path.split(dir)
d = os.path.realpath(d)
if not os.path.isdir(d):
d,_ = os.path.split(d)
if basePrefix:
basePrefix = os.path.realpath(basePrefix)
if str(dir).startswith(basePrefix):
name = str(dir)[len(basePrefix):]
name = str(d)
if str(d).startswith(basePrefix):
name = name[len(basePrefix):]
else:
_, name = os.path.split(dir)
_, name = os.path.split(d)
return name
def createPlugins(env, dir, **kw):
def createPlugins(env, directory, **kw):
""" investigate the given source directory to identify all contained source trees.
@return: a list of build nodes defining a plugin for each of these source trees.
"""
return [env.LumieraPlugin( getDirname(tree)
, srcSubtree(env, tree, appendCPP='LUMIERA_PLUGIN')
, srcSubtree(tree)
, **kw
)
for tree in findSrcTrees(dir)
for tree in findSrcTrees(directory)
]

186
admin/scons/IconSvgRenderer.py Executable file
View file

@ -0,0 +1,186 @@
#!/usr/bin/python
#
# IconSvgRenderer.py - Icon rendering utility script
#
# Copyright (C) Lumiera.org
# 2008, Joel Holdsworth <joel@airwebreathe.org.uk>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
import os
import sys
import getopt
import shutil
from xml.dom import minidom
rsvgPath = "./rsvg-convert"
artworkLayerPrefix = "artwork:"
#
# 2/2011 some notes by Ichthyo
# The purpose of this python script is
# - to parse a SVG
# - to invoke Inkscape to render this SVG into a raster image (icon)
#
# For the actual Cairo based SVG rendering we rely on an executable 'rsvg-convert',
# which is built during the Lumiera build process.
#
# Judging from the code and the actual SVGs, this seems to work as follows:
# The SVG contains a design to be rendered into raster images of various sizes.
# These sizes are determined by special rectangles, which act as bounding box and
# are placed on a special 'plate' layer, which is a child layer of the main
# 'artwork:' layer. The grid of the SVG is setup such as to result in pixel sizes
# suitable for icon generation. The actual size of the generated icons are then
# parsed from the height and width attributes of the mentioned bounding box
# rectangles.
#
# The parser seems to be rather simplistic; the sizes and positions need to be
# integral numbers. In one instance we had a float number in the y coordinate,
# which resulted in an invalid, zero sized output icon
#
#
def createDirectory (name):
try:
if os.path.isfile (name):
os.remove (name)
if not os.path.exists (name):
os.mkdir (name)
except:
print 'WARNING: createDirectory("%s") failed. Permission problems?' % name
def copyMergeDirectory (src, dst):
listing = os.listdir (src)
for file_name in listing:
src_file_path = os.path.join (src, file_name)
dst_file_path = os.path.join (dst, file_name)
shutil.copyfile (src_file_path, dst_file_path)
def getDocumentSize (svg_element):
width = float(svg_element.getAttribute("width"))
height = float(svg_element.getAttribute("height"))
return [width, height]
def findChildLayerElement (parent_element):
for node in parent_element.childNodes:
if node.nodeType == minidom.Node.ELEMENT_NODE:
if node.tagName == "g":
if node.getAttribute("inkscape:groupmode") == "layer":
return node
return None
def parsePlateLayer (layer):
rectangles = []
for node in layer.childNodes:
if node.nodeType == minidom.Node.ELEMENT_NODE:
if node.tagName == "rect":
x = float(node.getAttribute("x"))
y = float(node.getAttribute("y"))
width = float(node.getAttribute("width"))
height = float(node.getAttribute("height"))
rectangles.append([x, y, width, height])
return rectangles
def parseSVG (file_path):
print "Parsing " + file_path
svgdoc = minidom.parse (file_path)
for root_node in svgdoc.childNodes:
if root_node.nodeType == minidom.Node.ELEMENT_NODE:
if root_node.tagName == "svg":
size = getDocumentSize (root_node)
layer = findChildLayerElement (root_node)
if layer != None:
layer_name = layer.getAttribute ("inkscape:label")
if layer_name[:len(artworkLayerPrefix)] == artworkLayerPrefix:
artwork_name = layer_name[len(artworkLayerPrefix):]
plate = findChildLayerElement(layer)
if plate != None:
return artwork_name, size, parsePlateLayer(plate)
return None
def renderSvgRsvg (file_path, out_dir, artwork_name, rectangle, _doc_size):
# Prepare a Cairo context
width = int(rectangle[2])
height = int(rectangle[3])
if not os.path.exists(rsvgPath):
print "Error: executable %s not found." % rsvgPath
os.spawnlp(os.P_WAIT, rsvgPath, rsvgPath,
"--source-rect=%g:%g:%g:%g" % (rectangle[0], rectangle[1], width, height),
"--output=" + os.path.join(out_dir, "%gx%g/%s.png" % (width, height, artwork_name)),
file_path)
def renderSvgIcon (file_path, out_dir):
artwork_name, doc_size, rectangles = parseSVG (file_path)
for rectangle in rectangles:
renderSvgRsvg(file_path, out_dir, artwork_name, rectangle, doc_size)
def getTargetNames (file_path):
"""get a list of target names to be rendered from the given source SVG
usable to setup the build targets for SCons
"""
artwork_name, _ , rectangles = parseSVG (file_path)
return ["%gx%g/%s.png" % (rectangle[2], rectangle[3], artwork_name) for rectangle in rectangles ]
def printHelp():
print "render-icon.py SRCFILE.svg TARGETDIR"
print "An icon rendering utility script for lumiera"
def parseArguments(argv):
_optlist, args = getopt.getopt(argv, "")
if len(args) == 2:
return args[0], args[1]
printHelp()
return None, None
def main (argv):
in_path, out_dir = parseArguments(argv)
if not (in_path and out_dir):
print "Missing arguments in_path and out_dir."
sys.exit(1)
if os.path.isfile(out_dir):
print "Unable to use '%s' as output directory, because it\'s a file." % out_dir
sys.exit(1)
if not os.path.isdir(out_dir):
print "Output directory '%s' not found." % out_dir
sys.exit(1)
# Create the icons folders
createDirectory(os.path.join(out_dir, "48x48"))
createDirectory(os.path.join(out_dir, "32x32"))
createDirectory(os.path.join(out_dir, "24x24"))
createDirectory(os.path.join(out_dir, "22x22"))
createDirectory(os.path.join(out_dir, "16x16"))
renderSvgIcon (in_path, out_dir)
if __name__=="__main__":
main(sys.argv[1:])

97
admin/scons/LumieraEnvironment.py
View file

@ -22,11 +22,10 @@
#####################################################################
import os
from os import path
import SCons
import SCons.SConf
from SCons.Action import Action
from SCons.Environment import Environment
from Buildhelper import *
@ -38,9 +37,15 @@ class LumieraEnvironment(Environment):
This allows us to carry structured config data without
using global vars. Idea inspired by Ardour.
"""
def __init__(self, pathConfig, **kw):
Environment.__init__ (self,**kw)
self.path = Record (pathConfig)
def __init__(self, buildSetup, buildVars, **kw):
kw.update(VERSION = buildSetup.VERSION
,TARGDIR = buildSetup.TARGDIR
,DESTDIR = '$INSTALLDIR/$PREFIX'
,toolpath = [buildSetup.TOOLDIR ]
,variables = buildVars
)
Environment.__init__ (self, **kw)
self.path = Record (extract_localPathDefs(buildSetup)) # e.g. buildExe -> env.path.buildExe
self.libInfo = {}
self.Tool("BuilderGCH")
self.Tool("BuilderDoxygen")
@ -49,6 +54,7 @@ class LumieraEnvironment(Environment):
register_LumieraResourceBuilder(self)
register_LumieraCustomBuilders(self)
def Configure (self, *args, **kw):
kw['env'] = self
return apply(LumieraConfigContext, args, kw)
@ -123,7 +129,7 @@ def register_LumieraResourceBuilder(env):
used to generate png from the svg source using librsvg.
"""
import render_icon as renderer # load Joel's python script for invoking the rsvg-convert (SVG render)
import IconSvgRenderer as renderer # load Joel's python script for invoking the rsvg-convert (SVG render)
renderer.rsvgPath = env.subst("$TARGDIR/rsvg-convert")
def invokeRenderer(target, source, env):
@ -151,43 +157,44 @@ def register_LumieraResourceBuilder(env):
return (generateTargets, source)
def IconResource(env, source):
"""Copy icon pixmap to corresponding icon dir. """
subdir = getDirname(str(source))
toBuild = env.path.buildIcon+subdir
toInstall = env.path.installIcon+subdir
env.Install (toInstall, source)
return env.Install(toBuild, source)
"""Copy icon pixmap to corresponding icon dir. """
subdir = getDirname(str(source))
toBuild = env.path.buildIcon+subdir
toInstall = env.path.installIcon+subdir
env.Install (toInstall, source)
return env.Install(toBuild, source)
def GuiResource(env, source):
subdir = getDirname(str(source))
toBuild = env.path.buildUIRes+subdir
toInstall = env.path.installUIRes+subdir
env.Install (toInstall, source)
return env.Install(toBuild, source)
subdir = getDirname(str(source))
toBuild = env.path.buildUIRes+subdir
toInstall = env.path.installUIRes+subdir
env.Install (toInstall, source)
return env.Install(toBuild, source)
def ConfigData(env, source, targetDir=None):
""" install (copy) configuration- and metadata.
target dir is either the install location configured (in SConstruct),
or an explicitly given absolute or relative path segment, which might refer
to the location of the executable through the $ORIGIN token
"""
subdir = getDirname(str(source), env.path.srcConf) # removes source location path prefix
if targetDir:
if path.isabs(targetDir):
toBuild = toInstall = path.join(targetDir,subdir)
else:
if targetDir.startswith('$ORIGIN'):
targetDir = targetDir[len('$ORIGIN'):]
toBuild = path.join(env.path.buildExe, targetDir, subdir)
toInstall = path.join(env.path.installExe, targetDir, subdir)
else:
toBuild = path.join(env.path.buildConf, targetDir, subdir)
toInstall = path.join(env.path.installConf, targetDir, subdir)
else:
toBuild = path.join(env.path.buildConf,subdir)
toInstall = path.join(env.path.installConf,subdir)
env.Install (toInstall, source)
return env.Install(toBuild, source)
def ConfigData(env, prefix, source, targetDir=None):
""" install (copy) configuration- and metadata.
target dir is either the install location configured (in SConstruct),
or an explicitly given absolute or relative path segment, which might refer
to the location of the executable through the $ORIGIN token
"""
source = path.join(prefix,str(source))
subdir = getDirname(source, prefix) # removes source location path prefix
if targetDir:
if path.isabs(targetDir):
toBuild = toInstall = path.join(targetDir,subdir)
else:
if targetDir.startswith('$ORIGIN'):
targetDir = targetDir[len('$ORIGIN'):]
toBuild = path.join(env.path.buildExe, targetDir, subdir)
toInstall = path.join(env.path.installExe, targetDir, subdir)
else:
toBuild = path.join(env.path.buildConf, targetDir, subdir)
toInstall = path.join(env.path.installConf, targetDir, subdir)
else:
toBuild = path.join(env.path.buildConf,subdir)
toInstall = path.join(env.path.installConf,subdir)
env.Install (toInstall, source)
return env.Install(toBuild, source)
buildIcon = env.Builder( action = Action(invokeRenderer, "rendering Icon: $SOURCE --> $TARGETS")
@ -205,8 +212,8 @@ def register_LumieraResourceBuilder(env):
class WrappedStandardExeBuilder(SCons.Util.Proxy):
""" Helper to add customisations and default configurations to SCons standard builders.
The original builder object is wrapped and most calls are simply forwarded to this
wrapped object by Python magic. But some calls are intecepted in order to inject
suitalbe default configuration based on the project setup.
wrapped object by Python magic. But some calls are intercepted in order to inject
suitable default configuration based on the project setup.
"""
def __init__(self, originalBuilder):
@ -291,7 +298,7 @@ class LumieraModuleBuilder(WrappedStandardExeBuilder):
explicit spec, falling back on the lib filename
"""
if 'soname' in kw:
soname = self.subst(kw['soname']) # explicitely defined by user
soname = self.subst(kw['soname']) # explicitly defined by user
else: # else: use the library filename as DT_SONAME
if SCons.Util.is_String(target):
pathname = target.strip()
@ -331,7 +338,7 @@ class LumieraPluginBuilder(LumieraModuleBuilder):
def register_LumieraCustomBuilders (lumiEnv):
""" install the customised builder versions tightly integrated with our buildsystem.
""" install the customised builder versions tightly integrated with our build system.
Especially, these builders automatically add the build and installation locations
and set the RPATH and SONAME in a way to allow a relocatable Lumiera directory structure
"""
@ -362,7 +369,7 @@ def register_LumieraCustomBuilders (lumiEnv):
action = Action(makeLink, "Install link: $TARGET -> "+srcSpec)
env.Command (target,source, action)
# adding SymLink direclty as method on the environment object
# adding SymLink directly as method on the environment object
# Probably that should better be a real builder, but I couldn't figure out
# how to get the linktext through literally, which is necessary for relative links.
# Judging from the sourcecode of SCons.Builder.BuilderBase, there seems to be no way

81
admin/scons/Options.py Normal file
View file

@ -0,0 +1,81 @@
# -*- python -*-
##
## Options.py - SCons build: command line options and help
##
# Copyright (C) Lumiera.org
# 2012, Hermann Vosseler <Ichthyostega@web.de>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of
# the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#####################################################################
from SCons.Script import PathVariable, EnumVariable, BoolVariable, Help
def defineCmdlineVariables(buildVars):
""" several toggles and configuration variables can be set on the commandline,
current settings will be persisted in a options cache file.
you may define custom variable settings in a separate file.
Commandline will override both.
"""
buildVars.AddVariables(
('ARCHFLAGS', 'Set architecture-specific compilation flags (passed literally to gcc)','')
,('CC', 'Set the C compiler to use.', 'gcc')
,('CXX', 'Set the C++ compiler to use.', 'g++')
,PathVariable('CCACHE', 'Integrate with CCache', '', PathVariable.PathAccept)
,PathVariable('DISTCC', 'Invoke C/C++ compiler commands through DistCC', '', PathVariable.PathAccept)
,EnumVariable('BUILDLEVEL', 'NoBug build level for debugging', 'ALPHA', allowed_values=('ALPHA', 'BETA', 'RELEASE'))
,BoolVariable('DEBUG', 'Build with debugging information and no optimisations', False)
,BoolVariable('OPTIMIZE', 'Build with strong optimisation (-O3)', False)
,BoolVariable('VALGRIND', 'Run Testsuite under valgrind control', True)
,BoolVariable('VERBOSE', 'Print full build commands', False)
,('TESTSUITES', 'Run only Testsuites matching the given pattern', '')
# ,BoolVariable('OPENGL', 'Include support for OpenGL preview rendering', False)
# ,EnumVariable('DIST_TARGET', 'Build target architecture', 'auto',
# allowed_values=('auto', 'i386', 'i686', 'x86_64' ), ignorecase=2)
,PathVariable('PREFIX', 'Installation dir prefix', 'usr/local', PathVariable.PathAccept)
,PathVariable('INSTALLDIR', 'Root output directory for install. Final installation will happen in INSTALLDIR/PREFIX/... ', '/', PathVariable.PathIsDir)
,PathVariable('PKGLIBDIR', 'Installation dir for plugins, defaults to PREFIX/lib/lumiera/modules', '',PathVariable.PathAccept)
,PathVariable('PKGDATADIR', 'Installation dir for default config, usually PREFIX/share/lumiera', '',PathVariable.PathAccept)
)
def prepareOptionsHelp(buildVars,env):
prelude = """
USAGE: scons [-c] [OPTS] [key=val [key=val...]] [TARGETS]
Build and optionally install Lumiera.
Without specifying any target, just the (re)build target will run.
Add -c to the commandline to clean up anything a given target would produce
Special Targets:
build : just compile and link
research: build experimental code (might fail)
testcode: additionally compile the Testsuite
check : build and run the Testsuite
doc : generate documentation (Doxygen)
all : build and testcode and doc
install : install created artifacts at PREFIX
Configuration Options:
"""
Help(prelude + buildVars.GenerateHelpText(env))

149
admin/scons/Platform.py Normal file
View file

@ -0,0 +1,149 @@
# -*- python -*-
##
## Platform.py - SCons build: platform configuration and library detection
##
# Copyright (C) Lumiera.org
# 2012, Hermann Vosseler <Ichthyostega@web.de>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of
# the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#####################################################################
from SCons.Script import Exit
from Buildhelper import isCleanupOperation, isHelpRequest
def configure(env):
""" locate required libraries.
setup platform specific options.
Abort build in case of failure.
"""
if isCleanupOperation(env) or isHelpRequest():
return env # skip configure in these cases
conf = env.Configure()
# run all configuration checks in the build environment defined thus far
# Perform checks for prerequisites --------------------------------------------
problems = []
if not conf.TryAction('pkg-config --version > $TARGET')[0]:
problems.append('We need pkg-config for including library configurations, exiting.')
if not conf.CheckLibWithHeader('m', 'math.h','C'):
problems.append('Did not find math.h / libm.')
if not conf.CheckLibWithHeader('dl', 'dlfcn.h', 'C'):
problems.append('Functions for runtime dynamic loading not available.')
if not conf.CheckLibWithHeader('pthread', 'pthread.h', 'C'):
problems.append('Did not find the pthread lib or pthread.h.')
else:
conf.env.Append(CPPFLAGS = ' -DHAVE_PTHREAD')
conf.env.Append(CCFLAGS = ' -pthread')
if conf.CheckCHeader('execinfo.h'):
conf.env.Append(CPPFLAGS = ' -DHAVE_EXECINFO_H')
if conf.CheckCHeader('valgrind/valgrind.h'):
conf.env.Append(CPPFLAGS = ' -DHAVE_VALGRIND_H')
else:
print 'Valgrind not found. The use of Valgrind is optional; building without.'
if not conf.CheckPkgConfig('nobugmt', 201006.1):
problems.append('Did not find NoBug [http://www.lumiera.org/nobug_manual.html].')
else:
conf.env.mergeConf('nobugmt')
if not conf.CheckCXXHeader('tr1/memory'):
problems.append('We rely on the std::tr1 standard C++ extension for shared_ptr.')
if not conf.CheckCXXHeader('boost/config.hpp'):
problems.append('We need the C++ boost-libraries.')
else:
if not conf.CheckCXXHeader('boost/scoped_ptr.hpp'):
problems.append('We need boost::scoped_ptr (scoped_ptr.hpp).')
if not conf.CheckCXXHeader('boost/format.hpp'):
problems.append('We need boost::format (header).')
if not conf.CheckLibWithHeader('boost_program_options-mt','boost/program_options.hpp','C++'):
problems.append('We need boost::program_options (including binary lib for linking).')
if not conf.CheckLibWithHeader('boost_system-mt','boost/system/error_code.hpp','C++'):
problems.append('We need the boost::system support library (including binary lib).')
if not conf.CheckLibWithHeader('boost_filesystem-mt','boost/filesystem.hpp','C++'):
problems.append('We need the boost::filesystem lib (including binary lib for linking).')
if not conf.CheckLibWithHeader('boost_regex-mt','boost/regex.hpp','C++'):
problems.append('We need the boost regular expression lib (incl. binary lib for linking).')
if conf.CheckLib(symbol='clock_gettime'):
print 'Using function clock_gettime() as defined in the C-lib...'
else:
if not conf.CheckLib(symbol='clock_gettime', library='rt'):
problems.append('No library known to provide the clock_gettime() function.')
if not conf.CheckPkgConfig('gavl', 1.0):
problems.append('Did not find Gmerlin Audio Video Lib [http://gmerlin.sourceforge.net/gavl.html].')
else:
conf.env.mergeConf('gavl')
if not conf.CheckPkgConfig('alsa', '1.0.23'):
problems.append('Support for ALSA sound output is required')
if not conf.CheckPkgConfig('gtkmm-2.4', 2.8):
problems.append('Unable to configure GTK--')
if not conf.CheckPkgConfig('glibmm-2.4', '2.16'):
problems.append('Unable to configure Lib glib--')
if not conf.CheckPkgConfig('gthread-2.0', '2.12.4'):
problems.append('Need gthread support lib for glib-- based thread handling.')
if not conf.CheckPkgConfig('cairomm-1.0', 0.6):
problems.append('Unable to configure Cairo--')
verGDL = '2.27.1'
if not conf.CheckPkgConfig('gdl-1.0', verGDL, alias='gdl'):
print 'No sufficiently recent (>=%s) version of GDL found. Maybe use custom package gdl-lum?' % verGDL
if not conf.CheckPkgConfig('gdl-lum', verGDL, alias='gdl'):
problems.append('GNOME Docking Library not found. We either need a sufficiently recent GDL '
'version (>=%s), or the custom package "gdl-lum" from Lumiera.org.' % verGDL)
if not conf.CheckPkgConfig('librsvg-2.0', '2.18.1'):
problems.append('Need rsvg Library for rendering icons.')
if not conf.CheckCHeader(['X11/Xutil.h', 'X11/Xlib.h'],'<>'):
problems.append('Xlib.h and Xutil.h required. Please install libx11-dev.')
if not conf.CheckPkgConfig('xv') : problems.append('Need libXv...')
if not conf.CheckPkgConfig('xext'): problems.append('Need libXext.')
# report missing dependencies
if problems:
print "*** unable to build due to the following problems:"
for isue in problems:
print " * %s" % isue
print
print "build aborted."
Exit(1)
print "** Gathered Library Info: %s" % conf.env.libInfo.keys()
# create new env containing the finished configuration
return conf.Finish()

137
admin/scons/Setup.py Normal file
View file

@ -0,0 +1,137 @@
# -*- python -*-
##
## Setup.py - SCons build: setup, definitions and compiler flags
##
# Copyright (C) Lumiera.org
# 2012, Hermann Vosseler <Ichthyostega@web.de>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of
# the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#####################################################################
from SCons.Script import EnsurePythonVersion, EnsureSConsVersion, Variables, Decider
from LumieraEnvironment import *
from Buildhelper import *
import Options
#-------------------------------------------------------Configuration
TARGDIR = 'target'
VERSION = '0.pre.01'
TOOLDIR = './admin/scons' # SCons plugins
OPTCACHE = 'optcache'
CUSTOPTFILE = 'custom-options'
# these are accessible via env.path.xxxx
buildExe = '#$TARGDIR'
buildLib = '#$TARGDIR/modules'
buildPlug = '#$TARGDIR/modules'
buildIcon = '#$TARGDIR/gui/icons'
buildUIRes = '#$TARGDIR/'
buildConf = '#$TARGDIR/config'
installExe = '#$DESTDIR/lib/lumiera'
installLib = '#$DESTDIR/lib/lumiera/modules'
installPlug = '#$DESTDIR/lib/lumiera/modules'
installIcon = '#$DESTDIR/share/lumiera/icons'
installUIRes = '#$DESTDIR/share/lumiera/'
installConf = '#$DESTDIR/lib/lumiera/config'
#-------------------------------------------------------Configuration
buildSetup = Record(locals())
def defineBuildEnvironment():
""" create a custom build environment,
define the basic compiler and linker flags,
define locations in source and target tree,
parse the commandline and pick up options
"""
EnsureSConsVersion(1,0)
EnsurePythonVersion(2,4)
Decider('MD5-timestamp') # detect changed files by timestamp, then do a MD5
buildVars = Variables([OPTCACHE, CUSTOPTFILE])
Options.defineCmdlineVariables(buildVars)
env = LumieraEnvironment(buildSetup, buildVars)
env.Replace( CPPPATH =["#src"] # used to find includes, "#" means always absolute to build-root
, CPPDEFINES=['LUMIERA_VERSION='+VERSION ] # note: it's a list to append further defines
, CCFLAGS='-Wall -Wextra '
, CFLAGS='-std=gnu99'
)
handleVerboseMessages(env)
handleNoBugSwitches(env)
env.Append(CPPDEFINES = '_GNU_SOURCE')
appendCppDefine(env,'DEBUG','DEBUG', 'NDEBUG')
# appendCppDefine(env,'OPENGL','USE_OPENGL')
appendVal(env,'ARCHFLAGS','CCFLAGS') # for both C and C++
appendVal(env,'OPTIMIZE', 'CCFLAGS', val=' -O3')
appendVal(env,'DEBUG', 'CCFLAGS', val=' -ggdb')
# setup search path for Lumiera plugins
appendCppDefine(env,'PKGLIBDIR','LUMIERA_PLUGIN_PATH=\\"$PKGLIBDIR/:ORIGIN/modules\\"'
,'LUMIERA_PLUGIN_PATH=\\"ORIGIN/modules\\"')
appendCppDefine(env,'PKGDATADIR','LUMIERA_CONFIG_PATH=\\"$PKGLIBDIR/:.\\"'
,'LUMIERA_CONFIG_PATH=\\"$DESTDIR/share/lumiera/:.\\"')
Options.prepareOptionsHelp(buildVars,env)
buildVars.Save(OPTCACHE, env)
return env
def appendCppDefine(env,var,cppVar, elseVal=''):
if env[var]:
env.Append(CPPDEFINES = env.subst(cppVar) )
elif elseVal:
env.Append(CPPDEFINES = env.subst(elseVal))
def appendVal(env,var,targetVar,val=None):
if env[var]:
env.Append( **{targetVar: env.subst(val) or env[var]})
def handleNoBugSwitches(env):
""" set the build level for NoBug.
Release builds imply no DEBUG
whereas ALPHA and BETA require DEBUG
"""
level = env['BUILDLEVEL']
if level in ['ALPHA', 'BETA']:
if not env['DEBUG']:
print 'Warning: NoBug ALPHA or BETA builds requires DEBUG=yes, switching DEBUG on!'
env.Replace( DEBUG = 1 )
env.Append(CPPDEFINES = 'EBUG_'+level)
elif level == 'RELEASE':
env.Replace( DEBUG = 0 )
def handleVerboseMessages(env):
""" toggle verbose build output """
if not env['VERBOSE']:
# SetOption('silent', True)
env['CCCOMSTR'] = env['SHCCCOMSTR'] = " Compiling $SOURCE"
env['CXXCOMSTR'] = env['SHCXXCOMSTR'] = " Compiling++ $SOURCE"
env['LINKCOMSTR'] = " Linking --> $TARGET"
env['LDMODULECOMSTR'] = " creating module [ $TARGET ]"

3
admin/treeinfo.sh
View file

@ -1,2 +1,3 @@
#!/bin/sh
find -name DIR_INFO -printf '%-30h: ' -exec head -1 {} \; | sort | cut -c 1-92
COLUMNS=$(tput cols)
find -name DIR_INFO -printf '%-30h: ' -exec head -1 {} \; | sort -k1,1 | cut -c 1-${COLUMNS:-92}

9
configure.ac
View file

@ -110,7 +110,7 @@ AC_CHECK_FUNCS(clock_gettime,
AC_MSG_NOTICE([clock_gettime is in the clib]),
AC_CHECK_LIB(rt, clock_gettime,
[LUMIERA_LIBS="$LUMIERA_LIBS -lrt"],
AC_MSG_ERROR([No library providing clock_gettime found])
AC_MSG_ERROR([No library providing clock_gettime found])
)
)
@ -143,6 +143,13 @@ AC_CHECK_HEADER([boost/regex.hpp],
AC_MSG_ERROR([boost::regex missing (http://www.boost.org/)])
)
AC_CHECK_HEADER([boost/filesystem.hpp],
[LUMIERA_BOOST_LIBS="$LUMIERA_BOOST_LIBS -lboost_filesystem-mt"],
AC_MSG_ERROR([boost::filesystem missing (http://www.boost.org/)])
)
AC_LANG_POP([C++])
AC_SUBST(LUMIERA_BOOST_CFLAGS)
AC_SUBST(LUMIERA_BOOST_LIBS)

2
data/DIR_INFO
View file

@ -1 +1 @@
Additional configuration, icons, graphics etc. to be installed with Lumiera
Additional configuration, icons, graphics for install

25
data/SConscript Normal file
View file

@ -0,0 +1,25 @@
# -*- python -*-
##
## SConscript - SCons buildscript for Icons and Resources
##
from Buildhelper import scanSubtree
Import('env')
# define Icons to render and install
vector_icon_dir = 'icons/svg'
prerendered_icon_dir = 'icons/prerendered'
icons = ( [env.IconRender(f) for f in scanSubtree(vector_icon_dir, ['*.svg'])]
+ [env.IconResource(f) for f in scanSubtree(prerendered_icon_dir, ['*.png'])]
)
#define Configuration files to install (dir-prefix, name)
config = ( env.ConfigData('config','setup.ini', targetDir='$ORIGIN')
+ env.ConfigData('config','dummy_lumiera.ini')
)
Export('icons config')

26
data/icons/README Normal file
View file

@ -0,0 +1,26 @@
#
# Lumiera Icon Artwork
#
# Copyright (C) Lumiera.org
# 2008, Joel Holdsworth <joel@airwebreathe.org.uk>
#
# Icon Artwork and similar materials accompanying the Lumiera Application
# is dual-licensed under the GNU General Public License version 2 or above,
# and
# Creative Commons Attribution-ShareAlike 3.0 Unported License
#
#
This directory holds Icons and similar graphics resources for the Lumiera GUI.
- prerendered: Raster image Icons rendered into a selection of sizes
- svg: Scalable Vector Grahpics Icons, to be rendered into suitable sizes
by the build process. Rendering is done with the help of lib Cairo.
The build creates an executable from src/tools/rsvg-convert.c
The invocation of the icon rendering is done with the help of a
Python script IconSvgRenderer.py, which first parses the SVG document
and then invokes rsvg-convert to generate the raster images.

0
icons/prerendered/16x16/panel-assets.png → data/icons/prerendered/16x16/panel-assets.png
View file

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 540 B

After

Width:  |  Height:  |  Size: 540 B

0
icons/prerendered/16x16/panel-timeline.png → data/icons/prerendered/16x16/panel-timeline.png
View file

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 453 B

After

Width:  |  Height:  |  Size: 453 B

0
icons/prerendered/16x16/panel-viewer.png → data/icons/prerendered/16x16/panel-viewer.png
View file

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 631 B

After

Width:  |  Height:  |  Size: 631 B

0
icons/prerendered/22x22/panel-assets.png → data/icons/prerendered/22x22/panel-assets.png
View file

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 695 B

After

Width:  |  Height:  |  Size: 695 B

0
icons/prerendered/22x22/panel-viewer.png → data/icons/prerendered/22x22/panel-viewer.png
View file

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 788 B

After

Width:  |  Height:  |  Size: 788 B

0
icons/prerendered/32x32/panel-assets.png → data/icons/prerendered/32x32/panel-assets.png
View file

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 1 KiB

After

Width:  |  Height:  |  Size: 1 KiB

0
icons/prerendered/32x32/panel-viewer.png → data/icons/prerendered/32x32/panel-viewer.png
View file

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 1.5 KiB

After

Width:  |  Height:  |  Size: 1.5 KiB

0
icons/svg/app-icon.svg → data/icons/svg/app-icon.svg
View file

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 58 KiB

After

Width:  |  Height:  |  Size: 58 KiB

0
icons/svg/tool-arrow.svg → data/icons/svg/tool-arrow.svg
View file

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 19 KiB

After

Width:  |  Height:  |  Size: 19 KiB

0
icons/svg/tool-i-beam.svg → data/icons/svg/tool-i-beam.svg
View file

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 11 KiB

After

Width:  |  Height:  |  Size: 11 KiB

0
icons/svg/track-disabled.svg → data/icons/svg/track-disabled.svg
View file

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 7.2 KiB

After

Width:  |  Height:  |  Size: 7.2 KiB

0
icons/svg/track-enabled.svg → data/icons/svg/track-enabled.svg
View file

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 14 KiB

After

Width:  |  Height:  |  Size: 14 KiB

0
icons/svg/track-locked.svg → data/icons/svg/track-locked.svg
View file

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 14 KiB

After

Width:  |  Height:  |  Size: 14 KiB

0
icons/svg/track-unlocked.svg → data/icons/svg/track-unlocked.svg
View file

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 20 KiB

After

Width:  |  Height:  |  Size: 20 KiB

0
icons/timeline-panel.svg → data/icons/timeline-panel.svg
View file

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 14 KiB

After

Width:  |  Height:  |  Size: 14 KiB

17
doc/SConscript Normal file
View file

@ -0,0 +1,17 @@
# -*- python -*-
##
## SConscript - SCons buildscript for Documentation
##
from Buildhelper import scanSubtree
Import('env')
doxydoc = env.Doxygen('devel/Doxyfile')
# env.Install(dir = '$DESTDIR/share/doc/lumiera$VERSION/devel', source=documentation)
env.Clean (doxydoc, doxydoc + ['devel/,doxylog','devel/warnings.txt'])
Export('doxydoc')

1
doc/design/DIR_INFO Normal file
View file

@ -0,0 +1 @@
design and concepts

36
doc/design/architecture/ArchitectureSummary.txt Normal file
View file

@ -0,0 +1,36 @@
Lumiera Architecture: Executive Summary
=======================================
:Author: Hermann Voßeler
:Email: <Ichthyostega@web.de>
:Date: Sept 2010
//Menu: label Summary
.Three Layers
Besides the (obvious) separation in GUI frontend and render engine, editing core and application logic
are treated separately. Besides enforcing clearly defined interfaces, the ability for script driven
``headless'' operation and extensibility by plug-ins are deeply rooted in the overall design.
.Strong Separation
High-level and low-level realm remain cleanly separated, relying on a compiler-like translation mechanism.
This way, each part can be optimised for its purpose. The GUI exposes a feature-rich session model close
to the problem domain, while the engine operates on a performance optimised render nodes graph.
.Scheduling Micro Operations
High performance calculations in the engine are organised as micro tasks to be scheduled,
allowing for adaptation to current and future hardware, expected to operate increasingly parallelised.
.Intelligent IO
The engine is backed by an sophisticated frame cache and self adjusting I/O manager for maximising throughput
and bandwidth use.
.Stream Type System
Instead of relying on a single media framework, data streams are abstracted and classified, allowing to
handle most wiring, detection of possible connections and conversion automatically.
.Rules Based Configuration
Rather than driven by hard wired default behaviour or tedious manual operation, control flow is guided by
a set of rules shaped configuration, which scales better to large and complicated projects. Queries are
issued at decision points, allowing to build up a work environment customised and tailored for the needs
of specific projects.

212
doc/design/architecture/TimeQuant.txt Normal file
View file

@ -0,0 +1,212 @@
Time values and Time quantisation
=================================
:Author: Hermann Voßeler
:Email: <Ichthyostega@web.de>
:Date: Spring 2011
//Menu: label Time Quantisation
.Definitions
The term &raquo;Time&laquo; spans a variety of vastly different entities. +
Within a NLE we get to deal with various _flavours of time values._
continuous time::
without any additional assumptions, *points in time* can be specified with
arbitrary precision. The time values are just numbers; the point of reference
and the meaning is implicit. Within Lumiera, time is encoded as integral
number of _micro ticks,_ practically continuous
time distance::
a range of time, a *distance* on the time axis, measured with the same
arbitrary precision as time points. Distances can be determined by
_subtracting_ two time points, consequently they are _signed numbers._
offset::
a distance can be used to adjust (offset) a time or a duration: this means
applying a relative change. The _target_ of an offset operation is a time or
duration, while it's _argument_ is a distance (synonymous to offset).
duration::
the length of a time range yields a *time metric*.
+
the duration can be defined as the _absolute value_ of the offset between
start and endpoint of the time range. A duration always abstracts from the
time _when_ this duration or distance happens, the relation to any time scale
remains implicit
time span::
contrary to a mere duration, a *time interval* or time span is actually
_anchored_ at a specific point in time. It can be seen as a _special kind of
duration,_ which explicitly states the information _when_ this time span takes
place.
changing time::
Time values are _immutable,_ like numbers.
+
Only a *time variable* can be changed -- yet some of the special time entities
can recieve link:TimeMutation[mutation messages], allowing e.g. for
adjustments to a time interval selection from the GUI
''''
internal time::
While the basic continuous time values don't imply any commitment regarding
the time scale and origin to be used, actually, within the implementation of
the application, the meaning of time values is uniform and free of
contradictions. Thus effectively there is an *implementation time scale* --
but its scope of validity is _strictly limited to the implementation level of
a single application instance._ It is never exposed and never persisted. It
might not be reproducible over multiple instantiations of the application. The
implementation reserves the right to recalibrate this internal scale. Later,
when Lumiera gains the capability to run within a network of render nodes,
these instance connections will include a negotiation about the internal time
scale, which remains completely opaque to the outer world. This explains, why
`lumiera::Time` instances lack the ability to show their time value beyond
debugging purposes. This is to avoid confusion and to stress their opaque
nature.
wall clock and system time::
The core property of any external real world time is that it is _running_ --
we have to synchronise to an external time source. This implies the presence
of a _running synchronisation process,_ with the authority to adjust the time
base;
+
contrast this to the internal time, which is static and unconnected --
quantised time::
The *act of quantisation* transforms a continuous property into a *discrete*
structure. Prominent examples can be found in the domain of micro physics and
with digital information processing. In a broader sense, any measurement or
_quantification_ also encompasses a quantisation. Regarding time and time
measurement, quantisation means alignment to a predefined *time grid*.
Quantisation necessarily is an _irreversible process_ -- possible
additional information is discarded.
+
Note that quantisation introduces a *time origin* and a *reference scale*
frame count::
within the context of film and media editing, the specification of a *frame number*
is an especially important instance of quantisation.
+
all the properties of quantisation apply indeed to this special case: it is a
time measurement or specification, where the values are aligned to a grid, and
there is a reference time point where the counting starts (origin) and a
reference scale (frames per second). Handling of quantised time values in
Lumiera is defined such as to ensure the presence of all those bits of
information. Without such precautions, operating with bare frame numbers leads
itself to all kinds of confusions, mismatches, quantisation errors and
unnecessary limitations of functionality.
timecode::
Quantisation also is the foundation of all kinds of formalised time specifications
actually even a frame count is some kind of (informal) timecode -- other timecodes
employ a standardised format. //Every// presentation of time values and every
persistent storage and exchange of such values is based on time codes.
Yet quantisation and time code aren't identical: a given quantised time value
typically can be cast into multiple timecode formats.
Patterns for handling quantised time
------------------------------------
When it comes to actually handling quantised time values, several patterns are
conceivable for dealing with the quantisation operation and representing
quantised data. As guideline for judging these patterns, the general properties
of time quantisation, as detailed above, should be taken into account.
Quantising a time value means both _discarding information,_ while at the same
time _adding explicit information_ pertaining the assumptions of the context.
.casual handling
this is rather an frequently encountered *anti pattern*. When reading such
code, the most striking observation is the sense of general unawareness of the
problem, which is then ``discovered'' on a per case base, which leads to
numerous repetitions of the same basic undertakings, but done with individual
treatment of each instance (not so much copy-n-paste). +
Typical code smells:
* the rounding, modulo and subtract-base operations pertinent with scale
handling are seemingly inserted as bugfix
* local code path forks to circumvent or compensate for otherwise hard wired
calculations based on specific ways to invoke a function
* playing strikingly clever tricks or employing heuristics to "figure out" the
missing scale information from accessible context after the fact
* advertising support for some of the conceivable cases as special feature, or
adding it as plugin or extension module with limited scope
* linking parts of the necessary additional information to completely
unrelated other structures, thus causing code tangling and scattering
* result or behaviour of calculations depends on the way things are set up in
a seemingly contingent way, forcing users to stick to very specific procedures
and ordered steps.
.static typing
an analysis of the cases to be expected establishes common patterns and some
base cases, which are then represented by distinct types with well established
conversions. This can be combined with generic programming for the common parts.
Close to the data input, a factory establishes these statically typed values.
.Summary
****************************************************************************
Lumiera uses (fixed) static typing for the plain _time values_, with a class
wrapping a simple 64bit integer as common denominator. The strict typing is
used to enforce a sane conversion path to quantised time values. For these
grid alinged values, and especially for the common time codes, the principle
of _delayed quantisation_ is preferred; the implementation relies on runtime
typing with type erasure, but provides conversion to statically tagged values
for the most important special formats
image:{imgd}/uml.time-entities.png["Time and Time Quantisation in Lumiera"]
****************************************************************************
.tagged values
quantised values are explicitly created out of continuous values by a quantiser
entity. These quantised data values contain a copy of the original data,
adjusted to be exactly aligned with respect to the underlying time grid. In
addition, they carry a tag or ID to denote the respective scale, grid or
timecode system. This tag can be used later on to assess compatibility or to
recast values into another timecode system.
.delayed quantisation
with this approach, the information loss is delayed as long as possible.
Quantised time values are rather treated as promise for quantisation, while the
actual time data remains unaltered. Additionally, they carry a tag, or even a
direct link to the responsible quantiser instance. Effectively, these are
specialised time values, instances of a sub-concept, able to stand-in for
general time values, but exposing additional accessors to get a quantised value.
discussion
~~~~~~~~~~
For Lumiera, the static typing approach is of limited value -- it excels when
values belonging to different scales are actually treated differently. There are
such cases, but rather on the data handling level, e.g. sound samples are always
handled block wise. But regarding time values, the unifying aspect is more
important, which leads to prefering a dynamic (run time typed) approach, while
_erasing_ the special differences most of the time. Yet the dynamic and open
nature of the Lumiera high-level model favours the delayed quantisation pattern;
the same values may require different quantisation depending on the larger model
context an object is encountered in. This solution might be too general and
heavy weight at times though. Thus, for important special cases, the accessors
should return tagged values, preferably even with differing static type. Time
codes can be integrated this way, but most notably the *frame numbers* used
for addressing throughout the backend, can be implemented as such specifically
typed tagged values; the tag here denotes the quantiser and thus the underlying
grid -- it should be implemented as hash-ID for smooth integration with code
written in plain C.
At the level of individual timecode formats, we're lacking a common denominator;
thus it is preferrable to work with different concrete timecode classes through
_generic programming._ This way, each timecode format can expose operations
specific only to the given format. Especially, different timecode formats expose
different _component fields,_ modelled by the generic *Digxel* concept.
There is a common baseclass `TCode` though, which can be used as marker
or for _type erasure._
-> more on link:TimeUsage.html[usage situations]
////
// TODO integrate those cross links
+
-> Timecode link:TimecodeFormat[format and quantisation]
+
-> Quantiser link:QuantiserImpl[implementation details]
////

186
doc/design/architecture/TimeUsage.txt Normal file
View file

@ -0,0 +1,186 @@
Time Usage Situations
=====================
:Author: Hermann Voßeler
:Email: <Ichthyostega@web.de>
:Date: Spring 2011
//Menu: label Time Usage
the following collection of usage situations helps to shape the details of the
time values and time quantisation design. +
-> see also link:TimeQuant.html[more about time quantisation]
time position of an object::
indeed the term ``time position'' encompasses two quite different questions
+
* a time or timing specification within the object
* determining the time point in reference to an existing scale
time and length of an object::
basically the same situation, +
but the length of a time span (duration) could be treated in two ways for quantisation
+
* having a precise specification and then quantise the start and endpoint
* quantise the start position and then establish an (independently quantised length)
moving and resizing an object::
this can in itself be done in two different ways, and each of them can be
applied in a quantised flavour, which sums up to 8 possible combinations,
considering that position and length are 2 degrees of freedom.
+
* a variable can be _changed_ by an offset
* a variable can be _defined_ to a new value
* another (hidden) degree of freedom lies in how to apply an quantised offset
to an unquantised value (and reversed), because this operation might be done
both in the quantised or non-quantised domain, and also the result might be
(un)quantised
updating the playback position::
this can be seen as a practical application of the above; basically we can
choose to show the wall clock time or we can advance the playback position in
frame increments, thus denoting the frame currently in display. For video,
these distinctions may look moot, but they are indeed relevant for precise
audio editing, especially when combined with loop playback (recall that audio
is processed block wise, but the individual sample frames and thus the
possible loop positions are way finer than the processing block size)
dispatching individual frames for calculation::
when a link:PlayProcess[render or playback process] is created, at some point
we need to translate this logical unit (``calculation stream'') into individual
frame job entries. This requires to break continuous time into individual
frames, and then ennumerating these frames.
displaying time intervals::
for display, time intervals get _re-quantised_ into display array
coordinates. While evidently the display coordinates are themselves quantised
and we obviously don't want to cancel out the effect of an quantisation of the
values or intervals to be displayed (which means, we get two quantisations
chained up after each other), there remains the question if the display array
coordinates should be aligned to the grid of the _elements to be displayed,_
and especially if the allowed zoom factors should be limited. This decision
isn't an easy one, as it has an immediate and tangible effect on what can be
showed, how reversible and reproducible a view is and (especially note this!)
on the actual values which can be set and changed through the GUI.
time value arithmetic::
Client code as well as the internal implementation of time handling needs to
do arithmetic operations with time values. Time values are additive and
totally ordered. Distance, as calculated by subtraction, can be made into a
metric. Another and quite different question is to what extent a quantised
variant of this arithmetics is required.
relative placement::
because of the divergence between quantised and unquantised values the
question arises, if placement relative to another object refers to the raw
position or the already quantised position. Basically all the variations
discussed for _time and length of an object_ also do apply here.
notable issues
--------------
*Direct quantisation of length is not possible*. This is due to the non-linear
nature of all but the most trivial time grids: Already such a simple addition
like a start offset destroys linearity, and this still the more is true within a
compound grid where the grid spacing changes at some point. Thus, the length has
to be re-established at the target position of an time interval after each
change involving quantisation. Regarding the _strategy_ to apply when
re-establishing the length, it seems more appropriate to treat the object as an
entity which is moved, which means to do quantisation in two steps, first the
position, then the endpoint (the second option in the description above). But it
seems advisable not to hard wire that strategy -- better put it into the
quantiser.
We should note that the problems regarding quantised durations also carry over
to _offsets:_ it is difficult to *define the semantics of a quantised offset*.
Seemingly the only viable approach is to have a _intended offset,_ and then
to apply a re-quantisation to the target after applying the (raw) offset.
*When to materialise a quantisation*. Because of the basic intention to retain
information, we prefer to delay the actual applicatio of the quantisation operation
to the values stored internally as much as possible. But not materialising
immediately at quantisation has the downside of possibly accumulating off-grid
values without that being evident. Most notably, if we apply the raw offsets
produced by GUI interactions, the object's positions and lengthes are bound to
accumulate spurious information never intended by the user.
Thus, an especially important instance of that problem is *how to deal with
updates in a quantised environment*. If we handle quantisation stictly as a
view employed on output, we run into the problems with accumulating spurious
information. On the other hand, allowing for quantised changes inevitably pulls
in all the complexity of mixing quantised and non-quantised values. It would be
desirable somehow to move these distinctions out of the scope of this design and
offload them onto the client (code using these time classes).
Another closely related problem is *when to allow mutations*, if at all (-> see
link:TimeMutation[more here...]). We can't completely do away with mutations,
simply because we don't have a pure functional language at our disposal. The
whole concept of _reference semantics_ doesn't play so well with immutable
objects. The Lumiera high-level (session) model certainly relies on objects
intended to be _manipulated._ Thus we need a re-settable length field in
`MObject` and we need a time variable for position calculations. Yet we
could make any _derived objects_ into immutable descriptor records, which
certainly helps with parallelism.
The *problem with playback position* is -- that indeed it's an attempt to
conceptualise a non-existing entity. There is no such thing like ``the'' playback
position. Yet most applications I'm aware off _do_ employ this concept. Likely
they got trapped by the metaphor of the tape head, again. We should do away with
that. On playback, we should show a _projection of wall-clock time onto the
expected playback range_ -- not more, not less. It should be acknowledged that
there is _no direct link to the ongoing playback processes,_ besides the fact
that they're assumed to sync to wall-clock time as well. Recall, typically there
are multiple playback processes going on in compound, and each might run on a
different update rate. If we really want a _visual out-of-sync indicator,_ we
should treat that as a separate reporting facility and display it apart of the
playback cursor.
An interesting point to note for the *frame dispatch step* is the fact that
in this case quantised values and quantisation are approached in the reverse
direction, compared with the other uses. Here, after establishing a start point
on the time scale, we proceed with ennumerating distinct frames and later on
need to access the corresponding raw time, especially to find out about the
link:Segmentation[timeline segment] to address, or for retrieving parameter
automation. -> link:FrameDispatcher[see frame dispatching].
Note that the *display window might be treated as just an independent instance
of quantisation*. This is similar to the approach taken above for modifying
quantised time span values. When following this line of thought, we should
provide a special kind of time grid, the display coordinates. The origin of
these is always defined to the left (lower) side of the interval to be
displayed, and they are gauged in screen units (pixels or similar, as used by
the GUI toolkit set). The rest is handled by the general quantisation
mechanisms. The problem of aligning the display should be transformed into a
general facility to align grids, and solved for the general case. Doing so
solves the remaining problems with quantised value changes and with *specifying
relative placements* as well: If we choose to represent them as quantised
values, we might (or might not) also choose to apply this _grid-alignment function._
the complete time value usage cycle
-----------------------------------
The way time value and quantisation handling is designed in Lumiera creates a
typical usage path, which actually is a one-way route. We might start out with a
textual representation according to a specific *timecode* format. Assumed we
know the implicit underlying *time grid* (coordinate system, framerate), this
timecode string may be parsed. This brings us (back) to the very origin, which
is a raw `TimeValue` (*internal time* value). Now, this time value might be
manipulated, compared to other values, combined into a *time span* (time point
and duration -- the most basic notion of an _object_ to be manipulated in the
Session). Anyway, at some point these time values need to be related to some
*time scale* again, leading to *quantised* time values, which -- finally --
can be cast into a timecode format for external representation again, thus
closing the circle.
substantial problems to be solved
---------------------------------
* how to link:TimeGridAlignment[align multiple grids]
* how to integrate link:TimeMutation[modifications of quantised values]. ([green]#✔ solved#)
* how to isolate the Time/Quantisation part from the grid MetaAsset in the
session -> we use the link:Advice[Advice] system ([green]#✔ solved#)
* how to design the relation of Timecode, Timecode formatting and Quantisation
-> link:TimecodeFormat[more here] [yellow-background]#WIP#

20
doc/design/architecture/index.txt Normal file
View file

@ -0,0 +1,20 @@
Design Documents: Lumiera Architecture
======================================
This section will feature some rather high-level considerations regarding
the architecture of the application.
.Executive Summary
See link:ArchitectureSummary.html[here] for a short summary of the core architecture
ideas of the envisioned Lumiera Application.
.Rendering and Playback Control
This part of the architecture is concened with how to initiate, coordinate and control
the calculation processes necessary to generate visible / audible data from our source
media. This design is _not so much_ concerned with the actual calculations; indeed it
takes on an ``bird eye's view'', treating the actual render engine mostly just as an
opaque entity providing some service.
-> link:playRender.html[Design: Playback and Rendering]

191
doc/design/architecture/playRender.txt Normal file
View file

@ -0,0 +1,191 @@
Design: Playback and Render Control
===================================
:Author: Ichthyostega
:Date: 5/2011
//MENU: label Player subsystem
**************************************************************************************
This part of the architecture is concened with how to initiate, coordinate and control
the calculation processes necessary to generate visible / audible data from our source
media. This design is _not so much_ concerned with the actual calculations; indeed it
takes on an ``bird eye's view'', treating the actual render engine mostly just as an
opaque entity providing some service.
**************************************************************************************
Collecting initial drafts
-------------------------
At start of 2011, several design sketches emerged already, as an offspring of other
discussions and brainstorming.
* the link:/documentation/devel/rfc_pending/EngineInterfaceOverview.html[Engine Interface draft]
contains some fundamental terms and definitions.
* discussions regarding time values and timecode handling shed some light onto the requirements
to be fulfilled by a player subsystem
- link:http://article.gmane.org/gmane.comp.video.lumiera.general/2116[Timecode Widget and Framerates]
- link:http://article.gmane.org/gmane.comp.video.lumiera.general/2109[Time, Time-spans, Quatisation and Timecode]
- link:http://lumiera.org/wiki/renderengine.html#TimeQuant[Definition of Time entities]
from the implementation notes in the TiddlyWiki
Reasoning about the Player Subsystem's structure
------------------------------------------------
At first sight, the name ``Player'' might be surprising.
The full, complete and precise name would be something along
the lines of ``_render- and playback-process coordination subsystem_''.
Now, we need to shorten that into a single word. The more obvious
abbreviation (just by the importance) would be _Render_, but that
would be strikingly misleading, because everyone would take that
to be the render engine. Which leaves us with _Player_. A second
consideration highlights similarities with the structure of a
typical software player -- the chosen term turns out to be well
aligned with the actual nature of that subsystem.
Influences and Requirements
~~~~~~~~~~~~~~~~~~~~~~~~~~~
In accordance with the spirit of modern software development, the analysis
starts by determining which would be the _Service_ to be provided by such
a player. Consideration of the use cases determines the fundamental forces
to be _multiplicity_, combined with _differentiation in detail_, all under
the government of _time-bound delivery_, combined with _live reconfiguration_.
The basic service turns out to be the *performing of a preconfigured object model*.
This performance is *time based*. Multiple usage instances of this service can
be expected to coexist, each of which can be broken down into a set of *elementary
streams to be delivered in sync*. The parameters of delivery can be *reconfigured
on-the-fly*.
Modes of operation
^^^^^^^^^^^^^^^^^^
Delivery can be either _free-wheeling_, so to cover a predefined time interval
with fixed quality calculations (final render), and it can be _throttled_ for using
excess computation power (background rendering). Or -- alternatively -- delivery can
be _time-bound_ (for classical playback), following the projection of wall clock time
onto a predefined timeline interval (the so called ``playhead'' or ``playback cursor''
proceeding along a timeline).
Reconfiguration
^^^^^^^^^^^^^^^
Some of these operation modes need to be prepared to encounter an unpredictable live
reconfiguration, driven by user interactions:
- any part of background rendering can be invalidated and restarted, while other parts
should be re-integrated, possibly with adjusted position
- the linearly proceeding playback can at any time be
* paused
* reversed in direction
* playback speed adjusted
- the playback can be looped, with _unlimited_ adjustments of the loop boundaries any time.
Conclusions for the Design
~~~~~~~~~~~~~~~~~~~~~~~~~~
Based on these observations, the following design looks pretty much obvious:
Overall, the player subsystem can be described as ``play/render-this''-service.
Given an suitable (high-level) object, the Player has the ability to ``perform''
(play or render) it.
- the standard case is _playing a timeline_.
- it's conceivable to allow playing a selection of other objects,
like e.g. directly playing a clip or even media asset. In these cases,
its the player's job to prepare the necessary scaffolding on the fly.
Yet each such performance of an object is a _stateful instance_, a player application:
On application of the player service, the client gets a front-end handle, a *play-controller*,
which is a _state machine_. It provides states and transitions of the kind 'play', 'pause', 'ffwd',
'rew', 'goto', 'step', 'scrub' and similar. Moreover it maintains (or connects to) a distinct
playback location, and it can be hooked up with a play-control GUI widget
(or something simpler in case of a render process, which is free wheeling).
Each play-controller in turn gets associated with several *play/render-processes*,
one for each independent media stream (channel) to be produced. Of course this
isn't an operating system process; rather, each such process is a compound of entries
in a registration table, which serve the purpose of tying several other services together,
which we initiate and use in order to make that render process happen.
Most notably, we'll use the services of the actual engine, which provides us with kind of
a *calculation stream service*: the ability to deliver a sequence of calculated
data frames in a timely fashion.
When a client requests such an instance of the player service, we build up these
parts providing that service, which cascades down to the individual elements.
At that point, we need to pull and combine two kinds of informations:
- the ``what'' to render: this information stems from the session/model.
- the ``how'' to render: this information is guided by the derived output configuration.
Viewer and Output connection
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Creating a player instance binds together three partners: a _timeline_, a _viewer_
and _the engine_. While the timeline provides the content to play, the _viewer connection_
is crucial for working out the actual output sink(s) and thus the output format to use.
Thus, a viewer connection is prerequisite for creating a player instance.
Viewer connections exist as associations in the session/model -- as entities separate
from the player. Usually, a timeline has (at least) one viewer connection. But in case
such a connection is (still) missing, building a player instance recurs to the session
to get a suitable viewer _allocated_. The viewer connection can't be broken during the
lifetime of that player instance (or putting it the other way: breaking that viewer
connection, e.g. by forcing a different connection or by shutting down the viewer,
immediately terminates the player. This detaching works synchronously, i.e. it
blocks until all the allocated _output slots_ could be released.
Live switching
^^^^^^^^^^^^^^
While the viewer connection can be treated as fixed during the lifespan of a player
instance, several life switching and reconfiguration operations might happen any time:
The _model port_ (place where data is retrieved from calculation), the output characteristics
(framerate, direction) and the delivery goals (playback position, loop playing, scrubbing)
all may be changed during playback -- we need a way for the player to ``cancel'' and
reconfigure the backend services.
Frame quantisation
^^^^^^^^^^^^^^^^^^
Quantisation is a kind of rounding; like any kind of rounding, quantisation is
a dangerous operation because it kills information content.
Thus there are three fundamental guidelines when it comes to rounding
. don't do it
. do it at most once
. do it as late as possible
These may guide the process of finding the right place for the Quantiser(s) to
apply. We need some information flows to be joined in order to be able to do
the quantisation, which leaves us with just a few possible junction points
where to place quantisation: The backend, the GUI, the player, the session.
- putting it into the backend seems to be the most reasonable at first sight:
We can ``do away'' with nasty things soon, especially if they are technicalities,
``get a clean state soon'' -- and hasn't frame quantisation something to do
with media data, which is handled in the backend?
+
Well, actually, all of those are pitfalls to trap the unwary. About
cleanliness, well (sigh). Doing rounding soon will leave us with a huge
amount of degraded information flows throughout the whole system; thus the
general rule to do it as late as possible. Uncrippled information is
enablement. And last but not least: the frame quantisation is connected
to the _output_ format -- and the backend is likely within the whole
application the subsystem most remote and unaware of output requirements.
- rounding/quantising in the GUI is extremely common within media applications;
unfortunately there seems to be not a single rational argument supporting that habit.
Most of all, it violates the subsidiarity principle.
Which leaves us with the player and the session. Both positions could
arguably be supported. Here, a more careful consideration shows, that
the ``act of frame rounding'' can be decomposed: into the _act of quantisation_
and the _frame grid:. Basically its the session which has the ability
to form the *frame grid*, but it is lacking crucial information about
the output. Only when connecting both -- which is the essence of the
player -- frame quantisation can actually be performed. Thus, the
player is the natural location to perform that quantisation operation.

25
doc/design/backend/Scheduler.txt Normal file
View file

@ -0,0 +1,25 @@
The Scheduler
-------------
:Author: CehTeh
:Date: 6/2007
//MENU: label Scheduler
Scheduling is done with two priority queues, one for high priority jobs and one for low priority jobs.
These priority queues are ordered by absolute time values plus some job specific identified.
There are following (non exhaustive) kinds of jobs:
* started job
* job to be canceled
* unscheduled job
* dependency providing jobs
Jobs implement a kind of future. We try hard to avoid any blocking waits.
The Job scheduler runs singlethreaded. Its only task is to schedule and delegate jobs to worker threads,
by itself it will never do any extensive processing.
Each job has an pre configured behaviour for the case of failure or deadline miss.
Any canceling and expireing jobs gets noted in *Statistics* to adjust performance and timings
for optimal performance and I/O throughput.

111
doc/design/backend/index.txt
View file

@ -1,5 +1,114 @@
Design Documents: Backend
=========================
Eventually, this will have design documentation for the Backend.
What follows is a summary regarding the design of Lumiera's *Data Handling Backend*
This is the foundation layer responsible for any high performance or high volume
data access. Within Lumiera, ther are two main kinds how data is handled:
* The Session and the object models manipulated through the GUI is kept in memory.
It is backed by a _storage backend,_ which provides database like storage and
especially logging, replaying and ``Undo'' of all ongoing modifications..
* Media data is handled _frame wise_ -- as described below.
The backend uses *memory mapping* to make data available to the program.
This is somewhat different to the more common open/read/write/close file access,
while giving superior performance and much better memory utilization.
The data backend must be capable to handle more data than will fit into the memory
or even address space on 32 bit architectures. Moreover, a project might access more files
than the OS can handle at a any time, thus the for _Files used by the Backend,_ it needs a
*FilehandleCache* to manage file handle dynamically.
Which parts of a file are actually mapped to physical RAM is managed by the kernel;
it keeps a *FileMapCache* to manage the *FileMaps* we've set up.
In the End, the application itself only requests *Data Frames* from the Backend.
To minimize latency and optimize CPU utilization we have a *Prefetch thread* which operates
a *Scheduler* to render and cache frames which are _expected to be consumed soon_. The intention
is to manage the rendering _just in time_.
The prefetcher keeps *Statistics* for optimizing performance.
Accessing Files
---------------
+FileDescriptor+ is the superclass of all possible filetypes, it has a weak reference to a
+FileHandle+ which is managed in within the +FilehandleCache+. On creation, only the existence
(when reading) or access for write for new files are checked. The +FileDescriptor+ stores some
generic metadata about the underlying file and intended use. But the actual opening is done on demand.
The _content of files is memory mapped_ into the process address space.
This is managed by +FileMap+ entries and a +FileMapCache+.
File Handles
~~~~~~~~~~~~
A +FilehandleCache+ serves to store a finite maximum number of +FileHandles+ as a MRU list.
FileHandles are managed by the +FilehandleCache+; basically they are just storing the underlying OS file
handles and managed in a lazy/weak way, (re)opened when needed and aging in the cache when not needed,
since the amount of open file handles is limited aged ones will be closed and reused when the system
needs to open another file.
File Mapping
~~~~~~~~~~~~
The +FileMapCache+ keeps a list of +FileMaps+, which are currently not in use and subject of aging.
Each +FileMap+ object contains many +Frames+. The actual layout depends on the type of the File.
Mappings need to be _page aligned_ while Frames can be anywhere within a file and dynamically sized.
All established ++FileMap++s are managed together in a central +FileMapCache+.
Actually, +FileMap+ objects are transparent to the application. The upper layers will just
request Frames by position and size. Thus, the +File+ entities associate a filename with the underlying
low level File Descriptor and access
Frames
~~~~~~
+Frames+ are the smallest datablocks handled by the Backend. The application tells the Backend to make
Files available and from then on just requests Frames. Actually, those Frames are (references to) blocks
of continuous memory. They can be anything depending on the usage of the File (Video frames, encoder frames,
blocks of sound samples). Frames are referenced by a smart-pointer like object which manages the lifetime
and caching behavior.
Each frame referece can be in one out of three states:
readonly::
the backing +FileMap+ is checked out from the aging list, frames can be read
readwrite::
the backing +FileMap+ is checked out from the aging list, frames can be read and written
weak::
the +FileMap+ object is checked back into the aging list, the frame can't be accessed but we can
try to transform a weak reference into a readonly or readwrite reference
Frames can be addressed uniquely whenever a frame is not available. The backend can't serve a cached
version of the frame, a (probably recursive) rendering request will be issued.
Prefetching
~~~~~~~~~~~
There are 2 important points when we want to access data with low latency:
. Since we handle much more data than it will fit into most computers RAM.
The data which is backed in files has to be paged in and available when needed.
The +Prefetch+ Thread manages page hinting to the kernel (posix_madvise()..)
. Intermediate Frames must eventually be rendered to the cache.
The Backend will send +Renderjobs+ to the +Scheduler+.
Whenever something queries a +Frame+ from the backend it provides hints about what it is doing.
These hints contain:
* Timing constraints
- When will the +Frame+ be needed
- could we drop the request if it won't be available (rendered) in-time
* Priority of this job (as soon as possible, or just in time?)
* action (Playing forward, playing backward, tweaking, playback speed, recursive rendering of dependent frames)
.Notes
* The Backend will try to render related frames in groups.
* This means that following frames are scheduled with lower priority.
* Whenever the program really requests them the priority will be adjusted.
-> more about link:Scheduler.html[the Scheduling of calculation jobs]

15
doc/design/engine/OutputHandling.txt Normal file
View file

@ -0,0 +1,15 @@
Design: Output Handling
=======================
:Date: June 2011
:Author: Ichthyostega
//Menu: label Output handling
Some ideas....
- abstract away the actual technology used for output
- have generic *output designations* and translate them into an *output slot*
- the OutputSlot interface can be designed to match the requirements of the Engine
- assume a mechanism to handle timeouts, glitches and skips within each concrete OutputSlot implementation

3
doc/design/engine/index.txt
View file

@ -1,5 +1,6 @@
Design Documents: Renderengine
==============================
Eventually, this will have design documentation for the Engine.
This section contains design documents regarding the overall workings of the Render Engine,
and the handling of output generation and output connections.

9
doc/design/index.txt
View file

@ -1,6 +1,15 @@
Lumiera Design Documents
========================
// Menu : prepend child design/backend
// Menu : prepend child engine
// Menu : prepend child model
// Menu : prepend child design/gui
// Menu : prepend child architecture
// Menu : append child plugins
// Menu : append child workflow
The goal of Lumiera is to provide a professional tool for video editing on
GNU/Linux systems. The vision of the development team defines a modern design
for the core of a Non-Linear Editing software. One of the key aspect of Lumiera

96
doc/design/plugins/PluginBrainstorm.txt
View file

@ -3,12 +3,81 @@ Plugin Brainstorm
:Author: Christian Thäter
:Date: 2008-09-15
Raw Version
-----------
Lumiera will use a very simple and language neutral plugin system. The focus is on easy and independent distribution of plugins and small specific interfaces. Ultimate flexibility is of second concern.
.Concept
Plugins are just shared libraries which offer well defined Interfaces.
A Plugin may offer more than one interface and may in turn request/use interfaces
from other Plugins or from the main application.
[cpp]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Interface Definition
--------------------
Interfaces are declared in header files. They use some tool macros to give a convenient definition language.
Basically, Interfaces are fixed -- with the exception that new functions may be added.
link:PluginVersioning.html[Plugin Version Management] should stay out of the view most of the time.
Plugin interfaces are simple C structs with some metadata at the beginning and function prototypes
added at the end. With some macros we can map simple functions to versioned interfaces.
Compiled plugins will stay compatible even if the interface is extended, while sourcecode need maintenance.
An interface needs a name and a version. They define a block where the actual function prototypes can be added.
New prototypes have to be added at the end, existing prototypes must never be changed.
Each function prototype must be given with its different parts:
- return type
- name
- arguments list
- version.
.Example
[source,c]
-----------------------------------------
LUMIERA_INTERFACE(foo, 1,
LUMIERA_IPROTO(void, bar, (void)),
LUMIERA_IPROTO(int, baz, (int i))
);
LUMIERA_INTERFACE(foo, 2,
LUMIERA_IPROTO(void, bar, (void)),
LUMIERA_IPROTO(int, baz, (float i))
);
-----------------------------------------
Note that the version 2 interface _changed the parameter from int to float_ for the 'baz' function.
The interface/plugin framework will expand the above definitions into:
[source,c]
-----------------------------------------
struct lumiera_interface_foo_1
{
struct lumiera_interface interface_header_;
void (*bar) (void);
int (*baz) (int i);
};
struct lumiera_interface_foo_2
{
struct lumiera_interface interface_header_;
void (*bar) (void);
int (*baz) (float i);
};
-----------------------------------------
Implementation of Interfaces
----------------------------
Interfaces can be implemented either in core code or through plugins.
In each case, such an _instantiation_ of an interface means that actual functions are mapped
to the corresponding slots in the interface structure.
.Implementing an interface
[source,c]
-----------------------------------------
LUMIERA_INTERFACE_DECLARE (interface_descriptor, 0,
/* The following slots are some human-readable descriptions of certain properties */
LUMIERA_INTERFACE_SLOT (const char*, name, (LumieraInterface)),
@ -18,4 +87,21 @@ LUMIERA_INTERFACE_DECLARE (interface_descriptor, 0,
LUMIERA_INTERFACE_SLOT (const char*, license, (LumieraInterface))
/* TODO add more things here, dependencies, provisions etc */
);
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-----------------------------------------
Lumiera Plugin API
-------------------
The Lumiera Interface/Plugin framework provides some functions to manage Plugins.
Actually a user requests interfaces. The libraries which implement Plugins are managed transparently.
Interfaces are exported as instances and are not necessary singleton. This means that a single Plugin
can export the same interface type several times under different names. The naming rules for interfaces
need to be defined elsewhere.
loading and opening a Plugin
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Plugins are looked up in `$LUMIERA_PLUGIN_PATH`, which is a colon separated list of directories,
and then in a specific ``Lumiera plugin dir'', where standard plugins get installed alongside
with the Application

44
doc/design/plugins/PluginVersioning.txt Normal file
View file

@ -0,0 +1,44 @@
PluginVersioningCases
======================
:Author: MichaelPloujnikov
:Date Created: 200707121127
:Date Changed: 200707160404
:Count Changes: 46
//MENU: label Plugin Version
Compatibility matrix
--------------------
.Source compatibility
[grid="rows"]
`100`200`200~~~~
*~CALLER~ \ ^CALLEE^**,OLD^**^,NEW^**^
OLD,works,works but a recent interface definition must be available
NEW,works,works
~~~~
.Binary compatibility
[grid="rows"]
`100`200`200~~~~
*~CALLER~ \ ^CALLEE^**,OLD^**^,NEW^**^
OLD,works,works
NEW,caller gets "revision not sufficient" at runtime and should implement fallbacks,works
~~~~
^*^) CALLER is the user of an interface, CALLEE is the interface provider (usually a plugin)
^**^) OLD means an initial revision, NEW means some later revision of an interface
Observations
------------
Compiling a newer Plugin for some older main application release has some quirks (interface definitions are intended to be shipped with the main application). This should be rarely the case.
When compiling, older Plugins should be updated to new interface revisions.
Caller should provide a fallback to older interface revisions for binary compatibility.
Generally, sources should just be properly maintained and updated to use the most recent interfaces revision.
For binary compatibility everything will work well, provided that the caller kept proper fallback functionality for older interface revisions. Plugins which are independently distributed (packaged) in binary form don't need to be updated with every new main application release and just work.

12
doc/devel/rfc.txt → doc/devel/design_process.txt
View file

@ -2,16 +2,20 @@ Design Process
==============
//Menu: include rfc
//Menu: include rfc_final
//Menu: include rfc_pending
//Menu: include rfc_parked
//Menu: include rfc_dropped
//Menu: put child 'rfc_dropped' after 'rfc'
//Menu: put child 'rfc_dropped' after 'rfc_final'
//Menu: attach child 'rfc'
How it Works
------------
Design Process entries (RfC) can be created by anyone.
Design Process entries (Request for Comment) can be created by anyone.
Each entry goes through several stages until it's accepted (or rejected).
All our RfC entries are filed here, either in the link:rfc/index.html[RfC accepted] section,
All our RfC entries are filed here, either in the link:rfc_final/index.html[RfC accepted] section,
or as link:rfc_pending/index.html[pending RfC] or as link:rfc_dropped/index.html[RfC dropped].
* Every proposal starts out as _*Idea*_, allowing other people to review and comment on it, while still working out details.
@ -37,6 +41,6 @@ Handling of RfC entries in practice
.Adding a new Proposal:
. Check if there is no similar proposal below! If yes, contact it's author and contribute to that one.
. Think of a good/descriptive _Name-ID_ for the Proposal. It will be used to create a RfC sub-page, so no embedded spaces please.
. [red,yellow]#!TODO!# _please describe how to use the +rfc.sh+ here_.
. Use the './admin/rfc.sh' helper script to maintain the RFC's. Check out its link:../technical/infra/rfcsh.html[documentation].

0
doc/devel/images/fig132741.png → doc/devel/images/ProjectTimelineSequenceUML.png
View file

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 25 KiB

After

Width:  |  Height:  |  Size: 25 KiB

BIN
doc/devel/images/lumiera_big_graph.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 48 KiB

BIN
doc/devel/images/uml.time-entities.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 19 KiB

82
doc/devel/meeting_summary/2008-02-01.txt Normal file
View file

@ -0,0 +1,82 @@
Initial Lumiera Developers Meeting
==================================
:Date: 2008-02-01
//Menu: label 2008-02-01 the first Lumiera Developers Meeting
__Participants__
* hermanr
* cehteh
* ichthyo
* Dasypus
* gmerlin
* SimAV
* pippin
* Plough
* Plouj
* Velmont
Discuss the open points in [Lumiera/DesignProcess] --- do we need this formalism?
---------------------------------------------------------------------------------
At start of the project last summer, Cehteh made a http://www.pipapo.org/pipawiki/Lumiera/DesignProcess[design process proposal]. We will keeping this up, not for every implementation detail, but for major plans, wishes and design decisions. One point in the agenda of future meetings will be to work through proposals in the queue
* proposals in the "idea" state are not complete, can be just brainstorming or need further discussion. Comments please to the proposal page.
* proposals in the "draft" state are ready for conclusive discussion and will be treated in one of the next meetings
* "final" proposals are either "accepted" or "dropped". We don't differentiate the latter, but should write a short note why it was dropped.
* if there is need for more or finer grained categories, we'll extend the page and the template as needed or provide different views
The development model
---------------------
We employ a distributed model based on GIT. We want this repository to be as complete as possible, including documentation in embedded TiddlyWikis and Bug reports. Each dev has its own GIT repo, devs are pulling from each other, they are free to cherry pick and try to make the combined version work. Point is that everyone can clone the git, negotiate with the others what s(he) wants to do, and hack on. Every dev signs off his branch with an standardized signature. For small changes we provide a "Mob GIT", i.e. anonymously pushable git (which is untrusted of course). Cehteh is currently working on a git web frontend which makes the codebase in the mob-repo web-editable like a wiki.
Will we need a stable version or an official branch? not yet &mdash; as long as the team is small it will work more painless without. At some point, when the project is more mature, we will define an official branch. Later on we will have automated builds and regression test runs. As we do test-driven development anyways, it's just a question of someone setting up all the infrastructure, then we'll do it.
Ichthyo proposes a new requirement: All devs should ensure the "master" branch of their respective repositories always passes the compiler and the test suite. Work-In-Progress should be done on branches. Rationale: it is sufficient to pull from the master branches, and you can be sure the version you pulled worked for the originator.
A note on dependencies: it will be hard to target minimal dependencies for such a project, but we shall try not to bloat it unnecessarily. Sometimes it can be sensible to rather re-invent a feature &mdash; esp. when it fits into the core focus of the project &mdash; instead of depending on difficult to build and not sufficiently maintained external projects. But we should avoid reinventing things like GUI toolkits.
And, pretty much obvious, we try to stick to modern programming standards. Read: modules have interfaces, interfaces need some (minimal) documentation, and it is not allowed to bypass the interfaces and tangle everything in a wild fashion.
Currently, the project can be separated into three layers, which could evolve into sub-projects: Backend, Proc-Layer, GUI. For each part, the dev most deeply involved and most knowledgeable will take on the sometimes necessary leadership and have the last word in case of quarrels. Currently, Cehteh cares for the Backend and Ichthyo headed for the Proc-Layer. We have postponed any work on the GUI for now and don't participate in GUI toolkit discussions. If there is a dev willing to care for the GUI, collect proposals, care for usability and the users needs and finally code it up, then he will the one to decide what toolkit to use.
We plan to make the discussion about GPL v3 a point on the agenda of the next meeting.
Monthly meetings
----------------
* make it thursday, not friday
* time for now 21:00 GMT &mdash; if some (potential) participants have problems with this time, please speak up (maybe alternating times?)
* write a short status report for each mayor part *prior* to the meeting (saves us time). Maybe add an TODO list there
* go through the open issues for the design process drafts
* publish a protocol of each meeting on the (Cinelerra-CV, LibOpenvideo) Mailinglists, in the link:TiddlyWiki[] and on pipapo.org
* News, Protocols and the agenda of the next meeting can be found at link:MonthlyMeetings[this page here]
* next meeting on first Thursday in March (6.3.2008)
Who works on what, what are the short term goals, what tasks are open?
----------------------------------------------------------------------
*Ichthyo* works on the processing layer. Current goal is to get the core of the builder fleshed out. Next goal is to create a clip object (dummy), add it to the EDL, hand the EDL over to the builder and let the builder make the first (preliminary) render nodes. (note: left many details for later).
Ichthyo started coding his design draft and things seem to work out as intended. Some Keywords: Have a high-level model and a low level model. The former is comprised of the objects in the Session edited by the user, the latter is a network of completely configured render nodes, employing the same pull model as in Cinelerra-2. In between sits the Builder, translating high-level into low-level. This translation is done on demand (not for every frame).
Current state in this part is: basic asset manager is done, asset objects (forming a hierarchy) can be created and will be registered automatically, it is possible to create a clip-"MediaObject" from a media asset (without caring for any details regarding multichannel clips). Some support lib components are written, Session and data holders start up on demand and shutdown cleanly. The test suite is the only interesting executable, and this will remain so for quite some time. Currently writing the first parts of the Builder.
Further plans/ideas: Ichthyo is rather determined to embed a Prolog interpreter (YAP Prolog) to handle all sorts of configuration queries in a rule-driven fashion. Things Ichthyo can't do in the near future: caring for session loading/saving serialisation plus storage backends, caring for a DB based implementation of the asset manager and integration with production support software, target the scheduler which will receive any edit operations initiated from the GUI.
*Cehteh* is currently working on webgit, which is somewhat related inasmuch it will make small contributions to the mob repository much simpler. Previously he started with some foundation and support facilities. He plans to come back to the Backend implementation in about two weeks. The Backend is intended to handle all media (and even meta-)data as generalized frames. The render nodes network created by the Proc-Layer is completely stateless and all data is served from below. While it will be possible to address and access individual data within a frame (e.g. audio samples), frames are the smallest unit for memory and cache management. No plans to use a tiled memory model or to support frames larger than aprox. 20-40% of the available RAM.
Cehteh's design plan includes a scheduler to organize the access to the raw data, monitor the timings and prefetch data as needed. This scheduler will be configurable via quality preferences ("need realtime", "need perfect quality"). Further, there will be an elaborate caching scheme trying hard to avoid re-rendering any frames already calculated previously. Temporary data will be backed by files and thus swapped out &mdash; this swapout and size of temporary data is to be monitored and adjusted according to the current load &mdash; and all temporary data is kept as most-recent-used cache discipline. Incoming and outgoing footage shall mostly be handled by using mmaped buffers. The rationale is to avoid unnecessary copy from kernel to user space and wasting memory for an additional in-kernel buffer. Writing via a mmapped buffer is little more tricky; there will be a in-place writing which is used for indices and other precalculated data which needs updates, and the processing layer can query write buffers which are actually a small cache/ring and then comited to the file. Basically, mmapping is a clean solution if you can design for it, and it's portable (posix)
Things to do: object serialisation backend is sometime on Cehteh's schedule, but that's ahead and if someone else helps or takes over it would be OK. Even more true for a DB based backend for the asset manager.
*Gmerlin* plans to implement grayscale support for GAVL, so the upper layers can store arbitrary data in it.
About multithreading
~~~~~~~~~~~~~~~~~~~~
Since the render nodes are stateless they can be driven in multiple threads (but inter frame dependencies need to be resolved/serialized). Mostly the backend manages threads and does that quite conservatively (compared to Cinelerra-2 which massively creates separate threads for small tasks). Any edit operations initiated from GUI go to a scheduler in the middle layer, which enqueues and effectively serializes operations done to the "media objects" in the high-level model. The editing operations themselves are *not threadsafe* by design, they rely on being scheduled correctly. The builder is triggered from this Proc-Layer scheduler and starts in one separate thread, and when done, we swap whole parts of the render nodes network and then the backend can re(start) rendering as needed.
The naming discussion
---------------------
The discussion looks healthy so far... People can add new proposals on the http://www.pipapo.org/pipawiki/Lumiera/Names[pipawiki]. Interesting names are still coming in, so we should just let the name-choosing game go on for a while. And, btw, we _can_ depart from beeing similar to "Cinelerra" ;-)
Let's say, we need a person volonteering to guide/moderate the selection, going over the list and scratching inammprobiate ones. Criteria for good names being:
* should not be an existing project
* should be "googleable"
* should not be offensive
* should have one of the free top level domains (.net, .org)
* should be compatible with educational institutions (sorry, no pr0nedit :)
* should not obviously collide with trade marks

189
doc/devel/meeting_summary/2008-03-06.txt Normal file
View file

@ -0,0 +1,189 @@
2008-03-06 Lumiera Developers Meeting
=====================================
:Author: cehteh
:Date: 2008-03-07
__Participants__
* hermanr
* cehteh
* ichthyo
* Plouj
* SimAV
* akirad
* _MMA_
* rgareus
* ddennedy
* edrz
* anewcomb
* gmerlin
* oracle2025
* gisle
Boilerplate
-----------
Organization of this meeting
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
* cehteh writes the protocol
* hermanr does chairman
Leftovers from last meeting
~~~~~~~~~~~~~~~~~~~~~~~~~~~
We concluded there are no leftovers
Go through Ideas and Drafts in design process
---------------------------------------------
Idea: time_handling
~~~~~~~~~~~~~~~~~~~
http://www.pipapo.org/pipawiki/Lumiera/DesignProcess/time_handling[]
Point 1 is superseded by using gavl.
Points 2 and 3 are still valid.
Ichthyo asked gmerlin if there are any problems according gavl for the points 2 (position of frames) and 3 (position for keyframes and automation). Gmerlin acknowledges that he doesnt see any problems.
Oracle2025 brings interlacing on topic "are you aware that automations and keyframes could/should also apply to fields?". We agree that this must be handled "in interlacing, a frame is a pair of 2 subsequent fields".
Conclusion:
Refactor the proposal according to the discussion and work it out, make it draft then. Discuss about it on the next meeting
Idea: Unit tests in Python
~~~~~~~~~~~~~~~~~~~~~~~~~~
http://www.pipapo.org/pipawiki/Lumiera/DesignProcess/UnitTests_Python[]
We have a testsuite based on cehtehs 'test.sh' which superseedes this proposal.
Conclusion:
Drop it.
Idea: Todo Lists
~~~~~~~~~~~~~~~~
http://www.pipapo.org/pipawiki/Lumiera/DesignProcess/TodoLists[]
This Idea is in a very early state an not yet mature.
Cehteh explains: "I want something which doesn't need much human care and i want one big \'milestones' thing and a small \'mini-task' thing". Ichthyo refines this as "Roadmap" and "Near time task list". We agree that this shall not be too strict. There are some ideas floating, like adding this things to the testsuite or use the wiki. Ichthyo shows how he uses the tiddlywiki's task macro (http://ichthyostega.de/cin3/wiki/renderengine.html#PlanningNodeCreatorTool[]). He likes it but doubts its usefulness when it is used without guessing the hours/days of work.
Conclusion:
We use the Tiddlywiki task macro thing for now.
Draft: CCodingStyleGuide
~~~~~~~~~~~~~~~~~~~~~~~~
http://www.pipapo.org/pipawiki/Lumiera/DesignProcess/CCodingStyleGuide[]
Ichthyo tells that the discussion on the wiki page made this clear now. Cehteh explains that he uses this style with success for diffrent projects. We make clear that this is meant for C, in C++ we use namespaces. Overall we agree that code shall be self-documenting.
Conclusion:
Make it final.
Draft: DevelopmentFramework
~~~~~~~~~~~~~~~~~~~~~~~~~~~
http://www.pipapo.org/pipawiki/Lumiera/DesignProcess/DevelopmentFramework[]
Cehteh explains that he will transfer this propoal to a tiddlywiki covering compatibility guides and dependencies. We conclude that this wiki must reflect the actual practice (NoBug, Doxygen, test.sh) despite the proposal is not concrete yet.
A short discussion about build systems came up, we still use autotools and scons in parallel, delaying the decision later. oracle2025 mentioned that scons could be used for development and autotools for release. No decision about that everyone has different opinions. But we agree that it works as is.
Conclusion:
Cehteh will make the proposed wiki which shall be maintained to reflect the state and this topic will be finalized by that.
Draft: Skills Collection
~~~~~~~~~~~~~~~~~~~~~~~~
http://www.pipapo.org/pipawiki/Lumiera/DesignProcess/SkillsCollection[]
This might be only useful if there are more developers working on the project.
Conclusion:
Leave in draft state for now.
Draft: Architecture Overview
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
http://www.pipapo.org/pipawiki/Lumiera/DesignProcess/ArchitectureOverview[]
Ichthyo drawn a diagram showing the planned architecture. Cehteh raises concerns about codecs/plugins/effects in backend. After that there was some discussion about details. Cehteh suggests to place plugins in a extra box, gmerlin suggests that \'plugins' don't fit into the diagram, there should be "filters", "sources",..; Followed by some more discussion, showing that we basically agree and understand the design but the drawing needs some refinements.
Conclusion:
Good idea, needs some refinements, work in progress.
Call for Design
---------------
Ichthyo explains that the wants the overall design a bit more formalized. He put this topic on the agenda to remind that we have to work it out in link:DesignProcess[] and already started to make some design proposals.
Conclusion:
Things need to be worked out in link:DesignProcess[], take a look and review it.
Naming
------
Raffa, Velmont, goibhniu and others collected and managed proposed names past weeks which finally ended in a community voting. Results are at http://www.cs.cornell.edu/w8/~andru/cgi-perl/civs/results.pl?id=E_9fad0d1d10c23d38[]
Lumiera won, Lumiera.org is free and no one of in this meeting has objections against this name. Velmont offers to register and pay for the lumiera.org domain which will be hosted on our own server (see topic below). Hermanr raises concerns if it is ok when the name is registered on another site and by another person than the server, cehteh explains that he like this distribution where doable and no one other objects. Ichthyo asks about a short name for Lumiera which will be used for C++ namespaces and such kinds. After a short discussion we agree that we use the full name and no abbreviation. We now have to rename the source and the wiki. Anyone renames the sourcecode where he is responsible for. Cehteh will go over the pipapo.org wiki and rename things.
Finally we express our great thanks to the people who put the efforts into the name selection, thanks raffa, Velmont, goibhniu & co.
Conclusion:
Projectname is Lumiera, we have a lot things to rename.
Project Server, setup, organization, administration
---------------------------------------------------
Some people gathered together and rented a server at http://www.hetzner.de[hetzner.de] which will host some free project pages and personal sites. Cehteh is the one who signed for the server and officially respobsible for it.
The server is split into isolated \'vservers' which are created as needed. Idea is to work cooperatively to get the best out of its resources.
For our projects (cinelerra.org and lumiera.org) we now need volunteers who help setting it up and administrating things. Velmont offered to help with lumiera.org, hermanr takes care of cinelerra.org, raffa will help with maintaining the webpages. oracle2025 will care for developer chroots, build and test environments. cehteh will make a page on pipapo.org which reflects the current server setup http://www.pipapo.org/pipawiki/RootServerSetup[]. There are still a lot jobs to do!
Cehteh asked about how to manage emergency root access on the host/root server when he is unreachable. There where several suggestions between one root key for all who pay for the server to shared key schemes. We finally agreed that anyone who payed for the server can send a ssh key to ssh to be registered for emergency access.
The concrete server setup will be worked out next days. Ideas are one frontend proxy and a shared nameserver. A shared nameserver, A developer vserver shared between all developers. Maybe a shared mail server.
Hermanr asked about how to handle distribution of large media files. There was some discussion about bittorrent vs http. We concluded to work that out as needed.
In the forthcoming discussion about cehteh stresses that "it is a public server, if you place confidential information unencrypted on it, its you fault". Ssh keys shall be kept secret by the users but we can't enforce policies on those.
Ichthyo asks about how to setup lumiera.org (wiki?), oracle2025 suggests webgit, cehteh explains that webgit isn't ready yet but should be useable in a few days (pipapo.org will use that too). Velmont suggest a \'real' website for the time when non-developers want to use it. Git will be set up on lumiera.org the same way as it is currently set up on pipapo.org.
Hermanr asked Velmont about some help moving the bugs.cinelerra.org over, this might be little challenging since it needs an MTA and other things.
Ichthyo asks about keeping the Lumiera project pages on pipapo.org until the server is ready, this is ok.
Conclusions:
We have our own server which now needs volunteers for maintenance. The projects will slowly move there, until ready Lumiera stays on pipapo.org.
GPL3 pros cons, license rationale
---------------------------------
Ichthyo and cehteh would like to investigate a license change to GPLv3, neither of us has experience with problems this
might raise. Questions are if we lock us out from potential libraries which are GPLv2 only or lock out possible contributors because of unforeseen patent clauses. The others here thinking that "GPLv2 or later" would be most-compatible. Gmerlin tells that he uses some "GPLv2 only" mplayer code in gavl which is itself "GPLv2 or later". Cehteh explains that "GPLv2 only" code is a problem, where "GPLv2 or later" is not. Ichthyo raises concerns that the Support library may need to be LGPL, cehteh mentions that it would likely be enough if we only apply that to the plugin-loader. After all it turns out that there is a lot of confusion, misunderstanding and interpretation used with licenses and no one of us knows for sure. Finally we conclude that the interpretation and license enforcement is ruled by the copyright owner.
Conclusion:
Learn more about GPLv3, use GPLv2 or later and propose to switch to GPLv3 when we are ready to do public beta releases when possible.
Next meeting
------------
The next meeting will be at thuesday 3rd april 21:00GMT (if not rescheduled see below).
Ichthyo tells that Martin Ellison wrote he couldn't attend because of the time. So again we made a short discussion about changing/alternating timings to allow people in downunder/japan to attend. They have to speak up an make time suggestions.
Conclusion:
Next meeting 3rd april 21:00GMT. But Aussies, please speak up if you want the time changed!

290
doc/devel/meeting_summary/2008-04-03.txt Normal file
View file

@ -0,0 +1,290 @@
2008-04-03 Lumiera Developers Meeting
=====================================
:Author: Sami Kallinen
:Date: 2008-04-09
__Participants__
* cehteh
* gmerlin
* hermanr
* ichthyo
* joelholdsworth
* johncoswell
* rgareus
* sakalli
Boilerplate
-----------
Organization of this meeting
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
* sakalli writes the protocol, ichthyo will help reviewing it
Recurring Topics
~~~~~~~~~~~~~~~~
We concluded there are no leftovers and currently no Drafts from the design process to discuss
Call for volunteers
-------------------
(which tasks are to do, how can we interest people...?)
There was a sense that there are people willing to help but they might need direction or cannot do actual coding. But there is also a lot of tasks that could be done by beginners and non-programmers. These tasks are important and if someone who is not a programmer is willing to take them on that leaves more time for the programmers. Examples of such tasks:
* Protocol writing
* maintaining to do lists and isolating tasks
* constant code and documentation review
* unit/functional testing
* header file "task force" - some fundamentals discussed by the developers on libopenvideo should find their way into basic .h files
* installing and maintaining project tracking and automated builds/tests on the server
See http://www.lumiera.org/wiki/todo.html#Tasks[]
It was also noted that at current phase some basic development is still needed before we can accommodate a large number of coders doing small tasks and for now we need some subsystem owners. For GUI for example.
To make it easier for "beginners" to get aquainted with the program it was decided that one *official Lumiera GIT repository* should be established.
Project announcements/registration
----------------------------------
(freshmeat, gnu, etc...) any takers?
The discussion was about advertising the project on various sites. There were some concerns raised that it is too early to do too much publicity at the moment. That the right time should be shortly before the first public release. But there is also some value in registering to the project in planning/alpha stage to interest people such as prospective coders. It was noted that the http://www.linux.com/feature/126441[article on linux.com] resulted in a lot of people contacting ichthyo and cehteh. http://plot.bek.no/pipermail/piksel/2008-January/003328.html["I believe in Cinelerra"] by Leo Germani also generated a lot of traffic by developers. Some level of presence on mailing lists would be good although not too much just enough to indicate the project is going on.
Conclusion: volunteer needed to coordinate publicity (not overly urgent, but helpful)
Informal talk about the GUI
---------------------------
Firstly, as SimAV was not able to be present during the meeting a proposal by him was relayed by Ichthyo: GUI and backend should be completely separated and communicate via pipes. It was agreed that the GUI should be detached but some higher level message protocol rather than bare pipes should be used. Some comments:
* cehteh standalone gui is a nice to have, but implementation details have to be worked out
* gmerlin Video playback can be separated in a separate X connection
* cehteh suggests to start with an conventional, attached gui and just keep it in mind to make it detachable later
* rgareus IMHO you want to share memory for the video-frame buffer beween the player and backend - with pipes you'll go the video-jack way.
joelholdsworth is a new developer in the group and had showed interest in developing the GUI. he stated that he is interested in seeing a GUI that uses popular toolkits and that plays by the standards to create a consistent UI with the rest of the free desktop. For this reason he was advocating using GTKmm. There was no direct objections to GTKmm. Generally, there seems some preference for GTK. (Note this doesn't mean GNOME). Cehteh advocates the idea to use Lua-gtk, i.e. writing the GUI in a clean and lightweight scripting language.
Cario was suggested to be used for implementing the canvas in the timeline view.
The gui should provide a skeleton. Timeline and canvas should be controllable and extensible by plugins. Tracks can be nested. Use tiling windows and dockeable views and tools palettes. For example some configuration within session/renderpipeline may cause some pluggable elements to be added to the gui. A fader or a toggle/icon and so forth. Some plugins might render additional output (e.g. in previews). A video track renders thumbnails, automation curves on top.
Workflow is another important aspect to take into account in the design. A lot of information needs to come from editors and users in the design process. Different thoughts about the workflow:
* Should not make a predefined workflow. Just produce a tool that can be adapted to different workflows. It can be predefined, but should not be fixed.
* We dont have the resources to develop a "workflow" in the formal manner, we will go the "propose and comment" route.
* We should take the best ideas from popular competing products such as Final Cut Pro, AVID, Premiere plus After Effects.
Customisation is deemed important. Some points brought up in the discussion:
* sakalli: would be great if we could provide a fcp configuration and a avid configuration besides the default configuration as well. That could lure a lot of users quickly. On the other side, too much flexibility can be be a problem as well.
* cehteh: yes .. but such things are prolly costly to setup .. we need volunteers for anything optional
* Customization should be possible without recompile, but not too cheap.
* There should be a working skeleton or some fixed anchor but gui operations should be bound to a scripting language to achieve customization.
* The whole gui in a script with some performance critical widgets coded in C/C++
* Some parts should just be customizable, other parts should be kept fixed.
* cehteh suggested to start out by making a \'high level' description with no language/toolkit in mind .. what widgets do we need etc and after that to work on the skeleton mockup.
* cehteh / hermanr: Lumiera should allow to choose theme independently from desktop .. because what you have as your desktop-theme doesn't suit for video editing in many cases ... see ardour
ichthyo brings up the role of customization for the middle layer and gives an overview of his plans for the new participants.
He had good experiences with a rule based aproach in various projects. He wants to embed a Prolog interpreter that can answer to "configuration queries". There is a "builder" component in the middle layer to derive the render nodes graph based on a "high level model", which is visible to the user (in the GUI) and based on these configuration queries.
Rationale for using prolog is: The rules are very readable because they rather represent facts and relations, not the way "how dings are done" but the way "how things are related". Example: how to configure some effects when the footage is interlaced. Additionally, we want to have some global swithces, which could control these rules, like "I want maximum quality", or "I want as much as possible set up automatically". The prolog rules are stored in the session and can be editd by advanced users. Of course, there is a set of basic rules. This aspect of customisation isn't exactly a GUI issue but it would have consequences for the GUI.
some discussion:
* joelholdsworth: so we just need a way of exposing properties. This is reasonably straightforward when all you want is value, colour, string etc.
* rules within the middle layer/session could bind to some properties exposed in the GUI
* gmerlin joelholdsworth: Agree, but point out that this can become difficult
* cehteh joelholdsworth: skeleton and plugins who attach into some gui areas, so you can add custom widgets
* ichthyo: also the ability to attach "tags" to various objects, so the rules can bind on those tags
* cehteh: maybe a standard set of widgets, not really completely custom ones (gmerlin agrees)
* joelholdsworth: probably you need to be able to do both well
* joelholdsworth in many apps the value/colour/string set works perfectly well, but in luminierra the controls will likely need to be much richer
Some brainstorming points about usability:
* We need a realy good bezier widget for all sorts of curves.
* Faders that can be grabbed everywhere are nice like like ardours and Blender faders. Or others which have no big knob
* The controls in blender's node view would be a good modal to copy (not advocating GL UI, just using widgets and concepts).
One extra point:
* we have agreed to make all of the big interfaces between the layers as plain C interfaces, because this is the most widely supported common denominator.
GUI CONCLUSION
~~~~~~~~~~~~~~
* joelholdsworth is now the official GUI maintainer for Lumiera!
* sakalli, hermanr, ichthyo announced interest in contributing to discussions about GUI, workflows and usability. gmerlin noted that he knows some gtk secrets and is willing to help out here too.
Froscon application
-------------------
http://www.froscon.org/[Froscon] is a small nice german open source exhibition that is very very technical oriented. Cehteh will be there as well as Ichthyo. Question is wether we should have a booth, perhaps together with some other interested media projects. Could ask Richard and the ffmpeg/mplayer people. End of call for papers is in june.
We made no decision about official presence but most likely there will be a developer meeting if nothing else.
Timecode metadata discussion
----------------------------
This Discussion started out on the libopenvideo list: The question was wether we agree on one uniform format solution from decoder up to app level regarding timecode. And what this format would be: arithmetic or struct type. Regarding metadata (including type of timecode) the question is if we should work out a rather fixed data format, or go the route towards rather open description records.
Conclusion: Gmerlin does it the way it fits best for within the decoder, and Lumiera uses a more uniform time format for representing timecode in the higher app layers. Conversion functions will be concentrated in a library. Regarding metadata we will investigate further.
_A (shortended) digest of the timecode and metadata discussion is attached below_
Next meeting
------------
The next meeting will be at thuesday 8th may 21:00GMT
''''
.Timecode discussion
----
12:51 cehteh gmerlin: did you read my last mail about generalization of metadata?
12:51 cehteh key/value pairs
12:51 gmerlin not sure about that
12:51 cehteh i am pretty sure that i want that in lumiera
12:52 cehteh if gavl does that for us it would be even better
12:52 gmerlin first of all 2 different things:
12:52 gmerlin 1. arithmetic <-> struct
12:53 gmerlin 2. Generic <-> format specific
12:53 cehteh arithmetic for sure
12:53 gmerlin On the decoder API side, I want to export ideally only one type
12:54 gmerlin No no SMPTEXXX and ISO999908098
12:54 cehteh will that be attached to the frames in a intrusive way?
12:54 cehteh i rather leave the attaching up to the application
12:54 gmerlin intrusive?
12:55 cehteh struct frame { uint64_t timecode; } vs...
12:55 cehteh struct frame { struct metadata* something; }
12:55 cehteh or even vs something done at a api level
12:56 gmerlin struct lumiera_video_frame { gavl_video_frame_t * f; uint64_t timecode};
12:56 gmerlin Or add timecodes to gavl_video_frame_t
12:57 cehteh struct lumiera_video_frame { gavl_video_frame_t * f; struct metadata* timecode_n_stuff; }
12:57 ichthyo so I support a struct metadata*
12:58 cehteh ichthyo: timecodes might not be regular .. they should be optional, but anyways they dont cost when using an arithmetic type
12:58 cehteh i mean thats the purpose of timecodes .. to tag every frame and dont make assumptions that your footage is perfect
12:58 ichthyo but a struct metadata* would rather mean to keep it out of basic GAVL and have a libmetadata -- which may be integrated with GAVL -- right?
12:59 cehteh someome may have stolen a frame :P
12:59 cehteh thats gmerlins decision ...
12:59 gmerlin I'm against gavl integration of metadata
12:59 gmerlin gavl has a clear scope: Number crunching
01:00 cehteh gmerlin: you somehow have to pass it from the decoder to the application
01:01 cehteh i would prefer to introduce some sidechannel
01:01 gmerlin Yes, gmerlin_avdecoder will provide a separate function for getting the next timecode
01:01 gmerlin bgav_get_next_timecode()
01:01 cehteh gavl_get_frame (gavl_frame* place the frame here, metadata* place metadata there)
01:01 gmerlin or: bgav_timecode_from_pts()
01:03 gmerlin Some higher MXF profiles allow changing timecode parameneters in one stream
01:06 gmerlin It's sometimes impossible to separate demuxers and codecs
01:06 gmerlin better have them together in one lib
01:06 cehteh actually i would like to cache gavls internal states into the backend
01:08 gmerlin Ok, so timecodes are YYYYMMDDHHMMSSFF as int64_t
01:08 cehteh the indexing is something i prolly have some words about
01:09 cehteh timecodes are uint64_t as microseconds
01:09 cehteh well for lumiera :-P
01:10 cehteh YYYYMMDDHHMMSSFF as int64_t isnt that much better than a struct :P
01:11 ichthyo now, IMHO its mostly up to gmerlin to decide how it fits into GAVL
01:11 cehteh ack
01:11 gmerlin Not gavl
01:11 ichthyo gmerlin?
01:11 gmerlin gmerlin avdecoder yes.
01:11 cehteh in lumiera we will use usecs
01:11 ichthyo yes
01:12 ichthyo we will have a conversion function somewhere
01:12 gmerlin And will it include a calendar date?
01:12 cehteh doing it only once (avdecoder->usec) makes some sense
01:12 gmerlin usecs since when?
01:12 ichthyo either, we get directly usecs from avdecoder, if this fits in for gmerlin,
01:12 ichthyo or we'll get the format which works best for him and we do the conversion ourselves
01:12 cehteh defined elsewhere
01:12 cehteh reel-start or absolute time or whatever
01:13 ichthyo also, what sort of timecode this was based on (drop frame or not)
01:13 cehteh gmerlin: thats a important point .. just uint64_t does not suffice
01:13 cehteh it needs a tag describing what kind of timecode was used
01:14 cehteh no matter how you encode it
01:14 cehteh and iirc there are quite some timecodes
01:14 ichthyo because in this respect richard was right: we need to be able to reprocuce any value 1:1
01:14 cehteh yep
01:15 cehteh further i want to be able to convert between any kind of timecode in a fingersnip
01:15 gmerlin ichthyo: and that's impossible with usecs
01:15 cehteh gmerlin: its not
01:15 ichthyo with usecs alone: agreed
01:16 ichthyo with usecs + metadata it should be possible
01:16 ichthyo modulo some extreme cases (high speed cams, weeks of footage....)
01:16 cehteh yes .. you need framerate and other infos depending on the metadata
01:17 cehteh you need to be careful to do the math right so that inverse functions dont drift
01:17 gmerlin I'll export something lossless, which repduces MXF, DV and Quicktime timecodes losslessly
01:17 gmerlin You can then do what you want with them :)
01:17 cehteh thats why i would like to do it at only *one* place/library .. because its delicate and bugs are easier to spot/fix there
01:18 ichthyo and then there is some api to get at the additional metadata
01:18 cehteh yeah
01:18 ichthyo and beyond that: lets devise a libmetadata
01:18 cehteh just shove the data over .. YYYYMMDDHHMMSSFF as int64_t would be ok
01:18 gmerlin Ok.
01:18 cehteh we make a libmetadata (as part of our support lib)
01:19 cehteh somewhat independent so that it can be spun out
----
.about Metadata
----
01:19 cehteh gmerlin: well .. how bout other metadata
01:19 cehteh your decoder needs to pass that too
01:20 gmerlin like?
01:20 gmerlin Author, Artist title...
01:20 cehteh some cameras encode shutter speed, focus, gain control, colour balance
01:20 cehteh and other things
01:20 gmerlin Hmmm
01:20 gmerlin I think MXF exports some of these
01:21 ichthyo wav files can have additional chunks; whats with flac?
01:21 gmerlin flac has vorbis metadata
01:21 ichthyo important for the more modern sound formats
01:21 cehteh dunno .. just thinkin about pro codecs and raw film metadata
01:21 ichthyo just wanting to point out: things to come here,
01:22 ichthyo e.g for sound: what stream is what channel
01:22 gmerlin Channel locations are handles by gavl
01:22 gmerlin If you have a multichannel stream
01:22 cehteh well .. note that there is per-asset metadata you want to pass .. and per frame metadata
01:22 cehteh maybe even some more undiscovered cases :)
01:23 cehteh did someone say this is easy??
01:23 sakalli did any of you watch the video regarding red workflow that i sent to the mailing list? redcode raw holds everyting, asa, wb, etc... hope lumiera will be able to facilitate that as well.
01:23 ichthyo at some point I want to be able to support ambisonics
01:24 ichthyo and maybe some day there will be wave field sysnthesis in more broad use
01:24 cehteh the codec in the back needs to pass us these things else we cantb handle it
01:24 gmerlin well, the easiest would be to extend bgav_metadata_t
01:24 ichthyo rationale is: just be open
01:24 cehteh gmerlin: maybe we need to investigate this things more, nothing needs to be decided now
01:24 ichthyo ack
01:25 cehteh see my key/value metadata proposal .. at some point maybe you make a complete sidechannel just for metadata
01:25 cehteh decoder stuffs that in on demand and it is passed along with its own api like the bgav_timecode_from_pts()
01:26 cehteh that way you dont need to touch it more than necessary
01:26 gmerlin shouldn't that key/value stuff be unified with plugin parameters?
01:27 cehteh for lumiera, maybe, maybe not
01:27 ichthyo you mean: you have a common api for describing "values" ?
01:27 cehteh some plugin parameters are determined by math functions
01:27 gmerlin You have a key, a type and a value
01:27 cehteh maybe for caching them
01:27 cehteh the api might be unified .. but the storage not
01:28 gmerlin why?
01:28 ichthyo because any block of data could be in the metadata
01:28 cehteh metadata which belongs to the frame coming from the decoder is const
01:28 cehteh plugin parameters are volatile
01:33 gmerlin One other point (but too late for it now) are audio timecodes. I'll adress that on libopenvideo
01:33 cehteh we will write a little more sophisticated timecode library
01:34 cehteh gmerlin: that should be unified :)
----

244
doc/devel/meeting_summary/2008-05-08.txt Normal file
View file

@ -0,0 +1,244 @@
2008-05-08 Lumiera Developers Meeting
======================================
:Author: cehteh
:Date: 2008-05-15
__Participants__
* cehteh
* joelholdsworth
* ichthyo
* raffa
* Wes``Lappy
* gmerlin
Boilerplate
-----------
Organization of this meeting
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
* cehteh writes the protocol
Webpage-Infrastructure, Maintenance
-----------------------------------
Cehteh put this on topic because it's really urgent to bring up some
infrastructure to manage the information we produce.
The Lumiera pages on pipapo.org were always meant as intermediary
solution. Pipapo.org gets much spam on the moinmoin wiki and cehteh
expresses that he wants to move pipapo.org to a new infrastructure
based on asciidoc and git he created
(see http://git.pipapo.org/uWiki.html[]). This system is barely useable
and needs lots of work to be completed. It would be nice to use it for
Lumiera too, if the others agree. Maintaining and extending
(scripting) needs someone who knows shell scripting and doesn't need
to be done by the core devs freeing their resources to work on
Lumiera. Cehteh expresses that none of the people who proposed to
help before showed up yet. WesLappy tells he might help (addendum not
this week, because he is busy).
Next there was a suggestion by cehteh to convert the tiddlywikis to
asciidoc. Ichthyo expresses that he likes the tiddlywikis, Joel
mentions that they feel a little odd. Generally we all agree that
they have some problems in sense of workflow and merging.
Ichthyo makes the suggestion to keep the tiddlywikis as scrapbook and
build up \'official' documentation based on their content in
whatever-we-use-then (asciidoc) documentation system. All agree on
this approach.
Back to the new wiki, cehteh suggests to make stricter access rules to
prevent spam: "There will be a group of \'Validated' people which
following rules: only Validated people can edit general content and
\'Validated' people can add new people to \'Validated' themselves". That
means that we can have a lightweight self-administrating
authentication where new people have to be introduced by someone who
is already there.
Ichthyo suggests to send a reply to Serge G. who send an introduction
to the cinelerra mailing list expressing that he would like to help.
Raffa will take care of content/design as much her time/knowledge permits.
Conclusion
~~~~~~~~~~
* We really need someone to help with the webpage scripting.
* Documentation needs to be well organized and moved over.
* Content of the pipapo.org moinmoin wiki should be moved over.
* The new website should be well organized with a nice looking frontpage
* All other documentation should be below that
Whats pending in link:DesignProcess[]
-------------------------------------
MistakestoAvoid
~~~~~~~~~~~~~~~
http://www.pipapo.org/pipawiki/Lumiera/DesignProcess/MistakestoAvoid[]
rick_777 wrote a "MistakestoAvoid" page which explains some possible
gotchas. We basically agree with most points there while we already
decided to address some things differently than he suggested. One
noteable difference is that we do not intent to provide a windows
version of Lumiera, which was explained in serveral places. Cehteh
added some comments to the page explaining some things.
nothing more to discuss in \'Ideas', we go over to \'Drafts'
ArchitectureOverview
~~~~~~~~~~~~~~~~~~~~
http://www.pipapo.org/pipawiki/Lumiera/DesignProcess/ArchitectureOverview[]
Cehteh suggests to put that drawing under version control, as .fig.
Ichthyo explains that it is already a .SVG and that he does not like
.fig.
Conclusion: We agree to keep it as .SVG, add it to the repository and
improve/complete it.
GitSubmoduleTransistion
~~~~~~~~~~~~~~~~~~~~~~~
http://www.pipapo.org/pipawiki/Lumiera/DesignProcess/GitSubmoduleTransistion[]
Another suggestion made by cehteh is to make managing of subprojects
within the sourcetree easier. Joel and ichthyo oppose that it is not
really needed now and needs more understanding of git. Ichthyo
suggests that the documentation might be separated into its own git,
he further explains that the things are not settled yet and we don't
know if we will some rearrangements and movements of files between
modules.
Conclusion: We transform the documentation to a submodule and see how
it works. Other things will be decided much later.
GlobalInitialization
~~~~~~~~~~~~~~~~~~~~
http://www.pipapo.org/pipawiki/Lumiera/DesignProcess/GlobalInitialization[]
This topic is discussed and agreed on the link:DesignProcess[] page already.
Conclusion: finalize it
MasterRepositorySetup
~~~~~~~~~~~~~~~~~~~~~
http://www.pipapo.org/pipawiki/Lumiera/DesignProcess/MasterRepositorySetup[]
Setting up an master repository was decided on the last meeting.
cehteh now set one up which also does some automatic, but intended
fragile merging from subsystem maintainer branches.
1. it updates automatically for the maintainers repo for conflict
free fast-forwards
2. it can always be overridden with explicit pushes
The others agree on the setup.
Ichthyo stresses that maintainers shall watch that their \'master'
branch should compile cleanly and pass the testsuite always, test
which are not operational should be tagged as PLANNED. We all
agree, while cehteh's master is currently in a broken state (note:
fixed now).
Then a short discussion about building and synchronizing the docs
follows. Problem is that docs build in maintainer branches are
different for each branch, rsyncing them up to the server will always
change a lot of things. We agree that the \'official' docs shall be
build from the LUMIERA/master repository, preferably on the \'devel'
vserver which has to be set up.
Conclusions: Make this final, setup build environment in the devel
server and build docs there.
NoBugFlags
~~~~~~~~~~
http://www.pipapo.org/pipawiki/Lumiera/DesignProcess/NoBugFlags[]
Defining a debugging control hierarchy. This is work in progress and
cehteh explains that he works it out and deploys it, this depends on
the link:GlobalInitialization[] decided earlier.
Conclusion: accepted, finish and finalize it.
Further Technical Discussion
----------------------------
Ichthyo asks how the GUI will be pulled up. Since he didn't attend IRC
discussions recently he has no information yet whats going on. We
explain him that we already made some discussions. Integrate the GUI
into the build system probably linked as library, nothing finally
decided yet. Communication will go over the plugin/interface facility
(Plain C API). This things should be worked out and documented in
link:DesignProcess[].
Cehteh made a small tool `./admin/headercheck` which will gradually
extended to proof that headers are sufficiently selfstanding.
Progress
--------
Cehteh finished low level file handling and working in mmaping and
frameprovider next. When thats finished, the finalization of the
Plugin loader and interface definition things is the most urgent thing.
Ichthyo works on the builder internals and next on some sort of
"connection manager".
Joel goes on with GUI programming and integrating it into the source
tree next.
Gmerlin and cehteh discuss about how to handle the index files which
avdecoder uses internally. cehteh would like to manage them in the
Lumiera backend to, because filehandles are a precious resource.
Gmerlin explains that this index files are just loaded and kept in
memory. So this poses no problem for filehandle exhaustion. We will
discuss this further via email.
Cehteh suggests that we should think about some kind of
preferences/registry sometime next to store default configs.
Following a discussion about how messages are passed between GUI and
core. Generally we will use the interfaces defined by the plugin
system. Gmerlin says that he uses message queues in \'gmerlin', Joel
requests that a lot of things should be done synchronous and he wants
to avoid message queues. Cehteh suggests to use use the scheduler for
GUI things too. For the parts where the GUI interacts with the high
level proc model ichthyo and joel agree on working something
(synchronous) out. Ichthyo stresses that the edit core is not
threadsafe by design and relies on the backends scheduler.
The underlying builder might sometimes to expensive for synchronous
calls (ichthyo plans a rule system there) this needs to be worked out.
Ichthyo explains that the builder needs to detect cycles and check if
the high level model is translateable into the low level model (in
case plugins git removed and so on). Parts in the render graph which
are not \'doable' should be flagged as erroneous but not dropped since
one doesn't want to loose project information when loading a project
on a machine with different installed plugins. In any case it should be
possible that the GUI gets immediate synchronous feedback for such
things.
Next Meeting
------------
Next meeting is on 5th June 21:00 UTC

81
doc/devel/meeting_summary/2008-06-06.txt Normal file
View file

@ -0,0 +1,81 @@
2008-06-06 Lumiera Developers Meeting
=====================================
:Author: ichthyo
:Date: 2008-06-07
21:00 -23:15 UTC on #lumiera
Participants:
-------------
* cehteh
* ichthyo
* joelholdsworth
* rcbarnes
* raffa
Left over from the last meeting
-------------------------------
Nothing left over and no urgent topics. Seemingly, work is proceeding in all parts of the application.
Discuss Ideas and Drafts in design process
------------------------------------------
There are no new design proposals and no proposals that can be finalized.
Ichthyo points out that he's about to work out the details of some of his proposals, which are currently in "idea" state. Following that, most of the meeting is spent on discussing the details of two of these proposals.
Idea: Design the Render Nodes interface
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
http://www.pipapo.org/pipawiki/Lumiera/DesignProcess/DesignRenderNodesInterface[]
*Cehteh* points out that, as we are in the pre-alpha phase, interfaces may be growing on-demand. Later on, interface versions will be numbered. If needed, we could add a special "draft" or "experimental" tag, or, alternatively, use the common numbering scheme, where odd major version numbers denote the development line of an interface.
*Ichthyo* agrees, but adds he also meant "interface" in this proposal in a wider sense, like in "what do we need and require from a processing node". Knowing how generally Lumiera will handle the processing nodes while rendering helps him with defining and implementing the builder
__Conclusion__: "currently in work". For now, grow interfaces on demand.
Idea: Placement Metaphor used within the high-level view of Proc-Layer
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
http://www.pipapo.org/pipawiki/Lumiera/DesignProcess/ProcPlacementMetaphor[]
In the course of the discussion, *Ichthyo* explains the rationale
* one common mechanism for sticking objects together and putting them into the session
* either specify the "placement"-parameters (time, output destination, track) directly, link to another object's parameters, or derive some or all of those values from the context (up the tree of tracks)
* ability to build a system of high-level media objects (clips, effects...) which re-adjust automatically on change
* extensible to handle or derive some parameters based on conditions and rules, e.g. for semi-automatic wiring of the output destination based on tags
*Joelholdsworth* is concerned that this proposal may go too far and tries to tie things together which aren't really connected. While basically it's no problem having the time position of a clip either absolute, or derived by a link to another object, he can't see a clear benefit of controlling sound pan or video layer order from the placement. Pan, for example, is just an parameter value or interpolated curve, similar to colour correction or gamma adjustment. For the gui, he points out, it's probably better to stick to the object metaphor, so start time, output, layer number or sound pan would be properties of the object.
But that's exactly what Ichthyo wants to avoid. Obviously, this would be the standard solution employed by most current editing apps, and works reasonably well in average cases. But he is looking for a solution which covers this standard case, but also doesn't get into the way when dealing with advanced compositing, working with spatial sound systems (Ambisonics, Wave Field Synthesis) or stereoscopic (3D) video.
On the whole, there is no conclusion yet. Certainly, this proposal needs more discussion, parts need to be defined much more clear (esp. the "Pro" arguments), maybe parts of the functionality should be separated out.
While in this discussion, several aspects of the cooperation of GUI and Proc layer are considered.
* it is not necessary to make all of the Placement proposal visible to the GUI (and the user). Proc layer may as well provide a simplyfied and hard wired API for the most common properties (layer index, pan) and only use this part of the Placement concept for building and wiring the nodes.
* the adjustment of objects linked together by a placement can be handled as follows:
. GUI notifies Proc of a position/parameter change of one object and gets immediate, synchronous feedback (OK or fail)
. Proc detects the other linked objects affected by the change and notifies GUI (both synchronous and asynchronous is possible) to update information regarding those objects
. GUI pulls the necessary properties by calling Proc on a per object base.
* as a rule of thumb, GUI <--> Proc is mostly synchronous, while Backend <--> GUI is often asynchronous, but there are exceptions from the rule
* we have general _parameters_, which are automatible. These are represented as _control data connections between the nodes_. We certainly don't want to represent some things in this way, though. For example, the in/out points of clips are fixed values.
* in Ichthyo's concept, the Placement doesn't itself provide such parameter values/sources, rather it can be used to _find_ or _derive_ parameter sources.
* the node graph is built bottom up, starting at the source, then via the effects attached locally to a clip, further on upwards (directed by the tree of tracks) to be finally connected via global busses to the output ports. Rendering pulls from these output ports.
* Joelholdsworth, Cehteh, Ichthyo and Rcbarnes agree that the plain _node-editor_ approach is problematic in the context of a NLE. It shows too much details and fails to capture the temporal aspect. We strive at having node-like features and flexibility, but rather within the timeline.
* especially, the topology of the node graph isn't constant over the whole timeline. But it's the job of the builder in the Proc layer to deal with these complexities, the user shouldn't be overwhelmed with all those details.
Next meeting
------------
* some people in europe complained that 21:00 UTC is too late, esp. with daylight saving
* there was the proposal to alternate between the current schedule, and sunday 16:00 UTC
* but Joel can't attend on sunday afternoon for now
* so we settle down on thursday, 16:30
Next meeting: *Thursday 3. July 2008 16:30 UTC*

94
doc/devel/meeting_summary/2008-07-06.txt Normal file
View file

@ -0,0 +1,94 @@
2008-07-06 Lumiera Developers Meeting
=====================================
:Author: Teld
:Date: 2008-08-10
Participants
------------
* ichthyo
* cehteh
* jilt
* dmj726
* Teld
* \_nasa_
* alcarinque_
* MNO
* Wescotte
* weatherhead
Boilerplate
-----------
Organization of this meeting
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
* dmj726 intends to write the protocol
* ichtyo is chairman
* there is no agenda, so this is a freeform meeting
Leftovers from last meeting
~~~~~~~~~~~~~~~~~~~~~~~~~~~
There are no leftovers
Introduction of Lumiera
-----------------------
Because there are quite some new participants, an introduction of the project Lumiera is given
There are 3 core devs:
* ichthyo: proc layer, render graph, in the middle, C++, he maintains scons
* cehteh: backend serving data (from filesystem and so on), manages threads, using C, Posix System programming, maintains autotools and git
* joelholdsworth: GUI in C++/Gtkm
Other people involved:
* rcbarnes: ui designer and coder
* raffa: web content
* Simav: gives a hand when and where he can
* Teld: web infrastructure
The foundations of the design are already done but a lot of detail needs to be worked out. cehteh and ichtyo provide a non exhaustive list.
cehteh:
* improvement of the testsuite (simple Bash)
* start of a worker thread manager (Posix knowledge required)
* start of the scheduler (some Posix, good C coding)
* runtime profiler (little but advanced Posix realtime programming job)
* review of code and documentation
* system administration
* setup of a build/test environment on the server
* setup and maintain postfix/dovecot by a mail administrator
ichtyo:
* asset management (keeping track of all media files, captures, clips, scenes)
* session loading saving and the issues of compound documents
* session structure related issues to be able to Import/Export/Connect via e.g. OSC (Open Sound Control) or JACK
* flesh out the more high level "edit operations" and the interface to UNDO
* Prolog integration after the first integration round has been reached. The Prolog interpreter will do some of the more advanced configuration (example: if effect XYZ is used, then deinterlace beforehand).
* integration with some sort of production support software (like Celtx)
cehteh emphasizes that Lumiera is an open project, so anybody can jump in where he sees fit as long as he communicates and acknowledges with the persons involved. ichtyo points out that the plugin structure is very important: anything that is not legally completely clean (proprietary), should be factored out into a sister project and be loaded as plugin.
Issues and questions that come up
---------------------------------
* handling of config errors.
When the configuration has a syntax error, in 90% of the cases there will be no possibility to do anything sane. Therefore, the config system will be built to log a warning and the user code does not need to care. The user just gets an alert and the application continues to work.
* scripting language.
There will be a scripting interface. ichtyo does not want scripts everywhere, only at well defined interfaces. That implies also that scripts cannot just do anything, only that what is permitted in a controlled way. The meeting agrees on that. cehteh wants one default language and proposes Lua: light, simple.
Other members suggestions: Python, Ruby, Scheme. However, Python and Ruby are very slow. Scheme has many variants.
* editing on small devices (eeePC)
Problem: video editors GUIs are some of the most elaborate and complicated GUIs. However, basic functions consist of only a few buttons. Proxy editing could be a solution. It is decided that it is not a primary goal now. First the basics have to be implemented.
* uWiki.
uWiki is the glue between a markup driver (Asciidoc) and revision control (git). Haserl with Bash is used now. Haserl appears to have problems with conditional includes. Its limits have been reached while prototyping. Lua could very well be a valid replacement. It will be investigated. PHP is rejected because it is not stable and suffers from serious security problems.
* \'musical\' timeline in bars and beats
The questions of syncing and linking things together are already on the core agenda: the so-called placement concept. Discussion is still going on, especially with the GUI developer joelholdsworth. See for detailed information: http://www.pipapo.org/pipawiki/Lumiera/DesignProcess/ProcPlacementMetaphor[]
Next meeting
------------
The next meeting will be held Thursday, 7 August 19:00 UCT.

74
doc/devel/meeting_summary/2008-09-04.txt Normal file
View file

@ -0,0 +1,74 @@
2008-09-03 Lumiera Developers Meeting
=====================================
:Author: Joel Holdsworth
:Date: 2008-09-05
Participants
------------
* ichthyo
* cehteh
* joelholdsworth
* Plouj
* thorwil
* wildhostile
Boilerplate
-----------
Organization of this meeting
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
* joel
* ichtyo is chairman
* there is no agenda, so this is a freeform meeting
Leftovers from last meeting
~~~~~~~~~~~~~~~~~~~~~~~~~~~
There are no leftovers
Mailing Lists
-------------
* any improvements should be reported to cehteh
* need volunteers to be moderators. The only volunteers are cehteh and joelholdsworh_
* moderating the ML might not be that big a job - we might not get that much spam
* it's preferable to allow people to post to the mailing list even if they're not subscribed
* cehteh eventually wants the mailing list to interact with uwiki.
Plugin Interfaces
-----------------
* all plugin interaction, including manifest queries will happen through the system of interfaces
* this includes enumerating structured data within the plugins. It might sometimes be desirable to store this in a text file, but in this case still the list will be exported through the interfaces. If this were not the case, it would not be possible to make plugins which are wrappers of LADSPA plugins for example.
* For a lumiera plugin there is only one entry point, thats an interface itself which bootstraps all the other interfaces in that plugin.
Lib Dependancy Problems
-----------------------
* we've agreed to stick to one compatibility level: Debian/Lenny for perhaps up to 3 years from now
* others will be supported - RedHat etc.
Builds and Buildbot
-------------------
* ichthyo asks for help setting up a test/build server
* We now have build scripts based on scons and autotools.
* All current build systems have different problems.
* ichthyo will take care of the Debian packages for Lumiera and NoBug
Website Transition Plan
-----------------------
* There's lots of work to do on the website - transitioning etc. that the core developers don't want to get bogged down with. We need a web developer to volunteer.
* All the website stuff is stored in git, and works through Asciidoc, which is not difficult to learn.
Logo Design
-----------
* Postponed for the next meeting when joelholdsworth is around.
Next meeting
------------
The next meeting will be held Thursday, 2 oct 21:00 utc.
Comment
~~~~~~~
re "ichthyo asks for help setting up a test/build server" for "Debian"
see "Build Server" at http://www.pipapo.org/pipawiki/Lumiera/GuiBrainstorming?action=edit&editor=gui[]

232
doc/devel/meeting_summary/2008-10-10.txt Normal file
View file

@ -0,0 +1,232 @@
2008-10-10 Lumiera Developers Meeting
=====================================
:Author: Ichthyo
:Date: 2008-10-15
Oct 10, 2008 on #lumiera 19:00 - 23:15 UTC +
__Participants__
* cehteh
* ichthyo
* joelholdsworth
* alcarinque
* !KenSentMe
* Plouj
* raffa
* thorwil
* Victor_Sigma
* wildhostile
organisational
--------------
Not much of a fixed agenda this time.
+
Agreement to start with the Logo discussion, as this is of general interest, followed by the design drafts and similar dev topics.
The Lumiera Logo
----------------
Summary of the situation: discussion regarding a logo for Lumiera is going on sporadically since quite some time. Several people from the community have made proposals. Also, there was discussion about the criteria a logo would have to fulfil. Especially the core devs raised the bar and required quite some professional level of design. On the contrary, several members of the community were concerned that such a demanding attitude will hinder creativity in a domain which is far off from coding. Moreover, many people complained they are really excited about Lumiera and strongly want to participate in some manner, but find it very difficult in the current phase of the project to give any valuable contribution.
This summer, one of the proposals by http://farm4.static.flickr.com/3060/2927333034_ac94be80ea_b.jpg[Leandro Ribeiro] gained some popularity and especially was embraced by two of the core devs, while the GUI core dev wasn't convinced and http://www.pipapo.org/pipawiki/JoelHoldsworth/LogoThoughts[explained] his reservation. Prior to this meeting some people joined a brainstorming session and came up with http://www.pipapo.org/pipawiki/Lumiera/Logos?action=AttachFile&do=get&target=combo.png[another design] compiled of several proposals, which could meet the acceptance of the core devs. At the same time, Raffa made an argument for conducting a public contest, similar to the one which gave us the name of Lumiera. The situation for Lumiera is somewhat special. Usually, the community builds when the product is already minimally usable; we can't have users for now, but we have a lot of prospective users.
Thus, while basically it would be possible for the core devs to shorten the process by just picking a design which is acceptable for all, maybe after augmenting it a little, several of the arguments articulated this far are in favour of a more formal decision by a contest:
* it would allow for a lot of people to really participate
* it could help to shape a general (graphic) style for Lumiera
* it could underpin the fact Lumiera indeed is a collaborative effort
* it doesn't mean additional work for the core devs on the whole
* it helps spreading the word
Then, there is some discussion about the requirements. The core devs are concerned to keep up some quality level, because there is the possibility for the community to embrace a design, but when Lumiera enters the area it is intended to do, the standards of comparison will be different. The GIMP logo can be quoted as a negative example here.
Conclusion
~~~~~~~~~~
There will be a Lumiera Logo contest.
* we should further discuss requirements on the Mailinglist until the end of the next week
* the '''deadline for submissions''' will be the next meeting (Nov 12 2008)
* then, after a pre-selection phase, the vote shall be conducted prior to the December meeting.
Some minimal technical requirements will be enforced:
* the basic sign fits a compact rather quadratic bounding box (like 4:3)
* easy to print on various media (on posters, t-shirts, ..)
* basically a shape, using only a small number of solid colours (web safe)
* should work on different backgrounds, esp. not requiring white background
* the design should scale from microscopic size (favicon) up to big posters
* vector graphic source is mandatory
Besides, we give some artistic guidelines
* it should be recognisable
* it should contain something like a sign, not just "Lumiera" in a fancy font
* it should not rely on transparencies, gradients and subtle shades,
* it should be viable, possibly work well in different presentation styles, able to be simplified as well as graphically augmented
Raffa volunteers to help organizing the contest and the voting.
Recurring Topics: design proposals
----------------------------------
Discussion of open http://www.pipapo.org/pipawiki/Lumiera/DesignProcess[design process] drafts.
Mistakes to avoid
~~~~~~~~~~~~~~~~~
http://www.pipapo.org/pipawiki/Lumiera/DesignProcess/MistakestoAvoid[Mistakes to avoid in the Lumiera Design]
We are carrying this one along since quite some time and we'd like to get rid of it, either by reworking it or by dropping it as-is. Because it contains a mixture of things
* we fully agree to 80% of the statements made there, but we think those statements are so very basic and self-evident as to be considered off-topic here. We are aware of the recurring problems with open source video editing. That's why we are here.
* the proposal draws conclusions on two technically substantial points, at which we don't agree. And it fails to provide sufficient (technically sound) arguments to prove these statements.
While it is certainly ''desirable'' to be cross-platform as much as possible and especially '''target Microsoft Windows''', we don't see much possibilities with today's mainstream technology to build an application which is as technologically demanding as a video editor is. We would end up developing two or even three sister applications, or we are forced to sacrifice performance for portability. When put up to face such options, we have a clear preference to concentrate on a really free and open platform.
While it is certainly ''desirable'' to make the application as robust as possible, we don't see how '''using multiple separate processes''' could help us with this goal ''without creating major scalability or performance problems'' due to the use of shared memory. And, yet more important: we don't share the basic assumption made in the proposal, namely that video processing is inherently dangerous. We think the basic algorithms involved are sufficiently well-known and understandable to implement them in a sound manner.
__Conclusion__: drop it
on the question of separate processes
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
The only practical solution would be to separate the GUI. Separating backend and proc-layer doesn't make much sense, technically speaking. We re-considered this question. Joelholdsworth (GUI) and Ichthyo (Proc-layer)prefer running the GUI in-process and to avoid the additional hassle with shared memory. Cehteh (backend) doesn't care for know and would like to re-discuss it as an option later on. This is somewhat similar to the possibility of running Lumiera distributed over the network, which is a goal, but not an immediate one.
Tag Clouds
~~~~~~~~~~
http://www.pipapo.org/pipawiki/Lumiera/DesignProcess/TagCloudsOnResources[Tag clouds on resources]
Tags are considered very important. Meanwhile, we have a much more advanced proposal, which superseeds this one: http://www.pipapo.org/pipawiki/Lumiera/DesignProcess/DelectusShotEvaluator["Delectus"]
__Conclusion__: drop it
Overview of Lumiera Architecture
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
http://www.pipapo.org/pipawiki/Lumiera/DesignProcess/ArchitectureOverview[Architecture Overview]
The purpose of this drawing to give an first impression what subsystem is where and what the names mean. Since discussed last time, Ichthyo re-arranged the plugins as requested and added some details for the backend. Joelholdsworth points out that it's OK for him to show the GUI just as a single block here (and of course the GUI has much more internal structure). Cehteh adds that maintaining this drawing surely is a moving target, so we better remove the rendered image and just retain the textual description and link to the source (SVG), which is in GIT.
__Conclusion__: accept it, change the image to a link
EDLs as Meta-Clips
~~~~~~~~~~~~~~~~~~
http://www.pipapo.org/pipawiki/Lumiera/DesignProcess/EDLsAreMetaClips[EDLs are meta-clips]
This is just a high-level proposal, which doesn't go into technical details of implementing nested EDLs. It just poses the question "do we want to treat nested EDLs as being meta-clip" -- which we do.
__Conclusion__: accepted
The Builder
~~~~~~~~~~~
http://www.pipapo.org/pipawiki/Lumiera/DesignProcess/ProcBuilder[Builder in the Proc-Layer]
Similar to the previous one, this is a high-level proposal. It covers the fundamental approach Ichthyo takes within the Proc-Layer. Cehteh adds that he agrees to 98% and the remaining 2% are just matter of favour. Personally, he would have preferred one large graph which gets garbage collected (instead of the segmented graph)
__Conclusion__: accepted
High-level Model
~~~~~~~~~~~~~~~~
http://www.pipapo.org/pipawiki/Lumiera/DesignProcess/ProcHighLevelModel[Overview of the High-level model within the Proc-Layer]
Cehteh queries if this shouldn't be better moved over to the documentation? He is fine with the contents, but it seems to be a bit voluminous for a design proposal. Ichthyo asks to leave it there just for now, as he feels it still needs review.
__Conclusion__: leave it for now, maybe retract it from the design proposals and file it as documentation?
Lua scripting language
~~~~~~~~~~~~~~~~~~~~~~
http://www.pipapo.org/pipawiki/Lumiera/DesignProcess/ScriptingLanguage[use Lua as required scripting language]
All core devs agree with this decision. Joelholdsworth points out that he is fine with Lua, just he didn't want to write the GUI in it. Ichthyo adds that Lua is probably the best choice from today's mainstream scripting languages, because it is very lightweight. He further points out, that having Lua as ''required'' scripting language doesn't rule out using other popular languages (Python, Ruby) for scripting. Just they aren't required for running Lumiera. Cehteh will have a look at the details as soon as possible, but has currently other more urgent things in the queue. (Plouj shows interest to help here)
__Conclusion__: accepted
Time Handling
~~~~~~~~~~~~~
http://www.pipapo.org/pipawiki/Lumiera/DesignProcess/time_handling[time handling]
A long standing proposal; meanwhile we've decided to build on GAVL, which is now reflected in the text of this proposal too. Ichthyo points out he changed the rounding rule in this proposal from "mathematical" to "floor (x+0.5)". Cehteh asks if we should use a thin library layer on top of gavl, to centralize all the time calculations. There seems to be agreement that this should actually be done ''on demand.'' Joelholdsworth sais sometimes he'd prefer to have a simple raw type to represent time, because it makes calculations much easier. Ichthyo adds that internally (within the layers, but not on interfaces) one could use a thin C++ wrapper with overloaded operators, which is default-convertible to gavl_time.
__Conclusion__: accepted
Note: the proposed rigid testsuite for time handling is necessary only when we introduce a wrapper...
Interface naming convention
~~~~~~~~~~~~~~~~~~~~~~~~~~~
See the design proposal http://www.pipapo.org/pipawiki/Lumiera/DesignProcess/InterfaceNamespaces[Interface Namespaces]. While working on the plugin loader, '''Cehteh''' and '''nasa''' did some brainstorming about a plugin naming scheme and how to get unique interface identifiers. The idea is to use a distinctive prefix, like a condensed organisation domain or email name, or a GPG fingerprint or even a UID, followed by the interface namespace hierarchy. These would be combined into a single identifier (valid C name). The intention is to get non-ambiguous names, yet avoiding the need of a central registry.
__Conclusion__: accepted
general Topics
Config System
-------------
'''cehteh''' landed the first version of this subsystem and asked for review and testing. Currently it's "work with no progress", but it is basically usable (you can get values, you can set values by environment variables, but you can't persist). It should be augmented on demand, e.g. by adding the support for more value types (value types are a hint how to parse, and link the parsed value to a given C type).
Use of Namespaces
-----------------
Currently there is no clear uniform rule regarding namespaces use. *Joelholdsworth* places his code within lumiera::gui and below. He points out that external libs like Cairo, GTK, GLib will bring in their hierarchies and all of Lumiera should comply and put anything below a lumiera:: root. On the contrary, *Ichthyo* deliberately moved his implementation code away from the central lumiera:: interface hierarchy into shallow namespaces and avoids nesting. He argues that having all code below lumiera:: effectively makes this namespace global or forces it to be void of any function; rather he'd prefer to import every interface explicitly into the implementation namespace. *Cehteh* argues that having a global lumiera::, even if empty, would mark a general claim and stand for the uniformity of the project. Generally, there should be some correspondence between folders and namespaces.
No conclusion yet, to be discussed further.
Interface Definition Language
-----------------------------
In his work on the plugin loader, '''Cehteh''' created a first draft how to export an interface, and calls for review. An example can be found in http://www.lumiera.org/gitweb?p=lumiera/ct;a=blob;f=tests/backend/test-interfaces.c;h=fb1c4d30a34414767a313d24df60e679c96ad2a7;hb=7323114e77348995ccaf03417136aef7ee332776[tests/backend/test-interfaces.c]
An interface is a list of "slots" mapping functions. The interface descriptor is itself realised as interface, an thus can be versioned, extended and specialised. By use of some glue code and macros we create a simple Interface Definition Language
* after exporting a header with the C interface, including the types to be used...
* LUMIERA_INTERFACE_DECLARE creates an interface description (i.e. the official interface)
* implementing such an interface can be done in 3 major ways
* LUMIERA_INTERFACE_INSTANCE creates an instance on-the-fly (e.g. for descriptors)
* LUMIERA_EXPORT for defining a table of functions/interfaces to export from the core
* LUMIERA_PLUGIN for defining an interface table for functions located within a dynlib module (plugin)
* besides, with LUMIERA_INTERFACE_INLINE you can create a function on-the-fly while LUMIERA_INTERFACE_MAP maps onto an existing function directly
The plugin loading system cares for mapping the given implementation function to the interface slots. Interfaces from the core need to be registered before they can be used, while for plugins this is done automatically on loading the module. The client code later just calls {{{interface_open()}}} and {{{interface_close()}}} where it doesn't matter anymore if the invoked function is in the core or loaded from an dynlib (plugin); loader, registry and/or plugin-DB shall manage it transparently.
Version numbering starts with 1 and uses minor version numbers for compatible changes and major version numbers for everything that breaks existing asumptions. Version number zero is reserved for experimental work and per definition always the most recent version number.
The system is designed to be very flexible and extensible, but this foundation really needs thorough review.
Joelholdworth expresses some concern regarding the UIDs in octal notation used within the interface description. Cehteh explains they never need to be specified by client code. They are just distinctive IDs and provided for some special case within the plugin loader / serializer. He plans to provide a simple tool which automatically replaces a $LUIDGEN with such a bitstring. The octal notation was chosen, because it is the most compact albeit portable notation possible in C source code.
Conclusion
~~~~~~~~~~
looks good, agreement by all core devs.
+
Should be reviewed and investigated in detail to find any hidden problems.
Next meeting
------------
There were some problems with the meeting schedule. Using the first week of month seems to be problematic. We'll try with second wednesday...
The next meeting is scheduled for '''Wednesday Nov 12 2008 19:30 UTC''' at #lumiera

785
doc/devel/meeting_summary/2008-11-12.txt Normal file
View file

@ -0,0 +1,785 @@
2008-11-12 Lumiera Developers Meeting
=====================================
:Author: Ichthyostega
:Date: 2008-11-12
Dec 11, 2012 on #lumiera 20:00 - 01:15 UTC +
__Participants__
* cehteh
* ichthyo
* joelholdsworth
* raffa
_Summary and transcript supplemented by Ichthyo 11/2011_
Platform and lib dependency status
----------------------------------
Right away, there is agreement between all 3 core devs that the Lumiera project
should try to stay close to a *stable reference linux distribution*. Usually,
developers tend to stick to ``latest greatest'', causing lots of difficult to resolve
problems to packagers and ursers of their software. The Lumiera team wants to go into
the opposite direction: staying close to a mature and stable distro and rather try to
backport any newer libraries, when they are _really required_ for getting ahead with
the coding. Since _cehteh_ and _ichthyo_ are Debian users and _joelholdsworth_ tends
to use a moderately recent Ubuntu, the intention is to use Debian/stable as reference.
Conclusion
~~~~~~~~~~
* use *Debian/stable* (currently Lenny) as reference distribution
* care for backports of any newer libraries required
* use VMs to verify the build on a moderate selection of more recent distros
Problems with GUI Plugins
-------------------------
The core devs are aware of some challenging problems regarding plug-ins for the
GUI and effect plug-ins requiring an specific GUI control or option pane. Still the more
since the intention is for Lumiera to adapt _existing plug-in systems_ rather then
rolling yet-another new plug-in standard no one else cares to support. Initially,
specific Lumiera plug-ins will be provided only for some very basic modular functionality,
eg. fader, panner, blur.
_joelholdsworth_ points out that configurability can't replace real GUI and workflow design.
He quotes from a recent Blender GUI discussion
[quote, the Belnder UI reviews]
____________________________________________________________________
Lastly, I’d like to address another misconception, this time about customizability.
There has been a notion that the solution to most of the UI problems can be solved with added customizability.
The notion goes that if the UI is inefficient, the user can simply customize it herself to suit her needs.
This claim is wrong for several reasons:
- It is impossible to customize something you have not fully comprehended yet,
so it in no way helps the learning process.
- It makes an application less predictable, because the user cannot rely on the application
to always act the same.
- It takes focus away from the users content, and over to managing the application itself,
which is what we wanted to avoid in the first place.
- It undermines the portability of Blender, because you cannot rely on Blender
functioning the same way across different systems.
- Customizability does not equate to flexibility.
- 99% of Blenders user will use the default setup anyway.
This is not to say that customizability is always bad - having the ability to change the hotkeys
from the defaults to match another 3D application such as Maya or Softimage XSI can make it easier
for users of those applications to adapt to Blender.
____________________________________________________________________
Consequently, Joel wants to try just doing the UI _right_, then add a little bit of customization back in here and there.
Additionally, agreement is to have a kind-of ``perspective switcher'' (like in Eclipse), so saving different
panel layouts would be covered easily
.-- Discussion about foreseeable problems with GUI Plugins --
[caption="☉Transcript☉ "]
----------------------------
[2008-11-12 21:53:50] <ichthyo> I thought we could just discuss the problem informally
and share some thoughts what we think is most critical
[2008-11-12 22:06:01] <ichthyo> there are plugins which don't provide a GUI, and some plugin systems allow
for a standard method for creating plugin guis
[2008-11-12 22:05:46] <joelholdsworth> so bundling GUI together with plugins is easier now, right?
[2008-11-12 22:05:59] <joelholdsworth> we implement some kind of parameters system
[2008-11-12 22:06:03] <ichthyo> yes
[2008-11-12 22:06:14] <cehteh> joelholdsworth: thats up to you .. but the plugin system is ready
[2008-11-12 22:06:27] <joelholdsworth> yeah that's fine
[2008-11-12 22:06:38] <joelholdsworth> if you want to extend the GUI you just expose and interface for it
[2008-11-12 22:07:51] * cehteh thinks about some canvas widget which the plugin can use to draw stuff there
[2008-11-12 22:08:26] <joelholdsworth> cehteh: as in overlays on the video canvas?
[2008-11-12 22:09:19] <cehteh> joelholdsworth: up to you, not necessary overlays
[2008-11-12 22:05:07] <ichthyo> I see some problems we should thik about
[2008-11-12 22:06:50] <ichthyo> so, basically loading a processing routine from
a plugin and using it to process video/audio is no problem. Basically you define
some interface (first in C++) then translate it to the lumiera interface macros
but with the GUI there are several conceptual problems
[2008-11-12 22:07:35] <ichthyo> first: There are plugins which don't have a GUI. Thats fine
[2008-11-12 22:08:05] <ichthyo> we need to define a parameter system, as joel pointed out
[2008-11-12 22:08:38] <ichthyo> and then we need to fit the existing plugin systems, each one, to use this parameter system
[2008-11-12 22:08:45] <cehteh> thats type:key:value at simplest
[2008-11-12 22:08:51] <cehteh> maybe with some representation hints
[2008-11-12 22:08:58] <ichthyo> there is more about it
[2008-11-12 22:09:03] <ichthyo> representation hints
[2008-11-12 22:09:07] <ichthyo> type of gui control
[2008-11-12 22:09:15] <ichthyo> logarithmic/linear scale
[2008-11-12 22:09:17] <ichthyo> zero point
[2008-11-12 22:09:19] <ichthyo> range
[2008-11-12 22:09:27] <ichthyo> but it is manageable
[2008-11-12 22:09:38] <ichthyo> LADSPA is a prominent example
[2008-11-12 22:09:55] <ichthyo> then, the problem in the GUI is how to arrange those controls
[2008-11-12 22:10:20] <ichthyo> anyway, I think this alternative is the most simple one and the one which will
create the least problems
[2008-11-12 22:10:34] <joelholdsworth> ichthyo: I figured the UI will just expose parameters
[2008-11-12 22:10:42] <joelholdsworth> so you expose them semantically - maybe in a tree
[2008-11-12 22:10:51] <joelholdsworth> and add rich presentation hints
[2008-11-12 22:11:00] <joelholdsworth> that hit the UI how to lay out out
[2008-11-12 22:10:59] <cehteh> i think layout should be optional by some config/stylesheet
[2008-11-12 22:11:05] <ichthyo> cehteh: possible. But those things are already defined by the
plugin systems we want to wrap up.
[2008-11-12 22:13:01] <cehteh> you have a list (or maybe tree) of parameters .. presentation aside
... tree just because some parameters depend on the presence of some else
and you have a presentation ..
[2008-11-12 22:11:53] <ichthyo> next problem is:
[2008-11-12 22:12:16] <ichthyo> the plugins need to allocate some space in the GUI and should not clutter
the workspace this is a tough really tough interface design problem
because some plugins aren't just some "optional stuff", but wired
right into the core operation
[2008-11-12 22:12:54] <joelholdsworth> well ichthyo: I'm not planning to allow plugins to "take over"
handling workflow. they get to extend very specific parts of the UI
[2008-11-12 22:13:09] <joelholdsworth> e.g. params
[2008-11-12 22:13:11] <joelholdsworth> or the canvas
[2008-11-12 22:13:15] <joelholdsworth> or timeline
[2008-11-12 22:13:22] <joelholdsworth> and only in very specific ways
[2008-11-12 22:14:15] <joelholdsworth> ichthyo: and most of the time plugins wont extend anything --
their whole UI will be exposed semantically (with UI presentation hints)
(in addition to the plain semantics)
[2008-11-12 22:13:27] <ichthyo> thats fine
[2008-11-12 22:14:58] <ichthyo> but do you see the interface design problem?
basically you want to design a smooth handling. but now you need to rely
on some module which is plugged in and you don't know much about this module
how is it possible to tweak effects easily while you just don't know much
about those effects?
[2008-11-12 22:15:23] <joelholdsworth> yes I agree
[2008-11-12 22:15:48] <ichthyo> then there is another problem:
[2008-11-12 22:15:56] <ichthyo> key bindings and MIDI bindings
[2008-11-12 22:16:10] <ichthyo> you want to bind key shortcuts and external controllers to GUI elements
[2008-11-12 22:16:20] <ichthyo> and of course plugins need to be included into this system
[2008-11-12 22:16:35] <joelholdsworth> ok that's quite easy - you just right click on the control
(or click a tiny button) that opens up a menu that lets you set this stuff
[2008-11-12 22:16:59] <joelholdsworth> UI design? well that's what presentation hints help with
[2008-11-12 22:17:26] <joelholdsworth> they set size, layout, the appropriate type of slider
for example to expose this value
[2008-11-12 22:17:29] <ichthyo> and what happens when the respective plugin options window isn't opened at all?
[2008-11-12 22:17:47] <ichthyo> or when you have multiple instances of the same plugin?
[2008-11-12 22:17:48] <joelholdsworth> what do you mean?
[2008-11-12 22:18:11] <ichthyo> for example: you use a sound panner plugin
and you want to adjust the sound pan in 3 dimensions by key shortcuts,
or to bind them to some knobs on an external control surface
and now you have multiple instances of the same panner plugin
so we need a naming scheme
[2008-11-12 22:19:03] <cehteh> how i thought about this:
1) plugin provides the list of its parameters (maybe as tree)
2) there is a config file (config system) which gets installed
with the plugins which defines the default layout
3) user might override that in his own configs ...
but still only defaults for new instances of the things
4) the actual configured state is saved with the session
while all this config just is a definition which gets applied to a kindof stylesheet
[2008-11-12 22:20:05] <ichthyo> as tree for sure, because many plugins have nested windows for each instance
[2008-11-12 22:21:49] <ichthyo> which means we need a naming/addressing scheme
[2008-11-12 22:22:20] <ichthyo> ...which creates sort of a cascading scheme
[2008-11-12 22:22:42] <ichthyo> I mean, a naming/addressing scheme
[2008-11-12 22:23:04] <ichthyo> for distinguishing identical parameters in different instances of yet-unknown plugins
[2008-11-12 22:23:19] <ichthyo> and now consider: where should these key bindings attach to?
[2008-11-12 22:23:39] <ichthyo> to a specific instance, or to the suitable instance which is just in focus?
[2008-11-12 22:23:46] <ichthyo> or both (probably)
[2008-11-12 22:24:03] <ichthyo> you may want to bind a different knob to each different panner plugin
[2008-11-12 22:24:25] <ichthyo> or alternatively, you may want to bind "the pan knob" to the "current pannner"
[2008-11-12 22:24:42] <ichthyo> this would require that we introduce a focus concept
[2008-11-12 22:25:00] <ichthyo> I think thorwil proposed something similar in the last discussion
[2008-11-12 22:25:16] <ichthyo> so we have a notion of the "current track", "current clip"
[2008-11-12 22:25:49] <ichthyo> and then there is another problem with plugin GUIs
[2008-11-12 22:26:10] <ichthyo> many plugin systems, especially VST have custom GUIs
[2008-11-12 22:26:27] <ichthyo> and may not even work properly without using those
[2008-11-12 22:26:48] <ichthyo> which means, some GUI code in the plugin takes over. That is something I utterly dislike
[2008-11-12 22:33:29] <cehteh> no thats not needed
[2008-11-12 22:33:29] <cehteh> in the session each object has a uuid thats its address
[2008-11-12 22:33:29] <cehteh> each parameter has a uuid too(interface system)
[2008-11-12 22:33:31] <cehteh> so you can say parameter 0x2344545645654534 of object 0x4543765732525t643 ....
[2008-11-12 22:33:34] <cehteh> "yet unknown" doesnt exist .. it is either unknown or known :)
[2008-11-12 22:55:26] <ichthyo> well ..the problem is just: you can't avoid things like a user binding
some external control surface to a certain gui control. All I wanted is
to bring the problem to everyones attention, also, that we need a parameter system
and that this parameter system is probably tree-like, not just flat
[2008-11-12 22:57:06] <cehteh> i want anythnig be addressable with uuid(manipulating_function)+uuid(object)
[2008-11-12 22:58:41] <ichthyo> for me, this is just a opaque reference which is implemented in some way. So, assumed
I have the object representing the connectionn to the plugin, I can provide the GUI with
some means for re-addressing a certain parameter, e.g. for a key binding, and, of course
for re-establishing the same binding when re-creating the session from storage
[2008-11-12 23:00:50] <ichthyo> Further, with plugin guis instances there is yet another problem in that you probably
have more option windows of some plugins as you can show in any reasonable GUI
These issues were one of the reasons I made this "FeatureBundle" proposal some time ago,
Meaning there is a mechanism allowing you to bundle
- an existing external plugin
- an adaptation we do for connecting it to the gui
- maybe a script or gui
- customisation info we need for the gui layout
...and provide all together as a single bundle the user can install
[2008-11-12 23:11:15] <ichthyo> joelholdsworth: maybe you have already some ideas how to deal with all those plugin
option windows? they are a real problem for getting a smooth handling and workflow.
because: you'll spend a considerable amount of the whole project time with tewaking
[2008-11-12 23:12:28] <ichthyo> (yes, that's what I am about to point out....)
[2008-11-12 23:13:05] <ichthyo> ...if you work on tweaking, you typically have a small amount of plugin windows
which need to stay open with maybe subtabs for all active plugins
say some color correction on 2 tracks, and then you open and close variouos plugins
for several different clips ...plus, maybe the possibility to "pin" some of the plugin
windows, while others just allocate the next free slot
[2008-11-12 23:13:14] <joelholdsworth> I guess we need some modeless way of putting plugin controls in a panel
[2008-11-12 23:15:32] <joelholdsworth> so maybe having panels open up - or do it like inkscape with a scrollable panel
container that gets longer as more panels are opened
[2008-11-12 23:20:10] <cehteh> Cinelerra's plugin guis who constantly get lost behind the main window are really a mess
[2008-11-12 23:19:28] <ichthyo> joelholdsworth: have you already any ideas about focus handling?
[2008-11-12 23:20:15] <ichthyo> joelholdsworth: do we get a "current object", a "current track" or the like
which receives keybord input
[2008-11-12 23:20:37] <joelholdsworth> keybaord focus, yes - that was the plan
[2008-11-12 23:20:54] <ichthyo> thorwil recently pointed at the concept of having an "active element" within
a larger selected group of elements e.g. you have a "current clip" and within it
a currently active effect attachment and if you press the shortcut, you switch over
to the automation curve of this plugin if there is one, etc. etc...
[2008-11-12 23:21:44] <cehteh> think about future ... multiple elements have focus from different devices
[2008-11-12 23:22:06] <cehteh> multitouch ... and other controlers
[2008-11-12 23:22:06] <joelholdsworth> I think we have to treat keyboard focus as special
[2008-11-12 23:22:23] <cehteh> next X version will utilitze multiple mouse pointers
[2008-11-12 23:22:35] <ichthyo> e.g. what's with the mouse wheel?
[2008-11-12 23:22:42] <ichthyo> what's with a jog dial on a control surface
[2008-11-12 23:23:03] <ichthyo> or with the play/stop buttons on a control surface?
[2008-11-12 23:23:03] <joelholdsworth> my plan is to treat the jog dial as special
[2008-11-12 23:23:23] <joelholdsworth> yes again - special button assignment
[2008-11-12 23:23:23] <ichthyo> sort of like the mouse wheel?
[2008-11-12 23:23:34] <cehteh> imagine user wants a 2nd normal pc keyboard as video controler
[2008-11-12 23:23:42] <ichthyo> because all those rather follow sort of a focus
[2008-11-12 23:23:56] <ichthyo> maybe together with modifier keys
[2008-11-12 23:24:04] <joelholdsworth> yes again that would work via an auxialiry controller setup config
the primary keybaord and mouse work in the normal way
we can't have two keyboard focusses - it's just won't work
we can have a second keyboard - a board of hotkeys
[2008-11-12 23:25:21] <joelholdsworth> there's mouse, keyboard and auxiliary controllers
[2008-11-12 23:25:34] <joelholdsworth> and those controllers would be mapped to the UI using a cunning dialog
[2008-11-12 23:25:58] <ichthyo> and what I said some time ago today with the binding for the "current" pan
is a similar idea. Because also such a binding should rather follow the focus
it doesn't help if you bind it to a specific instance of the panner plugin
the moment you have more then 8 panners, it won't scale
rather, it would be helpful, if you had a "current" fade and pan and sound level,
which would follow the focus, so you could e.g. navigate the focus with the
right hand at the cursor keys, and leave the left hand on your control surface's knobs
please understand me right: I don't want to push features, and I will stay out of GUI
issues. All I want is that everyone in the project is aware of this problems
[2008-11-12 23:27:44] <joelholdsworth> yes ok
[2008-11-12 23:28:45] <joelholdsworth> I see the problem
[2008-11-12 23:28:58] <ichthyo> ok thanks
[2008-11-12 23:31:54] <ichthyo> but it's fine we are all aware of the problem and seemingly share the same approach
config system + some sort of automatic persistent references within proc + some
indirection + maybe a focus system
[2008-11-12 23:33:10] <ichthyo> and another point I want to note is: all of us are rather reluctant to allow plugins
to create GUIs on their own
[2008-11-12 23:33:27] <joelholdsworth> depends what you mean by that
[2008-11-12 23:33:29] <cehteh> i guess we cant avoid that
[2008-11-12 23:33:38] <ichthyo> but note: there is a great urge from some some users for such things
[2008-11-12 23:33:39] <cehteh> for really external plugins
[2008-11-12 23:33:47] <joelholdsworth> yes
[2008-11-12 23:34:01] <cehteh> i am more cared about them fireing threads up :P
[2008-11-12 23:34:13] <ichthyo> thats part of the problem
[2008-11-12 23:34:37] <ichthyo> some start registering their own key shortcuts and all sorts of horrible things
[2008-11-12 23:34:42] <joelholdsworth> if you let each plugin create every bit of it's ui, you get a lot of code
copy/paste and a lot of diverging GUI, so you need to "help" the plugins
by putting them in a strict framework. Then, on top of that we allow
extensions which allow a little more flexibility
[2008-11-12 23:35:20] <ichthyo> Steinberg failed to address this problem properly and VST bears on this heritage until today
[2008-11-12 23:35:35] <cehteh> make it in a way where the 'correct' way is also the most convenient for any programmer is
all and the best we can do
[2008-11-12 23:35:49] * ichthyo nods
[2008-11-12 23:36:04] <joelholdsworth> yes
[2008-11-12 23:36:07] <cehteh> note the 'any' ...
[2008-11-12 23:36:20] <ichthyo> :-P
[2008-11-12 23:36:26] <cehteh> means mad programmers, beginner programmers, experienced programmers, html programmers ...
[2008-11-12 23:36:52] <joelholdsworth> but I'm thinking we do need to remember that the majority of our plugin developers
will be part of this project team
[2008-11-12 23:38:09] <ichthyo> joelholdsworth: no, it won't be this way. Rather, there are some mainstream plugin systems.
People will foremost want to use plugins they know already, letting asside a small set of
core plugins, yes, those are the ones we provide as native lumiera plugins
[2008-11-12 23:38:42] <joelholdsworth> hmm ok
[2008-11-12 23:38:44] <ichthyo> a fader, a mask etc.
[2008-11-12 23:38:50] <ichthyo> but anything besides that
[2008-11-12 23:39:06] <ichthyo> every sound engenieer has his favorite eq and compressor
[2008-11-12 23:39:26] <ichthyo> and he will rather swich to another application than being forced
to use a compressor he isn't fine with. Same for advanced video stuff
----------------------------
Source code organisation
------------------------
In the weeks before that meeting some discussions regarding the start-up of the main application
took place, including the question how to load and pull up the Lumiera GTK GUI as a plug-in.
Moreover, _cehteh_ just ``landed'' the branch with the first draft of his new plugin/interface
system. Which prompted us to sort out the locations and arrangement of those parts of the source
dealing with the application framework. The decision was to place that code into a small library,
which we call the *Lumiera common application services* (+liblumieracommon.so+), to be kept separate
from the *support library* (+liblumiera.so+).
.-- Include dir and namespaces --
[caption="☉Transcript☉ "]
----------------------------
[2008-11-12 23:49:43] <ichthyo> There is the namespace issue we left open on the last meeting
[2008-11-12 23:50:34] <cehteh> its really oblivious that you dont need to reflect the C++ namespaces
1:1 in the directory structure. I still think having all our stuff into lumiera::
would be good .. but :)
[2008-11-12 23:51:33] <ichthyo> in any case, it is closely interconnected to the "interface namespace":
which means the following: there are some interfaces (opaque datatypes
plus forward decls for C, abstract base classes for C++)
plus the things defined via the interface/plugin system
this stuff is our "public interface"
[2008-11-12 23:53:01] <ichthyo> and I really try hard to keep it separate from the implementation
[2008-11-12 23:53:14] <cehteh> the interface/plugin system has now its own rules which are somewhat special
and good by that way. I am thinking about a interface_doc.h which can be used
to generate documentation from the interface language with some special gcc/cpp flags
[2008-11-12 23:53:18] <ichthyo> those interface parts will be contained in a "Lumiera sdk"
and probably all those interfaces (for C++) will go into namespace lumiera::
because it's just the best and most natural pick
and, as said, I really want us to try hard to get this interface part
a real interface part and cleanly separated from the implementation
meaning: I don't want *.c and *.cpp files in this package/directory/namespace,
unless they are really needed there
[2008-11-12 23:55:55] <cehteh> well interfaces themself need to be .c they need to be implemented
and the implementation might adapt the code for example the config system
sometimes returns internal state as success/failure indicator
but its interface just returns integers
[2008-11-12 23:56:13] <ichthyo> so the question regarding namespaces is connected to the question:
how is the implementation organized in relation to the interface part?
[2008-11-12 23:59:41] <cehteh> config_interface.h needs to be installed someday (with a better name maybe)
[2008-11-13 00:01:52] <cehteh> do we want a src/includes/ ?
[2008-11-13 00:02:04] <cehteh> for includes which get installed
[2008-11-13 00:00:07] <ichthyo> so, someone *using* the interface system probably will need to include config_interface.h
[2008-11-13 00:00:19] <ichthyo> but he doesn't need config.c
[2008-11-13 00:00:29] <cehteh> config_interface.c is the mapping/actual implementation ...
which gets linked in but the user doesn't need it
[2008-11-13 00:00:52] <ichthyo> yes, thats my point. So we should try to separate those cleanly.
[2008-11-12 23:59:38] <ichthyo> so lets assume src/lumiera contains our interface
(I just take it as an example to make my point more clear)
[2008-11-13 00:01:01] <ichthyo> Let me explain it on another more elaborate example
[2008-11-13 00:01:47] <ichthyo> I have a class "Session" which is located in a file session.hpp
[2008-11-13 00:01:58] <ichthyo> but it is just a ABC
[2008-11-13 00:02:18] <ichthyo> and I have a Class SessionImpl which is contained in sessionimpl.hpp and sessionimpl.cpp
[2008-11-13 00:02:44] <ichthyo> and in the "interface package" there should be *only* session.hpp
[2008-11-13 00:03:11] <ichthyo> probably this class Session should be moved into namespace lumiera::
[2008-11-13 00:03:16] <ichthyo> then the situation would be clean
[2008-11-13 00:03:32] <ichthyo> note: up to now i wrote only implementation code.
[2008-11-13 00:03:50] <cehteh> yes, me too
[2008-11-13 00:04:07] <ichthyo> but I already made the class hierarchy this way, i.e. everyone *uses* just the class Session
[2008-11-13 00:04:28] <ichthyo> and only the class SessManagerImpl knows of the implementation class SessionImpl
[2008-11-13 00:04:32] <ichthyo> and so on
[2008-11-13 00:04:50] <cehteh> well thats not important yet...
I currently have some 'mess' of includes because its just easier for now to mix them
all together and relative easy to clean up when done, and I don't have a differentiation
between interface and implementation headers .. because
a) its all implementation code yet
[2008-11-13 00:06:06] <cehteh> b) i was under the impression that i only wanted to export interfaces
over 'Interfaces'
[2008-11-13 00:06:25] <ichthyo> ...which is fine
[2008-11-13 00:08:24] <cehteh> so .. public interfaces ... place in:
[ ] src/include/*.h
[ ] src/*/*_interface.h
[2008-11-13 00:08:45] <ichthyo> I am for the former: put them into a separate directory
[2008-11-13 00:08:55] <ichthyo> put them in a clearly separated namespace
[2008-11-13 00:08:29] <ichthyo> but... on the long run I am sure we don't just get one "includes" package,
rather I guess we get a (very small) tree of interfaces.
That's the rationale behind my keeping the implementation namespaces completely apart
[2008-11-13 00:08:53] <cehteh> ok
[2008-11-13 00:09:43] <cehteh> the _interface.c is the glue code which can stay in the implementation dir imo
[2008-11-13 00:10:16] <cehteh> actually i wanted a include dir too .. i just didnt start because i wanted to ask you too
[2008-11-13 00:10:41] <cehteh> ah another thing i asked joel before .. i dump the tests dir out of the Doxyfile
[2008-11-13 00:10:58] <ichthyo> really?
[2008-11-13 00:11:09] <cehteh> main documentation should have less noise,
the include graphs are messed up with all test dependencies.
But we can make special doxyfiles for special purposes
Doxyfile.all Doxyfile.small Doxyfile.browser
[2008-11-13 00:13:34] <ichthyo> yes, thats fine.
So we make a "small" doc set with just the core facilities, and a "complete" doc set,
which includes: test cases and documentation on private members, so someone really
needing to find out how something works in detail can use the "complete" set.
At the end, I bet we will have almost 60% test code and 40% actual code
[2008-11-13 00:16:29] <ichthyo> ok, back to the include dir question:
My proposal would be to use src/lumiera/... and namespace lumiera::...
(and its children) for the public interfaces
[2008-11-13 00:17:04] <cehteh> mhm thats the usual reversal operation
[2008-11-13 00:17:31] <cehteh> src/lumiera/include/ ... gets installed in /usr/include/lumiera :P
[2008-11-13 00:17:40] <cehteh> ok you dont have the include
[2008-11-13 00:17:52] <cehteh> but i'd rather use the name include
[2008-11-13 00:17:59] <ichthyo> so src/lumiera/** -> /usr/include/lumiera/**
[2008-11-13 00:18:22] <ichthyo> this would make the namespaces match 100% to the directories
[2008-11-13 00:18:13] <cehteh> i dont like that much
[2008-11-13 00:18:18] <cehteh> better src/include
[2008-11-13 00:18:22] <cehteh> rationale:
[2008-11-13 00:18:43] <cehteh> when i look at some project i dont know and see a ./include
i know that there are the interfaces. If it is ./projectname then
i expect there a huge tree of code which makes the project
[2008-11-13 00:20:04] <cehteh> which is actually true for most project ...
at least i dont know anyone which names its includes projectname
[2008-11-13 00:20:14] <ichthyo> thats a good argument
[2008-11-13 00:20:23] <cehteh> the projectname/include is somewhat common .. but bit bloated
[2008-11-13 00:20:48] <cehteh> so better include
[2008-11-13 00:20:49] <ichthyo> yeah, thats a good point
[2008-11-13 00:21:04] <ichthyo> so namespace lumiera:: -> /src/include
[2008-11-13 00:21:35] <ichthyo> namespace lumiera::edit -> /src/include/edit (just as an example)
[2008-11-13 00:22:00] <ichthyo> while namespace proc::builder -> src/proc/builder
[2008-11-13 00:22:31] <cehteh> but i think keeping it flat would be ok
[2008-11-13 00:22:58] <ichthyo> I didn't say I want it. But it may happen to be necessary or just better,
when the root of the interfaces gets too big
[2008-11-13 00:23:00] <cehteh> at least i dont plan to make a tree behind it
[2008-11-13 00:23:08] <ichthyo> neither do I
[2008-11-13 00:23:11] <ichthyo> but... well
[2008-11-13 00:23:14] <cehteh> i doubt that it will get very big
[2008-11-13 00:23:29] <ichthyo> I've seen enough projects to prove the contrary
[2008-11-13 00:23:45] <ichthyo> and when an interface dir as 60 entries it starts to get messy
[2008-11-13 00:23:37] <cehteh> even with 100 files there it will still be manageable
[2008-11-13 00:24:02] <cehteh> and if not .. we may reconsider that before we release
[2008-11-13 00:24:38] <cehteh> a deeper hierarchy which is very sparse is much more annoying
than a dir with up to 100 files
[2008-11-13 00:24:49] <ichthyo> but anyway, the important point is I really want to try hard to get public interfaces
and implementation code cleanly separated
[2008-11-13 00:25:31] <cehteh> for C++ absolutely .. for C i less that a *little* bit
[2008-11-13 00:25:54] <cehteh> for performance reasons and because most of the backend wont end in public interfaces anyway
[2008-11-13 00:26:27] <ichthyo> just out of curiosity: what are the performance reasons?
[2008-11-13 00:26:51] <cehteh> i have some static inline accessor functions ..
which need to be in the .h and need full public structure definitions
which expose private details. I think these will stay private
but if not i wont really care.
[2008-11-13 00:27:06] <ichthyo> ah, I see
[2008-11-13 00:27:59] <ichthyo> probably those just count as "reasonable exceptions from the rule"
[2008-11-13 00:28:33] <cehteh> well C has no access protection at all .. you have to do whats allowed/documented
[2008-11-13 00:28:47] <cehteh> any improvisation counts as undefined behaviour .. point
[2008-11-13 00:28:26] <ichthyo> and besides: also within the implementation of one layer,
there are further interfaces, e.g the builder has an interface
[2008-11-13 00:32:03] <ichthyo> actually, I wanted to write something down, but couldn't find the time
[2008-11-13 00:32:17] <ichthyo> But I may well tell it just informally if that's ok ?
[2008-11-13 00:33:02] <ichthyo> I always assume we have separate interface namespace(s)
and implementation is in a different namespace then the exported interface
basically I see two scenarios:
so, the first scenario I'd call "libarry type"
it means that the implementation namespace is nested
eg namespace mylib <-- the stuff the library users include and use in their code
[2008-11-13 00:34:23] <ichthyo> namespace mylib::impl and mylib::impl::special <- the actual code making the library work
[2008-11-13 00:34:50] <ichthyo> second scenario is rather what I was aiming for
namespace lumiera <-- the interfaces
[2008-11-13 00:35:45] <ichthyo> namespace proc::builder:.... <-- the implementation has a completely separate hierarchy
for each subsystem
[2008-11-13 00:36:08] <ichthyo> my rationale for prefering the second aproach is:
[2008-11-13 00:36:32] <ichthyo> in our case, we are "implementation heavy".
We will certainly have much more implementation code than interfaces
we often will build more nested namespaces for the implementation,
but can live with just one shallow public interface
besides, you can just grab one directory and install it as "include",
the way we discussed
for a real library, the situation is reversed: Often you have much code close
to the interface, maybe even inline
e.g. think at boost:
half of the code is immediately inline within the interface classes you use
and the more technical parts are in shallow sub-namespaces below
the good side of course is, that the implementation code doesn't need
to pull in the interface, because he already sees it because of the nesting.
but for our situation, for me, the need to pull in explicitly any interface
you want to use or implement counts rather as a good thing
because it clearly documents what non-local parts you use
that was my rationale for keeping the implementation part
in a completely separate hierarchy.
[2008-11-13 00:43:17] <ichthyo> joelholdsworth: so you moved your gui code within namespace gui, is that correct?
[2008-11-13 00:43:50] <joelholdsworth> yes that's right!
[2008-11-13 00:43:59] <ichthyo> because, then the issue seems to be settled.
Originally I had yet more shallow trees.
But your argument with the matching directory names and the consistency
is an important point. So then, I'll go ahead and move my implementationn code
into a new root namespace proc and then start cleaning up the library part
[2008-11-13 00:44:47] <joelholdsworth> great :)
[2008-11-13 00:45:20] <ichthyo> meaning, you can expect quite some stuff moving between /src/common and /src/lib
[2008-11-13 00:45:40] <ichthyo> on the long run, probably the intention should be for "common" to disappear
[2008-11-13 00:46:08] <ichthyo> either stuff goes into namespace lumiera::, meaning it's an interface
and the header should go into "src/include" or stuff is heavy support lib
implementation stuff and thus goes below src/lib
----------------------------
.-- build system, plugin tree --
[caption="☉Transcript☉ "]
----------------------------
[2008-11-13 00:47:06] <ichthyo> which brings us to another question
[2008-11-13 00:47:18] <ichthyo> how do we organize building plugins?
because: one plugin may contain several source files and headers,
but needs to be linked into one module
[2008-11-13 00:47:55] <cehteh> plugins dir
[2008-11-13 00:48:11] <cehteh> and a tree there
[2008-11-13 00:48:22] <cehteh> plugins/video/effects
[2008-11-13 00:49:56] <cehteh> plugins/video/effects/foo/ ... with foobar.c foobaz.c links to foo.lum
[2008-11-13 00:52:38] <cehteh> plugins/video/effects/foo/foo.c builds foo.lum and
installs it in $(pkglibdir)/plugins/video/effects/foo.lum
[2008-11-13 00:48:24] <ichthyo> background is: I am aiming to get the build process as much rules-directed as possible
[2008-11-13 00:48:27] <ichthyo> thus:
[2008-11-13 00:48:54] <ichthyo> can we come up with a rule about how plugins will be built?
[2008-11-13 00:50:33] <ichthyo> again the notorious namespace question
[2008-11-13 00:51:08] <ichthyo> if it's a rather large plugin, e.g. a plugin providing an adapter for an
external media type libarary or such stuff then it probably can have nested namespaces
[2008-11-13 00:51:56] <cehteh> we really dont need a 1:1 relation between dirs and namespaces
[2008-11-13 00:52:08] <ichthyo> so, then, how can the build system tell at what level of the tree below plugins
we'd need to start with building one shared libarary?
[2008-11-13 00:52:46] <ichthyo> would the following rule be ok for you?
[2008-11-13 00:52:58] <ichthyo> start with dir src/plugins and descend
[2008-11-13 00:53:12] <ichthyo> depth-first tree search
[2008-11-13 00:53:28] <ichthyo> when you enter a directory which contains a real source file
[2008-11-13 00:53:46] <ichthyo> then build everything below it into one shared module
[2008-11-13 00:53:31] <cehteh> well i just suggesting here .. for the plugin loader ist relative simple
[2008-11-13 00:53:56] <cehteh> it just searches in the paths you give him (and doesnt descend itself)
[2008-11-13 00:54:19] <ichthyo> no, I really want to configure as few pathes as possible
[2008-11-13 00:54:23] <cehteh> if you think thats ok, then do it
[2008-11-13 00:54:57] <ichthyo> note, the limitation is: you can't just put some isolated code
file *.c *cpp in some of the root directories...
the moment you add a *c or *cpp file in some new sub tree,
you'll find a <subtree-rootdirname>.lum in the corresponding bin dir
[2008-11-13 00:55:17] <cehteh> thats prolly a good choice anyways
[2008-11-13 00:55:28] <cehteh> plugins shall be only at the leaves of the tree
[2008-11-13 00:55:42] <cehteh> be careful when linking
[2008-11-13 00:56:16] <cehteh> the autoconf build included a lot libs from configure which where not needed for plugins
[2008-11-13 00:56:57] <ichthyo> besides, I will set up a different "build environment" for plugins
[2008-11-13 00:57:16] <ichthyo> in SCons, each build environment has a fixed set of libraries attached
[2008-11-13 00:57:28] <cehteh> you likely need specialized ones for some plugins
[2008-11-13 00:57:43] <cehteh> but you can inherit them
[2008-11-13 00:58:02] <ichthyo> yes, I can inherit them
[2008-11-13 00:58:12] <ichthyo> and I match them from the directory name
[2008-11-13 00:58:20] <ichthyo> the rest is fully automatic
[2008-11-13 00:58:14] * cehteh thinks
[2008-11-13 00:58:38] <cehteh> a bare plugin doesnt need any lib
[2008-11-13 00:59:04] <ichthyo> not even the lumiera support lib?
[2008-11-13 00:59:11] <ichthyo> liblumiera.a
[2008-11-13 00:59:12] <ichthyo> ?
[2008-11-13 00:59:29] <cehteh> nope
[2008-11-13 00:59:43] <cehteh> only if it needs somthing from that
[2008-11-13 01:00:01] <cehteh> -DLUMIERA_PLUGIN for compiling plugins
[2008-11-13 01:00:11] <ichthyo> plus libdl
[2008-11-13 01:00:24] <cehteh> stop. no
[2008-11-13 01:00:34] <cehteh> libdl actually breaks the isolation :P
[2008-11-13 01:00:41] <ichthyo> ah, only for the part which loads the lib
[2008-11-13 01:00:51] <cehteh> the host needs libdl not the plugin
[2008-11-13 01:00:54] <ichthyo> but what is when a plugin wants to open another interface
[2008-11-13 01:01:11] <cehteh> it asks the host to do it
[2008-11-13 01:01:17] <ichthyo> doesn't it need to link against your plugin system impl?
[2008-11-13 01:01:32] <cehteh> nope .. just the interface.h
[2008-11-13 01:02:49] <cehteh> interface.h defines some functions .. but these are not available for the plugin either
but the nature of plugins is to extend the system ..
thus follows that they will need some more specific libs in many cases
[2008-11-13 01:02:29] <ichthyo> please understand my goal: I want to make the build as automatic and rules based as possible
[2008-11-13 01:02:58] <cehteh> yes ...
[2008-11-13 01:03:09] <ichthyo> for SCons, this is already mostly the case currently,
but the autotools build is a maintainance nightmare right now
because the paths and dependencies are so much hard wired there
[2008-11-13 01:03:24] <ichthyo> i spent already a considerable time to fix it again and again
[2008-11-13 01:03:52] <cehteh> huh i feel comfortable with autotools
[2008-11-13 01:04:04] <ichthyo> fine for you, but I fix it all the time
[2008-11-13 01:06:44] <ichthyo> for now I didn't use any special switches
[2008-11-13 01:06:59] <ichthyo> I just defined exampleplugin to be an dynamic module
[2008-11-13 01:07:10] <cehteh> sounds reasonable
[2008-11-13 01:07:41] <ichthyo> as said, in future I'll tell scons
"build everything below this root dir into a dynamic module"
and scons will derive the necessary compile and link commands
[2008-11-13 01:10:32] <ichthyo> ah, another little issue
[2008-11-13 01:10:42] <ichthyo> we have two tools directories currently
[2008-11-13 01:10:53] <ichthyo> one as "src/tool"
[2008-11-13 01:10:57] <ichthyo> and another as
[2008-11-13 01:11:12] <ichthyo> just "admin"
[2008-11-13 01:11:29] <ichthyo> within admin, there is the icon rendering and the vgsuppression
[2008-11-13 01:12:01] <ichthyo> the idea was, that all this special support tools would be in src/tool
[2008-11-13 01:12:52] <cehteh> admin are administrative scripts .. tools shall be moved into src/tool
[2008-11-13 01:13:00] <ichthyo> ok
[2008-11-13 01:13:07] <ichthyo> so joelholdsworth:
[2008-11-13 01:13:29] <ichthyo> would it be ok for you to move the icon building executable into src/tool ?
[2008-11-13 01:13:41] <ichthyo> and probably vgsuppression too?
[2008-11-13 01:14:00] <ichthyo> so the build process doesn't need to build anything in admin
[2008-11-13 01:15:29] <joelholdsworth> yes that's ok!
[2008-11-13 01:15:39] <joelholdsworth> that really is just a tool
[2008-11-13 01:15:43] <ichthyo> that is built lilke a tool, so I'd like to move it into src/tool
[2008-11-13 01:16:15] <joelholdsworth> it's the standard rsvg utility with some options added
[2008-11-13 01:16:58] <ichthyo> a propos: joelholdsworth:
I've built some additional directory checks into the accompanying python script
you have probably noticed that I call directly from the SConstruct into your
python script to invoke the icon rendering, and I built some additional checks
and move non-empty dirs away as "*.bak"
[2008-11-13 01:17:50] <joelholdsworth> I hadn't actually looked
[2008-11-13 01:17:59] <joelholdsworth> but yes that makes perfect snese
[2008-11-13 01:18:31] <joelholdsworth> actually it is supposed to overwrite any prior renderings
[2008-11-13 01:18:39] <ichthyo> ah?
[2008-11-13 01:19:00] <joelholdsworth> so if you change the SVG the PNGs get created fresh
[2008-11-13 01:19:01] <ichthyo> so I could change it and just do a recursive remove if the target dir exists?
[2008-11-13 01:19:10] <joelholdsworth> yes
[2008-11-13 01:19:32] <joelholdsworth> like obj files - it's supposed to just overwrite any previous bits
[2008-11-13 01:19:50] <joelholdsworth> well the targets are just build temporaries
[2008-11-13 01:20:01] <ichthyo> ok
[2008-11-13 01:22:25] <ichthyo> ok. then regariding the build system, the only bit of cleanup is the tests
[2008-11-13 01:22:46] <ichthyo> cehteh: we talked about it when I visited you
[2008-11-13 01:23:02] <ichthyo> it's not urgent right now
[2008-11-13 01:23:14] <ichthyo> but.. as we agreed at that time
[2008-11-13 01:23:39] <ichthyo> we should make a separate tree for the test-support code, maybe in a subdirectory of /tests
[2008-11-13 01:24:01] <ichthyo> and this tree of course reflects the main tree. That was the idea if I recall correct
[2008-11-13 01:25:11] <cehteh> ichthyo: yes .. i know
[2008-11-13 01:25:13] <cehteh> pending
[2008-11-13 01:25:18] <ichthyo> yes, not urgent
[2008-11-13 01:25:26] <ichthyo> works as-is for now
[2008-11-13 01:25:45] <ichthyo> I have some similar cleanup for my part
[2008-11-13 01:27:24] <ichthyo> btw: do we need "test plugins"?
[2008-11-13 01:27:52] <ichthyo> i.e. something similar to the src/plugin subdirs we discussed just a moment ago
[2008-11-13 01:28:53] <cehteh> you mean some which get installed?
[2008-11-13 01:29:09] <ichthyo> no, just plugins which are needed only from within tests
[2008-11-13 01:29:18] <cehteh> yes we need them
[2008-11-13 01:29:19] <ichthyo> i.e. /tests/plugin
[2008-11-13 01:29:39] <ichthyo> and everything below will be built exactly similar as everything below /src/plugin
[2008-11-13 01:29:44] <cehteh> i currently build them in place example_plugin.c
[2008-11-13 01:29:54] <ichthyo> just it will be only used for test runs
[2008-11-13 01:29:56] <ichthyo> yes, ok
[2008-11-13 01:32:18] <cehteh> another thing: shall plugins override existing interfaces or barf out?
[2008-11-13 01:32:33] <cehteh> both has pros and cons
[2008-11-13 01:32:46] <cehteh> both can easily circumvented with some effort
[2008-11-13 01:33:27] <ichthyo> ehm, I don't fully understand the situation.
Could you explain it? Lets say: we have interface A
now a plugin wants to provide an implementation of interface A
i.e. the client of this plugin would use interface A to access it
[2008-11-13 01:34:03] <cehteh> in a mail to you i explained that i would like mockup plugins which
override core functionality
[2008-11-13 01:35:07] <ichthyo> so, wouldn't the mockup just be another implementation of interface A ?
[2008-11-13 01:35:09] <cehteh> well you query an interface+implementation tuple
[2008-11-13 01:35:35] <cehteh> you cant just open a interface .. you open a implementation of an interface
[2008-11-13 01:35:49] <ichthyo> yes, but you use this (interface+implemntation) just for opening the interface, or?
[2008-11-13 01:36:05] <cehteh> so mocplugin can provide blurfoo
[2008-11-13 01:36:15] <cehteh> even if blurfoo.lum provides it too
[2008-11-13 01:40:32] <cehteh> imagine 2 *.lum modules implement and export exactly the same thing
(or at least announce it as exactly the same thing)
[2008-11-13 01:40:57] <ichthyo> I'd just try to relate it to the similar situation with classes
[2008-11-13 01:41:16] <ichthyo> you have an interface and now you have two subclasses
[2008-11-13 01:41:12] <cehteh> nah classes are different
[2008-11-13 01:41:27] <ichthyo> no they aren't. really different
[2008-11-13 01:41:43] <cehteh> subtle different
[2008-11-13 01:41:47] <ichthyo> so both subclasses claim to implement the contract which is defined by the interface
[2008-11-13 01:42:04] <ichthyo> now I have a factory which instantiates one of those subclasses
[2008-11-13 01:42:14] <ichthyo> or the other or both
[2008-11-13 01:42:18] <ichthyo> same situation as for the plugins
[2008-11-13 01:42:44] <cehteh> err no
[2008-11-13 01:42:48] <cehteh> you have only one class
[2008-11-13 01:42:49] <ichthyo> the client just wants an "instance" of the contract defined by the interface
[2008-11-13 01:43:01] <ichthyo> and doesn't care which one it is
[2008-11-13 01:43:14] <cehteh> you didnt understand my problem because it doesnt exist in C++
[2008-11-13 01:43:33] <cehteh> class foo {...};
[2008-11-13 01:43:43] <cehteh> foo this_is_it;
[2008-11-13 01:43:44] <cehteh> foo this_is_it;
[2008-11-13 01:43:57] <cehteh> ... redefinition of the same instance is my problem
[2008-11-13 01:44:08] <cehteh> not even exactly
[2008-11-13 01:44:37] <ichthyo> ah, in C++ the two would either be rejected by the compiler,
or they would be allowed if in different scopes
[2008-11-13 01:44:43] <cehteh> even worse the 2nd can say the it is a identical instance but it isnt
[2008-11-13 01:45:07] <cehteh> thats a good way to inject moc objects
[2008-11-13 01:45:15] <cehteh> so now my solution:
[2008-11-13 01:46:01] <cehteh> *thinking*
[2008-11-13 01:46:29] <cehteh> minor versions must differ, biggier minor will win
[2008-11-13 01:46:41] <cehteh> (newer one for certain)
[2008-11-13 01:46:53] <cehteh> thats intended
[2008-11-13 01:47:11] <cehteh> now a mock object might claim it is some *exact* same ..
[2008-11-13 01:47:57] <cehteh> that will be rejected .. so a mock object must do some heavy work
to throw out the old one from the nest
[2008-11-13 01:48:10] <cehteh> thats ok for testing purposes
[2008-11-13 01:48:12] <ichthyo> well... at some point you will be able to tell the loader:
load this module for use as interface A
[2008-11-13 01:48:20] <cehteh> nah
[2008-11-13 01:48:37] <cehteh> but you forget that modules != interfaces
[2008-11-13 01:49:07] <ichthyo> I understand the modules as being like a "subclass" of the interface
[2008-11-13 01:49:23] <ichthyo> meaning, each of them can "stand-in" for the interface
[2008-11-13 01:49:18] <cehteh> nope wrong
[2008-11-13 01:49:41] <cehteh> modules are a collection of interfaces
[2008-11-13 01:49:47] <cehteh> completely unrelated
[2008-11-13 01:49:54] <ichthyo> huh?
[2008-11-13 01:50:19] <cehteh> a module could provide a video effect and a gui plugin and a fancy format exporter at once
[2008-11-13 01:50:36] <cehteh> no one shall do that ... but modules are just collections of interfaces
[2008-11-13 01:50:37] <ichthyo> which would be 3 interfaces
[2008-11-13 01:50:41] <cehteh> yes
[2008-11-13 01:51:00] <ichthyo> but what then is the term for the implementation of an interface?
[2008-11-13 01:51:08] <ichthyo> because, at an extension point
[2008-11-13 01:51:21] <ichthyo> you just want to tell: this is the interface to be used at this extension point
[2008-11-13 01:51:28] <cehteh> but the module is just a bunch of interfaces .. and module itself doesnt give them a purpose
[2008-11-13 01:52:09] <ichthyo> thats clear. Something like the feature bundle could give it a speficic purpose,
or something similar, which itself is loaded as plugin
[2008-11-13 01:52:46] <cehteh> modules are just the bucket .. it is pretty unspecified what you put in
[2008-11-13 01:53:01] <ichthyo> yes thats fine, to start with
[2008-11-13 01:53:23] <ichthyo> anything more specific can be built on top of such a system
[2008-11-13 01:53:35] <ichthyo> for example take the video effect
at some point, you define an interface "VideoEffect"
probably this interface isn't loaded from a module, rather it's defined from the core,
but this doesn't matter for the point in question here
now, everything you can ever use as an implementation of this "VideoEffect"
must use this interface, because the client of the interface
(which in this case is the core which uses this interface for rendering)
doesn't care about the implementation and doesn't know any differences
----------------------------

72
doc/devel/meeting_summary/2008-12-10.txt Normal file
View file

@ -0,0 +1,72 @@
2008-12-10 Lumiera Developers Meeting
=====================================
:Author: Teld
:Date: 2009-01-12
Dec 10, 2008 on #lumiera 20:15 - 00:45 UTC +
__Participants__
* cehteh
* ichthyo
* joelholdsworth
* Teld
* tchy
* Divinespectrum
* raffa
* kirk
* Victorsigma
* pedro
* thorwil
The Lumiera Logo
----------------
The logo contest is a great success: http://www.pipapo.org/pipawiki/Lumiera/Logos[105 entries]. Everyone agrees that a preselection has to be made. Every attendee will have a look at the entries and selects 5 for the preselection list. Finally, 21 logos are selected. Together with a nice collage they are http://www.pipapo.org/pipawiki/Lumiera/Logos/Selection[presented] so that anyone online can vote for his/her favorite one. Voting will be possible from 20 december until 31 december at midnight.
Recurring Topics
----------------
Discussion of open http://www.pipapo.org/pipawiki/Lumiera/DesignProcess[design process] drafts.
EDL vs Sequence
~~~~~~~~~~~~~~~
There has been a long discussion on the ML about the term EDL. In the end almost agreed or was at least ok with the Sequence instead of EDL. The Sequence will be the building block and is a collection of media objects and a tree of tracks. A discussion about the UML showed in [wiki:self:../DesignProcess/TimelineSequenceOutput Project, Timeline(s), Sequence(s) and Output] and the relation of Timeline and Sequence follows.
__Conclusion__: the name Sequence will be used instead of EDL
Interfaces between layers
~~~~~~~~~~~~~~~~~~~~~~~~~
Ichtyo explains that the idea is that between layers only the C Language interfaces of the interface system are used. Wrappers have been made so that a convenient C++ interface is available.
Threads
~~~~~~~
The UI will mostly run on a single thread, i.e. gtk_main will run in this GUI thread and when it terminates, the GUI is "done" and signals back that it is done. Cehteh points out that really hard working threads should be managed by the scheduler in the backend. All agree that we should avoid thread cancellation
General Topics
--------------
Video format
~~~~~~~~~~~~
Kirk asks to what video format footage has to be converted so that it can be used in Lumiera. Ichtyo and cehteh point out that Lumiera will not be nailed down to one single format. A small number of formats that permit precise editing will be supported. Other formats may be available through plugins. The working horse is gmerlin, especially libgavl. Essentially Lumiera will try to figure out a conversion path in a lossless way.
Knowledge base
~~~~~~~~~~~~~~
Kirk has already collected lots of information on video formats, conversion from one into the other, formats supported out of the box by Quicktime, Windows Media Player and so on. The developers agree that this is much needed. Kirk proposes to help in this area.
Next meeting
------------
The next meeting will be at Wednesday 14 jan 21:00 UTC

122
doc/devel/meeting_summary/2009-01-14.txt Normal file
View file

@ -0,0 +1,122 @@
2009-01-14 Lumiera Developers Meeting
=====================================
:Author: Raffaella_Traniello_and_Goibhniu
:Date: 2009-02-25
Jan 14, 2009 on #lumiera starting 20:00 UTC
__Participants__
* cehteh
* ichthyo
* joelholdsworth
* raffa
* goibhniu
* Teld
* nikola_
* thorwil
* SimAV
* raevol
The Lumiera Logo - Workflow and GUI
-----------------------------------
Everyone is pretty happy with the results of the logo contest. The logo still needs some refinements, especially:
* the font
* angle of view
* single logo or multiple logos based on a theme (dynamic logo)
Logo work discussion has started on Mailing list but it is dying out.
Workflow and GUI discussion has already started on Mailing list. Keeping it effective and useful demands core devs time and energy.
Conclusion
~~~~~~~~~~
Two groups are created, each of these have someone responsible to bring them forward.
The group nudger is some director/moderator who keeps it going and nudges the people constantly, leading the discussion, not making the decisions alone. This 'supervisor' doesn't need to be someone to do the work of that group, but someone who can spend some efforts and time to coordinate it.
Work discussion will proceed on the Mailing List but following the directions stated during the IRC meetings. +
Messages will have [ARTWORK] or [LOGO] or [WORKFLOW] in the subject. [[BR]] Groups can schedule IRC meetings separated from the monthly meetings that are meant for general organization.
Group 1 [WORKFLOW]
^^^^^^^^^^^^^^^^^^
. director/supervisor: nikola_ (Nikola Duper)
Unlike other software where the GUI dictates the capabilities of the backend, Lumiera is being built from the bottom up to the GUI. So the foundations are built first to make the things accessible independently to the layout. +
The GUI layout is determined by the people who code it. This workflow group will be counsellor for the coding devs.[[BR]] The core dev responsible for the GUI is joelholdsworth (Joel Holdsworth).[[BR]] One of the group goals is to make the dev point of view dialogate with the editor point of view.
The group studies how a professional editor uses a NLE, defines the key-features and the main qualities (rock solid, simple, support for the main video formats, usable,...), define the static mainstream functionality as opposed to optional experimental ideas and funny widgets, discuss implementability in Lumiera with the devs. +
It discuss about UI design, to ease the pro workflow, to improve usability for amateurs and discoverability for newcomers. [[BR]] It thinks about ergonomics.
Group 2 [ARTWORK]
^^^^^^^^^^^^^^^^^
. director/supervisor: raffa (Raffaella Traniello)
The group discuss about logo, themes, the look and feel...
Getting the 'non-coders' community to help
------------------------------------------
goibhniu proposes an issue tracker.
The documentation needs love: it's currently horribly scattered all around. +
All the content needs to be converted to asciidoc and to the git repo at lumiera.org.[[BR]] Waiting for uWiki, the site will be in asciidoc+git [[BR]] The site needs a structure or other means to increase discoverability of all our information[[BR]] goibhniu proposes organizing a "sprint" to help get people involved and set a date for it (documentation migration/organisation)
Conclusion
~~~~~~~~~~
* goibhniu volunteers to set up Trac.
* raevol volunteers to maintain it.
* The link:AsciiDoc[] Day idea will be discussed at the next meeting.
Possible GUI-Integration: Display a frame?
------------------------------------------
There was some discussion about the need for a component which does the timing and communicates with the scheduler and where it should sit in the stack.
For example: a render controler would be `for (frame i=0; i < end; ++i){pull_frame_without_timing_constraints(i);}` and a player would be the same with timing constraints and possibly lower quality. The GUI just has to remember that a play button is pushed, and that a certain instance of the play control is associated with a certain playback task in the backend. It was decided that this doesn't need to be resolved at the moment and requires further discussion and planning. It was felt that device dependnet things shouldn't be in the backend since it may one day run remotely as a frame server. It may need to sit between the gui and the backend. It was noted that the backend will run on constraints rather than ticks. For now the focus is on getting a single frame into the gui and there will be time to flesh out final interfaces later.
The devs agreed to meet up on Saturday to get the latest version up and running.
http://www.lumiera.org/gitweb?p=lumiera/ct;a=blob;f=src/common/guifacade.cpp;h=7f781845ad79b4a3e2cd820b96b87f5e702a80d3;hb=8e89506d289f3986af133493a40e3705b7919c87#l64[]
(The thread shall be forked from within the GUI code not from the facade)
This was discussed and ichthyo has already pushed a new cleaned up state and resolved most of the dependency problems with the liblumiera. It raised the question: do we prefer a UI facade to a GUI facade or do we want an extra script facade someday?
It was discussed whether joelholdsworth should wait for cehteh to make a playback interface or use a dummy one for now. Cehteh wasn't ready to commit to a final interface yet but felt it was important to acknowledge the type of bidirectional interface that would be required. Icythyo has been using the link:GuiNotification[] as a dummy to implement an interface and get things working with loading the GUI plugin. Joelholdsworth already has displayer code which uses XVideo and falls back to GDK. It tells the buffer the address instead of being integrated with gavl, so cehteh will be able to hand it the address when ready. At the moment the buffers contain RGB (not gavl frames or anything). For the simple test joelholdsworth will need a buffer with RGB data and some metadata which cehteh and joelholdsworth planned to flesh out in the near future.
# Errors, Assertions, Checks, Dev- and Release-Builds
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
A new Nobug is nearly ready with new documentation (cehteh). The error handler has been improved, as announced on the ML. Cehteh emphasised the importance of using nobug everywhere and not `#if DEBUG`:
DEBUG is not a standard C/C++/buildsystem defined macro when it works its just coincidence but NDEBUG is standard. Neither does such code use nobug facilities nor respect nobug build levels.
in new nobug you can do:
NOBUG_IF_ALPHA(
alpha code here
)
it's the same for BETA RELEASE and there's a NOT variant for each but even then that should be avoided and higher level nobug facilities should be used instead.
(http://www.lumiera.org/nobug.html)[]
beta and release builds shall be possible with nobug soon .. waiting for metta to integrate the new test.sh
test.sh with regex is awesome because you can test output which contains no exact things like addresses, version numbers, times
and running the testsuite under valgrind works properly now
Workflow/Interface
~~~~~~~~~~~~~~~~~~
Ichthyo expressed the importance of not having dedicated kinds of tracks, but to be able to use any track for anything (audio, video etc.). This will require more work in the proc layer. Tracks are also not just layered but are trees which can fold. Any clip can contain N streams of various kinds. When a clip contains video and audio streams it still occupies just one track and the system automatically connects the audio to an audio bus etc. but the user is free to split off the audio as an independent clip and treat it separately from the video. The plan is to have various kinds of links between clips. This basic idea will need further refinement and consideration by the workflow team (Nikola et. al.) e.g. the user will need some way to direct the sound to a different bus even though it may be in a compound clip. By default all the video should be layered in the order of the tracks and a user should be able to say "don't mix the audio from track 4 and its child tracks directly to master but route it to the music subgroup". This is where the tree of tracks plays an important role because each track with its children forms a group.
There was further discussion on the general aims, objectives and focus of Lumiera and the reasoning behind them. Integration with control surfaces, jog wheels, sliders etc. should also be considered for the workflow. Nikola expressed the importance of stability and a simple and clean UI.
Next meeting
------------
Usually meetings are the second Wednesday each month on #lumiera at Freenode. The next meeting will be on Feb 11 - 20:00 UTC
Core dev's IRC meetings
~~~~~~~~~~~~~~~~~~~~~~~
cehteh proposes a recurrent core dev meeting, more frequent than the monthly meeting since there is the a lot of stuff ahead that needs communication.

118
doc/devel/meeting_summary/2009-09-09.txt Normal file
View file

@ -0,0 +1,118 @@
2009-09-09 Lumiera Developers Meeting
=====================================
Raffaella_Traniello_and_Hermann_Voßeler <prg@ichthyostega.de>
Sep 9, 2009 on +#lumiera+ +
^20:00 - 22:30 UTC^
.Participants
- cehteh
- ichthyo
- joelholdsworth
- andrewjames
- daniieel
- hermanr
- liveD
- raffa
- SimAV
Point 1: FrOSCon
----------------
Our participation to the conference was a success. We met other video
developers and people interested in Lumiera.
It has been a meeting opportunity for the community, it produced good and
reusable propaganda material (about 100 flyers distributed).
This success wouldn't be possible without a lot of people helping. +
So lets say: *Thank you all*!
We plan to be there again next year with another Lumiera meeting (possibly
all the 3 core devs present). _Ichthyo_ consideres to do a dedicate talk
(in German).
Participation as a group to other possible events has been considered for +
(in order of interest):
LAC, Piksel, OVC, Piksel Summer camp, Linuxtag, FSCONS.
To improve visibility _liveD_ proposes to extend the project also on the
new social network like facebook. +
The proposal is discussed but doesn't find supporters.
Point 2: Funding
----------------
Preparing for FrOSCon had some costs, paid by the people working in
Lumiera. Ideally, _cehteh_ says, people working on Lumiera should not have
to pay for working on it, no matter who pays.
He proposes to make a project partnership with FFIS (a German non-profit
organisation for founding free software).
That means we could officially collect donations through FFIS (people
donate to FFIS, to free software in general or specifically to Lumiera).
That could work out well for specific scopes, like covering travel,
lodging and diet for developer gatherings or for exhibition material and
server.
The proposal is accepted. _Cehteh_ will contact FFIS to propose them a partnership.
If later on there will be funding for development (not just covering expenses)
that would require a different approach and special communication with the
community.
_daniieel_ proposes to put a price to each request in "buglist". +
The proposal is discussed and dismissed.
Point 3: Website
----------------
The news in the homepage makes the project look alive. But this is not
enough to communicate the state of the Lumiera development. Two
solutions are proposed:
* The website needs to be reorganized and made accessible. andrewjames
volunteers to work on it.
* Quarterly development news will be published in the devel section of the
website. The devs will post drafts to the mailing list and encourage the
other devs to add their informations. Then someone shall aggregate this
and make a news webpage. That creates a new task: help collecting the
news and write them together.
The devs will communicate their achievements more often in small steps
to the mailing list
_raffa_ proposes monthly news on the homepage with more broad topics
(development, community, openvideo). +
The proposal gets no interest.
Point 4: Ticket Query clean-up
------------------------------
Trac allows for creation of custom ticket queries. We've accumulated quite some
custom reports, and _cehteh_ asks if some of them can be dropped (Ticket #225).
_Ichthyo_ points out that he uses some reports frequently, and considers some
others nice-to-have. Finally it turns out that only Report #5 isn't used by
anyone and can be dropped.
_Cehteh_ points out that the http://issues.lumiera.org/tags[TracTags] macro
is a convenient feature -- for example it allows to tag tickets by required skill.
Point 5: Coding
---------------
_Cehteh_ finished a first stable NoBug release and intends to give uWiki a boost
in the next weeks. _Ichthyo_ worked on a command frontend usable at the GUI/Proc
interface; in the next time, there will be some integration work to be done
together with _Joelholdsworth_ to get the GUI and high-level model connected.
A short discussion about the relevance of uWiki for Lumiera follows. Indeed,
it takes away some developer power, and _Joelholdsworth_ is concerned about that,
as the project is a bit underpowered generally. _Cehteh_ on the contrary considers
building a suitable (distributed) infrastructure as an important part of the
project.

208
doc/devel/meeting_summary/2011-03-09.txt Normal file
View file

@ -0,0 +1,208 @@
Lumiera Developers Meeting
==========================
:Author: Stefan Kangas
:Date: 2011-03-09
March 9, 2011 on +#lumiera+ +
^20:00 GMT - 23:30 GMT^
//Menu: label 2011-03-09
__Participants__
* Christian Thaeter (cehteh)
* Francesco Siddi (fsiddi)
* Hermann Voßeler (ichthyo)
* Odin Hørthe Omdal (Velmont)
* Raffaella Traniello (raffa)
* Stefan Kangas (skangas)
The New Website
---------------
The new website is finally online.
How do we proceed with the graphical layout?
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
21:37 <fsiddi> the template code is not ready yet
21:37 <fsiddi> and there are some incorrect uses of tags in the current one
21:37 <fsiddi> so i will go on coding a static html layout
21:38 <fsiddi> then will submit it to your critique and then we can make a template out of it
...
21:50 <ichthyo> fsiddi: I liked the way you provided a slightly larger content area
for the tutorial part
image:{img}/meetings/2011-03-09-mockup_dev_1.jpg["Website Proposal Mockup number 3", width=128, link="{img}/meetings/2011-03-09-mockup_dev_1.jpg"]
image:{img}/meetings/2011-03-09-mockup_dev_2.jpg["Website Proposal Mockup number 2", width=128, link="{img}/meetings/2011-03-09-mockup_dev_2.jpg"]
Discussion about the IFrame
~~~~~~~~~~~~~~~~~~~~~~~~~~~
The only serious alternative to IFrames seems to be SSI, since we want to keep
generating pages with asciidoc, and not have to regenerate the entire website
just because one page changed.
Conclusion was that we will stick with IFrames for now and change later when/if
that becomes a problem.
Old stuff in the repository
~~~~~~~~~~~~~~~~~~~~~~~~~~~
There is a folder /attic on the website.
This information is supposed to be merged with the rest of the website, and then
deleted from its current location. This work is on-going and considered a
"background task".
IRC Logs
--------
A discussion was brought up about how to handle IRC logs.
For privacy reasons we do not want to save them routinely in public
locations. If we put them on the web they will most probably be kept by Google
until the sun burns out.
*robots.txt* might be respected by Google or not. Surely some other search
engines will not respect it. Private logs or other logs not publically available
are fine though.
22:40 <cehteh> well ... IRC should *not* be for persistent documentation .. we shall
force/educate ourself to document things (and decisions) properly, irc
should be volatile
...
22:41 <ichthyo> in the past, sometimes we had very "contentful" dicussions
22:41 <ichthyo> in that cases we just saved a transcript
...
22:44 <ichthyo> so personally I prefer to make transcripts of the *really* contentful
discussions and put them in the public documentation
22:44 <ichthyo> "transcript" means cleaned up irc log
22:44 <cehteh> yes
22:45 <cehteh> so for meetings we should prolly do that
22:47 <cehteh> not further edited but just leave all join/parts/noise and silly
comments out
22:47 <cehteh> (and also rants and stuff)
22:48 <cehteh> only conclusions and most important arguments shall stay
22:48 <cehteh> maybe reorder it a bit as it happens that we talk about 2 things at the
same time
22:48 <cehteh> but also not too much work on it
22:49 <cehteh> and more importantly: document decisions and proposals in a more
offical way: that is 'rfc'
22:49 <skangas_> Agreed. It is more important to preserve information in an accessible
way for the future than focusing on mundane details like
join/part/order/ranting etc.
22:50 <skangas_> This means I personally always prefer shorter summaries of a
discussion to IRC logs, given that they do not leave anything
important out.
VoIP discussions over Mumble?
-----------------------------
There seems to be link:http://kitenet.net/~joey/blog/entry/12_hours_of_talking/[ongoing discussions]. on using VoIP more in Debian GNU/Linux.
It was decided that while the idea is nice, it will not be used for developer
meetings. We might use it for developer discussions outside meetings if we feel
the need though. This will probably be more common as the number of developers
increase.
Also, the point was brought up that GNU Emacs has really nice support for
several people sharing the same session. Everyone gets their own point, and with
some hacks it is even possible to color them differently. This needs to be done
on an untrusted remote server though -- Emacs can do anything and it is thus
highly insecure.
Version numbering
-----------------
Hermann has written an rfc for version numbers.
23:19 <cehteh> so in my words: we will have a usuall major.minor.patch for releases
23:20 <cehteh> and major+1~develtag for devel snapshots
23:20 <cehteh> where develtag is a timestamp
23:20 <ichthyo> and we *could* make a stream of development versions
23:21 <cehteh> YYYYMMDD should suffice
23:21 <cehteh> or maybe just monotonic increading from 1
23:21 <ichthyo> well... optional suffix+ number
23:21 <ichthyo> for the rare cases that you roll two releases a day
...
23:20 <ichthyo> and the third idea of my proposal: *not* use major.minor.patch before 1.0
23:24 <ichthyo> i.e. in the whole alpha... beta range
23:24 <ichthyo> we just do 0.01 0.02 ... 0.99
23:25 <skangas_> Does it go without saying that minor versions can go on above 1.9.0
to 1.10.0 etc?
23:25 <ichthyo> yes, I didn't mention that, but thats important
23:26 <ichthyo> its not nice, and usually you get a major first, but it can happen
23:27 <ichthyo> so i'd say, we have discussed it now, we could comment on it and then
revisit it next time (and maybe decide on the proposal then)
23:27 <skangas_> Do we want a stable version and a development version?
23:28 <skangas_> e.g. 1.0 is stable and 1.1 is unstable
23:28 <skangas_> I am thinking about the scheme that wine and linux is using.
23:28 <cehteh> i think linux won a lot with abadon this odd/even practice
23:28 <ichthyo> regarding stable versions
23:28 <ichthyo> personally, I don't like that idea
23:29 <skangas_> cehteh: I did not know they had abandoned it.
23:29 <cehteh> skangas: linux dont use that anymore
23:29 <cehteh> since 2.6 years ago
23:29 <ichthyo> because they tend to rot
23:29 <skangas_> ichthyo: The stable versions?
23:29 <cehteh> you just make a release and done .. everything after that is for the
next version
23:29 <ichthyo> yes
23:29 <cehteh> the only thing to discuss is how you count there
23:30 <ichthyo> also, it is more in line with the "extreme programming" style
23:30 <ichthyo> i.e. don't do too disruptive changes
23:30 <cehteh> if you want 'pre' or 'rc'
23:30 <ichthyo> but rather refactor often
23:30 <skangas_> OK.
23:31 <ichthyo> there is onre concern though
23:31 <ichthyo> there *needs* to be some stability then
23:31 <cehteh> releases get only bugfixes and *maybe* you can declare some release as
LTS ... but for lumiera i would think really hard about that and likely
deny that
23:31 <ichthyo> because people are working with the software
23:31 <ichthyo> but we want to have that with our compatibility scheme
23:31 <cehteh> (our interface system should provide backward compatibility)
23:31 <ichthyo> yes
23:31 <ichthyo> but anyway, it *might* turn out that we fail with that promise
23:32 <ichthyo> and then we'll have to reconsider a "stable" line
23:32 <cehteh> so maintaining some LTS while having another development branch will
suck developer resources
23:32 <ichthyo> yes
23:32 <ichthyo> so I'd really try hard to avoid that
23:33 <cehteh> well a stable line yes ... that is 'stable' until the next stable
release is there .. and then 'maintained' as in adding important
bugfixes for some (but not excessive) time
23:33 <cehteh> we need to care for a stable for sure
23:33 <ichthyo> indeed, but it shouldnt digress too much from the active development
23:33 <cehteh> but we dont want some really old release to get permanent updates and
care because we declared it as LTS
23:33 <skangas_> ichthyo: Is your RFC open to editing/additions?
23:34 <ichthyo> yes
23:34 <ichthyo> please go ahead, just add comments etc.
23:34 <cehteh> use the rfc.sh comment function :)
23:34 <skangas_> I was just thiking about writing this down in some way, that we
specifically do not want this.
23:23 <cehteh> also do we want 'rc' releases?
23:23 <cehteh> or do we just declare certain devel snapshots as rc (i like the later)
23:23 <ichthyo> yes, thats what I'd prefer too
23:24 <cehteh> devel snapshots should be automatic buildable by builddrone
23:25 <cehteh> I would like to make builddrone an automatic staging system
23:25 <cehteh> add some 'rules' there ..
23:26 <cehteh> if build and testsuite succeeds and test-install works and latests
commit doesnt contain WIP: then stage a devel snapshot
23:27 <cehteh> (eventually or maybe we might even put tests/bugs there which prevent
certain staging actions, finally all integration and complete releases
could be automatized)
23:26 <cehteh> remember in test.sh i reserved the 90-99* area for 'bugs'
23:26 <cehteh> we may put some blocking tests there which prevent staging
Next meeting
------------
The next meeting will be at Wednesday April 13 20:00 UTC.

642
doc/devel/meeting_summary/2011-04-13.txt Normal file
View file

@ -0,0 +1,642 @@
2011-04-13 Lumiera Developers Meeting
=====================================
:Author: Ichthyostega
:Date: 2011-4-14
April 13, 2011 on #lumiera 20:00 - 23:47 UTC +
__Participants__
* cehteh
* ichthyo
* fsiddi
* skangas
_Protocol written by Ichthyo_
.organisational
- IRC meeting summaries will be moved into the main Git tree, below 'doc/devel'
- mostly shortened and tidied IRC logs, plus summary of conclusions \+ decisions
- ``winter quarterly coding news'': this time just paste _Ichthyo's_ contribution into the website
- the next ``coding news'' probably in May?
New Website Page Layout
-----------------------
_Francesco Siddi_ has augmented his Layout proposal and already built up two page templtes to
cover most of the layout needs of the Lumiera website and documentation resources. However,
some points with the code generated by Asciidoc turned out to be problematic.
In a link:http://permalink.gmane.org/gmane.comp.video.lumiera.general/2330[message preceeding the meeting],
today _ichthyo_ highlighted some general concerns which might need further discussion and maybe
a decision.
utilising horizontal space
~~~~~~~~~~~~~~~~~~~~~~~~~~
Resolution of displays has been largely increased, especially on desktop computers, with
a tendency towards ``widescreen'' aspect ratio. While, on the other hand, text content
is still mostly vertically oriented. This leads to the question how to make use
of this additional space in horizontal direction, while still also supporting
the not-so large displays.
.-- Discussion of details --
[caption="☉Transcript☉ "]
----------------------------
[2011-04-13 22:55:35] <ichthyo> so my idea was just to let us discuss how we could use that additional space, when its available
[2011-04-13 22:56:08] <ichthyo> I mean, just lets discuss open ended -- what possibilities do we see for that?
[2011-04-13 22:55:42] <cehteh> if the screen is wide enough they should make 'some' use of it .. maybe just using biggier fonts
[2011-04-13 22:56:46] <ichthyo> using 2 columns would be one possibility, but that is tough and demading to get to work properly
[2011-04-13 22:57:02] <fsiddi> there are basically 2 ways
[2011-04-13 22:57:18] <fsiddi> 1 is to use liquid layout
[2011-04-13 22:57:49] <ichthyo> liquit layout means, that the content area just expands, right?
[2011-04-13 22:57:56] <fsiddi> like wikipedia yes
[2011-04-13 22:58:14] <cehteh> http://slashdot.org/ scales reasonable well with different screen sizes
[2011-04-13 22:58:52] <ichthyo> wikipedia is also an interesting example, because they use lots of floating block content
[2011-04-13 22:58:57] <ichthyo> tables, images, additional infos
[2011-04-13 22:59:20] <fsiddi> yes the mediawiki software is very powerful
[2011-04-13 22:59:38] <fsiddi> another *very* well done documentation is http://developer.apple.com/library/mac/navigation/
[2011-04-13 22:59:52] <fsiddi> (from a design point of view)
[2011-04-13 22:59:54] <fsiddi> :)
[2011-04-13 23:00:01] * cehteh likes (or rather demands) that browser zoom (ctrl-+) works well on the lumiera page
---------------------
using a »liquid layout«?
^^^^^^^^^^^^^^^^^^^^^^^^
.-- liquid ⟺ not liquid --
[caption="☉Transcript☉ "]
----------------------------
[2011-04-13 23:05:14] <fsiddi> ok, so apart from that, the discussion is liquid vs not liquid
[2011-04-13 23:05:31] <ichthyo> and also what possibilities there are
[2011-04-13 23:05:43] <fsiddi> ichthyo: what do you mean?
[2011-04-13 23:05:56] <ichthyo> e.g. just expanding the content area
[2011-04-13 23:06:04] <ichthyo> or also using a script to increase the font
[2011-04-13 23:06:13] <ichthyo> or using floating blocks in the sidebar
[2011-04-13 23:06:38] <ichthyo> (asciidoc has several elements which could float into the sidebar)
[2011-04-13 23:07:09] <ichthyo> and if we know that the site layout supports that, then we'll likely make use of that
when writing documentation
[2011-04-13 23:07:10] <cehteh> when you have a page with a lot (many sideful) of content then a side menu
(also footer/header) must be floating or can be left out
[2011-04-13 23:07:31] <cehteh> there is no point in having a side-menu only visible for 3% of the content
[2011-04-13 23:07:45] <ichthyo> ah yes, thats another point, i.e. how to treat the vertical scrolling
[2011-04-13 23:08:07] <ichthyo> it is possible e.g. to have the scrollbars on the content area
[2011-04-13 23:08:08] <cehteh> imo we need 2 classes of pages .. small ones and huge ones
[2011-04-13 23:08:22] <cehteh> scrollbars inside are inconvinient
[2011-04-13 23:08:18] <fsiddi> http://wiki.blender.org/index.php/Doc:Manual
[2011-04-13 23:08:37] <fsiddi> it has scrolling sidebar
[2011-04-13 23:09:08] <cehteh> but with js again
[2011-04-13 23:09:19] <cehteh> i seen that as CSS only solution i think
[2011-04-13 23:09:37] <cehteh> otherwise yes, thats an option ..
[2011-04-13 23:10:02] <ichthyo> cehteh: yes, you can get that just with CSS
[2011-04-13 23:10:23] <fsiddi> actually you need js to trigger the class to make it floating
[2011-04-13 23:10:27] <cehteh> back to my statement, do you agree that we need this 2 kinds of classes?
[2011-04-13 23:10:36] <fsiddi> i don't think so
[2011-04-13 23:10:42] * ichthyo neither
[2011-04-13 23:10:56] <fsiddi> cehteh, i think it will overcomplicate the backend
[2011-04-13 23:11:03] <ichthyo> because I guess 90% of the documentation pages will get into the "large, much content" category
[2011-04-13 23:11:05] <cehteh> really?
[2011-04-13 23:11:12] <cehteh> yes
[2011-04-13 23:11:19] <ichthyo> so we *do* have two templates already
[2011-04-13 23:11:22] <fsiddi> i think it's better to keep it flexible
[2011-04-13 23:11:26] <cehteh> and does this documentation need a sidebar?
[2011-04-13 23:11:26] <fsiddi> yes
[2011-04-13 23:11:35] <fsiddi> we use 2 templates
[2011-04-13 23:12:16] <ichthyo> one is the "top level", with the horizontal menu
[2011-04-13 23:12:24] <cehteh> this boils down to one for the general pages and one for documentation
[2011-04-13 23:12:25] <ichthyo> and the second one is the documentation template, right
[2011-04-13 23:12:44] <ichthyo> while those "general" pages are a handful
[2011-04-13 23:12:45] <fsiddi> yes
[2011-04-13 23:12:55] <cehteh> yes
[2011-04-13 23:12:56] <ichthyo> while the documentation pages will go into the hundred and more
[2011-04-13 23:14:08] <ichthyo> so I draw the conclusion, that the documentation pages are optimised for much content
[2011-04-13 23:14:19] <fsiddi> yes
[2011-04-13 23:13:30] <cehteh> ok .. settled
...
[2011-04-13 23:13:54] <fsiddi> i propose to keep the layout not liquid also in that pages
[2011-04-13 23:14:35] <fsiddi> if you prefer i can make them larger
[2011-04-13 23:14:49] <fsiddi> but for the moment i'd like to keep the same width
[2011-04-13 23:14:48] <ichthyo> what are the problems you see with a liquid layout?
[2011-04-13 23:15:08] <fsiddi> mostly a readability issue
[2011-04-13 23:15:29] <fsiddi> if a page is too large, it's unreadable
[2011-04-13 23:15:39] <ichthyo> well, I second that statement
[2011-04-13 23:15:53] <ichthyo> if a page goes over a certain amount of characters per line
[2011-04-13 23:16:10] <fsiddi> ichthyo: that's what i mean
[2011-04-13 23:15:53] <cehteh> can you scale the font on huge screens with css?
[2011-04-13 23:16:10] <ichthyo> cehteh: not in css2
[2011-04-13 23:16:15] <cehteh> ok
[2011-04-13 23:16:24] <ichthyo> java script would work
[2011-04-13 23:16:42] <cehteh> i opt for a reasonable big max size as already proposed
[2011-04-13 23:16:47] <ichthyo> well, but if we manage to limit those number of characters per line
[2011-04-13 23:16:55] <fsiddi> i can implement that
[2011-04-13 23:16:58] <cehteh> but dont make this too small
[2011-04-13 23:17:05] <ichthyo> then liquid layout might be more reasonable
[2011-04-13 23:17:33] <fsiddi> ok
[2011-04-13 23:17:36] <ichthyo> I am not 100% sure, but I recall having seen a layout, just based on CSS,
which achieved exactly that,
[2011-04-13 23:18:03] <ichthyo> i.e. limiting the maximum width, but allowing some liquid expansion below that
[2011-04-13 23:17:55] <fsiddi> i'm not sure that just CSS is possible
[2011-04-13 23:18:20] <ichthyo> If I recall right, it used several nested containers
---------------------
Regarding the handling of *vertical scrolling*, the conclusion was to stick to the defaults
as much as possible. Especially, scrollbars should rather be left to the browser, not added
to the content area. We might consider to fix the navigation block relative to the browser window
though, if that is doable with too much complications. Moreover, this navigation block, holding
the (vertical) tree-like menu, should be made sufficiently large in vertical direction, but might
show scrollbars in case of overflow.
Navigation menu
^^^^^^^^^^^^^^^
we still need to find a way how to make the new +JQuery+ based navigation menu open and highlight
the _current_ page automatically. _fsiddi_ and _ichthyo_ will work out a solution in a separate meeting
on IRC next week.
Colour scheme
^^^^^^^^^^^^^
.gray shades or using a colour scheme?
[caption="☉Transcript☉ "]
----------------------------
[2011-04-13 23:28:23] <ichthyo> I'd like to bring up the question regarding color
[2011-04-13 23:28:36] <ichthyo> and I'll ask especially you, fsiddi
[2011-04-13 23:28:44] <ichthyo> you know, colours are a matter of taste
[2011-04-13 23:29:15] <ichthyo> thus I'd say, as you did the general layout, you have an important say in that
[2011-04-13 23:28:59] <fsiddi> ah yes colors
[2011-04-13 23:29:11] <fsiddi> i try to keep it neutral
[2011-04-13 23:29:25] <ichthyo> you you would popose to stick to just gray shades?
[2011-04-13 23:29:34] <ichthyo> what is with the links?
[2011-04-13 23:29:39] <ichthyo> currently they are slate blue
[2011-04-13 23:29:45] <cehteh> thats ok for me
[2011-04-13 23:29:47] <fsiddi> i still have to work in them
[2011-04-13 23:29:55] <fsiddi> yes I did not color the links
[2011-04-13 23:30:13] <ichthyo> generally speaking, we should always try to limit the number of colours in a layout, IMHO
[2011-04-13 23:30:18] <cehteh> i dont think we shall use any fancy colors and maybe/if required then we add colors as markup
[2011-04-13 23:30:29] <ichthyo> but we could think of one or two "leading colours"
[2011-04-13 23:30:35] <fsiddi> i want to
[2011-04-13 23:30:43] <fsiddi> but did not put them in the css yet
[2011-04-13 23:30:53] <fsiddi> i'll do that soon
[2011-04-13 23:31:38] <fsiddi> so this is also for review soon :)
[2011-04-13 23:32:07] <cehteh> but colors are fixable and we dont have an official lumiera color (do we need one?)
[2011-04-13 23:32:22] <ichthyo> well... we could think about that
[2011-04-13 23:32:19] <fsiddi> i'll try to make up one
[2011-04-13 23:32:28] <cehteh> dunno :)
[2011-04-13 23:32:37] <ichthyo> according to my experience
[2011-04-13 23:32:47] <ichthyo> it helps a lot when you set a clear style guide early
[2011-04-13 23:33:41] <ichthyo> ok
---------------------
Webserver upgrade and the »reference distribution«
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.-- Server upgrade discussion --
[caption="☉Transcript☉ "]
----------------------------
[2011-04-14 00:25:59] <cehteh> ok next point:
[2011-04-14 00:26:07] <cehteh> - Webserver update to squeeze (new asciidoc, keep ichthyos hand
[2011-04-14 00:26:07] <cehteh> installed trac)
[2011-04-14 00:26:07] <cehteh> - Do we want to bump our 'reference' distribution to squeeze too?
[2011-04-14 00:26:19] <cehteh> ... webserver .. as soon as possible, but no urge
[2011-04-14 00:26:34] <cehteh> reference .. i just wanted to bring this up, imo there is no need
[2011-04-14 00:26:46] <ichthyo> personally, I will upgrade soon, next 2 weeks hopefully
[2011-04-14 00:27:13] <cehteh> yes i am on squeeze and even with backports already
[2011-04-14 00:27:29] <cehteh> so its prolly even better to have the reference on the devel server a bit behind
[2011-04-14 00:27:28] <ichthyo> I would propose to bump the "reference" the moment when we actually upgrade the
devserver + builddrone
[2011-04-14 00:27:51] <cehteh> well the devserver will be upgraded when we bump the reference
[2011-04-14 00:28:10] <cehteh> builddrone will be upgraded sometime next but thats not related to the reference
[2011-04-14 00:28:12] <ichthyo> but for now there is no problem also supporting lenny, but with the note that we'll
drop that support once we run into serious problems
[2011-04-14 00:28:22] <cehteh> yes
[2011-04-14 00:28:30] <cehteh> iirc that would be the reason to bump it
[2011-04-14 00:29:05] <ichthyo> well, IMHO, when we both are on squeeze, then effectively the reference is bumped :-P
[2011-04-14 00:29:40] <cehteh> nah .. the reference is about what builddrone reports to us too
[2011-04-14 00:29:15] <cehteh> i'd stay with lenny as long as we can so .. or maybe if the next stable gets froozen
then we can go to squeeze
[2011-04-14 00:29:51] <cehteh> and what skangas and other gui coders need also
[2011-04-14 00:30:15] <cehteh> i expect that gavl and gui dependencies will be a cause for a bump
[2011-04-14 00:32:50] <cehteh> summarize: bump it someday .. as need arises?
[2011-04-14 00:33:26] <cehteh> or even better. .. no decision yet .. we'll see when its time
---------------------
Website licensing and legal questions
-------------------------------------
.-- Discussion regarding the Web content license --
[caption="☉Transcript☉ "]
----------------------------
[2011-04-13 23:54:14] <skangas> For the record, I agree with what ichthyo said in his first e-mail.
[2011-04-13 23:54:55] <skangas> That "dual licensing under GPL and something comparable" is the best choice.
[2011-04-13 23:55:03] <skangas> Probably CC-BY-SA.
[2011-04-13 23:55:10] <ichthyo> my thinking too
[2011-04-13 23:56:05] <ichthyo> that is, for the web content, and the documentation
[2011-04-13 23:57:00] <ichthyo> cehteh: would that be ok for you too?
[2011-04-13 23:59:29] <cehteh> [23:58] <cehteh> mhm?
[2011-04-13 23:59:31] <cehteh> .. any answers beyond that?
[2011-04-13 23:59:55] <ichthyo> seems we had a short split or so
[2011-04-14 00:00:07] <ichthyo> skangas and myself just talked about licenses
[2011-04-14 00:00:09] <cehteh> i got completely disconnected
[2011-04-14 00:00:41] <ichthyo> [23:57] <ichthyo> cehteh: would that be ok for you too?
[2011-04-14 00:01:29] <cehteh> CC-BY-SA means?
[2011-04-14 00:01:47] <ichthyo> Creative commons 3.0 attribution share alike
[2011-04-14 00:02:39] <ichthyo> cehteh: CC-BY-SA is considered "equivalent in spirit" with the GPL
[2011-04-14 00:01:50] <cehteh> copyleft .. attribution .. share-alike?
[2011-04-14 00:02:25] <cehteh> this dualcicensing would be ok for me .. but a note
[2011-04-14 00:02:48] <cehteh> how about putting the documentation under a non-commercial license?
[2011-04-14 00:03:03] <ichthyo> that would throw us out of debian main
[2011-04-14 00:03:13] <cehteh> that is no one can use the lumiera docs for release printed docs and make profit from it
[2011-04-14 00:03:14] <ichthyo> debian considers such a license "non-free"
[2011-04-14 00:03:49] <ichthyo> well, I am not so much concerened about that
[2011-04-14 00:04:03] <ichthyo> because, the docs still remain available
[2011-04-14 00:04:16] <ichthyo> and if someone really goes through all the pain of creating printed stuff
[2011-04-14 00:04:31] <ichthyo> and does it well, then he deserves to make profit from that, IMHO
[2011-04-14 00:04:25] <cehteh> but i dislike the idea that someone makes profit with no work and no giving back
[2011-04-14 00:04:48] <ichthyo> I am sure that *does require* substantial amount of work
[2011-04-14 00:05:03] <ichthyo> if that person wants any chance to sell something
[2011-04-14 00:05:23] <ichthyo> just print a crappy web dump won't make anyone sell much copies, IMHO
[2011-04-14 00:04:59] <cehteh> then its ok for me
[2011-04-14 00:05:33] <cehteh> but i've seen publisher which just took the free docs do *little* editing and formating
and sell it for a lot money
[2011-04-14 00:05:53] <ichthyo> and, will he make substantial turnover with that?
[2011-04-14 00:06:15] <skangas> Yeah, I think the point here is that poor quality high prices do not sell.
[2011-04-14 00:06:16] <cehteh> btw GPL only would prevent that too .. the publisher has to release his tex,
m$ -word or whatever sources then
[2011-04-14 00:06:36] <cehteh> does -sa prevent that too?
[2011-04-14 00:06:36] <ichthyo> making modifications available under the same conditions, yes.
[2011-04-14 00:06:46] <cehteh> besides, he needs to point out *within* that printed docs, that the license is free
[2011-04-14 00:07:44] <cehteh> but possibly some people get upset when they contribute a lot and someone else makes the profit
[2011-04-14 00:07:44] <skangas> I do not see how it would damage our goals.
[2011-04-14 00:08:06] <skangas> It would be more in the spirit of not allowing the free-loaders easy access.
[2011-04-14 00:08:20] <cehteh> well as i saied, that was just a note/idea .. i am fine with the dual-licensing
[2011-04-14 00:08:39] <ichthyo> well... here is the standard argument: if someone creates additional value on top of what
we provide, i.e. make a quality print, process orders, ship the copies,
then fine for me, if he/she makes turnover with that
[2011-04-14 00:09:01] <cehteh> yes
[2011-04-14 00:09:08] <skangas> I do not care much for that argument.
[2011-04-14 00:09:16] <skangas> For me, it is more about getting into Debian main.
[2011-04-14 00:09:59] <skangas> And the fact that I do not care if some marginal publisher makes a bit of money off
of the Lumiera documentation.
[2011-04-14 00:09:10] <ichthyo> the online docs will always be more acurate, more up-to-date
[2011-04-14 00:09:13] <skangas> Not even if a big one does it; that would only mean more attention to the project.
[2011-04-14 00:09:32] <cehteh> i would like this if we even can point to him and announce that as the offical lumiera book
[2011-04-14 00:10:10] <cehteh> but there are so much publishers which just try to make money without being really involved
[2011-04-14 00:10:31] <skangas> cehteh, Yes, even publishers which only print Wikipedia articles.
[2011-04-14 00:10:46] <ichthyo> and btw, what is more realistic, given that lumiera becomes a real, professional app,
then maybe someone will provide lectures and training
and that is much more apt to create a good turnover and benefit
[2011-04-14 00:11:14] <cehteh> ichthyo: lectures and training involves work .. thats ok
[2011-04-14 00:11:42] <cehteh> anyeays ... dual license gpl and cc-by-sa
[2011-04-14 00:11:46] <cehteh> ok for me
[2011-04-14 00:11:55] <ichthyo> fine, seems case is settled
[2011-04-14 00:12:04] <cehteh> eh which gpl ? gpl2+ .. same as lumiera
[2011-04-14 00:12:08] <ichthyo> yes
[2011-04-14 00:12:15] <ichthyo> thats important
[2011-04-14 00:12:32] <cehteh> and we may bump lumiera to gplv3 when we release it (and we know all its dependencies)
[2011-04-14 00:12:39] <ichthyo> so you can move code / doc in both directions without problems
[2011-04-14 00:12:42] <skangas> GPLv2+ not GPLv2, right?
[2011-04-14 00:12:48] <cehteh> skangas: yes
[2011-04-14 00:12:51] <skangas> OK. Great.
[2011-04-14 00:13:14] <cehteh> we only use gplv2 now because we dont want to rule out to use external libs which may be v2 only
[2011-04-14 00:13:25] <ichthyo> and a rather firm promise to bump it to gpl3 when this works and we're approaching a release
[2011-04-14 00:13:40] <cehteh> or gplv4 :P
[2011-04-14 00:13:47] <ichthyo> yay!
[2011-04-14 00:13:51] <cehteh> btw duke nukem is delayed
[2011-04-14 00:14:02] <ichthyo> ouch, I'm surprised
---------------------
Conclusion
~~~~~~~~~~
* Website content and documentation will be dual-licensed GPL 2+ and CC-3.0-BY-SA
* _Ichthyo_ will clarify the actual licenses used on the ``license'' page
* moreover he'll care to add an _Impressum_ -- as required by german law
Trac spam
---------
After an increasing amount of Spam tickets in the link:http://issues.lumiera.org[Trac],
_Ichthyo_ installed the Trac antispam plugin (which required an upgrade to Trac 0.12,
actually repackaging a current version into an pre-release debian package, as the
official debian package isn't on the required level). After a bit of training,
the Bayesian filter successfully blocked any further spam tickets.
The remaining problem are spam user accounts though. To deal with that problem, _Ichthyo_
designed a custom SQL query based on some obvious heuristics, which seems to pinpoint
all spurious accounts. We'll try to execute that SQL for some time manually, and --
in case it behaves sane -- automate that cleanup as a cronjob in some months.
_Cehteh_ points out that this new policy should at least be anounced on the Mailinglist.
Recurring Topic: Design process entries
---------------------------------------
Discussion of open link:/documentation/devel/rfc.html[design process] drafts.
Since some time, no further discussion happened regarding the currently _pending_
RfC entries. Agreement is that we should again return to the former routine and
revisit the relevant design process entries in each developer meeting.
.-- the Application Install proposal --
[caption="☉Transcript☉ "]
----------------------------
[2011-04-14 00:48:25] <cehteh> http://lumiera.org/documentation/devel/rfc_pending/ApplicationInstall.html
[2011-04-14 00:48:40] <ichthyo> maybe only pick out some interestin ones or some which are quick to decide
[2011-04-14 00:49:06] <cehteh> well i want to go over all pending .. then we can put notes there "boring for the next meeting"
[2011-04-14 00:49:42] <cehteh> for example this application install .. is boring .. you did a lot work, imo you can finalize it
[2011-04-14 00:50:04] <cehteh> (i dint read it in detail now)
[2011-04-14 00:50:25] <cehteh> maybe we want another state "accepted" ..
[2011-04-14 00:50:44] <cehteh> that is the interesting things which we know we will not drop but which are not finalized yet
[2011-04-14 00:54:29] <cehteh> the application install came first... we need it, you did it well .. it could be 'finalized' or
rather that would be some 'acceepted' candidate ..
[2011-04-14 00:53:25] <ichthyo> I think, the existing states are enough
[2011-04-14 00:53:44] <ichthyo> either really discuss something and then decide, or just leave it in draft
[2011-04-14 00:54:05] <ichthyo> lets just postpone the application install and leave it in draft!
---------------------
Delectus Shot Evaluator
~~~~~~~~~~~~~~~~~~~~~~~
Agreement to _park_ it until someone else comes up to advance this topic further.
Clip Cataloging System
~~~~~~~~~~~~~~~~~~~~~~
similarily to _park_ until someone cares....
DesignParamAutomation
~~~~~~~~~~~~~~~~~~~~~
Keep _pending_ -- _ichthyo_ will expand on that
Lumiera Forward Iterator
~~~~~~~~~~~~~~~~~~~~~~~~
[caption="☉Transcript☉ "]
----------------------------
[2011-04-14 00:58:39] <cehteh> pending
[2011-04-14 00:58:48] <ichthyo> well, this is entirely an C++ topic
[2011-04-14 00:58:58] <ichthyo> I for my part vote for accepting it now
[2011-04-14 00:59:11] <ichthyo> I use this concept now since almost a year and it worked out well
[2011-04-14 00:59:25] <cehteh> yeah i think its more a trac ticket than a rfc
[2011-04-14 00:59:55] <ichthyo> well, it *is* something which would need discussion if there where more than one C++ developer
[2011-04-14 01:00:14] <cehteh> but if it works for you, i put it on 'maybe finalize' .. means i read through it and finalize it
when i have no objections
[2011-04-14 01:00:24] <ichthyo> yes, agreed
---------------------
Design the Render Nodes interface
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.-- Discussion of details --
[caption="☉Transcript☉ "]
----------------------------
[2011-04-14 01:01:04] <cehteh> thats definitely pending .. needs discussion
[2011-04-14 01:01:33] <ichthyo> yes
[2011-04-14 01:01:54] <cehteh> maybe we park it to get rid of it for now since this is months ahead?
[2011-04-14 01:02:28] <ichthyo> we could even drop it, but parking is ok
[2011-04-14 01:02:44] <ichthyo> that RfC basically sais: PLING PLING PLING, we need to discuss that
[2011-04-14 01:02:53] <cehteh> well, I wont drop it, its a nice place to document the intention about the design
[2011-04-14 01:03:10] <ichthyo> ok, so lets park it
---------------------
Developer Documentation Structure
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
--> see link:http://issues.lumiera.org/ticket/763[Ticket #763]
.-- Discussion of details --
[caption="☉Transcript☉ "]
---------------------
[2011-04-14 01:03:23] <cehteh> i check that, bring it up to date and finalize it?
[2011-04-14 01:03:36] <ichthyo> i have some objections against that, see my comment
[2011-04-14 01:04:00] <ichthyo> I think, the current structure is better than what that RfC proposes
[2011-04-14 01:04:05] <cehteh> yes .. thats what i meant with bring it up to date
[2011-04-14 01:04:39] <cehteh> i can keep it pending .. and then we can finalize it when you agree
[2011-04-14 01:04:49] <ichthyo> ok, so you will update it?
[2011-04-14 01:05:32] <cehteh> yes .. maybe not for next time but i put it on todo
---------------------
Engine Interface Overview
~~~~~~~~~~~~~~~~~~~~~~~~~
.-- Discussion of details --
[caption="☉Transcript☉ "]
----------------------------
[2011-04-14 01:05:50] <cehteh> pending .. there is lot to do?
[2011-04-14 01:06:02] <ichthyo> sort of
[2011-04-14 01:06:10] <ichthyo> basically that is a high level outline
[2011-04-14 01:06:19] <cehteh> this is a rather big thing maybe we drop it in favor of smaller rfc's
[2011-04-14 01:06:23] <ichthyo> it doesn't contain details
[2011-04-14 01:06:27] <cehteh> but for now leave it pending
[2011-04-14 01:06:45] <ichthyo> at the time I wrote that, you said it looks okish for you
[2011-04-14 01:06:58] <cehteh> yes .. quite possible
[2011-04-14 01:06:55] <ichthyo> this one would be really important to discuss soon
[2011-04-14 01:07:11] <cehteh> i need to catch up first ..duh
[2011-04-14 01:07:23] <ichthyo> note: it doesnt go into details, just sets a very high level outline how the overall process works
[2011-04-14 01:07:39] <ichthyo> I think *that one* is really needed to be accepted/ reworked soon
[2011-04-14 01:07:43] <cehteh> yes .. lets talk next meeting about that (or some time else)
[2011-04-14 01:07:57] <ichthyo> ok
[2011-04-14 01:08:05] <cehteh> so pending for now
---------------------
Feature Bundle
~~~~~~~~~~~~~~
Expected to be very important in the far future, but we don't have the
resources to work on that right now, so _park_ it.
Marble Mode
~~~~~~~~~~~
.-- Discussion of details --
[caption="☉Transcript☉ "]
----------------------------
[2011-04-14 01:08:51] <ichthyo> this is also a high level one
[2011-04-14 01:08:57] <ichthyo> I am much in favour of that
[2011-04-14 01:09:09] <ichthyo> but its really kind of conceptual
[2011-04-14 01:09:20] <cehteh> me too ... but its too early to finalize it or?
[2011-04-14 01:09:52] <cehteh> that would be a 'accepted' candidate too :)
[2011-04-14 01:09:55] <ichthyo> well, I would accept it...
[2011-04-14 01:10:23] <ichthyo> but I wrote it so you (and others) also think that over
[2011-04-14 01:10:27] <cehteh> i think 'final' should somethnig which wont be changed or discussed anymore unless we see problems
[2011-04-14 01:10:40] <cehteh> thats why i am thinking we may need an 'accepted' state
[2011-04-14 01:11:07] <cehteh> and this marble mode certainly needs a lot more discussion and work to be final
[2011-04-14 01:11:51] <ichthyo> mhm, but that RfC just proposes to go for that direction, not how to do all the details
[2011-04-14 01:11:58] <cehteh> or we just finalize it as 'concept' we want no matter how we implement it finally
[2011-04-14 01:12:05] <cehteh> yes ok then
[2011-04-14 01:12:10] <ichthyo> yes, ok then
---------------------
Normalized Device Coordinates
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
still very rough, but basically agreed. +
While it needs more work, it's a bit out of focus right now, so _park it.
Proc High Level Model and Placement concept
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
That's rather final by now. This link:http://lumiera.org/documentation/devel/rfc/ProcHighLevelModel.html[proposal]
meanwhile documents the existing design; it's up to date and didn't see significant
additions since almost two years. Generally agreed upon, so it's _final_ now.
The same holds true for the
link:http://lumiera.org/documentation/devel/rfc/ProcPlacementMetaphor.html[Placement] proposal
Render Optimizer, Resource Management and Profiling
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.-- Discussion of details --
[caption="☉Transcript☉ "]
----------------------------
[2011-04-14 01:17:19] <cehteh> park
[2011-04-14 01:17:32] <ichthyo> accept for me
[2011-04-14 01:17:56] <ichthyo> that is so much our common understanding meanwhile
[2011-04-14 01:18:02] <ichthyo> so, maybe just polish a bit
[2011-04-14 01:18:18] <cehteh> yes park because it needs some polishing before final
[2011-04-14 01:18:24] <ichthyo> (remove the "pro and con") and then we could accept it right away
[2011-04-14 01:18:28] <cehteh> i can put a note that its basically accepted
[2011-04-14 01:18:38] <cehteh> parking isnt neccessary bad :P
[2011-04-14 01:18:46] <ichthyo> ResourceManagement
[2011-04-14 01:18:49] <cehteh> (well again that would be a 'accept' candidate)
[2011-04-14 01:18:53] <ichthyo> needs some more work
[2011-04-14 01:19:01] <cehteh> that are 2 things .. yes
[2011-04-14 01:19:10] <cehteh> profiling and budgeting ..
[2011-04-14 01:19:21] <cehteh> i think i will implement them and then see how it works
[2011-04-14 01:19:40] <cehteh> (unless someone else shows better alternatives)
[2011-04-14 01:19:47] <ichthyo> if you change that and e.g. remove the code sample and make it really short and high level,
then we could accept it as a concept; but the implementation details need certainly more work
[2011-04-14 01:20:17] <cehteh> hey that code is the code i lost with the hdd crash .. i am happy that i put it there
[2011-04-14 01:20:27] <cehteh> yes it was a very rough idea
[2011-04-14 01:20:37] <cehteh> and not on schedule to implemented soon
[2011-04-14 01:20:45] <cehteh> i just sketched some code down
[2011-04-14 01:21:04] <cehteh> so pending or park?
[2011-04-14 01:21:08] <ichthyo> as said, would it be ok for you to remove that sketch and just leave it on that level of the
first sentences, because then we could accept it right away
[2011-04-14 01:21:45] <cehteh> ok pending for now .. i dont want to work on this currently .. other things are more important
[2011-04-14 01:21:47] <ichthyo> we both pretty much agree that we *want* some kind of budget managing and resource usage
---------------------
Roadmap
~~~~~~~
The link:http://lumiera.org/documentation/devel/rfc/Roadmap-first.html[Roadmap document]
was erroneously not marked as final; +
Seemingly it was decided upon in 2009 already ...
Stream Type System
~~~~~~~~~~~~~~~~~~
[caption="☉Transcript☉ "]
----------------------------
[2011-04-14 01:24:06] <ichthyo> very important for me -- my vote is for accept
[2011-04-14 01:24:16] <cehteh> we had some discussion how to maintain metadata ..
[2011-04-14 01:24:40] <cehteh> i vote for accept too but this metadata (which may decribe the type) needs work
[2011-04-14 01:24:49] <ichthyo> well, this one is really a heavyweight conceptual RfC
[2011-04-14 01:25:00] <ichthyo> it is not about *how* to maintain metadata
[2011-04-14 01:25:05] <cehteh> yes
[2011-04-14 01:25:08] <cehteh> so accept
[2011-04-14 01:25:18] <ichthyo> but it really contains far reaching decisions
[2011-04-14 01:25:27] <ichthyo> and terminology
[2011-04-14 01:25:51] <ichthyo> please see, I would be glad if there was some discussion about that
[2011-04-14 01:26:36] <ichthyo> but well, at the moment I am the only one thinking into more details of this whole topic
[2011-04-14 01:26:24] <cehteh> shall i leave it pending just to nag us?
[2011-04-14 01:26:50] <ichthyo> yes, leave it pending
[2011-04-14 01:27:10] <cehteh> i was thinking too about this .. but not consistently ...
[2011-04-14 01:27:24] <ichthyo> as said, I vote for accept, but I would really ask you to think it over
[2011-04-14 01:27:38] <cehteh> i think that needs some time to settle to the right point [tm]
[2011-04-14 01:27:43] <ichthyo> I'm not right away implementing it, but the implementation is rather trivial
[2011-04-14 01:27:48] <ichthyo> so leave it pending
---------------------
Threads Signals and important management tasks
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.-- Discussion of details --
[caption="☉Transcript☉ "]
----------------------------
[2011-04-14 01:28:33] <cehteh> we need to work together to implement this on the main .. but generally i think this can be
accepted with some refinements
[2011-04-14 01:28:14] <ichthyo> some time ago, we had a short discussion about that
[2011-04-14 01:28:55] <ichthyo> The bottom line was: I am in favour of that, but I rather would like not to build it
directly into that main thread, but have a dedicated thread for it
and at that time you said, that would not be a fundamental problem
[2011-04-14 01:29:30] <cehteh> i wonder why not the main thread .. but its ok for me
[2011-04-14 01:29:53] <ichthyo> my argument is keeping the code simple and dedicated to one purpose
[2011-04-14 01:30:05] <cehteh> the main thread is a kindof 'service thread' right?
[2011-04-14 01:30:09] <ichthyo> the main thread's purpose is to wake up and shut down the system
[2011-04-14 01:30:25] <ichthyo> and I'd prefer to let him do only that and nothing else
[2011-04-14 01:30:26] <cehteh> i dont want the cinelerra fail .. opening a thread for every purpose
[2011-04-14 01:30:58] <ichthyo> generally yes, but for the more fine grained things we have the scheduler
[2011-04-14 01:31:08] <cehteh> well ok .. from my pov the main thread is more than just that but a general sheepheard
[2011-04-14 01:31:39] <cehteh> not only starting and shutdown but also control the direction
[2011-04-14 01:32:08] <cehteh> but really this isnt much a difference, if you want an extra thread then let it be so
[2011-04-14 01:32:46] <ichthyo> I see the code complexity. It would just look okish for me if we split it into these two
[2011-04-14 01:33:26] <cehteh> there is not much complexity, the main thread waits currently ..
[2011-04-14 01:34:02] <cehteh> and it can get woken by signals or condvar (convars return when a signal arrives)
[2011-04-14 01:34:29] <cehteh> actually having an extra thread wont make the code simpler because then the main thread
needs to be rigged to ignore the signals. All other threads are created by us and can
implicitly be rigged in this way
[2011-04-14 01:34:53] <ichthyo> main thread handles that already, if I recall correct
[2011-04-14 01:35:05] <ichthyo> because it loops on the condition and goes to sleep again
[2011-04-14 01:35:29] <cehteh> yes but you didnt add any sigmask handling / signal blocking right?
[2011-04-14 01:35:44] <ichthyo> ah I see, thats still todo
[2011-04-14 01:35:51] <cehteh> anyways .. i accept it .. implementation pending
[2011-04-14 01:37:38] <cehteh> signal handling becomes a 'subsystem' then ... :)
[2011-04-14 01:37:46] <ichthyo> yes, thats what I mean
---------------------
Session structure -- Timelines, Sequences, Output
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
This link:http://lumiera.org/documentation/devel/rfc_pending/TimelineSequenceOutput.html[proposal]
can be considered definitively final. Key point is: we have multiple timelines and a sequence can be used
in multiple timelines. We pretty much agree on that, thus it counts as _finalised_ now.
Use Cases
~~~~~~~~~
This is an heavyweight proposal regarding the high-level design and general handling of the
Application. This would be really a topic to be discussed in conjection with the ``Workflow''
-- the idea was to have a working group focussed these topics entirely, but there is no one
in charge of that right now. Thus _park_ it for the time being.
Version Number Scheme
~~~~~~~~~~~~~~~~~~~~~
The proposal for link:http://lumiera.org/documentation/devel/rfc_pending/VersionNumberScheme.html[Version numbering]
is _accepted_ -- it's considered close to common practice and _ichthyo_ relied on it for the debian
package already.
'''''''''''''
Next meeting
------------
The next meeting will be as usual, at Wednesday May 11, 20:00 UTC

677
doc/devel/meeting_summary/2011-05-11.txt Normal file
View file

@ -0,0 +1,677 @@
2011-05-11 Lumiera Developers Meeting
=====================================
:Author: Ichthyostega
:Date: 2012-05-15
May 11, 2011 on #lumiera 20:00 - 23:23 UTC +
__Participants__
* cehteh
* ichthyo
* daylife
_Transcript prepared by Ichthyo_
Render Engine Interface
-----------------------
_Ichthyo_ had prepared a new
link:/documentation/devel/rfc_pending/EngineInterfaceSpec.html[RfC]
about the actual interface between the
link:/documentation/design/architecture/playRender.html[Player subsystem]
and the Renderengine/Scheduler.
The following discussion turned into considering several detaled and technical
topics regarding the Jobs to be scheduled, how to create and attach new jobs,
handle memory ...
- Engine Interface ⟺ Scheduler Interface
- Latency of GUI reaction
- Aborting of Jobs, rescheduling, Job dependencies
- Ressource handling, Callbacks after Modifications
- one central service ⟺ integrating with external libs/services
- explicit wait state on Jobs?
- Interfaces and components working together
- Responsibilities: who cares for what part
- Dedicated Time facility
- animating the GUI during playback
- interaction between proc and backend
- handling of prerequisite ressources
Conclusion
~~~~~~~~~~
_Ichthyo_ will take on responsibility for that interface and care to translate
the calls into actual Jobs to be handed to the Scheduler. The Scheduler interface
itself will remain rather low-level and straight forward. The _callback_ facilities
described in the mentioned RfC need to be implemented explicitly on top of this
Scheduler interface. Ichthyo will care for that, and the integration with the
Fixture datastructure.
Next meeting
------------
The next meeting will be as usual, at Wednesday June 8, 20:00 UTC
''''
.-- Engine Interface ⟺ Scheduler Interface --
[caption="☉Transcript☉ "]
----------------------------
[2011-05-11 22:08:54] <cehteh> ichthyo: how do we want to proceed with the engine interface,
discuss it point by point or only a overview and edit comments in the rfc?
[2011-05-11 22:09:08] <ichthyo> well I thought rather the latter...its a rough draft
[2011-05-11 22:10:52] <cehteh> well generally i agree with your proposal (i'm just reading it)
with some refinements/ideas i want to add
[2011-05-11 22:12:48] <ichthyo> well, probably the most important new view angle is that
I looked on all those 'play mode changes'
[2011-05-11 22:12:51] <cehteh> QoS: do we want constants there or could you think about a 'functor' which reacts more dynamic?
[2011-05-11 22:13:20] <cehteh> well your proposal sounded like constants (to be defined)
[2011-05-11 22:13:35] <ichthyo> yes, its constants, but it seems natural to extend that
[2011-05-11 22:13:47] <cehteh> i think about a functor or even a trait-class where you can query things
[2011-05-11 22:14:19] <cehteh> this class can then react more dynamic
[2011-05-11 22:14:22] <ichthyo> ok, yes, so that you could e.g. combine several independent degrees of freedom
[2011-05-11 22:14:31] <cehteh> yes
[2011-05-11 22:16:02] <cehteh> then we can ask precisely "shall we render this frame now?"...
instead generally "do we render frames under timing pressure?"
[2011-05-11 22:16:47] <cehteh> and of course also other questions, accuracy, syncronization and so on
(...)
[2011-05-11 22:18:17] <cehteh> the other thing is about sequences --
the backend delivers only single frames, you set up relations
(this frame comes after that frame)
[2011-05-11 22:18:44] <cehteh> i would like if we can put prefetching and driving sequences
into the player/scheduler and not in the backend at all
[2011-05-11 22:18:56] <ichthyo> yeah... I figured that there is some discrepancy
[2011-05-11 22:19:33] <cehteh> well for me there is no much difference,
i am already optimizing for sequences of course but
there is no need to pass that explicitly
[2011-05-11 22:19:33] <ichthyo> so somehow we'd find a way to come from such sequences down to
the individual frames which can be deliverd
[2011-05-11 22:20:22] <ichthyo> yet there is one nasty problem
[2011-05-11 22:20:42] <ichthyo> all those changes I'm writing about here are more or less connected to such a sequence a s awhole
[2011-05-11 22:22:02] <cehteh> a 'job' .. or more precicely a 'frame-render-job' can have some state,
like references to related (following) jobs or so on
[2011-05-11 22:22:15] <cehteh> i am not sure yet if we want to put that there or in the player
[2011-05-11 22:23:04] <cehteh> maybe even the player is better but that makes the player a little more complex
as it doesnt just need to shoot and forget but also care for a lot
(potentially aborted) jobs
----------------------------
.-- Latency of GUI reaction --
[caption="☉Transcript☉ "]
----------------------------
[2011-05-11 22:22:49] <ichthyo> e.g think about the following scenario:
[2011-05-11 22:23:06] <ichthyo> we're in the middle of playing, and now we want to switch to double speed playback
[2011-05-11 22:23:39] <cehteh> yes then the player cancels every other frame job and changes its timebase
[2011-05-11 22:23:56] <cehteh> or instead canceling it could put them on a lower priority
[2011-05-11 22:24:01] <ichthyo> that means, the player/scheduler/whatever needs to find out a point in time,
*when* this switch can be performed
[2011-05-11 22:24:13] <ichthyo> and then cancel all existing jobs after that point and reschedule new jobs
[2011-05-11 22:24:40] <cehteh> so that means the player might get considerably more heavy with all this bookkeeping
[2011-05-11 22:24:40] <ichthyo> note I thought it helps if we don't guarantee immediate reaction
[2011-05-11 22:25:00] <ichthyo> for the user it doesn't count if the speedup happens 100ms later
[2011-05-11 22:25:13] <cehteh> 100ms yes ... thats noticeable
[2011-05-11 22:25:22] <ichthyo> ok, 50ms or so
[2011-05-11 22:25:45] <daylife> 30 ms is noticable
[2011-05-11 22:26:41] <cehteh> there should be a fast-path for time and priority rescheduling
[2011-05-11 22:27:33] <cehteh> fast path == explcit interface for only that purpose
[2011-05-11 22:27:36] <daylife> I'm not sure what you're discussing exactly, but one of the things that bothers me most
about the Lightworks beta is the noticable lag when navigating the timeline,
playing, etc, in contrast to Avid, which immediately, no delay whatsoever,
responds to any user action
[2011-05-11 22:29:01] <cehteh> user actions should give *instant* feedback --
even if this feedback is only fake feedback in corner cases
[2011-05-11 22:29:21] * ichthyo nods
[2011-05-11 22:29:22] <daylife> editing is on that part a bit like playing piano. if there were any delay,
no matter how short, between pressing a key and hearing a note, it would be very irritating
[2011-05-11 22:29:32] <cehteh> exacty
[2011-05-11 22:29:39] <hermanr> Indeed
----------------------------
.-- Aborting of Jobs, rescheduling, Job dependencies --
[caption="☉Transcript☉ "]
----------------------------
[2011-05-11 22:33:02] <cehteh> but caveat .. maybe we need to reschedule dependent jobs too
[2011-05-11 22:33:24] <ichthyo> yeah... likely
[2011-05-11 22:33:38] <cehteh> so far you wanted to provide jobs in a conflictless nondeadlocking way
[2011-05-11 22:34:08] <cehteh> but for rescheduling that means likely that dependency information needs
to be passed and maintained in the scheduler as well
[2011-05-11 22:34:12] <ichthyo> that doesn't need to be immediately the level within the scheduler
[2011-05-11 22:34:29] <cehteh> well it would be much easier if it is
[2011-05-11 22:35:09] <ichthyo> well... thus the idea to allow for a (small) tolerance interval
[2011-05-11 22:35:14] <cehteh> then rescheduling automatically walks all dependencies, reschedules them too
and by that it can already see if we can met timing constraints
[2011-05-11 22:36:37] <cehteh> the 'reschedule' can answer if rescheduling is possible
[2011-05-11 22:37:07] <cehteh> so you just iterate over all frame-job to be rescheduled from shortest to longest time
[2011-05-11 22:37:36] <cehteh> the most urgend reschedules may return "hey this job is already running, cant reschedule"
[2011-05-11 22:37:49] <cehteh> or even "already done, nothing to do"
[2011-05-11 22:38:03] <cehteh> to "sorry, cant do that dave"
[2011-05-11 22:38:15] <cehteh> to "ok rescheduling acknowledged"
[2011-05-11 22:38:32] <cehteh> any other returns possible? i think thats all
[2011-05-11 22:39:39] <ichthyo> ok, but now, just by doing that walk, we've found out a time point when the change will happen
[2011-05-11 22:40:33] <ichthyo> and that would be the information interesting for the layers above
[2011-05-11 22:40:56] <ichthyo> e.g. to change the animation speed of the cursor, or whatever
[2011-05-11 22:41:40] <cehteh> if you want async rescheduling you make that a job by itself
[2011-05-11 22:42:41] <cehteh> so for the player it might reschedule the next 500ms worth of frames synchronously
and any later frames (background rendering) asynchronously
[2011-05-11 22:42:04] <ichthyo> or more concrete... when we do an 'abort'
[2011-05-11 22:42:12] <ichthyo> e.g. because the user changed the model
[2011-05-11 22:42:24] <ichthyo> then I'm sure I need that asynchroneous feedback
[2011-05-11 22:43:14] <ichthyo> ...i.e. that other part doesn't have a timer or scheduler on its own,
but rather relies on that callback
[2011-05-11 22:43:19] <cehteh> we never abort 'running' jobs .. only waiting or scheduled jobs
[2011-05-11 22:43:48] <ichthyo> but we can (and need) to 'abort' whole calculation streams
----------------------------
.-- Ressource handling, Callbacks after Modifications --
[caption="☉Transcript☉ "]
----------------------------
[2011-05-11 22:45:27] <cehteh> do you really need that feedback?
[2011-05-11 22:45:47] <cehteh> how about 'abort' gives instant feedback ..
[2011-05-11 22:45:53] <cehteh> or even 'never fails'
[2011-05-11 22:46:09] <cehteh> just abort a job and consider it dead from your point of view
[2011-05-11 22:46:14] <ichthyo> note: the point is: in that other part triggering the 'abort'
[2011-05-11 22:46:23] <ichthyo> there is no timer or scheduler running
[2011-05-11 22:46:44] <ichthyo> so it doesn't help to know *when* the abort is executed
[2011-05-11 22:47:03] <ichthyo> but I need to be notified *after* it has happened
[2011-05-11 22:47:09] <cehteh> jobs are self containing, any resources they need are attached to the job,
they should not reference any volatile data up in proc
[2011-05-11 22:47:39] <ichthyo> directly, yes of course -- but indirectly thats impossible
[2011-05-11 22:49:49] <ichthyo> in that concrete example, quite simply, because the higher layers can do some stuff
only after the jobs are gone
[2011-05-11 22:49:57] <ichthyo> e.g. close the session
[2011-05-11 22:50:03] <cehteh> really?
[2011-05-11 22:50:25] <ichthyo> well... I can't just blow away a session which is still used by ongoing jobs
[2011-05-11 22:51:04] <cehteh> jobs have no backpointers to the session
[2011-05-11 22:51:05] <ichthyo> of course I don't mutate the data which is referred by the jobs
[2011-05-11 22:52:09] <cehteh> nah not only reference data i am really thinking that jobs have absolutely
no references into the session
[2011-05-11 22:52:18] <cehteh> they have references into the backend data
[2011-05-11 22:52:23] <cehteh> but these are managed there
[2011-05-11 22:52:37] <cehteh> while closing a session doesnt require to shutdown the backend
[2011-05-11 22:53:01] <ichthyo> well, so to clarify that: there *is* some kind of back reference for sure.
Just it isn't direct. See for example: a job references an exit node
[2011-05-11 22:53:35] <ichthyo> and it references an output slot
[2011-05-11 22:53:38] <cehteh> really?
[2011-05-11 22:53:47] <cehteh> how about a functor there
[2011-05-11 22:53:50] <ichthyo> both are things which are somehow allocated from the session
[2011-05-11 22:53:55] <cehteh> jobs references a continutation job
[2011-05-11 22:54:16] <cehteh> which is an abstraction of the exit node
[2011-05-11 22:54:36] <cehteh> well anyways .. the callback *is* trivially possible
[2011-05-11 22:54:48] <cehteh> but do not expect that i want to do rescheduling for aborting jobs
[2011-05-11 22:55:30] <ichthyo> well... as said: jobs can't be aborted
[2011-05-11 22:55:38] <ichthyo> but whole calculation streams can
[2011-05-11 22:55:42] <cehteh> if you place a job to be executed in 1 hour and then abort this job,
this abort callback might be called not earlier than in one hour
(or if your really want we can add a time-limit there for rescheduling)
[2011-05-11 22:57:34] <ichthyo> no! this callback job would not be scheduled to that time of the original job
(1 hour in the future) but to that point when the scheduler can guarantee that
non of the old jobs will become active anymore because then I can dispose the data,
change connections, deregister output slots and so on.
Thus all I want for this respect is a guarantee that the working function of that job
can't get active anymore
[2011-05-11 23:00:41] <ichthyo> do you see my point?
[2011-05-11 23:00:46] <cehteh> yes i see your point
[2011-05-11 23:01:33] <ichthyo> it doesn't help me to know *now* that 3 frames into the future will still be calculated
and delivered, but starting with the 4.th frame, the abort will be in effect
[2011-05-11 23:01:44] <ichthyo> this information doesn't help me *now*
[2011-05-11 23:02:09] <ichthyo> because, until these 3 frames are delivered, I can't deregister the "output slot"
[2011-05-11 23:02:17] <ichthyo> e.g. disconect an audio client from Jack server
[2011-05-11 23:02:57] <cehteh> RAII :P .. garbage collecton would be better in this case
[2011-05-11 23:03:07] <ichthyo> not the slightest
[2011-05-11 23:04:10] <ichthyo> deregistering an external connection has nothing to do with how the resources are tracked.
[2011-05-11 23:04:22] <ichthyo> rather I need to know *when* its safe to do so
(...)
[2011-05-11 23:07:56] <ichthyo> cehteh: actually I'd put an adaptation layer in between.
When viewed from the player (or the session for that), I only want an high-level 'abort'
operation. Like I proposed in my RfC. I didn't expect to talk directly to the scheduler
or any part of the engine. So at the interface of the scheduler, that can be implemented
so that it fits in best there
[2011-05-11 23:08:18] <cehteh> well i am thinking in low level terms (as usual)
[2011-05-11 23:09:09] <cehteh> i write some low-level notes for the interface (nothing concrete)
----------------------------
.-- one central service ⟺ integrating with external libs/services --
[caption="☉Transcript☉ "]
----------------------------
[2011-05-11 23:09:58] <ichthyo> Just as a striking example
[2011-05-11 23:10:13] <ichthyo> think about the occasional crashes with the dummy player currently in the GUI
[2011-05-11 23:10:24] <ichthyo> that is exactly due to a lack of such a callback
[2011-05-11 23:10:26] <cehteh> yes while i was thinking we can decouple this completely, but maybe not
[2011-05-11 23:10:41] <ichthyo> the GUI closes the window. The only thing I can do is to block that closing
[2011-05-11 23:10:57] <ichthyo> but without external help, I don't know how long to block
[2011-05-11 23:11:33] <ichthyo> and I we don't block, then the last frame gets disposed into an output buffer
which is already gone, deregistered from XV and whatsoever
[2011-05-11 23:11:45] <ichthyo> not ever, but sometimes
[2011-05-11 23:12:18] <cehteh> well i was thinking that the scheduler and backend only operate on their own data
and you can just drop that on the floor from player and proc (and gui)
[2011-05-11 23:12:37] <cehteh> that means currently the player presents some buffers and the mockup player renders there
[2011-05-11 23:13:02] <cehteh> this renders into the gui provied buffer which the gui may release too early right?
[2011-05-11 23:13:34] <cehteh> but if this buffer is provided by the backend and the gui just closes down, not displaying it
anymore then nothing happens, the backend knows how to manage resources
[2011-05-11 23:14:29] <ichthyo> well... true, but a general mismatch
[2011-05-11 23:14:34] <ichthyo> usually it works the other way round
[2011-05-11 23:14:49] <ichthyo> I don't know how it's for ALSA, but also for jack it works the other way round
[2011-05-11 23:15:01] <ichthyo> all those systems assume that you have a thread running just to serve them
[2011-05-11 23:15:06] <cehteh> yes
[2011-05-11 23:15:13] <ichthyo> and require you to set up buffers to place things into
[2011-05-11 23:15:24] <ichthyo> Jack is even more rigourous
[2011-05-11 23:15:32] <cehteh> yes i see now
[2011-05-11 23:15:54] <ichthyo> so thats an important general mismatch we should come up with a nice solution maybe
[2011-05-11 23:16:13] <cehteh> but well i still wonder if we may put such a callback even more into the
backend "resource released" ...
[2011-05-11 23:16:18] <cehteh> mhm maybe not
[2011-05-11 23:16:48] <cehteh> has pros and cons .. not all resources have jobs associated
[2011-05-11 23:16:16] <ichthyo> of course it would be great, if the backend just delivers in time,
right into the buffer for output
[2011-05-11 23:16:39] <ichthyo> so that we don't need another buffering and copy over step on the output side
[2011-05-11 23:17:06] <cehteh> but still in real the jack buffers will come from the backend
[2011-05-11 23:17:13] <cehteh> so the backend can send a notify
[2011-05-11 23:21:23] <ichthyo> cehteh: AFAIK, the jack buffers come from the jack server
[2011-05-11 23:22:02] <ichthyo> same for any gui widgets to display video
[2011-05-11 23:21:44] <ichthyo> and not from the backend
[2011-05-11 23:21:49] <cehteh> ichthyo: the backend can still manage that
[2011-05-11 23:22:44] <ichthyo> so why should the backend try to manage buffers,
which are actually managed by an other (external) system??
[2011-05-11 23:22:47] <cehteh> generally i want it the other way around that the backend allocates/mmaps buffers,
but i know that there are a lot things which do it the other way around
[2011-05-11 23:23:02] <cehteh> that means we can do 2 things: copying into the destination buffer
[2011-05-11 23:23:11] <cehteh> or manage this destination buffers as well
[2011-05-11 23:24:08] <ichthyo> or (3rd option): let the last calculation step work into those external destination buffers
[2011-05-11 23:24:35] <cehteh> thats case 2 .. when we manage them, then we calculate there with our normal toolchain
[2011-05-11 23:25:03] <ichthyo> not quite; we use them, but we don't manage them
[2011-05-11 23:25:27] <cehteh> proc just asks for a buffer and doesnt need to care from where it comes
(RAM, Graphics card, audio server ...)
[2011-05-11 23:26:11] <cehteh> if we dont manage them but only 'use' then then we get some more code paths
for every different kind of buffer
[2011-05-11 23:26:08] <ichthyo> and thats not all
[2011-05-11 23:26:14] <ichthyo> there are similar problems
[2011-05-11 23:26:26] <ichthyo> e.g. jack doesn't run in our threading sytem
[2011-05-11 23:26:30] <cehteh> yes, rather shared memory
[2011-05-11 23:27:04] <ichthyo> so another question is: can we even be so precise?
we don't have low-latency permissions, as the jack server has
so that means some additional buffering or copying anyway
[2011-05-11 23:27:31] <cehteh> we manage buffers provided by jack
[2011-05-11 23:27:49] <cehteh> if we cant render in time, we cant copy in time for sure
[2011-05-11 23:27:55] <ichthyo> ah... if you call that 'managing' then yes
[2011-05-11 23:28:33] <cehteh> so we have different memory backend, first our own mmaped persistent file storage
[2011-05-11 23:28:46] <cehteh> then later maybe memory on the gpu
[2011-05-11 23:28:37] <ichthyo> nah: 'managing' means owning, allocating, deallocating
[2011-05-11 23:29:12] <cehteh> well we ask jack for memory and we tell jack when we dont need it anymore
[2011-05-11 23:29:33] <cehteh> and i think i can do that better in the backend just like any other memory resource
[2011-05-11 23:29:40] <ichthyo> but actually I don't see why we should
[2011-05-11 23:29:49] <ichthyo> why can't we just cooperate with these services?
[2011-05-11 23:29:56] <ichthyo> as everyone else does too?
[2011-05-11 23:30:01] <cehteh> i call that cooperation?
[2011-05-11 23:30:41] <ichthyo> I don't see why the backend needs to pseudo-manage things which are just out of the scope,
and actually owned and managed by someone else?
[2011-05-11 23:30:49] <cehteh> the question is only if every single thing does this ad-hoc --
aka the gui cares for video buffers, the audio player cares for sound buffers,
proc cares for rendering buffers and so on
or we have one backend which cares about all buffers
the coding effort is almost the same
[2011-05-11 23:31:46] <cehteh> but we get one 'memory' service which can deliver different classes of memory
(and later easily adaptable/extendable to new technologies)
[2011-05-11 23:32:01] <ichthyo> I'd rather doubt that. In all those cases, we're using already existing
libraries and solutions. and we're rather integrating with them
[2011-05-11 23:32:23] <cehteh> yes so there is no much effort doing so ..
[2011-05-11 23:32:54] <ichthyo> that extends even more to the media libs, we're intend to use
[2011-05-11 23:32:55] <cehteh> but since the backend always knows what memory is in use it can do a clean shutdown
[2011-05-11 23:33:18] <cehteh> without all this callbacks and asyncronous messages
[2011-05-11 23:33:34] <ichthyo> not really
[2011-05-11 23:33:49] <ichthyo> if we use external services, we're bound to the terms of these
[2011-05-11 23:33:56] <ichthyo> same for external libraries
[2011-05-11 23:34:14] <cehteh> yes
[2011-05-11 23:35:04] <ichthyo> if we have a nice and blastingly fast mmaped file read and buffer handling for it than fine
[2011-05-11 23:35:41] <ichthyo> then I'd use that exactly for those sevices, which *we* write, provide and manage
[2011-05-11 23:36:08] <cehteh> possibly we just do it as you saying, and we may implement it in the way as i think later
(and then try out and compare) .. or stay with your way
[2011-05-11 23:37:29] <ichthyo> maybe yes... probably I'd concentrate first on that core part which we provide,
because I'd guess it is the most important one
i.e. reading those source media files and the like
[2011-05-11 23:37:46] <cehteh> well i have some reasons to do it in my way --
for example if you connect the viewer somewhere in the graph,
then it would be nice if it can transparently use memory on the graphics card
instead copying over. Or think about render nodes and network transparency later
[2011-05-11 23:38:48] <cehteh> but this is future and i think it is not a fundamental design decision can can be
incrementally added when it makes sense and turns out to work well
[2011-05-11 23:39:07] <cehteh> first i am going for the mmaped file access and nothing else
[2011-05-11 23:39:07] <ichthyo> yes, probably better approaching it that way
----------------------------
.-- explicit wait state on Jobs? --
[caption="☉Transcript☉ "]
----------------------------
[2011-05-11 23:44:34] <ichthyo> another question.
[2011-05-11 23:44:51] <ichthyo> do our jobs have some mechanism to 'yield' or wait or similar?
[2011-05-11 23:45:10] <ichthyo> i.e. go in a wait state (and make room for other jobs to be scheduled meanwhile)?
[2011-05-11 23:45:34] <cehteh> waiting on other jobs
[2011-05-11 23:45:49] <cehteh> i am thinking if i make a generic waiting or only on jobs
[2011-05-11 23:45:54] <ichthyo> so you call into a funciton on the lumiera scheduler
[2011-05-11 23:45:55] <cehteh> i think only on jobs is good
[2011-05-11 23:46:06] <ichthyo> which blocks and reschedules other jobs meanwhile?
[2011-05-11 23:46:23] <cehteh> a running job?
[2011-05-11 23:46:34] <ichthyo> just brainstorming
[2011-05-11 23:46:45] <ichthyo> both is thinkable
[2011-05-11 23:46:56] <ichthyo> also it would be thinkable that we disallow any waits
[2011-05-11 23:47:03] <cehteh> nope i do not want that, rather split the jobs up into small pieces
[2011-05-11 23:47:13] <ichthyo> and just say, then the job function has to terminate and another job has to pick up
[2011-05-11 23:47:27] <cehteh> yes thats possible
[2011-05-11 23:48:00] <cehteh> but when a job runs it runs until its done (which might be a errorneous/abort on its own)
[2011-05-11 23:48:20] <cehteh> after running a job wont become rescheduled
[2011-05-11 23:48:49] <cehteh> if it turns out we need a yield i could implement it .. but not prematurely
[2011-05-11 23:49:44] <ichthyo> but likely for our purposes we don't even need that
[2011-05-11 23:50:16] <cehteh> yes and thats not domain of the scheduler
[2011-05-11 23:50:32] <cehteh> jobs can block anyways
[2011-05-11 23:50:40] <cehteh> in running state
[2011-05-11 23:50:53] <cehteh> no need to put them on a wait queue or whatever
[2011-05-11 23:51:31] <ichthyo> but of course the goal is to write them such that they don't block
[2011-05-11 23:52:11] <cehteh> and the only thing worthwile to wait for in a special wait state is for some other job
[2011-05-11 23:52:38] <cehteh> the next question is .. do we release this waiting jobs by some signal or by a job termination?
[2011-05-11 23:53:14] <cehteh> first can be more efficent and will be more complex
[2011-05-11 23:53:30] <cehteh> so for first i opt for the later, there is prolly anyways something to do
[2011-05-11 23:54:02] <ichthyo> how does this 'waiting' actually work?
[2011-05-11 23:54:16] <ichthyo> does the jobfunction call into some API of the scheduler?
[2011-05-11 23:54:21] <cehteh> i make a explicit waiting container (llist) jobs waiting sit there and are not scheduled
[2011-05-11 23:54:47] <cehteh> (or maybe that uses a priority queue to, to detect expired jobs)
[2011-05-11 23:55:02] <cehteh> the sheduler wraps the jobfunction
[2011-05-11 23:55:10] <ichthyo> ok
[2011-05-11 23:55:29] <ichthyo> so that means, he gets control only when the jobfunction exits
[2011-05-11 23:55:51] <cehteh> for simplicity and a first implementation yes i think that suffices
[2011-05-11 23:55:59] * ichthyo thinks the same
----------------------------
.-- Interfaces and components working together --
[caption="☉Transcript☉ "]
----------------------------
[2011-05-12 00:01:37] <cehteh> so now we talked about the scheduler interface --
which isnt really the engine interface --
BUT: the job structure needs to carry all resources involved around
[2011-05-12 00:02:34] <cehteh> think abut the W there are more interfaces involved
[2011-05-12 00:02:43] <ichthyo> ok... that brings up another closely related question
[2011-05-12 00:02:59] <ichthyo> the engine interface needs something like these calculation streams
[2011-05-12 00:03:19] <ichthyo> that means, that the individual job needs to cary a marker, so it can be associated
to a specific stream
[2011-05-12 00:02:59] <cehteh> first, asking the backend for a frame
[2011-05-12 00:03:58] <cehteh> i add at least one generic void* to the job where you can put any data along
[2011-05-12 00:04:12] <cehteh> type safety on the level above please :P
[2011-05-12 00:05:20] <ichthyo> I'm fine with that
[2011-05-12 00:05:27] <cehteh> (caveat about the lifetime and ownership of this void*)
[2011-05-12 00:05:52] <cehteh> if the job owns it then we need also a cleanup hook there
(...)
[2011-05-12 00:10:44] <cehteh> next would be a interface how to setup a job... possibly in multiple stages
[2011-05-12 00:11:59] <cehteh> job_new(); job_wait_for(another_job); job_data()=some_calculation_stream;
[2011-05-12 00:12:06] <cehteh> job_arm()
[2011-05-12 00:12:14] <cehteh> something like that
[2011-05-12 00:12:38] <ichthyo> sounds ok
[2011-05-12 00:12:53] <cehteh> well .. now i think it becomes clear that 'jobs' are our
message passing interface between proc and backend
[2011-05-12 00:13:15] <cehteh> you dont ask the backend directly for a frame you schedule a job delivering a frame
[2011-05-12 00:13:29] <cehteh> the backend schedules a job rendering a frame if its not cached
[2011-05-12 00:13:35] <ichthyo> yes, my thinking too
(...)
[2011-05-12 00:14:50] <cehteh> so 'job' is a baseclass .. but there can be highlevel subclasses
cache_fetch_job and render_frame_job
[2011-05-12 00:15:04] <ichthyo> really?
[2011-05-12 00:15:11] <cehteh> i wonder -- makes sense or?
[2011-05-12 00:15:38] <ichthyo> *could* be done, but somehow goes against that simplicity
[2011-05-12 00:15:53] <cehteh> I mean each of this class shares a lot, how abortion is handled and more
[2011-05-12 00:16:43] <cehteh> Ok on the low level you can always put a job together with all the little
details needed and then arm it. I expect that we have maybe half a dozen classes of jobs
[2011-05-12 00:17:03] <ichthyo> not only on the low level. That would also be the correct OO aproach
[2011-05-12 00:17:14] <ichthyo> "prefer composition over inheritance"
[2011-05-12 00:17:53] <ichthyo> because, if you have a care_fetch_job, etc etc, you end up switching on types
[2011-05-12 00:18:13] <cehteh> ok .. leave it simple
[2011-05-12 00:18:27] <ichthyo> better investigate data at the start, and then assemble a job which can just run as-is
[2011-05-12 00:19:01] <cehteh> ok
----------------------------
.-- Responsibilities: who cares for what part --
[caption="☉Transcript☉ "]
----------------------------
[2011-05-12 00:20:00] <ichthyo> to expand on that
[2011-05-12 00:20:11] <ichthyo> when we create a 'calculation stream'
[2011-05-12 00:20:18] <ichthyo> we get the following informations:
[2011-05-12 00:20:30] <ichthyo> - the timings (frame duration, stepping)
[2011-05-12 00:20:36] <ichthyo> - the exit node to use
[2011-05-12 00:20:42] <cehteh> I wonder if i have to care anything about a caclulation stream --
thats purely a player/render thing
[2011-05-12 00:21:15] <ichthyo> *If* your realm just starts at the scheduler, then the answer is "no"
[2011-05-12 00:21:22] <ichthyo> then you don't have to care for that
[2011-05-12 00:21:42] <ichthyo> that means, I would implement that engine interface, kindof like a facade
[2011-05-12 00:21:43] <cehteh> except i would prolly take some hints for optimization
[2011-05-12 00:22:17] <cehteh> so you can add at least add flags like ONESHOT, FORWARD, BACKWARD, RANDOM
[2011-05-12 00:22:19] <ichthyo> taking that approach would mean that I would also have to set up the times for the jobs, right?
[2011-05-12 00:27:39] <ichthyo> where starts the part which you care for?
[2011-05-12 00:27:59] <ichthyo> did I understand you correct, that you rather don't want to care for the calculation streams
[2011-05-12 00:28:07] <ichthyo> the callbacks and this stuff?
[2011-05-12 00:28:31] <cehteh> yes i think you can abstract them better, you have knowledge about the session and user interaction
[2011-05-12 00:28:37] <ichthyo> (I have no problem taking that responsibility, and implement it on top of the bare jobs)
[2011-05-12 00:29:38] <ichthyo> so that means, that I will also care for setting up the render nodes/invocation,
before packaging that into a job function, right?
[2011-05-12 00:29:57] <ichthyo> which would make sense, because I also implemented the render nodes themselves
[2011-05-12 00:30:02] <cehteh> yes
[2011-05-12 00:28:50] <ichthyo> ok
----------------------------
.-- Dedicated Time facility --
[caption="☉Transcript☉ "]
----------------------------
[2011-05-12 00:22:48] <cehteh> we once talked about that we want a timer facility (outside of player)
[2011-05-12 00:23:16] <cehteh> and a 3rd prio queue in the scheduler which is short and hard(er) realtime
[2011-05-12 00:23:21] <ichthyo> because e.g. we need such for animating the GUI
[2011-05-12 00:23:28] <ichthyo> i.e. show the playback position
[2011-05-12 00:23:29] <cehteh> but only the player knows the times
[2011-05-12 00:24:18] <cehteh> this 'timer' possibly wont be singleton too .. you can play unrelated things in parallel
[2011-05-12 00:24:51] <ichthyo> well... I'd rather expect that to be quite similar to the jobs
[2011-05-12 00:25:26] <cehteh> well i wonder if the normal jobs become bit more heavyweight
[2011-05-12 00:25:40] <cehteh> and the high prio queue gets its own interface
[2011-05-12 00:26:00] <cehteh> no full blown jobs, no waiting (inter-jobs)
[2011-05-12 00:26:10] <ichthyo> what do you think to add to the normal jobs, beyond what we just discussed?
[2011-05-12 00:26:15] <ichthyo> ah maybe the dependencies
[2011-05-12 00:26:23] <ichthyo> but besides that?
[2011-05-12 00:29:03] <cehteh> so there will be the full blown jobs
[2011-05-12 00:29:27] <cehteh> which can record dependencies on other jobs, have their state, a jobfunction
(...)
[2011-05-12 00:34:00] <cehteh> the exact setup for scheduling will likely become little more complicated
[2011-05-12 00:34:46] <cehteh> you dont just set a time when to render but timespans and priorities,
possibly twice for each queue -- at least my plan is that a job can be in the
background render queue and in the high prio queue (we should give this a better name) at once
[2011-05-12 00:36:52] <cehteh> this interface needs to be discussed ... but for now it suffices that we know its there
[2011-05-12 00:37:08] <cehteh> (also a reason to make the tinyjobs which are much simpler)
----------------------------
.-- animating the GUI during playback --
[caption="☉Transcript☉ "]
----------------------------
[2011-05-12 00:41:34] <ichthyo> cehteh: how would you deal with that 'animating' of the gui during playback?
[2011-05-12 00:41:49] <ichthyo> that is, updating the time, moving the playhead cursor etc.
[2011-05-12 00:42:02] <ichthyo> could that be a job too?
[2011-05-12 00:42:38] <cehteh> well summarize:
[2011-05-12 00:43:08] <cehteh> we use the 'normal' queues for rendering .. there are job dependencies which
eventually yield in a frame rendered. The actualy 'playout' whatever that means
is done by the realtime tinyjobs from a special time-syncronized queue
[2011-05-12 00:44:32] <cehteh> this is continutation based
[2011-05-12 00:45:09] <cehteh> the last thing in the normal queues is setup this playout (if still in time)
[2011-05-12 00:45:43] <cehteh> blinking the gui doesnt need to be hard realtime it could be set up there too --
or can you inject gui messages from other threads?
[2011-05-12 00:46:56] <ichthyo> in GTK, you invoke so called "signals"
which are functors. And they are wired such that the actual function
is then dispatched from the event thread
you can create that signals from any thread
GTK cares to execute them through his event mechanism
[2011-05-12 00:48:44] <cehteh> this signals can carry some payload (rendered thumbnail?)
[2011-05-12 00:48:56] <ichthyo> they are function objects
[2011-05-12 00:49:02] <ichthyo> so they can do anything they want
[2011-05-12 00:49:32] <ichthyo> typically you bind a method from some existing object
[2011-05-12 00:50:50] <cehteh> so we may add a notification layer there to give it the right distance for abstraction --
and the gui can maybe subscribe to certain notification as well as things which are
queried by the gui are send back there
[2011-05-12 00:51:07] <ichthyo> yes, my thinking too
[2011-05-12 00:51:02] <ichthyo> for our problem that means that there will be some signal(s) to update the timecode
[2011-05-12 00:51:23] <ichthyo> there is already even an interface called "GuiNotificationFacade"
[2011-05-12 00:51:30] <ichthyo> its defined through the interface system
[2011-05-12 00:51:41] <ichthyo> just Joel didn't get around filling it with life
[2011-05-12 00:51:51] <ichthyo> that was shortly before he left
(...)
[2011-05-12 00:56:13] <cehteh> and anything not sound and video playback, just gui update related can be lets say 50ms
off time -- for sure it would be a bit pita when automation curves lag noticeable
[2011-05-12 00:56:55] <ichthyo> so it could even be enqueued with a lower priority than frame calculations scheduled
for the same time
[2011-05-12 00:58:28] <ichthyo> well... cinelerra e.g. scrolls the whole timeline pane keeping the cursor in the middle
[2011-05-12 00:58:36] <cehteh> there are no much things which are time constraind in the gui but not video or sound
[2011-05-12 00:58:37] <ichthyo> this is nice for the handling
[2011-05-12 00:58:46] <cehteh> and depend on backend/proc
[2011-05-12 00:59:16] <ichthyo> anyway... that means that there might be thumbnails to process, and I take it that we
handle that somehow through our engine as well, just with lower priority
[2011-05-12 00:59:23] <cehteh> yes that costs a lot performance there ..
[2011-05-12 01:00:01] <ichthyo> as a *user* I really don't know what to prefer
[2011-05-12 01:00:10] <cehteh> btw if frames or thumbnails have higher priority might be a feature of 'perspectives'
[2011-05-12 01:00:24] <ichthyo> an application which burns quite some resources, but gets me a steady updating display
[2011-05-12 01:00:44] <ichthyo> or an application which freezes every 10 seconds when the cursor hits the right window border
[2011-05-12 01:00:54] <cehteh> haha
[2011-05-12 01:00:56] <ichthyo> (e.g. ardour has this behaviour)
[2011-05-12 01:00:59] <cehteh> neither is good
[2011-05-12 01:02:03] <ichthyo> probably the best solution (also the rather expensive to implement) is to make that GUI
update in several stages, and each with a different priority
[2011-05-12 01:02:22] <ichthyo> e.g. a wireframe has high priority, thumbnails and waveforms have low priority
[2011-05-12 01:03:29] <cehteh> well the timeline can be rendere into an extra canvas .. then displaying it with scrolling
is certainly more efficent than redraw it completely 60 times a second
[2011-05-12 01:03:37] <ichthyo> well... you could skip the thumbnails, yet still have an animated cursor and running timecode
[2011-05-12 01:04:15] <cehteh> yes but you dont need to care in the gui .. you just send the jobs with configured piority
and deadline and render when the data arrives
[2011-05-12 01:04:29] <ichthyo> yeah, thats what I mean
(...)
[2011-05-12 01:06:06] <cehteh> i am not gui programmer
[2011-05-12 01:06:53] <cehteh> but if, i would make a timeline in a canvas (or even multiple layers,
automation on top) and render that in background even with some safe areas around
which are not visible, then scolling in is just a fast hardware operation.
[2011-05-12 01:07:36] <ichthyo> exactly, thats the way I think we should do it
[2011-05-12 01:07:51] <cehteh> well i was already suprised that no one pre-renders menus when the mouse is hovering
over them and when the user clicks displaying them in instant
[2011-05-12 01:07:52] <ichthyo> ok... the key point is to do that rendering asynchroneously
[2011-05-12 01:08:05] <cehteh> thats how i would do it
[2011-05-12 01:08:47] <ichthyo> makes many GUI programmers feel queasy
[2011-05-12 01:08:58] <cehteh> hey we all like snappy guis
[2011-05-12 01:09:48] <ichthyo> actually, I must say it's kind of embarrasing when we consider the level of
functionality provided by many GUI toolkits even today
[2011-05-12 01:10:30] <ichthyo> basically the technology from 20 years ago
[2011-05-12 01:10:35] <cehteh> yes
[2011-05-12 01:10:53] <cehteh> and even with wayland and all new things it not going to change that fundamentals
[2011-05-12 01:11:02] <ichthyo> yes, indeed
[2011-05-12 01:11:20] <ichthyo> that is just an attempt to make that old model running faster, by offloading to the hardware
[2011-05-12 01:12:03] <cehteh> i would like a compositor on the toolkit level ... not as window server
[2011-05-12 01:13:14] <ichthyo> yes
[2011-05-12 01:13:19] <ichthyo> and even more
[2011-05-12 01:13:28] <ichthyo> the parts should be composed in a functional way
[2011-05-12 01:13:47] <ichthyo> and the toolkit would then invoke that functors in the right order and async
[2011-05-12 01:13:53] <cehteh> yep
----------------------------
.-- interaction between proc and backend --
[caption="☉Transcript☉ "]
----------------------------
[2011-05-12 01:14:46] <cehteh> the 'actual' interfaces how the backend queries a frame from proc and how proc
asks the backend for a frame .. do we need to work this out?
But i think about not working it offically out at all in a big way,
this interfaces prolly just happen and can be improved in small details
[2011-05-12 01:16:10] <ichthyo> in the (half finished) code in the ProcNode, I used placeholder / facade interfaces
[2011-05-12 01:16:21] <ichthyo> that is enough to set up the backbone
[2011-05-12 01:16:52] <ichthyo> that is, for node -> backend (to get a frame)
[2011-05-12 01:17:12] <cehteh> ah ok
[2011-05-12 01:17:18] <ichthyo> the other one will turn out, when I care for wiring up such a job, as we discussed today
[2011-05-12 01:17:31] <ichthyo> that is engine/player -> proc
[2011-05-12 01:17:50] <ichthyo> "node invocation interface"
[2011-05-12 01:18:22] <ichthyo> well, personally I think I'll first concentrate on the player, to get a rough outline there
[2011-05-12 01:18:36] <cehteh> yes that will show us the requirements
[2011-05-12 01:19:15] <ichthyo> and if I recall right, you were rather engaged into the framecache / frame database
[2011-05-12 01:20:10] <cehteh> the general interface is simple (in my head) .. but there are some dirty corners to worked out
(...)
[2011-05-12 01:20:48] <cehteh> if proc asks the backend for a frame, the backend may ask proc for rendering it --
and this going pingpoing with all dependenices, but the inital query must provide enough
information to construct all of the rest which involves setting up jobs which need
deadlines and priorities
[2011-05-12 01:22:05] <cehteh> or we may rethink this somehow
[2011-05-12 01:22:17] <cehteh> doing the ping-pong more in the higher levels
[2011-05-12 01:22:26] <cehteh> (no W then)
[2011-05-12 01:22:56] <ichthyo> well... still a W, but between two function levels a bit higher then
[2011-05-12 01:22:59] <cehteh> asking the cache if it has a frame, if not then the initiator is responsible
for setting up the rendering job
[2011-05-12 01:23:08] <cehteh> might become a M then :)
[2011-05-12 01:23:15] <ichthyo> yes, kindof that
[2011-05-12 01:23:56] <cehteh> well this needs some careful thinking --
thats an important design decision we can not easily revert
[2011-05-12 01:26:00] <ichthyo> well... until now I've kept that decision rather flexible.
That is, in the part I wrote in the nodes
[2011-05-12 01:26:40] <cehteh> for now .. eventually this becomes nailed down
[2011-05-12 01:27:35] <ichthyo> you recall, there was the controversy if we should split and schedule
every calculation step, even if it doesn't include fetching some aditional data
[2011-05-12 01:28:43] <cehteh> huh? ... well it wasnt about the fetching it was about finer granulared planning for rendering
[2011-05-12 01:29:00] <cehteh> note that when you render too early the caches cool down (or fetch too early)
[2011-05-12 01:29:59] <cehteh> so i was about discovery before fetching, and fetching in time, and then rendering
[2011-05-12 01:30:12] <cehteh> and possibly add some more steps as need arises
[2011-05-12 01:30:27] <ichthyo> yes... that was kind of the middle ground we agreed on
[2011-05-12 01:31:10] <ichthyo> I retained the possibility to decide if and where a chain of calculations for a single
frame should be split and scheduled as separate jobs
[2011-05-12 01:31:24] <cehteh> ah
[2011-05-12 01:34:15] <cehteh> run only the heavy lifting in single jobs
[2011-05-12 01:34:38] <ichthyo> I allways assume that 1 frame is one job
[2011-05-12 01:34:40] <ichthyo> at least
[2011-05-12 01:34:53] <cehteh> well refine 'rendering one frame is one job'
[2011-05-12 01:35:02] <ichthyo> the controversy was to what extend a single frame should be split into multiple jobs
[2011-05-12 01:35:10] <cehteh> i dont care if discovery over a sequence is one job for multiple frames
[2011-05-12 01:36:54] <ichthyo> at that point (more than a year ago), you insisted that every calculation
be done in a separate job, and I didn't agree to that
[2011-05-12 01:37:00] <cehteh> yes
[2011-05-12 01:37:08] <ichthyo> that is, even if its just the application of yet another effect to the same frame
[2011-05-12 01:37:09] <cehteh> i think we can try out and see
[2011-05-12 01:37:16] <ichthyo> yes, that was the conclusion
[2011-05-12 01:37:37] <ichthyo> while it is pretty much clear that the fetching of data should be a separate job; i.e. the IO wait
[2011-05-12 01:39:12] <cehteh> prolly scales well this way, but we may need refinements for branches in the graph
[2011-05-12 01:39:41] <ichthyo> especially for those cases, which combine several frames (e.g. time average, motion tracking)
----------------------------
.-- handling of prerequisite ressources --
[caption="☉Transcript☉ "]
----------------------------
[2011-05-12 01:43:35] <cehteh> resource transfer .. we talked about that
[2011-05-12 01:43:46] <cehteh> waiting for resources not jobs
[2011-05-12 01:44:26] <ichthyo> yes, of course .... I took that for granted
[2011-05-12 01:44:44] <ichthyo> otherwise it wouldn't make sense to have a separate job for fetching input data
[2011-05-12 01:45:09] <cehteh> well you remember the that writing has to be committed .. and then i wanted a short circuit to transfer a resource to another receiver than giving it back to the backend
[2011-05-12 01:47:31] <cehteh> if you want to accumulate render jobs then i strongly opt for doing this vertically (if ever)
[2011-05-12 01:47:40] <cehteh> never horizontally
[2011-05-12 01:47:43] <ichthyo> yes
[2011-05-12 01:48:11] <ichthyo> with the exception you just mentioned: planning further jobs could be done in a sway for a segment of time
[2011-05-12 01:48:44] <cehteh> yes but with a limit .. not a full sequence from the session
[2011-05-12 01:48:49] <ichthyo> e.g. the 2* 25 jobs for the next second
[2011-05-12 01:49:03] <ichthyo> (2 per frame, one for the resourece, one for the calculation)
[2011-05-12 01:49:08] <ichthyo> in a typical standard case
[2011-05-12 01:49:28] <ichthyo> and then add a followup job to schedule the next jobs for the next second
[2011-05-12 01:49:40] <ichthyo> just as an example
[2011-05-12 01:49:52] <cehteh> lets say dont run discovery for more than 20 frames .. and dont fetch more than 5 frames and render every single frame
[2011-05-12 01:50:07] <cehteh> even if the span between cuts is longer
[2011-05-12 01:50:22] <cehteh> this values of course may be configureable
[2011-05-12 01:50:26] <ichthyo> and its clear that the actual numbers are for optimisation
----------------------------

37
doc/devel/meeting_summary/2012-01-11.txt Normal file
View file

@ -0,0 +1,37 @@
2012-01-11 Lumiera Developers Meeting
=====================================
:Author: cehteh
:Date: 2012-01-11
Jan 11, 2011 on #lumiera 20:00 - 22:30
__Participants__
* cehteh
* ichthyo
* benn
* raffa
Conclusions
-----------
. _ichthyo_ removed most of the TiddlyWikis, and worked the content into the website
. _cehteh_ reports that Lumiera got another donation (75 €), arrangements with
the FFIS to get access (view) about the donations account are under way. We'll
ask donors then if they want to be published or stay anonymous and will set up
a wiki page listing donations and expenses.
. _ichthyo_ reworked the SCons build, as discussed last time
. _cehteh_ writes a very short RfC, to document that we're using SCons for now.
. possibly no one going to LAC, too far away
. we discussed a link checker / link resolver for the website.
The idea is to have a semi automatic tool, which is used locally when
authoring website content to find cross references.
. _benny_ and _ichthyo_ follow up on the
link:http://libregraphicsmag.com/[Libre Graphics Magazine]'s offer to promote
graphics and media projects and try to get into discussion with them and see
what can be done within our limited time.
. when it comes to have a working example for media file output, we stick to the
mainstream solutions *ffmpeg* and or *gstreamer*, but care not to lock ourselves
into a single solution. Concluded that we do this over plugin interfaces --
it mostly boils down to support ffmpeg -- besides we'll investigate simple
alternatives for basic output.

373
doc/devel/meeting_summary/index.txt Normal file
View file

@ -0,0 +1,373 @@
Past Meetings
=============
//Menu: sort children descending
Not all meetings gets summarized on this page. For example, there are no meeting
summaries at all available for 2010. This only means that there were not enough
things that needed deciding/documenting during these meetings to warrant someone
spending time on the task of writing a summary.
Anyone interested in Lumiera development is also encouraged to read mailing list
archives and other documentation.
************************
During Summer 2011 again the regular monthly meethings
were mostly informal detail discussions plus some
organisational planning. Not much to account for.
************************
11 Jan 2012
-----------
Topics
~~~~~~
* Website status. Clean-up of TiddlyWikis
* cross-links and link checking for the website
* promotion: Libre Graphics Magazine
* rendering to file: library solutions
Summary
^^^^^^^
* link:2012-01-11.html[Summary (by cehteh)]
14 Dec 2011
-----------
Topics
~~~~~~
* discussion about build systems.
* review of the SCons build scripts
* some details about the scheduler and invoking render jobs
////
Summary
^^^^^^^
* link:2011-12-14.html[IRC Transcript (prepared by XXXX)]
////
************************
During Summer 2011 we kept up our monthly IRC meetings -- rather casual exchange
for the time being.
- 9 Nov 2011
- 12 Oct 2011
- 14 Sep 2011
- 10 Aug 2011
- 13 Jul 2011
- 8 Jun 2011
************************
11 May 2011
-----------
Topics
~~~~~~
* detailed technical discussion about the Render Engine Interface
Summary
^^^^^^^
* link:2011-05-11.html[IRC Transcript (prepared by Ichthyo)]
13 Apr 2011
-----------
Topics
~~~~~~
* New Website Page Layout
+
liquid layout -- scrolling -- colours
* Website licensing and legal questions
* Trac spam
* Go through Ideas and Drafts in design process
+
accepted: ForwardIterator, HighLevelModel, Placement concept, Session structure
Summary
^^^^^^^
* link:2011-04-13.html[Summary (written by Hermann Voßeler)]
9 Mar 2011
----------
Topics
~~~~~~
* The New Website
* IRC Logs
* VoIP discussions over Mumble?
* Version numbering
Summary
^^^^^^^
* link:2011-03-09.html[Summary (written by Stefan Kangas)]
.informal meetings, no summaries
********
In 2010, the monthly meetings where rather informal gatherings,
without any relevant decisions beyond the organisation of conference activities.
During that time, usually the two remaining Lumiera core developers where alternatively
either prevented by real-life obligations, or otherwise just in ``deep coding mode''
********
9 Sep 2009
----------
Topics
~~~~~~
* FrOSCon
* Funding
* Website
* Ticket Query Clean-up
* Coding
Summary
^^^^^^^
* link:2009-09-09.html[Summary (written by raffa and Ichthyo)]
14 Jan 2009
-----------
Topics
~~~~~~
* organisational (protocol, agenda, leftovers?)
* Logo Contest results, elaborating the winner concept
* Forming a Workflow group
* Set up a Trac for issue tracking
* Getting the \'non-coders' community to help
* Possible GUI-Integration: Display a frame?
Summary
^^^^^^^
* link:2009-01-14.html[Summary (written by raffa)]
10 Dec 2008
-----------
Topics
~~~~~~
* organisational (protocol, agenda, leftovers?)
* Logo Contest progress, discussion about how to do the pre selection
* rename "EDL" into "Sequence"
* link:DesignProcess/TimelineSequenceOutput[Project, Timeline(s), Sequence(s) and Output]
* GUI/Proc Interface and collaboration
* Threading questions regarding gtk-main and main()
Summary
^^^^^^^
* link:2008-12-10.html[Summary (written by Teld)]
12 Nov 2008
-----------
Topics
~~~~~~
* organisational (protocol, agenda, leftovers?)
* Logo Contest progress
* uWiki Progress
* Builddrone, any helpers?
* Platform and lib dependency status
* Discussion about foreseeable problems with Gui Plugins
* Include dir and namespaces
* build system, plugin tree
Summary
^^^^^^^
* link:2008-11-12.html[Transcript (supplemented by Ichthyo in 2011)]
10 Oct 2008
-----------
Topics
~~~~~~
* organisational (protocol, agenda...)
* Lumiera Logo
* Design Process: Go through Ideas and Drafts
- Mistakes to avoid
- Tag clouds
- Lumiera Architecture
- EDLs as meta-clips
- Builder
- Lua scripting language
- Time handling
- Interface naming
* Config System (1st version landed)
* Use of Namespaces
* Interface Definition Language
Summary
^^^^^^^
* link:2008-10-10.html[Summary (written by Ichthyo)]
4 Sep 2008
----------
Topics
~~~~~~
* boilerplate (protocol, agenda, ...)
* Plugin Descriptors
* Mailinglist, setup, administration
* Lumiera-work Mailinglist (for TODOs)
* About Website transition plan
Summary
^^^^^^^
_(missing)_
7 Aug 2008
----------
Topics
~~~~~~
* Go through Ideas and Drafts in design process
* Config System (in the works)
* Scripting language plans
* GUI brainstorming page
Summary
^^^^^^^
_(missing)_
6 Jul 2008
----------
Topics
~~~~~~
* boilerplate (protocol, what's left from last meeting)
* Introduction of Lumiera
* handling of config errors.
* scripting language.
* editing on small devices (eeePC)
* uWiki.
* 'musical' timeline in bars and beats
Summary
^^^^^^^
* link:2008-07-06.html[Summary (written by Teld)]
6 Jun 2008
----------
Topics
~~~~~~
* boilerplate (protocol, what's left from last meeting)
* Go through Ideas and Drafts in design process
- processing node interface
- Placement concept
Summary
^^^^^^^
* link:2008-06-06.html[Summary (written by ichthyo)]
8 May 2008
----------
Topics
~~~~~~
* boilerplate (protocol, what's left from last meeting)
* Go through Ideas and Drafts in design process
* Website infrastructure
Summary
^^^^^^^
* link:2008-05-08.html[Summary (written by cehteh)]
3 Apr 2008
----------
Topics
~~~~~~
* boilerplate (protocol, what's left from last meeting)
* Go through Ideas and Drafts in design process
* Call for volunteers (which tasks are to do, how can we interest people) for wiki/todo.html
* Project announcements/registration (freshmeat, gnu, etc...) any takers?
* Informal talk about the GUI (We have some people who want to contribute now, what's going on, how can they start)
* froscon application (linux multimedia booth?)
* timecode-metadata discussion: convert to arithmetic time type early or late?
Summary
^^^^^^^
* link:2008-04-03.html[Summary (written by sakalli)]
6 Mar 2008
----------
Topics
~~~~~~
* boilerplate (protocol, whats left from last meeting, next meeting)
* Go through Ideas and Drafts in design process
* "Call for design" for some features relevant for the whole Application:
- parameters and automation
- render node interface
- handling of compound sessions
- session storage interface in general
* Project name
* Project Server, setup, organization, administration
* GPL3 pros cons, license rationale
Summary
^^^^^^^
* link:2008-03-06.html[Summary (written by cehteh)]
1 Feb 2008
----------
Topics
~~~~~~
* Who will write the protocol below
* Discuss the open points in link:Lumiera/DesignProcess[] do we need this formalism?
* Who works on what, what are the short term goals, what tasks are open
* State of the new-name search for the was-intended-as-cinelerra-3 project (no name discussion, just how we will progress!)
* Introduce the development model
* How to handle the regular meetings in future (day, time, where to put summaries)
Summary
^^^^^^^
* link:2008-02-01.html[Summary (written by Ichthyo)]

2
doc/devel/report.txt
View file

@ -1,7 +1,7 @@
Statistics and Reports
======================
//Menu: label Statisticts
//Menu: label Statistics
++++++++++++++++++++++++++++++++++++++

225
doc/devel/rfc/ApplicationInstall.txt Normal file
View file

@ -0,0 +1,225 @@
ApplicationInstall
==================
// please don't remove the //word: comments
[grid="all"]
`------------`-----------------------
*State* _Draft_
*Date* _Di 11 Jan 2011 17:00:55 CET_
*Proposed by* Ichthyostega <prg@ichthyostega.de>
-------------------------------------
[abstract]
*********************************************************************************
Lumiera should be a _freely relocatable_ application bundle.
Relying only on the relative folder structure within this bundle, the application
will be fully functional at any location, provided that the external library
dependencies are resolvable using the standard mechanisms of the platform.
The setup mechanism must be obvious, self-explanatory and must not rely
on compiled in magic or buildsystem trickery. Yet packaging into a FSH conforming
installation location should be supported by the same mechanisms.
*********************************************************************************
Description
-----------
//description: add a detailed description:
Lumiera is planned to become a large professional application bundle, relying
on several external resources for proper operation. An installed Lumiera
application will be more like Gimp, Blender, OpenOffice or Eclipse,
not like bash, autotools or emcas.
Besides that, it can be expected that Lumiera frequently will be used in a
project or studio like setup, where the application isn't installed, but just
unZIPped / unTARed and used as-is. Thus, it should be sufficient to unpack
the application bundle and point it to the session file and maybe the
media storage.
The Eclipse platform can serve as a model for the setup of an modern
application of that style: It can be just unpacked, and when looking
into the folder structure, the meaning of the parts is obvious, and the
basic bootstrap is controlled by two short text based INI files.
While Lumiera presumably won't get _that_ heavyweight and is clearly
not intended to become a general business application platform like OSGi --
the underlying principles can serve as a point of reference for modern
development standards.
This leads to the following conclusions:
- we need to define a standard folder layout for the bundle
- the application must not rely on any compiled-in absolute paths
- the executable should fetch the directly accompanying shared modules
- all other lib dependencies should be handled by the system mechanisms
- the bootstrap configuration likewise must not be compiled-in
- this configuration must be human readable and clearly exhibit its purpose
- the same system must be able to adapt to a FSH conforming installation layout
Judging from our current planning and the existing codebase, Lumiera
is on a good way in that direction, yet some cleanup needs to be done,
especially removing convenience shortcuts from the early days of development
and catching up with the repair of some traits of sloppyness here and there.
Library resolution
~~~~~~~~~~~~~~~~~~
In former days, it was common habit to compile-in a hard wired absolute
+RPATH+. This can be considered obsolete practice; for example, the Debian
policy forbids doing so. This is the result from numerous maintainability
problems in the past. On the other hand, the GNU linker and other modern
linkers support a relative resolution of shared modules directly accompanying
an specific executable. The Debian policy allows this, if and only if these
shared modules are installed with the same binary package and only used by
this specific executable(s). Together, this is exactly what we need to
solve our requirement.
Thus, the build process enables the new-style DT-tags in the Elf binary
and sets the +DT_RUNPATH+ with an value relative to +$ORIGIN+, which resolves
to the path of the currently executing binary. Moreover, it is _sufficient_
to set this on the initial executable _only,_ because this creates a common
searchpath for all lib resolution events in the scope of that loaded executable.
Besides that, we need to care that our private libraries have a unique +SONAME+,
in this case all starting with the prefix +liblumiera*+. Note moreover that this
new-style +DT_RUNPATH+ indeed _can_ be overridden by an +LD_LIBRARY_PATH+ in the
environment, should there be the need for very special experiments.
Bootstrap location
~~~~~~~~~~~~~~~~~~
Thus, a single relative library folder becomes the only hard wired start
configuration. In our case, the folder +$ORIGIN/modules+ was chosen. The
root of the package then holds all the binaries depending on these common
internal libraries, that is the +lumiera+ executable and any accompanying
special tools. As usual with such large application bundles, these get
only _symlinked_ into the +/usr/bin+ folder on installation.
For sake of clarity, after starting the executable, the _same location_
is used to load the bootstrap configuration. This configuration in turn
defines all further locations like the extended configuration, project
templates, plugin search path, the GUI module to load, the search path
for icons and GUI resources, project templates and similar basics.
Relative paths and the location of the executable
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
According to the general requirements outlined above, most paths should
be given in a relative fashion. Unfortunately there is no portable solution
for self-discovering the currently running executable. But at least there
is a solution for all current major platforms. Under Linux, this information
can be retrieved from the kernel through the +/proc+ filesystem.
Again for sake of clarity, the same token +$ORIGIN+ was chosen to denote
this executable location (note: this is _not_ the current working directory).
Moreover, due to the folder layout detailed above, this coincides with the
root of the application bundle, thus making for a self-explanatory convention.
Besides +$ORIGIN+, these search paths later on likely will contain locations
below the user's home directory, e.g. +~/.lumiera/themes+
Tasks
~~~~~
// List what needs to be done to implement this Proposal:
* identify what impedes such a modern setup procedure ([green]#✔ done#)
* rectify the folder structure created in the build target
directory ([green]#✔ done#)
* build the executables in a way to allow relative resolution of the
internal shared modules ([green]#✔ done#)
* replace the compiled-in path definitions for plugin loading by a
configurable bootstrap ([green]#✔#)
* add an working library implementation for a config loader ([green]#✔ done#)
* add a mechanism for establishing the path of the current execubable. +
This is _non-portable_ ([green]#✔ done#)
* wire the prepared API in the GUI to use this working config loader
for resolving GUI resources ([green]#✔ done#)
* try to extract the path search code from the existing config loader,
or build a new solution based on standard libraries ([green]#✔ done#)
* introduce an output root directory into the buildsystem, allowing
for package builds ([green]#✔#)
* define a _Debian packaging_ as proof-of-concept ([green]#✔ done#)
Discussion
~~~~~~~~~~
Pros
^^^^
* self-contained
* self-explanatory
* based on _best practices_
* conforming with FSH and Debian policy
Cons
^^^^
* requires work
* raises the bar at the implementation side
* requires an bootstrap sequence to be explicitly performed
on application startup
* breaks with some beloved habits of the Unix community
Alternatives
^^^^^^^^^^^^
//alternatives: explain alternatives and tell why they are not viable:
I can think of two alternatives
. dealing with all those problems _later_
. not making an concept, rather sticking to UNIX habits
The first alternative is indeed worth considering, because we're settling
some things to be really implemented way later, which bears some dangers.
But, on the other hand, it is a common practice known from extreme programming
to deliver early and regularly, which effectively means to set up the deploy
path of an application really early in the development cycle. The rationale
is that -- according to general experience -- the deployment always turns
up some very specific problems and constraints, which can be a serious
threat when discovered late in the development process.
The second alternative isn't really applicable IMHO. The original UNIX philosophy
breeds on an academic setup and really excels with small nifty commandline utils
combined by pipes, each specialised to do a single thing very well. These utils
are more like the objects within our implementation. The concept of large
application software bundles and desktop software was always a bit alien
within the classic UNIX environment.
Rationale
---------
//rationale: Give a concise summary why it should be done *this* way:
This RfC can be seen as an commitment to an professional approach and as
clarification: Traditionally, the Unix community hailed a lot of _black magic_
practices like compiled-in installation paths, macro magic, +sed+ and +awk+
trickery, inline code compiled on-the-fly, relying on very specific and
un-obvious behaviour of some build script, configuration via environment
variables and a lot of similar idioms. These practices might be adequate
in a quickly moving Research & Development setup, but turned out to be
not so helpful when it comes to industrial strength development,
as they are known to lead to maintenance problems.
//Conclusion
//----------
//conclusion: When approbate (this proposal becomes a Final)
// write some conclusions about its process:
Comments
--------
//comments: append below
.State -> Draft
There is now a complete implementation of this concept on my ``proc'' branch. +
Moreover, I was able to define an initial Debian packaging for Lumiera on top
of that implementation.
During that work, I had opportunity to visit various areas of the existing codebase,
which reminded me of several small issues, which seem to become unhealthy when lying
around unfixed for such a long time. Probably I'll start a clean-up initiative and
try to bring these points to discussion separately.
So 13 Feb 2011 20:04:00 CET Ichthyostega <prg@ichthyostega.de>
//endof_comments:

200
doc/devel/rfc/ApplicationStructure.txt Normal file
View file

@ -0,0 +1,200 @@
Design Process : Application Structure
======================================
[grid="all"]]
`------------`----------------------
*State* _Dropped_
*Date* _2008-11-05_
*Proposed by* link:ct[]
------------------------------------
Application Structure
---------------------
Here I am going to propose some more refined structure of the application and
its components.
Description
~~~~~~~~~~~
So far we came up with a simplified BACKEND/PROC/GUI structure where each of
this entities defines its own sub subcomponents. We agreed to glue that all
together with some portable versioned interfaces system, but details where not
laid out yet. At the time of this writing the interface system and plugin
loader are reasonable finished to be usable (some small refinements to do). We
recently discussed some details on IRC on how to engage this without a
definitive decision. The topic of this proposal is to make a detailed
description towards how the application components being glued together.
In the discussion mentioned above we concluded that we want a 'lumiera' binary
which in turn loads the optional parts as plugins. There was no consent what
this parts are actually be, except that the GUI should be optional for headless
operation. I suggested to make as much as possible pluginable to make it easier
to validate our interfaces and try different things out.
Now I introduce 'lumiera' here, this will become a new component in
./src/lumiera being the driver application for bootstraping all the rest:
Then our application structure looks somewhat like (please refine):
* the 'lumiera' loader
- commandline handling
- interface & plugin system
- session manager core
- configuration system
- lua scripting
* backend
- file and io handling
- caches
- streams
- threads
- scheduler
* proc
- asset management
- config rules system
- builder
- render graph management
* gui
- timelines
- viewers
- resources
- preferences
- ...
Furthermore the interface&plugin system is flexible enough to provide things
independently of their origin (if it is build in or a plugin/dynamic library).
So deployment (where to link these things) is secondary.
'lumiera' will then be the executable the user starts up, what exactly gets
initialized and booted up is then matter
of configuration and commmandline options (and maybe lua scripting?).
Tasks
^^^^^
* create the 'lumiera' directory
- setup the build system
- move config, plugin and interfaces therein
- lua support can be done later
* write the main() part of the application
- start config system
- parse commandline opts
* librificate all other components (backend, proc gui)
- define their lumiera interfaces
- decide if they shall be statically linked, becoming shared libs or plugins
This are rather distributed tasks, after the 'lumiera' being set up, all other
components have to be adapted to be loadable from it.
Pros
^^^^
* flexible plugin based architecture
- later: loads only things which are necessary for a given task
* very fast startup
* things which cant be used on a given environment can be left out (no gui on
a headless system, no $DISPLAY set)
* inter dependencies between interfaces and plugins are automatically tracked.
Cons
^^^^
Ichthyo raised concerns that this kind of flexibility might attract other
people to write things which are not in our intention and break future design
and compatibility. We need to carefully document and define interfaces that
people don't abuse those!
Alternatives
^^^^^^^^^^^^
We discussed the startup/main() through the GUI as it is currently done, it
would be also possible to produce some more executables (lumigui, luminode,
lumiserver, ....). But I think we agreed that a common loader is the best way
to go.
Rationale
~~~~~~~~~
I just think this is the best way to ensure a enduring design even for future
changes we can not forsee yet.
Comments
--------
We discussed this issue lately on IRC and I got the feeling we pretty much
agreed on it.
* we don't want to build a bunch of specialized executables, rather we build
one core app which pulls up optional parts after parsing the config
* we change the GUI to be loaded via the module/interfaces system
From reading the above text, this proposal seems to capture that. But I am
somewhat unsure if the purpose of this proposal isn't rather to load just a
micro kernel and the pull up components according to configuration. Because I
wouldn't accept such an architecture, and I clearly stated so right at the
beginning of our project. I accepted a very flexible and language neutral
plugin system on the condition the core remains in control, stays
''reasonable'' monolithic and componentization doesn't handicap us in creating
an architecture based on abstractions and exploiting the proven design
patterns.
It has that flexibility, yes. But that means not that we have to abuse it in
any way. The main() there and thus the bootstrap of the application is under
our tight control, if we want to reject scriptable/highly configurable
bootstrapping there then we can just do so. Thats more a social than a
technical decision. I personally don't like if a design is 'nannying' and puts
too much constraints into unforeseen areas. If the computer can do some task
better than we, it shall do it. This still means that I want to stay very much
in control, it should only do some tedious, error-prone managing tasks for me.
For example the interfaces system already tracks inter-dependencies between
plugins and interfaces automatically, without the programmer needs to care or
define anything. The interface system gets it right and we wont need to care
for the order initialization. I added that because I consider such as
absolutely important for plugins which might be supplied by third parties where
we have no control over. But I now realized that we can nicely use that for our
own internal things too. Imo thats some very valuable service.
-- link:ct[] [[DateTime(2008-11-08T06:26:18Z)]]
Some further minor details: We didn't finish the discussion about namespaces on
the last meeting. (I know I still have to write up a proposal showing the two
or three alternatives I see regarding namespace organisation). But probably,
"lumiera::" will be our top level interface namespace and then probably the
lumiera directory will be taken by that. I see no problem also putting some
startup facilities in there, but generally, it shouldn't contain implementation
code, only headers and abstract classes. If that's going to become a problem,
we should consider to use a separate package for the startup, e.g. "src/boot".
Another point is, you need not write a main, because there is already one.
Please have a look at it, especially with regards to the
[wiki:self:../GlobalInitialization global initialisation]. Further, last year
I've investigated boost::program_options and think it's fine. I use it for my
test class runner since then. I don't think there is any reason why we should
bother with parsing options (most config is pulled up from the session). I
don't think we get much program options, maybe something to set a GUI skin.
Moreover, I've written last year a thin wrapper around the commandline and
integrated it with the boost options parser such that user code can receive the
remaining options as a vector of std::strings. Please have a look at
link:http://git.lumiera.org/gitweb?p=LUMIERA;a=blob;f=tests/common/mainsuite.cpp;h=455bfd98effd0b7dbe6597f712a1bdfa35232308;hb=80e1e382f42512ebf2e10a802f77e50327b8fb73[the test class runner main]
for an usage example. I really want our Lumiera main to be clean and expressive
in the way showed there. Probably the most important part of the startup is
pulling up the session core; because of that I think most of the startup
process falls into the realm of the Proc-Layer. Within Proc, I don't want any
significant string manipulations done with C-strings and I don't want raw
arrays when we can use std::vector.
-- link:Ichthyostega[] [[DateTime(2008-11-06T19:28:13Z)]]
I 'dropped' this now because we do it somewhat differently now and I dont want
to document this here :P
-- link:ct[] [[DateTime(2009-02-03T17:28:28Z)]]
''''
Back to link:/documentation/devel/rfc.html[Lumiera Design Process overview]

15
doc/devel/rfc/CCodingStyleGuide.txt
View file

@ -65,16 +65,11 @@ For each `struct namespace_foo_struct` we have following typedefs:
[source,C]
----
typedef struct namespace_foo_struct namespace_foo; // canonical
typename
typedef const namespace_foo * const_NamespaceFoo; // pointer to const
object
typedef namespace_foo* link:NamespaceFoo[]; // canonical
pointer/handle
typedef namespace_foo ** link:NamespaceFoo[]_ref; // when intend to
mutate the handle itself
typedef const namespace_foo ** const_NamespaceFoo_ref; // for const object
handle
typedef struct namespace_foo_struct namespace_foo; // basic struct name
typedef namespace_foo* NamespaceFoo; // canonical object pointer/handle
typedef const namespace_foo * const_NamespaceFoo; // pointer to const object
typedef namespace_foo ** NamespaceFoo_ref; // when intend to mutate the handle itself
typedef const namespace_foo ** const_NamespaceFoo_ref; // same for const object handle
----
Examples:

10
doc/devel/rfc_pending/ClipCatalogingSystem.txt → doc/devel/rfc/ClipCatalogingSystem.txt
View file

@ -3,7 +3,7 @@ Design Process : Clip Cataloging System
[grid="all"]
`------------`-----------------------
*State* _Idea_
*State* _Parked_
*Date* _2008-07-26_
*Proposed by* link:JordanN[]
-------------------------------------
@ -117,4 +117,12 @@ without even having to "drive" the development..
--link:Tree[][[DateTime(2008-08-27T20:38:00NZ)]].
''''
Parked until someone cares
~~~~~~~~~~~~~~~~~~~~~~~~~~
Decided on Developer meeting
Do 14 Apr 2011 02:52:30 CEST Christian Thaeter
Back to link:/documentation/devel/rfc.html[Lumiera Design Process overview]

25
doc/devel/rfc/CodingStyle.txt
View file

@ -52,30 +52,14 @@ Tasks
* footers (or headers) to configure common editors to use this style by default
Pros
^^^^
Cons
^^^^
Alternatives
^^^^^^^^^^^^
Rationale
~~~~~~~~~
Conclusion
----------
we agreed on GNU style
-- link:ct[] [[DateTime(2007-07-03T04:04:22Z)]]
CT:: '2007-07-03 04:04'
@ -88,6 +72,8 @@ think all other reasons will lead us to nowhere!
Although I'm used to a BSD/KNF-like coding style I will try the GNU one. After
all, the wikipedia page mentions no disadvantages of that style :)
MichaelPloujnikov:: '2007-06-27 17:17'
I just proposed K&R because it is widely accepted. Personally, I was never very
fond of K&R style, I always prefered putting opening braces to the left. I
@ -96,5 +82,8 @@ ECLIPSE comes with presets for all this styles :-P ). Anyhow, I can adapt to
most any style. The only thing I really dislike is using tabs (with the
exeption of database DDLs and CSound files, where tab are actually helpful) :)
Ichthyo:: '2007-06-27 20:55'
''''
Back to link:/documentation/devel/rfc.html[Lumiera Design Process overview]

4
doc/devel/rfc/DIR_INFO
View file

@ -1 +1,3 @@
accepted design proposals
RFC pool
all other rfc dirs (sorted by state) link back to here

11
doc/devel/rfc_pending/DelectusShotEvaluator.txt → doc/devel/rfc/DelectusShotEvaluator.txt
View file

@ -1,6 +1,6 @@
[grid="all"]
`------------`-----------------------
*State* _Idea_
*State* _Parked_
*Date* _2008-09-21_
*Proposed by* link:nasa[]
-------------------------------------
@ -385,4 +385,13 @@ very expensive. I'm a sound guy, so I feel your pain...
- link:NicholasSA[] 2009-01-04
Parked
~~~~~~
Decided on Developer meeting, until someone wants to investigate this further.
Do 14 Apr 2011 02:52:30 CEST Christian Thaeter
Back to link:/documentation/devel/rfc.html[Lumiera Design Process overview]

84
doc/devel/rfc/DesignParamAutomation.txt Normal file
View file

@ -0,0 +1,84 @@
[grid="all"]
`------------`-----------------------
*State* _Idea_
*Date* _2008-03-06_
*Proposed by* link:Ichthyostega[]
-------------------------------------
Design the handling of Parameters and Automation
------------------------------------------------
Parameters of Plugin Components and/or Render Nodes play a role at various
levels of the application.
+
Thus it seems reasonable to do a formal requirements analysis and design prior
to coding.
Description
~~~~~~~~~~~
Regarding components directly participating in the render (which may be
implemented by plugins), we distinguish between *configuration* (static) and
*parameters* (dynamic). The point of reference for this distinction is the
render process: a plugin configuration may well be variable in some manner,
e.g. the plugin may provide different flavours of the same algorithm. But this
choice has to be fixed prior to feeding the corresponding plugin asset to the
builder. Contrary to such fixed configuration setup, the _parameters_ are
considered to be _variable_ during the rendering process. They can be changed
on-the-fly from GUI, and they may be automated. Probably, each Render Node will
have at least one such _parameter_ -- namely a bypass switch.
Tasks
^^^^^
* we need to work out an introspection mechanism for parameters
- asses what different types of parameters we need
- find out how much structured parameters will be (do simple values
suffice?)
- define how parameters can be discovered/enumerated
- define a naming scheme for parameters, so they can be addressed
unambiguously
* value parameters have a value range. Work out how to handle this
* parameters may need a specific presentation in the GUI
- linear/logarithmic scale, scale reference
- selecting the right widget
So...
. find out to which extend we need these properties
. find out what parts of the App will have what requirements?
. chose a best fitting implementation based on this information
A closely related issue is the handling of *Automation*. The current draft
calls for an abstract interface "ParamProvider", which just allows the
link:Plugin/RenderComponent[] to pull a current value, without knowing if the
ParamProvider is a GUI widget or an automation data set with interpolation. The
component using the param value should not need to do any interpolation. We
should re-asses and refine this draft as needed. Note: Render Nodes are
stateless; this creates some tricky situations.
Alternatives
^^^^^^^^^^^^
?? (any ideas?)
Rationale
~~~~~~~~~
Comments
--------
Back to link:/documentation/devel/rfc.html[Lumiera Design Process overview]

11
doc/devel/rfc_pending/DesignRenderNodesInterface.txt → doc/devel/rfc/DesignRenderNodesInterface.txt
View file

@ -1,6 +1,6 @@
[grid="all"]
`------------`-----------------------
*State* _Idea_
*State* _Parked_
*Date* _2008-03-06_
*Proposed by* link:Ichthyostega[]
-------------------------------------
@ -62,7 +62,7 @@ the invocation interface
this is intended to be a rather simple "call-style" interface, without much
possibilites to influence the way things are happening. You pull a node and
will find the results in a provided buffer or the cache, but you can't even
change the frame data type type of the result. Besides the node invocation,
change the frame data type type of the result. Besides the node invocation,
functions for collecting statistics will be accessible here too (Probably these
functions will be ''implemented'' in a classic-OO fashion by virtual functions,
but that's another story)
@ -118,4 +118,11 @@ Comments
--------
Parked
~~~~~~
We park this until we have time to revisit the details. It is accepted that we
need to design this interfaces.
Do 14 Apr 2011 03:06:42 CEST Christian Thaeter
Back to link:/documentation/devel/rfc.html[Lumiera Design Process overview]

136
doc/devel/rfc/DeveloperDocumentationStructure.txt Normal file
View file

@ -0,0 +1,136 @@
Developer Documentation Structure
=================================
// please don't remove the //word: comments
[grid="all"]
`------------`-----------------------
*State* _Idea_
*Date* _Mon Aug 2 18:03:25 2010_
*Proposed by* Christian Thaeter <ct@pipapo.org>
-------------------------------------
[abstract]
********************************************************************************
I describe here how to bring the Lumiera Developer Documentation into an simple
hierarchical structure. Previously we accumulated a lot Documentation which
ended in quite a few different places. This should be tidied up.
********************************************************************************
Description
-----------
//description: add a detailed description:
I propose to reorganize the developer documentation in the following way:
* make a 3 (or more, see below) level documentation structure:
1. The entry level becomes the 'Lumiera: The inner Core' document which shall
not go into details but give a hint what everything is made for. This
will be the first introductory doc for new developers.
2. second level are the RFC's which descibe the design as planned on a
general level, not going (except for some example snippets) into
implementation details.
3. the third level is the doxygen documentation which describes what
actually got implemented in detail. This can be further split into
an external reference and a internal part.
We using test-driven-development, our tests are our specifications. This leads
to the idea that ideas, design and intentions for tests should be documented
there too. In a higher level abstract human written form. I propose to use my
pipadoc documentation extractor (that means, writing asciidoc within the code as
special comments) for this.
Tasks
~~~~~
// List what would need to be done to implement this Proposal in a few words:
// * item ...
* Go over the old content of the asciidoced tiddlywikis, integrate it either in
the "Lumiera: The inner Core" document or write single RFC's for them.
* The 'proc' tiddlywiki is a bit special, we need a plan how to integrate this.
Possibly making a own document-dir for this, or refactor it in plenty RFC's.
This is ichthyos decision.
* Decide how to proceed with the UML model
Pros
^^^^
// add just a fact list/enumeration which make this suitable:
Much easier entry to the whole developer documentation. Reading the "Inner Core"
document should be sufficient to get a good idea about the Lumiera design and
layout. All details are linked from there and thus easily findable.
Cons
^^^^
// fact list of the known/considered bad implications:
There are some open ends yet, doxygen for example doesn't integrate nicely, we
possibly can't link to single doxygen entities since these have no permanent
link (to my understanding, to be investigated). Other parts like the UML model
are not yet decided and moving the other existing content over needs some (not
really much) work.
Alternatives
------------
//alternatives: explain alternatives and tell why they are not viable:
Spring 2010 we discussed and decided an overall website and documentation structure.
We could just stick to that.
Rationale
---------
//rationale: Describe why it should be done *this* way:
This approach fits nicely into our overall infrastructure and the way we wanted
to do things. Using git and asciidoc mostly, making the developer documentation
part of the source tree and reasonable easy available/maintainable to
developers.
//Conclusion
//----------
//conclusion: When approbated (this proposal becomes a Final)
// write some conclusions about its process:
Comments
--------
//comments: append below
* The general idea of having three levels, with 'The Inner Core' as entry point,
looks OK for me.
* beyond that -- we had a detailed discussion about the overall website structure,
which includes the documentation. Why should we overthrow these results now and
re-start the discussion? Lets just stick to this agreed on structure!
* especially I don't like the way this proposal tries to squeeze everything into
an completely uniform structure. It is simply not true that the RFCs are just the
second level, and doxygen would cover the 3^rd^ level. Look at the existing
documentation to see why.
- RFCs are a 'kind' of document, not a 'hierarchy level.' Indeed, our existing
RFCs span all three hierarchy levels, and this is OK so and should remain this
way. (And yes, I like the RFCs much and want to retain them)
- RFCs are well suited to topics requiring discussion and agreement by the whole
core developer team. I see no point in 'pseudo-RFC-ing' the individual design
decisions only relevant for an isolated part of the application and without
any potential for discussion.
- similarily, in the TiddlyWiki, besides just working notes (``extended brain'')
you'll find finished text pages belonging to all different levels, from very
high-level conceptual down to explanation of technical details, with
cross references and tags for categorisation (and this will be retained
when asciidocing the content).
* so my conclusion is rather having one overview text, and then the split into
*conceptual* and *technical* documentation, each of which has a separate sub
structure not necessarily congruent to the structure on the other half. RFCs,
UML model and doxygen are just separate and consistent bodies of documentation
and can be referred to from the main documentation. (I agree with the observation
regarding permanent links into doxygen. But I can't imagine there isn't some
existing solution to this problem)
-- link:Ichthyostega[] 2010-10-15
//endof_comments:

271
doc/devel/rfc/EngineInterfaceOverview.txt Normal file
View file

@ -0,0 +1,271 @@
Engine Interface Overview
=========================
:Date: 2010
// please don't remove the //word: comments
[grid="all"]
`------------`-----------------------
*State* _Draft_
*Date* _2010-04-16_
*Proposed by* link:Ichthyostega[]
-------------------------------------
Overview Engine Interface(s)
----------------------------
********************************************************************************
At the Engine Interfaces, Lumiera's Backend and Session get connected and work
together to produce rendered output. This design proposal intends to give an
overview of the connection points and facilities involved, to define some terms
and concepts and to provide a foundation for discussion and working out the
APIs in detail.
********************************************************************************
Participants
~~~~~~~~~~~~
*Render Process*:: represents an ongoing calculation as a whole
*Engine Model*:: encloses the details of the current engine configuration and
wiring
*Dispatcher*:: translates a render process into the (planned) invocation of
individual nodes
*Scheduler*:: cares for calculations actually to happen, in the right order
and just in time, if at all
*Node*:: abstraction of an processing unit, supports planning by the
dispatcher, allows to pull data, thereby driving the actual calculation.
Render Process
~~~~~~~~~~~~~~
The render process brackets an ongoing calculation as a whole. It is not to be
confused with a operating system process or thread; rather it is a point of
reference for the relevant entities in the GUI and Proc-Layer in need to
connect to such a "rendering", and it holds the specific definitions for this
calculation series. A render process
_corresponds to a single data stream_ to be rendered. Thus, when the play
controller of some timeline in the model is
in _playing_ or _paused_ state, typically multiple corresponding render
processes exist.
* there is an displayer- or output slot, which got allocated on creation
of the process
* the process disposes calculated data frames "into" this slot
* the process can be paused/started and stopped (aborted, halted).
* some processes allow for changing parameters dynamically (e.g. speed,
direction)
* each process has to ensure that the output/display slot gets closed or
released finally
.Process parameters
A process is linked to a single stream data format (a ->
link:StreamTypeSystem.html[stream implementation type]). +
It is configured with _frame quantisation_ and _timings_, and a _model port_
identifier and _channel selector_.
quantisation::
translates time values into frame numbers. (In the most general
case this is a function, connected to the session)
timings::
a definition to translate global model time units in real clock time,
including _alignment_ to an external _time grid_.
model port::
a point in the (high level) model where output can be produced. +
This might be a global pipe in one of the model's timelines, or
it might be a _probe point_.
channel::
within the session and high level model, details of the stream
implementation are abstracted. Typically, a global pipe (master bus
or subgroup) corresponds to a multichannel stream, and each of these
channels might be hooked up to an individual render process
(we have to work out if that's _always the case_ or just under
_some circumstances_)
[NOTE]
===================
While certainly the port and channel definition is fixed, unfortunately the
quantisation and the timings are'nt. The timings may be changed in the middle
of an ongoing render process, due to changed playback speed, shuffling or
requirements forwarded from chase-and-lock synchronisation to an external
source. We still need to discuss if Lumiera is going to support variable
framerates (several media professionals I've talked to were rather positive we
need to support that -- personally I'm still in doubt we do). Variable
framerates force us to determine the frame numbers by an integration over time
from a start position up to the time position in question. The relevant data to
be integrated is located in the session / high-level model; probably we'll then
create an excerpt of this data, but still the less quantisation will be a
function of time. Anyway, it is the render processes job to translate all kinds
of parameter changes into relevant internal API calls to reconfigure the
calculation process to fit.
===================
Engine Model (low-level Model)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
The low level model is a network of interconnected render nodes. It is created
by the build process to embody any configuration, setup and further
parametrisation derived from the high-level description within the session. But
the data structure of this node network is _opaque_ and considered an
implementation detail. It is not intended to be inspected and processed by
outward entities (contrast this to the high-level model within the session,
which provides an extensive discovery API and can be manipulated by model
mutating commands). We just provide a set of _query and information retrieval
functions_ to suit the needs of the calculation process. The engine model is
_not persisted._
* the engine model is partitioned by a _segmentation_ of the time axis.
Individual segments can be hot-swapped.
* the engine has _exit nodes,_ corresponding to the model ports mentioned above
* each exit node provides a stream type definition plus quantisation and
alignment constraints.
Thus, for any pair (port, time) it is possible to figure out a segment and an
exit node to serve this position. The segmentation(s) for multiple ports might
differ. To allow for effective dispatching, the model should provide
convenience functions to translate these informations into frame number ranges.
The mentioned quantisation and alignment constraints stem from the fact that
the underlying media source(s) are typically themselves quantised and the
timings might be manipulated within the processing chain. We might or might not
be able to shift the underlying media source
(it might be a live input or it might be tied to a fixed timecode)
Processing Node
~~~~~~~~~~~~~~~
In this context, a node is a conceptual entity: it is an elementary unit of
processing. It might indeed be a single invocation of a _processor_ (plugin or
similar processing function), or it might be a chain of nodes, a complete
subtree, it might _represent_ a data source (file, external input or peer in
case of distributed rendering), or it might stand for a pipeline implemented in
hardware. The actual decision about these possibilities happened during the
build process and can be configured by rules. Information about these decisions
is retained only insofar it is required for the processing, most of the
detailed type information is discarded after the wiring and configuration step.
As mentioned above, each node serves two distinct purposes, namely to assist
with the planning and dispatching, and to pull data by performing the
calculations.
Nodes can be considered _stateless_ -- pulling a node has no effect outside the
invocation context. While a node _might_ actually be configured to drive a
whole chain or subtree and propagate the pull request
_within_ this tree or chain internally, the node _never propagates a pull
request beyond its realm._ The pull()
call expects to be provided with all prerequisite data, intermediary and output
buffers.
Dispatching Step
~~~~~~~~~~~~~~~~
The dispatcher translates a render process (actually a _calculation stream_ as
part of a render process) into sequences of node invocations,
which then can be analysed further (including planning the invocation of
prerequisites) and scheduled. This mapping is assisted by the engine model API
(to find the right exit node in the right segment), the render process (for
quantisation) and the involved node's invocation API (to find the
prerequisites)
Node Invocation API
~~~~~~~~~~~~~~~~~~~
As nodes are stateless, they need to be embedded into an invocation context in
order to be of any use. The node invocation has two distinct stages and thus
the invocation API can be partitioned in two groups
Planning
^^^^^^^^
During the planning phase, the dispatcher retrieves various informations
necessary to _schedule_ the following pull call. These informations include
* reproducible invocation identifier, usable to label frames for caching
* opaque source identifier (owned by the backed) in case this node
represents a source
* prerequisite nodes
* index (channel) of the prerequisite's output to be fed as input buffer(s)
* number and size of the output buffers required
* additional memory required
* control data frame(s)
Node pull
^^^^^^^^^
* the pull call expects to be provided with all the resources announced during
the planning step
* moreover, the pull call needs to know (or some way to figure out) the time
coordinates
* after retrieving automation, the control flow forwards to the actual
processing function
* there is an result/error code (assuming the scheduler prefers error codes
over exceptions)
''''
Tasks
~~~~~
* find out if we need to support variable framerate
([green]#-> yes, implementation deferred#)
* find out about the exact handling of multichannel data streams ([green]#✔ done#)
* design and prototypical implementation of frame quantisation ([green]#✔ done#)
* design a buffer descriptor ([green]#✔ done#)
* design a buffer designation scheme [red]#TODO#
* expand on the node identification scheme [red]#TODO#
* clarify how control data frames can be addressed [red]#TODO#
Discussion
~~~~~~~~~~
Pros/Cons/Alternatives
^^^^^^^^^^^^^^^^^^^^^^
Currently we're focussing on how to implement _this_ concept, not on
evaluating alternatives. Especially the idea of scheduling individual frame jobs
is a core concept of Lumiera. This RfC tries to bridge from the session model to
an engine based on these concepts. It's the attempt to link two concepts already
defined and decided on....
Rationale
^^^^^^^^^
* allow for optimal resource use and avoid blocking of threads
* shift away complexity from the engine into the builder, which is by far not
so performance critical
* allow to adjust the actual behaviour of the engine in a wide range, based on
actual measurements
* create a code structure able to support the foreseeable extensions (hardware
and distributed rendering) without killing maintainability
//Conclusion
//----------
//conclusion: When approbate (this proposal becomes a Final)
// write some conclusions about its process:
Comments
--------
//comments: append below
.State -> Draft
Requirements and details of the design are sufficiently clear meanwhile.
Ther seems to be not much room for alternative approaches, given our
general planning for the application
Mi 11 Mai 2011 19:27:12 CEST Ichthyostega <prg@ichthyostega.de>
//endof_comments:
''''
Back to link:/documentation/devel/rfc.html[Lumiera Design Process overview]

234
doc/devel/rfc/EngineInterfaceSpec.txt Normal file
View file

@ -0,0 +1,234 @@
Engine Interface Spec
=====================
// please don't remove the //word: comments
[grid="all"]
`------------`-----------------------
*State* _Idea_
*Date* _Mi 11 Mai 2011 17:53:16 CEST_
*Proposed by* Ichthyostega <prg@ichthyostega.de>
-------------------------------------
[abstract]
********************************************************************************
The main service of the Renderengine is to deliver a stream of calculations
bound by timing constraints. The parameters of this delivery can be reconfigured
********************************************************************************
Try to start the integration and overall design of the Lumiera Render Engine.
Point of reference is the functionality other parts of the application are relying on.
Description
-----------
//description: add a detailed description:
The Engine is driven by the Player subsystem and draws on the low-level Model
(Render nodes network) for all local parameters and control data. The goal is
to deliver all the typical playback and rendering operations commonly found
in NLE applications (functional requirements). Moreover, this functionality
shall be delivered in a robust and reliable fashion, while making optimal
use of the available I/O bandwidth and computation power (non-functional
requirements).
Requirements Specification
~~~~~~~~~~~~~~~~~~~~~~~~~~
.functional
- simple calculation stream
- with or without defined end point
- deliver to fixed output buffer(s) with high timing precision
- ability to deliver individual data chunks (e.g. single frames)
- ``free wheeling'' operation for maximum calculation throughput
- throttled operation using idle calculation or bandwidth resources
- streams with regular stepping and arbitrary block sizes
- looping, seamless chaining of calculation streams with ongoing timing
- ability to pause and to change / combine any of the above any time
- ability to abort or change, providing reliable feedback on completion
- ability to relocate (shift in time) parts of an ongoing calculation stream
- support for chase-and-lock synchronisation
.non-functional
- protection against overload and deadlocks
- gracious degradation in case of problems
- maintain a configurable quality-of-service level
- utilise precisely the resources actually available
Functionality description in detail
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-> see the link:EngineInterfaceOverview.html[Engine/Interface overview] for
a description of the involved entities and for definitions for common terms.
Definitions
^^^^^^^^^^^
Calculation Stream::
A series of similar but parametrised calculations,
bound to deliver results in sequence and in accordance to timing constraints
Timed Delivery::
Calculation result data has to be delivered within a fixed time interval
immediately preceding the delivery deadline, so it can be consumed by
an (possibly external) output process without the need for further buffering
Output Slot::
An abstracted facility receiving calculated data; including a scheme
to organise the output buffers, which get handed over to an independent
thread running in parallel, or maybe even to an external process
Operation description
^^^^^^^^^^^^^^^^^^^^^
When *creating a calculation stream*, the _exit node_ and an already opened
_output slot_ are provided, plus the _timing parameters_ (frame duration,
stepping interval length, optional start and/or endtime)
A *free wheeling calculation stream* is created in a similar fashion, just
without timing constraints on the output delivery; i.e. just the output slot
is parametrised differently. This invocation is used to create a ``mixdown''
or ``final render'' to be saved into an output file.
A *background rendering mandate* is created analogous, but _without_ providing
an output slot. Rather, it is expected that the engine will cache the generated
data internally.
When calculation stream definitions are *chained*, the follow-up calculation
stream is expected to be delivered seamlessly after the preceding stream,
without interrupting the output timings.
*Looping* is a special kind of chained calculations, where the same segment
is delivered continuously. But note, the loop boundaries are not necessarily
aligned with the frame spacing or the output timing requirements.
For supporting *chase-and-lock*, the engine needs a mechanism to follow an
externally provided synchronisation goal, _without_ altering the output
delivery timings. Obviously, we need to build in a strategy for handling
this problem (because the solution is bound to be different for different
kinds of media).
The engine can expect the output slot to support *de-clicking* or *flicker
protection* -- yet the engine needs to signal precisely when this is necessary
Quality of service
^^^^^^^^^^^^^^^^^^
The Engine is expected to provide different quality-of-service classes,
which are requested as part of the definition parameters for a calculation stream.
- 'SYNC_PRIORITY' means to keep up to the delivery requirements,
even if this means failing to deliver data altogether.
- 'PERFECT_RESULT' means to deliver data perfect up to the definition,
even if this means violating the timing constraints.
- 'COMPROMISE' allows the engine to take some shortcuts in order to
deliver an roughly satisfactory behaviour. Likely there will be
multiple classes of compromise.
The quality of service is partially implemented directly by the engine
and partially passed on as parameter to the individual node invocations.
For example, the engine might decide to switch down to proxy media, while
actually the node network will perform the actual switch and reconfiguration.
The quality of service could be implemented as a strategy, to be consulted at
various decision points. The above cases would then be just some preconfigured
default strategies.
Reconfiguration and abort
^^^^^^^^^^^^^^^^^^^^^^^^^
There needs to be a separate ``control channel'' to cause various reconfigurations
during an ongoing calculation process. With the exception of the output slot, all
parameters defining an calculation stream might be changed on-the-fly -- including
the possibility to abort calculation altogether.
The engine is _not required_ to react on such change requests immediately or
synchronously. The goal is rather to integrate such changes seamlessly.
Yet we _require_...
- a guarantee that the change request is observed within some tolerance interval
(i.e. we may block waiting on the change to happen, without risking a deadlock)
- a reliable feedback _after_ the change has happened, by invoking a response signal
(functor/callback provided with the change request)
- a guarantee not to proceed with the original setup after this signalling
(read: after receiving this feedback, resources required only by the initial
setup may be deallocated)
Especially note that the following things might be changed in the middle of an
ongoing calculation:
- timing parameters of the calculation stream (frame durations, stepping interval)
- start and end time
- splitting and chaining of calculation streams (e.g introducing jumps)
- adjust the looping boundaries
- toggle _paused_ state
- change the exit node to use for pulling
- relocate the nominal time position of parts of the calculation stream;
especially we expect already calculated and cached data to be re-labeled
- invalidate parts of the (nominal) time axis, forcing recalculation
- abort individual calculation streams without interfering with others.
Tasks
~~~~~
// List what needs to be done to implement this Proposal:
* analyse requirements of the player subsystem ([green]#✔ done#)
* determine further informations needed during calculation [,yellow]#WIP#
* find out about timing requirements and constraints in detaill [red]#TODO#
* define the interface functions in detail [red]#TODO#
* prepare a test fixture with mock-up calculations [red]#TODO#
* implement the invocation backbone with stubbed functionality [red]#TODO#
Discussion
~~~~~~~~~~
Pros
^^^^
// add a fact list/enumeration which make this suitable:
// * foo
// * bar ...
Cons
^^^^
// fact list of the known/considered bad implications:
The requirements placed on life changes are quite high
Alternatives
^^^^^^^^^^^^
//alternatives: explain alternatives and tell why they are not viable:
Rationale
---------
//rationale: Give a concise summary why it should be done *this* way:
The expectations for the playback and render functionality of a NLE are
pretty much set. There isn't much room for reducing functionality.
So the goal for this RfC is to precisely define the inevitable
and break it down into tangible functionality on the implementation level.
//Conclusion
//----------
//conclusion: When approbate (this proposal becomes a Final)
// write some conclusions about its process:
Comments
--------
//comments: append below
Discussed in the May developers meeting. Seems to be basically acceptable.
_Cehteh_ proposed some small adjustments:
- making the _QualityOfService_ rather a strategy to be queried
- treating the rescheduling a bit separate from the other changes, because
that is very common and needs to be performant.
- introducing a separate scheduler/queue for time scheduled tasks, like
with rater soft realtime requirements
So 15 Mai 2011 00:55:24 CEST Ichthyostega <prg@ichthyostega.de>
//endof_comments:
''''
Back to link:/documentation/devel/rfc.html[Lumiera Design Process overview]

8
doc/devel/rfc_pending/FeatureBundle_PluggableModules.txt → doc/devel/rfc/FeatureBundle_PluggableModules.txt
View file

@ -1,6 +1,6 @@
[grid="all"]
`------------`-----------------------
*State* _Idea_
*State* _Parked_
*Date* _2008-09-03_
*Proposed by* link:Ichthyostega[]
-------------------------------------
@ -186,4 +186,10 @@ link:Plugin/Interface[] system self-describing this will also be used to
bootstrap a session on itself (by the serializer which is tightly integrated)
-- link:ct[] [[DateTime(2008-09-04T09:28:37Z)]] 2008-09-04 09:28:37
Parked
~~~~~~
Needs to ne reviewed some time later.
Do 14 Apr 2011 03:06:42 CEST Christian Thaeter
Back to link:/documentation/devel/rfc.html[Lumiera Design Process overview]

208
doc/devel/rfc/GitCommitMessageFormat.txt Normal file
View file

@ -0,0 +1,208 @@
Git Commit Message Format
=========================
// please don't remove the //word: comments
[grid="all"]
`------------`-----------------------
*State* _Final_
*Date* _Fr 31 Aug 2012 03:54:14 CEST_
*Proposed by* Christian Thaeter <ct@pipapo.org>
-------------------------------------
********************************************************************************
.Abstract
This RFC describes the format of commit messages, based on already used
practice.
********************************************************************************
Description
-----------
//description: add a detailed description:
Git commit messages are almost free form text, the only exception is that git
threats the first line in a commit message specially to form the header one
sees in various outputs.
Since we aim to automate some processing we refine the format for commit
messages sightly more.
.General
Commit Messages will be shown in space limited areas (lists on webpages,
emails, tooltips). Unlike source code where we are quite lax about line
lengths commit messages should not exceed 80 characters per line.
The first line is treated as Header as described below, followed by an empty
line and then the Body of the commit message. The Body is optional but
recommended.
This formalized keywords for headers are optional, if in doubt then don't use any.
But if one uses them, then use only one defined here since automatic processing
only knows about these.
.Header
The Header is free form text explaining the purpose of the commit in a few
words. It may start with one uppercased keyword and a colon if appobiate directly
followed by some (optional, defined elsewhere) metadata. This Keywords are
optional but recommended since automatic processing acts upon them. Normal commits
need these keywords and are just free form text.
To be exact, here is a regex matching valid Headers:
^[[:upper:]]+:[[:punct:]]?[[:alnum:][:punct:]]*
Legal headers are for example:
DONE: some feature
FIX:#1234 Segv when starting
RELEASE:1.0 Party
.List of Defined headers:
'WIP:'::
'Work in Progress', commits marked as this may be incomplete and fail
in various ways. For example automatic git-bisecting will skip them.
WIP should take precedence, if for example one needs multiple commits
to fix a bug, then only the final commit is tagged as FIX: and the
leading commits are WIP:
'FIX:'::
Bugfix. The Text should explain what error got fixed. A referenence to
a bug number is not optional and not needed.
'RFC:'::
This commit adds or modifies a RFC but doesn't touch the codebase
(tests/examples are permitted).
'DOC:'::
This commit only alters documentation but no code.
'STASH:'::
When 'git-stash' is not enough (for example one wants to move
unfinished changes changes to another repository or commit as backup and amend
things later). Normally such commits should not remain in a published
repository and not become merged.
'DRAFT:'::
Much like WIP: but will not break compilation and is sane to use. For
example mockups, documentation and skeleton code for new facilities
may use this.
'DONE:'::
Final commit/merge when some noteworthy part is done. The idea here is
that finished things could be easily filtered out for our quarterly reports.
'TODO:'::
This commit adds a documentation, comments or tests about something to
be done. Same rules as NoBug's TODO apply.
'FIXME:'::
This commit adds a documentation, comments or tests about something to
be fixed. Aka a known bug which can not be fixed instantly for some
reason. Same rules as NoBug's FIXME apply.
'PLANNED:'::
This commit adds a documentation, comments or tests about something
planned. Same rules as NoBug's PLANNED apply.
'ALPHA:'::
Notifies the CI system that this commit defines an ALPHA release, the CI
may take this and build an package accordingly
'BETA:'::
Notifies the CI system that this commit defines an BETA release, the CI
may take this and build an package accordingly
'RELEASE:'::
Notifies the CI system that this commit defines an production release, the CI
may take this and build an package accordingly
Note: This list will be updated as need arises
.Body
Adding a body is optional but recommended but for the most simple changes.
A body, if present should separate from the header by one empty line.
It is suggested not to make any lines longer than 80 characters and use
asciidoc formatting. In most cases this means just free form text and maybe
use of bulleted list.
Care should be taken to write clean understandable commit messages. In some
extent they may repeat the comments and documentation from the committed code
in a short form. Think that anyone else reading only the commit message should
understand whats going on.
Tasks
~~~~~
// List what needs to be done to implement this Proposal:
// * first step ([green]#✔ done#)
// * second step [,yellow]#WIP#
Discussion
~~~~~~~~~~
Pros
^^^^
// add a fact list/enumeration which make this suitable:
// * foo
// * bar ...
Cons
^^^^
// fact list of the known/considered bad implications:
Alternatives
^^^^^^^^^^^^
//alternatives: explain alternatives and tell why they are not viable:
Rationale
---------
//rationale: Give a concise summary why it should be done *this* way:
This RFC is based on existing practice, we almost done it this way. some minor
glitches are present in the project history (no colon after keywords,
lowercase keywords). Automatic processing becomes simpler when we formalize
these things in an unambigous way. Commits failing this definitions might
confuse the toolchain (builddrone) but this failures shall not be critical.
//Conclusion
//----------
//conclusion: When approbate (this proposal becomes a Final)
// write some conclusions about its process:
Comments
--------
//comments: append below
.State -> Final
//add reason
We decided on the sept. 2012 devel meeting to finlize this RFC.
Christian Thaeter:: 'Do 13 Sep 2012 03:57:23 CEST' ~<ct@pipapo.org>~
//endof_comments:
''''
Back to link:/documentation/devel/rfc.html[Lumiera Design Process overview]

0
doc/devel/rfc_dropped/GitSubmoduleTransistion.txt → doc/devel/rfc/GitSubmoduleTransistion.txt
View file

35
doc/devel/rfc_pending/LumieraForwardIterator.txt → doc/devel/rfc/LumieraForwardIterator.txt
View file

@ -3,7 +3,7 @@ Design Process: Lumiera Forward Iterator
[grid="all"]
`------------`-----------------------
*State* _Idea_
*State* _Final_
*Date* _2009-11-01_
*Proposed by* link:Ichthyostega[]
-------------------------------------
@ -94,7 +94,8 @@ relies on that interface for discovering session contents. Besides that, we
need more implementation experience.
Some existing iterators or collection-style interfaces should be retro-fitted.
See http://issues.lumiera.org/ticket/349[Ticket #349]. + Moreover, we need to
See http://issues.lumiera.org/ticket/349[Ticket #349]. +
Moreover, we need to
gain experience about mapping this concept down into a flat C-style API.
@ -133,6 +134,36 @@ compromising the clean APIs.
Comments
--------
//comments: append below
Now in use since more then a year, without turning up any serious problems.
The only _minor concern_ I can see is that this concept, as such, doesn't solve
the problem with exposing implementation details of the underlying container on the API.
Similar to STL Iterators, the actual implementation representation is only disguised
behind a 'typedef'. But, generally speaking, this is an inevitable consequence of the
``zero overhead'' abstraction. For the cases when an indirection (via VTable) is feasible,
I've created the 'IterSource' template, which sticks to this Lumiera Forward Iterator
concept, but provides an opaque frontend, allowing to decouple completely from the
actual implementation. Besides that, over time I've written several standard adapters
for the most common STL containers, plus Map, key and value extractors.
Ichthyostega:: 'Sa 16 Apr 2011 00:20:13 CEST'
minor change: removed support for post-increment. It doesn't fit with the concept
and caused serious problems in practice. A correct implementation of post-increment
would require a ``deep copy'' of any underlying data structures.
Ichthyostega:: 'Sa 07 Jan 2012 21:49:09 CET' ~<prg@ichthyostega.de>~
//endof_comments:
Final
~~~~~
Accepted on developer meeting
Do 14 Apr 2011 02:52:30 CEST Christian Thaeter
Back to link:/documentation/devel/rfc.html[Lumiera Design Process overview]

88
doc/devel/rfc/MakeSconsTheOfficialBuildSystem.txt Normal file
View file

@ -0,0 +1,88 @@
Make Scons the official build System
====================================
// please don't remove the //word: comments
[grid="all"]
`------------`-----------------------
*State* _Final_
*Date* _Mi 11 Jan 2012 21:45:58 CET_
*Proposed by* Christian Thaeter <ct@pipapo.org>
-------------------------------------
********************************************************************************
.Abstract
_Bless SCons the default build system for Lumiera._
********************************************************************************
Description
-----------
//description: add a detailed description:
So far we using autotools and scons in parallel. Over time the need arose to have one
reliable supported build system. This shall be SCons.
Tasks
~~~~~
// List what needs to be done to implement this Proposal:
// * first step ([green]#✔ done#)
// * second step [,yellow]#WIP#
Nothing to do except for releases SCons _must be working_ and all non functional
build systems will be stripped on the release branches.
////
Discussion
~~~~~~~~~~
Pros
^^^^
// add a fact list/enumeration which make this suitable:
// * foo
// * bar ...
Cons
^^^^
// fact list of the known/considered bad implications:
Alternatives
^^^^^^^^^^^^
//alternatives: explain alternatives and tell why they are not viable:
Rationale
---------
//rationale: Give a concise summary why it should be done *this* way:
////
Conclusion
----------
//conclusion: When approbate (this proposal becomes a Final)
// write some conclusions about its process:
result of discussions and evaluation during the last years
Comments
--------
//comments: append below
.State -> Final
//add reason
Decided on the December 2011 Developer meeting.
Christian Thaeter:: 'Wed 11 Jan 2012 22:28:36 CET' ~<ct@pipapo.org>~
//endof_comments:
''''
Back to link:/documentation/devel/rfc.html[Lumiera Design Process overview]

12
doc/devel/rfc_pending/MarbleMode.txt → doc/devel/rfc/MarbleMode.txt
View file

@ -1,6 +1,6 @@
[grid="all"]
`------------`-----------------------
*State* _Idea_
*State* _Final_
*Date* _2008-10-10_
*Proposed by* link:Ichthyostega[]
-------------------------------------
@ -133,5 +133,15 @@ should actively support both working styles.
Comments
--------
Final
~~~~~
Everyone likes this and we want to have this. But this is rather a concept
which needs a lot more discussion for the implementation. Further details may
follow where these thingsare worked out.
Do 14 Apr 2011 03:06:42 CEST Christian Thaeter
Back to link:/documentation/devel/rfc.html[Lumiera Design Process overview]

427
doc/devel/rfc/MistakestoAvoid.txt Normal file
View file

@ -0,0 +1,427 @@
Design Process : Mistakes to avoid
==================================
[grid="all"]
`------------`-----------------------
*State* _Dropped_
*Date* _2008-04-21_
*Proposed by* link:rick_777[]
-------------------------------------
Mistakes to avoid in the Lumiera design
---------------------------------------
As a multimedia user and experienced programmer, I've found various flaws
present in open source Non Linear Video editors. Here I will list the problems
and their proposed (or mandatory) solutions. Please forgive me if some of the
ideas here have already been approved, I wrote this text before reaching this
wiki.
Description
~~~~~~~~~~~
As a multimedia user and experienced programmer, I've found the following flaws
present in open source Non Linear Video editors (your mileage may vary) :
. Frequent crashes (which most of the time make you lose your work)
. Reinventing the wheel for every new project
. Lack of a user-friendly (and extensible) UI
. Lack of support for certain video formats or codecs
. Lack of documentation
. Lack of cross-platform support
. Dependency on scripted languages like Python, which make installation a mess
I will expand on the problems and their proposed (or mandatory) solutions.
1. Frequent crashes
~~~~~~~~~~~~~~~~~~~
[grid="all"]
`------------`------------------------------------------------------
*Problem* Frequent Crashes and unsaved work.
*Severity* CRITICAL.
*Solution* Isolating the UI from the rendering and data handling (also
improves the extensibility)
*Required* Yes
*Workarounds* Auto-save (however it's not a real solution for the problem)
--------------------------------------------------------------------
Working with multimedia (video / audio) editing is a magnet for segfaults
(crashes) due to the handling of pointers and compression algorithms. A bug in
a plugin (like in Audacity's low-pass filter) will crash and you suddenly
realize you lost your work - unless you have an auto-save feature, but that
doesn't go to the root of the problem.
My proposal is to move the low-level handling of video to a separate process,
which then will do the processing - if it crashes, the UI will only report an
error with a dialog (i.e. "the process crashed. Try again?"), but you work will
stay safe. I'm not sure of the implementation difficulties that arise from
having a shared memory buffer for rendering / processing, but one thing is
certain: Whenever you move the cursor or rewind a part of a clip in your
resources, the application isn't supposed to crash. Just moving the cursor
isn't a time-critical task, so perhaps we can use temporary files for this.
It's safer if you're not doing the final rendering.
Comments
^^^^^^^^
I am not sure yet about separating things into processes, generally it is clear
that this would be more robust but there are some performance impacts and
programming problems (massisve amounts of data in shared memory). But most
importantly, when a subprocess gets a job and crashes on it, it won't complete
the job, we don't have a choice except gracefully abort it. From a user
perspective "It doesn't work!" there is no much difference to a complete crash.
Well and yes we aim to make it crash proof rather, crashes a bugs and have to
be fixed, point.
Lumiera will never ever loose work, we don't plan to make a project file,
autosafe way. Lumiera will keep projects in an internal database like format
which consists of a Dumpfile and a contingous written logfile. After a
crash/powerdown whatever, this log just gets replayed. The advantages are
countless, imagine persistent, selective undo and so on. Any other format
(cinelerra2 XML, MXF, ...) will be realized by importer/exporter plugins.
-- link:ct[] [[DateTime(2008-04-21T11:27:23Z)]]
2. Reinventing the wheel for every new project
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
[grid="all"]
`------------`------------------------------------------------------
*Problem* Various projects compete and reinvent the wheel
*Severity* Serious (Slows down development time. A lot)
*Solution* Multi-tier design, turn the data handling into a backend and use
whatever UI you prefer
*Required* Yes. Better now that the project hasn't started
---------------------------------------------------------------------
Imagine the Linux kernel was tied to the window manager. You would have to
stick with KDE or GNOME and you couldn't improve it! Fortunately it's not like
that for Linux, but it is for some projects. If you want to change the user
interface from QT to wxWidgets or GTK you'll need to rewrite every module.
If you separate the UI from the project handling engine, you can simply fork
the project and change the UI to one that supports skinning, without having to
do the complicated video-processing stuff.
Separating the processes has an equivalent for web programming, it's called
"separation of concerns", or multi-tier design. When you suddenly change the
database engine, you don't need to change the whole program, just the database
module. Same goes for changing the UI from HTML to XML or Flash. If they're
separate modules that only communicate through a clearly defined API.
Example case 1: The Code::Blocks IDE. The compiling engine supports various
compilers, and the engine itself is only a plugin for the main editor. If the
compiler crashes, you only get an error, but the IDE doesn't crash (unless it's
the UI part that's doing something wrong).
Example case 2: Chessmaster. The user interface and speech synthesis stuff only
call the chess engine, called "theking.exe". Linux chess games also depend on
an engine to do the thinking.
So I suggest to split the project into four separate tiers (not necessarily
processes):
. User interface - communicates with the "project" tier, handles the user
events and does the calls.
. The project tier - the main part of the video editor. This one invokes the
renderer and decides which effects to apply, saving them as mere parameters
for later processing. It also tells you where the current pointer for the
track view is. Also calls the rendering engine for the current frame, or for
previews of a certain special effect. Note that if this process keeps running
even if the GUI crashes, later we can restart the GUI and keep working.
. The rendering engine - This one must be a separate process for the reasons
stated in problem #1. This also gives us the advantage that it can work on
the background while we keep working on the project (after all the project is
just a set of data stating which effects to apply to which tracks, and which
files are used for the tracks) - instead of just having a window saying
"Rendering, please wait". Even Adobe Premiere Pro suffered from this problem.
This means that if we put enough effort, we can surpass commercial software
in certain areas. Note that the rendering engine uses the same API than the
project tier, as it works on a copy of the project when doing the final
rendering.
. The video processing wrapper, which has interfaces for different video
processing toolkits (DirectX, GStreamer, etc). This also makes the project
cross-platform. Tiers 1 and 2 can go in one process, and the 3 and 4 in
another (this would make tier 2 a library which defines a C++ Class, and tier
4 would also be a library which is used by the rendering engine).
By separating the tiers, these can later become their own projects and overall
the community would receive great benefits.
Comments
^^^^^^^^
Please look at our design drafts, things will be separated (little different
than you describe here). We reuse things which are benefitful (gavl, ffmpeg,
..) but we are also aware that we reinvent the wheel for some things by
intention. Lumieras goal is not just to glue some existing libraries together
under a new gui, there are already a lot projects trying this way. We rather
aim for a ''Professional'' high performance Video editing solution which does
some things in a different (maybe more complex) way. We do not use existing
frameworks like MLT or gstreamer because we believe that these do not fit our
goals (gstreamer will be supported through plugins). We do not produce yet
another multimedia framework library (this only happen by coincidence) to be
used by others.
-- link:ct[] [[DateTime(2008-04-21T11:27:23Z)]]
3. Lack of a user-friendly and extensible UI.
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
[grid="all"]
`------------`------------------------------------------------------
*Problem* Often, editors provide a very poor and buggy interface.
Examples: Jahshaka doesn't even provide tooltips for the various tools, and
the documentation is poor; In Cinelerra I've noticed some bugs when using the
open dialog, I'd rather have the KDE one, thanks.
*Severity* From Annoying to Serious.
*Solution 1* Use a library that allows you to use different widget
libraries, like wxWidgets.
*Required* Recommended, but not obligatory.
*Solution 2* Write different user interfaces, but they'd be hard to maintain.
*Required*, No.
---------------------------------------------------------------------
This problem is complicated, we need a good framework for handling the tracks.
Perhaps this could become a separate project. Ideas are welcome.
Comments
^^^^^^^^
Joel started working on a GUI recently and making good progress. The UI should
finally be quite flexible as it mostly provides a skeletion where plugins
render to. We have quite a lot ideas about the UI and user input is welcome.
The UI is currently the most separate tier in the design, i'd like to make it a
plugin itself which is loaded when lumiera is started in a gui mode, but it is
to early to say how exactlly it will be integrated, except that we all agree
that GUI is optional and Lumiera can also run headless, script driven.
-- link:ct[] [[DateTime(2008-04-21T11:27:23Z)]]
4. Lack of support for certain video formats or codecs
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
[grid="all"]
`------------`------------------------------------------------------
*Problem* Lack of support for certain video formats or codecs.
*Severity* Critical.
*Workarounds* 1. Give a help page for the user to do his own conversion, but
this is very inelegant, annoying, and a waste of time. 2. Provide conversion
on the fly, and keep a separate "preprocessed" copy of the imported clip in a
separate directory. This is a nice middle ground, IMHO.
*Solution* Use a wrapper library as stated in problem # 2, having a
plugin-based design is recommended.
*Required* Yes.
---------------------------------------------------------------------
Some editors like Cinelerra are hardwired into using one format, or have a
phobia to certain formats / codecs (i.e. DivX AVI's). If we separate the
project editing engine from the video handling libraries, we can use unknown
formats by simply providing an input/output plugin. This would allows us to use
files encoded with lossless codecs like
http://lags.leetcode.net/codec.html[Lagarith]. This also provides forward
compatibility for future formats.
Comments
^^^^^^^^
Lumiera is a video editor we don't care (*cough*, not really true) about video
formats. Everything which comes In and goes Out is defined in plugins which
handle video formats. We currently decided to use 'gavl' because it is a nice
small library which does exactly what we want. Later on gstreamer and other
such kinds of decoder/encoder/processing-pipe libs will be realized.
-- link:ct[] [[DateTime(2008-04-21T11:27:23Z)]]
5. Lack of documentation
~~~~~~~~~~~~~~~~~~~~~~~~
[grid="all"]
`------------`------------------------------------------------------
*Problem* Some video editors have very poor documentation (and that's an
understatement *cough* Jahshaka *cough* )
*Severity* Critical.
*Solution* Have a team for the documentation.
*Required* Yes.
---------------------------------------------------------------------
Nuff said.
Comments
^^^^^^^^
Quote from Ohloh.net: (http://www.ohloh.net/projects/lumiera)[]
------------------------------------------------------------
Extremely well-commented source code
Lumiera is written mostly in C++. Across all C++ projects on Ohloh, 23% of all
source code lines are comments. For Lumiera, this figure is 46%. This very
impressive number of comments puts Lumiera among the best 10% of all C++
projects on Ohloh.
------------------------------------------------------------
Nuff saied... Oh well, about user docs we like to get that impressive ratings
there too, any helpers?
-- link:ct[] [[DateTime(2008-04-21T11:27:23Z)]]
6. Lack of cross-platform support
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
[grid="all"]
`------------`------------------------------------------------------
*Problem* Where's my Windows version?
*Severity* Blocker
*Solution* Use a cross-platform toolkit for the UI.
*Required* Depends, do you plan to make it Cross-Platform?
--------------------------------------------------------------------
A good example for this is the Code::Blocks IDE, which was thought of being
cross-platform from the beginning. Curiously, at first the project was
Windows-only, and its only F/OSS alternative was Dev-C++ from Bloodshed (eew).
Otherwise you'd have to stick with proprietary applications like Visual C++.
In Linux there were various IDE's, but they were Linux-only. Since Code::Blocks
uses a cross-platform toolkit (wxWidgets), it can be compiled either in Windows
and Linux. There are RPM's for various distros now that the first public
version (8.02) got out. I've heard that QT is also cross-platform, but I
haven't tried it yet.
Of course - if you separate the UI from the project engine, someone could make
his own Windows UI for the project. Now what needs to be taken care of, is that
the rendering libraries are cross-platform too.
Comments
^^^^^^^^
We refuse to make it cross platform intentionally. Most things are written
portable, POSIX compatible, some might need platform specific fixes. But our
target is primary Linux (because thats what we use) secondary any other Free OS
(hopefully we find some testers/maintainers for that). Lumiera ''might'' run on
OSX and patches will be accepted, but it is not a free platform so we don't
care by ourself. Windows due its diffrent system interfaces will be hard to
port, if someone wants to do that, have fun, we will accept patches to, but we
do not support it in *any* way by ourself.
-- link:ct[] [[DateTime(2008-04-21T11:27:23Z)]]
7. Dependency on scripted languages like Python, which make installation a mess
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
[grid="all"]
`------------`------------------------------------------------------
*Problem* Installation can be a mess if we depend on scripted languages.
*Severity* Annoying, the end user might just conform with another project
that "just works".
*Solution* Make it in C++ or other easily-compilable language.
*Required* VERY recommended.
---------------------------------------------------------------------
I've had to install several packages for my distro (whose repository is not as
large as others like Ubuntu's) from source. Some of them depend on very
esoteric scripted languages which I also need to install. And then the
libraries, etc. My suggestion is to free the end user from this burden, and
work on a common language, like C++.
Comments
^^^^^^^^
At some point a scripting language ''will'' be required, yet to drive the
testsuite, make headless rendering work and so on. We need to provide
installation instructions and/or even bundle this language with Lumiera. This
will likely become a small embedded language like Lua or some kind of forth (or
maybe some scheme?) it should not depend on strange modules which are not part
of the core scripting language distribution (or we shall provide them too),
needs to be worked out.
-- link:ct[] [[DateTime(2008-04-21T11:27:23Z)]]
Author's comments
^^^^^^^^^^^^^^^^^
Some of the measures stated in this document are optional, but separating the
processes for the rendering engine, editor and User Interface are the optimal
solution and required to avoid common problems.
Discussion
----------
Mostly we agree with the general statements in this Design Entry. But there are
some points which don't stand the test of a detailed technical discussion. For
example, you simply can't state it's a 'mistake' not to write code which
similarly runs on windows and *nix. Well. You could try to write it in Java.
See my point? While today it's quite feasible to write office stuff or banking
applications in a cross-platform manner, a video editor still is a different
kind of a beast.
A similar argumentation holds true for the question, wether or not to use
separate processes and IPC. While it certainly is a good idea to have the X
server or a database running in a separate process, the situation is really
quite different for editing video. Hopefully it's clear why.
Could you please rework this Design Entry in a way that we can finalize
(accept) it?
* Please remove the section about windows
* Please separate out things needing technical discussion and are not just
"mistakes", thus retaining only the big picture statements (on which we all
agree)
* How to secure the application against crashes
* If it is viable/desirable to run the gui in a separate process really needs
in-depth technical discussion (create a new Design Entry for this)
* How to deal with the dependencies problem in combination with
plugins/extensions and script languages
-- link:Ichthyostega[] [[DateTime(2008-10-05T01:51:50Z)]]
Conclusion
----------
The October.2008 dev meeting decided to 'drop' this design proposal as is.
Basically, this text just tells us "to make Lumiera good", and especially it
contains a mixture of topics
* We fully agree to 80% of the statements made there, but we think those
statements are so very basic and self-evident as to be considered off-topic
here. We are aware of the recurring problems with open source video editing.
That's why we are here.
* The proposal draws conclusions on two technically substantial points, at
which we don't agree. And it fails to provide sufficient (technically sound)
arguments to prove these statements.
While it is certainly 'desirable' to be cross-platform as much as possible and
especially '''target Microsoft Windows''', we don't see much possibilities with
today's mainstream technology to build an application which is as
technologically demanding as a video editor is. We would end up developing two
or even three sister applications, or we are forced to sacrifice performance
for portability. When put up to face such options, we have a clear preference
to concentrate on a really free and open platform.
While it is certainly 'desirable' to make the application as robust as
possible, we don't see how '''using multiple separate processes''' could help
us with this goal ''without creating major scalability or performance
problems'' due to the use of shared memory. And, yet more important: we don't
share the basic assumption made in the proposal, namely that video processing
is inherently dangerous. We think the basic algorithms involved are
sufficiently well-known and understandable to implement them in a sound manner.
''''
Back to link:/documentation/devel/rfc.html[Lumiera Design Process overview]

10
doc/devel/rfc_pending/NormalizedDeviceCoordinates.txt → doc/devel/rfc/NormalizedDeviceCoordinates.txt
View file

@ -1,6 +1,6 @@
[grid="all"]
`------------`-----------------------
*State* _Idea_
*State* _Parked_
*Date* _2009-01-14_
*Proposed by* link:ct[]
-------------------------------------
@ -101,4 +101,12 @@ framerate" and a fixed resolution in pixels. But we certainly can do better.
-- Ichthyostega 18:09:50
Parked
~~~~~~
deferred for later, generally accepted.
Do 14 Apr 2011 03:06:42 CEST Christian Thaeter
Back to link:/documentation/devel/rfc.html[Lumiera Design Process overview]

161
doc/devel/rfc/OfficialAssemblyLanguage.txt Normal file
View file

@ -0,0 +1,161 @@
Design Process : Official Assembly Language
===========================================
[grid="all"]
`------------`-----------------------
*State* _Dropped_
*Date* _2008-08-01_
*Proposed by* link:PercivalTiglao[]
-------------------------------------
Official Assembly Language
--------------------------
I describe here an optimization that might have to be be taken into account at
the design level. At very least, we should design our code with
auto-vectorization in mind. At the most, we can choose to manually write parts
of our code in assembly language and manually vectorize it using x86 SSE
Instructions or !PowerPC !AltiVec instructions. By keeping these instructions
in mind, we can easily achieve a large increase in speed.
Description
~~~~~~~~~~~
While the C / C++ core should be designed efficiently and as portable as
possible, nominating an official assembly language or an official platform can
create new routes for optimization. For example, the x86 SSE instruction set
can add / subtract 16 bytes in parallel (interpreted as 8-bit, 16-bit, 32-bit,
or 64-bit integers, or 32-bit/64-bit floats), with some instructions supporting
masks, blending, dot products, and other various instructions specifically
designed for media processing. While the specific assembly level optimizations
should be ignored for now, structuring our code in such a way to encourage a
style of programming suitable for SSE Optimization would make Lumiera
significantly faster in the long run. At very least, we should structure our
innermost loop in such a way that it is suitable for gcc's auto-vectorization.
The problem is that we will be splitting up our code. Bugs may appear on some
platforms where assembly-specific commands are, or perhaps the C/C++ code would
have bugs that the assembly code does not. We will be maintaining one more
codebase for the same set of code. Remember though, we don't have to do
assembly language now, we just leave enough room in the design to add
assembly-level libraries somewhere in our code.
Tasks
~~~~~
* Choose an "Official" assembly language / platform.
* Review the SIMD instructions avaliable for that assembly language.
* For example, the Pentium 2 supports MMX instructions. Pentium 3 supports MMX
and SSE Instructions. Early Pentium4s support MMX, SSE, and SSE2
instructions. Core Duo supports upto SSE4 instructions. AMD announced SSE5
instructions to come in 2009.
* Consider SIMD instructions while designing the Render Nodes and Effects
architecture.
* Write the whole application in C/C++ / Lua while leaving sections to optimize
in assembly later. (Probably simple tasks or a library written in C)
* Rewrite these sections in Assembly using only instructions we agreed upon.
Pros
~~~~
Assuming we go all the way with an official assembly language / platform...
* Significantly faster render and previews. (Even when using a high-level
library like http://www.pixelglow.com/macstl/valarray/[macstl valarray], we
can get 3.6x -- 16.2x the speed in our inner loop. We can probably expect
greater if we hand-optimize the assembly)
Cons
~~~~
* Earlier architectures of that family will be significantly slower or
unsupported
* Other architectures will rely on C / C++ port instead of optimized assembly
* Redundant Code
Alternatives
^^^^^^^^^^^^
* We only consider auto-vectorization -- GCC is attempting to convert trivial
loops into common SSE patterns. Newer or Higher level instructions may not be
supported by GCC. This is turned on
http://gcc.gnu.org/projects/tree-ssa/vectorization.html[in GCC4.3 with
specific compiler flags]
* We can consider assembly but we don't officially support it -- We leave the
holes there for people to patch up later. Unofficial ports may come up, and
maybe a few years down the line we can reconsider assembly and start to
reimplement it down the road.
* Find a SIMD library for C/C++ -- Intel's ICC and
http://gcc.gnu.org/onlinedocs/gcc-3.4.6/gcc/Vector-Extensions.html[GCC] both
have non-standard extensions to C that roughly translate to these
instructions. There is also the
http://www.pixelglow.com/macstl/valarray/[macstl valarray library] mentioned
earlier. Depending on the library, the extensions can be platform specific.
* Write in a language suitable for auto-vectorization -- Maybe there exists
some vector-based languages? Fortran might be one, but I don't really know.
Rationale
~~~~~~~~~
I think this is one of those few cases where the design can evolve in a way
that makes this kind of optimization impossible. As long as we try to keep this
optimization avaliable in the future, then we should be good.
Comments
--------
* I have to admit that I don't know too much about SSE instructions aside from
the fact that they can operate on 128-bits at once in parallel and there are
some cache tricks involved when using them. (you can move data in from memory
without bringing in the whole cache line). Nonetheless, keeping these
assembly level instructions in mind will ease optimization of this Video
Editor. Some of the instructions are high-level enough that they may effect
design decisions. Considering them now while we are still in early stages of
development might prove to be advantagous. Optimize early? Definitely not.
However, if we don't consider this means of optimization, we may design
ourselves into a situation where this kind of optimization becomes
impossible.
* I don't think we should change any major design decisions to allow for
vectorization. At most, we design a utility library that can be easily
optimized using SIMD instructions. Render Nodes and Effects can use this
library. When this library is optimized, then all Render Nodes and Effects
can be optimized as well. -- link:PercivalTiglao[]
[[DateTime(2008-08-01T16:12:11Z)]]
* Uhm, the Lumiera core (backend, proc, gui) doesn't do any numbercrunching.
This is all delegated to plugins (libgavl, effects, encoders). I think we
don't need any highly assembler/vector optimized code in the core (well, lets
see). This plugins and libraries are somewhat out of our scope and thats good
so, the people working on it know better than we how to optimize this stuff.
It might be even worthwile to try if when we leave all vectorization out, if
then the plugins can use the vector registers better and we gain overall
performance!
-- link:ct[] [[DateTime(2008-08-03T02:27:14Z)]]
* Another idea about a probably worthwhile optimization: gcc can instumentate
code for profileing and then do arc profileing and build it a second time
with feedback what it learnd from the profile runs, this mostly affects
branch prediction and can give a reasonable performance boost. If somone
likes challenges, prepare the build system to do this:
. build it with -fprofile-arcs
. profile it by running ''carefully'' selected benchmarks and tests.
. rebuild it again this time with -fbranch-probabilities
. PROFIT
-- link:ct[] [[DateTime(2008-08-03T02:27:14Z)]]
* I've discussed general ideas around, and I agree now that "core Lumiera" is
not the place to think of these kinds of optimizations. So I'll just move
this over to dropped. -- link:PercivalTiglao[]
[[DateTime(2008-08-04T18:33:58Z)]]
''''
Back to link:/documentation/devel/rfc.html[Lumiera Design Process overview]

10
doc/devel/rfc_pending/ProcHighLevelModel.txt → doc/devel/rfc/ProcHighLevelModel.txt
View file

@ -1,6 +1,6 @@
[grid="all"]
`------------`-----------------------
*State* _Draft_
*State* _Final_
*Date* _2008-08-16_
*Proposed by* link:Ichthyostega[]
-------------------------------------
@ -240,4 +240,12 @@ Comments
--------
Final
~~~~~
Description of the Lumiera high level model as is.
Do 14 Apr 2011 03:06:42 CEST Christian Thaeter
Back to link:/documentation/devel/rfc.html[Lumiera Design Process overview]

8
doc/devel/rfc_pending/ProcPlacementMetaphor.txt → doc/devel/rfc/ProcPlacementMetaphor.txt
View file

@ -1,6 +1,6 @@
[grid="all"]
`------------`-----------------------
*State* _Idea_
*State* _Final_
*Date* _2008-03-06_
*Proposed by* link:Ichthyostega[]
-------------------------------------
@ -228,4 +228,10 @@ benchmark in more ways than one, for some time.
. --link:Tree[][[DateTime(2008-08-23T12:54:00NZ)]].
Final
~~~~~
Do 14 Apr 2011 03:06:42 CEST Christian Thaeter
Back to link:/documentation/devel/rfc.html[Lumiera Design Process overview]

240
doc/devel/rfc/RefactorLiblumieraOut.txt Normal file
View file

@ -0,0 +1,240 @@
Refactor Liblumiera Out
=======================
// please don't remove the //word: comments
[grid="all"]
`------------`-----------------------
*State* _Idea_
*Date* _Fr 22 Apr 2011 10:46:50 CEST_
*Proposed by* Christian Thaeter <ct@pipapo.org>
-------------------------------------
[abstract]
********************************************************************************
liblumiera contains alot useful and reuseable code which is already in use by
other projects
********************************************************************************
Description
-----------
//description: add a detailed description:
Over the time we've put some efforts into the liblumiera. I've added
some from my code which predates the lumiera project which I am using
on many other projects. This now caused that I maintain this sources in
different unrelated projects and have to cross merge and update stuff
when I do updates and fixes somewhere. I think its time to factor the
reuseable parts out into a independent library (like glib does for
gtk), in fact I had this plan long ago.
.What parts are eligible for a standalone library
Anything which is something tool alike and useful for other projects and not
tied to Lumiera only. This are the algorithms/datastructures, allocators, tool
macros. Additionally some of the src/common things should be moved into the
library. I give some lists below.
.How to name it
Long time ago my plan was to name it 'ctlib' or 'cehlib' but meanwhile there is
enough code done by others. So I'd propose a more neutral name, still
'lumieralib' or 'lulib' would be approbiate. The only thing we have to account
for is that some parts which are too specific for Lumiera and should not be
integrated into this spinoff need either to stay in a lumiera-internal lib
(src/lib/) as currently or being moved to the respective subsystems using them
(src/backend, src/proc, src/common, ...), so the names should not clash.
.C, C++ ...
For myself I need the C parts, while there is C++ code which interfaces to the
C implementations and also a lot code which does nice C++ things on its own.
This possibly means that we should in fact make 2 packages out of this, one C
and one C++ library (where the C++ part is more than just the wrappers, but
also the tools and tricks which are currently in src/lib/ and reuseable).
.Who maintains it
Despite a spin of I think we don't want to change anything from our current
practice and maintain it by the Lumiera developers. For many parts I feel
responsible for it, but its really a part of the Lumiera codebase, despite
independently useable.
.How to maintain it
We need to decide about build system and documentation system. As build system
we may right start using scons. For documentation the situation is a but
different since some of my code uses pipadoc/asciidoc and other uses doxygen.
.What not to do
Some of the code is currently quite specific to Lumiera while it could be made
more generic. This is *NOT* subject of this RFC we may or may not do such a
refactoring but this RFC and any work resulting from this should only restrict
to simple things like necessary namespace and variable renaming and integration
in the build system.
C Parts
-------
Library
~~~~~~~
What belongs to the library
Containers
^^^^^^^^^^
* cuckoo hashing (cuckoo.c|h)
* linked lists (llist.h slist.h)
* cache lists (mrucache.c|h)
* splay trees (psplay.c|h)
* priority queues (not done yet)
Runtime tools
^^^^^^^^^^^^^
* error handling (error.h error.c) used by the other facilities too
* clib convinience wrapers (safeclib.c|h) needs better name, maybe refactor
into new facilities
Multithreading
^^^^^^^^^^^^^^
* locking, condition variables etc. (condition.c|h (rec)mutex.c|h, rwlock ...)
Memory management
^^^^^^^^^^^^^^^^^
* Memory pools (mpool.c|h)
* Temporary buffers (tmpbuf.c|h)
Metaprogramming
^^^^^^^^^^^^^^^
* preprecessor tools (ppmpl.h) move common preprocessor macros here
* polymorphic call helper for C (vcall.h)
Interface system and module loader
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
except for some hardcoded references to 'lumiera_org' and '.lum' plugin names
this is quite generic, possibly moving this over could be postponed, but might
eventually be done.
From 'src/common'
------
interface.c interfacedescriptor.h interface.h interfaceproxy.cpp
interfaceregistry.c interfaceregistry.h plugin.c plugin_dynlib.c plugin.h
------
The 'config' system could become a candidate too if it ever gets finished and
proved useful, but for the time being its better kept in Lumiera.
Not Library
~~~~~~~~~~~
Too specific to Lumiera:
-----
luid.c luid.h time.h
-----
C++ Parts
---------
For most of the C++ parts I am not sure, ichthyo should decided upon these
(please edit this here)
Library
~~~~~~~
These look 'generic' or wrap the C parts:
------
singleton-factory.hpp singleton.hpp singleton-policies.hpp
singleton-preconfigure.hpp singleton-ref.hpp singleton-subclass.hpp
sync-classlock.hpp sync.cpp sync.hpp thread-local.hpp
typed-allocation-manager.hpp typed-counter.hpp util.cpp util-foreach.hpp
util.hpp variant.hpp
------
Not Sure
~~~~~~~~
------
access-casted.hpp advice advice.hpp allocation-cluster.cpp
allocation-cluster.hpp bool-checkable.hpp cmdline.cpp cmdline.hpp del-stash.hpp
diagnostic-context.hpp element-tracker.hpp error.hpp (currently too
lumiera specific) exception.cpp (as before) factory.hpp format.hpp
frameid.hpp functor-util.hpp handle.hpp hash-indexed.hpp iter-adapter.hpp
iter-adapter-stl.hpp iter-source.hpp itertools.hpp iter-type-binding.hpp
lifecycle.cpp lifecycleregistry.hpp lumitime-fmt.hpp lumitime.hpp
multifact-arg.hpp multifact.hpp meta/* null-value.hpp observable-list.hpp
opaque-holder.hpp optional-ref.hpp p.hpp query.cpp query.hpp ref-array.hpp
ref-array-impl.hpp result.hpp scoped-holder.hpp scoped-holder-transfer.hpp
scoped-ptrvect.hpp searchpath.cpp searchpath.hpp sub-id.hpp symbol.hpp
symbol-impl.cpp visitor-dispatcher.hpp visitor.hpp visitor-policies.hpp
wrapper.hpp wrapperptr.hpp appstate.cpp appstate.hpp basic-setup.cpp
basic-setup.hpp DIR_INFO external guifacade.cpp instancehandle.hpp option.cpp
option.hpp query subsys.cpp subsys.hpp subsystem-runner.hpp
----
Not Library
~~~~~~~~~~~
------
logging.cpp nobug-init.cpp nobug-init.hpp streamtype.cpp streamtype.hpp test/*
time/* time.cpp tree.hpp
-----
Tasks
~~~~~
// List what needs to be done to implement this Proposal:
// * first step ([green]#✔ done#)
* decide on name, namespaces [,yellow]#WIP#
* create git repository, setup boilerplace (build system, legalese)
[,yellow]#WIP#
* move all code over into the git repos, refactor (namespace renames) ()
[,yellow]#WIP#
* make Lumiera use the new lib [,yellow]#WIP#
Discussion
~~~~~~~~~~
Pros
^^^^
// add a fact list/enumeration which make this suitable:
* I am already reuse much of its code, making it independent makes maintaining
it less burden
Cons
^^^^
// fact list of the known/considered bad implications:
* new packages, new dependencies for Lumiera instead 'batteries included'
Alternatives
^^^^^^^^^^^^
//alternatives: explain alternatives and tell why they are not viable:
Rationale
---------
//rationale: Give a concise summary why it should be done *this* way:
//Conclusion
//----------
//conclusion: When approbate (this proposal becomes a Final)
// write some conclusions about its process:
Comments
--------
//comments: append below
//endof_comments:
''''
Back to link:/documentation/devel/rfc.html[Lumiera Design Process overview]

13
doc/devel/rfc_pending/RenderOptimizer.txt → doc/devel/rfc/RenderOptimizer.txt
View file

@ -1,6 +1,6 @@
[grid="all"]
`------------`-----------------------
*State* _Idea_
*State* _Parked_
*Date* _2007-06-07_
*Proposed by* link:ct[]
-------------------------------------
@ -86,4 +86,15 @@ The good news is, that:
the matrix can be transformed from e.g. RGB to YUV, so these filters can
always work in both colorspaces directly
Parked
~~~~~~
Generally this is accepted but needs some more polishing when we go over it.
Do 14 Apr 2011 03:06:42 CEST Christian Thaeter
Back to link:/documentation/devel/rfc.html[Lumiera Design Process overview]

121
doc/devel/rfc/ResourceManagementBudgeting.txt Normal file
View file

@ -0,0 +1,121 @@
Resource Management: Budgeting
==============================
// please don't remove the //word: comments
[grid="all"]
`------------`-----------------------
*State* _Idea_
*Date* _Fri Jul 23 20:33:32 2010_
*Proposed by* Christian Thaeter <ct@pipapo.org>
-------------------------------------
[abstract]
******************************************************************************
The Profiler will give some Idea about how much Resources can me used to
optimally utilize the system. Knowing this number leads to the next challenge,
distributing the resources to different subsystems, jobs and objects. I here
introduce a budgeting system which takes care for this.
******************************************************************************
Description
-----------
//description: add a detailed description:
The idea is quite simple, for each kind of resource we have a global budget
manager which accounts for the available and used amounts of this resource.
Each user of a resource has its own account managing his share on the resource.
The system is completely voluntary giving only hints how much of a resource is
available for anyone.
[source,C]
------------------------------------------------------------------------------
typedef ssize_t budget_count;
struct budgetmanager
{
rwlock lock;
void (*callback)(); // called on resource shortage
(resource collector)
int sum_priorities; // sum of all client budgets .. each
client is granted available/(sum_priorities/own_priority) of the resource
budget_count available_prefs; // configuration from preferences
budget_count available_profile; // tuned by profiler
int available_factor; // how much % from prefs vs profile
budget_count available; // caclulated from above
budget_count allocated; // actively in use
};
struct budget
{
BudgetManager manager;
int priority;
budget_count allocated;
};
------------------------------------------------------------------------------
Tasks
~~~~~
// List what would need to be done to implement this Proposal in a few words:
// * item ...
Discussion
~~~~~~~~~~
Pros
^^^^
// add just a fact list/enumeration which make this suitable:
// * foo
// * bar ...
Cons
^^^^
// fact list of the known/considered bad implications:
Alternatives
^^^^^^^^^^^^
//alternatives: if possible explain/link alternatives and tell why they are not
viable:
Rationale
---------
//rationale: Describe why it should be done *this* way:
//Conclusion
//----------
//conclusion: When approbate (this proposal becomes a Final) write some
conclusions about its process:
Comments
--------
//comments: append below
//endof_comments:

182
doc/devel/rfc/ResourceManagementProfiling.txt Normal file
View file

@ -0,0 +1,182 @@
Resource Management: Profiling
==============================
// please don't remove the //word: comments
[grid="all"]
`------------`-----------------------
*State* _Idea_
*Date* _Fri Jul 23 19:34:29 2010_
*Proposed by* Christian Thaeter <ct@pipapo.org>
-------------------------------------
[abstract]
******************************************************************************
From the beginning on we planned some kind of 'profiling' to adapt dynamically
to workload and machine capabilities. I describe here how statistic data can be
gathered in a generic way. This will later work together with other components
tuning the system automatically.
******************************************************************************
Description
-----------
//description: add a detailed description:
I just introduce some ideas about the planned profiling framework here, nothing
is defined/matured yet this is certainly subject for futher discussion and
refinement.
.Requirements/Evaluation generic::
Profiling should be sufficiently abstracted to have a single set of
datastructures and algorithms to work on a broad range of subjects
being profiled. Moreover the profiling core just offers unitless
counters, semantic will be added on top of that on a higher level.
least possible overhead::
Profiling itself must not cost much, it must not block and should avoid
expensive operations. Simple integer arithmetic without divisions is
suggested.
accurate::
We may sample data on in stochastic way to reduce the overhead,
nevertheless data which gets sampled must be accurately stored and
processed without rounding losses and drifts.
transient values::
It's quite common that some values can be far off either in maximum or
in minimum direction, the system should adapt to this and recover from
such false alarms. Workload also changes over time we need to find some
way to measure the current/recent workload an grand total over the
whole application runtime is rather uninteresting. While it is also
important that we adapt slow enough not to get into some osccilating
cycle.
active or passive system::
Profiling can be only passive collecting data and let it be analyzed by
some other component or active triggering some action when some limits
are reached. I am yet a bit undecided and keep it open for both.
.Brainstorming in Code
[source,C]
------------------------------------------------------------------------------
typedef int64_t profile_value;
struct profile
{
ProfileVTable vtable;
/*
Using trylock for sampling makes it never contend on the lock but some
samples are lost. Should be ok.
*/
mutex_t lock; /* with trylock? */
/* statistics / running averages */
/* n being a small number 2-5 or so */
profile_value max[n]; /* n maximum values seen so far,
decreased by recovery */
profile_value min[n]; /* n minimum values seen so far,
increased by recovery */
/* store sum & count, but average calculation implies a division and will be
only done on demand */
profile_value count; /* count profile calls */
profile_value sum; /* sum up all calls, average =
sum/count */
/* current is the sampled value to be integrated */
/* trend is caclulated before theb new run_average */
profile_value trend; /* trend = (trend +
(run_average-current))>>1 */
/* we may need some slower diverging formula for running average */
profile_value run_average; /* run_average = (run_average +
current)>>1) */
/* active limits, define whats good and whats bad, calls back to vtable when
limit is hit */
profile_value max_limit;
profile_value min_limit;
/* do we want limits for trends too? */
/* we count how often we hit limits, a hit/miss ratio will give a good value
for optimization */
profile_value hit_cnt;
profile_value high_miss_cnt;
profile_value low_miss_cnt;
/* recovery state */
int rec_init;
int rec_current;
int rec_percent;
void* extra;
};
------------------------------------------------------------------------------
Tasks
~~~~~
// List what would need to be done to implement this Proposal in a few words:
// * item ...
Discussion
~~~~~~~~~~
Pros
^^^^
// add just a fact list/enumeration which make this suitable:
// * foo
// * bar ...
Cons
^^^^
// fact list of the known/considered bad implications:
Alternatives
^^^^^^^^^^^^
//alternatives: if possible explain/link alternatives and tell why they are not
viable:
Rationale
---------
//rationale: Describe why it should be done *this* way:
//Conclusion
//----------
//conclusion: When approbate (this proposal becomes a Final) write some
conclusions about its process:
Comments
--------
//comments: append below
//endof_comments:

2
doc/devel/rfc_pending/Roadmap-first.txt → doc/devel/rfc/Roadmap-first.txt
View file

@ -1,6 +1,6 @@
[grid="all"]
`------------`-----------------------
*State* _Idea_
*State* _Final_
*Date* _2009-01-30_
*Proposed by* link:Ichthyostega[]
-------------------------------------

199
doc/devel/rfc/SemanticTags.txt Normal file
View file

@ -0,0 +1,199 @@
Semantic tags
=============
// please don't remove the //word: comments
[grid="all"]
`------------`-----------------------
*State* _Idea_
*Date* _Do 30 Aug 2012 21:06:54 CEST_
*Proposed by* Christian Thaeter <ct@pipapo.org>
-------------------------------------
********************************************************************************
.Abstract
We have a lot documentation which needs to be cross referenced. Adding well the
known 'tags' concept and extend it slightly with some semantics will aid future
automatic processing.
********************************************************************************
Description
-----------
//description: add a detailed description:
Every document (including sourcecode) could extended with some metadata, aka tags
which are then used to build automatic crossreferences.
Commonly tags are just 'words' which are picked up and crossreferenceds. I propose
to extend this scheme slightly.
Overall this scheme must be very natual and easy to use. A user should not need to
know about the underlying machinery and a tag as in a single lowercase 'word' should
be sufficient in almost all cases. Moreover Tags should be optional.
.Ontology
To give tags some sematics we introduce a simple ontology:
- Tags can have namespaces, delimited by a dot 'foo.bar.baz'.
Tags are looked up from right to left 'baz' would suffice as long it is unique.
Non unique cases will be handled in context (sometimes non uniqunes is desired)
- We introduce simple "Is a" and "Has a" relationships. These are defined by the
casing of the tag: 'ALL_UPPERCASE' means "Is a" and anything else (including
mixed case) means "Has a". Note that for most cases the "Is a" relation will be
defined implicitly, ín normal cases one doesnt need to care.
- define some tag algebra for lookups (group tags by comma and semicolons, where
comma means 'and' and semicolon means 'or'). Used to query the tag database.
regex/globbing might become handy too.
.Implicit Tags
Tags can be implicit by generating them from the document:
- Derrive tags from the type and location of the Document.
RFC's are 'RFC', source files are 'SOURCE.C' and so on.
- Derrive Tags from the content of the document.
Asciidoc titles will be used here. A simple preprocessor
generates a tag from a title (make it CamelCase, simplify etc.)
The resulting tag is only used iff it is unique
.Use this tags
Tags are collected/discovered by some script which creates a tag-database
(possibly plaintext asciidoc files) as big project index linking back to the content,
details need to be worked out.
We create special asciidoc macros for crossreferencing tags for example: 'RFC:foobar'
'SOURCE:builder', details need to be worked out later.
Note: this Proposal is about including tags in the first place, processing them is only
suggested and left out for later.
Tasks
~~~~~
// List what needs to be done to implement this Proposal:
// * first step ([green]#✔ done#)
// * second step [,yellow]#WIP#
We need to define how to integrate tags in different documents syntactically.
For RFC's these will likely become a part of the initial table. in other Asciidoc
documents they could be a special comment or header. For Source files special comments
will be used.
Tags themself will be added lazily on demand (unless we find someone with the patience
to go over all documents and tag them properly).
Creating the infrastructure handling this tags (cross indexing etc) is not part of
this proposal, nevertheless we planning this since some time and it will be defined in
other RFC's.
Discussion
~~~~~~~~~~
Pros
^^^^
// add a fact list/enumeration which make this suitable:
// * foo
// * bar ...
* Gives a simple graspable way to build a cross reference over the whole project
Cons
^^^^
// fact list of the known/considered bad implications:
* adding tags and developing the tools manging them will take some time
Alternatives
^^^^^^^^^^^^
//alternatives: explain alternatives and tell why they are not viable:
We have the ht/dig search function over the Website which give a much simpler way to
find documents.
Rationale
---------
//rationale: Give a concise summary why it should be done *this* way:
It is very urgent and important that we make our content much easier accessible.
//Conclusion
//----------
//conclusion: When approbate (this proposal becomes a Final)
// write some conclusions about its process:
Comments
--------
//comments: append below
//edit comment
You may recall this proposal created some heated debate at the last developer meeting.
After thinking it over some time, I can see now more clearly what irritated me.
. for me, the proposal seems somewhat to lack focus. Right now we have some shortcomings at
rather basic operations when *authoring content* at the website. This proposal tends to be more
interested in some kind of automated content discovery.
. the term ``tag'' in this proposal is overlayed with different meanings. For one it means an attached
textual property of some document, but also it denotes to some kind of inferred categorisation.
I'd rather propose to stick to the former meaning (which is common place) and treat the latter
as one _source for data_ within an categorisation algorithm. This way, such categorisation
sources can remain an implementation detail and don't need to be fixed in an universal way.
. I have serious concerns against the _ontology_ part of the proposal. Not only is the syntax
unintuitive, but more importantly, this ontology is not well aligned with real world usage.
+
To underpin the last diagnosis, just look at the existing tags in our Wiki:
* automation (3)
* Builder (20)
* classes (6)
* Concepts (9)
* decision (19)
* def (90)
* design (43)
* discuss (19)
* draft (55)
* example (3)
* excludeMissing (6)
* GuiIntegration (6)
* img (40)
* impl (36)
* Model (22)
* operational (19)
* overview (20)
* Player (12)
* plugin (2)
* Rendering (24)
* rewrite (3)
* Rules (8)
* SessionLogic (30)
* spec (76)
* systemConfig (9)
* Types (4)
The absolute majority of these are neither _is-a_ nor _has-a_. The great thing with
tags, why everyone seems to love them, is exactly that they are *not formalized*.
You can just throw in some tags and keywords and use them for a plethora of
unrelated and unstructured purposes and generally just assume that your
reader will somehow ``get it''.
Ichthyostega:: 'Mi 10 Okt 2012 05:36:35 CEST' ~<prg@ichthyostega.de>~
//endof_comments:
''''
Back to link:/documentation/devel/rfc.html[Lumiera Design Process overview]

0
doc/devel/rfc_dropped/SkillsCollection.txt → doc/devel/rfc/SkillsCollection.txt
View file

268
doc/devel/rfc/StreamTypeSystem.txt Normal file
View file

@ -0,0 +1,268 @@
Stream Type System
==================
// please don't remove the //word: comments
[grid="all"]
`------------`-----------------------
*State* _Draft_
*Date* _2008-10-05_
*Proposed by* link:Ichthyostega[]
-------------------------------------
********************************************************************************
.Abstract
Especially in the Proc-Layer, within the Builder and at the interface to the
Engine we need sort of a framework to deal with different »kinds« of
media streams. +
This is the foundation to be able to define what can be connected and to
separate out generic parts and isolate specific parts.
********************************************************************************
Description
-----------
//description: add a detailed description:
The general idea is that we need meta information, and -- more precisely --
that _we_ need to control the structure of this metadata. Because it has
immediate consequences on the way the code can test and select the appropriate
path to deal with some data or a given case. This brings us in a difficult
situation:
* almost everything regarding media data and media handling is notoriously
convoluted
* because we can't hope ever to find a general umbrella, we need an extensible
solution
* we want to build on existing libraries rather then re-inventing media
processing.
* a library well suited for some processing task not necessarily has a type
classification system which fits our needs.
The proposed solution is to create an internal Stream Type System which acts as
a bridge to the detailed (implementation type) classification provided by the
library(s). Moreover, the approach was chosen especially in a way as to play
well with the rule based configuration, which is envisioned to play a central
role for some of the more advanced things possible within the session.
Terminology
~~~~~~~~~~~
* *Media* is comprised of a set of streams or channels
* *Stream* denotes a homogeneous flow of media data of a single kind
* *Channel* denotes an elementary stream, which -- _in the given context_ --
can't be decomposed any further
* all of these are delivered and processed in a smallest unit called *Frame*.
Each frame corresponds to a time interval.
* a *Buffer* is a data structure capable of holding one or multiple Frames of media data.
* the *Stream Type* describes the kind of media data contained in the stream
Concept of a Stream Type
~~~~~~~~~~~~~~~~~~~~~~~~
The Goal of our Stream Type system is to provide a framework for precisely
describing the ``kind'' of a media stream at hand. The central idea is to
structure the description/classification of streams into several levels.
A complete stream type (implemented by a stream type descriptor) contains
a tag or selection regarding each of these levels.
Levels of classification
^^^^^^^^^^^^^^^^^^^^^^^^
* Each media belongs to a fundamental *kind of media*, examples being _Video,
Image, Audio, MIDI, Text,..._ This is a simple Enum.
* Below the level of distinct kinds of media streams, within every kind we
have an open ended collection of *Prototypes*, which, within the high-level
model and for the purpose of wiring, act like the "overall type" of the
media stream. Everything belonging to a given Prototype is considered to be
roughly equivalent and can be linked together by automatic, lossless
conversions. Examples for Prototypes are: stereoscopic (3D) video versus the
common flat video lacking depth information, spatial audio systems
(Ambisonics, Wave Field Synthesis), panorama simulating sound systems (5.1,
7.1,...), binaural, stereophonic and monaural audio.
* Besides the distinction by prototypes, there are the various *media
implementation types*. This classification is not necessarily hierarchically
related to the prototype classification, while in practice commonly there
will be some sort of dependency. For example, both stereophonic and monaural
audio may be implemented as 96kHz 24bit PCM with just a different number of
channel streams, but we may as well have a dedicated stereo audio stream
with two channels multiplexed into a single stream.
Working with media stream implementations
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
For dealing with media streams of various implementation type, we need
_library_ routines, which also yield a _type classification system_ suitable
for their intended use. Most notably, for raw sound and video data we use the
http://gmerlin.sourceforge.net/[GAVL] library, which defines a fairly complete
classification system for buffers and streams. For the relevant operations in
the Proc-Layer, we access each such library by means of a Façade; it may sound
surprising, but actually we just need to access a very limited set of
operations, like allocating a buffer. _Within_ the Proc-Layer, the actual
implementation type is mostly opaque; all we need to know is if we can connect
two streams and get an conversion plugin.
Thus, to integrate an external library into Lumiera, we need explicitly to
implement such a Lib Façade for this specific case, but the intention is to be
able to add this Lib Façade implementation as a plugin (more precisely as a
"Feature Bundle", because it probably includes several plugins and some
additional rules)
Link between implementation type and prototype
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
At this point the rules based configuration comes into play. Mostly, to start
with, determining a suitable prototype for a given implementation type is sort
of a tagging operation. But it can be supported by heuristic rules and an
flexible configuration of defaults. For example, if confronted with a media
with 6 sound channels, we simply can't tell if it's a 5.1 sound source, or if
it's a pre mixed orchestra music arrangement to be routed to the final balance
mixing or if it's a prepared set of spot pick-ups and overdubbed dialogue. But a
heuristic rule defaulting to 5.1 would be a good starting point, while
individual projects should be able to set up very specific additional rules
(probably based on some internal tags, conventions on the source folder or the
like) to get a smooth workflow.
Moreover, the set of prototypes is deliberately kept open ended. Because some
projects need much more fine grained control than others. For example, it may
be sufficient to subsume any video under a single prototype and just rely on
automatic conversions, while other projects may want to distinguish between
digitized film and video NTSC and PAL. Meaning they would be kept in separate
pipes an couldn't be mixed automatically without manual intervention.
connections and conversions
^^^^^^^^^^^^^^^^^^^^^^^^^^^
* It is _impossible to connect_ media streams of different kind. Under some
circumstances there may be the possibility of a _transformation_ though. For
example, sound may be visualized, MIDI may control a sound synthesizer,
subtitle text may be rendered to a video overlay. Anyway, this includes some
degree of manual intervention.
* Streams subsumed by the same prototype may be _converted_ lossless and
automatically. Streams tagged with differing prototypes may be _rendered_
into each other.
* Conversions and judging the possibility of making connections at the level
of implementation types is coupled tightly to the used library; indeed, most
of the work to provide a Lib Façade consists of coming up with a generic
scheme to decide this question for media streams implemented by this
library.
Tasks
~~~~~
// List what needs to be done to implement this Proposal:
* draft the interfaces ([green]#✔ done#)
* define a fall-back and some basic behaviour for the relation between
implementation type and prototypes [,yellow]#WIP#
* find out if it is necessary to refer to types in a symbolic manner, or if it
is sufficient to have a ref to a descriptor record or Façade object.
* provide a Lib Façade for GAVL [,yellow]#WIP#
* evaluate if it's a good idea to handle (still) images as a separate distinct
kind of media
Discussion
~~~~~~~~~~
Alternatives
^^^^^^^^^^^^
//alternatives: explain alternatives and tell why they are not viable:
Instead of representing types by metadata, leave the distinction implicit and
instead implement the different behaviour directly in code. Have video tracks
and audio tracks. Make video clip objects and audio clip objects, each
utilising some specific flags, like sound is mono or stereo. Then either
switch, switch-on-type or scatter out the code into a bunch of virtual
functions. See the Cinelerra source code for details.
In short, following this route, Lumiera would be plagued by the same notorious
problems as most existing video/sound editing software. Which is, implicitly
assuming ``everyone'' just does ``normal'' things. Of course, users always were
and always will be clever enough to work around this assumption, but the problem
is, all those efforts will mostly stay isolated and can't crystallise into a
reusable extension. Users will do manual tricks, use some scripting or rely on
project organisation and conventions, which in turn creates more and more
coercion for the ``average'' user to just do ``normal'' things.
To make it clear: both approaches discussed here do work in practice, and it's
more a cultural issue, not a question guided by technical necessities to select
the one or the other.
Rationale
---------
//rationale: Give a concise summary why it should be done *this* way:
* use type metadata to factor out generic behaviour and make variations in
behaviour configurable.
* don't use a single classification scheme, because we deal with distinctions
and decisions on different levels of abstraction
* don't try to create an universal classification of media implementation type
properties, rather rely on the implementation libraries to provide already a
classification scheme well suited for _their_ needs.
* decouple the part of the classification guiding the decisions on the level
of the high level model from the raw implementation types, reduce the former
to a tagging operation.
* provide the possibility to incorporate very project specific knowledge as
rules.
//Conclusion
//----------
//conclusion: When approbate (this proposal becomes a Final)
// write some conclusions about its process:
Comments
--------
//comments: append below
As usual, see the
http://www.lumiera.org/wiki/renderengine.html#StreamType[Proc-Layer impl doku]
for more information and implementation details.
Practical implementation related note: I found I was blocked by this one in
further working out the details of the processing nodes wiring, and thus make
any advance on the builder and thus to know more precisely how to organize the
objects in the link:EDL/Session[]. Because I need a way to define a viable
abstraction for getting a buffer and working on frames. The reason is not
immediately obvious (because initially you could just use an opaque type). The
problem is related to the question what kind of structures I can assume for the
builder to work on for deciding on connections. Because at this point, the
high-level view (pipes) and the low level view (processing functions with a
number of inputs and outputs) need in some way to be connected.
The fact that we don't have a rule based system for deciding queries currently
is not much of a problem. A table with some pre configured default answers for
a small number of common query cases is enough to get the first clip rendered.
(Such a solution is already in place and working.) +
-- link:Ichthyostega[] 2008-10-05
Woops fast note, I didn't read this proposal completely yet. Stream types could
or maybe should be coopertatively handled together with the backend. Basically
the backend offers one to access regions of a file in a continous block, this
regions are addressed as "frames" (this are not necessary video frames). The
backend will keep indices which associate this memory management with the frame
number, plus adding the capabilitiy of per frame metadata. This indices get
abstracted by "indexing engines" it will be possible to have different kinds of
indices over one file (for example, one enumerating single frames, one
enumerating keyframes or gops). Such a indexing engine would be also the place
to attach per media metadata. From the proc layer it can then look like `struct
frameinfo* get_frame(unsigned num)` where `struct frameinfo` (not yet defined)
is something like `{ void* data; size_t size; struct metadata* meta; ...}` +
-- link:ct[] 2008-10-06
Needs Work
~~~~~~~~~~
There are a lot details to be worked out for an actual implementation but we
agreed that we want this concept as proposed here.
Do 14 Apr 2011 03:06:42 CEST Christian Thaeter
//endof_comments:
''''
Back to link:/documentation/devel/rfc.html[Lumiera Design Process overview]

155
doc/devel/rfc/SystematicMetadata.txt Normal file
View file

@ -0,0 +1,155 @@
SystematicMetadata
==================
// please don't remove the //word: comments
[grid="all"]
`------------`-----------------------
*State* _Idea_
*Date* _Mo 08 Okt 2012 04:39:16 CEST_
*Proposed by* Ichthyostega <prg@ichthyostega.de>
-------------------------------------
********************************************************************************
.Abstract
_give a short summary of this proposal_
********************************************************************************
Lumiera is a metadata processing application: _Data_ is _media data_, and everything
else is _metadata_. Since our basic decision is to rely on existing libraries for
handling data, the ``metadata part'' is what _we are building anew._
This RfC describes a fundamental approach towards metadata handling.
Description
-----------
//description: add a detailed description:
Metadata is conceived as a huge uniform tree. This tree is conceptual -- it is
never represented as a whole. In the implemented system, we only ever see parts
of this virtual tree being cast into concrete data representations. These parts
are like islands of explicitly defined and typed structure, yet they never need
to span the whole virtual model, and thus there never needs to be an universal
model data structure definition. Data structure becomes implementation detail.
Parts of the system talk to each other by _describing_ some subtree of metadata.
This description is _always pushed:_ the receiver implements an API allowing the
sender to navigate to some path-like scope and populate it with values, similar
to populating a filesystem. It is up to the receiver to assemble these information
into a suitable representation. Some receiver might invoke an object factory, while
another serialises data into an external textual or binary representation.
Abstract Metadata Model
~~~~~~~~~~~~~~~~~~~~~~~
The conceptual model for metadata is close to what the JSON format uses: +
There are primitive values as +null+, string, number and boolean. Compund values
can be arrays or records, the latter being a sub-scope populated with key-value pairs.
We might consider some extensions
* having data values similar to BSON of MongoDB: integrals, floats, timestamps
* introducing two _special magic keys_ for records: `"type"` and `"id"`
Sources and Overlays
~~~~~~~~~~~~~~~~~~~~
Metadata is delivered from _sources_, which can be _layered_. Similarly, on the
receiving side, there can be multiple _writeable layers_, with a routing strategy
to decide which writeable layer receives a given metadata element. This routing
is implemented within a pipeline connecting sender and receiver; if the default
routing strategy isn't sufficient, we can control the routing by introducing a
a meta-tree in some separate branch, this way making the metadata self-referential.
Some points to note
~~~~~~~~~~~~~~~~~~~
- this concept doesn't say anything about the actual meaning of the metadata elements,
since that is always determined by the receiver, based on the current context.
- likewise, this concept doesn't state anything about the actual interactions, the
involved parts and how the interaction is initiated and configured; this is considered
an external topic, which needs to be solved within the applicable context (e.g. the
session has a specific protocol how to retrieve a persisted session snapshot)
- there is no separate _system configuration_ -- configuration appears just as a
local record of key-value pairs, which is interpreted according to the context.
- in a similar vein, this concept deliberately doesn't state anything regarding the
handling of _defaults_, since these are so highly dependent on the actual context.
Tasks
~~~~~
// List what needs to be done to implement this Proposal:
// * first step ([green]#✔ done#)
* define the interaction API [yellow-background]#WIP#
* scrutinise this concept to find the pitfalls [yellow-background]#WIP#
* build a demonstration prototype, where the receiver fabricates an object [red yellow-background]#TBD#
Discussion
~~~~~~~~~~
Pros
^^^^
- the basic implementation is strikingly simple, much simpler than building
a huge data structure or any kind of serialisation/deserialisation scheme
- parts can be combined in an open fashion, we don't need a final concept up-front
- even complex routing and overlaying strategies become manageable, since they can be
treated in isolation, local for a given scope and apart from the storage representation
- library implementations for textual representations can be integrated.
Cons
^^^^
- the theoretical view is challenging and rather uncommon
- a naive implementation holds the whole data tree in memory twice
- how the coherent ``islands'' are combined is only a matter of invocation order
and thus dangerously flexible
Alternatives
^^^^^^^^^^^^
//alternatives: explain alternatives and tell why they are not viable:
The classical alternative is to define a common core data structure, which
needs to be finalised quickly. Isolated functional modules will then be written
to work on that common data set, which leads to a high degree of coupling.
Since this approach effectively doesn't scale well, what happens in practice is
that several independent storage and exchange systems start to exist in parallel,
e.g. system configuration, persisted object model, plug-in parameters, presentation
state.
Rationale
---------
//rationale: Give a concise summary why it should be done *this* way:
Basically common (meta) data could take on a lot of shapes between two extremes:
- the _precise typed structure_, which also is a contract
- the _open dynamic structure_, which leaves the contract implicit
The concept detailed in this RfC tries to reconcile those extremes by avoiding
a global concrete representation; +
this way the actual interaction -- with the necessity
of defining a contract -- is turned into a local problem.
//Conclusion
//----------
//conclusion: When approbate (this proposal becomes a Final)
// write some conclusions about its process:
Comments
--------
//comments: append below
//endof_comments:
''''
Back to link:/documentation/devel/rfc.html[Lumiera Design Process overview]

Some files were not shown because too many files have changed in this diff Show more