From 7a64f3764373cd870cdaad53b0dd24b4587f465f Mon Sep 17 00:00:00 2001 From: Ichthyostega Date: Fri, 19 Dec 2008 20:17:23 +0100 Subject: [PATCH] SCons: use two existing modules for integrating DistCC and CCache --- SConstruct | 12 +- admin/scons/BuilderDoxygen.py | 357 +++++++++++++++++----------------- admin/scons/BuilderGCH.py | 12 +- admin/scons/Buildhelper.py | 36 ++++ admin/scons/ToolCCache.py | 76 ++++++++ admin/scons/ToolDistCC.py | 65 +++++++ 6 files changed, 367 insertions(+), 191 deletions(-) create mode 100644 admin/scons/ToolCCache.py create mode 100644 admin/scons/ToolDistCC.py diff --git a/SConstruct b/SConstruct index d711313aa..d30b7c6c7 100644 --- a/SConstruct +++ b/SConstruct @@ -64,14 +64,12 @@ def setupBasicEnvironment(): ,toolpath = [TOOLDIR] ,tools = ["default", "BuilderGCH", "BuilderDoxygen"] ) + env.Tool("ToolDistCC") + env.Tool("ToolCCache") handleVerboseMessages(env) env.Append ( CCCOM=' -std=gnu99') env.Append ( SHCCCOM=' -std=gnu99') # workaround for a bug: CCCOM currently doesn't honour CFLAGS, only CCFLAGS - env.Replace( SETCC=env.subst('$SETCC ')) - env.Replace( CC=env['SETCC']) # possibly using another compiler command... - env.Replace( SETCXX=env.subst('$SETCXX ')) - env.Replace( CXX=env['SETCXX']) env.Replace( VERSION=VERSION , SRCDIR=SRCDIR , BINDIR=BINDIR @@ -147,8 +145,10 @@ def defineCmdlineOptions(): opts = Options([OPTIONSCACHEFILE, CUSTOPTIONSFILE]) opts.AddOptions( ('ARCHFLAGS', 'Set architecture-specific compilation flags (passed literally to gcc)','') - ,('SETCC', 'Set the C compiler to use. (you can use $CC to refer to the default)','$CC') - ,('SETCXX', 'Set the C++ compiler to use. (you can refer to $CXX)','$CXX') + ,('CC', 'Set the C compiler to use.', 'gcc') + ,('CXX', 'Set the C++ compiler to use.', 'g++') + ,PathOption('CCACHE', 'Integrate with CCache', '', PathOption.PathAccept) + ,PathOption('DISTCC', 'Invoke C/C++ compiler commands through DistCC', '', PathOption.PathAccept) ,EnumOption('BUILDLEVEL', 'NoBug build level for debugging', 'ALPHA', allowed_values=('ALPHA', 'BETA', 'RELEASE')) ,BoolOption('DEBUG', 'Build with debugging information and no optimisations', False) diff --git a/admin/scons/BuilderDoxygen.py b/admin/scons/BuilderDoxygen.py index 943fce946..a397ef32e 100644 --- a/admin/scons/BuilderDoxygen.py +++ b/admin/scons/BuilderDoxygen.py @@ -33,198 +33,197 @@ from fnmatch import fnmatch def DoxyfileParse(file_contents): - """ - Parse a Doxygen source file and return a dictionary of all the values. - Values will be strings and lists of strings. - """ - data = {} - - import shlex - lex = shlex.shlex(instream = file_contents, posix = True) - lex.wordchars += "*+./-:" - lex.whitespace = lex.whitespace.replace("\n", "") - lex.escape = "" - - lineno = lex.lineno - token = lex.get_token() - key = token # the first token should be a key - last_token = "" - key_token = False - next_key = False - new_data = True - - def append_data(data, key, new_data, token): - if new_data or len(data[key]) == 0: - data[key].append(token) - else: - data[key][-1] += token - - while token: - if token in ['\n']: - if last_token not in ['\\']: - key_token = True - elif token in ['\\']: - pass - elif key_token: - key = token - key_token = False - else: - if token == "+=": - if not data.has_key(key): + """ Parse a Doxygen source file and return a dictionary of all the values. + Values will be strings and lists of strings. + """ + data = {} + + import shlex + lex = shlex.shlex(instream = file_contents, posix = True) + lex.wordchars += "*+./-:" + lex.whitespace = lex.whitespace.replace("\n", "") + lex.escape = "" + + lineno = lex.lineno + token = lex.get_token() + key = token # the first token should be a key + last_token = "" + key_token = False + next_key = False + new_data = True + + def append_data(data, key, new_data, token): + if new_data or len(data[key]) == 0: + data[key].append(token) + else: + data[key][-1] += token + + while token: + if token in ['\n']: + if last_token not in ['\\']: + key_token = True + elif token in ['\\']: + pass + elif key_token: + key = token + key_token = False + else: + if token == "+=": + if not data.has_key(key): + data[key] = list() + elif token == "=": data[key] = list() - elif token == "=": - data[key] = list() - else: - append_data( data, key, new_data, token ) - new_data = True + else: + append_data( data, key, new_data, token ) + new_data = True + + last_token = token + token = lex.get_token() + + if last_token == '\\' and token != '\n': + new_data = False + append_data( data, key, new_data, '\\' ) + + # compress lists of len 1 into single strings + for (k, v) in data.items(): + if len(v) == 0: + data.pop(k) + + # items in the following list will be kept as lists and not converted to strings + if k in ["INPUT", "FILE_PATTERNS", "EXCLUDE_PATTERNS"]: + continue + + if len(v) == 1: + data[k] = v[0] + + return data - last_token = token - token = lex.get_token() - - if last_token == '\\' and token != '\n': - new_data = False - append_data( data, key, new_data, '\\' ) - - # compress lists of len 1 into single strings - for (k, v) in data.items(): - if len(v) == 0: - data.pop(k) - - # items in the following list will be kept as lists and not converted to strings - if k in ["INPUT", "FILE_PATTERNS", "EXCLUDE_PATTERNS"]: - continue - - if len(v) == 1: - data[k] = v[0] - - return data def DoxySourceScan(node, env, path): - """ - Doxygen Doxyfile source scanner. This should scan the Doxygen file and add - any files used to generate docs to the list of source files. - """ - default_file_patterns = [ - '*.c', '*.cc', '*.cxx', '*.cpp', '*.c++', '*.java', '*.ii', '*.ixx', - '*.ipp', '*.i++', '*.inl', '*.h', '*.hh ', '*.hxx', '*.hpp', '*.h++', - '*.idl', '*.odl', '*.cs', '*.php', '*.php3', '*.inc', '*.m', '*.mm', - '*.py', - ] - - default_exclude_patterns = [ - '*~', - ] - - sources = [] - - data = DoxyfileParse(node.get_contents()) - - if data.get("RECURSIVE", "NO") == "YES": - recursive = True - else: - recursive = False - - file_patterns = data.get("FILE_PATTERNS", default_file_patterns) - exclude_patterns = data.get("EXCLUDE_PATTERNS", default_exclude_patterns) - - # - # We're running in the top-level directory, but the doxygen - # configuration file is in the same directory as node; this means - # that relative pathnames in node must be adjusted before they can - # go onto the sources list - conf_dir = os.path.dirname(str(node)) - - for node in data.get("INPUT", []): - if not os.path.isabs(node): - node = os.path.join(conf_dir, node) - if os.path.isfile(node): - sources.append(node) - elif os.path.isdir(node): - if recursive: - for root, dirs, files in os.walk(node): - for f in files: - filename = os.path.join(root, f) - - pattern_check = reduce(lambda x, y: x or bool(fnmatch(filename, y)), file_patterns, False) - exclude_check = reduce(lambda x, y: x and fnmatch(filename, y), exclude_patterns, True) - - if pattern_check and not exclude_check: - sources.append(filename) - else: - for pattern in file_patterns: - sources.extend(glob.glob("/".join([node, pattern]))) - - sources = map( lambda path: env.File(path), sources ) - return sources + """ Doxygen Doxyfile source scanner. + This should scan the Doxygen file and add any files + used to generate docs to the list of source files. + """ + default_file_patterns = [ + '*.c', '*.cc', '*.cxx', '*.cpp', '*.c++', '*.java', '*.ii', '*.ixx', + '*.ipp', '*.i++', '*.inl', '*.h', '*.hh ', '*.hxx', '*.hpp', '*.h++', + '*.idl', '*.odl', '*.cs', '*.php', '*.php3', '*.inc', '*.m', '*.mm', + '*.py', + ] + + default_exclude_patterns = [ + '*~', + ] + + sources = [] + + data = DoxyfileParse(node.get_contents()) + + if data.get("RECURSIVE", "NO") == "YES": + recursive = True + else: + recursive = False + + file_patterns = data.get("FILE_PATTERNS", default_file_patterns) + exclude_patterns = data.get("EXCLUDE_PATTERNS", default_exclude_patterns) + + # + # We're running in the top-level directory, but the doxygen configuration file + # is in the same directory as node; this means that relative pathnames in node + # must be adjusted before they can go onto the sources list + conf_dir = os.path.dirname(str(node)) + + for node in data.get("INPUT", []): + if not os.path.isabs(node): + node = os.path.join(conf_dir, node) + if os.path.isfile(node): + sources.append(node) + elif os.path.isdir(node): + if recursive: + for root, dirs, files in os.walk(node): + for f in files: + filename = os.path.join(root, f) + + pattern_check = reduce(lambda x, y: x or bool(fnmatch(filename, y)), file_patterns, False) + exclude_check = reduce(lambda x, y: x and fnmatch(filename, y), exclude_patterns, True) + + if pattern_check and not exclude_check: + sources.append(filename) + else: + for pattern in file_patterns: + sources.extend(glob.glob("/".join([node, pattern]))) + + sources = map( lambda path: env.File(path), sources ) + return sources def DoxySourceScanCheck(node, env): - """Check if we should scan this file""" - return os.path.isfile(node.path) + """ Check if we should scan this file """ + return os.path.isfile(node.path) + def DoxyEmitter(source, target, env): - """Doxygen Doxyfile emitter""" - # possible output formats and their default values and output locations - output_formats = { - "HTML": ("YES", "html"), - "LATEX": ("YES", "latex"), - "RTF": ("NO", "rtf"), - "MAN": ("NO", "man"), - "XML": ("NO", "xml"), - } - - data = DoxyfileParse(source[0].get_contents()) - - targets = [] - out_dir = data.get("OUTPUT_DIRECTORY", ".") - - # add our output locations - for (k, v) in output_formats.items(): - if data.get("GENERATE_" + k, v[0]) == "YES": - targets.append(env.Dir( os.path.join(out_dir, data.get(k + "_OUTPUT", v[1]))) ) - - # don't clobber targets - for node in targets: - env.Precious(node) - - # set up cleaning stuff - for node in targets: - env.Clean(node, node) - - return (targets, source) + """ Doxygen Doxyfile emitter """ + # possible output formats and their default values and output locations + output_formats = { + "HTML": ("YES", "html"), + "LATEX": ("YES", "latex"), + "RTF": ("NO", "rtf"), + "MAN": ("NO", "man"), + "XML": ("NO", "xml"), + } + + data = DoxyfileParse(source[0].get_contents()) + + targets = [] + out_dir = data.get("OUTPUT_DIRECTORY", ".") + + # add our output locations + for (k, v) in output_formats.items(): + if data.get("GENERATE_" + k, v[0]) == "YES": + targets.append(env.Dir( os.path.join(out_dir, data.get(k + "_OUTPUT", v[1]))) ) + + # don't clobber targets + for node in targets: + env.Precious(node) + + # set up cleaning stuff + for node in targets: + env.Clean(node, node) + + return (targets, source) def generate(env): - """ - Add builders and construction variables for the - Doxygen tool. This is currently for Doxygen 1.4.6. - """ - doxyfile_scanner = env.Scanner( - DoxySourceScan, - "DoxySourceScan", - scan_check = DoxySourceScanCheck, - ) + """ Add builders and construction variables for the + Doxygen tool. This is currently for Doxygen 1.4.6. + """ + doxyfile_scanner = env.Scanner( + DoxySourceScan, + "DoxySourceScan", + scan_check = DoxySourceScanCheck, + ) + + import SCons.Builder + doxyfile_builder = SCons.Builder.Builder( + action = "cd ${SOURCE.dir} && (${DOXYGEN} ${SOURCE.file} 2>&1 |tee ,doxylog)", + emitter = DoxyEmitter, + target_factory = env.fs.Entry, + single_source = True, + source_scanner = doxyfile_scanner, + ) + + env.Append(BUILDERS = { + 'Doxygen': doxyfile_builder, + }) + + env.AppendUnique( + DOXYGEN = 'doxygen', + ) - import SCons.Builder - doxyfile_builder = SCons.Builder.Builder( - action = "cd ${SOURCE.dir} && (${DOXYGEN} ${SOURCE.file} 2>&1 |tee ,doxylog)", - emitter = DoxyEmitter, - target_factory = env.fs.Entry, - single_source = True, - source_scanner = doxyfile_scanner, - ) - - env.Append(BUILDERS = { - 'Doxygen': doxyfile_builder, - }) - - env.AppendUnique( - DOXYGEN = 'doxygen', - ) def exists(env): - """ - Make sure doxygen exists. - """ - return env.Detect("doxygen") + """ Make sure doxygen exists. + """ + return env.Detect("doxygen") diff --git a/admin/scons/BuilderGCH.py b/admin/scons/BuilderGCH.py index c522cb9cd..ec48edcdd 100644 --- a/admin/scons/BuilderGCH.py +++ b/admin/scons/BuilderGCH.py @@ -61,7 +61,7 @@ def setup_dependency(target,source,env, key): if header_path in [x.path for x in deps]: print "Precompiled header(%s) %s \t <--- %s" % (key,header_path,source[0]) env.Depends(target, header) - + def static_pch_emitter(target,source,env): SCons.Defaults.StaticObjectEmitter( target, source, env ) @@ -73,9 +73,9 @@ def shared_pch_emitter(target,source,env): setup_dependency(target,source,env, key='GCH-sh') return (target, source) + def generate(env): - """ - Add builders and construction variables for the Gch builder. + """ Add builders and construction variables for the Gch builder. """ env.Append(BUILDERS = { 'gch': env.Builder( @@ -87,7 +87,7 @@ def generate(env): target_factory = env.fs.File, ), }) - + try: bld = env['BUILDERS']['GCH'] bldsh = env['BUILDERS']['GCH-sh'] @@ -96,11 +96,11 @@ def generate(env): bldsh = GchShBuilder env['BUILDERS']['PrecompiledHeader'] = bld env['BUILDERS']['PrecompiledHeaderShared'] = bldsh - + env['GCHCOM'] = '$CXX -o $TARGET -x c++-header -c $CXXFLAGS $_CCCOMCOM $SOURCE' env['GCHSHCOM'] = '$CXX -o $TARGET -x c++-header -c $SHCXXFLAGS $_CCCOMCOM $SOURCE' env['GCHSUFFIX'] = '.gch' - + for suffix in SCons.Util.Split('.c .C .cc .cxx .cpp .c++'): env['BUILDERS']['StaticObject'].add_emitter( suffix, static_pch_emitter ) env['BUILDERS']['SharedObject'].add_emitter( suffix, shared_pch_emitter ) diff --git a/admin/scons/Buildhelper.py b/admin/scons/Buildhelper.py index 7eef32750..a8a76a600 100644 --- a/admin/scons/Buildhelper.py +++ b/admin/scons/Buildhelper.py @@ -112,7 +112,43 @@ def getDirname(dir): dir,_ = os.path.split(dir) _, name = os.path.split(dir) return name + + + +def checkCommandOption(env, optID, val=None, cmdName=None): + """ evaluate and verify an option, which may point at a command. + besides specifying a path, the option may read True, yes or 1, + denoting that the system default for this command should be used. + @return: True, if the key has been expanded and validated, + False, if this failed and the key was removed + """ + if not val: + if not env.get(optID): return False + else: + val = env.get(optID) + if val=='True' or val=='true' or val=='yes' or val=='1' or val == 1 : + if not cmdName: + print "WARNING: no default for %s, please specify a full path." % optID + del env[optID] + return False + else: + val = env.WhereIs(cmdName) + if not val: + print "WARNING: %s not found, please specify a full path" % cmdName + del env[optID] + return False + + if not os.path.isfile(val): + val = env.WhereIs(val) + + if val and os.path.isfile(val): + env[optID] = val + return True + else: + del env[optID] + return False + def RegisterIcon_Builder(env, renderer): diff --git a/admin/scons/ToolCCache.py b/admin/scons/ToolCCache.py new file mode 100644 index 000000000..7d93fb7a4 --- /dev/null +++ b/admin/scons/ToolCCache.py @@ -0,0 +1,76 @@ +# -*- python -*- +## +## ToolCCache.py - SCons tool for integrating with CCache compiler cache +## + +# Copyright (C) Lumiera.org and FreeOrion.org +# 2008, Hermann Vosseler +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of +# the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +##################################################################### + +# This SCons builder was extracted from http://www.freeorion.org/ +# FreeOrion is an open-source platform-independent galactic conquest game +# +# history: 12/2008 adapted for Lumiera build system + + +import os +from Buildhelper import * + + +def generate(env): + """ Modify the environment such as to redirect any + C/C++ compiler invocations through CCache, while using + CCache config variables found in the os.environment. + """ + if not exists(env): return + + assert env['CCACHE'] + if not env['CCACHE'] in env['CC']: + env['CC'] = env.subst('$CCACHE $CC') + if not env['CCACHE'] in env['CXX']: + env['CXX'] = env.subst('$CCACHE $CXX') + print env.subst("* Build using $CCACHE") + + for i in ['HOME' + ,'CCACHE_DIR' + ,'CCACHE_TEMPDIR' + ,'CCACHE_LOGFILE' + ,'CCACHE_PATH' + ,'CCACHE_CC' + ,'CCACHE_CPP2' + ,'CCACHE_PREFIX' + ,'CCACHE_DISABLE' + ,'CCACHE_READONLY' + ,'CCACHE_NOSTATS' + ,'CCACHE_NLEVELS' + ,'CCACHE_HARDLINK' + ,'CCACHE_RECACHE' + ,'CCACHE_UMASK' + ,'CCACHE_HASHDIR' + ,'CCACHE_UNIFY' + ,'CCACHE_EXTENSION' + ]: + if os.environ.has_key(i) and not env.has_key(i): + env['ENV'][i] = os.environ[i] + + + +def exists(env): + """ Ensure CCache is available. + """ + return checkCommandOption(env, 'CCACHE', cmdName='ccache') + diff --git a/admin/scons/ToolDistCC.py b/admin/scons/ToolDistCC.py new file mode 100644 index 000000000..ecc45d011 --- /dev/null +++ b/admin/scons/ToolDistCC.py @@ -0,0 +1,65 @@ +# -*- python -*- +## +## ToolDistCC.py - SCons tool for distributed compilation using DistCC +## + +# Copyright (C) Lumiera.org and FreeOrion.org +# 2008, Hermann Vosseler +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of +# the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +##################################################################### + +# This SCons builder was extracted from http://www.freeorion.org/ +# FreeOrion is an open-source platform-independent galactic conquest game +# +# history: 12/2008 adapted for Lumiera build system + + +import os +from Buildhelper import * + + +def generate(env): + """ Modify the environment such as to redirect any + C/C++ compiler invocations through DistCC. Additionally + pull in the environment config variables used by DistCC + """ + if not exists(env): return + + assert env['DISTCC'] + if not env['DISTCC'] in env['CC']: + env['CC'] = env.subst('$DISTCC $CC') + if not env['DISTCC'] in env['CXX']: + env['CXX'] = env.subst('$DISTCC $CXX') + print env.subst("* Build using $DISTCC") + for i in ['HOME' + ,'DISTCC_HOSTS' + ,'DISTCC_VERBOSE' + ,'DISTCC_FALLBACK' + ,'DISTCC_LOG' + ,'DISTCC_MMAP' + ,'DISTCC_SAVE_TEMPS' + ,'DISTCC_TCP_CORK' + ,'DISTCC_SSH' + ]: + if os.environ.has_key(i) and not env.has_key(i): + env['ENV'][i] = os.environ[i] + + +def exists(env): + """ Ensure DistCC exists. + """ + return checkCommandOption(env, 'DISTCC', cmdName='distcc') +