benn/lumiera_
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

SCons: use two existing modules for integrating DistCC and CCache

Browse source
This commit is contained in:
Fischlurch 2008-12-19 20:17:23 +01:00
parent 359144c528
commit 7a64f37643
6 changed files with 367 additions and 191 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
SConstruct
View file

@ -64,14 +64,12 @@ def setupBasicEnvironment():
,toolpath = [TOOLDIR]
,tools = ["default", "BuilderGCH", "BuilderDoxygen"]
)
env.Tool("ToolDistCC")
env.Tool("ToolCCache")
handleVerboseMessages(env)
env.Append ( CCCOM=' -std=gnu99')
env.Append ( SHCCCOM=' -std=gnu99') # workaround for a bug: CCCOM currently doesn't honour CFLAGS, only CCFLAGS
env.Replace( SETCC=env.subst('$SETCC '))
env.Replace( CC=env['SETCC']) # possibly using another compiler command...
env.Replace( SETCXX=env.subst('$SETCXX '))
env.Replace( CXX=env['SETCXX'])
env.Replace( VERSION=VERSION
, SRCDIR=SRCDIR
, BINDIR=BINDIR
@ -147,8 +145,10 @@ def defineCmdlineOptions():
opts = Options([OPTIONSCACHEFILE, CUSTOPTIONSFILE])
opts.AddOptions(
('ARCHFLAGS', 'Set architecture-specific compilation flags (passed literally to gcc)','')
,('SETCC', 'Set the C compiler to use. (you can use $CC to refer to the default)','$CC')
,('SETCXX', 'Set the C++ compiler to use. (you can refer to $CXX)','$CXX')
,('CC', 'Set the C compiler to use.', 'gcc')
,('CXX', 'Set the C++ compiler to use.', 'g++')
,PathOption('CCACHE', 'Integrate with CCache', '', PathOption.PathAccept)
,PathOption('DISTCC', 'Invoke C/C++ compiler commands through DistCC', '', PathOption.PathAccept)
,EnumOption('BUILDLEVEL', 'NoBug build level for debugging', 'ALPHA',
allowed_values=('ALPHA', 'BETA', 'RELEASE'))
,BoolOption('DEBUG', 'Build with debugging information and no optimisations', False)

357
admin/scons/BuilderDoxygen.py
View file

@ -33,198 +33,197 @@ from fnmatch import fnmatch
def DoxyfileParse(file_contents):
"""
Parse a Doxygen source file and return a dictionary of all the values.
Values will be strings and lists of strings.
"""
data = {}
import shlex
lex = shlex.shlex(instream = file_contents, posix = True)
lex.wordchars += "*+./-:"
lex.whitespace = lex.whitespace.replace("\n", "")
lex.escape = ""
lineno = lex.lineno
token = lex.get_token()
key = token # the first token should be a key
last_token = ""
key_token = False
next_key = False
new_data = True
def append_data(data, key, new_data, token):
if new_data or len(data[key]) == 0:
data[key].append(token)
else:
data[key][-1] += token
while token:
if token in ['\n']:
if last_token not in ['\\']:
key_token = True
elif token in ['\\']:
pass
elif key_token:
key = token
key_token = False
else:
if token == "+=":
if not data.has_key(key):
""" Parse a Doxygen source file and return a dictionary of all the values.
Values will be strings and lists of strings.
"""
data = {}
import shlex
lex = shlex.shlex(instream = file_contents, posix = True)
lex.wordchars += "*+./-:"
lex.whitespace = lex.whitespace.replace("\n", "")
lex.escape = ""
lineno = lex.lineno
token = lex.get_token()
key = token # the first token should be a key
last_token = ""
key_token = False
next_key = False
new_data = True
def append_data(data, key, new_data, token):
if new_data or len(data[key]) == 0:
data[key].append(token)
else:
data[key][-1] += token
while token:
if token in ['\n']:
if last_token not in ['\\']:
key_token = True
elif token in ['\\']:
pass
elif key_token:
key = token
key_token = False
else:
if token == "+=":
if not data.has_key(key):
data[key] = list()
elif token == "=":
data[key] = list()
elif token == "=":
data[key] = list()
else:
append_data( data, key, new_data, token )
new_data = True
else:
append_data( data, key, new_data, token )
new_data = True
last_token = token
token = lex.get_token()
if last_token == '\\' and token != '\n':
new_data = False
append_data( data, key, new_data, '\\' )
# compress lists of len 1 into single strings
for (k, v) in data.items():
if len(v) == 0:
data.pop(k)
# items in the following list will be kept as lists and not converted to strings
if k in ["INPUT", "FILE_PATTERNS", "EXCLUDE_PATTERNS"]:
continue
if len(v) == 1:
data[k] = v[0]
return data
last_token = token
token = lex.get_token()
if last_token == '\\' and token != '\n':
new_data = False
append_data( data, key, new_data, '\\' )
# compress lists of len 1 into single strings
for (k, v) in data.items():
if len(v) == 0:
data.pop(k)
# items in the following list will be kept as lists and not converted to strings
if k in ["INPUT", "FILE_PATTERNS", "EXCLUDE_PATTERNS"]:
continue
if len(v) == 1:
data[k] = v[0]
return data
def DoxySourceScan(node, env, path):
"""
Doxygen Doxyfile source scanner. This should scan the Doxygen file and add
any files used to generate docs to the list of source files.
"""
default_file_patterns = [
'*.c', '*.cc', '*.cxx', '*.cpp', '*.c++', '*.java', '*.ii', '*.ixx',
'*.ipp', '*.i++', '*.inl', '*.h', '*.hh ', '*.hxx', '*.hpp', '*.h++',
'*.idl', '*.odl', '*.cs', '*.php', '*.php3', '*.inc', '*.m', '*.mm',
'*.py',
]
default_exclude_patterns = [
'*~',
]
sources = []
data = DoxyfileParse(node.get_contents())
if data.get("RECURSIVE", "NO") == "YES":
recursive = True
else:
recursive = False
file_patterns = data.get("FILE_PATTERNS", default_file_patterns)
exclude_patterns = data.get("EXCLUDE_PATTERNS", default_exclude_patterns)
#
# We're running in the top-level directory, but the doxygen
# configuration file is in the same directory as node; this means
# that relative pathnames in node must be adjusted before they can
# go onto the sources list
conf_dir = os.path.dirname(str(node))
for node in data.get("INPUT", []):
if not os.path.isabs(node):
node = os.path.join(conf_dir, node)
if os.path.isfile(node):
sources.append(node)
elif os.path.isdir(node):
if recursive:
for root, dirs, files in os.walk(node):
for f in files:
filename = os.path.join(root, f)
pattern_check = reduce(lambda x, y: x or bool(fnmatch(filename, y)), file_patterns, False)
exclude_check = reduce(lambda x, y: x and fnmatch(filename, y), exclude_patterns, True)
if pattern_check and not exclude_check:
sources.append(filename)
else:
for pattern in file_patterns:
sources.extend(glob.glob("/".join([node, pattern])))
sources = map( lambda path: env.File(path), sources )
return sources
""" Doxygen Doxyfile source scanner.
This should scan the Doxygen file and add any files
used to generate docs to the list of source files.
"""
default_file_patterns = [
'*.c', '*.cc', '*.cxx', '*.cpp', '*.c++', '*.java', '*.ii', '*.ixx',
'*.ipp', '*.i++', '*.inl', '*.h', '*.hh ', '*.hxx', '*.hpp', '*.h++',
'*.idl', '*.odl', '*.cs', '*.php', '*.php3', '*.inc', '*.m', '*.mm',
'*.py',
]
default_exclude_patterns = [
'*~',
]
sources = []
data = DoxyfileParse(node.get_contents())
if data.get("RECURSIVE", "NO") == "YES":
recursive = True
else:
recursive = False
file_patterns = data.get("FILE_PATTERNS", default_file_patterns)
exclude_patterns = data.get("EXCLUDE_PATTERNS", default_exclude_patterns)
#
# We're running in the top-level directory, but the doxygen configuration file
# is in the same directory as node; this means that relative pathnames in node
# must be adjusted before they can go onto the sources list
conf_dir = os.path.dirname(str(node))
for node in data.get("INPUT", []):
if not os.path.isabs(node):
node = os.path.join(conf_dir, node)
if os.path.isfile(node):
sources.append(node)
elif os.path.isdir(node):
if recursive:
for root, dirs, files in os.walk(node):
for f in files:
filename = os.path.join(root, f)
pattern_check = reduce(lambda x, y: x or bool(fnmatch(filename, y)), file_patterns, False)
exclude_check = reduce(lambda x, y: x and fnmatch(filename, y), exclude_patterns, True)
if pattern_check and not exclude_check:
sources.append(filename)
else:
for pattern in file_patterns:
sources.extend(glob.glob("/".join([node, pattern])))
sources = map( lambda path: env.File(path), sources )
return sources
def DoxySourceScanCheck(node, env):
"""Check if we should scan this file"""
return os.path.isfile(node.path)
""" Check if we should scan this file """
return os.path.isfile(node.path)
def DoxyEmitter(source, target, env):
"""Doxygen Doxyfile emitter"""
# possible output formats and their default values and output locations
output_formats = {
"HTML": ("YES", "html"),
"LATEX": ("YES", "latex"),
"RTF": ("NO", "rtf"),
"MAN": ("NO", "man"),
"XML": ("NO", "xml"),
}
data = DoxyfileParse(source[0].get_contents())
targets = []
out_dir = data.get("OUTPUT_DIRECTORY", ".")
# add our output locations
for (k, v) in output_formats.items():
if data.get("GENERATE_" + k, v[0]) == "YES":
targets.append(env.Dir( os.path.join(out_dir, data.get(k + "_OUTPUT", v[1]))) )
# don't clobber targets
for node in targets:
env.Precious(node)
# set up cleaning stuff
for node in targets:
env.Clean(node, node)
return (targets, source)
""" Doxygen Doxyfile emitter """
# possible output formats and their default values and output locations
output_formats = {
"HTML": ("YES", "html"),
"LATEX": ("YES", "latex"),
"RTF": ("NO", "rtf"),
"MAN": ("NO", "man"),
"XML": ("NO", "xml"),
}
data = DoxyfileParse(source[0].get_contents())
targets = []
out_dir = data.get("OUTPUT_DIRECTORY", ".")
# add our output locations
for (k, v) in output_formats.items():
if data.get("GENERATE_" + k, v[0]) == "YES":
targets.append(env.Dir( os.path.join(out_dir, data.get(k + "_OUTPUT", v[1]))) )
# don't clobber targets
for node in targets:
env.Precious(node)
# set up cleaning stuff
for node in targets:
env.Clean(node, node)
return (targets, source)
def generate(env):
"""
Add builders and construction variables for the
Doxygen tool. This is currently for Doxygen 1.4.6.
"""
doxyfile_scanner = env.Scanner(
DoxySourceScan,
"DoxySourceScan",
scan_check = DoxySourceScanCheck,
)
""" Add builders and construction variables for the
Doxygen tool. This is currently for Doxygen 1.4.6.
"""
doxyfile_scanner = env.Scanner(
DoxySourceScan,
"DoxySourceScan",
scan_check = DoxySourceScanCheck,
)
import SCons.Builder
doxyfile_builder = SCons.Builder.Builder(
action = "cd ${SOURCE.dir} && (${DOXYGEN} ${SOURCE.file} 2>&1 |tee ,doxylog)",
emitter = DoxyEmitter,
target_factory = env.fs.Entry,
single_source = True,
source_scanner = doxyfile_scanner,
)
env.Append(BUILDERS = {
'Doxygen': doxyfile_builder,
})
env.AppendUnique(
DOXYGEN = 'doxygen',
)
import SCons.Builder
doxyfile_builder = SCons.Builder.Builder(
action = "cd ${SOURCE.dir} && (${DOXYGEN} ${SOURCE.file} 2>&1 |tee ,doxylog)",
emitter = DoxyEmitter,
target_factory = env.fs.Entry,
single_source = True,
source_scanner = doxyfile_scanner,
)
env.Append(BUILDERS = {
'Doxygen': doxyfile_builder,
})
env.AppendUnique(
DOXYGEN = 'doxygen',
)
def exists(env):
"""
Make sure doxygen exists.
"""
return env.Detect("doxygen")
""" Make sure doxygen exists.
"""
return env.Detect("doxygen")

12
admin/scons/BuilderGCH.py
View file

@ -61,7 +61,7 @@ def setup_dependency(target,source,env, key):
if header_path in [x.path for x in deps]:
print "Precompiled header(%s) %s \t <--- %s" % (key,header_path,source[0])
env.Depends(target, header)
def static_pch_emitter(target,source,env):
SCons.Defaults.StaticObjectEmitter( target, source, env )
@ -73,9 +73,9 @@ def shared_pch_emitter(target,source,env):
setup_dependency(target,source,env, key='GCH-sh')
return (target, source)
def generate(env):
"""
Add builders and construction variables for the Gch builder.
""" Add builders and construction variables for the Gch builder.
"""
env.Append(BUILDERS = {
'gch': env.Builder(
@ -87,7 +87,7 @@ def generate(env):
target_factory = env.fs.File,
),
})
try:
bld = env['BUILDERS']['GCH']
bldsh = env['BUILDERS']['GCH-sh']
@ -96,11 +96,11 @@ def generate(env):
bldsh = GchShBuilder
env['BUILDERS']['PrecompiledHeader'] = bld
env['BUILDERS']['PrecompiledHeaderShared'] = bldsh
env['GCHCOM'] = '$CXX -o $TARGET -x c++-header -c $CXXFLAGS $_CCCOMCOM $SOURCE'
env['GCHSHCOM'] = '$CXX -o $TARGET -x c++-header -c $SHCXXFLAGS $_CCCOMCOM $SOURCE'
env['GCHSUFFIX'] = '.gch'
for suffix in SCons.Util.Split('.c .C .cc .cxx .cpp .c++'):
env['BUILDERS']['StaticObject'].add_emitter( suffix, static_pch_emitter )
env['BUILDERS']['SharedObject'].add_emitter( suffix, shared_pch_emitter )

36
admin/scons/Buildhelper.py
View file

@ -112,7 +112,43 @@ def getDirname(dir):
dir,_ = os.path.split(dir)
_, name = os.path.split(dir)
return name
def checkCommandOption(env, optID, val=None, cmdName=None):
""" evaluate and verify an option, which may point at a command.
besides specifying a path, the option may read True, yes or 1,
denoting that the system default for this command should be used.
@return: True, if the key has been expanded and validated,
False, if this failed and the key was removed
"""
if not val:
if not env.get(optID): return False
else:
val = env.get(optID)
if val=='True' or val=='true' or val=='yes' or val=='1' or val == 1 :
if not cmdName:
print "WARNING: no default for %s, please specify a full path." % optID
del env[optID]
return False
else:
val = env.WhereIs(cmdName)
if not val:
print "WARNING: %s not found, please specify a full path" % cmdName
del env[optID]
return False
if not os.path.isfile(val):
val = env.WhereIs(val)
if val and os.path.isfile(val):
env[optID] = val
return True
else:
del env[optID]
return False
def RegisterIcon_Builder(env, renderer):

76
admin/scons/ToolCCache.py Normal file
View file

@ -0,0 +1,76 @@
# -*- python -*-
##
## ToolCCache.py - SCons tool for integrating with CCache compiler cache
##
# Copyright (C) Lumiera.org and FreeOrion.org
# 2008, Hermann Vosseler <Ichthyostega@web.de>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of
# the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#####################################################################
# This SCons builder was extracted from http://www.freeorion.org/
# FreeOrion is an open-source platform-independent galactic conquest game
#
# history: 12/2008 adapted for Lumiera build system
import os
from Buildhelper import *
def generate(env):
""" Modify the environment such as to redirect any
C/C++ compiler invocations through CCache, while using
CCache config variables found in the os.environment.
"""
if not exists(env): return
assert env['CCACHE']
if not env['CCACHE'] in env['CC']:
env['CC'] = env.subst('$CCACHE $CC')
if not env['CCACHE'] in env['CXX']:
env['CXX'] = env.subst('$CCACHE $CXX')
print env.subst("* Build using $CCACHE")
for i in ['HOME'
,'CCACHE_DIR'
,'CCACHE_TEMPDIR'
,'CCACHE_LOGFILE'
,'CCACHE_PATH'
,'CCACHE_CC'
,'CCACHE_CPP2'
,'CCACHE_PREFIX'
,'CCACHE_DISABLE'
,'CCACHE_READONLY'
,'CCACHE_NOSTATS'
,'CCACHE_NLEVELS'
,'CCACHE_HARDLINK'
,'CCACHE_RECACHE'
,'CCACHE_UMASK'
,'CCACHE_HASHDIR'
,'CCACHE_UNIFY'
,'CCACHE_EXTENSION'
]:
if os.environ.has_key(i) and not env.has_key(i):
env['ENV'][i] = os.environ[i]
def exists(env):
""" Ensure CCache is available.
"""
return checkCommandOption(env, 'CCACHE', cmdName='ccache')

65
admin/scons/ToolDistCC.py Normal file
View file

@ -0,0 +1,65 @@
# -*- python -*-
##
## ToolDistCC.py - SCons tool for distributed compilation using DistCC
##
# Copyright (C) Lumiera.org and FreeOrion.org
# 2008, Hermann Vosseler <Ichthyostega@web.de>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of
# the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#####################################################################
# This SCons builder was extracted from http://www.freeorion.org/
# FreeOrion is an open-source platform-independent galactic conquest game
#
# history: 12/2008 adapted for Lumiera build system
import os
from Buildhelper import *
def generate(env):
""" Modify the environment such as to redirect any
C/C++ compiler invocations through DistCC. Additionally
pull in the environment config variables used by DistCC
"""
if not exists(env): return
assert env['DISTCC']
if not env['DISTCC'] in env['CC']:
env['CC'] = env.subst('$DISTCC $CC')
if not env['DISTCC'] in env['CXX']:
env['CXX'] = env.subst('$DISTCC $CXX')
print env.subst("* Build using $DISTCC")
for i in ['HOME'
,'DISTCC_HOSTS'
,'DISTCC_VERBOSE'
,'DISTCC_FALLBACK'
,'DISTCC_LOG'
,'DISTCC_MMAP'
,'DISTCC_SAVE_TEMPS'
,'DISTCC_TCP_CORK'
,'DISTCC_SSH'
]:
if os.environ.has_key(i) and not env.has_key(i):
env['ENV'][i] = os.environ[i]
def exists(env):
""" Ensure DistCC exists.
"""
return checkCommandOption(env, 'DISTCC', cmdName='distcc')