From b30ca384cb14c5cca7595bb9ea27b0f187974cff Mon Sep 17 00:00:00 2001 From: Christian Thaeter Date: Sat, 9 Jan 2010 18:44:26 +0100 Subject: [PATCH] new test.sh|.conf from nobug --- tests/test.conf | 1 + tests/test.sh | 474 ++++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 397 insertions(+), 78 deletions(-) create mode 100755 tests/test.conf diff --git a/tests/test.conf b/tests/test.conf new file mode 100755 index 000000000..c86e03260 --- /dev/null +++ b/tests/test.conf @@ -0,0 +1 @@ +LOGSUPPRESS='^\(\*\*[0-9]*\*\* \)\?[0-9]\{10,\}: \(TRACE\|INFO\|NOTICE\|WARNING\|ERR\|TODO\|PLANNED\|FIXME\|DEPRECATED\|UNIMPLEMENTED\):' diff --git a/tests/test.sh b/tests/test.sh index 89de9ae54..19df30e38 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Copyright (C) Lumiera.org -# 2007 - 2008, Christian Thaeter +# 2007 - 2010, Christian Thaeter # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as @@ -25,35 +25,134 @@ # TESTMODE=FIRSTFAIL # stop testing on the first failure -export LC_ALL=C -NOBUG_LOGREGEX='^\(\*\*[0-9]*\*\* \)\?[0-9]\{10,\}: \(TRACE\|INFO\|NOTICE\|WARNING\|ERR\|TODO\|PLANNED\|FIXME\|DEPRECATED\|UNIMPLEMENTED\):' +#intro Test.sh +#intro ======= +#intro Christian Thäter +#intro +#intro A shell script driving software tests. +#intro + +#=intro +#=tests Writing tests +# =run +#=config +#=configf +# =make +# =control +#=valgrind +#=libtool +#_ +#_ Index +#_ ----- +#_ +#=index +#_ + +LC_ALL=C + +#config HEAD- Configuration; configuration; configure tests +#config +#config PARA LOGSUPPRESS; LOGSUPPRESS; suppress certain lines from stderr +#config LOGSUPPRESS='^\(\*\*[0-9]*\*\* \)\?[0-9]\{10,\}: \(TRACE\|INFO\|NOTICE\|WARNING\|ERR\):' +#config +#config Programms sometimes emit additional diagnostics on stderr which is volatile and not necessary for +#config validating the output the `LOGSUPRESS` variable can be set to a regex to filter this things out. +#config The default as shown above filters some NoBug annotations and non fatal logging out. +#config +LOGSUPPRESS='^\(\*\*[0-9]*\*\* \)\?[0-9]\{10,\}: \(TRACE\|INFO\|NOTICE\|WARNING\|ERR\):' + +#config PARA Resource Limits; ulimit; constrain resource limits +#config It is possible to set some limits for tests to protect the system against really broken cases. +#config Since running under valgrind takes consinderable more resources there are separate variants for +#config limits when running under valgrind. +#config +#config LIMIT_CPU=5 +#config Maximal CPU time the test may take after it will be killed with SIGXCPU. This protects agaist Lifelocks. +#config +#config LIMIT_TIME=10 +#config Maximal wall-time a test may take after this it will be killed with SIGKILL. Protects against Deadlocks. +#config +#config LIMIT_VSZ=524288 +#config Maximal virtual memory size the process may map, allocations/mappings will fail when this limit is reached. +#config Protects against memory leaks. +#config +#config LIMIT_VG_CPU=20 +#config LIMIT_VG_TIME=30 +#config LIMIT_VG_VSZ=524288 +#config Same variables again with limits when running under valgrind. +#config +LIMIT_CPU=5 +LIMIT_TIME=10 +LIMIT_VSZ=524288 +LIMIT_VG_CPU=20 +LIMIT_VG_TIME=30 +LIMIT_VG_VSZ=524288 + + +#configf HEAD~ Configuration Files; configuration files; define variables to configure the test +#configf +#configf `test.sh` reads config files from the following location if they are exist +#configf * 'test.conf' from the current directory +#configf * '$srcdir/test.conf' `$srcdir` is set by autotools +#configf * '$srcdir/tests/test.conf' `tests/` is suspected as default directory for tests +#configf * '$TEST_CONF' a user defineable variable to point to a config file +#configf +test -f 'test.conf' && source test.conf +test -n "$srcdir" -a -e "$srcdir/test.conf" && source "$srcdir/test.conf" +test -n "$srcdir" -a -e "$srcdir/tests/test.conf" && source "$srcdir/tests/test.conf" +test -n "$TEST_CONF" -a -e "$TEST_CONF" && source "$TEST_CONF" + + arg0="$0" -srcdir="$(dirname "$arg0")" -vgsuppression_mangle='/^\(\(==\)\|\(\*\*\)[0-9]*\(==\)\|\(\*\*\)\)\|\(\(\*\*[0-9]*\*\* \)\?[0-9]\{10,\}: ECHO:\)/d;' +TESTDIR="$(dirname "$arg0")" -ulimit -S -t 5 -v 524288 + +#libtool HEAD Libtool; libtool; support for libtool +#libtool When test.sh detects the presence of './libtool' it runs all tests with +#libtool `./libtool --mode=execute`. +#libtool +LIBTOOL_EX= +if test -x ./libtool; then + LIBTOOL_EX="./libtool --mode=execute" +fi + +#valgrind HEAD- Valgrind; valgrind; valgrind support +#valgrind Test are run under valgrind supervision by default, if not disabled. +#valgrind +#valgrind PARA VALGRINDFLAGS; VALGRINDFLAGS; control valgrind options +#valgrind VALGRINDFLAGS="--leak-check=yes --show-reachable=yes" +#valgrind +#valgrind `VALGRINDFLAGS` define the options which are passed to valgrind. This can be used to override +#valgrind the defaults or switching the valgrind tool. The special case `VALGRINDFLAGS=DISABLE` will disable +#valgrind valgrind for the tests. +#valgrind +#valgrind HEAD~ Generating Valgrind Suppression Files; vgsuppression; ignore false positives +#valgrind When there is a 'vgsuppression' executable in the current dir (build by something external) then +#valgrind test.sh uses this to generate a local 'vgsuppression.supp' file and uses that to suppress +#valgrind all errors generated by 'vgsuppression'. The Idea here is that one adds code which triggers known +#valgrind false positives in 'vgsuppression'. Care must be taken that this file is simple and does +#valgrind not generate true positives. +#valgrind +ulimit -S -t ${LIMIT_CPU:-5} -v ${LIMIT_VSZ:-524288} valgrind="" +LIMIT_TIME_REAL="$LIMIT_TIME" if [ "$VALGRINDFLAGS" = 'DISABLE' ]; then echo "valgrind explicit disabled" else if [ "$(which valgrind)" ]; then - ulimit -S -t 20 + ulimit -S -t ${ULIMIT_VG_CPU:-20} -v ${ULIMIT_VG_VSZ:-524288} + LIMIT_TIME_REAL="$LIMIT_VG_TIME" if [[ -x 'vgsuppression' ]]; then if [[ 'vgsuppression' -nt 'vgsuppression.supp' ]]; then echo 'generating valgrind supression file' - if [[ -x ".libs/vgsuppression" ]]; then - ./libtool --mode=execute valgrind --leak-check=yes --show-reachable=yes -q --gen-suppressions=all vgsuppression 2>&1 \ - | sed -e "$vgsuppression_mangle" >vgsuppression.supp - else - valgrind --leak-check=yes --show-reachable=yes -q --gen-suppressions=all ./vgsuppression 2>&1 \ - | sed -e "$vgsuppression_mangle" >vgsuppression.supp - fi + $LIBTOOL_EX $(which valgrind) ${VALGRINDFLAGS:---leak-check=yes --show-reachable=yes} -q --gen-suppressions=all vgsuppression 2>&1 \ + | awk '/^{/ {i = 1;} /^}/ {i = 0; print $0;} {if (i == 1) print $0;}' >vgsuppression.supp fi - valgrind="$(which valgrind) --leak-check=yes --show-reachable=no --suppressions=vgsuppression.supp -q $VALGRINDFLAGS" + valgrind="$(which valgrind) ${VALGRINDFLAGS:---leak-check=yes --show-reachable=no} --suppressions=vgsuppression.supp -q" else - valgrind="$(which valgrind) --leak-check=yes --show-reachable=no -q $VALGRINDFLAGS" + valgrind="$(which valgrind) ${VALGRINDFLAGS:---leak-check=yes --show-reachable=no -q}" fi else echo "no valgrind found, go without it" @@ -77,40 +176,193 @@ fi date >,testlog -function compare_regex() # rxfile plainfile +function compare_template() # template plainfile { - local regex + local template local line local miss local lineno=1 - local regexno=1 + local templateno=1 { - IFS='' read -u 3 -r regex || return 0 + IFS='' read -u 3 -r template || return 0 IFS='' read -u 4 -r line || { echo "no output"; return 1; } while true; do - if [[ $line =~ $regex ]]; then - IFS='' read -u 4 -r line || - if IFS='' read -u 3 -r regex; then - echo "premature end in output, expecting: '$regex':$regexno" - return 1 + local cmd="${template%%:*}:" + local arg="${template#*: }" + + case $cmd in + 'regex_cont:') + if [[ $line =~ $arg ]]; then + IFS='' read -u 4 -r line || + if IFS='' read -u 3 -r template; then + echo "premature end in output, expecting $template:$templateno" + return 1 + else + return 0 + fi + : $((++lineno)) + miss=0 else - return 0 + if [[ $((++miss)) -gt 1 ]]; then + echo -e "'$line':$lineno\ndoes not match\n$template:$templateno" + return 1 + fi + IFS='' read -u 3 -r template || { echo "more output than expected: '$line':$lineno"; return 1; } + : $((++templateno)) fi - : $((++lineno)) - miss=0 - else - if [[ $((++miss)) -gt 1 ]]; then - echo -e "'$line':$lineno\ndoes not match\n'$regex':$regexno" + ;; + 'literal:') + if [[ "$line" = "$arg" ]]; then + IFS='' read -u 3 -r template && IFS='' read -u 4 -r line || { + return 1 + } + else + echo -e "'$line':$lineno\ndoes not match\n$template:$templateno" return 1 fi - IFS='' read -u 3 -r regex || { echo "more output than expected: '$line':$lineno"; return 1; } - : $((++regexno)) - fi + ;; + *) + echo "UNKOWN MATCH COMMAND '$cmd'" 1>&2 + exit + ;; + esac done } 3<"$1" 4<"$2" } +#tests HEAD- Writing Tests; tests; how to write testsuites +#tests Tests are nothing more than bash scripts with some functions from the test.sh +#tests framework defined. Test.sh looks in the current directory for all files which ending in .test +#tests and runs them in alphabetical order. The selection of this tests can be constrained with the +#tests `TESTSUITES` environment variable. +#tests +#tests HEAD~ Testsuites; test files; writing tests +#tests It is common to start the name of the '.test' files with a 2 digi number to give them a proper +#tests order: '10foo.test', '20bar.test' and so on. Each such test should only test a certain aspect of +#tests the system. You have to select the testing binary with the `TESTING` function and then write +#tests certain TEST's defining how the test should react. Since tests are shell scripts it is possible +#tests to add some supplemental commands there to set and clean up the given test environment. +#tests +#tests HEAD^ TESTING; TESTING; set the test binary +#tests TESTING "message" test_program +#tests +#tests Selects the test binary for the follwing tests, prints an informal message. +#tests +#tests `message`:: +#tests message to be printed +#tests `test_program`:: +#tests an existing program to drive the tests or a shell function +#tests +#tests ---- +#tests TESTING "Testing a.out" ./a.out +#tests ---- +#tests +function TESTING() +{ + echo + echo "$1" + echo -e "\n#### $1, $TESTFILE, $2" >>,testlog + TESTBIN="$2" +} + +#tests HEAD^ TEST; TEST; single test +#tests TEST "title" arguments.. <>,send_stdin + 'if:') + if $arg; then + condstack="1$condstack" + else + condstack="0$condstack" + fi ;; - 'out') - echo "$arg" >>,expect_stdout + 'elseif:') + if [[ "${condstack:0:1}" = "0" ]]; then + if $arg; then + condstack="1${condstack:1}" + else + condstack="0${condstack:1}" + fi + else + condstack="2${condstack:1}" + fi ;; - 'err') - echo "$arg" >>,expect_stderr + 'else:') + if [[ "${condstack:0:1}" != "0" ]]; then + condstack="0${condstack:1}" + else + condstack="1${condstack:1}" + fi ;; - 'return') - expect_return="$arg" - ;; - '#'*|'') - : + 'endif:') + condstack="${condstack:1}" ;; *) - echo "UNKOWN TEST COMMAND '$cmd'" 1>&2 - exit + if [[ "${condstack:0:1}" = "1" ]]; then + case $cmd in + 'msg:') + echo "MSG $arg" + ;; + 'in:') + echo "$arg" >>,send_stdin + ;; + 'out:') + echo "regex_cont: $arg" >>,expect_stdout + ;; + 'err:') + echo "regex_cont: $arg" >>,expect_stderr + ;; + 'out-lit:') + echo "literal: $arg" >>,expect_stdout + ;; + 'err-lit:') + echo "literal: $arg" >>,expect_stderr + ;; + 'return:') + expect_return="$arg" + ;; + '#'*|':') + : + ;; + *) + echo "UNKOWN TEST COMMAND '$cmd'" 1>&2 + exit + ;; + esac + fi ;; esac done @@ -185,7 +486,7 @@ function TEST() if declare -F | grep $TESTBIN >&/dev/null; then CALL= elif test -x $TESTBIN; then - CALL="env $TESTBIN_PREFIX" + CALL="env $LIBTOOL_EX $valgrind" else CALL='-' echo -n >,stdout @@ -194,17 +495,31 @@ function TEST() fi if test "$CALL" != '-'; then - if test -f ,send_stdin; then - $CALL $TESTBIN "$@" <,send_stdin 2>,stderr >,stdout + ( + $CALL $TESTBIN "$@" <,send_stdin 2>,stderr >,stdout + echo $? >,return + ) & else - $CALL $TESTBIN "$@" 2>,stderr >,stdout + ( + $CALL $TESTBIN "$@" 2>,stderr >,stdout + echo $? >,return + ) & fi &>/dev/null - return=$? + pid=$! + + # watchdog + ( sleep $LIMIT_TIME_REAL && kill -KILL $pid ) &>/dev/null & + wpid=$! + wait $pid + return=$(<,return) + if [[ "$return" -le 128 ]]; then + kill -INT $wpid >&/dev/null + fi if test -f ,expect_stdout; then - grep -v "$NOBUG_LOGREGEX" <,stdout >,tmp - if ! compare_regex ,expect_stdout ,tmp >>,cmptmp; then + grep -v "$LOGSUPPRESS" <,stdout >,tmp + if ! compare_template ,expect_stdout ,tmp >>,cmptmp; then echo "unexpected data on stdout" >>,testtmp cat ,cmptmp >>,testtmp ((fails+=1)) @@ -213,9 +528,9 @@ function TEST() fi if test -f ,expect_stderr; then - grep -v "$NOBUG_LOGREGEX" <,stderr >,tmp + grep -v "$LOGSUPPRESS" <,stderr >,tmp cat ,tmp >>,testtmp - if ! compare_regex ,expect_stderr ,tmp >>,cmptmp; then + if ! compare_template ,expect_stderr ,tmp >>,cmptmp; then echo "unexpected data on stderr" >>,testtmp cat ,cmptmp >>,testtmp ((fails+=1)) @@ -256,6 +571,23 @@ function TEST() fi } +#tests HEAD^ PLANNED; PLANNED; deactivated test +#tests PLANNED "title" arguments.. <&2 - if test -f $i; then - source $i + echo "### $TESTFILE" >&2 + if test -f $TESTFILE; then + source $TESTFILE fi done echo @@ -295,20 +627,6 @@ function RUNTESTS() } } -function TESTING() -{ - echo - echo "$1" - echo -e "\n#### $1" >>,testlog - - if [[ -x ".libs/$2" ]]; then - TESTBIN_PREFIX="./libtool --mode=execute $valgrind" - else - TESTBIN_PREFIX="$valgrind" - fi - TESTBIN="$2" -} - TESTSUITES="${TESTSUITES}${1:+${TESTSUITES:+,}$1}" RUNTESTS