diff --git a/tests/test.sh b/tests/test.sh index 31c92da5b..7c2ca1427 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash # Copyright (C) Lumiera.org # 2007 - 2008, Christian Thaeter # @@ -26,18 +26,34 @@ # stop testing on the first failure export LC_ALL=C +NOBUG_LOGREGEX='^\(\*\*[0-9]*\*\* \)\?[0-9]\{10,\}: \(TRACE\|INFO\|NOTICE\|WARNING\|ERR\):' arg0="$0" srcdir="$(dirname "$arg0")" -ulimit -S -t 2 -v 524288 +ulimit -S -t 5 -v 524288 valgrind="" if [ "$VALGRINDFLAGS" = 'DISABLE' ]; then echo "valgrind explicit disabled" else if [ "$(which valgrind)" ]; then - valgrind="$(which valgrind) --leak-check=yes --show-reachable=yes -q $VALGRINDFLAGS" - ulimit -S -t 10 + ulimit -S -t 20 + if [[ -x 'vgsuppression' ]]; then + if [[ 'vgsuppression' -nt 'vgsuppression.supp' ]]; then + echo 'generating valgrind supression file' + + if [[ -x ".libs/vgsuppression" ]]; then + ./libtool --mode=execute valgrind --leak-check=yes --show-reachable=yes -q --gen-suppressions=all vgsuppression 2>&1 \ + | sed '/^\(==\)\|\(\*\*\)[0-9]*\(==\)\|\(\*\*\)/d;' >vgsuppression.supp + else + valgrind --leak-check=yes --show-reachable=yes -q --gen-suppressions=all ./vgsuppression 2>&1 \ + | sed '/^\(==\)\|\(\*\*\)[0-9]*\(==\)\|\(\*\*\)/d;' >vgsuppression.supp + fi + fi + valgrind="$(which valgrind) --leak-check=yes --show-reachable=no --suppressions=vgsuppression.supp -q $VALGRINDFLAGS" + else + valgrind="$(which valgrind) --leak-check=yes --show-reachable=no -q $VALGRINDFLAGS" + fi else echo "no valgrind found, go without it" fi @@ -50,6 +66,7 @@ TESTCNT=0 SKIPCNT=0 FAILCNT=0 + # the old testlog if existing will be used to check for previous test states if test -f ,testlog; then mv ,testlog ,testlog.pre @@ -59,6 +76,40 @@ fi date >,testlog +function compare_regex() # rxfile plainfile +{ + local regex + local line + local miss + local lineno=1 + local regexno=1 + { + IFS='' read -u 3 -r regex || return 0 + IFS='' read -u 4 -r line || { echo "no output"; return 1; } + while true; do + if [[ $line =~ $regex ]]; then + IFS='' read -u 4 -r line || + if IFS='' read -u 3 -r regex; then + echo "premature end in output, expecting: '$regex':$regexno" + return 1 + else + return 0 + fi + : $((++lineno)) + miss=0 + else + if [[ $((++miss)) -gt 1 ]]; then + echo -e "'$line':$lineno\ndoes not match\n'$regex':$regexno" + return 1 + fi + IFS='' read -u 3 -r regex || { echo "more output than expected: '$line':$lineno"; return 1; } + : $((++regexno)) + fi + done + } 3<"$1" 4<"$2" +} + + function TEST() { name="$1" @@ -69,7 +120,11 @@ function TEST() while read -r line; do cmd="${line%%:*}" - arg="${line#*: }" + arg="${line#*:}" + arg="${arg:1}" + if [[ ! "$arg" ]]; then + arg='^$' + fi expect_return=0 case $cmd in @@ -85,6 +140,9 @@ function TEST() 'return') expect_return="$arg" ;; + '#'*|'') + : + ;; *) echo "UNKOWN TEST COMMAND '$cmd'" 1>&2 exit @@ -128,36 +186,45 @@ function TEST() ((fails+=1)) else + if test -f ,send_stdin; then - cat ,send_stdin | $valgrind $TESTBIN "$@" 2>,stderr >,stdout + env $TESTBIN_PREFIX $TESTBIN "$@" <,send_stdin 2>,stderr >,stdout else - $valgrind $TESTBIN "$@" 2>,stderr >,stdout + env $TESTBIN_PREFIX $TESTBIN "$@" 2>,stderr >,stdout fi &>/dev/null return=$? if test -f ,expect_stdout; then - if ! cmp ,expect_stdout ,stdout &>/dev/null; then + grep -v "$NOBUG_LOGREGEX" <,stdout >,tmp + if ! compare_regex ,expect_stdout ,tmp >>,cmptmp; then echo "unexpected data on stdout" >>,testtmp - grep -v ': \(TRACE\|INFO\|NOTICE\|WARNING\|ERR\):' <,stdout >,tmp - diff -ua ,expect_stdout ,tmp >>,testtmp - rm ,tmp + cat ,cmptmp >>,testtmp ((fails+=1)) fi + rm ,tmp ,cmptmp fi if test -f ,expect_stderr; then - if ! cmp ,expect_stderr ,stderr &>/dev/null; then + grep -v "$NOBUG_LOGREGEX" <,stderr >,tmp + cat ,tmp >>,testtmp + if ! compare_regex ,expect_stderr ,tmp >>,cmptmp; then echo "unexpected data on stderr" >>,testtmp - grep -v ': \(TRACE\|INFO\|NOTICE\|WARNING\|ERR\):' <,stderr >,tmp - diff -ua ,expect_stderr ,tmp >>,testtmp - rm ,tmp + cat ,cmptmp >>,testtmp ((fails+=1)) fi + rm ,tmp ,cmptmp fi - if test "$expect_return" != "$return"; then - echo "unexpected return value $return" >>,testtmp - ((fails+=1)) + if [[ "${expect_return:0:1}" = '!' ]]; then + if [[ "${expect_return#\!}" = "$return" ]]; then + echo "unexpected return value $return, expected $expect_return" >>,testtmp + ((fails+=1)) + fi + else + if [[ "${expect_return}" != "$return" ]]; then + echo "unexpected return value $return, expected $expect_return" >>,testtmp + ((fails+=1)) + fi fi fi @@ -224,7 +291,14 @@ function TESTING() { echo echo "$1" - TESTBIN=$2 + echo -e "\n#### $1" >>,testlog + + if [[ -x ".libs/$2" ]]; then + TESTBIN_PREFIX="./libtool --mode=execute $valgrind" + else + TESTBIN_PREFIX="$valgrind" + fi + TESTBIN="$2" } TESTSUITES="${TESTSUITES}${1:+${TESTSUITES:+,}$1}"