diff --git a/tests/00test.tests b/tests/00test.tests index a0c4c9761..0b894982b 100644 --- a/tests/00test.tests +++ b/tests/00test.tests @@ -1,8 +1,13 @@ - TESTING "test system selftest" cat -TEST "test" < # Hermann Vosseler @@ -23,6 +22,11 @@ # # TESTMODE=FAST # run only tests which recently failed +# +# TESTMODE=FIRSTFAIL +# stop testing on the first failure + +export LC_ALL=C arg0="$0" srcdir=$(dirname "$arg0") @@ -47,6 +51,7 @@ TESTCNT=0 SKIPCNT=0 FAILCNT=0 +# the old testlog if existing will be used to check for previous test states if test -f ,testlog; then mv ,testlog ,testlog.pre else @@ -59,8 +64,34 @@ function TEST() { name="$1" shift - cat >,cmp - echo -n "" >,out + rm -f ,send_stdin + rm -f ,expect_stdout + rm -f ,expect_stderr + + while read -r line; do + cmd="${line%%:*}" + arg="${line#*: }" + expect_return=0 + + case $cmd in + 'in') + echo "$arg" >>,send_stdin + ;; + 'out') + echo "$arg" >>,expect_stdout + ;; + 'err') + echo "$arg" >>,expect_stderr + ;; + 'return') + expect_return=$arg + ;; + *) + echo "UNKOWN TEST COMMAND '$cmd'" 1>&2 + exit + ;; + esac + done echo -n "TEST $name: " echo -en "\nTEST $name: $* " >>,testlog @@ -87,16 +118,52 @@ function TEST() esac TESTCNT=$(($TESTCNT + 1)) - if $valgrind $TESTBIN "$@" 2>&1 | tee ,tmp | grep -v ': \(TRACE\|INFO\|NOTICE\|WARNING\|ERR\):' | cmp ,cmp - &>/dev/null; then + + fails=0 + + if test -f ,send_stdin; then + cat ,send_stdin | $valgrind $TESTBIN "$@" 2>,stderr >,stdout + return=$? + else + $valgrind $TESTBIN "$@" 2>,stderr >,stdout + return=$? + fi + + echo -n >,testtmp + + if test -f ,expect_stdout; then + if ! cmp ,expect_stdout ,stdout &>/dev/null; then + echo "unexpected data on stdout" >>,testtmp + grep -v ': \(TRACE\|INFO\|NOTICE\|WARNING\|ERR\):' <,stdout >,tmp + diff -ua ,expect_stdout ,tmp >>,testtmp + rm ,tmp + ((fails+=1)) + fi + fi + + if test -f ,expect_stderr; then + if ! cmp ,expect_stderr ,stderr &>/dev/null; then + echo "unexpected data on stderr" >>,testtmp + grep -v ': \(TRACE\|INFO\|NOTICE\|WARNING\|ERR\):' <,stderr >,tmp + diff -ua ,expect_stderr ,tmp >>,testtmp + rm ,tmp + ((fails+=1)) + fi + fi + + if test $expect_return != $return; then + echo "unexpected return value $return" >>,testtmp + ((fails+=1)) + fi + + if test $fails -eq 0; then echo ".. OK$MSGOK" echo ".. OK$MSGOK" >>,testlog else echo ".. FAILED$MSGFAIL"; echo ".. FAILED$MSGFAIL" >>,testlog - grep -v ': \(TRACE\|INFO\|NOTICE\|WARNING\|ERR\):' <,tmp >,out - diff -ua ,cmp ,out >>,testlog - # grep 'DEBUG:\|==.*==' <,tmp >>,testlog - cat ,tmp >>,testlog + cat ,testtmp >>,testlog + rm ,testtmp echo END >>,testlog FAILCNT=$(($FAILCNT + 1)) case $TESTMODE in @@ -123,7 +190,6 @@ function RUNTESTS() source $i done echo - rm ,cmp ,out ,tmp if [ $FAILCNT = 0 ]; then echo " ... PASSED $(($TESTCNT - $SKIPCNT)) TESTS, $SKIPCNT SKIPPED" #rm ,testlog diff --git a/wiki/index.html b/wiki/index.html index 51da7bc71..5e9b52bef 100755 --- a/wiki/index.html +++ b/wiki/index.html @@ -795,7 +795,7 @@ This distributed wiki might be used instead the pipapo.org wiki, investigate tha Wiki works. It is simple to use and just flexible enough to handle the task. I don't go to install any other software for such tasks on my server. While the design progresses I'd propose to move our work into git repositories and eventually phase this wiki pages out anyways. I'd rather like to start out distributed/git right away .. but git gives us only a fine storage layer, for a design process we need some good presentation layer (later when using git and starting the implementation everyones favorite editor serves for that) I have no better ideas yet to solve the presentation problem other than using this wiki (or maybe Bouml). -
+
This 'index.html' becomes the entry point of some tiddlywikis managed under git. There is a 'empty.html' in the same folder serving as template for generating new wikis. Please refrain from editing it.
 
 * I started a GitNotes where we will collect some information about git, howto and special setups
@@ -812,6 +812,7 @@ to get started, we create design drafts emphasizing different aspects and region
 * Ichthyo focuses mainly on the Render Engine and its interconnection to the EDL, [[see this separate page|renderengine.html]]
 * cehteh works on the data backend draft, see [[this page|backend.html]]
 * Some tools which don't fit somewhere else and are used everywhere are put into a [[Support Library|support_library.html]]
+* [[Description of the Test System|TestSh]]
 
 !Coding Structures
 next we should //start thinking// on how to organize several aspects of the practical coding...
@@ -3452,6 +3453,41 @@ Portions written by Luke Blanshard are hereby released into the public domain.
 <div macro="tasksum end"></div>
 <div class="tagglyTagging" macro="tagglyListWithSort"></div>
 <!--}}}-->
+
+
+
+
! The Test Script
+There is a simple test script in tests which will be used to drive various tests. All tests are run under valgrind control if available unless {{{VALGRINDFLAGS=DISABLE}}} is defined. This test script is integrated in the automake build and will be used when {{{make check}}} is called.
+
+! Options for running the Test Suite
+One may define {{{TESTMODE}}} containing any one of the following strings:
+* {{{FAST}}} only run tests which failed recently
+* {{{FIRSTFAIL}}} abort the tests at the first failure
+
+putting this together a very fast check (when using automake) while hacking on the source would look like:
+{{{
+VALGRINDFLAGS=DISABLE TESTMODE=FAST+FIRSTFAIL make check
+}}}
+This doesnt catch all errors, notably not regressions, but is useful to do coarse checks.
+
+Running the testsuite with everything enabled is just:
+{{{
+make check
+}}}
+
+! Writing Tests
+Tests are written in files named {{{NNname.tests}}} in the {{{tests}}} dir, where NN is a number defining the order of the various test files, 'name' should be a descriptive name about whats going to be tested.
+
+In a .tests file one has to define following:
+* {{{TESTING <description> <binary>}}} set the program binary to be tested to <binary>, <description> should be a string which is displayed as header before running following tests
+* {{{TEST <description> [optargs] <<END}}} run the previously set binary with [optargs], <description> is displayed when running this test. A detailed test spec must follow this command and be terminated with {{{END}}} on a single line. Test specs can contain following statements:
+** {{{in: <line>}}} send <line> to stdin of the test program
+** {{{out: <line>}}} expect <line> at the stdout of the test program
+** {{{err: <line>}}} expect <line> at the stderr of the test program
+** {{{return: <status>}}} expect <status> as exit status of the program
+
+If no {{{out:}}} or {{{err:}}} is given, stdout and stderr are not considered as part of the test. If no {{{return:}}} is given, then 0 is expected.
+