Test suite runner: pass exit code = 5 when having caught an exception

src/lib/test/suite.cpp

@@ -123,6 +123,10 @@ namespace test {
   /** "magic" groupID containing all registered testcases */
   const string Suite::ALLGROUP = "ALL";
   
+  /** exit code returned when any individual test threw */
+  const int Suite::EXCEPTION_THROWN = 5;
+  const int Suite::TEST_OK = 0;
+
   
   
   /** create a suite comprised of all the testcases 
@@ -132,6 +136,7 @@ namespace test {
    */
   Suite::Suite(string groupID) 
     : groupID_(groupID)
+    , exitCode_(0)
   {
     REQUIRE( !isnil(groupID) );
     TRACE(test, "Test-Suite( groupID=%s )\n", groupID.c_str () );
@@ -139,6 +144,15 @@ namespace test {
     if (!testcases.getGroup(groupID))
       throw lumiera::error::Invalid ("empty testsuite");
   }
+  
+  
+  int
+  Suite::getExitCode ()  const
+    {
+      return exitCode_;
+    }
+  
+  
     
 #define VALID(test,testID) \
   ASSERT ((test), "NULL testcase launcher for test '%s' found in testsuite '%s'", groupID_.c_str(),testID.c_str());
@@ -146,10 +160,14 @@ namespace test {
   
   namespace { // internal helper for launching with error logging
     
-    void
+    int
     invokeTestCase (Test& theTest, Arg cmdline)
     {
-      try { theTest.run (cmdline); }
+      try 
+        {
+          theTest.run (cmdline);
+          return Suite::TEST_OK;
+        }
       catch (lumiera::Error& failure)
         {
           lumiera_err errorID = lumiera_error(); // reset error flag
@@ -157,6 +175,7 @@ namespace test {
           cerr << "***            : " << failure.what() << endl;
           ERROR (test,     "Error state %s", errorID);
           WARN  (progress, "Caught exception %s", failure.what());
+          return Suite::EXCEPTION_THROWN;
     }   }
   }
   
@@ -188,7 +207,7 @@ namespace test {
             Launcher* test = (*tests)[testID];
             cmdline.erase (cmdline.begin());
             VALID (test,testID);
-            invokeTestCase (*(*test)(), cmdline);  // TODO confusing statement, improve definition of test collection datatype Ticket #289
+            exitCode_ |= invokeTestCase (*(*test)(), cmdline);  // TODO confusing statement, improve definition of test collection datatype Ticket #289
             return;
       }   }
     
@@ -199,7 +218,7 @@ namespace test {
         std::cout << "\n  ----------"<< i->first<< "----------\n";
         Launcher* test = (i->second);
         VALID (test, i->first);
-        invokeTestCase (*(*test)(), cmdline); // actually no cmdline arguments
+        exitCode_ |= invokeTestCase (*(*test)(), cmdline); // actually no cmdline arguments
       }
   }
   

src/lib/test/suite.hpp

@@ -66,14 +66,18 @@ namespace test {
   class Suite
     {
       string groupID_;
+      int exitCode_;
       
     public:
       Suite (string groupID);
       void run (Arg cmdline);
       void describe ();
+      int getExitCode ()  const;
       static void enrol (Launcher *test, string testID, string groups);
       
       static const string ALLGROUP;
+      static const int TEST_OK;
+      static const int EXCEPTION_THROWN;
     };
   
   

tests/lib/helloworldtest.cpp

@@ -21,48 +21,48 @@
 * *****************************************************/
 
 
-#include <iostream>
 #include "lib/test/run.hpp"
-
 #include "lib/util.hpp"
-using util::isnil;
 
 #include <boost/lexical_cast.hpp>
+#include <iostream>
+
 using boost::lexical_cast;
+using util::isnil;
 
 
-namespace lumiera
-  {
-  namespace test
+namespace lumiera{
+namespace test   {
+  
+  /******************************************
+   * Hellooooooo the world is a test
+   * @test demo of using the test framework
+   */
+  class HelloWorld_test : public Test
     {
-
-    /******************************************
-     * Helooooooo to the world of TDD
-     * @test demo of using the test framework
-     */
-    class HelloWorld_test : public Test
+      virtual void run(Arg arg) 
       {
-        virtual void run(Arg arg) 
-        {
-          int num= isnil(arg)?  1 : lexical_cast<int> (arg[1]);
-          
-          for ( ; 0 < num-- ; )
-            greeting();
-        } 
+        int num= isnil(arg)?  1 : lexical_cast<int> (arg[1]);
         
-        void greeting() 
-        { 
-          std::cout << "This is how the world ends...\n"; 
-        }
-      };
-    
+        for ( ; 0 < num-- ; )
+          greeting();
+      } 
       
-
-    /** Register this test class to be invoked in some test groups (suites) */
-    Launch<HelloWorld_test> run_HelloWorld_test("HelloWorld_test","unit common");
-    
-    // NOTE: you may use the Macro "LAUNCHER" in run.hpp to simplify this Registration
-    
-  } // namespace test
-
-} // namespace lumiera
+      void greeting() 
+      { 
+        std::cout << "This is how the world ends...\n";
+      }
+    };
+  
+  
+  
+  
+  
+  /** Register this test class to be invoked in some test groups (suites) */
+  Launch<HelloWorld_test> run_HelloWorld_test("HelloWorld_test","unit common");
+  
+  // NOTE: you may use the Macro "LAUNCHER" in run.hpp to simplify this Registration
+  
+  
+  
+}} // namespace lumiera::test

tests/lib/mainsuite.cpp

@@ -32,7 +32,10 @@ using lumiera::ON_GLOBAL_SHUTDOWN;
 
 /** run all tests or any single test specified in the first
  *  command line argument.
- *  Note: to ease debugging, we don't catch any exceptions. 
+ *  @return exit code signalling if any exception was caught
+ *          at the level of individual test cases
+ *  @throws any further exceptions raised outside of test cases
+ *  @throws any library and external exceptions (not lumiera::Exception)  
  */
 int main (int argc, const char* argv[])
 {
@@ -47,5 +50,5 @@ int main (int argc, const char* argv[])
     suite.run (args);
 
   LifecycleHook::trigger (ON_GLOBAL_SHUTDOWN);
-  return 0;
+  return suite.getExitCode();
 }