Library: integrate into the Lumiera code base

- reformat in Lumieara-GNU style
- use the Lumiera exceptions
- use Lumiera format-string frontend
- use lib/util

NOTE: I am the original author of the code introduced here,
and thus I can re-license it under GPL 2+

admin/scons/Platform.py

@@ -79,6 +79,9 @@ def configure(env):
     if not conf.CheckCXXHeader('functional'):
         problems.append('We rely on the C++11 functor objects.')
     
+    if not conf.CheckLibWithHeader('stdc++fs', 'filesystem', 'C++'):
+        problems.append('We need the C++17 filesystem support.')
+    
     if not conf.CheckCXXHeader('boost/config.hpp'):
         problems.append('We need the C++ boost-libraries.')
     else:

src/lib/stat/csv.hpp

@@ -1,21 +1,24 @@
 /*
- *  csv - parser and encoder
+  CSV.hpp  -  Parser and Encoder for CSV data
- *
+
- *  Copyright 2021, Hermann Vosseler <Ichthyostega@web.de>
+  Copyright (C)         Lumiera.org
- *
+    2022,               Hermann Vosseler <Ichthyostega@web.de>
- *  This file is part of the Yoshimi-Testsuite, which is free software:
+
- *  you can redistribute and/or modify it under the terms of the GNU
+  This program is free software; you can redistribute it and/or
- *  General Public License as published by the Free Software Foundation,
+  modify it under the terms of the GNU General Public License as
- *  either version 3 of the License, or (at your option) any later version.
+  published by the Free Software Foundation; either version 2 of
- *
+  the License, or (at your option) any later version.
- *  Yoshimi-Testsuite is distributed in the hope that it will be useful,
+
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  This program is distributed in the hope that it will be useful,
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  GNU General Public License for more details.
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *
+  GNU General Public License for more details.
- *  You should have received a copy of the GNU General Public License
+
- *  along with yoshimi.  If not, see <http://www.gnu.org/licenses/>.
+  You should have received a copy of the GNU General Public License
- ***************************************************************/
+  along with this program; if not, write to the Free Software
+  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+*/
 
 
 /** @file csv.hpp
@@ -36,159 +39,171 @@
  ** - only quoted fields may contain whitespace or comma
  ** - no escaping of quotes, i.e. no quotes within quotes
  ** [RFC 4180]: https://datatracker.ietf.org/doc/html/rfc4180
- ** 
+ ** @see lib::stat::DataFile
- ** @todo WIP as of 9/21
+ **
- ** @see util::DataFile used for [Timing statistics](\ref TimingObservation.hpp)
- ** 
  */
 
 
+#ifndef LIB_STAT_CSV_H
+#define LIB_STAT_CSV_H
 
-#ifndef TESTRUNNER_UTIL_CSV_HPP_
+#include "lib/error.hpp"
-#define TESTRUNNER_UTIL_CSV_HPP_
+#include "lib/nocopy.hpp"
-
+#include "lib/format-string.hpp"
-
+#include "lib/format-obj.hpp"
-#include "util/error.hpp"
+#include "lib/stat/regex.hpp"
-#include "util/format.hpp"
-#include "util/regex.hpp"
 
 #include <limits>
 #include <string>
 
+namespace lib {
+namespace stat {
+  
+  namespace error = lumiera::error;
 
-namespace util {
+  using util::_Fmt;
-
+  using util::toString;
-using std::regex;
+  using std::string;
-using std::string;
+  using std::regex;
-
+  
-namespace { // Implementation details...
+  
-
+  namespace { // Implementation details...
-    const string MATCH_SINGLE_TOKEN {R"~(([^,;"\s]*)\s*)~"};
+  
-    const string MATCH_QUOTED_TOKEN {R"~("([^"]*)"\s*)~"};
+    const string MATCH_SINGLE_TOKEN { R"~(([^,;"\s]*)\s*)~"};
-    const string MATCH_DELIMITER    {R"~((?:^|,|;)\s*)~"};
+    const string MATCH_QUOTED_TOKEN { R"~("([^"]*)"\s*)~" };
-
+    const string MATCH_DELIMITER    { R"~((?:^|,|;)\s*)~" };
+    
     const regex ACCEPT_FIELD{ MATCH_DELIMITER + "(?:"+ MATCH_QUOTED_TOKEN +"|"+ MATCH_SINGLE_TOKEN +")"
                             , regex::optimize};
-
+    
-
     template<typename VAL>
-    inline string format4Csv(VAL const& val)
+    inline string
+    format4Csv (VAL const& val)
     {
-        std::ostringstream oss;
+      std::ostringstream oss;
-        oss.precision(std::numeric_limits<VAL>::digits10);
+      oss.precision (std::numeric_limits<VAL>::digits10);  /////////////////////////////OOO herausfinden ob hier lexical_cast genügt ==> dann toString()
-        oss << val;
+      oss << val;
-        return oss.str();
+      return oss.str();
     }
-    inline string format4Csv(string const& val)
+    
+    inline string
+    format4Csv (string const& val)
     {
-        return '"'+val+'"';
+      return '"'+val+'"';
     }
-    inline string format4Csv(bool boo)
+    
+    inline string
+    format4Csv (bool boo)
     {
-        return formatVal(boo);
+      return util::showBool(boo);  ///////////////////////OOO würde toSting() das korrekt hinbekommen
     }
-
+  }//(End)Implementation
-}//(End)Implementation
+  
-
+  
-
+  /**
-/**
+   * Format and append a data value to a CSV string representation
- * Parser to split one line of CSV data into fields.
+   */
- * @remarks iterator-like throw-away object
+  template<typename VAL>
- *  - the `bool` evaluation indicates more fields to extract
+  inline void
- *  - dereference to get the field as string
+  appendCsvField (string& csv, VAL const& val)
- *  - increment to move to the next field
+  {
- * @throws error::Invalid on CSV format violation
+    csv += (0 == csv.length()? "":",")
- */
+         + format4Csv(val);
-class CsvLine
+  }
+  
+  
+  /**
+   * Parser to split one line of CSV data into fields.
+   * @remarks iterator-like throw-away object
+   *  - the `bool` evaluation indicates more fields to extract
+   *  - dereference to get the field as string
+   *  - increment to move to the next field
+   * @throws error::Invalid on CSV format violation
+   * @todo 3/24 should be rewritten as Lumiera Forward Iterator
+   */
+  class CsvLine
     : util::NonCopyable
-    , MatchSeq
+    , util::MatchSeq
-{
+    {
-    string const& line_;
+      string const& line_;
-    size_t   field_;
+      size_t   field_;
-    iterator curr_;
+      iterator curr_;
-    size_t   pos_;
+      size_t   pos_;
 
-public:
+    public:
-    CsvLine(string const& line)
+      CsvLine (string const& line)
         : MatchSeq(line, ACCEPT_FIELD)
         , line_{line}
         , field_{0}
         , curr_{MatchSeq::begin()}
         , pos_{0}
-    { }
+        { }
-
+      
-    explicit operator bool()
+      explicit operator bool()  const
-    {
+        {
-        return isValid();
+          return isValid ();
-    }
+        }
-
+      
-    string operator*()
+      string operator*()  const
-    {
+        {
-        if (not isValid()) fail();
+          if (not isValid ()) fail();
-        auto& mat = *curr_;
+          auto& mat = *curr_;
-        return mat[2].matched? mat[2]
+          return mat[2].matched? mat[2]
-                             : mat[1];
+                               : mat[1];
-    }
+        }
-
+      
-    void operator++()
+      void
-    {
+      operator++()
-        if (not isValid())
+        {
+          if (not isValid())
             fail();
-        pos_ = curr_->position() + curr_->length();
+          pos_ = curr_->position() + curr_->length();
-        ++curr_;
+          ++curr_;
-        if (pos_ < line_.length() and not isValid())
+          if (pos_ < line_.length() and not isValid())
-            fail();
+            fail ();
-        ++field_;
+          ++field_;
-    }
+        }
-
+      
-    size_t getParsedFieldCnt()
+      size_t
-    {
+      getParsedFieldCnt()
-        return field_;
+        {
-    }
+          return field_;
-
+        }
-    bool isValid()
+      
-    {
+      bool
-        return curr_ != end()
+      isValid()  const
-           and curr_->position() == pos_
+        {
-           and not curr_->empty();
+          return curr_ != end()
-    }
+             and pos_ == size_t(curr_->position())
-
+             and not curr_->empty();
-    bool isParseFail()
+        }
-    {
+      
-        return curr_ != end()
+      bool
-           and not isValid();
+      isParseFail()  const
-    }
+        {
-
+          return curr_ != end()
-    void fail()
+             and not isValid();
-    {
+        }
-        if (curr_ == end())
+      
-            if (pos_ >= line_.length())
+      void
-                throw error::Invalid("Only "+formatVal(field_)+" data fields. Line:"+line_);
+      fail()  const
-            else
+        {
-                throw error::Invalid("Garbage after last field. Line:"
+          if (curr_ == end())
-                                    +line_.substr(0,pos_)+"|↯|"+line_.substr(pos_));
+              if (pos_ >= line_.length())
-        else
+                  throw error::Invalid{_Fmt{"Only %d data fields. Line:%s"}
-            if (pos_ != curr_->position())
+                                           % field_ % line_};
-                throw error::Invalid("Garbage before field("+formatVal(field_+1)+"):"
+              else
-                                    +line_.substr(0,pos_)+"|↯|"+line_.substr(pos_));
+                  throw error::Invalid{_Fmt{"Garbage after last field. Line:%s|↯|%s"}
-            else
+                                           % line_.substr(0,pos_) % line_.substr(pos_)};
-                throw error::Invalid("CSV parse floundered. Line:"+line_);
+          else
-    }
+              if (pos_ != curr_->position())
-};
+                  throw error::Invalid{_Fmt{"Garbage before field(%d):%s|↯|%s"}
-
+                                           % (field_+1)
-
+                                           % line_.substr(0,pos_) % line_.substr(pos_)};
-
+              else
-/**
+                  throw error::Invalid{"CSV parse floundered. Line:"+toString(line_)};
- * Format and append a data value to a CSV string representation
+        }
- */
+    };
-template<typename VAL>
+  
-inline void appendCsvField(string& csv, VAL const& val)
+}} // namespace lib::stat
-{
+#endif /*LIB_STAT_CSV_H*/
-    csv += (0 == csv.length()? "":",")
-         + format4Csv(val);
-}
-
-
-
-} // namespace util
-#endif /*TESTRUNNER_UTIL_CSV_HPP_*/

src/lib/stat/data.hpp

@@ -1,41 +1,51 @@
 /*
- *  data - read and write a table with CSV data
+  DATA.hpp  -  read and write a table with CSV data
- *
+
- *  Copyright 2021, Hermann Vosseler <Ichthyostega@web.de>
+  Copyright (C)         Lumiera.org
- *
+    2022,               Hermann Vosseler <Ichthyostega@web.de>
- *  This file is part of the Yoshimi-Testsuite, which is free software:
+
- *  you can redistribute and/or modify it under the terms of the GNU
+  This program is free software; you can redistribute it and/or
- *  General Public License as published by the Free Software Foundation,
+  modify it under the terms of the GNU General Public License as
- *  either version 3 of the License, or (at your option) any later version.
+  published by the Free Software Foundation; either version 2 of
- *
+  the License, or (at your option) any later version.
- *  Yoshimi-Testsuite is distributed in the hope that it will be useful,
+
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  This program is distributed in the hope that it will be useful,
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  GNU General Public License for more details.
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *
+  GNU General Public License for more details.
- *  You should have received a copy of the GNU General Public License
+
- *  along with yoshimi.  If not, see <http://www.gnu.org/licenses/>.
+  You should have received a copy of the GNU General Public License
- ***************************************************************/
+  along with this program; if not, write to the Free Software
+  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+*/
 
 
 /** @file data.hpp
  ** Manage a table with time series data, stored persistently as CSV.
- ** The Yoshimi Testsuite captures timing data, to detect the possible performance
+ ** In the context of observations, configuration, calibration and QA, a series
- ** impact of code reworking. Due to the statistical nature of timing measurements
+ ** of measurement data taken over time is often evaluated statistically, to distill
- ** and the dependency on the run environment, it is not sufficient just to rely on
+ ** typical averages, variances and trends. Short of using a database, a modest
- ** a single measurement to establish the runtime characteristics of a given test;
- ** rather, the statistical trend of the timings observed over several consecutive
- ** runs of the Testsuite must be established. Short of using a database, a modest
  ** amount of numeric data can be maintained in CSV files, which also allows for
  ** further manual evaluation within a spreadsheet or statistics application.
- **
+ ** The CSV format as such can be quite elaborate, yet for the purpose of
+ ** saving and later reading back some values generated by the application
+ ** itself, supporting a limited format flavour is sufficient:
+ ** - first line is a header line and used to verify the storage format
+ ** - one record per line, embedded line breaks prohibited
+ ** - fields separated by comma, semicolon tolerated
+ ** - fields are trimmed and may be empty
+ ** - a field may be double quoted
+ ** - only quoted fields may contain whitespace or comma
+ ** - no escaping of quotes, i.e. no quotes within quotes
+ ** 
  ** As a fundamental building block, this header provides a data table template
  ** with flexible column configuration to hold arbitrary, explicitly typed values.
  ** This solution is statically typed and does not carry any runtime type information;
  ** the actual data table object is then defined and accessed by means of _accessor_
  ** components for each column of data. A tuple of _current values_ corresponding to
  ** the most recent row of data can be accessed directly through these sub-components.
- **
+ ** 
  ** # Usage
  ** Create an actual instantiation of the DataFile template, passing a structure
  ** with util::Column descriptors. You may then directly access the values of the
@@ -43,40 +53,39 @@
  ** @note mandatory to define a method `allColumns()`
  ** \code
  ** struct Storage
- ** {
+ **   {
  **     Column<string> name{"theName"};
  **     Column<int>    n{"counter"};
  **     Column<double> x{"X value"};
  **     Column<double> y{"Y value"};
  **
  **     auto allColumns(){ return std::tie(name,n,x,y); }
- ** };
+ **   };
- **
+ ** 
- ** using Dataz = util::DataFile<Storage>;
+ ** using Dataz = lib::stat::DataFile<Storage>;
- **
+ ** 
  ** Dataz daz("filename.csv");
- **
+ ** 
  ** daz.x = 123e-4;
  ** daz.y = -12345e-6;
- **
+ ** 
  ** std::vector<int>& counters = daz.n.data;
  ** \endcode
  **
- ** @see TimingObservation.hpp usage
- **
  */
 
 
 
-#ifndef TESTRUNNER_UTIL_DATA_HPP_
+#ifndef LIB_STAT_DATA_H
-#define TESTRUNNER_UTIL_DATA_HPP_
+#define LIB_STAT_DATA_H
 
 
-#include "util/nocopy.hpp"
+#include "lib/error.hpp"
-#include "util/error.hpp"
+#include "lib/nocopy.hpp"
-#include "util/utils.hpp"
+#include "lib/stat/file.hpp"
-#include "util/file.hpp"
+#include "lib/stat/csv.hpp"
-#include "util/csv.hpp"
+#include "lib/format-string.hpp"
+#include "lib/util.hpp"
 
 #include <type_traits>
 #include <utility>
@@ -88,331 +97,382 @@
 #include <tuple>
 
 
-namespace util {
+namespace lib {
-
+namespace stat{
-using std::tuple;
+  
-using std::vector;
+  namespace error = lumiera::error;
-using std::string;
+  
-
+  using std::move;
-
+  using std::tuple;
-
+  using std::vector;
-/**
+  using std::string;
- * Helper: perform some arbitrary operation on each element of a tuple.
+  using util::isnil;
- * @note the given functor must be generic, since each position of the tuple
+  using util::unConst;
- *       may hold a data element of different type.
+  using util::_Fmt;
- * @remark credits to David Vandevoorde (member of C++ committee) for using
+  using util::min;
- *       std::apply to unpack the tuple's contents into an argument pack and
+  
- *       then using a fold expression with the comma operator.
+  
- */
+  
-template<class FUN, typename...ELMS>
+  /**
-void forEach(tuple<ELMS...>&& tuple, FUN fun)
+   * Helper: perform some arbitrary operation on each element of a tuple.
-{
+   * @note the given functor must be generic, since each position of the tuple
-    std::apply([&fun](auto&... elms)
+   *       may hold a data element of different type.
-                    {
+   * @remark credits to David Vandevoorde (member of C++ committee) for using
+   *       std::apply to unpack the tuple's contents into an argument pack and
+   *       then using a fold expression with the comma operator.
+   */
+  template<class FUN, typename...ELMS>
+  void forEach(tuple<ELMS...>&& tuple, FUN fun)
+  {
+      std::apply([&fun](auto&... elms)
+                      {
                         (fun(elms), ...);
-                    }
+                      }
-              ,tuple);
+                ,tuple);
-}
+  }
-
+  
-
+  
-
+  
-/**
+  /** parse string representation into typed value */
- * Descriptor and Accessor for a data column within a DataFile table.
+  template<typename TAR>
- * @tparam VAL type of values contained within this column;
+  inline TAR
- *             this type must be _default constructible_ and _copyable._
+  parseAs (string const& encodedVal)
- */
+  {
-template<typename VAL>
+      std::istringstream converter{encodedVal};
-struct Column : util::NonCopyable
+      TAR value;
-{
+      converter >> value;
-    string header;
+      if (converter.fail())
-    vector<VAL> data;
+          throw error::Invalid{_Fmt{"unable to parse \"%s\""} % encodedVal};
-
+      return value;
-    using ValueType = VAL;
+  }
-
+  
-
+  template<>
-    Column(string headerID)
+  inline bool
+  parseAs(string const& encodedBool)
+  {
+      return util::boolVal(encodedBool);
+  }
+  template<>
+  inline string
+  parseAs(string const& string)
+  {
+      return string; // pass-through (even if empty)
+  }
+  
+  
+  
+  
+  
+  /**
+   * Descriptor and Accessor for a data column within a DataFile table.
+   * @tparam VAL type of values contained within this column;
+   *             this type must be _default constructible_ and _copyable._
+   */
+  template<typename VAL>
+  struct Column
+    : util::NonCopyable
+    {
+      string header;
+      vector<VAL> data;
+      
+      using ValueType = VAL;
+      
+      
+      Column (string headerID)
         : header{headerID}
         , data{}
-    { }
+        { }
-
+      
-
+      
-    VAL& get()
+      VAL&
-    {
+      get()
-        if (isnil(data))
-            throw error::State("No rows in DataTable yet");
-        return data.back();
-    }
-
-    operator VAL&()
-    {
-        return get();
-    }
-
-    operator VAL const&()  const
-    {
-        return unConst(this)->get();
-    }
-
-    template<typename X>
-    VAL& operator=(X&& newVal)
-    {
-        return get() = std::forward<X>(newVal);
-    }
-};
-
-
-
-
-
-/**
- * Table with data values, stored persistently as CSV file.
- * Each row within the table represents a data record, holding a sequence
- * of values. Values are statically typed per column, i.e. one column may hold
- * strings, while the next column holds doubles. For actual usage it is thus necessary
- * to define the column layout, through a sequence of [column Descriptors](\ref util::Column).
- *
- * # Usage
- * Actually those Column objects serve as descriptors, but also as accessors -- and they hold
- * the actual data storage for each column, which is a `std::vector<VAL>` of value type `VAL`.
- * There is always a _current record_ -- corresponding to the actual data value and the newest
- * data row. For persistent storage, the sequence of rows is _reversed,_ so the newest data
- * appears at the top of the CSV file.
- * @tparam TAB a struct comprised of several Column objects, which hold the data and
- *         provide access to values of this specific column. Moreover, this type _must define_
- *         a function `allColumns()` to return a tuple with references to these column fields;
- *         the order of fields within this tuple also defines the order of columns
- *         within the table and persistent CSV storage.
- * @see suite::step::TimingObservation (relevant usage example)
- */
-template<class TAB>
-class DataFile
-    : public TAB
-    , util::NonCopyable
-{
-    fs::path filename_;
-
-public:
-    DataFile(fs::path csvFile)
-        : filename_{consolidated(csvFile)}
-    {
-        loadData();
-    }
-
-
-    /* === Data Access === */
-
-    static constexpr size_t columnCnt = std::tuple_size_v<decltype(std::declval<TAB>().allColumns())>;
-
-    bool empty()  const
-    {
-        return 0 == this->size();
-    }
-
-    size_t size()  const
-    {
-        if (0 == columnCnt) return 0;
-        size_t rowCnt = std::numeric_limits<size_t>::max();
-        forEach(unConst(this)->allColumns(),
-                [&](auto& col)
-                {
-                    rowCnt = std::min(rowCnt, col.data.size());
-                }); // the smallest number of data points found in any column
-        return rowCnt;
-    }
-
-    string dumpCSV()  const
-    {
-        string csv;
-        for (uint i=0; i < size(); ++i)
-            csv += formatCSVRow(i) + '\n';
-        return csv;
-    }
-
-
-
-    /* === Manipulation === */
-
-    void newRow()
-    {
-        forEach(TAB::allColumns(),
-                [](auto& col)
-                {
-                    col.data.resize(col.data.size()+1);
-                });
-    }
-
-    void dupRow()
-    {
-        if (empty())
-            newRow();
-        else
-            forEach(TAB::allColumns(),
-                [](auto& col)
-                {
-                    col.data.emplace_back(col.data.back());
-                });
-    }
-
-    void dropLastRow()
-    {
-        if (not empty())
-            forEach(TAB::allColumns(),
-                [](auto& col)
-                {
-                    size_t siz = col.data.size();
-                    col.data.resize(siz>0? siz-1 : 0);
-                });
-    }
-
-    void reserve(size_t expectedCapacity)
-    {
-        forEach(TAB::allColumns(),
-                [=](auto& col)
-                {
-                    col.data.reserve(expectedCapacity);
-                });
-    }
-
-
-    /** @param lineLimit number of rows to retain, back from the newest */
-    void save(size_t lineLimit =std::numeric_limits<size_t>::max(), bool backupOld =false)
-    {
-        fs::path newFilename{filename_};
-        newFilename += ".tmp";
-
-        std::ofstream csvFile{newFilename, std::ios_base::out | std::ios_base::trunc};
-        if (not csvFile.good())
-            throw error::State("Unable to create CSV output file "+formatVal(newFilename));
-        saveData(csvFile, lineLimit);
-
-        if (backupOld)
         {
-            fs::path oldFile{filename_};
+          if (isnil (data))
-            oldFile += ".bak";
+              throw error::State{"No rows in DataTable yet"};
-            if (fs::exists(filename_))
+          return data.back();
-                fs::rename(filename_, oldFile);
         }
-        fs::rename(newFilename, filename_);
+      
-        filename_ = consolidated(filename_); // lock onto absolute path
+      operator VAL&()
-    }
+        {
-
+          return get();
-
+        }
-
+      
-private: /* === Implementation === */
+      operator VAL const&()  const
-
+        {
-    void loadData()
+          return unConst(this)->get();
+        }
+      
+      template<typename X>
+      VAL& operator= (X&& newVal)
+      {
+          return get() = std::forward<X> (newVal);
+      }
+    };
+  
+  
+  
+  
+  
+  /******************************************************************************************//**
+   * Table with data values, stored persistently as CSV file.
+   * Each row within the table represents a data record, holding a sequence
+   * of values. Values are statically typed per column, i.e. one column may hold
+   * strings, while the next column holds doubles. For actual usage it is thus necessary
+   * to define the column layout, through a sequence of [column Descriptors](\ref util::Column).
+   * 
+   * # Usage
+   * Actually those Column objects serve as descriptors, but also as accessors — and they hold
+   * the actual data storage for each column, which is a `std::vector<VAL>` of value type `VAL`.
+   * There is always a _current record_ — corresponding to the actual data value and the newest
+   * data row. For persistent storage, the sequence of rows is _reversed,_ so the newest data
+   * appears at the top of the CSV file.
+   * @tparam TAB a struct comprised of several Column objects, which hold the data and
+   *         provide access to values of this specific column. Moreover, this type _must define_
+   *         a function `allColumns()` to return a tuple with references to these column fields;
+   *         the order of fields within this tuple also defines the order of columns
+   *         within the table and persistent CSV storage.
+   */
+  template<class TAB>
+  class DataFile
+      : public TAB
+      , util::NonCopyable
     {
-        if (not (filename_.parent_path().empty()
+      fs::path filename_;
-                 or fs::exists(filename_.parent_path())))
+      
-            throw error::Invalid("DataFile("+formatVal(filename_.filename())
+    public:
-                                +") shall be placed into nonexistent directory "
+      DataFile(fs::path csvFile)
-                                +formatVal(filename_.parent_path()));
+        : filename_{fs::consolidated (csvFile)}
-        if (not fs::exists(filename_))
+        {
+          loadData();
+        }
+      
+      
+      /* === Data Access === */
+      
+      static constexpr size_t columnCnt = std::tuple_size_v<decltype(std::declval<TAB>().allColumns())>;
+      
+      bool
+      empty()  const
+        {
+          return 0 == this->size();
+        }
+      
+      size_t
+      size()  const
+        {
+          if (0 == columnCnt) return 0;
+          size_t rowCnt = std::numeric_limits<size_t>::max();
+          forEach (unConst(this)->allColumns()
+                  ,[&](auto& col)
+                    {
+                      rowCnt = min (rowCnt, col.data.size());
+                    }); // the smallest number of data points found in any column
+          return rowCnt;
+        }
+      
+      string
+      dumpCSV()  const
+        {
+          string csv;
+          for (uint i=0; i < size(); ++i)
+            csv += formatCSVRow(i) + '\n';
+          return csv;
+        }
+      
+      
+      
+      /* === Manipulation === */
+      
+      void
+      newRow()
+        {
+            forEach (TAB::allColumns()
+                    ,[](auto& col)
+                      {
+                        col.data.resize (col.data.size()+1);
+                      });
+        }
+      
+      void
+      dupRow()
+        {
+          if (empty())
+            newRow();
+          else
+            forEach (TAB::allColumns()
+                    ,[](auto& col)
+                      {
+                        col.data.emplace_back (col.data.back());
+                      });
+        }
+      
+      void
+      dropLastRow()
+        {
+          if (not empty())
+            forEach (TAB::allColumns()
+                    ,[](auto& col)
+                      {
+                        size_t siz = col.data.size();
+                        col.data.resize (siz>0? siz-1 : 0);
+                      });
+        }
+      
+      void
+      reserve (size_t expectedCapacity)
+        {
+            forEach (TAB::allColumns()
+                    ,[=](auto& col)
+                      {
+                        col.data.reserve(expectedCapacity);
+                      });
+        }
+      
+      
+      /** @param lineLimit number of rows to retain, back from the newest */
+      void
+      save (size_t lineLimit =std::numeric_limits<size_t>::max()
+           ,bool backupOld =false)
+        {
+          fs::path newFilename{filename_};
+          newFilename += ".tmp";
+          
+          std::ofstream csvFile{newFilename, std::ios_base::out | std::ios_base::trunc};
+          if (not csvFile.good())
+            throw error::State{_Fmt{"Unable to create CSV output file %s"}
+                                   % newFilename};
+          saveData (csvFile, lineLimit);
+          
+          if (backupOld)
+            {
+              fs::path oldFile{filename_};
+              oldFile += ".bak";
+              if (fs::exists (filename_))
+                  fs::rename (filename_, oldFile);
+            }
+          fs::rename (newFilename, filename_);
+          filename_ = fs::consolidated(filename_);
+        }                // lock onto absolute path
+      
+      
+      
+    private: /* === Implementation === */
+      
+      void
+      loadData()
+        {
+          if (not (filename_.parent_path().empty()
+                   or fs::exists(filename_.parent_path())))
+            throw error::Invalid{_Fmt{"DataFile(%s) placed into nonexistent directory %s"}
+                                     % filename_.filename() % filename_.parent_path()};
+          if (not fs::exists(filename_))
             return; // leave the table empty
-
+          
-        std::ifstream csvFile(filename_);
+          std::ifstream csvFile{filename_};
-        if (not csvFile.good())
+          if (not csvFile.good())
-            throw error::Misconfig{"unable to read CSV data file "+formatVal(filename_)};
+            throw error::Config{_Fmt{"unable to read CSV data file %s"} % filename_};
-
+          
-        std::deque<string> rawLines;
+          std::deque<string> rawLines;
-        for (string line; std::getline(csvFile, line); )
+          for (string line; std::getline(csvFile, line); )
-            rawLines.emplace_back(move(line));
+            rawLines.emplace_back (move(line));
-
+          
-        if (rawLines.size() < 1) return;
+          if (rawLines.size() < 1) return;
-        verifyHeaderSpec(rawLines[0]);
+          verifyHeaderSpec (rawLines[0]);
-
+          
-        // we know the number of rows now...
+          // we know the number of rows now...
-        reserve(rawLines.size() - 1);
+          reserve (rawLines.size() - 1);
-
+          
-        // storage in file is backwards, with newest data on top
+          // storage in file is backwards, with newest data on top
-        for (size_t row = rawLines.size()-1; 0<row; --row)
+          for (size_t row = rawLines.size()-1; 0<row; --row)
             if (not isnil(rawLines[row]))
-                appendRowFromCSV(rawLines[row]);
+              appendRowFromCSV (rawLines[row]);
-    }
+        }
-
+      
-
+      
-    void saveData(std::ofstream& csvFile, size_t lineLimit)
+      void
-    {
+      saveData (std::ofstream& csvFile, size_t lineLimit)
-        csvFile << generateHeaderSpec() << "\n";
+        {
-        if (empty())
+          csvFile << generateHeaderSpec() << "\n";
+          if (empty())
             return;
-        lineLimit = size() > lineLimit? size()-lineLimit : 0;
+          lineLimit = size() > lineLimit? size()-lineLimit : 0;
-        // store newest data first, possibly discard old data
+          // store newest data first, possibly discard old data
-        for (size_t row = size(); lineLimit < row; --row)
+          for (size_t row = size(); lineLimit < row; --row)
             csvFile << formatCSVRow(row-1) << "\n";
-    }
+        }
-
+      
-
+      
-    void verifyHeaderSpec(string headerLine)
+      void
-    {
+      verifyHeaderSpec (string headerLine)
-        CsvLine header(headerLine);
+        {
-        forEach(TAB::allColumns(),
+          CsvLine header{headerLine};
-                [&](auto& col)
+          forEach (TAB::allColumns()
-                {
+                  ,[&](auto& col)
-                    if (*header != col.header)
+                    {
-                        throw error::Invalid("Header mismatch in CSV file "+formatVal(filename_)
+                      if (*header != col.header)
-                                            +". Expecting column("+formatVal(col.header)
+                        throw error::Invalid{_Fmt{"Header mismatch in CSV file %s. "
-                                            +") but found "+formatVal(*header));
+                                                  "Expecting column(%s) but found \"%s\""}
-                    ++header;
+                                                 % filename_ % col.header % *header};
-                });
+                      ++header;
-    }
+                    });
-
+        }
-    string generateHeaderSpec()
+      
-    {
+      string
-        string csv;
+      generateHeaderSpec()
-        forEach(TAB::allColumns(),
+        {
-                [&](auto& col)
+          string csv;
-                {
+          forEach (TAB::allColumns()
-                    appendCsvField(csv, col.header);
+                  ,[&](auto& col)
-                });
+                    {
-        return csv;
+                      appendCsvField (csv, col.header);
-    }
+                    });
-
+          return csv;
-
+        }
-    void appendRowFromCSV(string line)
+      
-    {
+      
-        newRow();
+      void
-        CsvLine csv(line);
+      appendRowFromCSV (string line)
-        forEach(TAB::allColumns(),
+        {
-                [&](auto& col)
+          newRow();
-                {
+          CsvLine csv(line);
-                    if (!csv)
+          forEach (TAB::allColumns()
+                  ,[&](auto& col)
+                    {
+                      if (not csv)
                         if (csv.isParseFail())
-                            csv.fail();
+                          csv.fail();
                         else
-                            throw error::Invalid("Insufficient data; only "
+                          throw error::Invalid{_Fmt{"Insufficient data; only %d fields, %d expected. Line:%s"}
-                                                +str(csv.getParsedFieldCnt())
+                                                   % csv.getParsedFieldCnt() % columnCnt % line};
-                                                +" fields, "+str(columnCnt)
+                      
-                                                +" expected. Line="+line);
+                      using Value = typename std::remove_reference<decltype(col)>::type::ValueType;
-
+                      col.get() = parseAs<Value>(*csv);
-                    using Value = typename std::remove_reference<decltype(col)>::type::ValueType;
+                      ++csv;
-                    col.get() = parseAs<Value>(*csv);
+                    });
-                    ++csv;
+          if (csv)
-                });
+            throw error::Invalid{_Fmt{"Excess data fields in CSV. Expect %d fields. Line:%s"}
-        if (csv)
+                                     % columnCnt % line};
-            throw error::Invalid("Excess data fields in CSV. Expect "+str(columnCnt)+" fields. Line="+line);
+        }
-    }
+      
-
+      
-
+      string
-    string formatCSVRow(size_t rownum)  const
+      formatCSVRow (size_t rownum)  const
-    {
+        {
-        if (this->empty())
+          if (this->empty())
-            throw error::LogicBroken("Attempt to access data from empty DataTable.");
+            throw error::Logic{"Attempt to access data from empty DataTable."};
-        if (rownum >= this->size())
+          if (rownum >= this->size())
-            throw error::LogicBroken("Attempt to access row #"+str(rownum)
+            throw error::Logic{_Fmt{"Attempt to access row #%d beyond range [0..%d]."}
-                                    +" beyond range [0.."+str(size()-1)+"].");
+                                   % rownum % (size()-1)};
-
+          
-        string csvLine;
+          string csvLine;
-        forEach(unConst(this)->allColumns(),
+          forEach (unConst(this)->allColumns()
-                [&](auto& col)
+                  ,[&](auto& col)
-                {
+                    {
-                    appendCsvField(csvLine, col.data.at(rownum));
+                      appendCsvField (csvLine, col.data.at(rownum));
-                });
+                    });
-        return csvLine;
+          return csvLine;
-    }
+        }
-};
+    };
-
+  
-
+}} // namespace lib::stat
-
+#endif /*LIB_STAT_DATA_H*/
-} // namespace util
-#endif /*TESTRUNNER_UTIL_DATA_HPP_*/

src/lib/stat/error.hpp

@@ -1,114 +0,0 @@
-/*
- *  error - exceptions and error handling helpers
- *
- *  Copyright 2021, Hermann Vosseler <Ichthyostega@web.de>
- *
- *  This file is part of the Yoshimi-Testsuite, which is free software:
- *  you can redistribute and/or modify it under the terms of the GNU
- *  General Public License as published by the Free Software Foundation,
- *  either version 3 of the License, or (at your option) any later version.
- *
- *  Yoshimi-Testsuite is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU General Public License
- *  along with yoshimi.  If not, see <http://www.gnu.org/licenses/>.
- ***************************************************************/
-
-
-/** @file error.hpp
- ** Definition of semantic exception classes and helpers for error handling.
- ** - error::LogicBroken : violation of the application's internal logic assumptions.
- **                        Typically raising this exception implies a programming error.
- ** - error::Misconfig   : settings in configuration files or commandline miss expectations.
- ** - error::ToDo        : marker for "Stubs" or planned functionality during development.
- ** - error::State       : unexpected state or failure in system call.
- **
- ** \par Shortcuts and Helpers
- ** - Macro \ref UNIMPLEMENTED : shortcut for raising a error::ToDo
- **
- ** @todo WIP as of 7/21
- **
- */
-
-
-
-#ifndef TESTRUNNER_UTIL_ERROR_HPP_
-#define TESTRUNNER_UTIL_ERROR_HPP_
-
-
-#include <stdexcept>
-#include <string>
-
-using std::string;
-
-
-
-namespace error {
-
-using std::logic_error;
-
-
-class LogicBroken : public logic_error
-{
-public:
-    LogicBroken(string msg)
-        : logic_error{"LogicBroken: " + msg}
-    { }
-};
-
-
-class Misconfig : public logic_error
-{
-public:
-    Misconfig(string msg)
-        : logic_error{"Misconfig: "+msg}
-    { }
-};
-
-
-class Invalid : public logic_error
-{
-public:
-    Invalid(string msg)
-        : logic_error{"Invalid Data: "+msg}
-    { }
-};
-
-
-class State : public logic_error
-{
-public:
-    State(string msg)
-        : logic_error{"Unforeseen state: "+msg}
-    { }
-};
-
-
-class FailedLaunch : public State
-{
-public:
-    FailedLaunch(string msg)
-        : State{"Launch of Test Case failed -- "+msg}
-    { }
-};
-
-
-class ToDo : public logic_error
-{
-public:
-    ToDo(string msg) :
-        logic_error{"UNIMPLEMENTED: "+msg}
-    { }
-};
-
-
-} // namespace error
-
-
-#define UNIMPLEMENTED(_MSG_) \
-    throw error::ToDo(_MSG_)
-
-#endif /*TESTRUNNER_UTIL_ERROR_HPP_*/

src/lib/stat/file.cpp

@@ -0,0 +1,41 @@
+/*
+  File  -  collection of functions supporting unit testing
+
+  Copyright (C)         Lumiera.org
+    2009,               Hermann Vosseler <Ichthyostega@web.de>
+
+  This program is free software; you can redistribute it and/or
+  modify it under the terms of the GNU General Public License as
+  published by the Free Software Foundation; either version 2 of
+  the License, or (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program; if not, write to the Free Software
+  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+* *****************************************************/
+
+
+/** @file file.cpp
+ ** Storage and implementation details for filesystem helpers.
+ ** @todo 3/24 consider to establish some central location to define basic literals.
+ */
+
+
+#include "lib/stat/file.hpp"
+
+
+namespace std::filesystem {
+  const string UNIX_HOMEDIR_SYMBOL{"~"};
+  lib::Literal UNIX_HOMEDIR_ENV{"HOME"};
+}
+
+namespace util {
+  using fs::UNIX_HOMEDIR_SYMBOL;
+  using fs::UNIX_HOMEDIR_ENV;
+}

src/lib/stat/file.hpp

@@ -1,40 +1,46 @@
 /*
- *  file - filesystem access and helpers
+  FILE.hpp  -  Filesystem access and helpers
- *
+
- *  Copyright 2021, Hermann Vosseler <Ichthyostega@web.de>
+  Copyright (C)         Lumiera.org
- *
+    2022,               Hermann Vosseler <Ichthyostega@web.de>
- *  This file is part of the Yoshimi-Testsuite, which is free software:
+
- *  you can redistribute and/or modify it under the terms of the GNU
+  This program is free software; you can redistribute it and/or
- *  General Public License as published by the Free Software Foundation,
+  modify it under the terms of the GNU General Public License as
- *  either version 3 of the License, or (at your option) any later version.
+  published by the Free Software Foundation; either version 2 of
- *
+  the License, or (at your option) any later version.
- *  Yoshimi-Testsuite is distributed in the hope that it will be useful,
+
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  This program is distributed in the hope that it will be useful,
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  GNU General Public License for more details.
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *
+  GNU General Public License for more details.
- *  You should have received a copy of the GNU General Public License
+
- *  along with yoshimi.  If not, see <http://www.gnu.org/licenses/>.
+  You should have received a copy of the GNU General Public License
- ***************************************************************/
+  along with this program; if not, write to the Free Software
+  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+*/
 
 
 /** @file file.hpp
  ** Includes the C++ Filesystem library and provides some convenience helpers.
  ** The `std::filesystem` library allows for portable access to file and directory handling
- ** functions; this header maps these functions with a convenient `fs::` namespace prefix,
+ ** functions; this header maps these functions with an concise `fs::` namespace prefix,
- ** and offers some convenience extensions, which are _"slightly non-portable"_ (they were
+ ** and offers some convenience extensions, which are _„slightly non-portable“_ (meaning they
- ** developed on Linux and "should" work on Unix like systems; adapters for exotic operating
+ ** were developed on Linux and „should“ work on Unix like systems; adapters for exotic operating
  ** systems could be added here when necessary...)
- **
+ ** 
+ ** @todo 3/2024 should be part of generic utilities
  */
 
 
 
-#ifndef TESTRUNNER_UTIL_FILE_HPP_
+#ifndef LIB_STAT_FILE_H
-#define TESTRUNNER_UTIL_FILE_HPP_
+#define LIB_STAT_FILE_H
 
 
-#include "util/error.hpp"
+#include "lib/error.hpp"
+#include "lib/symbol.hpp"
+#include "lib/meta/util.hpp"
 
 #include <filesystem>
 #include <cstdlib>
@@ -42,43 +48,54 @@
 
 namespace fs = std::filesystem;
 namespace std::filesystem {
-
+  
-const string       UNIX_HOMEDIR_SYMBOL = "~";
+  extern const string UNIX_HOMEDIR_SYMBOL;
-const char * const UNIX_HOMEDIR_ENV    = "HOME";
+  extern lib::Literal UNIX_HOMEDIR_ENV;
-
+  
-
+  
-inline fs::path getHomePath()
+  inline fs::path
-{
+  getHomePath()
+  {
     auto home = std::getenv(UNIX_HOMEDIR_ENV);
     if (not home)
-        throw error::Misconfig("Program environment doesn't define $HOME (Unix home directory).");
+        throw lumiera::error::Config{"Program environment doesn't define $HOME (Unix home directory)."};
     return fs::path{home};
-}
+  }
-
+  
-
+  
-/** resolves symlinks, `~` (Unix home dir) and relative specs
+  /** resolves symlinks, `~` (Unix home dir) and relative specs
- * @return absolute canonical form if the path exists;
+   * @return absolute canonical form if the path exists;
- *         otherwise only the home directory is expanded */
+   *         otherwise only the home directory is expanded */
-inline fs::path consolidated(fs::path rawPath)
+  inline fs::path
-{
+  consolidated (fs::path rawPath)
-    if (rawPath.empty())
+  {
-        return rawPath;
+      if (rawPath.empty())
-    if (UNIX_HOMEDIR_SYMBOL == *rawPath.begin())
+          return rawPath;
-        rawPath = getHomePath() / rawPath.lexically_proximate(UNIX_HOMEDIR_SYMBOL);
+      if (UNIX_HOMEDIR_SYMBOL == *rawPath.begin())
-
+          rawPath = getHomePath() / rawPath.lexically_proximate(UNIX_HOMEDIR_SYMBOL);
-    return fs::exists(rawPath)? fs::absolute(
+  
-                                fs::canonical(rawPath))
+      return fs::exists(rawPath)? fs::absolute(
-                              : rawPath;
+                                  fs::canonical(rawPath))
-}
+                                : rawPath;
-
+  }
 }//(End)namespace fs
 
+
+
 namespace util {
-
+  
-inline string formatVal(fs::path path)
+  /** specialisation: render filesystem path as double quoted string */
-{
+  template<>
-    return "\""+string{path}+"\"";
+  struct StringConv<fs::path, void>
-}
+    {
-
+      static std::string
+      invoke (fs::path path) noexcept
+      try {
+          return "\""+std::string{path}+"\"";
+        }
+      catch(...)
+        { return lib::meta::FAILURE_INDICATOR; }
+    };
+  
 }//(End)namespace util
-#endif /*TESTRUNNER_UTIL_TEE_HPP_*/
+#endif /*LIB_STAT_FILE_H*/

src/lib/stat/format.hpp

@@ -1,108 +0,0 @@
-/*
- *  format - collection of test formatting helpers
- *
- *  Copyright 2021, Hermann Vosseler <Ichthyostega@web.de>
- *
- *  This file is part of the Yoshimi-Testsuite, which is free software:
- *  you can redistribute and/or modify it under the terms of the GNU
- *  General Public License as published by the Free Software Foundation,
- *  either version 3 of the License, or (at your option) any later version.
- *
- *  Yoshimi-Testsuite is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU General Public License
- *  along with yoshimi.  If not, see <http://www.gnu.org/licenses/>.
- ***************************************************************/
-
-
-/** @file format.hpp
- ** Collection of helper functions for text and number output and formatting.
- ** @todo WIP as of 7/21
- **
- */
-
-
-
-#ifndef TESTRUNNER_UTIL_FORMAT_HPP_
-#define TESTRUNNER_UTIL_FORMAT_HPP_
-
-
-#include "util/utils.hpp"
-
-#include <string>
-#include <sstream>
-
-using std::string;
-
-
-
-namespace util
-{
-
-
-/** format number as string */
-template<typename NUM>
-inline string str(NUM n)
-{
-   std::ostringstream oss;
-   oss << n;
-   return oss.str();
-}
-
-
-template<typename X>
-inline string formatVal(X x)
-{
-    return str(x);
-}
-
-inline string formatVal(string s)
-{
-    return "\""+s+"\"";
-}
-
-inline string formatVal(bool yes)
-{
-    return yes? "true":"false";
-}
-
-inline string formatVal(float f)
-{
-   std::ostringstream oss;
-   oss.precision(3);
-   oss.width(5);
-   oss << f;
-   return oss.str();
-}
-
-
-/** parse string representation into typed value */
-template<typename TAR>
-inline TAR parseAs(string const& encodedVal)
-{
-    std::istringstream converter{encodedVal};
-    TAR value;
-    converter >> value;
-    if (converter.fail())
-        throw error::Invalid("unable to parse "+formatVal(encodedVal));
-    return value;
-}
-
-template<>
-inline bool parseAs(string const& encodedBool)
-{
-    return util::boolVal(encodedBool);
-}
-template<>
-inline string parseAs(string const& string)
-{
-    return string; // pass-through (even if empty)
-}
-
-
-
-}//namespace util
-#endif /*TESTRUNNER_UTIL_FORMAT_HPP_*/

src/lib/stat/regex.hpp

@@ -1,33 +1,35 @@
 /*
- *  regex - helpers for working with regular expressions
+  REGEX.hpp  -  helpers for working with regular expressions
- *
+
- *  Copyright 2021, Hermann Vosseler <Ichthyostega@web.de>
+  Copyright (C)         Lumiera.org
- *
+    2022,               Hermann Vosseler <Ichthyostega@web.de>
- *  This file is part of the Yoshimi-Testsuite, which is free software:
+
- *  you can redistribute and/or modify it under the terms of the GNU
+  This program is free software; you can redistribute it and/or
- *  General Public License as published by the Free Software Foundation,
+  modify it under the terms of the GNU General Public License as
- *  either version 3 of the License, or (at your option) any later version.
+  published by the Free Software Foundation; either version 2 of
- *
+  the License, or (at your option) any later version.
- *  Yoshimi-Testsuite is distributed in the hope that it will be useful,
+
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  This program is distributed in the hope that it will be useful,
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  GNU General Public License for more details.
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *
+  GNU General Public License for more details.
- *  You should have received a copy of the GNU General Public License
+
- *  along with yoshimi.  If not, see <http://www.gnu.org/licenses/>.
+  You should have received a copy of the GNU General Public License
- ***************************************************************/
+  along with this program; if not, write to the Free Software
+  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+*/
 
 
 /** @file regex.hpp
  ** Convenience wrappers and helpers for dealing with regular expressions.
- ** @see suite::step::ExeLauncher::ExeLauncher
+ ** 
- **
+ ** @todo 3/2024 should be with the generic utils, might be a Lumiera Forward Iterator
  */
 
 
-
+#ifndef LIB_STAT_REGEX_H
-#ifndef TESTRUNNER_UTIL_REGEX_HPP_
+#define LIB_STAT_REGEX_H
-#define TESTRUNNER_UTIL_REGEX_HPP_
 
 
 #include <regex>
@@ -35,53 +37,25 @@
 #include <optional>
 
 namespace util {
-
+  
-using std::regex;
+  using std::regex;
-using std::smatch;
+  using std::smatch;
-using std::string;
+  using std::string;
-
+  
-
+  
-
+  
-/** wrapped regex iterator to allow usage in foreach loops */
+  /** wrapped regex iterator to allow usage in foreach loops */
-struct MatchSeq
+  struct MatchSeq
     : std::sregex_iterator
-{
+    {
-    MatchSeq(string const& toParse, regex const& regex)
+      MatchSeq (string const& toParse, regex const& regex)
         : std::sregex_iterator{toParse.begin(), toParse.end(), regex}
-    { }
+        { }
-
+  
-    using iterator = std::sregex_iterator;
+      using iterator = std::sregex_iterator;
-    iterator begin() { return *this; }
+      iterator begin() const { return *this; }
-    iterator end()   { return iterator(); }
+      iterator end()   const { return iterator(); }
-};
+    };
-
+  
-
+} // namespace util
-
+#endif/*LIB_STAT_REGEX_H*/
-/** encapsulated regex buildable from string */
-class Matcher
-{
-    std::optional<regex> pattern_;
-
-public:
-    Matcher()  = default;
-    Matcher(string const& regexDefinition)
-        : pattern_{regexDefinition.empty()? std::nullopt
-                                          : std::make_optional<regex>(regexDefinition, regex::optimize)}
-    { }
-    // standard copy acceptable
-
-    explicit operator bool()  const
-    {
-        return bool(pattern_);
-    }
-
-    bool matchesWithin(string probe)  const
-    {
-        return pattern_? std::regex_search(probe, *pattern_)
-                       : true; // undefined pattern matches everything
-    }
-};
-
-
-}//(End)namespace util
-#endif /*TESTRUNNER_UTIL_REGEX_HPP_*/

src/lib/stat/statistic.hpp

@@ -1,21 +1,24 @@
 /*
- *  statistic - helpers for generic statistics calculations
+  STATISTIC.hpp  -  helpers for generic statistics calculations
- *
+
- *  Copyright 2021, Hermann Vosseler <Ichthyostega@web.de>
+  Copyright (C)         Lumiera.org
- *
+    2022,               Hermann Vosseler <Ichthyostega@web.de>
- *  This file is part of the Yoshimi-Testsuite, which is free software:
+
- *  you can redistribute and/or modify it under the terms of the GNU
+  This program is free software; you can redistribute it and/or
- *  General Public License as published by the Free Software Foundation,
+  modify it under the terms of the GNU General Public License as
- *  either version 3 of the License, or (at your option) any later version.
+  published by the Free Software Foundation; either version 2 of
- *
+  the License, or (at your option) any later version.
- *  Yoshimi-Testsuite is distributed in the hope that it will be useful,
+
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  This program is distributed in the hope that it will be useful,
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  GNU General Public License for more details.
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *
+  GNU General Public License for more details.
- *  You should have received a copy of the GNU General Public License
+
- *  along with yoshimi.  If not, see <http://www.gnu.org/licenses/>.
+  You should have received a copy of the GNU General Public License
- ***************************************************************/
+  along with this program; if not, write to the Free Software
+  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+*/
 
 
 /** @file statistic.cpp
@@ -28,230 +31,264 @@
 
 
 
-#ifndef TESTRUNNER_UTIL_STATISTIC_HPP_
+#ifndef LIB_STAT_STATISTIC_H
-#define TESTRUNNER_UTIL_STATISTIC_HPP_
+#define LIB_STAT_STATISTIC_H
 
 
-#include "util/error.hpp"
+#include "lib/error.hpp"
-#include "util/nocopy.hpp"
+#include "lib/nocopy.hpp"
-#include "util/format.hpp"
+#include "lib/format-string.hpp"
-#include "util/utils.hpp"
+#include "lib/util.hpp"
 
+#include <utility>
 #include <vector>
 #include <array>
 #include <tuple>
 #include <cmath>
 
-namespace util {
+namespace lib {
-
+namespace stat{
-using std::fabs;
+  
-using std::array;
+  namespace error = lumiera::error;
-using std::tuple;
+  
-using std::make_tuple;
+  using std::fabs;
-
+  using std::array;
-using VecD = std::vector<double>;
+  using std::tuple;
-
+  using std::make_tuple;
-/** helper to unpack a std::tuple into a homogeneous std::array */
+  using std::forward;
-template<typename TUP>
+  using std::move;
-constexpr auto array_from_tuple(TUP&& tuple)
+  using util::min;
-{
+  using util::max;
-    constexpr auto makeArray = [](auto&& ... x){ return std::array{std::forward<decltype(x)>(x) ... }; };
+  using util::isnil;
-    return std::apply(makeArray, std::forward<TUP>(tuple));
+  using util::_Fmt;
-}
+  
-
+  using VecD = std::vector<double>;
-template<size_t places>
+  
-inline double round(double val)
+  
-{
+  
-    constexpr double shiftFac = pow(10.0,places);
+  /** helper to unpack a std::tuple into a homogeneous std::array */
-    return std::round(val * shiftFac)/shiftFac;
+  template<typename TUP>
-}
+  constexpr auto
-
+  array_from_tuple (TUP&& tuple)
-
+  {
-
+    constexpr auto makeArray = [](auto&& ... x)
-
+      {
-/**
+        return std::array{forward<decltype(x)> (x) ...};
- * Read-only view into a segment within a sequence of data
+      };
- * @tparam D value type of the data series
+    return std::apply (makeArray, forward<TUP> (tuple));
- * @remark simplistic workaround since we don't support C++20 yet
+  }
- * @todo replace by const std::span
+  
- */
+  template<size_t places>
-template<typename D>
+  inline double
-class DataSpan
+  round (double val)
+  {
+    constexpr double shift{pow(10.0, places)};
+    return std::round(val*shift) / shift;
+  }
+  
+  
+  
+  
+  /**
+   * Read-only view into a segment within a sequence of data
+   * @tparam D value type of the data series
+   * @remark simplistic workaround since we don't support C++20 yet
+   * @todo replace by const std::span
+   */
+  template<typename D>
+  class DataSpan
     : util::Cloneable
-{
+    {
-    const D* const b_{nullptr};
+      const D* const b_{nullptr};
-    const D* const e_{nullptr};
+      const D* const e_{nullptr};
-
+      
-public:
+    public:
-    DataSpan() = default;
+      DataSpan() = default;
-    DataSpan(D const& begin, D const& end)
+      DataSpan (D const& begin, D const& end)
         : b_{&begin}
         , e_{&end}
-    {   if (e_ < b_) throw error::Invalid("End point before begin."); }
+        {
-
+          if (e_ < b_)
-    template<class CON>
+            throw error::Invalid{"End point before begin."};
-    DataSpan(CON const& container)
+        }
+      
+      template<class CON>
+      DataSpan (CON const& container)
         : DataSpan{*std::begin(container), *std::end(container)}
-    { }
+        { }
-
+      
-
+      
-    using iterator = const D*;
+      using iterator = const D*;
-
+      
-    size_t size()  const { return e_ - b_; }
+      size_t size()  const { return e_ - b_; }
-    bool empty()   const { return b_ == e_; }
+      bool empty()   const { return b_ == e_;}
-
+      
-    iterator begin() const { return b_; }
+      iterator begin() const { return b_; }
-    iterator end()   const { return e_; }
+      iterator end()   const { return e_; }
-
+      
-    D const& operator[](size_t i) const { return b_ + i; }
+      D const& operator[](size_t i) const { return b_ + i; }
-    D const& at(size_t i) const
+      D const& at(size_t i)  const
-    {
+        {
-        if (i >= size()) throw error::Invalid("Index "+str(i)+" beyond size="+str(size()));
+          if (i >= size())
-        return this->operator[](i);
+            throw error::Invalid{_Fmt{"Index %d beyond size=%d"}
-    }
+                                     % i % size()};
-};
+          return this->operator[](i);
-
+        }
-
+    };
-
+  
-/** summation of variances, for error propagation: √Σe² */
+  
-template<typename... NUMS>
+  
-inline double errorSum(NUMS ...vals)
+  /** summation of variances, for error propagation: √Σe² */
-{
+  template<typename... NUMS>
+  inline double
+  errorSum (NUMS ...vals)
+  {
     auto sqr = [](auto val){ return val*val; };
     return sqrt((sqr(vals)+ ... + 0.0));
-}
+  }
-
+  
-
+  
-
+  
-template<typename D>
+  template<typename D>
-inline double average(DataSpan<D> const& data)
+  inline double
-{
+  average (DataSpan<D> const& data)
+  {
     if (isnil(data)) return 0.0;
     double sum = 0.0;
     for (auto val : data)
-        sum += val;
+      sum += val;
     return sum / data.size();
-}
+  }
-
+  
-template<typename D>
+  template<typename D>
-inline double sdev(DataSpan<D> const& data, D mean)
+  inline double
-{
+  sdev (DataSpan<D> const& data, D mean)
+  {
     if (isnil(data)) return 0.0;
     double sdev = 0.0;
     for (auto val : data)
-    {
+      {
         D offset = val - mean;
         sdev += offset*offset;
-    }
+      }
     size_t n = data.size();
     sdev /= n<2? 1: n-1;
-    return sqrt(sdev);
+    return sqrt (sdev);
-}
+  }
-
+  
-inline double sdev(VecD const& data, double mean)
+  inline double
-{   return sdev(DataSpan<double>{data}, mean); }
+  sdev (VecD const& data, double mean)
-
+  {
-
+    return sdev(DataSpan<double>{data}, mean);
-
+  }
-inline DataSpan<double> lastN(VecD const& data, size_t n)
+  
-{
+  
-    n = std::min(n, data.size());
+  
+  inline DataSpan<double>
+  lastN (VecD const& data, size_t n)
+  {
+    n = min (n, data.size());
     size_t oldest = data.size() - n;
     return DataSpan<double>{data[oldest], *data.end()};
-}
+  }
-
+  
-inline double averageLastN(VecD const& data, size_t n)
+  inline double
-{
+  averageLastN (VecD const& data, size_t n)
-    return average(lastN(data,n));
+  {
-}
+    return average (lastN (data,n));
-
+  }
-inline double sdevLastN(VecD const& data, size_t n, double mean)
+  
-{
+  inline double
-    return sdev(lastN(data,n), mean);
+  sdevLastN (VecD const& data, size_t n, double mean)
-}
+  {
-
+    return sdev (lastN (data,n), mean);
-
+  }
-/** "building blocks" for mean, variance and covariance of time series data */
+  
-template<typename D>
+  
-inline auto computeStatSums(DataSpan<D> const& series)
+  /** "building blocks" for mean, variance and covariance of time series data */
-{
+  template<typename D>
+  inline auto
+  computeStatSums (DataSpan<D> const& series)
+  {
     double ysum = 0.0;
     double yysum = 0.0;
     double xysum = 0.0;
     size_t x = 0;
     for (auto& y : series)
-    {
+      {
         ysum += y;
         yysum += y*y;
         xysum += x*y;
         ++x;
-    }
+      }
-    return make_tuple(ysum,yysum, xysum);
+    return make_tuple (ysum,yysum, xysum);
-}
+  }
-
+  
-
+  
-/**
+  /**
- * Single data point used for linear regression.
+   * Single data point used for linear regression.
- * Simple case: single predictor variable (x).
+   * Simple case: single predictor variable (x).
- * @remark including a weight factor
+   * @remark including a weight factor
- */
+   */
-struct RegressionPoint
+  struct RegressionPoint
-{
+    {
-    double x;
+      double x;
-    double y;
+      double y;
-    double w;
+      double w;
-};
+    };
-
+  
-using RegressionData = std::vector<RegressionPoint>;
+  using RegressionData = std::vector<RegressionPoint>;
-
+  
-
+  
-/** "building blocks" for weighted mean, weighted variance and covariance */
+  /** "building blocks" for weighted mean, weighted variance and covariance */
-inline auto computeWeightedStatSums(DataSpan<RegressionPoint> const& points)
+  inline auto
-{
+  computeWeightedStatSums (DataSpan<RegressionPoint> const& points)
+  {
     std::array<double,6> sums;
     sums.fill(0.0);
     auto& [wsum, wxsum, wysum, wxxsum, wyysum, wxysum] = sums;
     for (auto& p : points)
-    {
+      {
         wsum += p.w;
         wxsum += p.w * p.x;
         wysum += p.w * p.y;
         wxxsum += p.w * p.x*p.x;
         wyysum += p.w * p.y*p.y;
         wxysum += p.w * p.x*p.y;
-    }
+      }
     return sums;
-}
+  }
-
+  
-/**
+  /**
- * Compute simple linear regression with a single predictor variable (x).
+   * Compute simple linear regression with a single predictor variable (x).
- * @param points 2D data to fit the linear model with, including weights.
+   * @param points 2D data to fit the linear model with, including weights.
- * @return the computed linear model `b + a·x`, and the resulting fit
+   * @return the computed linear model `b + a·x`, and the resulting fit
- *         - socket (constant offset `b`)
+   *       - socket (constant offset `b`)
- *         - gradient (linear factor `a`)
+   *       - gradient (linear factor `a`)
- *         - a vector with a predicted `y` value for each `x` value
+   *       - a vector with a predicted `y` value for each `x` value
- *         - a vector with the error, i.e `Δ = y - y_predicted`
+   *       - a vector with the error, i.e `Δ = y - y_predicted`
- *         - correlation between x and y values
+   *       - correlation between x and y values
- *         - maximum absolute delta
+   *       - maximum absolute delta
- *         - delta standard deviation
+   *       - delta standard deviation
- */
+   */
-inline auto computeLinearRegression(DataSpan<RegressionPoint> const& points)
+  inline auto
-{
+  computeLinearRegression (DataSpan<RegressionPoint> const& points)
+  {
     auto [wsum, wxsum, wysum, wxxsum, wyysum, wxysum] = computeWeightedStatSums(points);
-
+    
     double xm = wxsum / wsum;                                       // weighted mean x = 1/Σw · Σwx
     double ym = wysum / wsum;
     double varx = wxxsum + xm*xm * wsum - 2*xm * wxsum;             // Σw · x-Variance = Σw(x-xm)²
     double vary = wyysum + ym*ym * wsum - 2*ym * wysum;
     double cova = wxysum + xm*ym * wsum - ym * wxsum - xm * wysum;  // Σw · Covariance = Σw(x-xm)(y-ym)
-
+    
     // Linear Regression minimising σ²
     double gradient = cova / varx;                                  // gradient = correlation · σy / σx ; σ = √Variance
     double socket   = ym - gradient * xm;                           // Regression line:  Y-ym = gradient · (x-xm)  ; set x≔0 yields socket
-
+    
     // Correlation (Pearson's r)
     double correlation = wyysum==0.0? 1.0 : gradient * sqrt(varx/vary);
-
+    
     // calculate error Δ for all measurement points
     size_t n = points.size();
     VecD predicted;  predicted.reserve(n);
@@ -259,66 +296,71 @@ inline auto computeLinearRegression(DataSpan<RegressionPoint> const& points)
     double maxDelta = 0.0;
     double variance = 0.0;
     for (auto& p : points)
-    {
+      {
         double y_pred = socket + gradient * p.x;
         double delta  = p.y - y_pred;
-        predicted.push_back(y_pred);
+        predicted.push_back (y_pred);
-        deltas.push_back(delta);
+        deltas.push_back (delta);
-        maxDelta = std::max(maxDelta, fabs(delta));
+        maxDelta = max (maxDelta, fabs(delta));
         variance += p.w * delta*delta;
-    }
+      }
     variance /= wsum * (n<=2? 1 : (n-2)/double(n)); // N-2 because it's an estimation,
                                                     // based on 2 other estimated values (socket,gradient)
-    return make_tuple(socket,gradient
+    return make_tuple (socket,gradient
-                     ,move(predicted)
+                      ,move(predicted)
-                     ,move(deltas)
+                      ,move(deltas)
-                     ,correlation
+                      ,correlation
-                     ,maxDelta
+                      ,maxDelta
-                     ,sqrt(variance)
+                      ,sqrt(variance)
-                     );
+                      );
-}
+  }
-
+  
-inline auto computeLinearRegression(RegressionData const& points)
+  inline auto
-{   return computeLinearRegression(DataSpan<RegressionPoint>{points}); }
+  computeLinearRegression (RegressionData const& points)
-
+  {
-
+    return computeLinearRegression(DataSpan<RegressionPoint>{points});
-
+  }
-/**
+  
- * Compute linear regression over a time series with zero-based indices.
+  
- * @remark using the indices as x-values, the calculations for a regression line
+  
- *         can be simplified, using the known closed formula for a sum of integers,
+  /**
- *         shifting the indices to 0…n-1 (leaving out the 0 and 0² term)
+   * Compute linear regression over a time series with zero-based indices.
- *         - `1+…+n = n·(n+1)/2`
+   * @remark using the indices as x-values, the calculations for a regression line
- *         - `1+…+n² = n·(n+1)·(2n+1)/6`
+   *         can be simplified, using the known closed formula for a sum of integers,
- * @return `(socket,gradient)` to describe the regression line y = socket + gradient · i
+   *         shifting the indices to 0…n-1 (leaving out the 0 and 0² term)
- */
+   *         - `1+…+n = n·(n+1)/2`
-template<typename D>
+   *         - `1+…+n² = n·(n+1)·(2n+1)/6`
-inline auto computeTimeSeriesLinearRegression(DataSpan<D> const& series)
+   * @return `(socket,gradient)` to describe the regression line y = socket + gradient · i
-{
+   */
+  template<typename D>
+  inline auto
+  computeTimeSeriesLinearRegression (DataSpan<D> const& series)
+  {
     if (series.size() < 2) return make_tuple(0.0,0.0,0.0);
-
+    
     auto [ysum,yysum, xysum] = computeStatSums(series);
-
+    
     size_t n = series.size();
     double im = (n-1)/2.0;                     // mean of zero-based indices i ∈ {0 … n-1}
     double ym = ysum / n;                      // mean y
     double varx = (n-1)*(n+1)/12.0;            // variance of zero-based indices Σ(i-im)² / n
     double vary = yysum/n - ym*ym;             // variance of data values  Σ(y-ym)² / n
-    double cova = xysum  - ysum *(n-1)/2;      // Time series Covariance = Σ(i-im)(y-ym) = Σiy + im·ym·n - ym·Σi - im·Σy; use n*ym = Σy
+    double cova = xysum  - ysum *(n-1)/2;      // Time series Covariance = Σ(i-im)(y-ym) = Σiy + im·ym·n - ym·Σi - im·Σy; use n·ym ≙ Σy
-
+    
     // Linear Regression minimising σ²
     double gradient = cova / (n*varx);         // Gradient = Correlation · σy / σx ; σ = √Variance;  Correlation = Covariance /(√Σx √Σy)
     double socket   = ym - gradient * im;      // Regression line:  Y-ym = Gradient · (i-im)  ; set i≔0 yields socket
-
+    
     // Correlation (Pearson's r)
     double correlation = yysum==0.0? 1.0 : gradient * sqrt(varx/vary);
-    return make_tuple(socket,gradient,correlation);
+    return make_tuple (socket,gradient,correlation);
-}
+  }
-
+  
-inline auto computeTimeSeriesLinearRegression(VecD const& series)
+  inline auto
-{   return computeTimeSeriesLinearRegression(DataSpan<double>{series}); }
+  computeTimeSeriesLinearRegression (VecD const& series)
-
+  {
-
+    return computeTimeSeriesLinearRegression(DataSpan<double>{series});
-
+  }
-}//(End)namespace util
+  
-#endif /*TESTRUNNER_UTIL_STATISTIC_HPP_*/
+}} // namespace lib::stat
+#endif /*LIB_STAT_STATISTIC_H*/

src/lib/stat/utils.cpp

@@ -1,68 +0,0 @@
-/*
- *  utils - collection of general purpose helpers and tools
- *
- *  Copyright 2021, Hermann Vosseler <Ichthyostega@web.de>
- *
- *  This file is part of the Yoshimi-Testsuite, which is free software:
- *  you can redistribute and/or modify it under the terms of the GNU
- *  General Public License as published by the Free Software Foundation,
- *  either version 3 of the License, or (at your option) any later version.
- *
- *  Yoshimi-Testsuite is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU General Public License
- *  along with yoshimi.  If not, see <http://www.gnu.org/licenses/>.
- ***************************************************************/
-
-
-/** @file utils.cpp
- ** Implementation details for some of the generic utils.
- ** 
- ** @todo WIP as of 7/21
- **
- */
-
-
-#include "util/utils.hpp"
-#include "util/error.hpp"
-
-#include <regex>
-
-using std::regex;
-
-namespace util {
-
-namespace {
-    const regex TRUE_TOKENS { "\\s*(true|yes|on|1|\\+)\\s*",  regex::icase | regex::optimize };
-    const regex FALSE_TOKENS{ "\\s*(false|no|off|0|\\-)\\s*", regex::icase | regex::optimize };
-
-    const regex TRIMMER{"\\s*(.*?)\\s*"};
-}
-
-bool boolVal(string const& textForm)
-{
-    if (regex_match(textForm, TRUE_TOKENS))
-        return true;
-    if (regex_match(textForm, FALSE_TOKENS))
-        return false;
-    throw error::Invalid{"String '"+textForm+"' can not be interpreted as bool value" };
-}
-
-bool isYes (string const& textForm) noexcept
-{
-    return regex_match (textForm, TRUE_TOKENS);
-}
-
-
-string trimmed(string text)
-{
-    std::smatch mat;
-    regex_match(text, mat, TRIMMER);
-    return mat[1];
-}
-
-
-}//(End)namespace util

src/lib/stat/utils.hpp

@@ -1,234 +0,0 @@
-/*
- *  utils - collection of general purpose helpers and tools
- *
- *  Copyright 2021, Hermann Vosseler <Ichthyostega@web.de>
- *
- *  This file is part of the Yoshimi-Testsuite, which is free software:
- *  you can redistribute and/or modify it under the terms of the GNU
- *  General Public License as published by the Free Software Foundation,
- *  either version 3 of the License, or (at your option) any later version.
- *
- *  Yoshimi-Testsuite is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU General Public License
- *  along with yoshimi.  If not, see <http://www.gnu.org/licenses/>.
- ***************************************************************/
-
-
-/** @file utils.hpp
- ** Collection of helper functions and abbreviations used to simplify code.
- ** - \ref isnil(arg) checks if the argument is "empty"; argument can be a string or a container
- ** - some helper functions for working with strings (`startsWith`, `endsWith`, `removePrefix|Suffix`)
- ** - \ref trim(string) extracts the content without leading and trailing whitespace
- ** - \ref boolVal() and \ref isYes() interpret a string as boolean value
- ** - \ref contains() generic containment check for maps, strings and iterable containers
- ** - Macro \ref STRINGIFY
- ** @todo WIP as of 7/21
- **
- */
-
-
-
-#ifndef TESTRUNNER_UTIL_UTILS_HPP_
-#define TESTRUNNER_UTIL_UTILS_HPP_
-
-
-#include <algorithm>
-#include <string>
-#include <set>
-
-using std::string;
-
-using uint = unsigned int;
-
-
-namespace util {
-
-/*  ======== generic empty check =========  */
-
-/** a family of util functions providing a "no value whatsoever" test.
- *  Works on strings and all STL containers, includes NULL test for pointers
- */
-template<class CONT>
-inline bool isnil(const CONT &container)
-{
-    return container.empty();
-}
-
-template<class CONT>
-inline bool isnil(const CONT *pContainer)
-{
-    return !pContainer or pContainer->empty();
-}
-
-template<class CONT>
-inline bool isnil(CONT *pContainer)
-{
-    return !pContainer or pContainer->empty();
-}
-
-inline bool isnil(const char *pCStr)
-{
-    return !pCStr or !(*pCStr);
-}
-
-/** check if string starts with a given prefix */
-inline bool startsWith(string const &str, string const &prefix)
-{
-    return 0 == str.rfind(prefix, 0);
-}
-
-inline bool startsWith(string const &str, const char *prefix)
-{
-    return 0 == str.rfind(prefix, 0);
-}
-
-/** check if string ends with the given suffix */
-inline bool endsWith(string const &str, string const &suffix)
-{
-    size_t l = suffix.length();
-    if (l > str.length())
-        return false;
-    size_t pos = str.length() - l;
-    return pos == str.find(suffix, pos);
-}
-
-inline bool endsWith(string const &str, const char *suffix)
-{
-    return endsWith(str, string(suffix));
-}
-
-inline void removePrefix(string &str, string const &prefix)
-{
-    if (not startsWith(str, prefix))
-        return;
-    str = str.substr(prefix.length());
-}
-
-inline void removeSuffix(string &str, string const &suffix)
-{
-    if (not endsWith(str, suffix))
-        return;
-    str.resize(str.length() - suffix.length());
-}
-
-inline string replace(string src, string toFind, string replacement)
-{
-    for (size_t pos = src.find(toFind, 0);
-         pos != string::npos && toFind.size();
-         pos = src.find(toFind, pos+replacement.size())
-        )
-        src.replace(pos, toFind.size(), replacement);
-    return src;
-}
-
-
-/** shortcut for containment test on a map */
-template<typename MAP>
-inline bool contains(MAP &map, typename MAP::key_type const &key)
-{
-    return map.find(key) != map.end();
-}
-
-/** shortcut for set value containment test */
-template<typename T>
-inline bool contains(std::set<T> const &set, T const &val)
-{
-    return set.end() != set.find(val);
-}
-
-/** shortcut for string value containment test */
-template<typename T>
-inline bool contains(std::string const &str, const T &val)
-{
-    return str.find(val) != std::string::npos;
-}
-
-/** shortcut for brute-force containment test
- *  in any sequential container */
-template<typename SEQ>
-inline bool contains(SEQ const &cont, typename SEQ::const_reference val)
-{
-    typename SEQ::const_iterator begin = cont.begin();
-    typename SEQ::const_iterator end = cont.end();
-
-    return end != std::find(begin, end, val);
-}
-
-
-/** @internal helper type for #backwards */
-template <typename ITA>
-struct ReverseIterationAdapter { ITA& iterable; };
-
-template <typename ITA>
-auto begin (ReverseIterationAdapter<ITA> adapt)
-{
-    return std::rbegin(adapt.iterable);
-}
-
-template <typename ITA>
-auto end (ReverseIterationAdapter<ITA> adapt)
-{
-    return std::rend(adapt.iterable);
-}
-
-/**
- * Adapter to iterate backwards in a "foreach" loop.
- * @tparam ITA a STL compatible container with back iteration capability.
- * @remark solution based on the [Stackoverflow] from 2015 by [Prikso NAI].
- *
- * [Stackoverflow]: https://stackoverflow.com/a/28139075
- * [Prikso NAI]: https://stackoverflow.com/users/3970469/prikso-nai
- */
-template <typename ITA>
-inline ReverseIterationAdapter<ITA>
-backwards (ITA&& iterable)
-{
-    return { iterable };
-}
-
-
-/**
- * Shortcut for casting away `const`.
- * @warning Use with care. Can be very handy to simplify defining
- *          const and non-const variants of member functions though.
- */
-template<class OBJ>
-inline OBJ* unConst (const OBJ* o)
-{
-return const_cast<OBJ*> (o);
-}
-
-template<class OBJ>
-inline OBJ& unConst (OBJ const& ro)
-{
-return const_cast<OBJ&> (ro);
-}
-
-
-/** @return content without leading or trailing whitespace */
-string trimmed(string);
-
-/** interpret the given text as boolean value
- * @throws error::Invalid when the text is not any valid bool token
- * @remark allowed are `true false yes no on off 1 0 + -` in upper and lower case
- */
-bool boolVal(string const& textForm);
-
-/** evaluate the given text form as boolean value for `true`
- * @note other than (\ref boolVal), this function treats _everything else_ as `false`
- */
-bool isYes (string const& textForm) noexcept;
-
-} // namespace util
-
-
-/** this macro wraps its parameter into a cstring literal */
-#define STRINGIFY(TOKEN) __STRNGFY(TOKEN)
-#define __STRNGFY(TOKEN) #TOKEN
-
-
-#endif /*TESTRUNNER_UTIL_UTILS_HPP_*/

wiki/thinkPad.ichthyo.mm

@@ -111771,6 +111771,106 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
 <node CREATED="1708744009942" ID="ID_1661435132" MODIFIED="1708744019908" TEXT="sicherstellen da&#xdf; die Grenzpunkte mit dabei sind"/>
 </node>
 </node>
+<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1710079791943" ID="ID_1402055509" MODIFIED="1710080437833" TEXT="Integration">
+<linktarget COLOR="#9d4168" DESTINATION="ID_1402055509" ENDARROW="Default" ENDINCLINATION="-761;122;" ID="Arrow_ID_397225736" SOURCE="ID_1781115298" STARTARROW="None" STARTINCLINATION="-165;-9;"/>
+<icon BUILTIN="flag-yellow"/>
+<node BACKGROUND_COLOR="#eef0c5" COLOR="#990000" CREATED="1710079800390" ID="ID_746814828" MODIFIED="1710124936401" TEXT="meine Statistik- und CSV Hilfsmittel von Yoshimi-Test einbringen">
+<icon BUILTIN="pencil"/>
+<node COLOR="#338800" CREATED="1710085462347" ID="ID_963243204" MODIFIED="1710114559233" TEXT="zusammenh&#xe4;ngenden Code-Cluster identifizieren">
+<icon BUILTIN="button_ok"/>
+<node CREATED="1710085659317" ID="ID_1596614133" MODIFIED="1710087280098" TEXT="statistic.hpp"/>
+<node CREATED="1710085720690" ID="ID_1192903835" MODIFIED="1710087283601" TEXT="data.hpp"/>
+<node CREATED="1710085706676" ID="ID_1438550945" MODIFIED="1710085719148" TEXT="csv.hpp"/>
+<node CREATED="1710085711443" ID="ID_661927366" MODIFIED="1710085717174" TEXT="file.hpp"/>
+<node CREATED="1710087255899" ID="ID_1405213598" MODIFIED="1710087276129" TEXT="regex.hpp"/>
+</node>
+<node BACKGROUND_COLOR="#c8c0b6" COLOR="#338800" CREATED="1710085818861" ID="ID_311148994" MODIFIED="1710114625741" TEXT="Historie extrahieren">
+<icon BUILTIN="button_ok"/>
+<node BACKGROUND_COLOR="#ccb59b" COLOR="#6e2a38" CREATED="1710089706576" ID="ID_902559893" MODIFIED="1710089727345" TEXT="eingesch&#xe4;tzt als Einmal-Aktion">
+<font ITALIC="true" NAME="SansSerif" SIZE="14"/>
+<icon BUILTIN="yes"/>
+<node CREATED="1710089730179" ID="ID_597450586" MODIFIED="1710089743699" TEXT="kann daher einmalig eine vor-gefilterte Historie konstruieren"/>
+<node CREATED="1710089754973" ID="ID_864520632" MODIFIED="1710089845717" TEXT="mache noch auf dem separaten Branch einen Vorbereitungs-Commit"/>
+<node CREATED="1710089859192" ID="ID_449981342" MODIFIED="1710089932978" TEXT="danch per git subtree add einbringen"/>
+<node CREATED="1710089895487" ID="ID_1694612812" MODIFIED="1710089925698" TEXT="eventuelle sp&#xe4;tere &#xc4;nderungen m&#xfc;ssen dann manuell per merge/rebase nachgezogen werden"/>
+</node>
+<node COLOR="#338800" CREATED="1710085830281" ID="ID_1502487564" MODIFIED="1710114568091" TEXT="Git filter-branch">
+<icon BUILTIN="button_ok"/>
+<node CREATED="1710090239621" ID="ID_1765645982" MODIFIED="1710090268743" TEXT="das Yoshimi-test - Repo als remote vorr&#xfc;bergehend ankoppeln"/>
+<node CREATED="1710090274604" ID="ID_1300808535" MODIFIED="1710090290818" TEXT="einen git filter-branch im Lumiera-Repo lokal ausf&#xfc;hren"/>
+</node>
+<node COLOR="#338800" CREATED="1710086220247" ID="ID_1505106911" MODIFIED="1710114587320" TEXT="Git-Historien zusammenkoppeln">
+<icon BUILTIN="button_ok"/>
+<node COLOR="#435e98" CREATED="1710114589130" ID="ID_1762940850" MODIFIED="1710114618631" TEXT="ours-merge"/>
+<node COLOR="#435e98" CREATED="1710114593043" ID="ID_1270108404" MODIFIED="1710114618631" TEXT="git read-tree"/>
+</node>
+</node>
+<node BACKGROUND_COLOR="#eef0c5" COLOR="#990000" CREATED="1710085479914" ID="ID_1739991591" MODIFIED="1710114641924" TEXT="minimale Integration in die Lumiera-Codebase">
+<icon BUILTIN="pencil"/>
+<node COLOR="#338800" CREATED="1710085492857" ID="ID_4363413" MODIFIED="1710124911666" TEXT="Lumiera-GNU-Style">
+<icon BUILTIN="button_ok"/>
+</node>
+<node COLOR="#338800" CREATED="1710085504127" ID="ID_1115022840" MODIFIED="1710124914512" TEXT="Namespace lib::stat">
+<icon BUILTIN="button_ok"/>
+</node>
+<node COLOR="#338800" CREATED="1710085513597" ID="ID_1623287366" MODIFIED="1710124916786" TEXT="Fehler durch Lumiera-Errors ausdr&#xfc;cken">
+<icon BUILTIN="button_ok"/>
+</node>
+<node BACKGROUND_COLOR="#eef0c5" COLOR="#990000" CREATED="1710085528964" ID="ID_197737968" MODIFIED="1710124923723" TEXT="Lumiera&apos;s Format-Funktionen verwenden">
+<icon BUILTIN="pencil"/>
+<node BACKGROUND_COLOR="#f0d5c5" COLOR="#990033" CREATED="1710114655354" ID="ID_685893002" MODIFIED="1710114679967" TEXT="zu kl&#xe4;ren: vollst&#xe4;ndige floatingpoint-Repr&#xe4;sentation?">
+<icon BUILTIN="help"/>
+<node CREATED="1710114695652" ID="ID_323709862" MODIFIED="1710114697384" TEXT="std::numeric_limits&lt;VAL&gt;::digits10"/>
+<node CREATED="1710114699873" ID="ID_496274436" MODIFIED="1710114711248" TEXT="util::toString sollte lexical_cast verwenden...">
+<node CREATED="1710114712306" ID="ID_44388315" MODIFIED="1710114716318" TEXT="pr&#xfc;fen da&#xdf; dem so ist"/>
+<node CREATED="1710114717256" ID="ID_894833593" MODIFIED="1710114727188" TEXT="herausfinden was lexical-cast macht"/>
+</node>
+<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1710114730104" ID="ID_1025510498" MODIFIED="1710114735134" TEXT="empirisch vergleichen">
+<icon BUILTIN="flag-yellow"/>
+</node>
+</node>
+<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1710115014482" ID="ID_1500611682" MODIFIED="1710115023467" TEXT="kl&#xe4;ren: bool-Format">
+<icon BUILTIN="flag-yellow"/>
+</node>
+</node>
+</node>
+<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1710085571918" ID="ID_1847964804" MODIFIED="1710085586443" TEXT="per Unit-Test dokumentieren">
+<icon BUILTIN="flag-yellow"/>
+</node>
+</node>
+<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1710079820663" ID="ID_346209836" MODIFIED="1710079838390" TEXT="ein gnuplot-Skript generieren">
+<icon BUILTIN="flag-yellow"/>
+<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1710079859600" ID="ID_1865864348" MODIFIED="1710079870844" TEXT="Skript aufbauen">
+<icon BUILTIN="flag-yellow"/>
+<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1710079895431" ID="ID_129580336" MODIFIED="1710079910502" TEXT="inline-CSV lesen">
+<icon BUILTIN="flag-yellow"/>
+</node>
+<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1710079911349" ID="ID_87181546" MODIFIED="1710079915901" TEXT="Diagramm zeichnen">
+<icon BUILTIN="flag-yellow"/>
+</node>
+<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1710079916553" ID="ID_1466798395" MODIFIED="1710079922053" TEXT="Legende hinzuf&#xfc;gen">
+<icon BUILTIN="flag-yellow"/>
+</node>
+</node>
+<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1710079848190" ID="ID_227233521" MODIFIED="1710079870844" TEXT="Darstellung ausarbeiten">
+<icon BUILTIN="flag-yellow"/>
+<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1710079926851" ID="ID_1457237228" MODIFIED="1710079939594" TEXT="Scatter-Plot">
+<icon BUILTIN="flag-yellow"/>
+</node>
+<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1710079932704" ID="ID_1073652291" MODIFIED="1710079939594" TEXT="Regressionslinie">
+<icon BUILTIN="flag-yellow"/>
+</node>
+<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1710079942505" ID="ID_1703688749" MODIFIED="1710080060314" TEXT="Darstellung f&#xfc;r sekund&#xe4;re Werte">
+<icon BUILTIN="flag-yellow"/>
+<node CREATED="1710080061651" ID="ID_1708271153" MODIFIED="1710080077627" TEXT="&#x2205; concurrency"/>
+<node CREATED="1710080078359" ID="ID_1753709562" MODIFIED="1710080082366" TEXT="&#x2205; job time"/>
+<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1710080104427" ID="ID_1362230214" MODIFIED="1710080121358" TEXT="sollte erweiterbar sein">
+<icon BUILTIN="yes"/>
+</node>
+</node>
+</node>
+</node>
+</node>
 </node>
 <node BACKGROUND_COLOR="#eef0c5" COLOR="#990000" CREATED="1708744078861" ID="ID_1023448351" MODIFIED="1708744085468" TEXT="Integration / Untersuchungen">
 <icon BUILTIN="pencil"/>
@@ -111913,8 +112013,8 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
 <icon BUILTIN="closed"/>
 </node>
 </node>
-<node BACKGROUND_COLOR="#eef0c5" COLOR="#990000" CREATED="1709917304689" ID="ID_1675166139" MODIFIED="1709918087976" TEXT="Setup-2">
+<node COLOR="#338800" CREATED="1709917304689" ID="ID_1675166139" MODIFIED="1710080405027" TEXT="Setup-2">
-<icon BUILTIN="pencil"/>
+<icon BUILTIN="button_ok"/>
 <node CREATED="1709917309616" ID="ID_790939741" MODIFIED="1709917341840" TEXT="&#x27f9; brauche also doch eine spezielle Regel">
 <node CREATED="1709917609861" ID="ID_1607140185" MODIFIED="1709917789356" TEXT="jede Node prunen...">
 <icon BUILTIN="idea"/>
@@ -111983,9 +112083,36 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
 <node CREATED="1709917935300" ID="ID_1144264892" MODIFIED="1709917943590" TEXT="Ergebnisse">
 <node CREATED="1709917946410" ID="ID_1422608617" MODIFIED="1709917968051" TEXT="eine lineare Punktwolke mit ein paar &#xbb;outliern&#xab; oberhalb / unterhalb"/>
 <node CREATED="1709917990340" ID="ID_178677368" MODIFIED="1709917997831" TEXT="Konstanter Offeset bei 2.5ms"/>
-<node CREATED="1709918013177" ID="ID_115401143" MODIFIED="1709918025811" TEXT="Concurrency nur zwischen 1.5 und 2.5"/>
+<node CREATED="1709918013177" ID="ID_115401143" MODIFIED="1709918025811" TEXT="Concurrency nur zwischen 1.5 und 2.5">
+<node CREATED="1710080204406" HGAP="27" ID="ID_758385950" MODIFIED="1710080369175" TEXT="bei der kurzen Laufzeit nicht anders zu erwarten" VSHIFT="1">
+<richcontent TYPE="NOTE"><html>
+  <head/>
+  <body>
+    <p>
+      ...da zur&#252;ckkehrende Worker die Vorfahrt haben, und die typsiche R&#252;stzeit ~100&#181;s betr&#228;gt, besteht noch hinreichend Wahrscheinlichkeit f&#252;r Contention
+    </p>
+  </body>
+</html></richcontent>
+<font NAME="SansSerif" SIZE="11"/>
+</node>
+</node>
 <node CREATED="1709918027533" ID="ID_1460800650" MODIFIED="1709918071661" TEXT="&#x2205;t zeigt das &#xfc;bliche Verhalten (~700 ..900 &#xb5;s)"/>
 </node>
+<node BACKGROUND_COLOR="#c8c0b6" COLOR="#435e98" CREATED="1710080150365" HGAP="7" ID="ID_704993753" MODIFIED="1710080186753" VSHIFT="21">
+<richcontent TYPE="NODE"><html>
+  <head/>
+  <body>
+    <p>
+      <u>Stand</u>: nun bereit f&#252;r eigentliche Messungen
+    </p>
+  </body>
+</html></richcontent>
+<icon BUILTIN="forward"/>
+</node>
+</node>
+<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1710080381906" ID="ID_1781115298" MODIFIED="1710080445458" TEXT="brauche einfache Darstellbarkeit der Ergebnisse">
+<arrowlink COLOR="#9d4168" DESTINATION="ID_1402055509" ENDARROW="Default" ENDINCLINATION="-761;122;" ID="Arrow_ID_397225736" STARTARROW="None" STARTINCLINATION="-165;-9;"/>
+<icon BUILTIN="flag-yellow"/>
 </node>
 </node>
 </node>