lumiera_/src/lib/stat/data.hpp

/*
  DATA.hpp  -  read and write a table with CSV data

  Copyright (C)         Lumiera.org
    2022,               Hermann Vosseler <Ichthyostega@web.de>

  This program is free software; you can redistribute it and/or
  modify it under the terms of the GNU General Public License as
  published by the Free Software Foundation; either version 2 of
  the License, or (at your option) any later version.

  This program is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU General Public License for more details.

  You should have received a copy of the GNU General Public License
  along with this program; if not, write to the Free Software
  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.

*/


/** @file data.hpp
 ** Manage a table with time series data, stored persistently as CSV.
 ** In the context of observations, configuration, calibration and QA, a series
 ** of measurement data taken over time is often evaluated statistically, to distill
 ** typical averages, variances and trends. Short of using a database, a modest
 ** amount of numeric data can be maintained in CSV files, which also allows for
 ** further manual evaluation within a spreadsheet or statistics application.
 ** The CSV format as such can be quite elaborate, yet for the purpose of
 ** saving and later reading back some values generated by the application
 ** itself, supporting a limited format flavour is sufficient:
 ** - first line is a header line and used to verify the storage format
 ** - one record per line, embedded line breaks prohibited
 ** - fields separated by comma, semicolon tolerated
 ** - fields are trimmed and may be empty
 ** - a field may be double quoted
 ** - only quoted fields may contain whitespace or comma
 ** - no escaping of quotes, i.e. no quotes within quotes
 ** 
 ** As a fundamental building block, this header provides a data table template
 ** with flexible column configuration to hold arbitrary, explicitly typed values.
 ** This solution is statically typed and does not carry any runtime type information;
 ** the actual data table object is then defined and accessed by means of _accessor_
 ** components for each column of data. A tuple of _current values_ corresponding to
 ** the most recent row of data can be accessed directly through these sub-components.
 ** 
 ** # Usage
 ** Create an actual instantiation of the DataFile template, passing a structure
 ** with util::Column descriptors. You may then directly access the values of the
 ** _actual column_ or save/load from a persistent CSV file.
 ** @note mandatory to define a method `allColumns()`
 ** \code
 ** struct Storage
 **   {
 **     Column<string> name{"theName"};
 **     Column<int>    n{"counter"};
 **     Column<double> x{"X value"};
 **     Column<double> y{"Y value"};
 **
 **     auto allColumns(){ return std::tie(name,n,x,y); }
 **   };
 ** 
 ** using Dataz = lib::stat::DataFile<Storage>;
 ** 
 ** Dataz daz("filename.csv");
 ** 
 ** daz.x = 123e-4;
 ** daz.y = -12345e-6;
 ** 
 ** std::vector<int>& counters = daz.n.data;
 ** \endcode
 ** 
 ** @see DataCSV_test
 **
 */



#ifndef LIB_STAT_DATA_H
#define LIB_STAT_DATA_H


#include "lib/error.hpp"
#include "lib/nocopy.hpp"
#include "lib/stat/file.hpp"
#include "lib/stat/csv.hpp"
#include "lib/format-string.hpp"
#include "lib/util.hpp"

#include <type_traits>
#include <utility>
#include <fstream>
#include <vector>
#include <string>
#include <limits>
#include <deque>
#include <tuple>


namespace lib {
namespace stat{
  
  namespace error = lumiera::error;
  
  using std::move;
  using std::tuple;
  using std::vector;
  using std::string;
  using util::isnil;
  using util::unConst;
  using util::_Fmt;
  using util::min;
  
  
  
  /**
   * Helper: perform some arbitrary operation on each element of a tuple.
   * @note the given functor must be generic, since each position of the tuple
   *       may hold a data element of different type.
   * @remark credits to David Vandevoorde (member of C++ committee) for using
   *       std::apply to unpack the tuple's contents into an argument pack and
   *       then using a fold expression with the comma operator.
   */
  template<class FUN, typename...ELMS>
  void forEach(tuple<ELMS...>&& tuple, FUN fun)
  {
      std::apply([&fun](auto&... elms)
                      {
                        (fun(elms), ...);
                      }
                ,tuple);
  }
  
  
  
  /** parse string representation into typed value */
  template<typename TAR>
  inline TAR
  parseAs (string const& encodedVal)
  {
      std::istringstream converter{encodedVal};
      TAR value;
      converter >> value;
      if (converter.fail())
          throw error::Invalid{_Fmt{"unable to parse \"%s\""} % encodedVal};
      return value;
  }
  
  template<>
  inline bool
  parseAs(string const& encodedBool)
  {
      return util::boolVal(encodedBool);
  }
  template<>
  inline string
  parseAs(string const& string)
  {
      return string; // pass-through (even if empty)
  }
  
  
  
  
  
  /**
   * Descriptor and Accessor for a data column within a DataFile table.
   * @tparam VAL type of values contained within this column;
   *             this type must be _default constructible_ and _copyable._
   */
  template<typename VAL>
  struct Column
    : util::NonCopyable
    {
      string header;
      vector<VAL> data;
      
      using ValueType = VAL;
      
      
      Column (string headerID)
        : header{headerID}
        , data{}
        { }
      
      
      VAL&
      get()
        {
          if (isnil (data))
              throw error::State{"No rows in DataTable yet"};
          return data.back();
        }
      
      operator VAL&()
        {
          return get();
        }
      
      operator VAL const&()  const
        {
          return unConst(this)->get();
        }
      
      template<typename X>
      VAL& operator= (X&& newVal)
      {
          return get() = std::forward<X> (newVal);
      }
    };
  
  
  
  
  
  /******************************************************************************************//**
   * Table with data values, stored persistently as CSV file.
   * Each row within the table represents a data record, holding a sequence
   * of values. Values are statically typed per column, i.e. one column may hold
   * strings, while the next column holds doubles. For actual usage it is thus necessary
   * to define the column layout, through a sequence of [column Descriptors](\ref util::Column).
   * 
   * # Usage
   * Actually those Column objects serve as descriptors, but also as accessors — and they hold
   * the actual data storage for each column, which is a `std::vector<VAL>` of value type `VAL`.
   * There is always a _current record_ — corresponding to the actual data value and the newest
   * data row. For persistent storage, the sequence of rows is _reversed,_ so the newest data
   * appears at the top of the CSV file.
   * @tparam TAB a struct comprised of several Column objects, which hold the data and
   *         provide access to values of this specific column. Moreover, this type _must define_
   *         a function `allColumns()` to return a tuple with references to these column fields;
   *         the order of fields within this tuple also defines the order of columns
   *         within the table and persistent CSV storage.
   */
  template<class TAB>
  class DataFile
      : public TAB
      , util::NonCopyable
    {
      fs::path filename_;
      
    public:
      DataFile(fs::path csvFile ="")
        : filename_{fs::consolidated (csvFile)}
        {
          loadData();
        }
      
      
      /* === Data Access === */
      
      static constexpr size_t columnCnt = std::tuple_size_v<decltype(std::declval<TAB>().allColumns())>;
      
      bool
      empty()  const
        {
          return 0 == this->size();
        }
      
      size_t
      size()  const
        {
          if (0 == columnCnt) return 0;
          size_t rowCnt = std::numeric_limits<size_t>::max();
          forEach (unConst(this)->allColumns()
                  ,[&](auto& col)
                    {
                      rowCnt = min (rowCnt, col.data.size());
                    }); // the smallest number of data points found in any column
          return rowCnt;
        }
      
      string
      dumpCSV()  const
        {
          string csv;
          for (uint i=0; i < size(); ++i)
            csv += formatCSVRow(i) + '\n';
          return csv;
        }
      
      
      
      /* === Manipulation === */
      
      void
      newRow()
        {
            forEach (TAB::allColumns()
                    ,[](auto& col)
                      {
                        col.data.resize (col.data.size()+1);
                      });
        }
      
      void
      dupRow()
        {
          if (empty())
            newRow();
          else
            forEach (TAB::allColumns()
                    ,[](auto& col)
                      {
                        col.data.emplace_back (col.data.back());
                      });
        }
      
      void
      dropLastRow()
        {
          if (not empty())
            forEach (TAB::allColumns()
                    ,[](auto& col)
                      {
                        size_t siz = col.data.size();
                        col.data.resize (siz>0? siz-1 : 0);
                      });
        }
      
      void
      reserve (size_t expectedCapacity)
        {
            forEach (TAB::allColumns()
                    ,[=](auto& col)
                      {
                        col.data.reserve(expectedCapacity);
                      });
        }
      
      
      /** @param lineLimit number of rows to retain, back from the newest */
      void
      save (size_t lineLimit =std::numeric_limits<size_t>::max()
           ,bool backupOld =false)
        {
          if (filename_.empty())
            throw error::Logic{"Unable to save DataFile without filename given."};
          
          fs::path newFilename{filename_};
          newFilename += ".tmp";
          
          std::ofstream csvFile{newFilename, std::ios_base::out | std::ios_base::trunc};
          if (not csvFile.good())
            throw error::State{_Fmt{"Unable to create CSV output file %s"}
                                   % newFilename};
          saveData (csvFile, lineLimit);
          
          if (backupOld)
            {
              fs::path oldFile{filename_};
              oldFile += ".bak";
              if (fs::exists (filename_))
                  fs::rename (filename_, oldFile);
            }
          fs::rename (newFilename, filename_);
          filename_ = fs::consolidated(filename_);
        }                // lock onto absolute path
      
      
      void
      saveAs (fs::path newStorage
             ,size_t lineLimit =std::numeric_limits<size_t>::max())
        {
          newStorage = fs::consolidated (newStorage);
          if (fs::exists(newStorage))
            throw error::Invalid{_Fmt{"Storing DataFile rejected: target %s exists already"}
                                     % newStorage};
          if (not (newStorage.parent_path().empty()
                   or fs::exists(newStorage.parent_path())))
            throw error::Invalid{_Fmt{"DataFile(%s) placed into nonexistent directory %s"}
                                     % newStorage.filename() % newStorage.parent_path()};
          filename_ = newStorage;
          save (lineLimit);
        }
      
      
    private: /* === Implementation === */
      
      void
      loadData()
        {
          if (not (filename_.parent_path().empty()
                   or fs::exists(filename_.parent_path())))
            throw error::Invalid{_Fmt{"DataFile(%s) placed into nonexistent directory %s"}
                                     % filename_.filename() % filename_.parent_path()};
          if (not fs::exists(filename_))
            return; // leave the table empty
          
          std::ifstream csvFile{filename_};
          if (not csvFile.good())
            throw error::Config{_Fmt{"unable to read CSV data file %s"} % filename_};
          
          std::deque<string> rawLines;
          for (string line; std::getline(csvFile, line); )
            rawLines.emplace_back (move(line));
          
          if (rawLines.size() < 1) return;
          verifyHeaderSpec (rawLines[0]);
          
          // we know the number of rows now...
          reserve (rawLines.size() - 1);
          
          // storage in file is backwards, with newest data on top
          for (size_t row = rawLines.size()-1; 0<row; --row)
            if (not isnil(rawLines[row]))
              appendRowFromCSV (rawLines[row]);
        }
      
      
      void
      saveData (std::ofstream& csvFile, size_t lineLimit)
        {
          csvFile << generateHeaderSpec() << "\n";
          if (empty())
            return;
          lineLimit = size() > lineLimit? size()-lineLimit : 0;
          // store newest data first, possibly discard old data
          for (size_t row = size(); lineLimit < row; --row)
            csvFile << formatCSVRow(row-1) << "\n";
        }
      
      
      void
      verifyHeaderSpec (string headerLine)
        {
          CsvLine header{headerLine};
          forEach (TAB::allColumns()
                  ,[&](auto& col)
                    {
                      if (*header != col.header)
                        throw error::Invalid{_Fmt{"Header mismatch in CSV file %s. "
                                                  "Expecting column(%s) but found \"%s\""}
                                                 % filename_ % col.header % *header};
                      ++header;
                    });
        }
      
      string
      generateHeaderSpec()
        {
          string csv;
          forEach (TAB::allColumns()
                  ,[&](auto& col)
                    {
                      appendCsvField (csv, col.header);
                    });
          return csv;
        }
      
      
      void
      appendRowFromCSV (string line)
        {
          newRow();
          CsvLine csv(line);
          forEach (TAB::allColumns()
                  ,[&](auto& col)
                    {
                      if (not csv)
                        {
                          if (csv.isParseFail())
                            csv.fail();
                          else
                            throw error::Invalid{_Fmt{"Insufficient data; only %d fields, %d expected. Line:%s"}
                                                     % csv.getParsedFieldCnt() % columnCnt % line};
                        }
                      
                      using Value = typename std::remove_reference<decltype(col)>::type::ValueType;
                      col.get() = parseAs<Value>(*csv);
                      ++csv;
                    });
          if (csv)
            throw error::Invalid{_Fmt{"Excess data fields in CSV. Expect %d fields. Line:%s"}
                                     % columnCnt % line};
        }
      
      
      string
      formatCSVRow (size_t rownum)  const
        {
          if (this->empty())
            throw error::Logic{"Attempt to access data from empty DataTable."};
          if (rownum >= this->size())
            throw error::Logic{_Fmt{"Attempt to access row #%d beyond range [0..%d]."}
                                   % rownum % (size()-1)};
          
          string csvLine;
          forEach (unConst(this)->allColumns()
                  ,[&](auto& col)
                    {
                      appendCsvField (csvLine, col.data.at(rownum));
                    });
          return csvLine;
        }
    };
  
}} // namespace lib::stat
#endif /*LIB_STAT_DATA_H*/