ichthyo/LUMIERA.clone
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Library: complete coverage of CSV data table including storage

Browse source 
also encompasses some coverage for the simplistic CSV format
implemented as storage backend for this data table
This commit is contained in:
Fischlurch 2024-03-14 23:44:39 +01:00
parent 3b3600379a
commit 599407deea
6 changed files with 176 additions and 111 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
src/lib/stat/csv.hpp
View file

@ -104,6 +104,35 @@ namespace stat {
}
/** parse string representation into typed value */
template<typename TAR>
inline TAR
parseAs (string const& encodedVal)
{
std::istringstream converter{encodedVal};
TAR value;
converter >> value;
if (converter.fail())
throw error::Invalid{_Fmt{"unable to parse \"%s\""} % encodedVal};
return value;
}
template<>
inline bool
parseAs(string const& encodedBool)
{
return util::boolVal(encodedBool);
}
template<>
inline string
parseAs(string const& string)
{
return string; // pass-through (even if empty)
}
/**
* Parser to split one line of CSV data into fields.
* @remarks iterator-like throw-away object
@ -123,7 +152,7 @@ namespace stat {
size_t pos_;
public:
CsvLine (string const& line)
CsvLine (string& line) // NOTE: string must exist elsewhere
: MatchSeq(line, ACCEPT_FIELD)
, line_{line}
, field_{0}

30
src/lib/stat/data.hpp
View file

@ -22,7 +22,7 @@
/** @file data.hpp
** Manage a table with time series data, stored persistently as CSV.
** Manage a table with data records, stored persistently as CSV.
** In the context of observations, configuration, calibration and QA, a series
** of measurement data taken over time is often evaluated statistically, to distill
** typical averages, variances and trends. Short of using a database, a modest
@ -135,34 +135,6 @@ namespace stat{
/** parse string representation into typed value */
template<typename TAR>
inline TAR
parseAs (string const& encodedVal)
{
std::istringstream converter{encodedVal};
TAR value;
converter >> value;
if (converter.fail())
throw error::Invalid{_Fmt{"unable to parse \"%s\""} % encodedVal};
return value;
}
template<>
inline bool
parseAs(string const& encodedBool)
{
return util::boolVal(encodedBool);
}
template<>
inline string
parseAs(string const& string)
{
return string; // pass-through (even if empty)
}
/**

2
tests/15library.tests
View file

@ -180,7 +180,7 @@ return: 0
END
PLANNED "Filesystem manipulations" FileSupport_test <<END
TEST "Filesystem manipulations" FileSupport_test <<END
return: 0
END

2
tests/16calculation.tests
View file

@ -2,7 +2,7 @@ TESTING "Library Test Suite: calculation" ./test-suite --group=calculation
PLANNED "Data Table with CSV storage" DataCSV_test <<END
TEST "Data Table with CSV storage" DataCSV_test <<END
return: 0
END

185
tests/library/stat/data-csv-test.cpp
View file

@ -27,31 +27,23 @@
#include "lib/test/run.hpp"
#include "lib/test/test-helper.hpp"
#include "lib/test/temp-dir.hpp"
#include "lib/stat/data.hpp"
//#include "lib/time/timevalue.hpp"
//#include "lib/error.hpp"
//#include "lib/util-foreach.hpp"
#include "lib/time/timevalue.hpp"
#include "lib/format-cout.hpp"
#include "lib/util.hpp"
#include "lib/format-cout.hpp" ///////////////////////TODO
#include "lib/test/diagnostic-output.hpp" ///////////////////////TODO
//#include <functional>
#include <sstream>
#include <string>
#include <vector>
//using lumiera::Error;
//using lumiera::LUMIERA_ERROR_EXCEPTION;
//using lumiera::error::LUMIERA_ERROR_ASSERTION;
//using lib::time::TimeVar;
//using lib::time::Time;
//using boost::algorithm::is_lower;
//using boost::algorithm::is_digit;
using util::isnil;
//using std::function;
using lib::time::Time;
using lib::test::TempDir;
using std::make_tuple;
using std::string;
using std::vector;
using std::make_tuple;
namespace lib {
@ -60,14 +52,15 @@ namespace test{
namespace {//Setup for test
/** Define the layout of a data row */
struct TableForm
{
Column<string> id{"ID"};
Column<string> id{"ID"}; // ◁────── names given here must match first storage line
Column<double> val{"Value"};
Column<int> off{"Offset"};
auto allColumns()
auto allColumns() // ◁────────── mandatory function; defines actual sequence of columns
{ return std::tie(id
,val
,off
@ -80,7 +73,6 @@ namespace test{
}//(End)Test setup
using error::LUMIERA_ERROR_STATE;
using error::LUMIERA_ERROR_INVALID;
@ -232,63 +224,126 @@ namespace test{
void
verify_CSV_Format()
{
string line;
double val = 1.0 / 3;
CHECK (util::toString(val) == "0.33333333"_expect );
CHECK (util::showDecimal(val) == "0.333333333333333"_expect );
CHECK (util::showComplete(val) == "0.33333333333333331"_expect);
CHECK (boost::lexical_cast<string>(val) == "0.33333333333333331"_expect);
CHECK (format4Csv(double(1) / 3) == "0.333333333333333"_expect );
CHECK (format4Csv(float(1) / 3) == "0.333333"_expect );
CHECK (format4Csv(f128(1) / 3) == "0.333333333333333333"_expect);
CHECK (format4Csv(bool(1)) == "true"_expect );
CHECK (format4Csv(bool(0)) == "false"_expect);
CHECK (format4Csv("Starship-3") == "\"Starship-3\""_expect ); // 3rd test today ;-)
CHECK (format4Csv(Time(1,2,25,13)) == "\"13:25:02.001\""_expect);
string line;
int64_t ii = -100000;
bool boo = true;
appendCsvField (line, ii);
CHECK (line == "-100000"_expect);
SHOW_EXPR(val)
SHOW_EXPL(val)
SHOW_EXPR(std::to_string(val))
SHOW_EXPR(format4Csv(val));
SHOW_EXPR(util::showDecimal(val))
SHOW_EXPR(util::showComplete(val))
double vval = parseAs<double>(format4Csv(val));
SHOW_EXPR(vval)
SHOW_EXPL(vval)
SHOW_EXPR(std::to_string(vval))
SHOW_EXPR(format4Csv(vval));
SHOW_EXPR(util::showDecimal(val))
SHOW_EXPR(util::showComplete(val))
vval = parseAs<double>(boost::lexical_cast<string>(val));
SHOW_EXPR(vval)
SHOW_EXPL(vval)
SHOW_EXPR(std::to_string(vval))
SHOW_EXPR(format4Csv(vval));
SHOW_EXPR(util::showDecimal(val))
SHOW_EXPR(util::showComplete(val))
bool boo;
SHOW_EXPR(boo);
SHOW_EXPL(boo)
SHOW_EXPR(std::to_string(boo))
SHOW_EXPR(format4Csv(boo));
SHOW_EXPR(format4Csv(-42));
SHOW_EXPR(format4Csv(uint64_t(-42)));
auto moo = f128(1) / 3;
SHOW_EXPR(moo)
SHOW_EXPL(moo)
SHOW_EXPR(std::to_string(moo))
SHOW_EXPR(format4Csv(moo));
SHOW_EXPR(util::showDecimal(moo))
SHOW_EXPR(util::showComplete(moo))
auto oo = 1.0f / 3;
SHOW_EXPR(oo)
SHOW_EXPL(oo)
SHOW_EXPR(std::to_string(oo))
SHOW_EXPR(format4Csv(oo));
SHOW_EXPR(util::showDecimal(oo))
SHOW_EXPR(util::showComplete(oo))
SHOW_EXPR(format4Csv(lib::time::Time(1,2,3,4)));
appendCsvField (line, val);
CHECK (line == "-100000,0.333333333333333"_expect);
appendCsvField (line, boo);
CHECK (line == "-100000,0.333333333333333,true"_expect);
appendCsvField (line, "Raptor");
CHECK (line == "-100000,0.333333333333333,true,\"Raptor\""_expect);
CsvLine csvLine(line);
CHECK (csvLine.isValid());
CHECK (*csvLine == "-100000"_expect);
CHECK (-100000 == parseAs<int>(*csvLine));
++csvLine;
CHECK (csvLine.isValid());
CHECK (*csvLine == "0.333333333333333"_expect);
CHECK (0.333333343f == parseAs<float>(*csvLine));
++csvLine;
CHECK (csvLine.isValid());
CHECK (*csvLine == "true"_expect);
CHECK (true == parseAs<bool>(*csvLine));
++csvLine;
CHECK (csvLine.isValid());
CHECK (*csvLine == "Raptor"_expect);
CHECK ("Raptor" == parseAs<string>(*csvLine));
++csvLine;
CHECK (not csvLine.isValid());
line = " ◐0◑. ; \t \"' \" \n ,oh my ;";
CsvLine horror(line);
CHECK ("◐0◑." == *horror); // as far as our CSV format is concerned, this is valid
CHECK (0 == horror.getParsedFieldCnt());
++horror;
CHECK (1 == horror.getParsedFieldCnt());
CHECK ("' " == *horror);
++horror;
CHECK ("oh" == *horror);
CHECK (2 == horror.getParsedFieldCnt());
// next field is not quoted, but contains space
VERIFY_FAIL (",oh |↯|my ;", ++horror );
CHECK (not horror.isValid());
CHECK (horror.isParseFail());
}
/** @test verify a table backed by persistent CSV data
*/
/** @test verify a table backed by persistent CSV data */
void
verify_persistentDataFile()
{
UNIMPLEMENTED("Arrrröööh");
TempDir temp;
// prepare a data file to load into the table...
fs::path f = temp.makeFile("dataz.csv");
std::ofstream content{f};
content << R"("ID", "Value", "Offset")"<<endl
<< R"( "one" , 5.5 ; +1 )"<<endl
<< R"(;" 0 ";0)" <<endl; // ◁────── demonstrating some leeway in storage format
content.close();
TestTab dat{f};
CHECK (2 == dat.size());
CHECK ("ID" == dat.id.header);
CHECK ("Value" == dat.val.header);
CHECK ("Offset" == dat.off.header);
//Note: data is reversed in storage — last/newest line first
CHECK ("one" == string{dat.id});
CHECK ( 5.5 == dat.val);
CHECK ( 1 == dat.off);
CHECK (dat.id.data == vector<string>({"","one"}));
CHECK (dat.val.data == vector<double>({0 ,5.5 }));
CHECK (dat.off.data == vector<int> ({0 ,1 }));
// can modify some values....
dat.id = "mid";
dat.dupRow();
dat.id = "last";
dat.off *= -1;
// can dump the contents as CSV
CHECK (dat.dumpCSV() ==
R"("",0,0
"mid",5.5,1
"last",5.5,-1
)"_expect);
// save complete table in current state, overwriting on disk
dat.save();
// read back data rewritten on disk...
std::ifstream readback{f};
std::ostringstream inBuff;
inBuff << readback.rdbuf();
CHECK (inBuff.str() ==
R"("ID","Value","Offset"
"last",5.5,-1
"mid",5.5,1
"",0,0
)"_expect);
// note again the reversed order in storage: last line at top
}
};

37
wiki/thinkPad.ichthyo.mm
View file

@ -112013,8 +112013,8 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
<node COLOR="#435e98" CREATED="1710114593043" ID="ID_1270108404" MODIFIED="1710114618631" TEXT="git read-tree"/>
</node>
</node>
<node BACKGROUND_COLOR="#eef0c5" COLOR="#990000" CREATED="1710085479914" ID="ID_1739991591" MODIFIED="1710114641924" TEXT="minimale Integration in die Lumiera-Codebase">
<icon BUILTIN="pencil"/>
<node COLOR="#338800" CREATED="1710085479914" ID="ID_1739991591" MODIFIED="1710466419905" TEXT="minimale Integration in die Lumiera-Codebase">
<icon BUILTIN="button_ok"/>
<node COLOR="#338800" CREATED="1710085492857" ID="ID_4363413" MODIFIED="1710124911666" TEXT="Lumiera-GNU-Style">
<icon BUILTIN="button_ok"/>
</node>
@ -112024,9 +112024,9 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
<node COLOR="#338800" CREATED="1710085513597" ID="ID_1623287366" MODIFIED="1710124916786" TEXT="Fehler durch Lumiera-Errors ausdr&#xfc;cken">
<icon BUILTIN="button_ok"/>
</node>
<node BACKGROUND_COLOR="#eef0c5" COLOR="#990000" CREATED="1710085528964" ID="ID_197737968" MODIFIED="1710124923723" TEXT="Lumiera&apos;s Format-Funktionen verwenden">
<icon BUILTIN="pencil"/>
<node BACKGROUND_COLOR="#f0d5c5" COLOR="#990033" CREATED="1710114655354" ID="ID_685893002" MODIFIED="1710114679967" TEXT="zu kl&#xe4;ren: vollst&#xe4;ndige floatingpoint-Repr&#xe4;sentation?">
<node COLOR="#338800" CREATED="1710085528964" ID="ID_197737968" MODIFIED="1710460904274" TEXT="Lumiera&apos;s Format-Funktionen verwenden">
<icon BUILTIN="button_ok"/>
<node COLOR="#435e98" CREATED="1710114655354" ID="ID_685893002" MODIFIED="1710460902224" TEXT="zu kl&#xe4;ren: vollst&#xe4;ndige floatingpoint-Repr&#xe4;sentation?">
<icon BUILTIN="help"/>
<node CREATED="1710114695652" ID="ID_323709862" MODIFIED="1710114697384" TEXT="std::numeric_limits&lt;VAL&gt;::digits10"/>
<node CREATED="1710114699873" ID="ID_496274436" MODIFIED="1710114711248" TEXT="util::toString sollte lexical_cast verwenden...">
@ -112110,8 +112110,9 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
<font ITALIC="true" NAME="SansSerif" SIZE="14"/>
<icon BUILTIN="yes"/>
</node>
<node BACKGROUND_COLOR="#eef0c5" COLOR="#990000" CREATED="1710377772401" ID="ID_96383520" MODIFIED="1710432999927" TEXT="Funktion umschreiben und in das Format-Framework integrieren">
<icon BUILTIN="pencil"/>
</node>
<node COLOR="#338800" CREATED="1710377772401" ID="ID_96383520" MODIFIED="1710460889268" TEXT="Funktion umschreiben und in das Format-Framework integrieren">
<icon BUILTIN="button_ok"/>
<node COLOR="#338800" CREATED="1710433003432" ID="ID_1577076163" MODIFIED="1710433096279" TEXT="neue explizite Format-Funktionen">
<icon BUILTIN="button_ok"/>
<node COLOR="#435e98" CREATED="1710433012315" ID="ID_1406333897" MODIFIED="1710433101135" TEXT="showDecimal &#x27f6; digits10"/>
@ -112123,9 +112124,8 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
<node COLOR="#338800" CREATED="1710433074435" ID="ID_1236251004" MODIFIED="1710433095113" TEXT="format4csv : bleibt als generisches front-End">
<icon BUILTIN="button_ok"/>
</node>
<node BACKGROUND_COLOR="#fdfdcf" COLOR="#ff0000" CREATED="1710433088707" ID="ID_1543435810" MODIFIED="1710433093241" TEXT="Testabdeckung">
<icon BUILTIN="flag-pink"/>
</node>
<node COLOR="#338800" CREATED="1710433088707" ID="ID_1543435810" MODIFIED="1710460887865" TEXT="Testabdeckung">
<icon BUILTIN="button_ok"/>
</node>
</node>
</node>
@ -112165,8 +112165,8 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
<icon BUILTIN="yes"/>
</node>
</node>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1710085571918" ID="ID_1847964804" MODIFIED="1710085586443" TEXT="per Unit-Test dokumentieren">
<icon BUILTIN="flag-yellow"/>
<node BACKGROUND_COLOR="#eef0c5" COLOR="#990000" CREATED="1710085571918" ID="ID_1847964804" MODIFIED="1710466417063" TEXT="per Unit-Test dokumentieren">
<icon BUILTIN="pencil"/>
<node CREATED="1710167567738" ID="ID_116953186" MODIFIED="1710168515030" TEXT="neue Test-Gruppe: 16calculation.tests">
<node CREATED="1710167587540" ID="ID_1196727143" MODIFIED="1710167645967" TEXT="group=calculation"/>
<node CREATED="1710167598409" ID="ID_498080750" MODIFIED="1710167615767" TEXT="Mathematik, aber auch Farbr&#xe4;ume und Signalverarbeitung"/>
@ -112186,8 +112186,8 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
<node CREATED="1710170641389" ID="ID_39926847" MODIFIED="1710170645400" TEXT="auch bei Exception"/>
</node>
</node>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1710163662386" ID="ID_694159551" MODIFIED="1710170478456" TEXT="DataCSV_test">
<icon BUILTIN="flag-yellow"/>
<node COLOR="#338800" CREATED="1710163662386" ID="ID_694159551" MODIFIED="1710466414212" TEXT="DataCSV_test">
<icon BUILTIN="button_ok"/>
<node COLOR="#338800" CREATED="1710351225837" ID="ID_1756484657" MODIFIED="1710352395458" TEXT="simpleUsage">
<icon BUILTIN="button_ok"/>
<node COLOR="#435e98" CREATED="1710352396692" ID="ID_101637286" MODIFIED="1710352560707" TEXT="f&#xfc;hre hiermit Tabellen ohne File-Storage ein">
@ -112195,6 +112195,15 @@ Date:&#160;&#160;&#160;Thu Apr 20 18:53:17 2023 +0200<br/>
<icon BUILTIN="yes"/>
</node>
</node>
<node COLOR="#338800" CREATED="1710460918482" ID="ID_85656633" MODIFIED="1710460921103" TEXT="verify_rowHandling">
<icon BUILTIN="button_ok"/>
</node>
<node COLOR="#338800" CREATED="1710460928285" ID="ID_864668027" MODIFIED="1710460932163" TEXT="verify_CSV_Format">
<icon BUILTIN="button_ok"/>
</node>
<node COLOR="#338800" CREATED="1710460938455" ID="ID_757574171" MODIFIED="1710466412661" TEXT="verify_persistentDataFile">
<icon BUILTIN="button_ok"/>
</node>
</node>
<node BACKGROUND_COLOR="#eee5c3" COLOR="#990000" CREATED="1710163672834" ID="ID_778295326" MODIFIED="1710170479867" TEXT="Statistic_test">
<icon BUILTIN="flag-yellow"/>