Commit a1d43491 authored by Marco Amagliani's avatar Marco Amagliani

merge in main branch of a consistent number of changes in the test system from dev/amaglim.

1. improvements in ERROR and warning checksum, with an apprpriate description of the error and of the differences.
2. New layout for difference report, that provides an faster way to see differences between testsrun.
3. Improved the internal test architecturesuing a new virutal interface for all kind of streams.
4. Use of string to pass data to parse insde the test sytem (instead of streams).
5. Improved documentation.

This takes all changes submit in dev/amaglim between CL14056 and CL14114.

[git-p4: depot-paths = "//ReForm/ReForm/main/Base/": change = 14114]
parent 42db4263
......@@ -35,6 +35,6 @@ IOutputStream& operator<<(IOutputStream& _os, const CodeLink& _lnk);
}//namespace Base
#define BASE_CODELINK Base::CodeLink(__FUNCTION__, __FILE__, __LINE__)
#define BASE_CODELINK ::Base::CodeLink(__FUNCTION__, __FILE__, __LINE__)
#endif//BASE_CODELINK_HH_INCLUDED
......@@ -32,7 +32,8 @@
#else // DEB_ON
#include "Base/Utils/IOutputStream.hh"
#include "Base/Code/CodeLink.hh"
#include "Base/Utils/OStringStream.hh"
#include <string>
#include <vector>
......@@ -70,10 +71,6 @@ public:
bool pass(const int _lvl) const { return _lvl <= lvl_; }
Stream& stream();
Base::Command end() const { return Base::Command::END; }
Base::Command end_lf() const { return Base::Command::END_LF; }
Base::Command end_err() const { return Base::Command::END_ERR; }
};
//! This is a private implementation for Stream
......@@ -98,9 +95,6 @@ public:
);
~Stream();
const std::string& string() const;
const char* c_str() const { return string().c_str(); }
Base::IOutputStream& print(const int);
Base::IOutputStream& print(const size_t);
Base::IOutputStream& print(const double);
......@@ -122,10 +116,8 @@ private:
friend class Controller;
};
extern void warning(const std::string& _wrng, const char* const _fnct,
const char* const _file, const int _line);
extern void error(const std::string& _err, const char* const _fnct,
const char* const _file, const int _line);
extern void warning(const std::string& _wrng, const ::Base::CodeLink& _lnk);
extern void error(const std::string& _err, const ::Base::CodeLink& _lnk);
}//namespace Debug
......@@ -134,7 +126,7 @@ extern void error(const std::string& _err, const char* const _fnct,
//TODO: This should use an atomic thread-safe static int(s)
#define DEB_enter_func static int deb_nmbr = 0; \
static int deb_lvl = Debug::INVALID_LEVEL; \
Debug::Enter deb(__FILE__, __FUNCTION__, deb_nmbr, deb_lvl);
::Debug::Enter deb(__FILE__, __FUNCTION__, deb_nmbr, deb_lvl);
#define DEB_only(CC) CC
......@@ -143,19 +135,19 @@ extern void error(const std::string& _err, const char* const _fnct,
#define DEB_out(LL, AA) DEB_out_if(true, LL, AA)
#define DEB_out_if(CC, LL, AA) { if (deb.pass(LL) && (CC)) \
{ deb.stream() << AA << deb.end(); } }
{ deb.stream() << AA << ::Base::Command::END; } }
#define DEB_line(LL, AA) DEB_line_if(true, LL, AA)
#define DEB_line_if(CC, LL, AA) { if (deb.pass(LL) && (CC)) \
{ deb.stream() << AA << deb.end_lf(); } }
{ deb.stream() << AA << ::Base::LF; } }
#define DEB_warning(LL, AA) DEB_warning_if(true, LL, AA)
#define DEB_warning_if(CC, LL, AA) { if (deb.pass(LL) && (CC)) \
{ Debug::Stream strm; strm << AA; \
Debug::warning(strm.string(), __FUNCTION__, __FILE__, __LINE__); } }
{ ::Base::OStringStream strm; strm << AA; \
::Debug::warning(strm.str, BASE_CODELINK); } }
#define DEB_error(AA) { Debug::Stream strm; strm << AA; \
Debug::error(strm.string(), __FUNCTION__, __FILE__, __LINE__); }
#define DEB_error(AA) { ::Base::OStringStream strm; strm << AA; \
::Debug::error(strm.str, BASE_CODELINK); }
#define DEB_error_if(CC, AA) { if (CC) DEB_error(AA); }
// Stream does not fulfill ostream. If you want to exploit an existing
......
......@@ -13,7 +13,6 @@
#include "Base/Code/CodeLink.hh"
#include "Base/Utils/ThrowError.hh"
#include "Base/Utils/Environment.hh"
#include "Base/Test/IChecksum.hh"
#include "Base/Test/ChecksumDebugEvent.hh"
#ifdef DEB_ON
......@@ -809,27 +808,18 @@ Stream& global_stream()
}//namespace
// put test checksum tag and separators in the required format
void warning(const std::string& _wrng, const char* const _fnct,
const char* const _file, const int _line)
void warning(const std::string& _wrng, const Base::CodeLink& _lnk)
{
Base::CodeLink code_link(_fnct, _file, _line);
TEST_only(Test::Checksum::Debug::warning.record(_wrng, code_link));
global_stream() << WARNING << ": " << _wrng << code_link << Base::Command::END_LF;
TEST_only(Test::Checksum::Debug::warning.record(_wrng, _lnk));
global_stream() << WARNING << ": " << _wrng << _lnk << Base::LF;
}
void error(const std::string& _err, const char* const _fnct,
const char* const _file, const int _line)
void error(const std::string& _err, const Base::CodeLink& _lnk)
{
Base::CodeLink code_link(_fnct, _file, _line);
TEST_only(Test::Checksum::Debug::error.record(_err, code_link));
global_stream() << ERROR << ": " << _err << code_link << Base::Command::END_ERR;
TEST_only(Test::Checksum::Debug::error.record(_err, _lnk));
global_stream() << ERROR << ": " << _err << _lnk << Base::Command::END_ERR;
}
#undef TRIGGER_POINT
//////////////////////////////////////////////////////////////////////////
Enter::Enter(const char* const _flnm, const char* const _fnct,
......@@ -993,8 +983,6 @@ Stream::~Stream()
}
}
const std::string& Stream::string() const { return dfile()->string(); }
Base::IOutputStream& Stream::print(const int _i)
{
dfile_->print(_i);
......
set(my_headers
${CMAKE_CURRENT_SOURCE_DIR}/Checksum.hh
${CMAKE_CURRENT_SOURCE_DIR}/ChecksumDebugEvent.hh
${CMAKE_CURRENT_SOURCE_DIR}/ChecksumFile.hh
${CMAKE_CURRENT_SOURCE_DIR}/ChecksumLogValueT.hh
${CMAKE_CURRENT_SOURCE_DIR}/IChecksum.hh
${CMAKE_CURRENT_SOURCE_DIR}/ChecksumNumberT.hh
${CMAKE_CURRENT_SOURCE_DIR}/types.hh
${CMAKE_CURRENT_SOURCE_DIR}/TestResult.hh
${CMAKE_CURRENT_SOURCE_DIR}/ReadValuesT.hh
PARENT_SCOPE
)
SET(my_t_impls
${CMAKE_CURRENT_SOURCE_DIR}/ChecksumLogValueT.cc
PARENT_SCOPE
)
set(my_sources
${CMAKE_CURRENT_SOURCE_DIR}/Checksum.cc
${CMAKE_CURRENT_SOURCE_DIR}/ChecksumDebugEvent.cc
${CMAKE_CURRENT_SOURCE_DIR}/ChecksumFile.cc
${CMAKE_CURRENT_SOURCE_DIR}/ChecksumLogValue.cc
${CMAKE_CURRENT_SOURCE_DIR}/IChecksum.cc
PARENT_SCOPE
)
#ifdef TEST_ON
#include "Checksum.hh"
#include <fstream>
#include <iostream>
namespace Test {
namespace Checksum {
namespace {
Registry& registry_modify()
{
static Registry chksm_reg;
return chksm_reg;
}
}//namespace
const Registry& registry()
{
return registry_modify();
}
///////////////////////////////////////////////////////////////////////////////
// class Checksum implementation
Object::Object(const char* _name)
: name_(_name)
{
auto pos = registry_modify().emplace(name_, this);
if (!pos.second)
{
std::cout << "Duplicate checksum definition: " << _name << std::endl;
throw;
}
}
Difference Object::compare_data(const String& _old, const String& _new) const
{
if (_old == _new)
return Difference::EQUAL;
String diff;
diff.resize((std::max(_old.length(), _new.length())));
for (int i = 0; i < diff.size(); ++i)
{
diff[i] = i < _old.length() && i < _new.length() && _old[i] == _new[i] ?
' ' : '*';
}
return Difference(Difference::UNKNOWN, diff);
}
Difference Object::compare(
const Path& /*_old_path*/,
const Record& _old_rcrd,
const Path& /*_new_path*/,
const Record& _new_rcrd
) const
{
const auto old_rslt_type = _old_rcrd.rslt.type();
const auto new_rslt_type = _new_rcrd.rslt.type();
auto data_diff = compare_data(_old_rcrd.data, _new_rcrd.data);
if (old_rslt_type == new_rslt_type) // same result, so just data compare
return data_diff;
// result types are different, qualify the difference
#define DIFFERENCE(TYPE) data_diff += Difference(Difference::TYPE)
switch (old_rslt_type)
{
case Result::OK :
{
return new_rslt_type == Result::WARNING ?
DIFFERENCE(SUSPICIOUS) : DIFFERENCE(REGRESSED);
}
case Result::WARNING :
{
return new_rslt_type == Result::OK ?
DIFFERENCE(IMPROVED) : DIFFERENCE(REGRESSED);
}
default : // ERROR, CRASH, HANG, etc
{
return new_rslt_type == Result::OK || new_rslt_type == Result::WARNING ?
DIFFERENCE(IMPROVED) : DIFFERENCE(UNKNOWN);
}
}
}
void Object::add(const Result& _rslt, const String& _data)
{
static Base::OutputStreamAdaptT<std::ofstream> test_str(REPORT_FILENAME);
test_str << _rslt << " " << name() << ": " << _data << Base::LF;
}
}//Checksum
}//namespace Test
#endif//TEST_ON
// (C) Copyright 2016 by Autodesk, Inc.
//
// The information contained herein is confidential, proprietary
// to Autodesk, Inc., and considered a trade secret as defined
// in section 499C of the penal code of the State of California.
// Use of this information by anyone other than authorized
// employees of Autodesk, Inc. is granted only under a written
// non-disclosure agreement, expressly prescribing the scope
// and manner of such use.
#ifndef BASE_ICHECKSUM_HH_INCLUDE
#define BASE_ICHECKSUM_HH_INCLUDE
#ifndef TEST_ON
#define TEST_only(CC)
#else
#include <Base/Test/TestResult.hh>
#include <Base/Utils/OStringStream.hh>
#include <map>
#include <sstream>
namespace Test {
namespace Checksum {
//! typedef String, this is used a lot in this namespace
typedef std::string String;
//! The checksum record
struct Record
{
Record() {}
Record(const Result& _rslt, const String _data) : rslt(_rslt), data(_data) {}
Result rslt; //! record result
String data; //!< recorded "data" (as string, can be parsed)
};
//! The difference found by the IChecksum::compare() operation
class Difference
{
public:
enum Type
{
EQUAL,
UNKNOWN,
IMPROVED,
NEGLEGIBLE,
SUSPICIOUS,
REGRESSED
};
static const char* const type_text(const Type _type)
{
static const char dscr[][32] =
{
"EQUAL",
"UNKNOWN",
"IMPROVED",
"NEGLEGIBLE",
"SUSPICIOUS",
"REGRESSED"
};
return dscr[_type];
}
Difference(const Type _type = EQUAL, const String& _dscr = String())
: type_(_type), dscr_(_dscr)
{}
const Type type() const { return type_; }
bool equal() const { return type() == EQUAL; }
bool operator==(const Difference& _othr) const
{
return type_ == _othr.type_ && dscr_ == _othr.dscr_;
}
Difference& operator+=(const Difference& _othr)
{
if (type_ < _othr.type_)
type_ = _othr.type_;
if (dscr_.empty())
dscr_ = _othr.dscr_;
else if (!_othr.dscr_.empty())
dscr_ += "; " + _othr.dscr_;
return *this;
}
Difference& operator+=(const Difference::Type& _type)
{
if (type_ < _type)
type_ = type_;
return *this;
}
const String& description() const
{
return dscr_;
}
const char* const type_text() const { return type_text(type_); }
friend Base::IOutputStream& operator<<(Base::IOutputStream& _os,
Difference& _diff)
{
// TODO: use string description array
return _os << _diff.type_text() << " " << _diff.dscr_;
}
private:
Type type_;
String dscr_;
};
/*!
Base class for test checksums. Whatever check we want to add in the test system,
it must be an instance of a class derived from Checksum. All derived classes
must be instantiated as global variables.
*/
class Object
{
public:
/*!
Performs an automatic registration of the new checksum in a
global list, and verifies that the name is unique.
*/
Object(const char* const _name);
//! Checksum name.
const char* const name() const { return name_; }
//! Add a record the checksum (generic version)
template <typename T>
void record(const Result& _rslt, const T& _data)
{
Base::OStringStream strm;
strm << _data;
add(_rslt, strm.str);
}
//! Add a record of the checksum (public version)
void record(const Result& _rslt, const String& _data) { add(_rslt, _data); }
/*!
Compare two existing records (old and new).
Returns a qualification and a description of the difference.
The default implementation has some intelligence in comparing the record
results, but compares the the data simply as strings (no parsing).
*/
virtual Difference compare(
const Path& _old_path, //!<[in] Path to the left record
const Record& _old_rcrd, //!<[in] "Left" record
const Path& _new_path, //!<[in] Path to the right record
const Record& _new_rcrd //!<[in] "Right" record
) const;
protected:
//! Add a record of the checksum (protected version)
void add(const Result& _rslt, const String& _data);
//! Compare the data, the default implementation does a string comparison
virtual Difference compare_data(const String& _old, const String& _new) const;
private:
const char* const name_;
private:
Object(const Object&);
Object* operator=(const Object&);
};
/*!
Definition of the checksums registry. It is a map from a string (that is the
checksum name to an IChecksum.
*/
typedef std::map<String, Object*> Registry;
/*!
Function to get a static map with all the registered checksums.
*/
const Registry& registry();
}//namespace Checksum
}//namespace Test
#define TEST_only(CC) CC
#endif//TEST_ON
#endif//BASE_ICHECKSUM_HH_INCLUDE
......@@ -22,52 +22,43 @@ namespace Checksum {
namespace Debug {
// put test checksum tag and separators in the required format
const char* const LINK_MARK = " reported ";
void Event::record(const std::string& _evnt, const Base::CodeLink& _lnk)
{
Result res(name() == ::Debug::ERROR ? Result::ERROR : Result::WARNING);
IChecksum::record(res) << "#" << nmbr_++ << ": " << _evnt << " reported "
<< _lnk << Base::LF;
Base::OStringStream strm;
strm << "#" << nmbr_++ << ": " << _evnt << LINK_MARK << _lnk;
add(name() == ::Debug::ERROR ? Result::ERROR : Result::WARNING,
strm.str);
}
void Event::record_number() const
void Event::record_number()
{
Result res(nmbr_ == 0 ? Result::OK : // the checksum will appear in all cases
(name() == ::Debug::ERROR ? Result::ERROR : Result::WARNING));
IChecksum::record(res) << " total#: " << nmbr_ << Base::LF;
Base::OStringStream strm;
strm << "total#: " << nmbr_;
add(
nmbr_ == 0 ? Result::OK :
(name() == ::Debug::ERROR ? Result::ERROR : Result::WARNING),
strm.str);
}
Severity Event::compare(
IStream& _str0,
IStream& _str1,
OStream& _os,
const Path* /*_dir0*/,
const Path* /*_dir1*/
) const
{
std::string line0, line1;
std::getline(_str0, (line0));
std::getline(_str1, (line1));
Difference Event::compare_data(const String& _old, const String& _new) const
{
// exclude the code link from the comparison
auto old_tmp = _old;
auto new_tmp = _new;
auto old_link_pos = old_tmp.rfind(LINK_MARK);
auto new_link_pos = new_tmp.rfind(LINK_MARK);
if (line0 == line1)
if (old_link_pos != String::npos && new_link_pos != String::npos)
{
_os << line0;
return 0;
old_tmp.resize(old_link_pos);
new_tmp.resize(new_link_pos);
}
std::string diff;
diff.resize((std::max(line0.length(), line1.length())));
for (int i = 0; i < diff.size(); ++i)
{
diff[i] = i < line0.length() && i < line1.length() && line0[i] == line1[i] ?
' ' : '*';
}
_os << Base::LF <<
"\tL: " << line0 << Base::LF <<
"\tR: " << line1 << Base::LF <<
"\tD: " << diff;
return 1;
}
return Object::compare_data(old_tmp, new_tmp);
}
Event error(::Debug::ERROR);
Event warning(::Debug::WARNING);
......
......@@ -12,7 +12,7 @@
#define BASE_CHECKSUMCOUNT_HH_INCLUDE
#include <Base/Debug/DebOut.hh>
#include <Base/Test/IChecksum.hh>
#include <Base/Test/Checksum.hh>
#if defined(TEST_ON) && defined(DEB_ON)
......@@ -27,22 +27,17 @@ namespace Test {
namespace Checksum {
namespace Debug {
class Event : public IChecksum
class Event : public Object
{
public:
Event(const char* _name) : IChecksum(_name), nmbr_(0) {}
Event(const char* _name) : Object(_name), nmbr_(0) {}
virtual void record(const std::string& _evnt, const Base::CodeLink& _lnk);
virtual void record_number();
virtual Severity compare(
IStream& _str0, //!<[in] First stream with checksum data
IStream& _str1, //!<[in] Second stream with checksum data
OStream& _os, //!<[out] Description of the difference.
const Path* _dir0 = nullptr, //!<[in] FIrst test directory
const Path* _dir1 = nullptr //!<[in] Second test directory
) const;
virtual void record_number() const;
protected:
//! Implement "smarter" comparison
virtual Difference compare_data(const String& _old, const String& _new) const;
protected:
int nmbr_;
......
......@@ -11,21 +11,16 @@
#ifdef TEST_ON
#include "ChecksumFile.hh"
#include "Base/Utils/OStringStream.hh"
#include <boost/functional/hash.hpp>
#include <boost/filesystem.hpp>
#include <fstream>
namespace Test {
namespace Checksum {
File::File(std::string _name)
: LogValueT<std::string>((_name += File::tag()).c_str()),
mod_chksm_name_(std::move(_name))
{}
void File::record(const char* _flnm)
{
// Reads the file, makes a has number form its data and finally record
......@@ -41,115 +36,13 @@ void File::record(const char* _flnm)
file_size += fstr.gcount();
boost::hash_combine(file_hash, buf_hash);
}
std::string log_val = std::string("\"") + _flnm + '\"' +
" (Size: " + std::to_string(file_size) + ")" +
" (Hash: " + std::to_string(file_hash) + ")";
LogValueT<std::string>::record(Result::OK, log_val);
}
namespace {
namespace fs = boost::filesystem;
/*! Compares two files. First compare their size. If the seize is the same finds
out if they are bitwise equal.
*/
struct CompareTwoFiles
{
CompareTwoFiles(const Path& _flnm_0, const Path& _flnm_1)
{
if (!fs::is_regular_file(_flnm_0) || !fs::is_regular_file(_flnm_1))
sizes_[0] = sizes_[1] = 0;
else
{
sizes_[0] = fs::file_size(_flnm_0);
sizes_[1] = fs::file_size(_flnm_1);
paths_[0] = _flnm_0;
paths_[1] = _flnm_1;
}
}
size_t size(const size_t _ind) const { return sizes_[_ind]; }
bool same_size() const { return sizes_[0] == sizes_[1]; }
double size_difference() const
{
return double(sizes_[1]) - double(sizes_[0]);
}
bool bitwise_equal() const
{
if (!valid() || !same_size())
return false;
std::array<char, 100> buff_a, buff_b;
std::ifstream str_a(paths_[0].string());
std::ifstream str_b(paths_[1].string());
bool bit_wise_eq = true;
while (bit_wise_eq)
{
bool eof_a = !str_a.read(buff_a.data(), buff_a.size());