Commit bdf6ed30 authored by Martin Marinov's avatar Martin Marinov
Browse files

ReForm, DOcloud: Added DOCloudConfig, and some minor fixes.

[git-p4: depot-paths = "//ReForm/ReForm/main/CoMISo/": change = 12200]
parent c6a14a95
......@@ -6,6 +6,7 @@ SET(my_headers
${CMAKE_CURRENT_SOURCE_DIR}/CPLEXSolver.hh
${CMAKE_CURRENT_SOURCE_DIR}/cURLpp.hh
${CMAKE_CURRENT_SOURCE_DIR}/DOCloudCache.hh
${CMAKE_CURRENT_SOURCE_DIR}/DOCloudConfig.hh
${CMAKE_CURRENT_SOURCE_DIR}/DOCloudJob.hh
${CMAKE_CURRENT_SOURCE_DIR}/DOCloudSolver.hh
${CMAKE_CURRENT_SOURCE_DIR}/GurobiHelper.hh
......
......@@ -6,6 +6,8 @@
//=============================================================================
#include "DOCloudCache.hh"
#include "DOCloudConfig.hh"
#include <Base/Utils/OutcomeUtils.hh>
#include <Base/Debug/DebOut.hh>
......@@ -13,6 +15,7 @@
#include <iomanip>
#include <cctype>
#include <functional>
#include <sstream>
#include <boost/filesystem.hpp>
......@@ -27,31 +30,6 @@ namespace DOcloud {
namespace {
const std::string& cache_directory()
{
static std::string cache_dir =
"\\\\camfs1\\General_access\\Martin_Marinov\\ReForm\\Cache\\";
static bool already_read = false;
if (!already_read)
{
already_read = true;
const char* env_cache_dir = getenv("ReFormCacheDir");
if (env_cache_dir != nullptr && env_cache_dir[0] != 0)
{
cache_dir = env_cache_dir;
if (cache_dir.back() != '\\')
cache_dir += '\\'; // Eventually add '\' to the directory string.
}
}
return cache_dir;
}
//
std::string full_cache_filename(const std::string& _filename)
{
return cache_directory() + _filename;
}
// Create a new temporary exclusive file without extension that is used to
// prevent write or read operation on files with the same name and extension
// .lp or .dat. while the cache is being written. This is the only class that
......@@ -116,11 +94,12 @@ bool save_file(const std::string& _filename, const std::string& _file_cnts)
// Finds a key string from the file name. This string will be used as file name
// where to store the related cached data.
void string_to_key(const std::string& _str, std::string& _key)
std::string string_to_hash(const std::string& _str)
{
// 1. Writes the file length.
const std::hash<std::string> hash_fn;
_key = std::to_string(hash_fn(_str));
std::stringstream strm;
strm << std::hex << hash_fn(_str);
return strm.str();
}
const size_t NO_SOLUTION_CODE = UINT_MAX;
......@@ -173,25 +152,25 @@ bool save_data(const std::string& _filename,
} // namespace
Cache::Cache(const std::string& _mip_lp)
: mip_lp_(_mip_lp), hash_(string_to_hash(mip_lp_)), found_(false)
{
DEB_enter_func;
DEB_line(2, "Cache hash: " << hash_);
}
bool Cache::restore_result(
std::string& _lp_prbl_str, // string containing the .lp problem
std::vector<double>& _x, // result.
double& _obj_val) // objective function value.
bool Cache::restore_result(std::vector<double>& _x, double& _obj_val)
{
DEB_enter_func;
found_ = false;
lp_file_cnts_ = std::move(_lp_prbl_str);
if (cache_directory().empty())
const auto* cache_loc = Config::query().cache_location();
if (cache_loc == nullptr) // cache location not provided, disabale the cache
return false;
string_to_key(lp_file_cnts_, key_);
DEB_line(2, "Key cache name: " << key_);
key_ += '_';
for (size_t iter_nmbr = 0; iter_nmbr < 10; ++iter_nmbr)
{
last_filename_ = full_cache_filename(key_) + std::to_string(iter_nmbr);
filename_ = cache_loc + hash_ + '_' + std::to_string(iter_nmbr);
std::string dat_filename(last_filename_ + ".dat");
std::string dat_filename(filename_ + ".dat");
boost::system::error_code err_cod;
if (!boost::filesystem::exists(
boost::filesystem::path(dat_filename.c_str()), err_cod) ||
......@@ -203,16 +182,16 @@ bool Cache::restore_result(
break;
}
if (FileLock::active(last_filename_))
if (FileLock::active(filename_))
break;
std::string cache_cnts;
if (!load_file(last_filename_ + ".lp", cache_cnts))
if (!load_file(filename_ + ".lp", cache_cnts))
break;
if (cache_cnts == lp_file_cnts_)
if (cache_cnts == mip_lp_)
{
found_ = load_data(last_filename_ + ".dat", _x, _obj_val);
found_ = load_data(filename_ + ".dat", _x, _obj_val);
return found_;
}
}
......@@ -233,10 +212,10 @@ public:
if (success_)
return;
// Removes files eventually written if there has been any kind of failure.
for (const auto& f_name : used_files_)
for (const auto& filename : used_files_)
{
if (!f_name.empty())
std::remove(f_name.c_str());
if (!filename.empty())
std::remove(filename.c_str());
}
}
......@@ -269,11 +248,11 @@ void
Cache::store_result(const std::vector<double>& _x, const double& _obj_val)
{
DEB_enter_func;
THROW_OUTCOME_if(found_, TODO); /* Multiple store of Docloud cache value. */
if (!last_filename_.empty())
THROW_OUTCOME_if(found_, TODO); /* Multiple store of DOcloud cache value. */
if (!filename_.empty())
{
CacheSaver saver;
saver.save(last_filename_, _x, _obj_val, lp_file_cnts_);
saver.save(filename_, _x, _obj_val, mip_lp_);
}
}
......
......@@ -21,27 +21,27 @@ namespace DOcloud {
class Cache
{
public:
Cache() : found_(false) {}
// Manage the cache for an optimization problem stored in .lp format
Cache(const std::string& _mip_lp);
const std::string& hash() const { return hash_; }
bool restore_result(
std::string& _file_name, // .lp file defining the optimization problem
std::vector<double>& _x, // result.
double& _obj_val); // objective function value.
double& _obj_val // objective function value.
);
// We can store the result for the given .lp file. This makes sense only we have
// not found cache data for the given .lp file, and in order to avoid data
// corruption this function fails if the data have been found.
void store_result(const std::vector<double>& _x, const double& _obj_val);
const std::string& get_lp_content() { return lp_file_cnts_; }
private:
std::string key_; // String generated from .lp file content data.
// This is a sort of hash key generate form the .lp
// file content.
std::string last_filename_;// Last name we have tried to get cached data.
std::string lp_file_cnts_; // Content of the input .lp file.
bool found_; // Remembers if we have found a cache for the input
// .lp file.
const std::string& mip_lp_; // The MIP represented in the .lp format
const std::string hash_; // hask for the lp_ problem
bool found_; // Remembers if we have found a cache for the input problem
std::string filename_; // File name to access the cached data (no extension)
};
} // namespace DOcloud
......
//=============================================================================
//
// CLASS DOCloudSolver
//
//=============================================================================
#ifndef COMISO_DOCLOUDCONFIG_HH
#define COMISO_DOCLOUDCONFIG_HH
//== COMPILE-TIME PACKAGE REQUIREMENTS ========================================
#include <CoMISo/Config/config.hh>
#if COMISO_DOCLOUD_AVAILABLE
#include <string>
//== NAMESPACES ===============================================================
namespace COMISO {
namespace DOcloud {
//== CLASS DEFINITION =========================================================
/**
Configuration options for the IBM Decision Optimization Cloud.
*/
class Config
{
public:
// access the configuration singleton
static const Config& query();
static Config& modify();
void set_root_url(const char* const _root_url);
const char* root_url() const { return root_url_.data(); }
void set_api_key(const char* _api_key);
const char* api_key() const { return api_key_.data(); }
void set_infeasible_timeout(const int _infs_time);
int infeasible_timeout() const { return infs_time_; }
void set_feasible_timeout(const int _fsbl_time);
int feasible_timeout() const { return fsbl_time_; }
void set_cache_location(const char* const _cache_loc);
const char* cache_location() const { return cache_loc_.data(); }
private:
std::string root_url_;
std::string api_key_;
int infs_time_;
int fsbl_time_;
std::string cache_loc_;
private:
Config();
static Config& object();
};
//=============================================================================
} // namespace DOcloud
} // namespace COMISO
//=============================================================================
#endif // COMISO_DOCLOUD_AVAILABLE
//=============================================================================
#endif // COMISO_DOCLOUDCONFIG_HH
//=============================================================================
......@@ -7,6 +7,7 @@
#include "DOcloudJob.hh"
#if COMISO_DOCLOUD_AVAILABLE
#include "DOCloudConfig.hh"
#include <Base/Utils/OutcomeUtils.hh>
#include <Base/Debug/DebUtils.hh>
......@@ -21,11 +22,36 @@ DEB_module("DOcloud")
namespace COMISO {
namespace DOcloud {
static char* root_url__ =
"https://api-oaas.docloud.ibmcloud.com/job_manager/rest/v1/jobs";
static std::string api_key__ =
"X-IBM-Client-Id: api_0821c92f-0f2b-4ea5-be24-ecc9cd7695dd";
static char* app_type__ = "Content-Type: application/json";
//////////////////////////////////////////////////////////////////////////
// Config
static const char* app_type__ = "Content-Type: application/json";
Config::Config()
: root_url_("https://api-oaas.docloud.ibmcloud.com/job_manager/rest/v1/jobs"),
api_key_("X-IBM-Client-Id: api_0821c92f-0f2b-4ea5-be24-ecc9cd7695dd"),
infs_time_(300), fsbl_time_(15),
cache_loc_("\\\\camfs1\\General_access\\Martin_Marinov\\ReForm\\Cache\\")
{
const char* env_cache_dir = getenv("ReFormCacheDir");
if (env_cache_dir != nullptr && env_cache_dir[0] != 0)
{
cache_loc_ = env_cache_dir;
if (cache_loc_.back() != '\\')
cache_loc_ += '\\'; // Eventually add '\' to the directory string.
}
}
Config& Config::object()
{
// TODO: implement MT-lock
static Config config;
return config;
}
//////////////////////////////////////////////////////////////////////////
// Config
const Config& Config::query() { return object(); }
Config& Config::modify() { return object(); }
class HeaderTokens
{
......@@ -59,8 +85,8 @@ public:
typedef std::vector<std::string>::const_iterator const_iterator;
const_iterator begin() const { return tkns_.begin();}
const_iterator end() const { return tkns_.end();}
const_iterator begin() const { return tkns_.begin(); }
const_iterator end() const { return tkns_.end(); }
private:
std::vector<std::string> tkns_;
......@@ -185,7 +211,7 @@ Job::~Job()
}
del.set_url(url_.data());
del.add_http_header(api_key__.c_str());
del.add_http_header(Config::query().api_key());
del.perform();
delete stts_;
......@@ -201,8 +227,8 @@ void Job::make()
cURLpp::Post post(post_loc);
THROW_OUTCOME_if(!post.valid(), TODO); //Failed to initialize the request
post.set_url(root_url__);
post.add_http_header(api_key__.c_str());
post.set_url(Config::query().root_url());
post.add_http_header(Config::query().api_key());
post.add_http_header(app_type__);
post.perform();
......@@ -216,21 +242,21 @@ void Job::make()
stts_ = new JsonTokens;
}
void Job::upload(cURLpp::Upload& _upload)
void Job::upload(cURLpp::Upload& _upld)
{
THROW_OUTCOME_if(!_upload.valid(), TODO); //Failed to initialize the request
THROW_OUTCOME_if(!_upld.valid(), TODO); //Failed to initialize the request
auto url = url_ + "/attachments/" + filename_ + "/blob";
_upload.set_url(url.data());
_upload.add_http_header(api_key__.c_str());
_upload.perform();
HttpStatus http_stat(_upload);
_upld.set_url(url.data());
_upld.add_http_header(Config::query().api_key());
_upld.perform();
HttpStatus http_stat(_upld);
http_stat.check(204);
}
void Job::upload()
{
if (file_buf_ == nullptr)
if (file_buf_.empty())
{// file is not buffered into memory
cURLpp::UploadFile upld(filename_);
upload(upld);
......@@ -249,7 +275,7 @@ void Job::start()
auto url = url_ + "/execute";
post.set_url(url.data());
post.add_http_header(api_key__.c_str());
post.add_http_header(Config::query().api_key());
post.add_http_header(app_type__);
post.perform();
HttpStatus http_stat(post);
......@@ -266,7 +292,7 @@ void Job::sync_status()
cURLpp::Get get;
THROW_OUTCOME_if(!get.valid(), TODO); //Failed to initialize the request
get.set_url(url_.data());
get.add_http_header(api_key__.c_str());
get.add_http_header(Config::query().api_key());
get.perform();
HttpStatus http_stat(get);
http_stat.check(200);
......@@ -305,7 +331,7 @@ void Job::sync_log()
const std::string url = url_ + "/log/items?start=" +
std::to_string(log_seq_idx_) + "&continuous=true";
get.set_url(url.data());
get.add_http_header(api_key__.c_str());
get.add_http_header(Config::query().api_key());
get.perform();
HttpStatus http_stat(get);
http_stat.check(200);
......@@ -401,7 +427,7 @@ void Job::abort()
THROW_OUTCOME_if(!del.valid(), TODO); //Failed to initialize the request
const std::string url = url_ + "/execute";
del.set_url(url.data());
del.add_http_header(api_key__.c_str());
del.add_http_header(Config::query().api_key());
del.perform();
HttpStatus http_stat(del);
......@@ -437,7 +463,7 @@ double Job::solution(std::vector<double>& _x) const
auto url = url_ + "/attachments/solution.json/blob";
get.set_url(url.data());
get.add_http_header(api_key__.c_str());
get.add_http_header(Config::query().api_key());
get.perform();
HttpStatus http_stat(get);
......
......@@ -25,7 +25,7 @@ class JsonTokens;
class Job : public cURLpp::Session
{
public:
Job(const char* _filename, const char* _file_buf = nullptr)
Job(const std::string& _filename, const std::string& _file_buf)
: filename_(_filename), file_buf_(_file_buf), stts_(nullptr)
{}
~Job();
......@@ -54,12 +54,12 @@ public:
protected:
void make();
void start();
void upload(cURLpp::Upload& _upload);
void upload(cURLpp::Upload& _upld);
void upload();
private:
const char* filename_;
const char* file_buf_;
const std::string filename_;
const std::string file_buf_;
std::string url_;
JsonTokens* stts_;
// these variables are initialized in start()
......
......@@ -278,21 +278,22 @@ void DOCloudSolver::solve(
"DOCloudSolver received a problem with non-constant gradient!");
std::vector<double> x(_problem->n_unknowns(), 0.0); // solution
std::string lp_prbl_str =
DOcloud::create_lp_string(_problem, _constraints, _discrete_constraints, x);
const std::string mip_lp = DOcloud::create_lp_string(_problem, _constraints,
_discrete_constraints, x);
double obj_val;
DOcloud::Cache cache;
if (cache.restore_result(lp_prbl_str, x, obj_val))
DOcloud::Cache cache(mip_lp);
if (cache.restore_result(x, obj_val))
DEB_line(3, "MIP cached.")
else
{
DEB_line(1, "MIP not cached, computing optimization.");
DOcloud::Job job("DoClouProblem.lp", cache.get_lp_content().data());
const std::string lp_hash = cache.hash() + ".lp";
DOcloud::Job job(lp_hash, mip_lp);
job.setup();
job.wait();
obj_val = job.solution(x);
//cache.store_result(x, obj_val);
cache.store_result(x, obj_val);
}
THROW_OUTCOME_if(x.empty(), MIPS_NO_SOLUTION);
......
......@@ -106,7 +106,7 @@ private:
class UploadData : public Upload
{
public:
UploadData(const std::string& _buf) : buf_(_buf) {}
explicit UploadData(const std::string& _buf) : buf_(_buf) {}
protected:
virtual size_t send_data();
......@@ -115,7 +115,7 @@ private:
class Buffer
{
public:
Buffer(const std::string& _data)
explicit Buffer(const std::string& _data)
: ptr_(_data.c_str()), len_(_data.size()), pos_(0)
{}
......@@ -123,13 +123,13 @@ private:
const size_t _n_elem, void* _from_buf);
size_t length() const { return len_; }
private:
const char* ptr_;
size_t len_;
size_t pos_;
};
class Buffer;
Buffer buf_;
};
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment