#include "MarginServiceCore.h" #include "Position.h" #include "lib/config/config.h" #include "lib/api/services/AuthService.pb.h" #include "lib/utils/bts_assert.h" #include "lib/utils/Uuid.h" #include "lib/utils/Dispatcher.h" #include "lib/utils/bts_chrono.h" #include "lib/logging/logging.h" #include "lib/utils/helper_macros.h" #include "lib/utils/datetime.h" #include "lib/config/protos/span.pb.h" #include "curl/curl.h" #include "pugixml/pugixml.hpp" #include #include #include #include using bts::api::position::MarginResult; namespace bts { namespace position { using MarginMap = boost::container::flat_map; struct MarginServiceCore::PImpl { utils::Dispatcher& dispatcher; MarginMap last_results, last_valid_results; std::string url_base, core_user, core_password, post_margin_path, get_margin_path; bool verbose; int throttle; PImpl(utils::Dispatcher& disp) : dispatcher(disp), last_results(), last_valid_results(), //api_bts_span BT5Margin! url_base("https://cmecorenr.cmegroup.com/MarginServiceApi/1.4/"), core_user("api_bts1"), core_password("Summer123"), post_margin_path("margins/?complete=true"), get_margin_path("margins/"), verbose(false), throttle(30000) { if(config::get().login() != "") core_user = config::get().login(); if(config::get().password() != "") core_password = config::get().password(); if(config::get().url() != "") url_base = config::get().url(); auto res = curl_global_init(CURL_GLOBAL_DEFAULT); if(res != CURLE_OK) { LOG(Gen,Error,"curl_global_init() failed: {}", std::string(curl_easy_strerror(res))); } } ~PImpl() { curl_global_cleanup(); } struct CurlMsg { std::vector data; }; static size_t create_reply_callback(void* contents, size_t size, size_t nmemb, void* userp) { size_t realsize = size * nmemb; CurlMsg& mem = *(CurlMsg*)userp; mem.data.reserve(mem.data.size() + realsize + 1); for(size_t i = 0; i < realsize; i++) mem.data.push_back(static_cast(contents)[i]); mem.data.push_back(0); return realsize; } static void dump(const char* text, FILE* stream, unsigned char* ptr, size_t size) { size_t i; size_t c; unsigned int width=0x10; fprintf(stream, "%s, %10.10ld bytes (0x%8.8lx)\n", text, (long)size, (long)size); for(i=0; i= 0x20 && ptr[i+c] < 0x80) ? ptr[i+c] : '.'; fputc(x, stream); } fputc('\n', stream); /* newline */ } } static int debug_callback(CURL*, curl_infotype type, char *data, size_t size, void *userp) { const char *text; UNUSED(userp); switch (type) { case CURLINFO_TEXT: //fprintf(stderr, "== Info: %s", data); text = "=> Info:"; break; case CURLINFO_HEADER_OUT: text = "=> Send header"; break; case CURLINFO_DATA_OUT: text = "=> Send data"; break; case CURLINFO_SSL_DATA_OUT: text = "=> Send SSL data"; break; case CURLINFO_HEADER_IN: text = "<= Recv header"; break; case CURLINFO_DATA_IN: text = "<= Recv data"; break; case CURLINFO_SSL_DATA_IN: text = "<= Recv SSL data"; break; default: return 0; } dump(text, stderr, (unsigned char *)data, size); dump(text, stdout, (unsigned char *)data, size); return 0; } struct curl_slist* setup_url_and_headers(CURL* curl, const std::string& xpath) { std::string full_url = url_base + xpath; LOG(Gen,Info,"CORE Margin URL: {}",full_url); curl_easy_setopt(curl, CURLOPT_URL, full_url.c_str()); //curl_easy_setopt(curl, CURLOPT_URL, "https://cmecorenr.cmegroup.com/MarginServiceApi/1.4/margins/"); curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, debug_callback); if(verbose) curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L); curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L); curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L); curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L); //HEADERS struct curl_slist *chunk = nullptr; chunk = curl_slist_append(chunk, std::string("username:" + core_user).c_str()); chunk = curl_slist_append(chunk, std::string("password:" + core_password).c_str()); chunk = curl_slist_append(chunk, "content-type:application/xml"); curl_easy_setopt(curl, CURLOPT_HTTPHEADER, chunk); curl_easy_setopt(curl, CURLOPT_USERAGENT, "libcurl-agent/1.0"); return chunk; } bool post(const std::string& xpath, const std::string& payload, CurlMsg& reply) { bool succ = false; CURL* curl = curl_easy_init(); if (curl) { struct curl_slist* header = setup_url_and_headers(curl, xpath); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, create_reply_callback); curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)&reply); curl_easy_setopt(curl, CURLOPT_POSTFIELDS, payload.c_str()); LOG(Gen,Info,"CORE Margin Payload: {}", payload); auto res = curl_easy_perform(curl); if(res == CURLE_OK) { std::string resp(reply.data.begin(), reply.data.end()); LOG(Gen,Info,"CORE Margin Post Response: {}", resp); succ = true; } else { std::string error(curl_easy_strerror(res)); LOG(Gen,Error,"POST curl_easy_perform() failed: {}", error); //std::cout << fmt::format("CORE Margin CURL Error: ", error); } curl_easy_cleanup(curl); if(header) curl_slist_free_all(header); } else { LOG(Gen,Error,"curl_easy_init() failed\n"); } return succ; } bool get(const std::string& xpath, CurlMsg& reply) { bool succ = false; CURL* curl = curl_easy_init(); if (curl) { struct curl_slist* header = setup_url_and_headers(curl, xpath); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, create_reply_callback); curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)&reply); auto res = curl_easy_perform(curl); if(res == CURLE_OK) { std::string resp(reply.data.begin(), reply.data.end()); LOG(Gen,Info,"CORE Margin GET Response: {}", resp); succ = true; } else { LOG(Gen,Error,"GET curl_easy_perform() failed: {}", std::string(curl_easy_strerror(res))); } curl_easy_cleanup(curl); if(header) curl_slist_free_all(header); } else { LOG(Gen,Error,"curl_easy_init() failed\n"); } return succ; } //For futures- it is only necessary to populate one of the following: Ticker, or Maturity Code and Clearing Code, or Maturity Code and Product Name. //so Ticker, Net Positions void create_margin_request(const std::string& portfolio_id, pugi::xml_document& doc, pugi::xml_node& trades_node) { auto att = doc.append_attribute("standalone"); att.set_value("yes"); auto ns = doc.append_child("ns2:marginReq"); auto ns2 = ns.append_attribute("xmlns:ns2"); ns2.set_value("http://cmegroup.com/schema/core/1.4"); auto user = ns.append_attribute("reqUserId"); user.set_value(core_user.c_str()); auto child = ns.append_child("margin"); child = child.append_child("transactions"); child = child.append_child("transaction"); child.append_attribute("type").set_value("TRADE"); //this doesn't seem to work, but we are ok with CME assigning a new portfolio id //child.append_attribute("portfolioId").set_value(portfolio_id.c_str()); UNUSED(portfolio_id); child = child.append_child("payload"); child.append_attribute("format").set_value("CSV"); child.append_attribute("encoding").set_value("STRING"); trades_node = child.append_child("string"); } bool parse_margin_response(const CurlMsg& msg, std::string& portfolio_id, std::string& calculation_id) { std::string str(msg.data.begin(), msg.data.end()); pugi::xml_document doc; doc.load_string(str.c_str()); auto child = doc.child("ns2:marginRpt"); child = child.child("margin"); if(child) { portfolio_id = child.attribute("portfolioId").as_string(); calculation_id = child.attribute("id").as_string(); return true; } else return false; } void parse_margin_calculation(const CurlMsg& msg, MarginResult& result) { result.set_status(MarginResult::Error); std::string str(msg.data.begin(), msg.data.end()); pugi::xml_document doc; doc.load_string(str.c_str()); auto child = doc.child("ns2:marginRpt"); child = child.child("margin"); if(child) { result.set_settle_indicator(child.attribute("settleInd").as_string()); result.set_settle_qualifier(child.attribute("settleQual").as_string()); result.set_portfolio_id(child.attribute("portfolioId").as_string()); result.set_as_of_time(child.attribute("asOfTime").as_string()); result.set_calculation_id(child.attribute("id").as_string()); auto val = child.child("amounts"); if(val && result.portfolio_id() != "") { result.set_opt_value(val.attribute("optVal").as_double()); result.set_nonopt_value(val.attribute("nonOptVal").as_double()); result.set_maint_margin(val.attribute("maint").as_double()); result.set_initial_margin(val.attribute("init").as_double()); result.set_status(MarginResult::NewValid); } } } struct xml_string_writer: pugi::xml_writer { std::string& result; xml_string_writer(std::string& res) : result(res) {} virtual void write(const void* data, size_t size) { result.append(static_cast(data), size); } }; bool compute_margin(const std::string& account_name, const std::string& portfolid_id, const std::vector& pos, std::function cb) { pugi::xml_document doc; pugi::xml_node trades_node; create_margin_request(portfolid_id, doc, trades_node); std::string request = "\nFirm Id, Acct Id, Ticker Symbol, Net Positions\n"; size_t count = 0; for(const auto& i : pos) { int p = i.position.value() * (i.side == api::SIDE_SELL ? -1 : 1); request += fmt::format("BTS, {}, {}, {}, {}", account_name, i.security_desc, p, "\n"); count++; } if(count > 0) { trades_node.append_child(pugi::node_pcdata).set_value(request.c_str()); std::string xml; xml_string_writer writer(xml); doc.save(writer); CurlMsg reply; post(post_margin_path, xml, reply); std::string portid_resp, calcid_resp; if(parse_margin_response(reply, portid_resp, calcid_resp)) { std::function do_web; do_web = [=]() { MarginResult margin; margin.set_status(MarginResult::Error); margin.set_account(account_name); CurlMsg reply2; bool done = false; try { if(get(get_margin_path + calcid_resp, reply2)) { parse_margin_calculation(reply2, margin); cb(margin); done = true; } } catch(const std::exception& ex) { LOG(Gen, Error, "Exception in parse_margin_calculation {}", std::string(ex.what())); } catch(...) { LOG(Gen, Error, "Unknown Exception in parse_margin_calculation"); } if(!done) dispatcher.post_copy(do_web, 5000); }; dispatcher.post_copy(do_web, 5000); return true; } } return false; } }; MarginServiceCore::MarginServiceCore(utils::Dispatcher& disp) : me(new PImpl(disp)) { } MarginServiceCore::~MarginServiceCore() {} void MarginServiceCore::compute_margin(const std::string& acct, const std::vector& pos, std::function cb) { me->dispatcher.post([=] () { query_margin_webapi(acct, pos, cb); }, me->throttle); } bool MarginServiceCore::query_margin_webapi(const std::string& acct, const std::vector& pos, std::function cb) { utils::Uuid guid; try { return me->compute_margin(acct, guid.to_text_string(), pos, [=] (const MarginResult& m) { me->last_results.insert_or_assign(acct, m); if(m.status() == MarginResult::NewValid) me->last_valid_results.insert_or_assign(acct, m); cb(m); }); } catch(const std::exception& ex) { LOG(Gen, Error, "Exception in query_margin_webapi {}", std::string(ex.what())); return false; } catch(...) { LOG(Gen, Error, "Unknown Exception in query_margin_webapi"); return false; } } void MarginServiceCore::set_verbose_debugging(bool verb) { me->verbose = verb; } bool MarginServiceCore::get_last_valid_result(const std::string& acct, MarginResult& result) { auto it = me->last_valid_results.find(acct); if(it != me->last_valid_results.end()) { result = it->second; return true; } return false; } bool MarginServiceCore::get_last_result(const std::string& acct, MarginResult& result) { auto it = me->last_results.find(acct); if(it != me->last_results.end()) { result = it->second; return true; } return false; } } }