Merge branch 'development' of bitbucket.org:bluetradesys/spark into development
This commit is contained in:
commit
fecbcbb6b6
86 files changed
+2592
-1080
No files matched your search
@@ -138,7 +138,7 @@
|
||||
_dispose = Disposable.Create(resources.ToArray());
|
||||
|
||||
static string fmt_sec(TimeAndSalesRowVM row) => row.security.ToString();
|
||||
static string fmt_time(TimeAndSalesRowVM row) => row.time.ToString("h:mm:ss tt");
|
||||
static string fmt_time(TimeAndSalesRowVM row) => row.time.ToString("h:mm:ss.ffffff");
|
||||
static string fmt_price(TimeAndSalesRowVM row) => row.security.spec_or_default().format(row.price);
|
||||
static string fmt_aggressor(TimeAndSalesRowVM row) => row.aggressor_side.ToString();
|
||||
static string fmt_volume(TimeAndSalesRowVM row) => new TickFactors(1, 100).format(row.volume);
|
||||
|
||||
@@ -46,7 +46,7 @@
|
||||
Filled = 56,
|
||||
Symbol = 60,
|
||||
Time = 112,
|
||||
TimeOfDay = 64,
|
||||
TimeOfDay = 84,
|
||||
TimeInForce = 47,
|
||||
User = 53,
|
||||
Aggressor = 80,
|
||||
|
||||
@@ -69,8 +69,16 @@ namespace mbts {
|
||||
public static bool is_valid(this RegionalTickFactors regional_tick) =>
|
||||
regional_tick.low_tick.is_valid() && regional_tick.high_tick.is_valid();
|
||||
|
||||
public static bool is_integer(this TickFactors tick) =>
|
||||
tick.tick_unit == tick.tick_base;
|
||||
public static bool is_integer(this TickFactors tick) {
|
||||
if (tick.tick_unit == tick.tick_base) {
|
||||
return true;
|
||||
}
|
||||
if (tick.tick_unit > tick.tick_base && tick.tick_base != 0) {
|
||||
var units_per_base = (int)(tick.tick_unit / tick.tick_base);
|
||||
return units_per_base * tick.tick_base == tick.tick_unit;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public static TickFactors tick_factors(this RegionalTickFactors regional_tick, double value) =>
|
||||
value > regional_tick.low_tick_inclusive_end ?
|
||||
|
||||
@@ -295,6 +295,9 @@ namespace mbts.spark {
|
||||
var ticks_per_dollar = (uint)(tf.low_tick.tick_base / tf.low_tick.tick_unit);
|
||||
// if ticks_per_row is greater than ticks_per_dollar, round to a multiple of ticks_per_dollar
|
||||
if (ticks_per_row > ticks_per_dollar) {
|
||||
if (ticks_per_dollar == 0) {
|
||||
return null;
|
||||
}
|
||||
return (uint)(ticks_per_dollar * Math.Floor((double)(ticks_per_row / ticks_per_dollar)));
|
||||
} else if (ticks_per_row == ticks_per_dollar) {
|
||||
return ticks_per_row;
|
||||
|
||||
@@ -14,4 +14,5 @@ message DatabentoMarketDataConfig {
|
||||
string key = 60;
|
||||
string dataset = 70;
|
||||
bool filter_securities_one_year= 80;
|
||||
bool load_reference_data = 90;
|
||||
}
|
||||
@@ -156,6 +156,8 @@ AdminServiceUtils::resolve_connection(
|
||||
const bts::api::admin::ExchangeConnection *rv = nullptr;
|
||||
switch (sec.exchange()) {
|
||||
case api::Venue::VENUE_GLBX: {
|
||||
//CME EXECUTION NEEDS TO BE UPDATED, probably merge with VENUE_BINANCE_USDS_FUTURES
|
||||
|
||||
auto deployment_type =
|
||||
boost::to_lower_copy(config::get<config::DeploymentType>().type());
|
||||
rv = resolve_cme_connection(data, account, sec);
|
||||
@@ -227,6 +229,7 @@ void AdminServiceUtils::fill_connections_from_config(
|
||||
connection.set_name(c.name());
|
||||
|
||||
switch (c.venue()) {
|
||||
//CME EXECUTION NEEDS TO BE UPDATED, probably merge default case
|
||||
case bts::api::VENUE_GLBX:
|
||||
if (c.cme_market_segment_id() == 0)
|
||||
connection.mutable_cme_connection()->mutable_cgw_connection();
|
||||
@@ -236,10 +239,7 @@ void AdminServiceUtils::fill_connections_from_config(
|
||||
->set_market_segment_id(c.cme_market_segment_id());
|
||||
add_connection(admin_data, connection);
|
||||
break;
|
||||
case bts::api::VENUE_CBOE:
|
||||
break;
|
||||
case bts::api::VENUE_TAIFEX:
|
||||
break;
|
||||
|
||||
case bts::api::VENUE_SIM:
|
||||
case bts::api::VENUE_SIMICE:
|
||||
connection.mutable_sim_connection();
|
||||
@@ -250,6 +250,8 @@ void AdminServiceUtils::fill_connections_from_config(
|
||||
c.sender_compid());
|
||||
add_connection(admin_data, connection);
|
||||
break;
|
||||
case bts::api::VENUE_TAIFEX:
|
||||
case bts::api::VENUE_CBOE:
|
||||
default:
|
||||
connection.mutable_general_connection()->set_exchange(c.venue());
|
||||
add_connection(admin_data, connection);
|
||||
@@ -571,6 +573,13 @@ bts::api::orderman::RoutingRecord AdminServiceUtils::get_routing_record(
|
||||
return rv;
|
||||
}
|
||||
switch (sec.exchange()) {
|
||||
case api::Venue::VENUE_CBOE:{//subaccount not added
|
||||
rv.set_session("cboe_1");
|
||||
rv.set_account(account_ptr->name());
|
||||
rv.set_operator_id(user_ptr->name());
|
||||
out_result.set_is_success(true);
|
||||
return rv;
|
||||
}
|
||||
case api::Venue::VENUE_DERIBIT: {
|
||||
rv.set_session("deribit_1");
|
||||
rv.set_account(account_ptr->name());
|
||||
@@ -733,6 +742,13 @@ bts::api::orderman::RoutingRecord AdminServiceUtils::get_routing_record(
|
||||
rv.set_customer_or_firm(sim_account_data->origin());
|
||||
return rv;
|
||||
} break;
|
||||
case api::Venue::VENUE_TAIFEX:{
|
||||
rv.set_session("taifex_1");
|
||||
rv.set_account(account_ptr->name());
|
||||
rv.set_operator_id(user_ptr->name());
|
||||
out_result.set_is_success(true);
|
||||
return rv;
|
||||
}
|
||||
case api::VENUE_FTX: // TODO
|
||||
case api::VENUE_BINANCE: // TODO
|
||||
case api::VENUE_BINANCE_US:
|
||||
@@ -741,8 +757,6 @@ bts::api::orderman::RoutingRecord AdminServiceUtils::get_routing_record(
|
||||
case api::VENUE_OKX:
|
||||
case api::Venue::VENUE_UNSET:
|
||||
case api::Venue::VENUE_BTS:
|
||||
case api::Venue::VENUE_CBOE:
|
||||
case api::Venue::VENUE_TAIFEX:
|
||||
case api::Venue::VENUE_CBOECFE:
|
||||
case api::Venue::Venue_INT_MIN_SENTINEL_DO_NOT_USE_:
|
||||
case api::Venue::Venue_INT_MAX_SENTINEL_DO_NOT_USE_:
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
MarexFixCfg {
|
||||
value{
|
||||
customer_or_firm :0
|
||||
handl_inst: 2
|
||||
rule80a: 1
|
||||
heart_bt_int : 30
|
||||
counterparty_port :22754
|
||||
name: "trafix"
|
||||
msg_type: "Order_Single"
|
||||
protocol_version: "FIX_42"
|
||||
sender_sub_id: "BTS"
|
||||
target_sub_id: "TRFXMRX"
|
||||
client_id: "BTS"
|
||||
counterparty_host : "74.217.55.230"
|
||||
open_close :"O"
|
||||
max_retries: 500
|
||||
retry_interval: 600
|
||||
ex_destination : "W"
|
||||
security_exchange : "CBOE"
|
||||
currency : "USD"
|
||||
}
|
||||
}
|
||||
@@ -6,8 +6,8 @@ tickConfig
|
||||
tick_entry {key:"TXF",value:1},
|
||||
tick_entry {key:"MXF",value:1},
|
||||
tick_entry {key:"TMF",value:1},
|
||||
tick_entry {key:"CDF",value:1},
|
||||
tick_entry {key:"QFF",value:1},
|
||||
tick_entry {key:"CDF",value:5},
|
||||
tick_entry {key:"QFF",value:5},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
syntax = "proto3";
|
||||
package bts.config;
|
||||
message MarexFixCfg {
|
||||
enum HandlInst {
|
||||
UNDEFINED = 0;
|
||||
DMA = 1;
|
||||
ALGO = 2;
|
||||
CARE = 3;
|
||||
}
|
||||
enum Rule80A {
|
||||
UNUSED = 0;
|
||||
A = 1;
|
||||
P = 2;
|
||||
}
|
||||
string msg_type = 1;
|
||||
string protocol_version = 2;
|
||||
HandlInst handl_inst = 3;
|
||||
string sender_sub_id = 4;
|
||||
string target_sub_id = 5;
|
||||
string client_id = 6;
|
||||
Rule80A rule80a = 7;
|
||||
string counterparty_host = 8;
|
||||
int32 heart_bt_int = 9;
|
||||
int32 counterparty_port = 10;
|
||||
uint32 customer_or_firm =11;
|
||||
string open_close =12;
|
||||
string name = 13;
|
||||
int32 max_retries = 14;
|
||||
uint32 retry_interval = 15;
|
||||
string ex_destination = 16;
|
||||
string security_exchange = 17;
|
||||
string currency = 18;
|
||||
}
|
||||
@@ -50,7 +50,9 @@ class TimeSpan {
|
||||
constexpr bool operator==(TimeSpan lhs, TimeSpan rhs) noexcept;
|
||||
constexpr bool operator!=(TimeSpan lhs, TimeSpan rhs) noexcept;
|
||||
constexpr bool operator<(TimeSpan lhs, TimeSpan rhs) noexcept;
|
||||
constexpr bool operator<=(TimeSpan lhs, TimeSpan rhs) noexcept;
|
||||
constexpr bool operator>(TimeSpan lhs, TimeSpan rhs) noexcept;
|
||||
constexpr bool operator>=(TimeSpan lhs, TimeSpan rhs) noexcept;
|
||||
|
||||
std::ostream &operator<<(std::ostream &out, const TimeSpan &ts) noexcept;
|
||||
|
||||
@@ -116,10 +118,18 @@ constexpr bool operator<(TimeSpan lhs, TimeSpan rhs) noexcept {
|
||||
return lhs.nanos() < rhs.nanos();
|
||||
}
|
||||
|
||||
constexpr bool operator<=(TimeSpan lhs, TimeSpan rhs) noexcept {
|
||||
return lhs.nanos() <= rhs.nanos();
|
||||
}
|
||||
|
||||
constexpr bool operator>(TimeSpan lhs, TimeSpan rhs) noexcept {
|
||||
return lhs.nanos() > rhs.nanos();
|
||||
}
|
||||
|
||||
constexpr bool operator>=(TimeSpan lhs, TimeSpan rhs) noexcept {
|
||||
return lhs.nanos() >= rhs.nanos();
|
||||
}
|
||||
|
||||
} // namespace bts::core
|
||||
|
||||
FMT_IMPL(bts::core::TimeSpan, t, "{}", t.to_iso_string());
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
#include "lib/config/config.h"
|
||||
#include "lib/config/protos/orders_fills_cleanup.pb.h"
|
||||
#include "lib/md/common.h"
|
||||
#include "lib/utils/datetime.h"
|
||||
|
||||
#include <boost/date_time/posix_time/posix_time.hpp>
|
||||
@@ -11,51 +12,15 @@
|
||||
namespace bts {
|
||||
namespace securities {
|
||||
|
||||
std::string to_str(api::Security const &sec) {
|
||||
std::string serialized;
|
||||
google::protobuf::TextFormat::PrintToString(sec, &serialized);
|
||||
return serialized;
|
||||
}
|
||||
|
||||
std::pair<bool, api::Security> from_str(std::string const &str) {
|
||||
api::Security result;
|
||||
auto success = google::protobuf::TextFormat::ParseFromString(str, &result);
|
||||
return std::make_pair(success, result);
|
||||
}
|
||||
|
||||
std::string descriptor(api::Security const &sec) {
|
||||
switch (sec.ExchangeData_case()) {
|
||||
case api::Security::kTaifex:
|
||||
return sec.taifex().security_desc();
|
||||
case api::Security::kGlbx:
|
||||
return sec.glbx().security_desc();
|
||||
case api::Security::kCboe:
|
||||
return sec.cboe().security_desc();
|
||||
case api::Security::kCfe:
|
||||
return sec.cfe().security_desc();
|
||||
case api::Security::kIce:
|
||||
return sec.ice().security_desc();
|
||||
case api::Security::kDeribit:
|
||||
return sec.deribit().security_desc();
|
||||
case api::Security::kFtx:
|
||||
return sec.ftx().security_desc();
|
||||
case api::Security::kBinance:
|
||||
return sec.binance().security_desc();
|
||||
case api::Security::kBinanceUsdsFutures:
|
||||
return sec.binance_usds_futures().security_desc();
|
||||
case api::Security::kBinanceUs:
|
||||
return sec.binance_us().security_desc();
|
||||
case api::Security::kCoinbase:
|
||||
return sec.coinbase().security_desc();
|
||||
case api::Security::kOkx:
|
||||
return sec.okx().security_desc();
|
||||
default:
|
||||
return "No descriptor for non glbx / ice exchange";
|
||||
}
|
||||
}
|
||||
using bts::api::Security;
|
||||
using bts::api::Venue;
|
||||
|
||||
api::SecurityType security_type(api::Security const &security) {
|
||||
switch (security.Contract_case()) {
|
||||
return security_type(security.Contract_case());
|
||||
}
|
||||
|
||||
api::SecurityType security_type(api::Security::ContractCase contract_case) {
|
||||
switch (contract_case) {
|
||||
case api::Security::ContractCase::kFuture:
|
||||
return api::SecurityType::SECURITYTYPE_FUTURE;
|
||||
case api::Security::ContractCase::kOption:
|
||||
@@ -70,16 +35,6 @@ api::SecurityType security_type(api::Security const &security) {
|
||||
return api::SecurityType::SECURITYTYPE_UNSET;
|
||||
}
|
||||
return api::SecurityType::SECURITYTYPE_UNSET;
|
||||
// if (sec.has_future())
|
||||
// return api::SecurityType::SECURITYTYPE_FUTURE;
|
||||
// else if (sec.has_option())
|
||||
// return api::SecurityType::SECURITYTYPE_OPTION;
|
||||
// else if (sec.has_spread())
|
||||
// return api::SecurityType::SECURITYTYPE_SPREAD;
|
||||
// else if (sec.has_synthetic())
|
||||
// return api::SecurityType::SECURITYTYPE_SYNTHETIC;
|
||||
// else
|
||||
// return api::SecurityType::SECURITYTYPE_UNSET;
|
||||
}
|
||||
|
||||
boost::gregorian::date smallest_leg(
|
||||
@@ -140,5 +95,91 @@ bool is_expired(api::Security const &security) {
|
||||
return is_expired(security, datetime::local_now());
|
||||
}
|
||||
|
||||
void write_date_string(
|
||||
uint32_t year, uint32_t month, uint32_t day, std::ostringstream &result) {
|
||||
result << std::setw(2) << std::setfill('0') << month << "/" << std::setw(2)
|
||||
<< std::setfill('0') << day << "/" << year - 2000;
|
||||
}
|
||||
|
||||
std::ostringstream &write_leg_descriptor(
|
||||
std::ostringstream &result, char separator, const api::ContractLeg &leg) {
|
||||
bts_assert(leg.security().id() != md::invalid_security_key);
|
||||
// Keep the string small, yet readable for diagnostic purposes.
|
||||
result << (leg.side() == api::SIDE_BUY ? '+' : '-') << separator
|
||||
<< leg.quantity() << "*(i=" << leg.security().id()
|
||||
<< ",m=" << leg.price_multiplier() << ",d=" << leg.leg_delta() << ")";
|
||||
return result;
|
||||
}
|
||||
|
||||
std::ostringstream &write_future_descriptor(
|
||||
std::ostringstream &result, const api::FutureContract &future) {
|
||||
write_date_string(
|
||||
future.expiration_year(), future.expiration_month(),
|
||||
future.expiration_day(), result);
|
||||
return result;
|
||||
}
|
||||
|
||||
std::ostringstream &write_option_descriptor(
|
||||
std::ostringstream &result,
|
||||
char separator,
|
||||
const api::OptionContract &option) {
|
||||
write_date_string(
|
||||
option.expiration_year(), option.expiration_month(),
|
||||
option.expiration_day(), result);
|
||||
result << separator << option.strike()
|
||||
<< (option.call_put() == api::PUTORCALL_CALL ? 'C' : 'P');
|
||||
return result;
|
||||
}
|
||||
|
||||
std::ostringstream &write_legs_descriptor(
|
||||
std::ostringstream &result,
|
||||
char separator,
|
||||
const google::protobuf::RepeatedPtrField<api::ContractLeg> &legs) {
|
||||
bts_assert(legs.size() > 0);
|
||||
auto it = legs.begin();
|
||||
write_leg_descriptor(result, separator, *it);
|
||||
while (++it != legs.end()) {
|
||||
result << separator;
|
||||
write_leg_descriptor(result, separator, *it);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
std::string unique_descriptor(const api::Security &security) {
|
||||
const char separator = ' ';
|
||||
std::ostringstream result;
|
||||
result << api::Venue_Name(security.exchange()).substr(6) << separator
|
||||
<< api::SecurityType_Name(security_type(security)).substr(12)
|
||||
<< descriptor(security);
|
||||
switch (security.Contract_case()) {
|
||||
case Security::ContractCase::kOption:
|
||||
result << separator;
|
||||
write_option_descriptor(result, separator, security.option());
|
||||
break;
|
||||
case Security::ContractCase::kFuture:
|
||||
result << separator;
|
||||
write_future_descriptor(result, security.future());
|
||||
break;
|
||||
case Security::ContractCase::kSpot:
|
||||
break;
|
||||
case Security::ContractCase::kSpread:
|
||||
result << separator;
|
||||
write_legs_descriptor(result, separator, security.spread().legs());
|
||||
break;
|
||||
case Security::ContractCase::kSynthetic: {
|
||||
result << "pc="
|
||||
<< api::SyntheticContract::PriceCalc_Name(
|
||||
security.synthetic().price_calc());
|
||||
write_legs_descriptor(result, separator, security.synthetic().legs());
|
||||
break;
|
||||
}
|
||||
default:
|
||||
bts_assert_msg(
|
||||
false, "unique_descriptor failed due to invalid contract case");
|
||||
break;
|
||||
}
|
||||
return result.str();
|
||||
}
|
||||
|
||||
} // namespace securities
|
||||
} // namespace bts
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
#include "lib/core/FMT_IMPL.h"
|
||||
#include "lib/domain_model/Security.pb.h"
|
||||
#include "lib/domain_model/SecurityType.h"
|
||||
#include "lib/domain_model/Side.h"
|
||||
#include "lib/domain_model/Venue.h"
|
||||
#include "lib/domain_model/common.h"
|
||||
@@ -25,11 +26,27 @@ class ICEExchangeData;
|
||||
} // namespace api
|
||||
|
||||
namespace securities {
|
||||
SecurityKey security_key(api::Security const &);
|
||||
std::string to_str(api::Security const &);
|
||||
std::pair<bool, api::Security> from_str(std::string const &);
|
||||
std::string descriptor(api::Security const &);
|
||||
|
||||
/**
|
||||
* Gets a descriptor string for a security (should be unique within a particular exchange).
|
||||
*/
|
||||
const std::string &descriptor(
|
||||
const api::Security &sec, const std::string &default_rv = "UNDEFINED");
|
||||
|
||||
/**
|
||||
* Gets a descriptor string for a security that is unique within the entire
|
||||
* system (not just exchange).
|
||||
*/
|
||||
std::string unique_descriptor(const api::Security &security);
|
||||
|
||||
api::SecurityType security_type(api::Security const &);
|
||||
api::SecurityType security_type(api::Security::ContractCase contract_case);
|
||||
|
||||
void write_date_string(
|
||||
std::uint32_t year,
|
||||
std::uint32_t month,
|
||||
std::uint32_t day,
|
||||
std::ostringstream &result);
|
||||
|
||||
inline std::string const &side_name(api::Side side) {
|
||||
return api::Side_Name(side);
|
||||
@@ -234,6 +251,41 @@ inline Price tick_size(api::Security const &sec) {
|
||||
return Price(sec.contract_spec().low_tick());
|
||||
}
|
||||
|
||||
inline const std::string &descriptor(
|
||||
const api::Security &sec, const std::string &default_rv) {
|
||||
switch (sec.ExchangeData_case()) {
|
||||
case api::Security::kGlbx:
|
||||
return sec.glbx().security_desc();
|
||||
case api::Security::kCboe:
|
||||
return sec.cboe().security_desc();
|
||||
case api::Security::kCfe:
|
||||
return sec.cfe().security_desc();
|
||||
case api::Security::kIce:
|
||||
return sec.ice().security_desc();
|
||||
case api::Security::kDeribit:
|
||||
return sec.deribit().security_desc();
|
||||
case api::Security::kFtx:
|
||||
return sec.ftx().security_desc();
|
||||
case api::Security::kBinance:
|
||||
return sec.binance().security_desc();
|
||||
case api::Security::kBinanceUsdsFutures:
|
||||
return sec.binance_usds_futures().security_desc();
|
||||
case api::Security::kBinanceUs:
|
||||
return sec.binance_us().security_desc();
|
||||
case api::Security::kCoinbase:
|
||||
return sec.coinbase().security_desc();
|
||||
case api::Security::kOkx:
|
||||
return sec.okx().security_desc();
|
||||
case api::Security::kTaifex:
|
||||
return sec.taifex().security_desc();
|
||||
default: {
|
||||
if (sec.has_synthetic()) {
|
||||
return sec.synthetic().name();
|
||||
}
|
||||
return default_rv;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Looking to round md::IntPrice? Look in IntPrice.h
|
||||
|
||||
// computes price_2 - price_1 in ticks
|
||||
@@ -314,3 +366,5 @@ inline bool is_spread_of_strips(const bts::api::Security &sec) {
|
||||
|
||||
} // namespace securities
|
||||
} // namespace bts
|
||||
|
||||
FMT_IMPL(bts::api::Security, s, "{}", bts::securities::descriptor(s));
|
||||
@@ -0,0 +1,30 @@
|
||||
#pragma once
|
||||
|
||||
#include "lib/domain_model/Security.pb.h"
|
||||
#include "lib/core/FMT_IMPL.h"
|
||||
#include "lib/core/EnumMacros.h"
|
||||
#include "lib/io/JsonReader.h"
|
||||
#include "lib/logging/logging.h"
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
namespace bts::api {
|
||||
|
||||
PB_ENUM_FROM_JSON(SecurityType)
|
||||
PB_ENUM_TO_JSON(SecurityType)
|
||||
|
||||
std::ostream &operator<<(std::ostream &out, const SecurityType &type) noexcept;
|
||||
|
||||
//////////////////////////////////////////////////
|
||||
// Definition
|
||||
//////////////////////////////////////////////////
|
||||
|
||||
inline std::ostream &operator<<(
|
||||
std::ostream &out, const SecurityType &type) noexcept {
|
||||
out << SecurityType_Name(type);
|
||||
return out;
|
||||
}
|
||||
|
||||
} // namespace bts::api
|
||||
|
||||
FMT_IMPL(bts::api::SecurityType, t, "{}", bts::api::SecurityType_Name(t));
|
||||
@@ -13,6 +13,17 @@ namespace bts::api {
|
||||
PB_ENUM_FROM_JSON(Venue)
|
||||
PB_ENUM_TO_JSON(Venue)
|
||||
|
||||
std::ostream &operator<<(std::ostream &out, Venue venue) noexcept;
|
||||
|
||||
//////////////////////////////////////////////////
|
||||
// Definition
|
||||
//////////////////////////////////////////////////
|
||||
|
||||
inline std::ostream &operator<<(std::ostream &out, Venue venue) noexcept {
|
||||
out << Venue_Name(venue);
|
||||
return out;
|
||||
}
|
||||
|
||||
} // namespace bts::api
|
||||
|
||||
FMT_IMPL(bts::api::Venue, v, "{}", bts::api::Venue_Name(v));
|
||||
@@ -3,6 +3,7 @@
|
||||
#include "ShmDumper.h"
|
||||
|
||||
#include "../Shm/MDStructure.h"
|
||||
#include "lib/domain_model/Security.h"
|
||||
#include "lib/io/JsonDumper.h"
|
||||
#include "lib/logging/logging.h"
|
||||
#include "lib/utils/SharedList.h"
|
||||
@@ -51,85 +52,8 @@ string dump_header(const ColdHeader &header) {
|
||||
return JsonDumper::dump(header);
|
||||
}
|
||||
|
||||
void write_date_string(
|
||||
uint32_t year, uint32_t month, uint32_t day, std::ostringstream &result);
|
||||
string ShmDumper::dump_security(const FixedSecurity &security) {
|
||||
ostringstream res;
|
||||
res << "{";
|
||||
res << "'key': " << security.key << ", ";
|
||||
res << "'symbol': '" << security.symbol() << "', ";
|
||||
res << "'type': '" << api::SecurityType_Name(security.type) << "', ";
|
||||
res << "'exchange': '" << api::Venue_Name(security.exchange) << "', ";
|
||||
res << "'alt_symbol': '" << security.alt_symbol() << "', ";
|
||||
res << "'cfi_code': '" << security.cfi_code() << "', ";
|
||||
res << "'security_desc': '" << security.security_desc() << "', ";
|
||||
res << "'platform_id': '" << security.platform_id() << "', ";
|
||||
res << "'display_factor': " << (int)security.display_factor << ", ";
|
||||
res << "'numeric_id': " << security.numeric_id << ", ";
|
||||
res << "'cboe_class_key': " << security.cboe_class_key << ", ";
|
||||
res << "'main_frac': " << (int)security.cme_main_fraction << ", ";
|
||||
res << "'sub_fract': " << (int)security.cme_sub_fraction << ", ";
|
||||
res << "'price_format': " << (int)security.cme_price_display_format << ", ";
|
||||
res << "'market_segment': " << (int)security.cme_market_segment_id << ", ";
|
||||
|
||||
res << "'price_break': " << security.price_break << ", ";
|
||||
res << "'low_tick': " << security.low_tick << ", ";
|
||||
res << "'high_tick': " << security.high_tick << ", ";
|
||||
res << "'cpp': " << security.cash_per_point << ", ";
|
||||
|
||||
res << "'market_description ': " << security.ice_market_description()
|
||||
<< ", ";
|
||||
res << "'cleared_alias ': " << security.ice_cleared_alias() << ", ";
|
||||
res << "'product_description ': " << security.ice_product_name() << ", ";
|
||||
res << "'hub_alias ': " << security.ice_hub_alias() << ", ";
|
||||
res << "'strip_name': " << security.ice_strip_name() << ", ";
|
||||
res << "'mic_code': " << security.mic_code() << ", ";
|
||||
|
||||
switch (security.type) {
|
||||
case SECURITYTYPE_SPOT:
|
||||
break;
|
||||
case SECURITYTYPE_FUTURE: {
|
||||
auto *psec = static_cast<const FixedFutureSecurity *>(&security);
|
||||
res << "'expiration': '";
|
||||
write_date_string(
|
||||
psec->expiration_year, psec->expiration_month, psec->expiration_day,
|
||||
res);
|
||||
res << "' ";
|
||||
break;
|
||||
}
|
||||
case SECURITYTYPE_OPTION: {
|
||||
auto *psec = static_cast<const FixedOptionSecurity *>(&security);
|
||||
res << "'expiration': '";
|
||||
write_date_string(
|
||||
psec->expiration_year, psec->expiration_month, psec->expiration_day,
|
||||
res);
|
||||
res << "', ";
|
||||
res << "'strike': " << psec->strike << ", ";
|
||||
res << "'callput': '" << api::PutOrCall_Name(psec->call_put) << "' ";
|
||||
break;
|
||||
}
|
||||
case SECURITYTYPE_SPREAD: {
|
||||
auto *psec = static_cast<const FixedSpreadSecurityXL *>(&security);
|
||||
res << "'legs': [";
|
||||
for (size_t x = 0; x < psec->leg_count(); ++x) {
|
||||
auto &leg = psec->legs[x];
|
||||
if (x > 0)
|
||||
res << ",";
|
||||
res << "{ 'key': " << leg.key << ", 'exkey': " << leg.exchange_key
|
||||
<< ", 'quantity': " << leg.leg_quantity
|
||||
<< ", 'leg_price': " << leg.leg_price
|
||||
<< ", 'delta': " << leg.leg_delta << ", 'side': '"
|
||||
<< api::Side_Name(leg.leg_side) << "' }";
|
||||
}
|
||||
res << "]";
|
||||
break;
|
||||
}
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
res << "}";
|
||||
return res.str();
|
||||
return JsonDumper::dump(security);
|
||||
}
|
||||
|
||||
string dump(datetime::Date date) {
|
||||
|
||||
@@ -23,12 +23,12 @@
|
||||
#include "lib/domain_model/Security.pb.h"
|
||||
#include "lib/domain_model/common.h"
|
||||
#include "lib/logging/logging.h"
|
||||
#include "lib/md/GeneralSecurityIndex.h"
|
||||
#include "lib/md/IMarketDataListener.h"
|
||||
#include "lib/md/IntPrice.h"
|
||||
#include "lib/md/MDProducer.h"
|
||||
#include "lib/md/MDProducerHandle.h"
|
||||
#include "lib/md/PriceSize.h"
|
||||
#include "lib/md/SecurityIDRecorder.h"
|
||||
#include "lib/md/SecurityIndex.h"
|
||||
#include "lib/md/Shm/FixedSecurity.h"
|
||||
#include "lib/md/Shm/MDStructure.h"
|
||||
@@ -274,10 +274,11 @@ bool update_incremental(const IMarketDataListener::BookOp &op) {
|
||||
};
|
||||
|
||||
struct GeneralMarketDataListener::PImpl {
|
||||
const api::domain::Venue venue;
|
||||
// if this is set to true, then the security key passed on in the
|
||||
// on_securities function is used. If this is not set to true, then a new key
|
||||
// is generated
|
||||
const bool use_given_sec_key = true;
|
||||
const bool use_given_sec_key = false;
|
||||
const std::string cc_name_prefix = "cc_security";
|
||||
size_t n_bk_updates = 0;
|
||||
bts::persistence::IDGenerator &id_gen =
|
||||
@@ -296,12 +297,15 @@ struct GeneralMarketDataListener::PImpl {
|
||||
MDArray asks = {};
|
||||
IncrementalBookData inc_book_data = {};
|
||||
explicit PImpl(
|
||||
api::domain::Venue venue,
|
||||
GeneralSecurityIndex &index,
|
||||
MDProducer &md_producer,
|
||||
std::shared_ptr<bts::orderman::SimExecutionGateway> sim_ex_gwy)
|
||||
: index(index), md_producer(md_producer), sim_ex_gwy(sim_ex_gwy) {
|
||||
if (sim_ex_gwy != nullptr) {
|
||||
sim_ex_gwy->set_producers(&index);
|
||||
: venue(venue), index(index),
|
||||
md_producer(md_producer),
|
||||
sim_ex_gwy(std::move(sim_ex_gwy)) {
|
||||
if (this->sim_ex_gwy != nullptr) {
|
||||
this->sim_ex_gwy->set_producers(&this->index);
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -363,6 +367,7 @@ void copy_books(
|
||||
GeneralMarketDataListener::GeneralMarketDataListener(
|
||||
GeneralSecurityIndex &index, MDProducer &md_producer, api::Venue venue)
|
||||
: me(std::make_unique<PImpl>(
|
||||
domain::DomainSecurityUtils::convert(venue),
|
||||
index,
|
||||
md_producer,
|
||||
bts::orderman::SimExecutionGatewayFactory::get()->get_getayeway(
|
||||
@@ -379,7 +384,7 @@ void GeneralMarketDataListener::on_securities(
|
||||
LOG(md, Info, "GeneralMarketDataListener::on_securities n_secs: {}",
|
||||
complete_security_list.size());
|
||||
for (auto s_ptr : complete_security_list) {
|
||||
domain::Security sec = *s_ptr;
|
||||
const domain::Security &sec = *s_ptr;
|
||||
const auto &s_comm = sec.security_common;
|
||||
CCSecSearchResult search_result = _find_sec(sec, true);
|
||||
LOG(md, Info,
|
||||
@@ -503,11 +508,7 @@ CCSecSearchResult GeneralMarketDataListener::_find_sec(
|
||||
if (add_if_absent) {
|
||||
uint64_t bts_id = cc_id.value();
|
||||
if (!me->use_given_sec_key) {
|
||||
auto logical_key = get_logical_key(sec);
|
||||
auto new_key_opt =
|
||||
me->id_gen.get_id(me->cc_name_prefix, logical_key, true);
|
||||
bts_assert(new_key_opt.has_value());
|
||||
bts_id = new_key_opt.value();
|
||||
bts_id = me->id_gen.get_or_add(me->venue, sec.type(), sec.desc());
|
||||
}
|
||||
me->cc_sec_id_map.insert({cc_id, {bts_id}});
|
||||
LOG(md, Info,
|
||||
@@ -654,9 +655,7 @@ uint64_t GeneralMarketDataListener::_generate_security_key(
|
||||
auto logical_key = get_logical_key(sec);
|
||||
// if (me->use_given_sec_key)
|
||||
// return sec.security_common.id.value();
|
||||
auto rv = me->id_gen.get_id(me->cc_name_prefix, logical_key, true);
|
||||
bts_assert(rv.has_value());
|
||||
return rv.value();
|
||||
return me->id_gen.get_or_add(me->venue, sec.type(), sec.desc());
|
||||
}
|
||||
|
||||
void GeneralMarketDataListener::_possibly_match_with_sim_orders(
|
||||
|
||||
@@ -61,26 +61,6 @@ datetime::Date sec_expiration_cut_off() {
|
||||
return datetime::Date(pt.date().year(), pt.date().month(), pt.date().day());
|
||||
}
|
||||
|
||||
void apply_compatibilit_mutations(Security &security) {
|
||||
// Migrate from old tick stuff
|
||||
if (security.has_glbx()) {
|
||||
if (security.has_glbx()) {
|
||||
if (security.contract_spec().main_fraction() == 0) {
|
||||
security.mutable_contract_spec()->set_main_fraction(
|
||||
security.glbx().do_not_use_main_fraction());
|
||||
}
|
||||
if (security.contract_spec().sub_fraction() == 0) {
|
||||
security.mutable_contract_spec()->set_sub_fraction(
|
||||
security.glbx().do_not_use_sub_fraction());
|
||||
}
|
||||
if (security.contract_spec().price_display_format() == 0) {
|
||||
security.mutable_contract_spec()->set_price_display_format(
|
||||
security.glbx().do_not_use_price_display_format());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
size_t load_securities(
|
||||
std::function<bool(const UserSymbol &symbo)> const &sym_filter,
|
||||
const std::function<void(const api::Security &)> &callback) {
|
||||
@@ -104,7 +84,6 @@ size_t load_securities(
|
||||
if (sym_filter(us)) {
|
||||
auto blob = select_stmt.getColumn("security_blob");
|
||||
if (security.ParseFromArray(blob, blob.getBytes())) {
|
||||
apply_compatibilit_mutations(security);
|
||||
bts_assert(security_key == security.id());
|
||||
count++;
|
||||
callback(security);
|
||||
@@ -146,7 +125,6 @@ SecurityPtr find_security(SecurityKey key) {
|
||||
auto result = std::make_shared<Security>();
|
||||
auto blob = select.getColumn(0);
|
||||
if (result->ParseFromArray(blob, blob.getBytes())) {
|
||||
apply_compatibilit_mutations(*result);
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
+148
-161
@@ -1,18 +1,25 @@
|
||||
#include "lib/domain_model/Security.h"
|
||||
#ifndef _WIN32
|
||||
#include "SecurityIDRecorder.h"
|
||||
#include "lib/domain_model/Security.h"
|
||||
#include "lib/persistence/IStore.h"
|
||||
#include "lib/persistence/SqliteStore.h"
|
||||
#include <boost/pool/object_pool.hpp>
|
||||
|
||||
#include "lib/config/config.h"
|
||||
#include "lib/logging/logging.h"
|
||||
#include "lib/utils/Sqlite_utils.h"
|
||||
#include "lib/utils/checked_arithmetic.h"
|
||||
#include <boost/filesystem.hpp>
|
||||
#include <tbb/concurrent_queue.h>
|
||||
|
||||
#include "lib/config/protos/market_data.pb.h"
|
||||
#include "lib/config/protos/system_config.pb.h"
|
||||
|
||||
#include <memory>
|
||||
#include <sqlite/sqlite3.h>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <tbb/concurrent_queue.h>
|
||||
|
||||
using namespace std;
|
||||
using namespace bts::utils;
|
||||
@@ -21,22 +28,55 @@ using std::size_t;
|
||||
namespace bts {
|
||||
namespace md {
|
||||
|
||||
using bts::api::Security;
|
||||
using bts::api::SecurityType;
|
||||
using bts::api::Venue;
|
||||
using bts::config::SystemConfig;
|
||||
using bts::persistence::IStore;
|
||||
using bts::persistence::SqliteStore;
|
||||
|
||||
uint64_t exchange_id(const api::Security &security);
|
||||
|
||||
struct SecurityIDEntry {
|
||||
api::Security security;
|
||||
SecurityIDRecorder::Callback callback;
|
||||
static std::unique_ptr<IStore> get_default_store() {
|
||||
return std::make_unique<SqliteStore>("key_ids");
|
||||
}
|
||||
|
||||
static persistence::StoreRow get_row(
|
||||
Venue venue,
|
||||
SecurityType type,
|
||||
const std::string &name,
|
||||
const std::string &prefix = "") {
|
||||
return persistence::StoreRow{
|
||||
fmt::format(
|
||||
"{}{}.{}", std::string{prefix}, Venue_Name(venue).substr(6),
|
||||
SecurityType_Name(type).substr(13)),
|
||||
name, ""};
|
||||
}
|
||||
|
||||
static persistence::StoreRow get_row(
|
||||
const api::Security &s, const std::string &prefix = "") {
|
||||
return get_row(s.exchange(), bts::securities::security_type(s), prefix);
|
||||
}
|
||||
|
||||
struct SecurityIdRequest {
|
||||
Venue venue;
|
||||
SecurityType type;
|
||||
std::string name;
|
||||
std::function<void(Venue, SecurityType, const std::string &, std::uint32_t)>
|
||||
callback;
|
||||
};
|
||||
|
||||
struct SecurityIDRecorderState {
|
||||
SecurityIDRecorderState(SecurityIDRecorderParams &p)
|
||||
: work(), pending_handle(), thread_id(), db(nullptr, sqlite3_close),
|
||||
timer(), lookup_key(p.lookup_key), unresolved_legs() {}
|
||||
SecurityIDRecorderState(
|
||||
std::unique_ptr<IStore> store, KeyLookupOption lookup_key)
|
||||
: store(store == nullptr ? get_default_store() : std::move(store)),
|
||||
pending_handle(), thread_id(), timer(), lookup_key(lookup_key),
|
||||
unresolved_legs() {}
|
||||
|
||||
tbb::concurrent_queue<std::shared_ptr<SecurityIDEntry>> work;
|
||||
std::unique_ptr<IStore> store{nullptr};
|
||||
tbb::concurrent_queue<std::shared_ptr<SecurityIdRequest>> id_request_queue;
|
||||
tbb::concurrent_queue<MDProducerHandle> pending_handle;
|
||||
std::thread::id thread_id;
|
||||
sqlite3_ptr db;
|
||||
unique_ptr<DispatcherTimer> timer;
|
||||
|
||||
KeyLookupOption lookup_key;
|
||||
@@ -59,51 +99,41 @@ struct SecurityIDRecorderState {
|
||||
void work_handles();
|
||||
void work_db();
|
||||
|
||||
void check_access() { bts_assert(thread_id == std::this_thread::get_id()); }
|
||||
void assign_key(
|
||||
const api::Security &security,
|
||||
const SecurityIDRecorder::Callback &key_assigned);
|
||||
void assign_key(
|
||||
api::Venue venue,
|
||||
api::SecurityType type,
|
||||
const std::string &name,
|
||||
std::function<void(
|
||||
api::Venue, api::SecurityType, const std::string &, std::uint32_t)>
|
||||
key_assigned);
|
||||
|
||||
void check_access() { bts_assert(thread_id == std::this_thread::get_id()); }
|
||||
void add_unresolved_spread(MDProducerHandle &handle);
|
||||
security_key search(const std::string &descriptor);
|
||||
security_key insert(const std::string &descriptor, SecurityType);
|
||||
security_key search(const persistence::StoreRow &row);
|
||||
security_key insert(const persistence::StoreRow &row);
|
||||
};
|
||||
|
||||
SecurityIDRecorder::SecurityIDRecorder(SecurityIDRecorderParams params)
|
||||
: _self(new SecurityIDRecorderState(params)) {
|
||||
if (params.db) {
|
||||
_self->db = std::move(params.db);
|
||||
}
|
||||
SecurityIDRecorder::SecurityIDRecorder(
|
||||
std::unique_ptr<persistence::IStore> store,
|
||||
KeyLookupOption lookup_key,
|
||||
utils::Dispatcher *dispatcher)
|
||||
: _self(std::make_unique<SecurityIDRecorderState>(
|
||||
store == nullptr
|
||||
? std::make_unique<persistence::SqliteStore>("key_ids")
|
||||
: std::move(store),
|
||||
lookup_key)) {
|
||||
init();
|
||||
if (params.dispatcher) {
|
||||
_self->run(*params.dispatcher);
|
||||
if (dispatcher != nullptr) {
|
||||
_self->run(*dispatcher);
|
||||
}
|
||||
}
|
||||
|
||||
SecurityIDRecorder::~SecurityIDRecorder() {}
|
||||
|
||||
void SecurityIDRecorder::init() {
|
||||
LOG(md, Info, "SecurityIDRecorder::init");
|
||||
string db_dir =
|
||||
config::get<config::SystemConfig>().data_path() + "databases/md/";
|
||||
string db_path = db_dir + "/security_keys.sqlite";
|
||||
|
||||
if (!boost::filesystem::exists(db_dir)) {
|
||||
boost::filesystem::create_directories(db_dir);
|
||||
}
|
||||
|
||||
if (!_self->db) {
|
||||
_self->db = sqlite::open_db(db_path.c_str());
|
||||
bts_assert(_self->db);
|
||||
}
|
||||
|
||||
// Set a long busy timeout
|
||||
sqlite3_busy_timeout(_self->db.get(), 9999);
|
||||
sqlite::exec_nofail(_self->db, "PRAGMA journal_mode=WAL");
|
||||
sqlite::exec_nofail(_self->db, "PRAGMA syncronous=0");
|
||||
sqlite::exec_nofail(
|
||||
_self->db, "CREATE TABLE IF NOT EXISTS security_keys (descriptor STRING "
|
||||
"NOT NULL PRIMARY KEY, type int);");
|
||||
}
|
||||
void SecurityIDRecorder::init() {}
|
||||
|
||||
void SecurityIDRecorder::ensure_key(MDProducerHandle handle) {
|
||||
LOG(md, Info, "SecurityIDRecorder::ensure_key desc: {} key: {} num_id: {}",
|
||||
@@ -113,37 +143,42 @@ void SecurityIDRecorder::ensure_key(MDProducerHandle handle) {
|
||||
}
|
||||
|
||||
void SecurityIDRecorder::assign_key(
|
||||
const api::Security &security,
|
||||
const SecurityIDRecorder::Callback &callback) {
|
||||
LOG(md, Info, "SecurityIDRecorder::assign_key sec: {}",
|
||||
security.DebugString());
|
||||
if (security.Contract_case() == Security::ContractCase::kSpread) {
|
||||
for (const auto &leg : security.spread().legs()) {
|
||||
if (leg.security().id() == invalid_security_key) {
|
||||
LOG(md, Error, "Security {} has legs which will never be resolved.",
|
||||
security.has_glbx() ? security.glbx().security_desc()
|
||||
: security.symbol());
|
||||
// No point in trying to add this to the queue
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
_self->assign_key(security, callback);
|
||||
const api::Security &security, const Callback &callback) {
|
||||
assign_key(
|
||||
security.exchange(), bts::securities::security_type(security),
|
||||
bts::securities::descriptor(security),
|
||||
[callback, msec = security](
|
||||
api::Venue, api::SecurityType, const std::string &,
|
||||
uint32_t id) mutable {
|
||||
msec.set_id(id);
|
||||
callback(msec);
|
||||
});
|
||||
}
|
||||
|
||||
void SecurityIDRecorderState::assign_key(
|
||||
const api::Security &security,
|
||||
const SecurityIDRecorder::Callback &callback) {
|
||||
LOG(md, Info, "SecurityIDRecorder::assign_key sec: {}",
|
||||
bts::securities::descriptor(security));
|
||||
if (security.id() == invalid_security_key) {
|
||||
auto work = std::make_shared<SecurityIDEntry>();
|
||||
work->security = security;
|
||||
work->callback = callback;
|
||||
LOG(md, Info, "SecurityIDRecorderState::assign_key work.push d: {}",
|
||||
bts::securities::descriptor(work->security));
|
||||
this->work.push(work);
|
||||
}
|
||||
api::Venue venue,
|
||||
api::SecurityType type,
|
||||
const std::string &name,
|
||||
std::function<
|
||||
void(api::Venue, api::SecurityType, const std::string &, std::uint32_t)>
|
||||
callback) {
|
||||
LOG(md, Info, "SecurityIDRecorder::assign_key {} {} {}", venue, type, name);
|
||||
auto id_req = std::make_shared<SecurityIdRequest>();
|
||||
id_req->venue = venue;
|
||||
id_req->type = type;
|
||||
id_req->name = name;
|
||||
id_req->callback = callback;
|
||||
id_request_queue.push(id_req);
|
||||
}
|
||||
|
||||
void SecurityIDRecorder::assign_key(
|
||||
api::Venue venue,
|
||||
api::SecurityType type,
|
||||
const std::string &name,
|
||||
std::function<
|
||||
void(api::Venue, api::SecurityType, const std::string &, std::uint32_t)>
|
||||
callback) {
|
||||
_self->assign_key(venue, type, name, std::move(callback));
|
||||
}
|
||||
|
||||
void SecurityIDRecorder::do_work() { _self->do_work(); }
|
||||
@@ -220,9 +255,12 @@ void SecurityIDRecorderState::work_handles() {
|
||||
retry.push_back(handle);
|
||||
} else {
|
||||
convert_security(handle.security(), &security);
|
||||
assign_key(security, [handle](const api::Security &security) mutable {
|
||||
handle.set_key(security.id());
|
||||
});
|
||||
assign_key(
|
||||
security.exchange(), bts::securities::security_type(security),
|
||||
bts::securities::descriptor(security),
|
||||
[handle](
|
||||
api::Venue, api::SecurityType, const std::string &,
|
||||
uint32_t id) mutable { handle.set_key(id); });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -239,33 +277,30 @@ void SecurityIDRecorderState::work_db() {
|
||||
// Then persist that list
|
||||
// Once a security has a key it is public so record it in the security
|
||||
// database
|
||||
std::shared_ptr<SecurityIDEntry> temp;
|
||||
vector<std::shared_ptr<SecurityIDEntry>> new_entries;
|
||||
std::shared_ptr<SecurityIdRequest> temp;
|
||||
vector<std::pair<uint32_t, std::shared_ptr<SecurityIdRequest>>> new_entries;
|
||||
// SPARK-1446 Need to try to add existing securities to the db in case we
|
||||
// deleted them by hand
|
||||
vector<std::shared_ptr<SecurityIDEntry>> existing_securities;
|
||||
vector<std::shared_ptr<SecurityIDEntry>> defer;
|
||||
vector<string> new_descriptors;
|
||||
vector<std::shared_ptr<SecurityIdRequest>> existing_securities;
|
||||
vector<persistence::StoreRow> new_rows;
|
||||
size_t count = 0;
|
||||
string descriptor;
|
||||
while (work.try_pop(temp)) {
|
||||
descriptor = security_descriptor(temp->security);
|
||||
|
||||
// If we're unable to create a descriptor try again later
|
||||
if (descriptor == "") {
|
||||
defer.push_back(std::move(temp));
|
||||
continue;
|
||||
while (id_request_queue.try_pop(temp)) {
|
||||
security_key key = invalid_security_key;
|
||||
auto row = get_row(temp->venue, temp->type, temp->name);
|
||||
try {
|
||||
key = search(row);
|
||||
} catch (const std::overflow_error &ex) {
|
||||
LOG(persist, Error,
|
||||
"SecurityIDRecorderState::work_db. Row search threw ex: {}",
|
||||
std::string{ex.what()});
|
||||
}
|
||||
|
||||
security_key key = search(descriptor);
|
||||
if (key != invalid_security_key) {
|
||||
temp->security.set_id(key);
|
||||
temp->callback(temp->security);
|
||||
temp->callback(temp->venue, temp->type, temp->name, key);
|
||||
existing_securities.push_back(std::move(temp));
|
||||
} else {
|
||||
// always push in parallel
|
||||
new_entries.push_back(std::move(temp));
|
||||
new_descriptors.push_back(descriptor);
|
||||
new_entries.push_back(std::make_pair(key, std::move(temp)));
|
||||
new_rows.push_back(std::move(row));
|
||||
++count;
|
||||
// Limit the amount of time we spend writing to sqlite during startup
|
||||
// to avoid blocking the heartbeat
|
||||
@@ -275,95 +310,47 @@ void SecurityIDRecorderState::work_db() {
|
||||
}
|
||||
}
|
||||
|
||||
for (auto &d : defer) {
|
||||
LOG(md, Info, "SecurityIDRecorderState::work_db work.push d: {}",
|
||||
bts::securities::descriptor(d->security));
|
||||
work.push(std::move(d));
|
||||
}
|
||||
|
||||
if (!new_entries.empty()) {
|
||||
sqlite::exec_nofail(db.get(), "BEGIN TRANSACTION");
|
||||
|
||||
store->insert_many_resolve_ids(new_rows);
|
||||
for (size_t x = 0; x < new_entries.size(); ++x) {
|
||||
Security &security = new_entries[x]->security;
|
||||
security_key key =
|
||||
insert(new_descriptors[x], securitytype_of(security));
|
||||
bts_assert_d(key);
|
||||
security.set_id(key);
|
||||
auto &entry = new_entries[x];
|
||||
auto &row = new_rows[x];
|
||||
entry.first = row.id;
|
||||
bts_assert_d(entry.first != invalid_security_key);
|
||||
}
|
||||
|
||||
sqlite::exec_nofail(db.get(), "COMMIT TRANSACTION");
|
||||
}
|
||||
|
||||
if (!new_entries.empty()) {
|
||||
for (size_t x = 0; x < new_entries.size(); ++x) {
|
||||
Security &security = new_entries[x]->security;
|
||||
new_entries[x]->callback(security);
|
||||
auto &[id, req] = new_entries[x];
|
||||
req->callback(req->venue, req->type, req->name, id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void SecurityIDRecorder::bulk_insert(
|
||||
const std::vector<const api::Security *> &data, const char *prefix) {
|
||||
LOG(md, Info, "SecurityIDRecorder::bulk_insert size: {}", data.size());
|
||||
if (data.size() == 0)
|
||||
return;
|
||||
sqlite::exec_nofail(_self->db.get(), "BEGIN TRANSACTION");
|
||||
std::string descriptor;
|
||||
{
|
||||
auto insert_stmt = sqlite::prepare_query(
|
||||
_self->db, "INSERT INTO security_keys (rowid, descriptor, type) "
|
||||
"values (?, ?, ?)");
|
||||
for (const api::Security *sec : data) {
|
||||
descriptor = prefix + security_descriptor(*sec);
|
||||
auto type = securitytype_of(*sec);
|
||||
|
||||
sqlite3_bind_int(insert_stmt.get(), 1, sec->id());
|
||||
sqlite3_bind_text(
|
||||
insert_stmt.get(), 2, descriptor.c_str(), descriptor.size(),
|
||||
SQLITE_STATIC);
|
||||
sqlite3_bind_int(insert_stmt.get(), 3, (int)type);
|
||||
sqlite3_step(insert_stmt.get());
|
||||
sqlite3_reset(insert_stmt.get());
|
||||
}
|
||||
std::vector<persistence::StoreRow> rows;
|
||||
rows.reserve(data.size());
|
||||
for (const auto *s : data) {
|
||||
rows.push_back(get_row(*s, prefix));
|
||||
}
|
||||
sqlite::exec_nofail(_self->db.get(), "COMMIT TRANSACTION");
|
||||
_self->store->insert_many_resolve_ids(rows);
|
||||
}
|
||||
|
||||
security_key SecurityIDRecorderState::search(const std::string &descriptor) {
|
||||
LOG(md, Info, "SecurityIDRecorderState::search descriptor: {}", descriptor);
|
||||
auto select_stmt = sqlite::prepare_query(
|
||||
db, "SELECT rowid from security_keys where descriptor=? LIMIT 1;");
|
||||
sqlite3_bind_text(
|
||||
select_stmt.get(), 1, descriptor.c_str(), descriptor.size(),
|
||||
SQLITE_STATIC);
|
||||
int result = sqlite3_step(select_stmt.get());
|
||||
if (result == SQLITE_DONE || result == SQLITE_ROW) {
|
||||
return sqlite3_column_int(select_stmt.get(), 0);
|
||||
security_key SecurityIDRecorderState::search(const persistence::StoreRow &row) {
|
||||
auto [exists, rowid, val] = store->retrieve_one(row.ns, row.key);
|
||||
if (exists) {
|
||||
if (!val.empty()) {
|
||||
bts_assert(stol(val) == rowid);
|
||||
}
|
||||
return system::math::checked_ltou(rowid);
|
||||
}
|
||||
return invalid_security_key;
|
||||
}
|
||||
|
||||
security_key SecurityIDRecorderState::insert(
|
||||
const std::string &descriptor, SecurityType type) {
|
||||
LOG(md, Info, "SecurityIDRecorderState::insert descriptor: {}", descriptor);
|
||||
auto insert_stmt = sqlite::prepare_query(
|
||||
db, "INSERT INTO security_keys (descriptor, type) values (?, ?)");
|
||||
sqlite3_bind_text(
|
||||
insert_stmt.get(), 1, descriptor.c_str(), descriptor.size(),
|
||||
SQLITE_STATIC);
|
||||
sqlite3_bind_int(insert_stmt.get(), 2, (int)type);
|
||||
int result = sqlite3_step(insert_stmt.get());
|
||||
if (result == SQLITE_DONE || result == SQLITE_ROW) {
|
||||
return sqlite3_last_insert_rowid(db.get());
|
||||
}
|
||||
// Just in case our insert failed due to duplicate row
|
||||
auto key = search(descriptor);
|
||||
if (key == invalid_security_key) {
|
||||
LOG(md, Error, "Failed to assign security key. SQLite return code: {}",
|
||||
result);
|
||||
}
|
||||
return key;
|
||||
security_key SecurityIDRecorderState::insert(const persistence::StoreRow &row) {
|
||||
return store->insert(row.ns, row.key, row.value);
|
||||
}
|
||||
|
||||
} // namespace md
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
#ifndef _WIN32
|
||||
#include "MDProducerHandle.h"
|
||||
#include "common.h"
|
||||
|
||||
#include "lib/persistence/IStore.h"
|
||||
#include "lib/utils/Sqlite_utils.h"
|
||||
#include <atomic>
|
||||
#include <functional>
|
||||
@@ -22,26 +22,15 @@ namespace md {
|
||||
|
||||
typedef const std::function<SecurityKey(ExchangeKey)> KeyLookup;
|
||||
typedef std::optional<KeyLookup> KeyLookupOption;
|
||||
struct SecurityIDRecorderParams {
|
||||
SecurityIDRecorderParams()
|
||||
: dispatcher(), db(sqlite::null_db()), lookup_key() {}
|
||||
SecurityIDRecorderParams(utils::Dispatcher &d)
|
||||
: dispatcher(&d), db(sqlite::null_db()), lookup_key() {}
|
||||
SecurityIDRecorderParams(utils::Dispatcher &d, KeyLookup &get_key)
|
||||
: dispatcher(&d), db(sqlite::null_db()), lookup_key(get_key) {}
|
||||
SecurityIDRecorderParams(sqlite3_ptr &db)
|
||||
: dispatcher(), db(std::move(db)), lookup_key() {}
|
||||
SecurityIDRecorderParams(sqlite3_ptr &db, KeyLookup &get_key)
|
||||
: dispatcher(), db(std::move(db)), lookup_key(get_key) {}
|
||||
utils::Dispatcher *dispatcher;
|
||||
sqlite3_ptr db;
|
||||
KeyLookupOption lookup_key;
|
||||
};
|
||||
|
||||
struct SecurityIDRecorderState;
|
||||
|
||||
class SecurityIDRecorder {
|
||||
public:
|
||||
SecurityIDRecorder(SecurityIDRecorderParams);
|
||||
SecurityIDRecorder(
|
||||
std::unique_ptr<persistence::IStore> store = nullptr,
|
||||
KeyLookupOption lookup_key = std::nullopt,
|
||||
utils::Dispatcher *dispatcher = nullptr);
|
||||
~SecurityIDRecorder();
|
||||
typedef std::function<void(const api::Security &)> Callback;
|
||||
|
||||
@@ -49,6 +38,13 @@ class SecurityIDRecorder {
|
||||
void ensure_key(MDProducerHandle handle);
|
||||
// Slow (copies security) - for use by services
|
||||
void assign_key(const api::Security &security, const Callback &key_assigned);
|
||||
void assign_key(
|
||||
api::Venue venue,
|
||||
api::SecurityType type,
|
||||
const std::string &name,
|
||||
std::function<void(
|
||||
api::Venue, api::SecurityType, const std::string &, std::uint32_t)>
|
||||
key_assigned);
|
||||
|
||||
// If a dispatcher is not provided you must call work periodically to do
|
||||
// lookups and flush the reserved ids to disk.
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
#include "lib/domain_model/Security.pb.h"
|
||||
#include "lib/domain_model/Security.h"
|
||||
#include "lib/utils/bts_assert.h"
|
||||
#ifndef NO_CPP0X
|
||||
|
||||
@@ -59,81 +59,6 @@ FixedSpreadLeg::FixedSpreadLeg()
|
||||
: key(invalid_security_key), leg_side(Side_MIN), leg_quantity(0),
|
||||
leg_price(0), leg_delta(1) {}
|
||||
|
||||
void write_date_string(
|
||||
uint32_t year, uint32_t month, uint32_t day, std::ostringstream &result) {
|
||||
result << std::setw(2) << std::setfill('0') << month << "/" << std::setw(2)
|
||||
<< std::setfill('0') << day << "/" << year - 2000;
|
||||
}
|
||||
|
||||
void write_leg_descriptor(
|
||||
std::ostringstream &result, char separator, const api::ContractLeg &leg) {
|
||||
result << separator << leg.security().id() << "-q" << leg.quantity() << "p"
|
||||
<< leg.price_multiplier() << "d" << leg.leg_delta()
|
||||
<< (leg.side() == SIDE_BUY ? "BUY" : "SELL");
|
||||
}
|
||||
|
||||
std::string security_descriptor(const api::Security &security) {
|
||||
const char separator = ' ';
|
||||
std::ostringstream result;
|
||||
if (security.exchange() == Venue::VENUE_ICE ||
|
||||
security.exchange() == Venue::VENUE_SIMICE)
|
||||
result << security.ice().security_desc() << separator
|
||||
<< Venue_Name(security.exchange());
|
||||
else
|
||||
result << security.symbol() << separator
|
||||
<< Venue_Name(security.exchange());
|
||||
switch (security.Contract_case()) {
|
||||
case Security::ContractCase::kOption: {
|
||||
result << separator;
|
||||
const auto &o = security.option();
|
||||
write_date_string(
|
||||
o.expiration_year(), o.expiration_month(), o.expiration_day(),
|
||||
result);
|
||||
result << separator << o.strike()
|
||||
<< (o.call_put() == PUTORCALL_CALL ? 'C' : 'P');
|
||||
break;
|
||||
}
|
||||
case Security::ContractCase::kFuture: {
|
||||
result << separator;
|
||||
const auto &f = security.future();
|
||||
write_date_string(
|
||||
f.expiration_year(), f.expiration_month(), f.expiration_day(),
|
||||
result);
|
||||
break;
|
||||
}
|
||||
case Security::ContractCase::kSpot: {
|
||||
break;
|
||||
}
|
||||
case Security::ContractCase::kSpread: {
|
||||
// If the security key of a leg is invalid return an error
|
||||
const auto &s = security.spread();
|
||||
bts_assert(s.legs_size() > 0);
|
||||
for (const auto &leg : s.legs()) {
|
||||
if (leg.security().id() == invalid_security_key)
|
||||
return "";
|
||||
write_leg_descriptor(result, separator, leg);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case Security::ContractCase::kSynthetic: {
|
||||
const auto &s = security.synthetic();
|
||||
bts_assert(s.legs_size() > 0);
|
||||
result << separator << s.name()
|
||||
<< " pc=" << SyntheticContract::PriceCalc_Name(s.price_calc());
|
||||
for (const auto &leg : s.legs()) {
|
||||
if (leg.security().id() == invalid_security_key)
|
||||
return ""; // unlikey with synthetics
|
||||
write_leg_descriptor(result, separator, leg);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case Security::ContractCase::CONTRACT_NOT_SET:
|
||||
bts_assert(
|
||||
security.Contract_case() != Security::ContractCase::CONTRACT_NOT_SET);
|
||||
}
|
||||
return result.str();
|
||||
}
|
||||
|
||||
void get_contract_spec(
|
||||
const FixedSecurity &source, /*out*/ api::ContractSpecification &spec) {
|
||||
spec.set_price_break(source.price_break);
|
||||
@@ -292,26 +217,12 @@ void convert_security(const FixedSecurity &source, Security *presult) {
|
||||
get_contract_spec(source, *dest.mutable_contract_spec());
|
||||
}
|
||||
|
||||
SecurityType securitytype_of_contractcase(Security::ContractCase cc) {
|
||||
switch (cc) {
|
||||
case Security::ContractCase::kFuture:
|
||||
return SECURITYTYPE_FUTURE;
|
||||
case Security::ContractCase::kOption:
|
||||
return SECURITYTYPE_OPTION;
|
||||
case Security::ContractCase::kSpot:
|
||||
return api::SECURITYTYPE_SPOT;
|
||||
case Security::ContractCase::kSpread:
|
||||
return SECURITYTYPE_SPREAD;
|
||||
case Security::ContractCase::kSynthetic:
|
||||
return SECURITYTYPE_SYNTHETIC;
|
||||
case Security::ContractCase::CONTRACT_NOT_SET:
|
||||
bts_assert(cc != Security::ContractCase::CONTRACT_NOT_SET);
|
||||
}
|
||||
bts_assert(false);
|
||||
}
|
||||
SecurityType securitytype_of(const Security &s) {
|
||||
return securitytype_of_contractcase(s.Contract_case());
|
||||
auto type = bts::securities::security_type(s.Contract_case());
|
||||
bts_assert(type != api::SecurityType::SECURITYTYPE_UNSET);
|
||||
return type;
|
||||
}
|
||||
|
||||
size_t num_legs(const Security &sec) {
|
||||
if (sec.Contract_case() == api::Security::ContractCase::kSpread) {
|
||||
return sec.spread().legs_size();
|
||||
@@ -359,7 +270,7 @@ uint64_t exchange_id(const Security &security) {
|
||||
void convert_security(
|
||||
const Security &src, FixedSecurity &dest, bool set_security_key) {
|
||||
dest.key = set_security_key ? src.id() : invalid_security_key;
|
||||
dest.type = securitytype_of_contractcase(src.Contract_case());
|
||||
dest.type = bts::securities::security_type(src.Contract_case());
|
||||
dest.symbol(src.symbol());
|
||||
dest.currency(src.currency());
|
||||
dest.exchange = src.exchange();
|
||||
@@ -374,8 +285,8 @@ void convert_security(
|
||||
dest.cme_market_segment_id = src.glbx().market_segment_id();
|
||||
break;
|
||||
case Security::ExchangeDataCase::kCboe:
|
||||
dest.numeric_id = src.deribit().exchange_id();
|
||||
dest.security_desc(src.deribit().security_desc());
|
||||
dest.numeric_id = src.cboe().exchange_id();
|
||||
dest.security_desc(src.cboe().security_desc());
|
||||
dest.cboe_class_key = src.cboe().class_key();
|
||||
break;
|
||||
case Security::ExchangeDataCase::kTaifex:
|
||||
|
||||
@@ -208,8 +208,6 @@ typedef FixedSpreadSecurity<16> FixedSpreadSecurityS;
|
||||
typedef FixedSpreadSecurity<128> FixedSpreadSecurityL;
|
||||
typedef FixedSpreadSecurity<512> FixedSpreadSecurityXL;
|
||||
|
||||
std::string security_descriptor(const api::Security &);
|
||||
|
||||
// Convert a fixedsecurity to a normal Security - caveat: spreads will be
|
||||
// normalized
|
||||
Security convert_security(const FixedSecurity &);
|
||||
@@ -217,10 +215,9 @@ void convert_security(const FixedSecurity &, Security *);
|
||||
void convert_security(
|
||||
const Security &src, FixedSecurity &dest, bool set_security_key = false);
|
||||
|
||||
SecurityType securitytype_of_contractcase(Security::ContractCase cc);
|
||||
SecurityType securitytype_of(const Security &);
|
||||
size_t num_legs(const Security &);
|
||||
uint64_t exchange_id(const Security &security);
|
||||
SecurityType securitytype_of(const api::Security &);
|
||||
size_t num_legs(const api::Security &);
|
||||
uint64_t exchange_id(const api::Security &security);
|
||||
|
||||
void to_json(nlohmann::json &j, const FixedSpreadLeg &f);
|
||||
|
||||
|
||||
@@ -35,8 +35,9 @@ typedef SyntheticSecurityServiceBase::MetadataStatefulPublisher
|
||||
|
||||
struct SyntheticSecurityService::Pimpl {
|
||||
Pimpl(const MetadataStatefulPublisher &fn)
|
||||
: recorder(utils::Dispatcher::background()), synthetics(), on_security(),
|
||||
meta(), on_metadata(), metadata_publisher(fn) {}
|
||||
: recorder(nullptr, std::nullopt, &utils::Dispatcher::background()),
|
||||
synthetics(), on_security(), meta(), on_metadata(),
|
||||
metadata_publisher(fn) {}
|
||||
SecurityIDRecorder recorder;
|
||||
std::unordered_map<SecurityKey, Security> synthetics;
|
||||
utils::Event<Security> on_security;
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
#pragma once
|
||||
|
||||
#include "lib/os/spark_data_directory.h"
|
||||
|
||||
#include <boost/filesystem/operations.hpp>
|
||||
#include <boost/filesystem/path.hpp>
|
||||
|
||||
namespace bts::md {
|
||||
|
||||
struct Mdp3DataDirectory {
|
||||
boost::filesystem::path log_dir() const { return root.log_dir() / "onix"; }
|
||||
|
||||
boost::filesystem::path data_dir() const { return root.root() / "cme_mdp3"; }
|
||||
|
||||
boost::filesystem::path config_file() const {
|
||||
return data_dir() / "config.xml";
|
||||
}
|
||||
|
||||
boost::filesystem::path templates_file() const {
|
||||
return data_dir() / "templates.xml";
|
||||
}
|
||||
|
||||
void create_dirs() {
|
||||
boost::filesystem::create_directories(data_dir());
|
||||
boost::filesystem::create_directories(log_dir());
|
||||
}
|
||||
|
||||
private:
|
||||
spark::SparkDataDirectory root{};
|
||||
};
|
||||
|
||||
} // namespace bts::md
|
||||
@@ -9,6 +9,7 @@
|
||||
#include "lib/config/config.h"
|
||||
#include "lib/metrics/metrics.h"
|
||||
#include "lib/os/os.h"
|
||||
#include "lib/persistence/SqliteStore.h"
|
||||
#include "lib/utils/Dispatcher.h"
|
||||
#include "lib/utils/Shuffle.h"
|
||||
#include "lib/utils/bts_chrono.h"
|
||||
@@ -160,12 +161,10 @@ class Co2Tests {
|
||||
#ifndef _WIN32
|
||||
void test_security_id_recorder(const std::string &db_name) {
|
||||
// Verify callbacks happen after work complete
|
||||
auto db1 = sqlite::open_db(db_name.c_str());
|
||||
REQUIRE(db1);
|
||||
auto db2 = sqlite::open_db(db_name.c_str());
|
||||
REQUIRE(db2);
|
||||
|
||||
SecurityIDRecorder idsvc{{db1}}, idsvc2{{db2}};
|
||||
SecurityIDRecorder idsvc{
|
||||
std::make_unique<persistence::SqliteStore>(db_name)};
|
||||
SecurityIDRecorder idsvc2{
|
||||
std::make_unique<persistence::SqliteStore>(db_name)};
|
||||
security_key result = invalid_security_key;
|
||||
Security s1 = mkoption("FOOX", invalid_security_key, 2018, 1, 20, 1850);
|
||||
Security f1 = mkfuture("FOO", invalid_security_key, 2018, 1, 20);
|
||||
@@ -215,18 +214,18 @@ class Co2Tests {
|
||||
std::make_unique<Co2TestInstrument>(s1, cold_header, hot_header);
|
||||
|
||||
// Add spread to SecurityIDRecorder and work it
|
||||
auto db = sqlite::open_db(":memory:");
|
||||
auto db = std::make_unique<persistence::SqliteStore>(":memory:");
|
||||
REQUIRE(db);
|
||||
|
||||
SecurityIDRecorder idsvc{
|
||||
{db, [&](ExchangeKey k) -> SecurityKey {
|
||||
for (const auto &fut :
|
||||
{f1handle->producer(), f2handle->producer()}) {
|
||||
if (k.value() == fut.security().numeric_id)
|
||||
return SecurityKey(fut.key());
|
||||
}
|
||||
return SecurityKey(invalid_security_key);
|
||||
}}};
|
||||
std::move(db), [&](ExchangeKey k) -> SecurityKey {
|
||||
for (const auto &fut :
|
||||
{f1handle->producer(), f2handle->producer()}) {
|
||||
if (k.value() == fut.security().numeric_id)
|
||||
return SecurityKey(fut.key());
|
||||
}
|
||||
return SecurityKey(invalid_security_key);
|
||||
}};
|
||||
|
||||
auto workwork = [&]() {
|
||||
idsvc.do_work();
|
||||
|
||||
@@ -4,9 +4,12 @@
|
||||
#include "lib/orderman/gateways/CCAPIExecutionGateway.h"
|
||||
// #include "lib/orderman/gateways/EMExecutionGateway.h"
|
||||
#include "lib/orderman/gateways/FakeExecutionGateway.h"
|
||||
#include "lib/orderman/gateways/GeneralExecutionGateway.h"
|
||||
#include "lib/orderman/gateways/GLBXExecutionGateway.h"
|
||||
#include "lib/orderman/gateways/GLBXiLink3ExecutionGateway.h"
|
||||
#include "lib/orderman/gateways/ICEExecutionGateway.h"
|
||||
#include "lib/orderman/gateways/IFIXGenericAdapter.h"
|
||||
#include "lib/orderman/gateways/OnixFIXAdapter.h"
|
||||
#include "lib/orderman/gateways/sim/SimExecutionGatewayFactory.h"
|
||||
#include "lib/orderman/gateways/sim/SimExecutionGatewayUniqueWrapper.h"
|
||||
#include "lib/tracing/PerfLog.h"
|
||||
@@ -21,23 +24,30 @@ std::unique_ptr<IExecutionGateway> ExecutionGatewayFactory::create_gateway(
|
||||
const config::JetFIX::Connection &conn,
|
||||
trace::PerfLog &pl,
|
||||
md::SecurityOracle o) {
|
||||
LOG(OrderMan, Info, "ExecutionGatewayFactory::create_gateway conn: {}",
|
||||
utils::pb::to_string(conn));
|
||||
if (SimExecutionGatewayFactory::is_sim_execution_on(conn.venue())) {
|
||||
return std::make_unique<SimExecutionGatewayUniqueWrapper>(conn.venue());
|
||||
}
|
||||
LOG(OrderMan, Info, "ExecutionGatewayFactory::create_gateway conn: {}",
|
||||
utils::pb::to_string(conn));
|
||||
if (conn.venue() == bts::api::VENUE_ICE ||
|
||||
conn.venue() == bts::api::VENUE_SIMICE)
|
||||
return std::make_unique<ICEExecutionGateway>(conn, pl, o);
|
||||
else if (conn.venue() == bts::api::VENUE_GLBX) {
|
||||
if (conn.use_ilink3())
|
||||
return std::make_unique<ilink3::GLBXiLink3ExecutionGateway>(
|
||||
conn, pl, o);
|
||||
return std::make_unique<GLBXExecutionGateway>(conn, pl, o);
|
||||
} else if (conn.venue() == bts::api::VENUE_DERIBIT)
|
||||
// return std::make_unique<EMExecutionGateway>(o);
|
||||
return std::make_unique<CCAPIExecutionGateway>(conn.venue(), o);
|
||||
return std::make_unique<FakeExecutionGateway>(o);
|
||||
switch(conn.venue()){
|
||||
case bts::api::VENUE_ICE:
|
||||
case bts::api::VENUE_SIMICE:
|
||||
return std::make_unique<ICEExecutionGateway>(conn, pl, o);
|
||||
case bts::api::VENUE_GLBX:{
|
||||
if (conn.use_ilink3())
|
||||
return std::make_unique<ilink3::GLBXiLink3ExecutionGateway>( conn, pl, o);
|
||||
return std::make_unique<GLBXExecutionGateway>(conn, pl, o);
|
||||
}
|
||||
case bts::api::VENUE_DERIBIT:
|
||||
return std::make_unique<CCAPIExecutionGateway>(conn.venue(), o);
|
||||
case bts::api::VENUE_CBOE:
|
||||
{
|
||||
std::unique_ptr<IFIXGenericAdapter> adapter = std::make_unique<OnixFIXAdapter>(o);
|
||||
return std::make_unique<GeneralExecutionGateway>(conn.venue(),std::move(adapter),o);
|
||||
}
|
||||
default:
|
||||
return std::make_unique<FakeExecutionGateway>(o);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace bts::orderman
|
||||
@@ -0,0 +1,86 @@
|
||||
#include "GeneralExecutionGateway.h"
|
||||
#include "lib/domain_model/Security.pb.h"
|
||||
#include "lib/logging/logging.h"
|
||||
#include "lib/orderman/CreateOrderMsg.h"
|
||||
#include "lib/orderman/OrderMsg.h"
|
||||
#include "lib/orderman/gateways/IExecutionGateway.h"
|
||||
#include <memory>
|
||||
#include <type_traits>
|
||||
|
||||
//Acts as a gateway, On One side is the ordermanager and on the other onixfix listener and onixfix adapter
|
||||
namespace bts::orderman {
|
||||
struct GeneralExecutionGateway::PImpl {
|
||||
PImpl(bts::api::Venue venue,
|
||||
std::unique_ptr<IFIXGenericAdapter> adapter,
|
||||
md::SecurityOracle o)
|
||||
: venue(venue), _adapter(std::move(adapter)), sec_oracle(o) {}
|
||||
bts::api::Venue venue;
|
||||
std::unique_ptr<IFIXGenericAdapter> _adapter;
|
||||
md::SecurityOracle sec_oracle;
|
||||
OnExeReport on_exe_report;
|
||||
OnGwStatusChange on_gw_status_change;
|
||||
OnCancelReject on_cancel_reject;
|
||||
OnExchangePositionState on_exchange_position_state;
|
||||
OnExchangeOpenOrdersState on_exchange_open_orders;
|
||||
};
|
||||
OnGwStatusChange &GeneralExecutionGateway::on_gw_status_change() {
|
||||
return me->on_gw_status_change;
|
||||
}
|
||||
// Modified constructor to accept the adapter from outside
|
||||
GeneralExecutionGateway::GeneralExecutionGateway(bts::api::Venue venue,
|
||||
std::unique_ptr<IFIXGenericAdapter> adapter, md::SecurityOracle o)
|
||||
: IExecutionGateway(), me(new PImpl(venue, std::move(adapter),o)) {
|
||||
LOG(OrderMan, Info, "ExecutionGateway::ExecutionGateway constructor");
|
||||
}
|
||||
GeneralExecutionGateway::~GeneralExecutionGateway() {}
|
||||
void GeneralExecutionGateway::start() {
|
||||
me->on_gw_status_change(true);
|
||||
me->_adapter->start();
|
||||
me->_adapter->register_callback(me->on_exe_report);
|
||||
LOG(OrderMan,Info,"ExecutionGateway::start started" );
|
||||
}
|
||||
void GeneralExecutionGateway::send_msgs(
|
||||
std::vector<OrderMsg> const &msgs, trace::CorID) {
|
||||
LOG(OrderMan,Info,"ExecutionGateway::send_msgs" );
|
||||
for (const auto &msg : msgs) {
|
||||
std::visit(
|
||||
[this](auto &&order) {
|
||||
using T = std::decay_t<decltype(order)>;
|
||||
if constexpr (std::is_same_v<T, CreateOrderMsg>) {
|
||||
LOG(OrderMan,Warn,"ExecutionGateway::CreateOrderMsg" );
|
||||
me->_adapter->launch_new_order(order);
|
||||
|
||||
} else if constexpr (std::is_same_v<T, CancelOrderMsg>) {
|
||||
LOG(OrderMan,Warn,"ExecutionGateway::CancelOrderMsg" );
|
||||
me->_adapter->cancel_order(order);
|
||||
} else if constexpr (std::is_same_v<T, CancelReplaceOrderMsg>) {
|
||||
LOG(OrderMan,Warn,"ExecutionGateway::CancelReplaceOrderMsg" );
|
||||
me->_adapter->modify_order(order);
|
||||
} else if constexpr (std::is_same_v<T, RFQMsg>) {
|
||||
// Handle RFQMsg if necessary
|
||||
// adapter->handle_rfq(order); // Example placeholder
|
||||
}
|
||||
},
|
||||
msg);
|
||||
}
|
||||
}
|
||||
void GeneralExecutionGateway::dispatch_incoming_fix_messages() {}
|
||||
void GeneralExecutionGateway::on_securities(
|
||||
const mercury::Update<NewSecuritiesEvent> &) {}
|
||||
void GeneralExecutionGateway::on_admin_data(
|
||||
const mercury::Update<NewAdminDataEvent> &) {}
|
||||
void GeneralExecutionGateway::send_fix(const jetfix::OutFIXPtr &, trace::CorID) {}
|
||||
OnExeReport &GeneralExecutionGateway::on_exe_report() { return me->on_exe_report; }
|
||||
OnCancelReject &GeneralExecutionGateway::on_cancel_reject() {
|
||||
static OnCancelReject default_cancel_reject;
|
||||
return default_cancel_reject;
|
||||
}
|
||||
OnExchangePositionState &GeneralExecutionGateway::on_exchange_position_state() {
|
||||
static OnExchangePositionState default_exchange_position_state;
|
||||
return default_exchange_position_state;
|
||||
}
|
||||
OnExchangeOpenOrdersState &GeneralExecutionGateway::on_exchange_open_orders() {
|
||||
static OnExchangeOpenOrdersState default_exchange_open_orders;
|
||||
return default_exchange_open_orders;
|
||||
}
|
||||
} //namespace bts::orderman
|
||||
@@ -0,0 +1,29 @@
|
||||
#pragma once
|
||||
#include "lib/orderman/gateways/IExecutionGateway.h"
|
||||
#include "lib/orderman/gateways/IFIXGenericAdapter.h"
|
||||
namespace bts {
|
||||
namespace orderman {
|
||||
class GeneralExecutionGateway : public IExecutionGateway {
|
||||
public:
|
||||
GeneralExecutionGateway(bts::api::Venue venue,std::unique_ptr<IFIXGenericAdapter> adapter,
|
||||
md::SecurityOracle o);
|
||||
virtual ~GeneralExecutionGateway();
|
||||
void start() override;
|
||||
void send_msgs(std::vector<OrderMsg> const & msgs, trace::CorID) override;
|
||||
void dispatch_incoming_fix_messages() override;
|
||||
void on_securities(const mercury::Update<NewSecuritiesEvent> &) override;
|
||||
void on_admin_data(const mercury::Update<NewAdminDataEvent> &) override;
|
||||
void send_fix(const jetfix::OutFIXPtr &, trace::CorID) override;
|
||||
OnExeReport &on_exe_report() override;
|
||||
OnGwStatusChange &on_gw_status_change() override;
|
||||
OnCancelReject &on_cancel_reject() override;
|
||||
OnExchangePositionState &on_exchange_position_state() override;
|
||||
OnExchangeOpenOrdersState &on_exchange_open_orders() override;
|
||||
private:
|
||||
struct PImpl;
|
||||
std::unique_ptr<PImpl> me;
|
||||
|
||||
|
||||
};
|
||||
} //namespace bts
|
||||
} //namespace orderman
|
||||
@@ -1,3 +1,6 @@
|
||||
#include "IFIXGenericAdapter.h"
|
||||
|
||||
namespace bts::orderman {}
|
||||
namespace bts::orderman {
|
||||
|
||||
IFIXGenericAdapter::~IFIXGenericAdapter(){}
|
||||
}
|
||||
@@ -1,22 +1,13 @@
|
||||
#pragma once
|
||||
|
||||
#include "lib/api/services/OrderService.pb.h"
|
||||
#include "lib/config/protos/jetfix.pb.h"
|
||||
#include "lib/domain_model/Security.pb.h"
|
||||
#include "lib/jetfix/codec/MessageDefinitions.h"
|
||||
#include "lib/jetfix/common/OutFIXPtr.h"
|
||||
#include "lib/md/SecurityOracle.h"
|
||||
#include "lib/mercury/events/Update.h"
|
||||
#include "lib/orderman/CancelOrderMsg.h"
|
||||
#include "lib/orderman/CancelReplaceOrderMsg.h"
|
||||
#include "lib/orderman/CreateOrderMsg.h"
|
||||
#include "lib/orderman/OrderMsg.h"
|
||||
#include "lib/orderman/orderman_utils/ExchangeOpenOrders.h"
|
||||
#include "lib/orderman/orderman_utils/ExchangePosition.h"
|
||||
#include "lib/orderman/orderman_utils/orderman_common.h"
|
||||
#include "lib/utils/bts_no_copy.h"
|
||||
#include "lib/tracing/PerfLog.h"
|
||||
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
#include <variant>
|
||||
|
||||
|
||||
|
||||
namespace bts::orderman {
|
||||
|
||||
@@ -29,33 +20,28 @@ class NewAdminDataEvent;
|
||||
class ExecutionReport;
|
||||
class CancelReject;
|
||||
|
||||
struct OrderStateWithFills {
|
||||
api::orderman::OrderState ordr_state;
|
||||
std::vector<api::orderman::Fill> fills = {};
|
||||
};
|
||||
|
||||
using OnOrderUpdate = std::function<void(OrderStateWithFills const &)>;
|
||||
using OnExecutionReport = std::function<void(ExecutionReport const &, const trace::CorID &)>;
|
||||
|
||||
// fix execution gateway for a single FIX session
|
||||
class IFIXGenericAdapter {
|
||||
public:
|
||||
IFIXGenericAdapter() = default;
|
||||
virtual ~IFIXGenericAdapter() {}
|
||||
virtual ~IFIXGenericAdapter();
|
||||
|
||||
virtual void start() = 0;
|
||||
// this callback should be added in a list, and on receving an update from the other side, it should be passed here
|
||||
virtual void register_callback(const OnOrderUpdate &);
|
||||
virtual void launch_new_order(const CreateOrderMsg &);
|
||||
virtual void modify_order(const CancelReplaceOrderMsg &);
|
||||
virtual void cancel_order(const CancelOrderMsg &);
|
||||
// // this callback should be added in a list, and on receving an update from the other side, it should be passed here
|
||||
virtual void register_callback(const OnExecutionReport &) =0 ;
|
||||
virtual void launch_new_order(const CreateOrderMsg &) =0;
|
||||
virtual void modify_order(const CancelReplaceOrderMsg &) = 0;
|
||||
virtual void cancel_order(const CancelOrderMsg &) =0;
|
||||
// as new securities are added
|
||||
virtual void update_securities(
|
||||
const mercury::Update<NewSecuritiesEvent> &) = 0;
|
||||
virtual void update_admin_data(
|
||||
const mercury::Update<NewAdminDataEvent> &) = 0;
|
||||
// virtual void update_securities(
|
||||
// const mercury::Update<NewSecuritiesEvent> &) = 0;
|
||||
// virtual void update_admin_data(
|
||||
// const mercury::Update<NewAdminDataEvent> &) = 0;
|
||||
|
||||
private:
|
||||
BTS_NO_COPY(IFIXGenericAdapter);
|
||||
// private:
|
||||
// BTS_NO_COPY(IFIXGenericAdapter);
|
||||
};
|
||||
|
||||
} // namespace bts::orderman
|
||||
@@ -0,0 +1,377 @@
|
||||
#include "OnixFIXAdapter.h"
|
||||
#include "OnixS/FIXEngine/FIX/Engine.h"
|
||||
#include "lib/config/protos/marex_fix_cfg.pb.h"
|
||||
#include "lib/domain_model/Security.pb.h"
|
||||
#include "lib/logging/logging.h"
|
||||
#include "lib/orderman/CancelOrderMsg.h"
|
||||
#include "lib/orderman/CancelReplaceOrderMsg.h"
|
||||
#include "lib/orderman/CreateOrderMsg.h"
|
||||
#include "lib/orderman/gateways/OnixFIXListener.h"
|
||||
#include "lib/orderman/gateways/TrafixOrderStateHandler.h"
|
||||
#include "lib/orderman/orderman_utils/orderman_common.h"
|
||||
#include "lib/utils/bts_assert.h"
|
||||
#include "lib/utils/helper_macros.h"
|
||||
#include <OnixS/FIXEngine/FIX/Exception.h>
|
||||
#include <OnixS/FIXEngine/FIX/FIX42.h>
|
||||
#include <OnixS/FIXEngine/FIX/Message.h>
|
||||
#include <OnixS/FIXEngine/FIX/Numeric.h>
|
||||
#include <OnixS/FIXEngine/FIX/ProtocolVersion.h>
|
||||
#include <cassert>
|
||||
#include <chrono>
|
||||
#include <memory>
|
||||
#include <quickfix/FixFields.h>
|
||||
#include <ratio>
|
||||
#include <string>
|
||||
#include <unistd.h>
|
||||
|
||||
namespace bts::orderman {
|
||||
void handleException(
|
||||
const std::string &context, const std::string &error) {
|
||||
bts_assert_msgf(false, "{} Exception: {}", context, error);
|
||||
}
|
||||
OnixFIXAdapter::OnixFIXAdapter(md::SecurityOracle sec_oracle)
|
||||
: state_handler(std::make_shared<TrafixOrderStateHandler>(sec_oracle)),
|
||||
listener(state_handler) {
|
||||
try {
|
||||
auto onix_cfg = config::get<config::MarexFixCfg>();
|
||||
LOG(OrderMan, Warn, "Initializing OnixFIXAdapter");
|
||||
OnixS::FIX::Engine::
|
||||
init(); // OnixS::FIX::Engine::init() needs to run before initialising session
|
||||
LOG(OrderMan, Diag, "Initializing OnixFIXAdapter engine initailes trying login");
|
||||
session = std::make_unique<Session>(
|
||||
onix_cfg.sender_sub_id(), onix_cfg.target_sub_id(),
|
||||
OnixS::FIX::ProtocolVersion::FIX_42, &listener);
|
||||
} catch (const OnixS::FIX::Exception &ex) {
|
||||
const std::string err = ex.what();
|
||||
LOG(OrderMan, Error, "Initializing OnixFIXAdapter ...{}",err);
|
||||
Engine::shutdown();
|
||||
handleException("OnixFIXAdapter::Construcotr init", err);
|
||||
}
|
||||
}
|
||||
OnixFIXAdapter::~OnixFIXAdapter() {
|
||||
try {
|
||||
if (session)
|
||||
session->logout();
|
||||
Engine::instance()->unregisterListener();
|
||||
Engine::shutdown();
|
||||
} catch (const OnixS::FIX::Exception &ex) {
|
||||
const std::string err = ex.what();
|
||||
handleException("OnixFIXAdapter::Destructor ", err);
|
||||
}
|
||||
}
|
||||
void OnixFIXAdapter::start() {
|
||||
auto cfg = config::get<config::MarexFixCfg>();
|
||||
const std::string counterpartyHost = cfg.counterparty_host();
|
||||
const int heartBtInt = cfg.heart_bt_int();
|
||||
const int counterpartyPort = cfg.counterparty_port();
|
||||
const int retryInterval = cfg.retry_interval(); // Add this field in MarexFixCfg
|
||||
const int maxRetries = cfg.max_retries(); // Optional: Set a limit on retries
|
||||
|
||||
int attempt = 0;
|
||||
|
||||
while (true) {
|
||||
try {
|
||||
LOG(OrderMan, Warn, "OnixFIXAdapter::start Attempting login (Attempt: {})", attempt + 1);
|
||||
session->logonAsInitiator(counterpartyHost, counterpartyPort, heartBtInt);
|
||||
LOG(OrderMan, Warn, "OnixFIXAdapter::start Login successful");
|
||||
break; // Exit loop on success
|
||||
} catch (const OnixS::FIX::Exception &ex) {
|
||||
const std::string err = ex.what();
|
||||
LOG(OrderMan, Error, "OnixFIXAdapter::start Login failed: {}. Retrying in {} seconds.", err, retryInterval);
|
||||
|
||||
if (maxRetries > 0 && ++attempt >= maxRetries) {
|
||||
Engine::shutdown();
|
||||
LOG(OrderMan, Error, "OnixFIXAdapter::start Exceeded max retries ({}). Aborting.", maxRetries);
|
||||
handleException("OnixFIXAdapter::start", err);
|
||||
break; // Exit if retries are exhausted
|
||||
}
|
||||
std::this_thread::sleep_for(std::chrono::seconds(std::max(60,retryInterval)));
|
||||
}
|
||||
}
|
||||
}
|
||||
template <typename T>
|
||||
uint translateEnum(T enum_value, const std::map<int, int> &translationMap) {
|
||||
auto it = translationMap.find(enum_value);
|
||||
bts_assert(it != translationMap.end()); // Using assert instead of bts_assert
|
||||
return it->second;
|
||||
}
|
||||
|
||||
uint translateTimeInForce(api::orderman::TimeInForce time_in_force) {
|
||||
//0 = Day 1 = GTC 2 = At the Open 3 = IOC 4 = FOK 5 = GTX
|
||||
static const std::map<int, int> tifMap = {
|
||||
{api::orderman::TimeInForce::TIME_IN_FORCE_DAY, 0},
|
||||
{api::orderman::TimeInForce::TIME_IN_FORCE_GTC, 1},
|
||||
{api::orderman::TimeInForce::TIME_IN_FORCE_IOC, 3}};
|
||||
|
||||
return translateEnum(time_in_force, tifMap);
|
||||
}
|
||||
|
||||
uint translateOrderType(api::orderman::NativeOrderType order_type) {
|
||||
//1 = Market 2 = Limit 3 = Stop 4 = Stop Limit 5 = Market on close A = On Close B = Limit on Close
|
||||
static const std::map<int, int> orderTypeMap = {
|
||||
{api::orderman::NativeOrderType::NATIVE_ORDER_TYPE_MARKET, 1},
|
||||
{api::orderman::NativeOrderType::NATIVE_ORDER_TYPE_LIMIT, 2},
|
||||
{api::orderman::NativeOrderType::NATIVE_ORDER_TYPE_STOP, 3},
|
||||
{api::orderman::NativeOrderType::NATIVE_ORDER_TYPE_STOP_LIMIT, 4}};
|
||||
|
||||
return translateEnum(order_type, orderTypeMap);
|
||||
}
|
||||
uint translateSide(api::Side side) {
|
||||
//1 = Buy 2 = Sell 5 = Sell short 6 = Sell short Exempt
|
||||
static const std::map<int, int> sideMap = {
|
||||
{api::Side::SIDE_BUY, 1}, {api::Side::SIDE_SELL, 2}};
|
||||
|
||||
return translateEnum(side, sideMap);
|
||||
}
|
||||
|
||||
std::string pad_string(
|
||||
const std::string &str, size_t length, char padding = ' ') {
|
||||
if (str.size() > length) {
|
||||
return str.substr(0, length);
|
||||
}
|
||||
return str + std::string(length - str.size(), padding);
|
||||
}
|
||||
|
||||
// Function to create the OSI Symbol
|
||||
static std::string create_osi_symbol(
|
||||
const std::string &symbol,
|
||||
const uint32_t &year,
|
||||
const uint32_t &month,
|
||||
const uint32_t &day,
|
||||
const char &call_put_indicator,
|
||||
const double &strike_price) {
|
||||
if (symbol.size() > 6) {
|
||||
throw std::invalid_argument("Symbol length cannot exceed 6 characters.");
|
||||
}
|
||||
|
||||
std::ostringstream osi_stream;
|
||||
|
||||
// Symbol: 6 bytes, padded with spaces
|
||||
osi_stream << pad_string(symbol, 6);
|
||||
|
||||
// Year: last 2 digits of the year
|
||||
osi_stream << std::setw(2) << std::setfill('0') << (year % 100);
|
||||
|
||||
// Month: 2 digits
|
||||
osi_stream << std::setw(2) << std::setfill('0') << month;
|
||||
|
||||
// Day: 2 digits
|
||||
osi_stream << std::setw(2) << std::setfill('0') << day;
|
||||
|
||||
// Call/Put Indicator: 1 byte
|
||||
osi_stream << call_put_indicator;
|
||||
|
||||
// Strike Price: 5 bytes for dollars, 3 bytes for decimal part
|
||||
int strike_dollars = static_cast<int>(strike_price);
|
||||
int strike_decimals =
|
||||
static_cast<int>((strike_price - strike_dollars) * 1000);
|
||||
osi_stream << std::setw(5) << std::setfill('0') << strike_dollars;
|
||||
osi_stream << std::setw(3) << std::setfill('0') << strike_decimals;
|
||||
|
||||
return osi_stream.str();
|
||||
}
|
||||
|
||||
void parse_option(Message &order, const api::Security &sec) {
|
||||
auto osi = create_osi_symbol(
|
||||
sec.symbol(), sec.option().expiration_year(),
|
||||
sec.option().expiration_month(), sec.option().expiration_day(),
|
||||
sec.option().call_put() == bts::api::PutOrCall::PUTORCALL_CALL ? 'c'
|
||||
: 'p',
|
||||
sec.option().strike());
|
||||
std::ostringstream month_stream;
|
||||
month_stream << std::setw(2) << std::setfill('0')
|
||||
<< sec.option().expiration_month();
|
||||
std::string month = month_stream.str();
|
||||
LOG(OrderMan, Warn, "OnixFIXAdapter::parse_option parsing osi:{}", osi);
|
||||
order
|
||||
.set(
|
||||
OnixS::FIX::FIX42::Tags::PutOrCall,
|
||||
sec.option().call_put() == bts::api::PutOrCall::PUTORCALL_CALL ? "1"
|
||||
: "0")
|
||||
.set(
|
||||
OnixS::FIX::FIX42::Tags::StrikePrice,
|
||||
std::to_string(sec.option().strike()))
|
||||
.set(
|
||||
OnixS::FIX::FIX42::Tags::MaturityMonthYear,
|
||||
std::to_string(sec.option().expiration_year()) + month)
|
||||
.set(
|
||||
OnixS::FIX::FIX42::Tags::MaturityDay,
|
||||
std::to_string(sec.option().expiration_day()))
|
||||
.set(OnixS::FIX::FIX42::Tags::SecurityType, "OPT")
|
||||
.set(OnixS::FIX::FIX42::Tags::Symbol, sec.symbol());
|
||||
}
|
||||
|
||||
void add_cfg_fields(Message &order, const config::MarexFixCfg &cfg) {
|
||||
order
|
||||
.set(
|
||||
OnixS::FIX::FIX42::Tags::HandlInst,
|
||||
cfg.handl_inst()) // 1: DMA 2: ALGO 3:CARE
|
||||
.set(
|
||||
OnixS::FIX::FIX42::Tags::Rule80A,
|
||||
cfg.Rule80A_Name(cfg.rule80a())) //A = AGENCY P = PRINCIPAL
|
||||
.set(OnixS::FIX::FIX42::Tags::SenderSubID, cfg.sender_sub_id())
|
||||
.set(OnixS::FIX::FIX42::Tags::TargetSubID, cfg.target_sub_id())
|
||||
.set(OnixS::FIX::FIX42::Tags::ClientID, cfg.client_id())
|
||||
.set(OnixS::FIX::FIX42::Tags::CustomerOrFirm, cfg.customer_or_firm())
|
||||
.set(OnixS::FIX::FIX42::Tags::OpenClose, cfg.open_close());
|
||||
order.set(FIX42::Tags::Currency,cfg.currency());
|
||||
order.set(FIX42::Tags::SecurityExchange,cfg.security_exchange());
|
||||
order.set(FIX42::Tags::ExDestination,cfg.ex_destination());
|
||||
}
|
||||
void populate_common_order_fields(Message &order, const CreateOrderMsg &msg) {
|
||||
order.set(OnixS::FIX::FIX42::Tags::Account, msg.routing_record->account())
|
||||
.set(
|
||||
OnixS::FIX::FIX42::Tags::TimeInForce,
|
||||
translateTimeInForce(msg.time_in_force))
|
||||
.set(
|
||||
OnixS::FIX::FIX42::Tags::OrdType, translateOrderType(msg.order_type))
|
||||
.set(OnixS::FIX::FIX42::Tags::Side, translateSide(msg.side))
|
||||
.set(
|
||||
OnixS::FIX::FIX42::Tags::Price,
|
||||
static_cast<float>(msg.price.value()))
|
||||
.set(OnixS::FIX::FIX42::Tags::OrderQty, msg.quantity.value());
|
||||
if (msg.security->has_option()) {
|
||||
parse_option(order, *msg.security);
|
||||
} else {
|
||||
order.set(OnixS::FIX::FIX42::Tags::Symbol, msg.security->symbol())
|
||||
.set(OnixS::FIX::FIX42::Tags::SecurityType, "FUT") //need to fix this
|
||||
.set(OnixS::FIX::FIX42::Tags::Currency, msg.security->currency());
|
||||
}
|
||||
}
|
||||
void OnixFIXAdapter::modify_order(
|
||||
const CancelReplaceOrderMsg &msg) { //confirm will have everything
|
||||
try {
|
||||
|
||||
//use this fileOnixS.FixEngineCpp-Ubuntu2204-gcc1210-c++17-x64-4.10.1/OnixS.FixEngineCpp-Ubuntu2204-gcc1210-c++17-x64-4.10.1/doc/html/group__double-decimal-conversion.html
|
||||
|
||||
LOG(OrderMan, Warn, "OnixFIXAdapter::modify_order modifying order{}",
|
||||
msg);
|
||||
auto order_cfg = config::get<config::MarexFixCfg>();
|
||||
auto id =
|
||||
state_handler->get_marex_key(OrderKey(msg.create_fields.key.value()));
|
||||
|
||||
if (id == "") {
|
||||
LOG(OrderMan, Error,
|
||||
"OnixFIXAdapter::modify_order could not find clorid");
|
||||
return;
|
||||
}
|
||||
auto new_id = state_handler->new_id(id);
|
||||
|
||||
Message order(
|
||||
OnixS::FIX::FIX42::Values::MsgType::Order_Cancel_Replace_Request,
|
||||
OnixS::FIX::ProtocolVersion::FIX_42);
|
||||
order.set(OnixS::FIX::FIX42::Tags::OrigClOrdID, id)
|
||||
.set(OnixS::FIX::FIX42::Tags::ClOrdID, new_id) // ID for order
|
||||
.set(
|
||||
OnixS::FIX::FIX42::Tags::TransactTime,
|
||||
OnixS::FIX::Timestamp::utc(),
|
||||
OnixS::FIX::TimestampFormat::YYYYMMDDHHMMSSMsec);
|
||||
|
||||
add_cfg_fields(order, order_cfg);
|
||||
populate_common_order_fields(order, msg.create_fields);
|
||||
order.validate();
|
||||
LOG(OrderMan, Warn, "order Validated");
|
||||
order.set(OnixFIXAdapter::FIX_TAG_SMART_ROUTING, "SMART");
|
||||
std::string order_string = order.toString('|');
|
||||
state_handler->modifyOrder(msg, new_id);
|
||||
session->send(&order);
|
||||
LOG(OrderMan, Warn, "Order update sent :{}", order_string);
|
||||
} catch (const OnixS::FIX::Exception &ex) {
|
||||
const std::string err = ex.what();
|
||||
handleException("Modify order", err);
|
||||
}
|
||||
}
|
||||
|
||||
void OnixFIXAdapter::launch_new_order(const CreateOrderMsg &msg) {
|
||||
LOG(OrderMan, Warn, "launch order {}", msg);
|
||||
|
||||
try {
|
||||
LOG(OrderMan, Warn, "OnixFIXAdapter::launch_new_order creating order");
|
||||
auto order_cfg = config::get<config::MarexFixCfg>();
|
||||
|
||||
auto new_id =
|
||||
state_handler->new_id(":" + std::to_string(msg.key.value()));
|
||||
LOG(OrderMan, Warn, "OnixFIXAdapter::launch_new_order new id:'{}'",
|
||||
new_id);
|
||||
std::string symbol = msg.security->symbol();
|
||||
Message order(
|
||||
OnixS::FIX::FIX42::Values::MsgType::Order_Single,
|
||||
OnixS::FIX::ProtocolVersion::FIX_42);
|
||||
order
|
||||
.set(OnixS::FIX::FIX42::Tags::ClOrdID, new_id) // ID for order
|
||||
.set(
|
||||
OnixS::FIX::FIX42::Tags::TransactTime,
|
||||
OnixS::FIX::Timestamp::utc(),
|
||||
OnixS::FIX::TimestampFormat::YYYYMMDDHHMMSSMsec);
|
||||
add_cfg_fields(order, order_cfg);
|
||||
populate_common_order_fields(order, msg);
|
||||
|
||||
std::string order_string = order.toString('|');
|
||||
std::string order_json = order.toJson();
|
||||
order.validate();
|
||||
LOG(OrderMan, Warn, "order Validated");
|
||||
order.set(OnixFIXAdapter::FIX_TAG_SMART_ROUTING, "SMART");
|
||||
state_handler->launchOrder(msg, new_id);
|
||||
// listener.fire_dummy_report(msg);
|
||||
session->send(&order);
|
||||
LOG(OrderMan, Warn, "Order sent :{} json:{}", order_string, order_json);
|
||||
|
||||
}
|
||||
// Shut down all reactors if they exist.
|
||||
catch (const OnixS::FIX::Exception &ex) {
|
||||
const std::string err = ex.what();
|
||||
handleException("Launch order", err);
|
||||
}
|
||||
}
|
||||
void OnixFIXAdapter::register_callback(const OnExecutionReport &cb) {
|
||||
LOG(OrderMan, Warn, "OnixFIXAdapter::register_callback callback registered");
|
||||
listener.setCallback(cb);
|
||||
}
|
||||
void OnixFIXAdapter::cancel_order(const CancelOrderMsg &msg) {
|
||||
LOG(OrderMan, Warn, "cancel order {}", msg);
|
||||
try {
|
||||
auto id = state_handler->get_marex_key(OrderKey(msg.key.value()));
|
||||
auto order_cfg = config::get<config::MarexFixCfg>();
|
||||
|
||||
if (id == "") {
|
||||
LOG(OrderMan, Error,
|
||||
"OnixFIXAdapter::modify_order could not find clorid");
|
||||
return;
|
||||
}
|
||||
auto new_id = state_handler->new_id(id);
|
||||
Message order_cancel(
|
||||
OnixS::FIX::FIX42::Values::MsgType::Order_Cancel_Request,
|
||||
OnixS::FIX::ProtocolVersion::FIX_42);
|
||||
order_cancel.set(
|
||||
OnixS::FIX::FIX42::Tags::HandlInst, order_cfg.handl_inst());
|
||||
order_cancel.set(OnixS::FIX::FIX42::Tags::ClOrdID, new_id);
|
||||
order_cancel.set(OnixS::FIX::FIX42::Tags::OrigClOrdID, id)
|
||||
.set(OnixS::FIX::FIX42::Tags::Side, translateSide(msg.side))
|
||||
.set(
|
||||
OnixS::FIX::FIX42::Tags::TransactTime,
|
||||
OnixS::FIX::Timestamp::utc(),
|
||||
OnixS::FIX::TimestampFormat::YYYYMMDDHHMMSSMsec);
|
||||
if (msg.security->has_option()) {
|
||||
parse_option(order_cancel, *msg.security);
|
||||
} else {
|
||||
order_cancel
|
||||
.set(OnixS::FIX::FIX42::Tags::Symbol, msg.security->symbol())
|
||||
.set(OnixS::FIX::FIX42::Tags::Currency, msg.security->currency());
|
||||
}
|
||||
std::string order_str = order_cancel.toString('|');
|
||||
LOG(OrderMan, Warn, "order_cancel Created: {}", order_str);
|
||||
// order_cancel.validate(); fails becasuse of HandlInst
|
||||
LOG(OrderMan, Warn, "order Validated");
|
||||
order_cancel.set(OnixFIXAdapter::FIX_TAG_SMART_ROUTING, "SMART");
|
||||
state_handler->cancelOrder(msg, new_id);
|
||||
session->send(&order_cancel);
|
||||
LOG(OrderMan, Warn, "order cancel sent");
|
||||
|
||||
}
|
||||
// Shut down all reactors if they exist.
|
||||
catch (const OnixS::FIX::Exception &ex) {
|
||||
const std::string err = ex.what();
|
||||
handleException("Cacnel order", err);
|
||||
}
|
||||
}
|
||||
} // namespace bts::orderman
|
||||
@@ -0,0 +1,27 @@
|
||||
#pragma once
|
||||
|
||||
#include "IFIXGenericAdapter.h"
|
||||
#include "lib/orderman/gateways/OnixFIXListener.h"
|
||||
#include "lib/orderman/gateways/TrafixOrderStateHandler.h"
|
||||
#include <memory>
|
||||
|
||||
namespace bts::orderman {
|
||||
|
||||
class OnixFIXAdapter : public IFIXGenericAdapter {
|
||||
static constexpr int FIX_TAG_SMART_ROUTING = 847;
|
||||
public:
|
||||
OnixFIXAdapter(md::SecurityOracle sec_oracle);
|
||||
~OnixFIXAdapter() override;
|
||||
void register_callback(const OnExecutionReport &report) override;
|
||||
void launch_new_order(const CreateOrderMsg &msg) override;
|
||||
void cancel_order(const CancelOrderMsg &msg) override;
|
||||
void modify_order(const CancelReplaceOrderMsg &) override;
|
||||
void start() override;
|
||||
|
||||
private:
|
||||
std::shared_ptr<TrafixOrderStateHandler> state_handler;
|
||||
OnixFixListener listener;
|
||||
std::unique_ptr<Session> session;
|
||||
};
|
||||
|
||||
} // namespace bts::orderman
|
||||
@@ -0,0 +1,264 @@
|
||||
#include "OnixFIXListener.h"
|
||||
|
||||
#include "lib/api/services/OrderService.pb.h"
|
||||
#include "lib/domain_model/Security.pb.h"
|
||||
#include "lib/domain_model/common.h"
|
||||
#include "lib/logging/logging.h"
|
||||
#include "lib/orderman/gateways/IExecutionGateway.h"
|
||||
#include "lib/orderman/gateways/IFIXGenericAdapter.h"
|
||||
#include "lib/orderman/orderman_utils/ExecutionReport.h"
|
||||
#include "lib/orderman/orderman_utils/orderman_common.h"
|
||||
#include "lib/utils/Uuid.h"
|
||||
#include "lib/utils/bts_assert.h"
|
||||
#include <OnixS/FIXEngine/FIX/FIX42.h>
|
||||
#include <OnixS/FIXEngine/FIX/Tag.h>
|
||||
#include <string>
|
||||
|
||||
namespace bts::orderman {
|
||||
ExecutionReport OnixFixListener::fire_dummy_report(CreateOrderMsg msg) {
|
||||
ExecutionReport *er_p = exe_report_pool.construct();
|
||||
ExecutionReport &er = *er_p;
|
||||
std::string account_name = "BTSIM";
|
||||
std::array<char, 12> an_array = {};
|
||||
size_t i = 0;
|
||||
for (const char c : account_name) {
|
||||
an_array[i++] = c;
|
||||
}
|
||||
er.account_name = AccountName(an_array);
|
||||
to_array(msg.routing_record->account(), &er.account_name.value());
|
||||
to_array(
|
||||
msg.routing_record->operator_id(), &er.operator_id.value());
|
||||
er.order_key = msg.key;
|
||||
er.msg_seq_num = 0;
|
||||
er.security = msg.security;
|
||||
er.leg_security = nullptr;
|
||||
er.side = msg.side;
|
||||
er.time_in_force = msg.time_in_force;
|
||||
er.order_type = msg.order_type;
|
||||
er.exec_type = api::orderman::EXEC_TYPE_NEW;
|
||||
er.order_status = api::orderman::NATIVE_ORDER_STATUS_NEW;
|
||||
er.manual_order_indicator = api::orderman::MANUAL_ORDER_INDICATOR_NO;
|
||||
auto s = std::to_string(msg.key.value());
|
||||
std::array<char, 17> idAsArray{};
|
||||
std::copy_n(s.begin(), std::min(s.size()-1, idAsArray.size() - 1), idAsArray.begin());
|
||||
er.exchange_order_id = ExchangeOrderID{idAsArray};
|
||||
er.price = msg.price;
|
||||
er.quantity = msg.quantity;
|
||||
er.filled_quantity = Quantity(0);
|
||||
er.last_shares = Quantity(0);
|
||||
er.msg = "Message Placed";
|
||||
er.exchange = msg.security->exchange();
|
||||
// Set transaction time
|
||||
ExchangeTimeStamp64 now = ExchangeTimeStamp64(datetime::unix_nano_now());
|
||||
er.transact_time = now;
|
||||
cb(er,{});
|
||||
return er;
|
||||
}
|
||||
|
||||
namespace {
|
||||
ExecutionReport createDefaultExecutionReport() {
|
||||
ExecutionReport er;
|
||||
er.exchange = api::VENUE_CBOE;
|
||||
std::string account_name = "BTSIM";
|
||||
std::string ops_id = "op_id_bts";
|
||||
|
||||
std::array<char, 12> account_array = {};
|
||||
std::copy_n(account_name.begin(), std::min(account_name.size(), account_array.size()), account_array.begin());
|
||||
er.account_name = AccountName(account_array);
|
||||
|
||||
to_array(ops_id, &er.operator_id.value());
|
||||
return er;
|
||||
}
|
||||
|
||||
void populateExecTypeAndOrderStatus(ExecutionReport &er, int exec_type_value) {
|
||||
switch (exec_type_value) {
|
||||
case 0:
|
||||
er.exec_type = api::orderman::ExecType::EXEC_TYPE_NEW;
|
||||
er.order_status = api::orderman::NATIVE_ORDER_STATUS_NEW;
|
||||
break;
|
||||
case 1:
|
||||
er.exec_type = api::orderman::ExecType::EXEC_TYPE_PARTIAL_FILL;
|
||||
er.order_status = api::orderman::NATIVE_ORDER_STATUS_PARTIALLY_FILLED;
|
||||
break;
|
||||
case 2:
|
||||
er.exec_type = api::orderman::ExecType::EXEC_TYPE_FILL;
|
||||
er.order_status = api::orderman::NATIVE_ORDER_STATUS_FILLED;
|
||||
break;
|
||||
case 4:
|
||||
er.exec_type = api::orderman::ExecType::EXEC_TYPE_CANCELED;
|
||||
er.order_status = api::orderman::NATIVE_ORDER_STATUS_CANCELED;
|
||||
break;
|
||||
case 5:
|
||||
er.exec_type = api::orderman::ExecType::EXEC_TYPE_REPLACED;
|
||||
er.order_status = api::orderman::NATIVE_ORDER_STATUS_REPLACED;
|
||||
break;
|
||||
case 8:
|
||||
er.exec_type = api::orderman::ExecType::EXEC_TYPE_REJECTED;
|
||||
er.order_status = api::orderman::NATIVE_ORDER_STATUS_REJECTED;
|
||||
break;
|
||||
default:
|
||||
er.exec_type = api::orderman::ExecType::EXEC_TYPE_UNDEFINED;
|
||||
er.order_status = api::orderman::NATIVE_ORDER_STATUS_UNDEFINED;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
void populateCreateOrderFields(ExecutionReport &er, const CreateOrderMsg &create_order_msg) {
|
||||
er.security = create_order_msg.security;
|
||||
er.order_type = create_order_msg.order_type;
|
||||
er.time_in_force = create_order_msg.time_in_force;
|
||||
er.manual_order_indicator = create_order_msg.manual_order_indicator;
|
||||
|
||||
to_array(create_order_msg.routing_record->account(), &er.account_name.value());
|
||||
to_array(create_order_msg.routing_record->operator_id(), &er.operator_id.value());
|
||||
er.exchange = create_order_msg.security->exchange();
|
||||
}
|
||||
|
||||
std::array<char, 17> stringToFixedArray(const std::string &s) {
|
||||
std::array<char, 17> idAsArray{};
|
||||
std::copy_n(s.begin(), std::min(s.size(), idAsArray.size() - 1), idAsArray.begin());
|
||||
return idAsArray;
|
||||
}
|
||||
}
|
||||
ExecutionReport OnixFixListener::get_exe_report(Message msg) {
|
||||
ExecutionReport *er_p = exe_report_pool.construct();
|
||||
ExecutionReport &er = *er_p;
|
||||
|
||||
auto msg_str = msg.toString();
|
||||
LOG(OrderMan,Info, "OnixFixListener::get_exe_report {}", msg_str);
|
||||
|
||||
// Extract order key and create order message
|
||||
er.order_key = state_handler->getOrderKey(msg.get(FIX42::Tags::ClOrdID).toString()); //OrderKey(extractUInt(msg.get(FIX42::Tags::ClOrdID).toString()));
|
||||
auto m = state_handler->getCreateOrderMsg(er.order_key);
|
||||
|
||||
if(m.has_value()){
|
||||
populateCreateOrderFields(er, m.value());
|
||||
}else{ //handle this later
|
||||
LOG(OrderMan,Error, "OnixFixListener::get_exe_report {} could not find create order message", msg_str);
|
||||
er = createDefaultExecutionReport();
|
||||
}
|
||||
// Populate fields from the message
|
||||
|
||||
int tmp;
|
||||
msg.get(FIX42::Tags::MsgSeqNum).toNumber(tmp);
|
||||
er.msg_seq_num = tmp;
|
||||
|
||||
er.leg_security = nullptr;
|
||||
|
||||
er.side = msg.get(FIX42::Tags::Side).toString()=="1"?api::Side::SIDE_BUY:api::Side::SIDE_SELL;
|
||||
|
||||
int exec_type_value = 9; // Default to undefined
|
||||
msg.get(FIX42::Tags::ExecType).toNumber(exec_type_value);
|
||||
populateExecTypeAndOrderStatus(er, exec_type_value);
|
||||
|
||||
auto price_str = msg.get(FIX42::Tags::Price);
|
||||
er.price = Price(price_str.valid() ? std::stod(price_str.toString()) : 0);
|
||||
|
||||
auto order_qty_str = msg.get(FIX42::Tags::OrderQty);
|
||||
er.quantity = Quantity(order_qty_str.valid() ? std::stoi(order_qty_str.toString()) : 0);
|
||||
|
||||
auto leaves_qty_str = msg.get(FIX42::Tags::LeavesQty);
|
||||
er.filled_quantity = leaves_qty_str.valid() ? er.quantity - Quantity(std::stoi(leaves_qty_str.toString())) : er.quantity;
|
||||
|
||||
auto last_shares_str = msg.get(FIX42::Tags::LastShares);
|
||||
er.last_shares = Quantity(last_shares_str.valid() ? std::stoi(last_shares_str.toString()) : 0);
|
||||
|
||||
auto avg_px_str = msg.get(FIX42::Tags::AvgPx);
|
||||
er.avg_price = Price(avg_px_str.valid() ? std::stod(avg_px_str.toString()) : 0);
|
||||
|
||||
auto text_str = msg.get(FIX42::Tags::Text).toString();
|
||||
er.msg = text_str.substr(0, std::min(text_str.length(), size_t(15)));
|
||||
|
||||
auto exec_id_str = msg.get(FIX42::Tags::ExecID);
|
||||
std::string fill_id = exec_id_str.valid() ? exec_id_str.toString() : bts::utils::Uuid().to_text_string();
|
||||
to_array(fill_id, &er.exchange_fill_id.value());
|
||||
|
||||
auto last_px_str = msg.get(FIX42::Tags::LastPx);
|
||||
er.last_price = Price(last_px_str.valid() ? std::stod(last_px_str.toString()) : 0);
|
||||
|
||||
|
||||
er.exchange_order_id = ExchangeOrderID{stringToFixedArray(msg.get(FIX42::Tags::OrderID).toString())};
|
||||
|
||||
|
||||
// er.transact_time = ExchangeTimeStamp64(datetime::unix_nano_now());
|
||||
//exchgane gives this "TransactTime": "20241111-16:36:08.243",
|
||||
// er.transact_time =
|
||||
// Set transaction time
|
||||
ExchangeTimeStamp64 now = ExchangeTimeStamp64(datetime::unix_nano_now());
|
||||
er.transact_time = now;
|
||||
|
||||
return er;
|
||||
}
|
||||
|
||||
void OnixFixListener::onInboundApplicationMsg(Message &msg, Session * /*sn*/) {
|
||||
std::string msg_str = msg.toString();
|
||||
std::string msg_json = msg.toJson();
|
||||
|
||||
FieldValueRef t = msg.type();
|
||||
char message_type;
|
||||
if(!t.toChar(message_type)){
|
||||
LOG(OrderMan, Warn,
|
||||
"no message type message: {} As json {}",
|
||||
msg_str, msg_json);
|
||||
return;
|
||||
}
|
||||
LOG(OrderMan, Warn,
|
||||
"Incoming application-level message: {} As json {}, type: {}",
|
||||
msg_str, msg_json,message_type);
|
||||
|
||||
switch (message_type) {
|
||||
case '8':
|
||||
{
|
||||
//exection report
|
||||
auto er = get_exe_report(msg);
|
||||
if(er.security==nullptr){
|
||||
LOG(OrderMan, Error,"DID not have sec");
|
||||
return;
|
||||
}
|
||||
LOG(OrderMan, Error,"msg {}",er.msg);
|
||||
cb(er,{});
|
||||
}
|
||||
}
|
||||
|
||||
// LOG(OrderMan, Warn,
|
||||
// "onInboundApplicationMsg::inbound firing er: {}",
|
||||
// er->to_string());
|
||||
|
||||
// cb(*er, {});
|
||||
}
|
||||
|
||||
|
||||
void OnixFixListener::onInboundSessionMsg(Message & msg, Session * /*sn*/)
|
||||
{
|
||||
std::string msg_str = msg.toString();
|
||||
std::string msg_json = msg.toJson();
|
||||
LOG( OrderMan, Warn,
|
||||
"Incoming session-level message:\n {} As json {}",
|
||||
msg_str,
|
||||
msg_json);
|
||||
}
|
||||
|
||||
void OnixFixListener::onStateChange(SessionState::Enum newState, SessionState::Enum prevState,
|
||||
Session * /*sn*/)
|
||||
{
|
||||
std::string prevst = SessionState::toString(prevState);
|
||||
std::string newst = SessionState::toString(newState);
|
||||
LOG(OrderMan, Warn,
|
||||
"Session's state is changed, prevState = {} , newState= {}",
|
||||
prevst,
|
||||
newst);
|
||||
}
|
||||
|
||||
void OnixFixListener::onError(ErrorReason::Enum /*reason*/, const std::string & description, Session * /*sn*/)
|
||||
{
|
||||
LOG( OrderMan, Error, "Session-level error: {}", description);
|
||||
}
|
||||
|
||||
void OnixFixListener::onWarning(WarningReason::Enum /*reason*/, const std::string & description,
|
||||
Session * /*sn*/)
|
||||
{
|
||||
LOG( OrderMan, Error, "Session-level warning: {}", description);
|
||||
}
|
||||
|
||||
|
||||
} // namespace bts::orderman
|
||||
@@ -0,0 +1,52 @@
|
||||
#pragma once
|
||||
|
||||
#include "lib/orderman/CreateOrderMsg.h"
|
||||
#include "lib/orderman/gateways/IExecutionGateway.h"
|
||||
#include "lib/orderman/gateways/TrafixOrderStateHandler.h"
|
||||
#include "lib/orderman/orderman_utils/ExecutionReport.h"
|
||||
#include <OnixS/FIXEngine.h>
|
||||
#include <OnixS/FIXEngine/FIX/Message.h>
|
||||
#include <boost/pool/object_pool.hpp>
|
||||
|
||||
namespace bts::orderman {
|
||||
class TrafixOrderStateHandler;
|
||||
using namespace OnixS::FIX;
|
||||
class OnixFixListener : public OnixS::FIX::ISessionListener {
|
||||
public:
|
||||
OnixFixListener( std::shared_ptr<TrafixOrderStateHandler> state_handler) : state_handler(state_handler),cb(dummy) {
|
||||
LOG(OrderMan, Info, "OnixFixListener::cb set callback registered");
|
||||
}
|
||||
std::shared_ptr<TrafixOrderStateHandler> state_handler;
|
||||
unsigned long exchange_oid = 0;
|
||||
/// Is called when the application-level message is received from the counterparty.
|
||||
void onInboundApplicationMsg(Message &msg, Session *sn)
|
||||
ONIXS_FIXENGINE_FINAL;
|
||||
// Set the real callback function
|
||||
void setCallback(const OnExeReport &callback) {
|
||||
LOG(OrderMan, Info, "OnixFixListener::setCallback callback registered");
|
||||
cb = callback;
|
||||
}
|
||||
/// Is called when the session-level message is received from the counterparty.
|
||||
void onInboundSessionMsg(Message &msg, Session *sn) ONIXS_FIXENGINE_FINAL;
|
||||
void onStateChange(
|
||||
SessionState::Enum newState,
|
||||
SessionState::Enum prevState,
|
||||
Session *sn) ONIXS_FIXENGINE_FINAL;
|
||||
OnExeReport cb;
|
||||
ExecutionReport fire_dummy_report(CreateOrderMsg msg);
|
||||
ExecutionReport get_exe_report(Message msg);
|
||||
static void dummy(const ExecutionReport &, const trace::CorID &) {
|
||||
LOG(OrderMan, Error, "OnixFixListener::Dummy fired");
|
||||
}
|
||||
void onError(
|
||||
ErrorReason::Enum reason,
|
||||
const std::string &description,
|
||||
Session *sn) ONIXS_FIXENGINE_FINAL;
|
||||
void onWarning(
|
||||
WarningReason::Enum reason,
|
||||
const std::string &description,
|
||||
Session *sn) ONIXS_FIXENGINE_FINAL;
|
||||
boost::object_pool<ExecutionReport> exe_report_pool;
|
||||
};
|
||||
|
||||
} // namespace bts::orderman
|
||||
@@ -0,0 +1,88 @@
|
||||
#include "TrafixOrderStateHandler.h"
|
||||
#include "lib/logging/logging.h"
|
||||
#include "lib/orderman/CreateOrderMsg.h"
|
||||
#include "lib/orderman/order_factory/IOrderFactory.h"
|
||||
#include "lib/orderman/orderman_utils/orderman_common.h"
|
||||
#include <cstddef>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
|
||||
namespace bts::orderman {
|
||||
|
||||
TrafixOrderStateHandler::TrafixOrderStateHandler(md::SecurityOracle sec_oracle)
|
||||
: pimpl_(std::make_unique<PImpl>(sec_oracle)) {}
|
||||
|
||||
TrafixOrderStateHandler::~TrafixOrderStateHandler() = default;
|
||||
|
||||
TrafixOrderStateHandler::OrderInfo* TrafixOrderStateHandler::get_order_info(const OrderKey&key) const {
|
||||
auto it = pimpl_->orders.find(key);
|
||||
if (it != pimpl_->orders.end()) {
|
||||
return &it->second;
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
|
||||
OrderKey TrafixOrderStateHandler::launchOrder(const CreateOrderMsg& msg, const std::string& clor_id) {
|
||||
std::lock_guard<std::mutex> lock(pimpl_->orders_mutex);
|
||||
pimpl_->orders.emplace(msg.key, OrderInfo(msg, clor_id));
|
||||
return msg.key;
|
||||
}
|
||||
OrderKey TrafixOrderStateHandler::modifyOrder(const CancelReplaceOrderMsg& msg, const std::string& clor_id) {
|
||||
std::lock_guard<std::mutex> lock(pimpl_->orders_mutex);
|
||||
LOG(md,Info,"Modifying Order: {}, id:{}",msg,clor_id);
|
||||
auto v = get_order_info(msg.create_fields.key);
|
||||
if(v==nullptr) return null_order_key();
|
||||
LOG(md,Info,"Modifying found order info Order: {}, id:{}",msg,clor_id);
|
||||
v->update(msg, clor_id);
|
||||
return msg.create_fields.key;
|
||||
}
|
||||
OrderKey TrafixOrderStateHandler::cancelOrder(const CancelOrderMsg& msg, const std::string& clor_id) {
|
||||
std::lock_guard<std::mutex> lock(pimpl_->orders_mutex);
|
||||
LOG(md,Info,"canceling Order: {}, id:{}",msg,clor_id);
|
||||
auto v = get_order_info(msg.key);
|
||||
if(v==nullptr) return null_order_key();
|
||||
std::string cancel_id = clor_id;
|
||||
v->cancel(msg, cancel_id);
|
||||
return msg.key;
|
||||
}
|
||||
//static
|
||||
std::string TrafixOrderStateHandler::new_id(const std::string& current_id) {
|
||||
LOG(OrderMan, Warn, "new_id {}", current_id);
|
||||
int update_number = getUpdateNumber(current_id)+1;
|
||||
LOG(OrderMan, Warn, "new_id {}, update number {}", current_id,update_number);
|
||||
|
||||
return std::to_string(update_number) + ":" + std::to_string(getOrderKey(current_id).value());
|
||||
}
|
||||
OrderKey TrafixOrderStateHandler::getOrderKey(const std::string& id) {
|
||||
std::size_t pos = id.find(':');
|
||||
if (pos == std::string::npos) return null_order_key();
|
||||
return OrderKey(std::stoi(id.substr(pos + 1)));
|
||||
}
|
||||
int TrafixOrderStateHandler::getUpdateNumber(const std::string& id) {
|
||||
std::size_t pos = id.find(':');
|
||||
if(pos==std::string::npos || pos ==0){
|
||||
return 0;
|
||||
}
|
||||
std::string prefix = id.substr(0, pos);
|
||||
bts_assert_msgf(std::all_of(prefix.begin(), prefix.end(), ::isdigit), "Failed to get update number id:{}",id);
|
||||
return std::stoi(prefix);
|
||||
}
|
||||
|
||||
std::optional<CreateOrderMsg> TrafixOrderStateHandler::getCreateOrderMsg(const OrderKey& key) const {
|
||||
std::lock_guard<std::mutex> lock(pimpl_->orders_mutex);
|
||||
auto v = get_order_info(key);
|
||||
if(v!=nullptr) return v->order_msg;
|
||||
LOG(md, Error, "Order with key {} not found.", key);
|
||||
return std::nullopt;
|
||||
}
|
||||
std::string TrafixOrderStateHandler::get_marex_key(const OrderKey &key) {
|
||||
std::lock_guard<std::mutex> lock(pimpl_->orders_mutex);
|
||||
|
||||
auto v = get_order_info(key);
|
||||
if(v!=nullptr) return v->current_id;
|
||||
return "";
|
||||
}
|
||||
|
||||
} // namespace b
|
||||
@@ -0,0 +1,61 @@
|
||||
#pragma once
|
||||
#include "IFIXGenericAdapter.h"
|
||||
#include "lib/orderman/CancelOrderMsg.h"
|
||||
#include "lib/orderman/CreateOrderMsg.h"
|
||||
#include "lib/orderman/gateways/OnixFIXListener.h"
|
||||
#include "lib/orderman/order_factory/IOrderFactory.h"
|
||||
#include "lib/orderman/orderman_utils/orderman_common.h"
|
||||
#include "lib/utils/bts_assert.h"
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
|
||||
namespace bts::orderman {
|
||||
|
||||
class TrafixOrderStateHandler {
|
||||
public:
|
||||
explicit TrafixOrderStateHandler(md::SecurityOracle sec_oracle);
|
||||
~TrafixOrderStateHandler();
|
||||
|
||||
OrderKey launchOrder(const CreateOrderMsg& msg, const std::string& clor_id);
|
||||
std::optional<CreateOrderMsg> getCreateOrderMsg(const OrderKey& key) const;
|
||||
std::string get_marex_key(const OrderKey &key) ;
|
||||
OrderKey modifyOrder(const CancelReplaceOrderMsg& msg, const std::string& clor_id);
|
||||
OrderKey cancelOrder(const CancelOrderMsg& msg, const std::string& clor_id);
|
||||
static std::string new_id(const std::string& current_id);
|
||||
static OrderKey getOrderKey(const std::string& id);
|
||||
static int getUpdateNumber(const std::string& id);
|
||||
|
||||
private:
|
||||
struct OrderInfo {
|
||||
explicit OrderInfo(const CreateOrderMsg& msg, const std::string& id)
|
||||
: current_id(id), order_msg(msg) {}
|
||||
void update(const CancelReplaceOrderMsg& msg, const std::string& new_id) {
|
||||
LOG(md,Info,"OrderInfo:update Order: {}, id:{}",msg,new_id);
|
||||
|
||||
upd_msg = msg;
|
||||
current_id = new_id;
|
||||
}
|
||||
|
||||
void cancel(const CancelOrderMsg& msg, const std::string& new_id) {
|
||||
cancel_msg = msg;
|
||||
current_id = new_id;
|
||||
}
|
||||
std::string current_id;
|
||||
CreateOrderMsg order_msg;
|
||||
CancelReplaceOrderMsg upd_msg;
|
||||
CancelOrderMsg cancel_msg;
|
||||
};
|
||||
OrderInfo* get_order_info(const OrderKey&key) const;
|
||||
struct PImpl {
|
||||
explicit PImpl(md::SecurityOracle sec_oracle) : sec_oracle(sec_oracle) {}
|
||||
mutable std::mutex orders_mutex;//need to use this every time orders are accesed
|
||||
//can be improved later on
|
||||
std::unordered_map<OrderKey, OrderInfo> orders;
|
||||
md::SecurityOracle sec_oracle;
|
||||
};
|
||||
|
||||
std::unique_ptr<PImpl> pimpl_;
|
||||
};
|
||||
}
|
||||
@@ -129,7 +129,7 @@ NativeOrderBase *OrderFactory::make_order(const api::Venue exchange) {
|
||||
return new ICENativeOrder();
|
||||
else if (exchange == api::VENUE_GLBX || exchange == api::VENUE_SIM)
|
||||
return new GLBXNativeOrder();
|
||||
return new EMNativeOrder();
|
||||
return new EMNativeOrder(exchange);
|
||||
}
|
||||
|
||||
NativeOrderBase *OrderFactory::make_native_order(
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
#include "lib/config/protos/toggles.pb.h"
|
||||
#include "lib/domain/OrderStatus.h"
|
||||
#include "lib/domain_model/Security.h"
|
||||
#include "lib/domain_model/Security.pb.h"
|
||||
#include "lib/domain_model/common.h"
|
||||
#include "lib/jetfix/codec/FIXFragment.h"
|
||||
#include "lib/jetfix/codec/FieldDefinitions.h"
|
||||
@@ -52,10 +53,10 @@ struct InflightUpdateState {
|
||||
};
|
||||
|
||||
struct EMNativeOrder::PImpl {
|
||||
PImpl()
|
||||
PImpl(const api::Venue exchange)
|
||||
: instance_id(EMNativeOrder::count++), services(nullptr),
|
||||
session_services(), sec(nullptr), previous_working_position(0),
|
||||
definition(), state(), exchange_order_id() {}
|
||||
definition(), state(), exchange_order_id(), exchange(exchange) {}
|
||||
const int instance_id;
|
||||
bool inited = false;
|
||||
OrderServices *services;
|
||||
@@ -67,7 +68,7 @@ struct EMNativeOrder::PImpl {
|
||||
OrderDefinition definition;
|
||||
OrderState state;
|
||||
std::optional<ExchangeOrderIDVar> exchange_order_id;
|
||||
|
||||
const api::Venue exchange;
|
||||
OrderServices::ExeReportSubID exe_report_sid;
|
||||
OrderServices::CancelRejectSubID cancel_reject_sid;
|
||||
// If a cancel is sent before the order placement is ack'ed this is set to
|
||||
@@ -82,7 +83,7 @@ struct EMNativeOrder::PImpl {
|
||||
RoutingRecord routing_record;
|
||||
};
|
||||
|
||||
EMNativeOrder::EMNativeOrder() : me(new PImpl()) {
|
||||
EMNativeOrder::EMNativeOrder(const api::Venue exchange) : me(new PImpl(exchange)) {
|
||||
LOG(OrderMan, Info,
|
||||
"EMNativeOrder::EMNativeOrder instance_id: {} t_id: {} t_name: {}",
|
||||
me->instance_id, os::get_thread_id(), os::get_application_name());
|
||||
@@ -342,6 +343,10 @@ OrderOpResult EMNativeOrder::on_cancel() {
|
||||
case ORDER_STATUS_WORKING: {
|
||||
// TODO, CancelOrderMsg is not completely filled
|
||||
CancelOrderMsg cxl_msg = {};
|
||||
cxl_msg.side = me->definition.side();
|
||||
cxl_msg.security = me->sec;
|
||||
// cxl_msg.exchange_order_id=me->exchange_order_id;//handle this
|
||||
cxl_msg.routing_record = &me->routing_record;
|
||||
cxl_msg.key = key();
|
||||
LOG(OrderMan, Info, "EMNativeOrder queue_order_msg for cxl");
|
||||
me->services->queue_order_msg(*me->session_services, cxl_msg);
|
||||
@@ -501,11 +506,22 @@ OrderOpResult EMNativeOrder::_on_activate(
|
||||
me->state.set_filled_quantity(0);
|
||||
me->state.set_last_price(0);
|
||||
me->state.set_avg_price(0);
|
||||
|
||||
me->session_services = &me->services->get_session_services(
|
||||
get_native_state(me->state).routing_record().session());
|
||||
// TODO: just to get going
|
||||
me->session_services = &me->services->get_session_services("deribit_1");
|
||||
std::string session_name;
|
||||
switch (me->exchange) {
|
||||
case api::VENUE_CBOE:
|
||||
session_name = "cboe_1";
|
||||
break;
|
||||
case api::VENUE_TAIFEX:
|
||||
session_name="taifex_1";
|
||||
break;
|
||||
default:
|
||||
session_name ="deribit_1";
|
||||
|
||||
}
|
||||
me->session_services = &me->services->get_session_services(session_name);
|
||||
|
||||
// Checks for the following:
|
||||
// 1. Gateway connection
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include "lib/api/services/OrderService.pb.h"
|
||||
#include "lib/domain_model/Security.pb.h"
|
||||
#include "lib/domain_model/common.h"
|
||||
#include "lib/orderman/CreateOrderMsg.h"
|
||||
#include "lib/orderman/orders/NativeOrderBase.h"
|
||||
@@ -19,7 +20,7 @@ class SessionServices;
|
||||
// Wrapper around Emumba's order
|
||||
class EMNativeOrder : public NativeOrderBase {
|
||||
public:
|
||||
EMNativeOrder();
|
||||
EMNativeOrder(const api::Venue exchange = api::VENUE_DERIBIT);
|
||||
~EMNativeOrder() override;
|
||||
api::orderman::OrderDefinition const &definition() const override;
|
||||
void on_init_new(
|
||||
|
||||
@@ -478,7 +478,7 @@ OrderOpResult GLBXNativeOrder::on_activate(
|
||||
now.to_uint64(), (now - corid.app_arrival).nanoseconds());
|
||||
// in sim mode, we dont need this,
|
||||
me->encoder->send_create(*me->services, *me->session_services);
|
||||
if (me->in_sim_mode) {
|
||||
if (me->in_sim_mode) { //CME EXECUTION NEEDS TO BE UPDATED
|
||||
CreateOrderMsg c_msg = {};
|
||||
_create_create_msg(c_msg);
|
||||
me->services->queue_order_msg(*me->session_services, c_msg);
|
||||
@@ -579,7 +579,7 @@ OrderOpResult GLBXNativeOrder::on_update(api::orderman::Update const &update) {
|
||||
auto p = Price(update.price());
|
||||
auto q = Quantity(update.quantity());
|
||||
auto send_update_in_sim_mode = [&]() {
|
||||
if (me->in_sim_mode) {
|
||||
if (me->in_sim_mode) { //CME EXECUTION NEEDS TO BE UPDATED
|
||||
CancelReplaceOrderMsg cxl_repl_msg = {};
|
||||
_create_create_msg(cxl_repl_msg.create_fields);
|
||||
Price new_p = Price(update.price());
|
||||
@@ -799,7 +799,7 @@ OrderOpResult GLBXNativeOrder::on_cancel() {
|
||||
me->encoder->update_cancel_encoding(*me->services);
|
||||
me->encoder->send_cancel(*me->services, *me->session_services);
|
||||
// TODO: check for sim mode here
|
||||
if (me->in_sim_mode) {
|
||||
if (me->in_sim_mode) { //CME EXECUTION NEEDS TO BE UPDATED
|
||||
CancelOrderMsg cxl_msg = {};
|
||||
cxl_msg.key = key();
|
||||
LOG(OrderMan, Info, "GLBXNativeOrder queue_order_msg for cxl");
|
||||
@@ -1381,7 +1381,7 @@ void GLBXNativeOrder::_create_create_msg(CreateOrderMsg &c) {
|
||||
bts::orderman::get_quantity(me->state, StateQuantityType::OrderQuantity);
|
||||
c.security = me->sec;
|
||||
c.side = me->definition.side();
|
||||
c.order_type = get_glbx_native_ord_type(me->definition);
|
||||
c.order_type = get_glbx_native_ord_type(me->definition); //CME EXECUTION NEEDS TO BE UPDATED
|
||||
c.time_in_force = me->definition.time_in_force();
|
||||
c.order_type = api::orderman::NATIVE_ORDER_TYPE_LIMIT;
|
||||
c.manual_order_indicator = me->definition.manual_order_indicator();
|
||||
|
||||
@@ -277,6 +277,7 @@ void iLink2OrderEncoder::update_exchange_order_id(ExchangeOrderIDVar const &v) {
|
||||
auto oid = to_trim_str(std::get<ExchangeOrderID>(v));
|
||||
me->update_encoding.add<jetfix::fix::OrderID>(oid);
|
||||
me->cancel_encoding.add<jetfix::fix::OrderID>(oid);
|
||||
//CME EXECUTION NEEDS TO BE UPDATED, can be commented out
|
||||
}
|
||||
|
||||
void iLink2OrderEncoder::update_create_order_encoding(
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
#include "spark_data_directory.h"
|
||||
#include "lib/config/config.h"
|
||||
#include "lib/config/protos/system_config.pb.h"
|
||||
#include "lib/os/os.h"
|
||||
|
||||
#include <boost/filesystem/path.hpp>
|
||||
#include <fmt/format.h>
|
||||
#include <string>
|
||||
|
||||
namespace bts::spark {
|
||||
|
||||
SparkDataDirectory::SparkDataDirectory()
|
||||
: SparkDataDirectory(config::get<config::SystemConfig>().data_path()) {}
|
||||
|
||||
SparkDataDirectory::SparkDataDirectory(const std::string &root) : root_(root) {}
|
||||
|
||||
boost::filesystem::path SparkDataDirectory::root() const { return root_; }
|
||||
|
||||
boost::filesystem::path SparkDataDirectory::log_dir() const {
|
||||
return root_ / fmt::format("logging.{}/", os::get_application_name());
|
||||
}
|
||||
|
||||
boost::filesystem::path SparkDataDirectory::log_file() const {
|
||||
return log_dir() / "log";
|
||||
}
|
||||
|
||||
boost::filesystem::path SparkDataDirectory::log_cache_file() const {
|
||||
return log_dir() / "log_cache";
|
||||
}
|
||||
|
||||
boost::filesystem::path SparkDataDirectory::database_dir(
|
||||
const std::string &name) const {
|
||||
return root_ / "databases" / (name + "/");
|
||||
}
|
||||
|
||||
boost::filesystem::path SparkDataDirectory::database_file(
|
||||
const std::string &name) const {
|
||||
return root_ / "databases" / name / (name + ".sqlite");
|
||||
}
|
||||
|
||||
boost::filesystem::path SparkDataDirectory::database_file(
|
||||
const std::string &dir_name, const std::string &db_name) const {
|
||||
return root_ / "databases" / dir_name / (db_name + ".sqlite");
|
||||
}
|
||||
|
||||
} // namespace bts::spark
|
||||
@@ -0,0 +1,28 @@
|
||||
#pragma once
|
||||
|
||||
#include <boost/filesystem/path.hpp>
|
||||
#include <string>
|
||||
|
||||
namespace bts::spark {
|
||||
|
||||
class SparkDataDirectory {
|
||||
public:
|
||||
SparkDataDirectory();
|
||||
|
||||
explicit SparkDataDirectory(const std::string &root);
|
||||
|
||||
boost::filesystem::path root() const;
|
||||
boost::filesystem::path log_dir() const;
|
||||
boost::filesystem::path log_file() const;
|
||||
boost::filesystem::path log_cache_file() const;
|
||||
|
||||
boost::filesystem::path database_dir(const std::string &name) const;
|
||||
boost::filesystem::path database_file(const std::string &name) const;
|
||||
boost::filesystem::path database_file(
|
||||
const std::string &dir_name, const std::string &db_name) const;
|
||||
|
||||
private:
|
||||
boost::filesystem::path root_;
|
||||
};
|
||||
|
||||
} // namespace bts::spark
|
||||
@@ -9,6 +9,8 @@
|
||||
|
||||
namespace bts::persistence {
|
||||
|
||||
using bts::api::domain::SecurityType;
|
||||
using bts::api::domain::SecurityType_Name;
|
||||
using bts::api::domain::Venue;
|
||||
using bts::api::domain::Venue_Name;
|
||||
using std::string;
|
||||
@@ -52,24 +54,27 @@ uint64_t get_next_id(
|
||||
}
|
||||
|
||||
std::optional<uint64_t> IDGenerator::get_id(
|
||||
const Venue &venue, const string &name, bool add_if_new) {
|
||||
return get_id(Venue_Name(venue), name, add_if_new);
|
||||
const Venue &venue,
|
||||
SecurityType type,
|
||||
const string &name,
|
||||
bool add_if_new) {
|
||||
std::string ns = fmt::format(
|
||||
"{}.{}", Venue_Name(venue).substr(6),
|
||||
SecurityType_Name(type).substr(14));
|
||||
return get_id(ns, name, add_if_new);
|
||||
}
|
||||
|
||||
std::optional<uint64_t> IDGenerator::get_id(
|
||||
const std::string &ns, const std::string &key, bool add_if_new) {
|
||||
std::lock_guard<std::mutex> myLock(_instance_mx);
|
||||
auto [exists, val] = me->kv_store->retrieve_one(ns, key);
|
||||
auto [exists, rowid, val] = me->kv_store->retrieve_one(ns, key);
|
||||
if (exists) {
|
||||
uint64_t rv = str_to_id(val);
|
||||
LOG(Gen, Diag, "IDGenerator::get_id ns: {} key:{} id: {}", ns, key, rv);
|
||||
return std::optional<uint64_t>(rv);
|
||||
|
||||
LOG(Gen, Diag, "IDGenerator::get_id ns: {} key: id: {}", ns, key, rowid);
|
||||
return std::optional<uint64_t>(rowid);
|
||||
} else {
|
||||
if (add_if_new) {
|
||||
auto all = me->kv_store->retrieve_all();
|
||||
uint64_t new_id = get_next_id(all);
|
||||
me->kv_store->insert(ns, key, std::to_string(new_id));
|
||||
auto new_id = me->kv_store->insert(ns, key, "");
|
||||
LOG(Gen, Diag, "IDGenerator::get_id ns: {} key: id: {}", ns, key,
|
||||
new_id);
|
||||
return std::optional<uint64_t>(new_id);
|
||||
@@ -80,15 +85,11 @@ std::optional<uint64_t> IDGenerator::get_id(
|
||||
}
|
||||
}
|
||||
|
||||
std::uint64_t IDGenerator::get_or_add(const Venue &venue, const string &name) {
|
||||
return get_or_add(Venue_Name(venue), name);
|
||||
}
|
||||
|
||||
std::uint64_t IDGenerator::get_or_add(
|
||||
const std::string &ns, const std::string &key) {
|
||||
auto id = get_id(ns, key, true);
|
||||
bts_assert(id.has_value());
|
||||
return *id;
|
||||
api::domain::Venue venue,
|
||||
api::domain::SecurityType type,
|
||||
const std::string &name) {
|
||||
return *get_id(venue, type, name, true);
|
||||
}
|
||||
|
||||
IDGenerator::~IDGenerator() {}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
#pragma once
|
||||
|
||||
#include "lib/api/domain/SecurityType.pb.h"
|
||||
#include "lib/api/domain/Venue.pb.h"
|
||||
|
||||
#include <OnixS/CME/MDH/Integral.h>
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
@@ -21,6 +23,7 @@ class IDGenerator {
|
||||
|
||||
std::optional<uint64_t> get_id(
|
||||
const api::domain::Venue &venue,
|
||||
api::domain::SecurityType type,
|
||||
const std::string &name,
|
||||
bool add_if_new = false);
|
||||
|
||||
@@ -28,9 +31,9 @@ class IDGenerator {
|
||||
const std::string &ns, const std::string &key, bool add_if_new);
|
||||
|
||||
std::uint64_t get_or_add(
|
||||
const api::domain::Venue &venue, const std::string &name);
|
||||
|
||||
std::uint64_t get_or_add(const std::string &ns, const std::string &key);
|
||||
api::domain::Venue venue,
|
||||
api::domain::SecurityType type,
|
||||
const std::string &name);
|
||||
|
||||
private:
|
||||
static std::shared_ptr<IDGenerator> instance_ptr_;
|
||||
@@ -38,6 +41,10 @@ class IDGenerator {
|
||||
|
||||
struct PImpl;
|
||||
std::unique_ptr<PImpl> me;
|
||||
|
||||
// These methods are private to force callers to use the [public] overloads with venue, type and
|
||||
// name params, to ensure consistent formatting.
|
||||
std::uint64_t get_or_add(const std::string &ns, const std::string &key);
|
||||
};
|
||||
|
||||
} // namespace bts::persistence
|
||||
@@ -1,8 +1,13 @@
|
||||
#pragma once
|
||||
|
||||
#include "lib/persistence/store_row.h"
|
||||
|
||||
#include <cstdint>
|
||||
#include <map>
|
||||
#include <set>
|
||||
#include <string>
|
||||
#include <tuple>
|
||||
#include <vector>
|
||||
|
||||
namespace bts {
|
||||
namespace persistence {
|
||||
@@ -11,17 +16,45 @@ class IStore {
|
||||
public:
|
||||
virtual std::map<std::string, std::map<std::string, std::string>>
|
||||
retrieve_all() = 0;
|
||||
virtual std::pair<bool, std::string>
|
||||
retrieve_one(std::string const &ns, std::string const &k) = 0;
|
||||
|
||||
/**
|
||||
* Finds the row with the specified namespace and key.
|
||||
*
|
||||
* @return std::tuple<bool, std::int64, std::string> The first item indicates
|
||||
* whether or not a row was found. For successful queries, the second and third
|
||||
* items are the rowid and value, respectively.
|
||||
*/
|
||||
virtual std::tuple<bool, std::int64_t, std::string> retrieve_one(
|
||||
std::string const &ns, std::string const &k) = 0;
|
||||
virtual void
|
||||
remove(std::string const &ns, std::set<std::string> const &keys) = 0;
|
||||
virtual void remove_all(std::string const &ns) = 0;
|
||||
virtual void insert(
|
||||
virtual std::int64_t insert(
|
||||
std::string const &ns, std::string const &k, std::string const &v) = 0;
|
||||
|
||||
void insert_many(const std::vector<StoreRow> &rows);
|
||||
void insert_many_resolve_ids(std::vector<StoreRow> &rows);
|
||||
|
||||
virtual void update(
|
||||
std::string const &ns, std::string const &k, std::string const &v) = 0;
|
||||
virtual ~IStore();
|
||||
};
|
||||
|
||||
//////////////////////////////////////////////////
|
||||
// Definition
|
||||
//////////////////////////////////////////////////
|
||||
|
||||
inline void IStore::insert_many(const std::vector<StoreRow> &rows) {
|
||||
for (const auto &row : rows) {
|
||||
insert(row.ns, row.key, row.value);
|
||||
}
|
||||
}
|
||||
|
||||
inline void IStore::insert_many_resolve_ids(std::vector<StoreRow> &rows) {
|
||||
for (auto &row : rows) {
|
||||
row.id = insert(row.ns, row.key, row.value);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace persistence
|
||||
} // namespace bts
|
||||
+248
-122
@@ -18,82 +18,112 @@
|
||||
namespace bts {
|
||||
namespace persistence {
|
||||
|
||||
using bts::config::SystemConfig;
|
||||
|
||||
static boost::filesystem::path get_db_path(const std::string &str) {
|
||||
for (char c : str) {
|
||||
switch (c) {
|
||||
case '.':
|
||||
case '/':
|
||||
case '\\':
|
||||
// Looks like 'str' is already a path
|
||||
return {str};
|
||||
}
|
||||
}
|
||||
return boost::filesystem::path{config::get<SystemConfig>().data_path()} /
|
||||
"databases" / str / (str + ".sqlite");
|
||||
}
|
||||
|
||||
struct SqliteStore::PImpl {
|
||||
PImpl(std::string const &store_root, std::string const &db_name)
|
||||
: db(db_name), db_path(persistence::storage_location(db, store_root)),
|
||||
conn(nullptr, sqlite3_close), insert(nullptr, sqlite3_finalize),
|
||||
remove(nullptr, sqlite3_finalize),
|
||||
remove_all(nullptr, sqlite3_finalize),
|
||||
update(nullptr, sqlite3_finalize) {
|
||||
std::string const db;
|
||||
boost::filesystem::path db_path;
|
||||
sqlite3_ptr conn{nullptr, sqlite3_close};
|
||||
sqlite3_stmt_ptr insert{nullptr, sqlite3_finalize};
|
||||
sqlite3_stmt_ptr insert_many{nullptr, sqlite3_finalize};
|
||||
sqlite3_stmt_ptr insert_with_id{nullptr, sqlite3_finalize};
|
||||
sqlite3_stmt_ptr remove{nullptr, sqlite3_finalize};
|
||||
sqlite3_stmt_ptr remove_all{nullptr, sqlite3_finalize};
|
||||
sqlite3_stmt_ptr update{nullptr, sqlite3_finalize};
|
||||
|
||||
PImpl(const boost::filesystem::path &db_path)
|
||||
: db(db_path.stem().string()), db_path(db_path) {
|
||||
bts_assert(db.size());
|
||||
}
|
||||
|
||||
std::string const db;
|
||||
boost::filesystem::path db_path;
|
||||
sqlite3_ptr conn;
|
||||
sqlite3_stmt_ptr insert;
|
||||
sqlite3_stmt_ptr remove;
|
||||
sqlite3_stmt_ptr remove_all;
|
||||
sqlite3_stmt_ptr update;
|
||||
BTS_NO_COPY(PImpl);
|
||||
PImpl(std::string const &store_root, std::string const &db_name)
|
||||
: PImpl(boost::filesystem::path(store_root) / (db_name + ".sqlite")) {}
|
||||
|
||||
explicit PImpl(sqlite3_ptr conn) : db(), db_path(), conn(std::move(conn)) {}
|
||||
|
||||
PImpl(const PImpl &) = delete;
|
||||
PImpl &operator=(const PImpl &) = delete;
|
||||
};
|
||||
|
||||
SqliteStore::SqliteStore(std::string const &db)
|
||||
: SqliteStore(config::get<config::SystemConfig>().data_path(), db) {}
|
||||
: me(std::make_unique<PImpl>(get_db_path(db))) {
|
||||
LOG(persist, Info, "SqliteStore ctr. db: {}; db_path: {}", db,
|
||||
me->db_path.string());
|
||||
init_db();
|
||||
}
|
||||
|
||||
SqliteStore::SqliteStore(std::string const &root_path, std::string const &db)
|
||||
: me(new PImpl(root_path, db)) {
|
||||
SqliteStore::SqliteStore(sqlite3_ptr db)
|
||||
: me(std::make_unique<PImpl>(std::move(db))) {
|
||||
LOG(persist, Info, "SqliteStore ctr. db: <imported ptr>");
|
||||
init_db();
|
||||
}
|
||||
|
||||
void SqliteStore::init_db() {
|
||||
auto path = me->db_path;
|
||||
auto parent = path.parent_path();
|
||||
const auto path = me->db_path;
|
||||
|
||||
if (!boost::filesystem::is_directory(parent))
|
||||
bts_assert(boost::filesystem::create_directories(parent));
|
||||
|
||||
if (!boost::filesystem::exists(path)) {
|
||||
LOG(persist, Warn, "{} database does not exist yet. Creating it at {}",
|
||||
me->db, path.string());
|
||||
|
||||
me->conn = sqlite::open_db(path.c_str());
|
||||
bts_assert_msg(me->conn, path.c_str());
|
||||
|
||||
const char *create_schema_sql =
|
||||
"create table store ( ns text, key text, value text)";
|
||||
sqlite::exec_nofail(me->conn, create_schema_sql);
|
||||
|
||||
const char *key_index_sql =
|
||||
"create unique index key_index on store(ns,key)";
|
||||
sqlite::exec_nofail(me->conn, key_index_sql);
|
||||
|
||||
const char *ns_index_sql = "create index ns_index on store(ns)";
|
||||
sqlite::exec_nofail(me->conn, ns_index_sql);
|
||||
} else {
|
||||
me->conn = sqlite::open_db(path.c_str());
|
||||
if (!path.empty()) {
|
||||
if (!boost::filesystem::is_directory(path.parent_path())) {
|
||||
bts_assert(boost::filesystem::create_directories(path.parent_path()));
|
||||
}
|
||||
if (!boost::filesystem::exists(path)) {
|
||||
LOG(persist, Warn, "{} database does not exist yet. Creating it at {}",
|
||||
me->db, path.string());
|
||||
}
|
||||
}
|
||||
|
||||
auto &conn = me->conn;
|
||||
if (conn == nullptr) {
|
||||
conn = sqlite::open_db(path.c_str());
|
||||
bts_assert(conn != nullptr);
|
||||
}
|
||||
sqlite::exec_nofail(
|
||||
conn,
|
||||
"CREATE TABLE IF NOT EXISTS store (ns TEXT, key TEXT, value TEXT)");
|
||||
sqlite::exec_nofail(
|
||||
conn, "CREATE UNIQUE INDEX IF NOT EXISTS key_index on store(ns, key)");
|
||||
sqlite::exec_nofail(
|
||||
conn, "CREATE INDEX IF NOT EXISTS ns_index on store(ns)");
|
||||
conn = sqlite::open_db(path.c_str());
|
||||
|
||||
// Asynchronous for speed
|
||||
sqlite::exec_nofail(me->conn, "PRAGMA synchronous = 0;");
|
||||
sqlite::exec_nofail(me->conn, "PRAGMA journal_mode=WAL");
|
||||
sqlite::exec_nofail(conn, "PRAGMA synchronous = 0;");
|
||||
sqlite::exec_nofail(conn, "PRAGMA journal_mode=WAL");
|
||||
|
||||
// prepare insert statement
|
||||
const char *insert_sql = "insert into store (ns, key, value) values(?,?,?)";
|
||||
me->insert = sqlite::prepare_query(me->conn, insert_sql);
|
||||
me->insert = sqlite::prepare_query(conn, insert_sql);
|
||||
|
||||
// prepare insert_with_id statement
|
||||
const char *insert_with_id_sql =
|
||||
"insert into store (rowid, ns, key, value) values (?,?,?,?)";
|
||||
me->insert_with_id = sqlite::prepare_query(conn, insert_with_id_sql);
|
||||
|
||||
// prepare update statement
|
||||
const char *update_sql =
|
||||
"update store set value = ? where ns = ? and key = ?";
|
||||
me->update = sqlite::prepare_query(me->conn, update_sql);
|
||||
me->update = sqlite::prepare_query(conn, update_sql);
|
||||
|
||||
// prepare remove statement
|
||||
const char *remove_all_sql = "delete from store where ns = ?";
|
||||
me->remove_all = sqlite::prepare_query(me->conn, remove_all_sql);
|
||||
me->remove_all = sqlite::prepare_query(conn, remove_all_sql);
|
||||
|
||||
// prepare remove statement
|
||||
const char *remove_sql = "delete from store where ns = ? and key = ?";
|
||||
me->remove = sqlite::prepare_query(me->conn, remove_sql);
|
||||
me->remove = sqlite::prepare_query(conn, remove_sql);
|
||||
}
|
||||
|
||||
SqliteStore::~SqliteStore() {}
|
||||
@@ -122,25 +152,27 @@ SqliteStore::Data SqliteStore::retrieve_all() {
|
||||
return result;
|
||||
}
|
||||
|
||||
std::pair<bool, std::string>
|
||||
SqliteStore::retrieve_one(std::string const &ns, std::string const &k) {
|
||||
std::tuple<bool, std::int64_t, std::string> SqliteStore::retrieve_one(
|
||||
std::string const &ns, std::string const &k) {
|
||||
auto query_str = fmt::format(
|
||||
"select ns,key,value from store where ns='{}' and key='{}'", ns, k);
|
||||
"select rowid,ns,key,value from store where ns='{}' and key='{}'", ns,
|
||||
k);
|
||||
sqlite3_stmt_ptr query = sqlite::prepare_query(me->conn, query_str.c_str());
|
||||
|
||||
std::pair<bool, std::string> result{false, ""};
|
||||
|
||||
auto result = std::make_tuple<bool, std::int64_t, std::string>(false, 0, "");
|
||||
auto pq = query.get();
|
||||
auto sqlite_result = sqlite3_step(pq);
|
||||
if (sqlite_result == SQLITE_ROW) {
|
||||
std::int64_t rowid = sqlite3_column_int64(pq, 0);
|
||||
std::string ns(
|
||||
(char *)sqlite3_column_text(pq, 0), sqlite3_column_bytes(pq, 0));
|
||||
std::string key(
|
||||
(char *)sqlite3_column_text(pq, 1), sqlite3_column_bytes(pq, 1));
|
||||
std::string key(
|
||||
(char *)sqlite3_column_text(pq, 2), sqlite3_column_bytes(pq, 2));
|
||||
std::string val(
|
||||
(char *)sqlite3_column_blob(pq, 2), sqlite3_column_bytes(pq, 2));
|
||||
result.first = true;
|
||||
result.second = std::move(val);
|
||||
(char *)sqlite3_column_blob(pq, 3), sqlite3_column_bytes(pq, 3));
|
||||
std::get<0>(result) = true;
|
||||
std::get<1>(result) = rowid;
|
||||
std::get<2>(result) = std::move(val);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
@@ -149,91 +181,185 @@ void SqliteStore::remove(
|
||||
std::string const &ns, std::set<std::string> const &keys) {
|
||||
bts_assert(ns.size());
|
||||
|
||||
for (auto &k : keys) {
|
||||
SQLITE_CHK(
|
||||
sqlite3_bind_text(
|
||||
me->remove.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT),
|
||||
me->conn);
|
||||
SQLITE_CHK(
|
||||
sqlite3_bind_text(
|
||||
me->remove.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT),
|
||||
me->conn);
|
||||
auto r = sqlite3_step(me->remove.get());
|
||||
if (r != SQLITE_DONE) {
|
||||
LOG(persist, Error,
|
||||
"Could not delete row in store. Will retry once. error_code: {} "
|
||||
"ns:{}, k: {}",
|
||||
r, ns, k);
|
||||
sleep_for(2s);
|
||||
r = sqlite3_step(me->remove.get());
|
||||
try {
|
||||
for (auto &k : keys) {
|
||||
SQLITE_CHK(
|
||||
sqlite3_bind_text(
|
||||
me->remove.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT),
|
||||
me->conn);
|
||||
SQLITE_CHK(
|
||||
sqlite3_bind_text(
|
||||
me->remove.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT),
|
||||
me->conn);
|
||||
auto r = sqlite3_step(me->remove.get());
|
||||
if (r != SQLITE_DONE) {
|
||||
LOG(persist, Error,
|
||||
"Failed a second time to delete row. Bailing out. error_code: "
|
||||
"{}",
|
||||
r);
|
||||
bts_assert(r == SQLITE_DONE);
|
||||
"Could not delete row in store. Will retry once. error_code: "
|
||||
"{} "
|
||||
"ns:{}, k: {}",
|
||||
r, ns, k);
|
||||
sleep_for(2s);
|
||||
r = sqlite3_step(me->remove.get());
|
||||
if (r != SQLITE_DONE) {
|
||||
LOG(persist, Error,
|
||||
"Failed a second time to delete row. Bailing out. "
|
||||
"error_code: "
|
||||
"{}",
|
||||
r);
|
||||
bts_assert(r == SQLITE_DONE);
|
||||
}
|
||||
}
|
||||
SQLITE_CHK(sqlite3_reset(me->remove.get()), me->conn);
|
||||
}
|
||||
SQLITE_CHK(sqlite3_reset(me->remove.get()), me->conn);
|
||||
} catch (const std::exception &ex) {
|
||||
LOG(persist, Error, "SqliteStore::remove threw. what: {}",
|
||||
std::string{ex.what()});
|
||||
}
|
||||
}
|
||||
|
||||
void SqliteStore::remove_all(std::string const &ns) {
|
||||
bts_assert(ns.size());
|
||||
SQLITE_CHK(
|
||||
sqlite3_bind_text(
|
||||
me->remove_all.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT),
|
||||
me->conn);
|
||||
auto r = sqlite3_step(me->remove_all.get());
|
||||
if (r != SQLITE_DONE) {
|
||||
LOG(persist, Error,
|
||||
"Could not delete all rows in store. Will retry once. error_code: {} "
|
||||
"ns:{}",
|
||||
r, ns);
|
||||
sleep_for(2s);
|
||||
r = sqlite3_step(me->remove_all.get());
|
||||
try {
|
||||
SQLITE_CHK(
|
||||
sqlite3_bind_text(
|
||||
me->remove_all.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT),
|
||||
me->conn);
|
||||
auto r = sqlite3_step(me->remove_all.get());
|
||||
if (r != SQLITE_DONE) {
|
||||
LOG(persist, Error,
|
||||
"Failed a second time to delete all rows. Bailing out. "
|
||||
"error_code: "
|
||||
"{}",
|
||||
r);
|
||||
bts_assert(r == SQLITE_DONE);
|
||||
"Could not delete all rows in store. Will retry once. error_code: "
|
||||
"{} "
|
||||
"ns:{}",
|
||||
r, ns);
|
||||
sleep_for(2s);
|
||||
r = sqlite3_step(me->remove_all.get());
|
||||
if (r != SQLITE_DONE) {
|
||||
LOG(persist, Error,
|
||||
"Failed a second time to delete all rows. Bailing out. "
|
||||
"error_code: "
|
||||
"{}",
|
||||
r);
|
||||
bts_assert(r == SQLITE_DONE);
|
||||
}
|
||||
}
|
||||
SQLITE_CHK(sqlite3_reset(me->remove_all.get()), me->conn);
|
||||
} catch (const std::exception &ex) {
|
||||
LOG(persist, Error, "SqliteStore::remove_all threw. what: {}",
|
||||
std::string{ex.what()});
|
||||
}
|
||||
SQLITE_CHK(sqlite3_reset(me->remove_all.get()), me->conn);
|
||||
}
|
||||
|
||||
void SqliteStore::insert(
|
||||
std::int64_t SqliteStore::insert(
|
||||
std::string const &ns, std::string const &k, std::string const &v) {
|
||||
SQLITE_CHK(
|
||||
sqlite3_bind_text(
|
||||
me->insert.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT),
|
||||
me->conn);
|
||||
SQLITE_CHK(
|
||||
sqlite3_bind_text(
|
||||
me->insert.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT),
|
||||
me->conn);
|
||||
SQLITE_CHK(
|
||||
sqlite3_bind_text(
|
||||
me->insert.get(), 3, v.c_str(), v.size(), SQLITE_TRANSIENT),
|
||||
me->conn);
|
||||
auto r = sqlite3_step(me->insert.get());
|
||||
if (r != SQLITE_DONE) {
|
||||
LOG(persist, Error,
|
||||
"Could not insert row in store. Will retry once. error_code: {} ns: "
|
||||
"{}, k: {}, v: {}",
|
||||
r, ns, k, v);
|
||||
sleep_for(2s);
|
||||
r = sqlite3_step(me->insert.get());
|
||||
try {
|
||||
SQLITE_CHK(
|
||||
sqlite3_bind_text(
|
||||
me->insert.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT),
|
||||
me->conn);
|
||||
SQLITE_CHK(
|
||||
sqlite3_bind_text(
|
||||
me->insert.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT),
|
||||
me->conn);
|
||||
SQLITE_CHK(
|
||||
sqlite3_bind_text(
|
||||
me->insert.get(), 3, v.c_str(), v.size(), SQLITE_TRANSIENT),
|
||||
me->conn);
|
||||
auto r = sqlite3_step(me->insert.get());
|
||||
if (r != SQLITE_DONE) {
|
||||
LOG(persist, Error,
|
||||
"Failed a second time to insert row. Bailing out. error_code: {}",
|
||||
r);
|
||||
bts_assert(r == SQLITE_DONE);
|
||||
"Could not insert row in store. Will retry once. error_code: {} "
|
||||
"ns: "
|
||||
"{}, k: {}, v: {}",
|
||||
r, ns, k, v);
|
||||
sleep_for(2s);
|
||||
r = sqlite3_step(me->insert.get());
|
||||
if (r != SQLITE_DONE) {
|
||||
LOG(persist, Error,
|
||||
"Failed a second time to insert row. Bailing out. error_code: "
|
||||
"{}",
|
||||
r);
|
||||
bts_assert(r == SQLITE_DONE);
|
||||
}
|
||||
}
|
||||
const auto rowid = sqlite3_last_insert_rowid(me->conn.get());
|
||||
SQLITE_CHK(sqlite3_reset(me->insert.get()), me->conn);
|
||||
return rowid;
|
||||
} catch (const std::exception &ex) {
|
||||
LOG(persist, Error, "SqliteStore::insert threw. what: {}",
|
||||
std::string{ex.what()});
|
||||
}
|
||||
SQLITE_CHK(sqlite3_reset(me->insert.get()), me->conn);
|
||||
return 0;
|
||||
}
|
||||
|
||||
void SqliteStore::insert_many(const std::vector<StoreRow> &rows) {
|
||||
LOG(md, Info, "SqliteStore::insert_many size: {}", rows.size());
|
||||
if (rows.empty()) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
sqlite::exec_nofail(me->conn.get(), "BEGIN TRANSACTION");
|
||||
sqlite3 *conn_rawp = me->conn.get();
|
||||
sqlite3_stmt *insert_rawp = me->insert.get();
|
||||
sqlite3_stmt *insert_with_id_rawp = me->insert_with_id.get();
|
||||
for (const auto &row : rows) {
|
||||
int col_idx = 1;
|
||||
sqlite3_stmt *stmt;
|
||||
if (row.id > 0) {
|
||||
stmt = insert_with_id_rawp;
|
||||
sqlite3_bind_int(stmt, col_idx++, row.id);
|
||||
} else {
|
||||
stmt = insert_rawp;
|
||||
}
|
||||
sqlite3_bind_text(
|
||||
stmt, col_idx++, row.ns.c_str(), row.ns.size(), SQLITE_STATIC);
|
||||
sqlite3_bind_text(
|
||||
stmt, col_idx, row.key.c_str(), row.key.size(), SQLITE_STATIC);
|
||||
int result = sqlite3_step(stmt);
|
||||
if (result == SQLITE_DONE || result == SQLITE_ROW) {
|
||||
sqlite3_last_insert_rowid(conn_rawp);
|
||||
}
|
||||
sqlite3_reset(stmt);
|
||||
}
|
||||
sqlite::exec_nofail(me->conn.get(), "COMMIT TRANSACTION");
|
||||
} catch (const std::exception &ex) {
|
||||
LOG(persist, Error, "SqliteStore::insert_many threw. what: {}",
|
||||
std::string{ex.what()});
|
||||
}
|
||||
}
|
||||
|
||||
void SqliteStore::insert_many_resolve_ids(std::vector<StoreRow> &rows) {
|
||||
LOG(md, Info, "SqliteStore::insert_many_resolve_ids size: {}", rows.size());
|
||||
if (rows.empty()) {
|
||||
return;
|
||||
}
|
||||
sqlite::exec_nofail(me->conn.get(), "BEGIN TRANSACTION");
|
||||
sqlite3 *conn_rawp = me->conn.get();
|
||||
sqlite3_stmt *insert_rawp = me->insert.get();
|
||||
sqlite3_stmt *insert_with_id_rawp = me->insert_with_id.get();
|
||||
for (auto &row : rows) {
|
||||
int col_idx = 1;
|
||||
sqlite3_stmt *stmt;
|
||||
if (row.id > 0) {
|
||||
stmt = insert_with_id_rawp;
|
||||
sqlite3_bind_int(stmt, col_idx++, row.id);
|
||||
} else {
|
||||
stmt = insert_rawp;
|
||||
}
|
||||
sqlite3_bind_text(
|
||||
stmt, col_idx++, row.ns.c_str(), row.ns.size(), SQLITE_STATIC);
|
||||
sqlite3_bind_text(
|
||||
stmt, col_idx, row.key.c_str(), row.key.size(), SQLITE_STATIC);
|
||||
int result = sqlite3_step(stmt);
|
||||
if (result == SQLITE_DONE || result == SQLITE_ROW) {
|
||||
if (row.id > 0) {
|
||||
bts_assert(sqlite3_last_insert_rowid(conn_rawp) == row.id);
|
||||
} else {
|
||||
row.id = sqlite3_last_insert_rowid(conn_rawp);
|
||||
}
|
||||
}
|
||||
sqlite3_reset(stmt);
|
||||
}
|
||||
sqlite::exec_nofail(me->conn.get(), "COMMIT TRANSACTION");
|
||||
}
|
||||
|
||||
void SqliteStore::update(
|
||||
|
||||
@@ -1,29 +1,39 @@
|
||||
#pragma once
|
||||
|
||||
#include "IStore.h"
|
||||
#include "lib/persistence/store_row.h"
|
||||
#include "lib/utils/Sqlite_utils.h"
|
||||
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
namespace bts {
|
||||
namespace persistence {
|
||||
|
||||
class SqliteStore : public IStore {
|
||||
public:
|
||||
SqliteStore(std::string const &storage_root, std::string const &db);
|
||||
SqliteStore(std::string const &db);
|
||||
explicit SqliteStore(sqlite3_ptr db);
|
||||
~SqliteStore();
|
||||
|
||||
static SqliteStore from_path(const std::string &full_path);
|
||||
|
||||
typedef std::map<std::string, std::map<std::string, std::string>> Data;
|
||||
Data retrieve_all() override;
|
||||
std::pair<bool, std::string>
|
||||
retrieve_one(std::string const &ns, std::string const &k) override;
|
||||
|
||||
void
|
||||
remove(std::string const &ns, std::set<std::string> const &keys) override;
|
||||
std::tuple<bool, std::int64_t, std::string> retrieve_one(
|
||||
std::string const &ns, std::string const &k) override;
|
||||
|
||||
void remove(
|
||||
std::string const &ns, std::set<std::string> const &keys) override;
|
||||
void remove_all(std::string const &ns) override;
|
||||
void insert(
|
||||
std::int64_t insert(
|
||||
std::string const &ns,
|
||||
std::string const &k,
|
||||
std::string const &v) override;
|
||||
void insert_many(const std::vector<StoreRow> &rows);
|
||||
void insert_many_resolve_ids(std::vector<StoreRow> &rows);
|
||||
void update(
|
||||
std::string const &ns,
|
||||
std::string const &k,
|
||||
@@ -33,6 +43,8 @@ class SqliteStore : public IStore {
|
||||
static std::string storage_location(std::string const &db);
|
||||
|
||||
private:
|
||||
SqliteStore();
|
||||
|
||||
struct PImpl;
|
||||
std::unique_ptr<PImpl> me;
|
||||
void init_db();
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
|
||||
namespace bts::persistence {
|
||||
|
||||
struct StoreRow {
|
||||
std::string ns;
|
||||
std::string key;
|
||||
std::string value;
|
||||
/**
|
||||
* Represents the unique 'rowid' in the sqlite db. A value of 0 indicates that
|
||||
* the rowid isn't known, and should either be generated by the db (for inserts),
|
||||
* or resolved by finding the row with matching ns, key, value (for retrieves).
|
||||
*/
|
||||
std::int64_t id{0};
|
||||
};
|
||||
|
||||
} // namespace bts::persistence
|
||||
@@ -16,7 +16,8 @@ namespace persistence {
|
||||
|
||||
TEST_CASE("PersistenceService Sqlite storage", "[persistence]") {
|
||||
utils::TempDir db_dir;
|
||||
SqliteStore st(db_dir.name(), "test_db");
|
||||
SqliteStore st(
|
||||
(boost::filesystem::path{db_dir.name()} / "test_db.sqlite").string());
|
||||
|
||||
SECTION("initial_db_is_empty") { REQUIRE(st.retrieve_all().empty()); }
|
||||
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <limits>
|
||||
#include <stdexcept>
|
||||
|
||||
namespace bts::system::math {
|
||||
|
||||
static std::uint32_t checked_ltou(std::int64_t l);
|
||||
static std::uint64_t checked_ltoul(std::int64_t l);
|
||||
|
||||
//////////////////////////////////////////////////
|
||||
// Definition
|
||||
//////////////////////////////////////////////////
|
||||
|
||||
inline std::uint32_t checked_ltou(std::int64_t l) {
|
||||
if (l < 0) {
|
||||
throw std::overflow_error{
|
||||
"Failed to convert int64 to uint32 (value < 0)"};
|
||||
}
|
||||
if (l >
|
||||
static_cast<std::int64_t>(std::numeric_limits<std::uint32_t>::max())) {
|
||||
throw std::overflow_error{
|
||||
"Failed to convert int64 to uint32. value > max)"};
|
||||
}
|
||||
return static_cast<std::uint32_t>(l);
|
||||
}
|
||||
|
||||
inline std::uint64_t checked_ltoul(std::int64_t l) {
|
||||
if (l < 0) {
|
||||
throw std::overflow_error{
|
||||
"Failed to convert int64 to uint64 (value < 0)"};
|
||||
}
|
||||
return static_cast<std::uint64_t>(l);
|
||||
}
|
||||
|
||||
} // namespace bts::system::math
|
||||
@@ -52,7 +52,7 @@ void add_or_update_to_table(
|
||||
const std::string &ns,
|
||||
const std::string &key,
|
||||
const std::string &val) {
|
||||
auto [exists, existing_val] = store->retrieve_one(ns, key);
|
||||
auto [exists, rowid, existing_val] = store->retrieve_one(ns, key);
|
||||
LOG(Gen, Info,
|
||||
"add_or_update ns: {} key: {} val: {} exists: {} existing_val: {}", ns,
|
||||
key, val, exists, existing_val);
|
||||
@@ -386,7 +386,8 @@ void AnalyticsServiceServer::save_report(
|
||||
const api::Response<::bts::api::ResultMessage> &resp) {
|
||||
Report report = report_arg;
|
||||
std::string key = std::to_string(report.id());
|
||||
auto [present, val] = me->reports_store->retrieve_one(me->reports_ns, key);
|
||||
auto [present, rowid, val] =
|
||||
me->reports_store->retrieve_one(me->reports_ns, key);
|
||||
if (!present) {
|
||||
int new_id = get_max_id(me->reports_store.get(), me->reports_ns) + 1;
|
||||
report.set_id(new_id);
|
||||
@@ -408,7 +409,8 @@ void AnalyticsServiceServer::delete_report(
|
||||
const bts::api::analytics::IntWrapper &id,
|
||||
const api::Response<::bts::api::ResultMessage> &resp) {
|
||||
std::string key = std::to_string(id.data());
|
||||
auto [present, val] = me->reports_store->retrieve_one(me->reports_ns, key);
|
||||
auto [present, rowid, val] =
|
||||
me->reports_store->retrieve_one(me->reports_ns, key);
|
||||
api::ResultMessage res;
|
||||
res.set_success(false);
|
||||
if (!present) {
|
||||
|
||||
@@ -581,10 +581,11 @@ void negate_id(api::Security *sec) {
|
||||
}
|
||||
|
||||
void ExchangeSimApplication::persist_book(OrderBook const &b) {
|
||||
auto existing = me->store->retrieve_one("books", b.sec_desc());
|
||||
auto [exists, rowid, oldval] =
|
||||
me->store->retrieve_one("books", b.sec_desc());
|
||||
auto val = utils::pb::to_string(b);
|
||||
|
||||
if (existing.first)
|
||||
if (exists)
|
||||
me->store->update("books", b.sec_desc(), val);
|
||||
else
|
||||
me->store->insert("books", b.sec_desc(), val);
|
||||
|
||||
@@ -246,21 +246,23 @@ void MarketDataGateway::unsubscribe(MarketDataListenerID id) {
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
void MarketDataGateway::start_glbx(const bts::api::md::MarketDataConfig &cfg){
|
||||
for (const auto &glbx : cfg.glbx()) {
|
||||
if(glbx.use_databento()){
|
||||
if(!glbx.use_databento()){
|
||||
_start_onix_glbx(glbx);
|
||||
return;
|
||||
}
|
||||
auto p = me_->venue_listeners.emplace_back(
|
||||
std::make_shared<VenueListenerData>(bts::api::VENUE_GLBX));
|
||||
// Capture `p->listener` and `me_->sec_id_store` by value
|
||||
auto listener = p->listener;
|
||||
auto sec_id_store = me_->sec_id_store;
|
||||
std::string tid_name = "glbx";
|
||||
bts::threading::make_thread(tid_name, [&]() {
|
||||
bts::threading::make_thread(tid_name, [listener, sec_id_store, glbx](){
|
||||
api::md::glbx::GlbxMarketDatahandler::run(
|
||||
p->listener,
|
||||
me_->sec_id_store,
|
||||
listener,
|
||||
sec_id_store,
|
||||
glbx);
|
||||
}).detach();
|
||||
}
|
||||
else{
|
||||
_start_onix_glbx(glbx);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
static void run_onixs() {
|
||||
LOG(md, Info, "MarketDataGateway::run_onixs t_id:{} t_name",
|
||||
@@ -295,7 +297,10 @@ void MarketDataGateway::_start_onix_glbx(
|
||||
"MarketDataGateway::_start_onix_glbx starting t_id:{} t_name:{}. "
|
||||
"cfg:\n{}",
|
||||
os::get_thread_id(), os::get_thread_name(), cfg.DebugString());
|
||||
bts::md::run_onixs();
|
||||
bts::threading::make_thread("Onix CME", [&]() {
|
||||
bts::md::run_onixs();
|
||||
}).detach();
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -374,35 +379,38 @@ void MarketDataGateway::_start_replays(
|
||||
|
||||
void MarketDataGateway::start_taifex(const bts::api::md::MarketDataConfig &cfg) {
|
||||
// taifex
|
||||
int counter = 0;
|
||||
for (const auto &taifex : cfg.taifex()) {
|
||||
counter++;
|
||||
auto p = me_->venue_listeners.emplace_back(
|
||||
std::make_shared<VenueListenerData>(bts::api::VENUE_TAIFEX));
|
||||
std::string tid_name = "taifex" +std::to_string(counter);
|
||||
bts::threading::make_thread(tid_name, [&]() {
|
||||
// Capture `p->listener` and `me_->sec_id_store` by value
|
||||
auto listener = p->listener;
|
||||
auto sec_id_store = me_->sec_id_store;
|
||||
std::string tid_name = "taifex";
|
||||
bts::threading::make_thread(tid_name, [listener, sec_id_store, taifex]() {
|
||||
bts::md::taifex::TaifexDataGateway::run(
|
||||
taifex, p->listener, me_->sec_id_store);
|
||||
taifex, listener, sec_id_store);
|
||||
}).detach();
|
||||
|
||||
}
|
||||
}
|
||||
void MarketDataGateway::start_cboe(const bts::api::md::MarketDataConfig &cfg){
|
||||
// cboe
|
||||
void MarketDataGateway::start_cboe(const bts::api::md::MarketDataConfig &cfg) {
|
||||
for (const api::md::CboeMarketDataConfig &cboe : cfg.cboe()) {
|
||||
auto p = me_->venue_listeners.emplace_back(
|
||||
std::make_shared<VenueListenerData>(bts::api::VENUE_CBOE));
|
||||
std::string tid_name = "CBOE";
|
||||
bts::threading::make_thread(tid_name, [&]() {
|
||||
api::md::cboe::CboeMarketDatahandler::run(
|
||||
p->listener,
|
||||
me_->sec_id_store,
|
||||
cboe);
|
||||
}).detach();
|
||||
auto p = me_->venue_listeners.emplace_back(
|
||||
std::make_shared<VenueListenerData>(bts::api::VENUE_CBOE));
|
||||
std::string tid_name = "CBOE";
|
||||
|
||||
// Capture `p->listener` and `me_->sec_id_store` by value
|
||||
auto listener = p->listener;
|
||||
auto sec_id_store = me_->sec_id_store;
|
||||
|
||||
bts::threading::make_thread(tid_name, [listener, sec_id_store, cboe]() {
|
||||
api::md::cboe::CboeMarketDatahandler::run(
|
||||
listener,
|
||||
sec_id_store,
|
||||
cboe);
|
||||
}).detach();
|
||||
|
||||
UNUSED(cboe);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
} // namespace bts::md
|
||||
@@ -10,20 +10,21 @@
|
||||
|
||||
namespace bts::md::binance {
|
||||
|
||||
using bts::api::domain::SecurityType;
|
||||
using bts::domain::SecurityID;
|
||||
using bts::domain::SecurityMarketDataStatistics;
|
||||
using std::pair;
|
||||
using std::string;
|
||||
using std::tuple;
|
||||
using std::vector;
|
||||
|
||||
#define VENUE_NAME (Venue_Name(Venue::BINANCE))
|
||||
|
||||
struct BinancePriceStatisticsAdaptor::PImpl {
|
||||
vector<pair<string, SecurityID>> securities;
|
||||
vector<tuple<SecurityType, string, SecurityID>> securities;
|
||||
IMarketDataListener *listener;
|
||||
|
||||
SecurityID get_sec_id(const string &sec_desc) const {
|
||||
for (const auto &[desc, id] : securities) {
|
||||
for (const auto &[type, desc, id] : securities) {
|
||||
if (desc == sec_desc) {
|
||||
return id;
|
||||
}
|
||||
@@ -36,7 +37,8 @@ struct BinancePriceStatisticsAdaptor::PImpl {
|
||||
};
|
||||
|
||||
BinancePriceStatisticsAdaptor::BinancePriceStatisticsAdaptor(
|
||||
vector<pair<string, SecurityID>> securities, IMarketDataListener *listener)
|
||||
vector<tuple<SecurityType, string, SecurityID>> securities,
|
||||
IMarketDataListener *listener)
|
||||
: me_(std::make_unique<PImpl>()) {
|
||||
bts_assert(listener != nullptr);
|
||||
me_->securities = std::move(securities);
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
#pragma once
|
||||
|
||||
#include "lib/api/domain/SecurityType.pb.h"
|
||||
#include "lib/domain/SecurityID.h"
|
||||
#include "server/sparkd/md/ccapi/ICCApiMessageHandler.h"
|
||||
|
||||
#include <string>
|
||||
#include <tuple>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
@@ -20,7 +22,10 @@ namespace bts::md::binance {
|
||||
class BinancePriceStatisticsAdaptor : public ICCApiMessageHandler {
|
||||
public:
|
||||
BinancePriceStatisticsAdaptor(
|
||||
std::vector<std::pair<std::string, domain::SecurityID>> securities,
|
||||
std::vector<std::tuple<
|
||||
api::domain::SecurityType,
|
||||
std::string,
|
||||
domain::SecurityID>> securities,
|
||||
IMarketDataListener *listener);
|
||||
|
||||
~BinancePriceStatisticsAdaptor();
|
||||
|
||||
@@ -49,6 +49,7 @@ using std::vector;
|
||||
#define _WARN(...) _LOG(Warn, __VA_ARGS__)
|
||||
#define _ERROR(...) _LOG(Error, __VA_ARGS__)
|
||||
|
||||
#define VENUE Venue::VENUE_BINANCE
|
||||
#define VENUE_NAME (Venue_Name(Venue::VENUE_BINANCE))
|
||||
#define CCAPI_VENUE_NAME CCAPI_EXCHANGE_NAME_BINANCE
|
||||
#define BINANCE_MSG_TYPE_NAME(x) \
|
||||
@@ -156,7 +157,8 @@ struct BinanceSecurityRequestor::PImpl {
|
||||
for (auto &sec : added_secs) {
|
||||
// Since the ids haven't been set yet, we should expect them to be 0
|
||||
bts_assert(sec.id().value() == 0);
|
||||
SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec.desc())};
|
||||
SecurityID sec_id{
|
||||
sec_id_store->get_or_add(VENUE, sec.type(), sec.desc())};
|
||||
sec.security_common.id = sec_id;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,6 +34,7 @@
|
||||
|
||||
namespace bts::md::binance {
|
||||
|
||||
using bts::api::domain::SecurityType;
|
||||
using bts::api::domain::Venue;
|
||||
using bts::api::domain::Venue_Name;
|
||||
using bts::api::md::BinanceMarketDataConfig;
|
||||
@@ -50,6 +51,7 @@ using std::unique_ptr;
|
||||
using std::unordered_map;
|
||||
using std::vector;
|
||||
|
||||
#define VENUE Venue::VENUE_BINANCE
|
||||
#define VENUE_NAME (Venue_Name(Venue::VENUE_BINANCE))
|
||||
#define CCAPI_VENUE_NAME CCAPI_EXCHANGE_NAME_BINANCE
|
||||
|
||||
@@ -123,14 +125,15 @@ static CCApiCorrelationID mk_stats_corr_id(const string &desc) {
|
||||
}
|
||||
|
||||
static ccapi::Request mk_stats_req(
|
||||
const vector<string> &sec_descs, uint32_t batch_id) {
|
||||
const vector<std::pair<SecurityType, string>> &sec_descs,
|
||||
uint32_t batch_id) {
|
||||
const auto corr_id = mk_stats_corr_id(FMT("BATCH_{}", batch_id));
|
||||
ccapi::Request req{
|
||||
ccapi::Request::Operation::GENERIC_PUBLIC_REQUEST, CCAPI_VENUE_NAME};
|
||||
req.setCorrelationId(corr_id.to_string());
|
||||
auto quoted_sec_descs =
|
||||
sec_descs |
|
||||
ranges::views::transform([](const auto &s) { return "\"" + s + "\""; });
|
||||
sec_descs | ranges::views::transform(
|
||||
[](const auto &s) { return "\"" + s.second + "\""; });
|
||||
const string symbols_query_arg = FMT_JOIN(quoted_sec_descs, ",");
|
||||
req.appendParam(
|
||||
{{CCAPI_HTTP_PATH, BINANCE_PRICE_STATS_PATH},
|
||||
@@ -209,16 +212,18 @@ struct BinanceSubscription::PImpl {
|
||||
sec_requestor.request();
|
||||
}
|
||||
|
||||
DepthHandler &mk_depth_handler(const string &sec_desc) {
|
||||
SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
|
||||
DepthHandler &mk_depth_handler(
|
||||
SecurityType sec_type, const string &sec_desc) {
|
||||
SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
|
||||
auto [it, is_new] =
|
||||
depth_handlers.try_emplace(sec_desc, sec_id, listener);
|
||||
bts_assert(is_new);
|
||||
return it->second;
|
||||
}
|
||||
|
||||
TickerHandler &mk_ticker_handler(const string &sec_desc) {
|
||||
SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
|
||||
TickerHandler &mk_ticker_handler(
|
||||
SecurityType sec_type, const string &sec_desc) {
|
||||
SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
|
||||
auto [it, is_new] =
|
||||
ticker_handlers.try_emplace(sec_desc, sec_id, listener);
|
||||
bts_assert_msgf(
|
||||
@@ -227,23 +232,25 @@ struct BinanceSubscription::PImpl {
|
||||
}
|
||||
|
||||
StatsHandler &mk_stats_handler(
|
||||
const vector<string> &sec_descs, uint32_t hnd_id) {
|
||||
vector<pair<string, SecurityID>> sec_desc_id_pairs;
|
||||
sec_desc_id_pairs.reserve(sec_descs.size());
|
||||
for (const auto &sec_desc : sec_descs) {
|
||||
sec_desc_id_pairs.emplace_back(
|
||||
sec_desc,
|
||||
SecurityID{sec_id_store->get_or_add(VENUE_NAME, sec_desc)});
|
||||
const vector<std::pair<SecurityType, string>> &sec_descs,
|
||||
uint32_t hnd_id) {
|
||||
vector<std::tuple<SecurityType, string, SecurityID>> sec_desc_id_tuples;
|
||||
sec_desc_id_tuples.reserve(sec_descs.size());
|
||||
for (auto [type, name] : sec_descs) {
|
||||
SecurityID id{sec_id_store->get_or_add(VENUE, type, name)};
|
||||
sec_desc_id_tuples.emplace_back(type, std::move(name), id);
|
||||
}
|
||||
auto [it, is_new] = stats_handlers.try_emplace(
|
||||
hnd_id, std::make_unique<StatsHandler>(sec_desc_id_pairs, listener));
|
||||
hnd_id,
|
||||
std::make_unique<StatsHandler>(sec_desc_id_tuples, listener));
|
||||
bts_assert_msgf(
|
||||
is_new, "StatsHandler already created with id '{}'", hnd_id);
|
||||
return *it->second;
|
||||
}
|
||||
|
||||
void add_stats_handler(
|
||||
const vector<string> &sec_descs, CCApiSession::BatchRequest &reqs) {
|
||||
const vector<std::pair<SecurityType, string>> &sec_descs,
|
||||
CCApiSession::BatchRequest &reqs) {
|
||||
if (sec_descs.empty()) {
|
||||
return;
|
||||
}
|
||||
@@ -268,14 +275,14 @@ struct BinanceSubscription::PImpl {
|
||||
CCApiSession::BatchRequest stats_reqs;
|
||||
|
||||
const int N_SECS_PER_STATS_REQ = 16;
|
||||
vector<string> curr_stats_sec_descs;
|
||||
for (const auto &sec_desc : securities) {
|
||||
vector<std::pair<SecurityType, string>> curr_stats_sec_descs;
|
||||
for (const auto &[type, desc] : securities) {
|
||||
depth_subs.push_back(
|
||||
{mk_depth_sub(sec_desc, depth_opts), &mk_depth_handler(sec_desc)});
|
||||
{mk_depth_sub(desc, depth_opts), &mk_depth_handler(type, desc)});
|
||||
ticker_subs.push_back(
|
||||
{mk_ticker_sub(sec_desc, ticker_opts),
|
||||
&mk_ticker_handler(sec_desc)});
|
||||
curr_stats_sec_descs.push_back(sec_desc);
|
||||
{mk_ticker_sub(desc, ticker_opts),
|
||||
&mk_ticker_handler(type, desc)});
|
||||
curr_stats_sec_descs.push_back(std::make_pair(type, desc));
|
||||
if (curr_stats_sec_descs.size() == N_SECS_PER_STATS_REQ) {
|
||||
add_stats_handler(curr_stats_sec_descs, stats_reqs);
|
||||
curr_stats_sec_descs.clear();
|
||||
@@ -299,7 +306,7 @@ struct BinanceSubscription::PImpl {
|
||||
*/
|
||||
void on_securities(vector<Security> secs) {
|
||||
// 'sec_descs' is used to make depth subscriptions
|
||||
vector<string> sec_descs;
|
||||
vector<pair<SecurityType, string>> sec_descs;
|
||||
sec_descs.reserve(secs.size());
|
||||
// 'sec_ptrs' is passed to IMarketDataListeners
|
||||
vector<const Security *> sec_ptrs;
|
||||
@@ -312,7 +319,8 @@ struct BinanceSubscription::PImpl {
|
||||
id, std::make_unique<Security>(std::move(s)));
|
||||
bts_assert_msgf(
|
||||
is_new, "Received duplicate Binance security '{}'", desc);
|
||||
sec_descs.push_back(std::move(desc));
|
||||
sec_descs.push_back(
|
||||
std::make_pair(it->second->type(), std::move(desc)));
|
||||
// add ptr to entry in 'securities' (not the loop var, we moved that)
|
||||
sec_ptrs.push_back(it->second.get());
|
||||
}
|
||||
|
||||
@@ -10,20 +10,21 @@
|
||||
|
||||
namespace bts::md::binance_us {
|
||||
|
||||
using bts::api::domain::SecurityType;
|
||||
using bts::domain::SecurityID;
|
||||
using bts::domain::SecurityMarketDataStatistics;
|
||||
using std::pair;
|
||||
using std::string;
|
||||
using std::tuple;
|
||||
using std::vector;
|
||||
|
||||
#define VENUE_NAME (Venue_Name(Venue::BINANCE))
|
||||
|
||||
struct BinanceUsPriceStatisticsAdaptor::PImpl {
|
||||
vector<pair<string, SecurityID>> securities;
|
||||
vector<tuple<SecurityType, string, SecurityID>> securities;
|
||||
IMarketDataListener *listener;
|
||||
|
||||
SecurityID get_sec_id(const string &sec_desc) const {
|
||||
for (const auto &[desc, id] : securities) {
|
||||
for (const auto &[type, desc, id] : securities) {
|
||||
if (desc == sec_desc) {
|
||||
return id;
|
||||
}
|
||||
@@ -36,7 +37,8 @@ struct BinanceUsPriceStatisticsAdaptor::PImpl {
|
||||
};
|
||||
|
||||
BinanceUsPriceStatisticsAdaptor::BinanceUsPriceStatisticsAdaptor(
|
||||
vector<pair<string, SecurityID>> securities, IMarketDataListener *listener)
|
||||
vector<tuple<SecurityType, string, SecurityID>> securities,
|
||||
IMarketDataListener *listener)
|
||||
: me_(std::make_unique<PImpl>()) {
|
||||
bts_assert(listener != nullptr);
|
||||
me_->securities = std::move(securities);
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
#pragma once
|
||||
|
||||
#include "lib/api/domain/SecurityType.pb.h"
|
||||
#include "lib/domain/SecurityID.h"
|
||||
#include "server/sparkd/md/ccapi/ICCApiMessageHandler.h"
|
||||
|
||||
#include <string>
|
||||
#include <tuple>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
@@ -25,7 +27,10 @@ namespace bts::md::binance_us {
|
||||
class BinanceUsPriceStatisticsAdaptor : public ICCApiMessageHandler {
|
||||
public:
|
||||
BinanceUsPriceStatisticsAdaptor(
|
||||
std::vector<std::pair<std::string, domain::SecurityID>> securities,
|
||||
std::vector<std::tuple<
|
||||
api::domain::SecurityType,
|
||||
std::string,
|
||||
domain::SecurityID>> securities,
|
||||
IMarketDataListener *listener);
|
||||
|
||||
~BinanceUsPriceStatisticsAdaptor();
|
||||
|
||||
@@ -156,7 +156,8 @@ struct BinanceUsSecurityRequestor::PImpl {
|
||||
for (auto &sec : added_secs) {
|
||||
// Since the ids haven't been set yet, we should expect them to be 0
|
||||
bts_assert(sec.id().value() == 0);
|
||||
SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec.desc())};
|
||||
SecurityID sec_id{
|
||||
sec_id_store->get_or_add(VENUE, sec.type(), sec.desc())};
|
||||
sec.security_common.id = sec_id;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,6 +36,7 @@
|
||||
|
||||
namespace bts::md::binance_us {
|
||||
|
||||
using bts::api::domain::SecurityType;
|
||||
using bts::api::domain::Venue;
|
||||
using bts::api::domain::Venue_Name;
|
||||
using bts::api::md::BinanceUsMarketDataConfig;
|
||||
@@ -47,11 +48,13 @@ using bts::io::QueryParams;
|
||||
using bts::persistence::IDGenerator;
|
||||
using std::pair;
|
||||
using std::string;
|
||||
using std::tuple;
|
||||
using std::uint32_t;
|
||||
using std::unique_ptr;
|
||||
using std::unordered_map;
|
||||
using std::vector;
|
||||
|
||||
#define VENUE Venue::VENUE_BINANCE_US
|
||||
#define VENUE_NAME (Venue_Name(Venue::VENUE_BINANCE_US))
|
||||
#define CCAPI_VENUE_NAME CCAPI_EXCHANGE_NAME_BINANCE_US
|
||||
|
||||
@@ -126,14 +129,14 @@ static CCApiCorrelationID mk_stats_corr_id(const string &desc) {
|
||||
}
|
||||
|
||||
static ccapi::Request mk_stats_req(
|
||||
const vector<string> &sec_descs, uint32_t batch_id) {
|
||||
const vector<pair<SecurityType, string>> &sec_descs, uint32_t batch_id) {
|
||||
const auto corr_id = mk_stats_corr_id(FMT("BATCH_{}", batch_id));
|
||||
ccapi::Request req{
|
||||
ccapi::Request::Operation::GENERIC_PUBLIC_REQUEST, CCAPI_VENUE_NAME};
|
||||
req.setCorrelationId(corr_id.to_string());
|
||||
auto quoted_sec_descs =
|
||||
sec_descs |
|
||||
ranges::views::transform([](const auto &s) { return "\"" + s + "\""; });
|
||||
sec_descs | ranges::views::transform(
|
||||
[](const auto &s) { return "\"" + s.second + "\""; });
|
||||
const string symbols_query_arg = FMT_JOIN(quoted_sec_descs, ",");
|
||||
req.appendParam(
|
||||
{{CCAPI_HTTP_PATH, BINANCE_US_PRICE_STATS_PATH},
|
||||
@@ -212,16 +215,18 @@ struct BinanceUsSubscription::PImpl {
|
||||
sec_requestor.request();
|
||||
}
|
||||
|
||||
DepthHandler &mk_depth_handler(const string &sec_desc) {
|
||||
SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
|
||||
DepthHandler &mk_depth_handler(
|
||||
SecurityType sec_type, const string &sec_desc) {
|
||||
SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
|
||||
auto [it, is_new] =
|
||||
depth_handlers.try_emplace(sec_desc, sec_id, listener);
|
||||
bts_assert(is_new);
|
||||
return it->second;
|
||||
}
|
||||
|
||||
TickerHandler &mk_ticker_handler(const string &sec_desc) {
|
||||
SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
|
||||
TickerHandler &mk_ticker_handler(
|
||||
SecurityType sec_type, const string &sec_desc) {
|
||||
SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
|
||||
auto [it, is_new] =
|
||||
ticker_handlers.try_emplace(sec_desc, sec_id, listener);
|
||||
bts_assert_msgf(
|
||||
@@ -230,23 +235,25 @@ struct BinanceUsSubscription::PImpl {
|
||||
}
|
||||
|
||||
StatsHandler &mk_stats_handler(
|
||||
const vector<string> &sec_descs, uint32_t hnd_id) {
|
||||
vector<pair<string, SecurityID>> sec_desc_id_pairs;
|
||||
sec_desc_id_pairs.reserve(sec_descs.size());
|
||||
for (const auto &sec_desc : sec_descs) {
|
||||
sec_desc_id_pairs.emplace_back(
|
||||
sec_desc,
|
||||
SecurityID{sec_id_store->get_or_add(VENUE_NAME, sec_desc)});
|
||||
const vector<std::pair<SecurityType, string>> &sec_descs,
|
||||
uint32_t hnd_id) {
|
||||
vector<tuple<SecurityType, string, SecurityID>> sec_desc_id_tuples;
|
||||
sec_desc_id_tuples.reserve(sec_descs.size());
|
||||
for (auto [type, name] : sec_descs) {
|
||||
SecurityID id{sec_id_store->get_or_add(VENUE, type, name)};
|
||||
sec_desc_id_tuples.emplace_back(type, std::move(name), id);
|
||||
}
|
||||
auto [it, is_new] = stats_handlers.try_emplace(
|
||||
hnd_id, std::make_unique<StatsHandler>(sec_desc_id_pairs, listener));
|
||||
hnd_id,
|
||||
std::make_unique<StatsHandler>(sec_desc_id_tuples, listener));
|
||||
bts_assert_msgf(
|
||||
is_new, "StatsHandler already created with id '{}'", hnd_id);
|
||||
return *it->second;
|
||||
}
|
||||
|
||||
void add_stats_handler(
|
||||
const vector<string> &sec_descs, CCApiSession::BatchRequest &reqs) {
|
||||
const vector<pair<SecurityType, string>> &sec_descs,
|
||||
CCApiSession::BatchRequest &reqs) {
|
||||
if (sec_descs.empty()) {
|
||||
return;
|
||||
}
|
||||
@@ -271,14 +278,14 @@ struct BinanceUsSubscription::PImpl {
|
||||
CCApiSession::BatchRequest stats_reqs;
|
||||
|
||||
const int N_SECS_PER_STATS_REQ = 16;
|
||||
vector<string> curr_stats_sec_descs;
|
||||
for (const auto &sec_desc : securities) {
|
||||
vector<pair<SecurityType, string>> curr_stats_sec_descs;
|
||||
for (const auto &[type, name] : securities) {
|
||||
depth_subs.push_back(
|
||||
{mk_depth_sub(sec_desc, depth_opts), &mk_depth_handler(sec_desc)});
|
||||
{mk_depth_sub(name, depth_opts), &mk_depth_handler(type, name)});
|
||||
ticker_subs.push_back(
|
||||
{mk_ticker_sub(sec_desc, ticker_opts),
|
||||
&mk_ticker_handler(sec_desc)});
|
||||
curr_stats_sec_descs.push_back(sec_desc);
|
||||
{mk_ticker_sub(name, ticker_opts),
|
||||
&mk_ticker_handler(type, name)});
|
||||
curr_stats_sec_descs.push_back(std::make_pair(type, name));
|
||||
if (curr_stats_sec_descs.size() == N_SECS_PER_STATS_REQ) {
|
||||
add_stats_handler(curr_stats_sec_descs, stats_reqs);
|
||||
curr_stats_sec_descs.clear();
|
||||
@@ -302,7 +309,7 @@ struct BinanceUsSubscription::PImpl {
|
||||
*/
|
||||
void on_securities(vector<Security> secs) {
|
||||
// 'sec_descs' is used to make depth subscriptions
|
||||
vector<string> sec_descs;
|
||||
vector<pair<SecurityType, string>> sec_descs;
|
||||
sec_descs.reserve(secs.size());
|
||||
// 'sec_ptrs' is passed to IMarketDataListeners
|
||||
vector<const Security *> sec_ptrs;
|
||||
@@ -315,7 +322,7 @@ struct BinanceUsSubscription::PImpl {
|
||||
id, std::make_unique<Security>(std::move(s)));
|
||||
bts_assert_msgf(
|
||||
is_new, "Received duplicate Binance US security '{}'", desc);
|
||||
sec_descs.push_back(std::move(desc));
|
||||
sec_descs.push_back(std::make_pair(s.type(), std::move(desc)));
|
||||
// add ptr to entry in 'securities' (not the loop var, we moved that)
|
||||
sec_ptrs.push_back(it->second.get());
|
||||
}
|
||||
|
||||
@@ -48,6 +48,7 @@ using std::vector;
|
||||
#define _WARN(...) _LOG(Warn, __VA_ARGS__)
|
||||
#define _ERROR(...) _LOG(Error, __VA_ARGS__)
|
||||
|
||||
#define VENUE Venue::VENUE_BINANCE_USDS_FUTURES
|
||||
#define VENUE_NAME (Venue_Name(Venue::VENUE_BINANCE_USDS_FUTURES))
|
||||
#define CCAPI_VENUE_NAME CCAPI_EXCHANGE_NAME_BINANCE_USDS_FUTURES
|
||||
#define VENUE_PRICE_STATS_PATH BINANCE_USDS_FUTURES_PRICE_STATS_PATH
|
||||
@@ -157,7 +158,8 @@ struct BinanceUsdsFuturesSecurityRequestor::PImpl {
|
||||
// the ids, if it does (for whatever reason), better to assert/crash
|
||||
// than quietly let a potential bug slip through the cracks.
|
||||
bts_assert(sec.id().value() == 0);
|
||||
SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec.desc())};
|
||||
SecurityID sec_id{
|
||||
sec_id_store->get_or_add(VENUE, sec.type(), sec.desc())};
|
||||
sec.security_common.id = sec_id;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,6 +30,7 @@
|
||||
|
||||
namespace bts::md::binance_usds_futures {
|
||||
|
||||
using bts::api::domain::SecurityType;
|
||||
using bts::api::domain::Venue;
|
||||
using bts::api::domain::Venue_Name;
|
||||
using bts::api::md::BinanceUsdsFuturesMarketDataConfig;
|
||||
@@ -39,12 +40,14 @@ using bts::domain::SecurityID;
|
||||
using bts::io::ProtoPrinter;
|
||||
using bts::io::QueryParams;
|
||||
using bts::persistence::IDGenerator;
|
||||
using std::pair;
|
||||
using std::string;
|
||||
using std::uint32_t;
|
||||
using std::unique_ptr;
|
||||
using std::unordered_map;
|
||||
using std::vector;
|
||||
|
||||
#define VENUE Venue::VENUE_BINANCE_USDS_FUTURES
|
||||
#define VENUE_NAME (Venue_Name(Venue::VENUE_BINANCE_USDS_FUTURES))
|
||||
#define CCAPI_VENUE_NAME CCAPI_EXCHANGE_NAME_BINANCE_USDS_FUTURES
|
||||
#define VENUE_PRICE_STATS_PATH BINANCE_USDS_FUTURES_PRICE_STATS_PATH
|
||||
@@ -192,8 +195,8 @@ struct BinanceUsdsFuturesSubscription::PImpl {
|
||||
sec_requestor.request();
|
||||
}
|
||||
|
||||
SecurityID get_sec_id(const string &sec_desc) {
|
||||
return SecurityID{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
|
||||
SecurityID get_sec_id(SecurityType sec_type, const string &sec_desc) {
|
||||
return SecurityID{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -211,8 +214,9 @@ struct BinanceUsdsFuturesSubscription::PImpl {
|
||||
return sec_desc;
|
||||
}
|
||||
|
||||
DepthHandler &add_depth_handler(const string &sec_desc) {
|
||||
const auto sec_id = get_sec_id(sec_desc);
|
||||
DepthHandler &add_depth_handler(
|
||||
SecurityType sec_type, const string &sec_desc) {
|
||||
const auto sec_id = get_sec_id(sec_type, sec_desc);
|
||||
auto [it, is_new] =
|
||||
depth_handlers.try_emplace(sec_desc, sec_id, listener);
|
||||
bts_assert_msgf(
|
||||
@@ -220,8 +224,9 @@ struct BinanceUsdsFuturesSubscription::PImpl {
|
||||
return it->second;
|
||||
}
|
||||
|
||||
TickerHandler &add_ticker_handler(const string &sec_desc) {
|
||||
const auto sec_id = get_sec_id(sec_desc);
|
||||
TickerHandler &add_ticker_handler(
|
||||
SecurityType sec_type, const string &sec_desc) {
|
||||
const auto sec_id = get_sec_id(sec_type, sec_desc);
|
||||
auto [it, is_new] =
|
||||
ticker_handlers.try_emplace(sec_desc, sec_id, listener);
|
||||
bts_assert_msgf(
|
||||
@@ -229,8 +234,9 @@ struct BinanceUsdsFuturesSubscription::PImpl {
|
||||
return it->second;
|
||||
}
|
||||
|
||||
StatsHandler &add_stats_handler(const string &sec_desc) {
|
||||
const auto sec_id = get_sec_id(sec_desc);
|
||||
StatsHandler &add_stats_handler(
|
||||
SecurityType sec_type, const string &sec_desc) {
|
||||
const auto sec_id = get_sec_id(sec_type, sec_desc);
|
||||
auto [it, is_new] =
|
||||
stats_handlers.try_emplace(sec_desc, sec_id, listener);
|
||||
bts_assert_msgf(
|
||||
@@ -252,16 +258,16 @@ struct BinanceUsdsFuturesSubscription::PImpl {
|
||||
CCApiSession::BatchSubscription ticker_subs;
|
||||
CCApiSession::BatchRequest stats_reqs;
|
||||
|
||||
for (const auto &sec_desc : securities) {
|
||||
const auto sec_desc_for_reqs = get_sec_desc_for_reqs(sec_desc);
|
||||
for (auto [type, name] : securities) {
|
||||
const auto sec_desc_for_reqs = get_sec_desc_for_reqs(name);
|
||||
depth_subs.push_back(
|
||||
{mk_depth_sub(sec_desc_for_reqs, depth_opts),
|
||||
&add_depth_handler(sec_desc)});
|
||||
&add_depth_handler(type, name)});
|
||||
ticker_subs.push_back(
|
||||
{mk_ticker_sub(sec_desc_for_reqs, ticker_opts),
|
||||
&add_ticker_handler(sec_desc)});
|
||||
&add_ticker_handler(type, name)});
|
||||
stats_reqs.push_back(
|
||||
{mk_stats_req(sec_desc), &add_stats_handler(sec_desc)});
|
||||
{mk_stats_req(name), &add_stats_handler(type, name)});
|
||||
}
|
||||
|
||||
session.subscribe(
|
||||
@@ -278,7 +284,7 @@ struct BinanceUsdsFuturesSubscription::PImpl {
|
||||
*/
|
||||
void on_securities(vector<Security> secs) {
|
||||
// 'sec_descs' is used to make depth subscriptions
|
||||
vector<string> sec_descs;
|
||||
vector<pair<SecurityType, string>> sec_descs;
|
||||
sec_descs.reserve(secs.size());
|
||||
// 'sec_ptrs' is passed to IMarketDataListeners
|
||||
vector<const Security *> sec_ptrs;
|
||||
@@ -291,7 +297,7 @@ struct BinanceUsdsFuturesSubscription::PImpl {
|
||||
id, std::make_unique<Security>(std::move(s)));
|
||||
bts_assert_msgf(
|
||||
is_new, "Received duplicate Binance USDS Future '{}'", desc);
|
||||
sec_descs.push_back(std::move(desc));
|
||||
sec_descs.push_back(std::make_pair(s.type(), std::move(desc)));
|
||||
// add ptr to entry in 'securities' (not the loop var, we moved that)
|
||||
sec_ptrs.push_back(it->second.get());
|
||||
}
|
||||
|
||||
@@ -30,7 +30,8 @@ void CboeMarketDatahandler::run(
|
||||
base_path + config.databento_settings().bbo_file_name(),
|
||||
config.databento_settings().key(),
|
||||
config.databento_settings().dataset(),
|
||||
config.databento_settings().filter_securities_one_year());
|
||||
config.databento_settings().filter_securities_one_year(),
|
||||
config.databento_settings().load_reference_data());
|
||||
}
|
||||
|
||||
} // namespace bts::md::cboe
|
||||
@@ -147,7 +147,8 @@ struct CoinbaseSecurityRequestor::PImpl {
|
||||
for (auto &sec : added_secs) {
|
||||
// Since the ids haven't been set yet, we should expect them to be 0
|
||||
bts_assert(sec.id().value() == 0);
|
||||
SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec.desc())};
|
||||
SecurityID sec_id{
|
||||
sec_id_store->get_or_add(VENUE, sec.type(), sec.desc())};
|
||||
sec.security_common.id = sec_id;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,6 +25,9 @@
|
||||
#include <unordered_map>
|
||||
|
||||
namespace bts::md::coinbase {
|
||||
|
||||
using bts::api::domain::SecurityType;
|
||||
using bts::api::domain::Venue;
|
||||
using bts::api::md::CoinbaseMarketDataConfig;
|
||||
using bts::domain::Security;
|
||||
using bts::domain::SecurityCriteria;
|
||||
@@ -32,10 +35,13 @@ using bts::domain::SecurityID;
|
||||
using bts::io::ProtoPrinter;
|
||||
using bts::io::QueryParams;
|
||||
using bts::persistence::IDGenerator;
|
||||
using std::pair;
|
||||
using std::string;
|
||||
using std::unique_ptr;
|
||||
using std::unordered_map;
|
||||
using std::vector;
|
||||
|
||||
#define VENUE Venue::VENUE_COINBASE
|
||||
#define VENUE_NAME (Venue_Name(Venue::VENUE_COINBASE))
|
||||
#define CCAPI_VENUE_NAME CCAPI_EXCHANGE_NAME_COINBASE
|
||||
|
||||
@@ -183,24 +189,29 @@ struct CoinbaseSubscription::PImpl {
|
||||
sec_requestor.callback().bind<&PImpl::on_securities>(*this);
|
||||
sec_requestor.request();
|
||||
}
|
||||
StatsHandler &mk_stats_handler(const string &sec_desc) {
|
||||
auto sec_id = SecurityID{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
|
||||
|
||||
StatsHandler &mk_stats_handler(
|
||||
SecurityType sec_type, const string &sec_desc) {
|
||||
auto sec_id =
|
||||
SecurityID{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
|
||||
auto [it, is_new] =
|
||||
stats_handlers.try_emplace(sec_desc, sec_id, listener);
|
||||
bts_assert_msgf(
|
||||
is_new, "StatsHandler already created with id '{}'", sec_id);
|
||||
return it->second;
|
||||
}
|
||||
DepthHandler &mk_depth_handler(const string &sec_desc) {
|
||||
SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
|
||||
DepthHandler &mk_depth_handler(
|
||||
SecurityType sec_type, const string &sec_desc) {
|
||||
SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
|
||||
auto [it, is_new] =
|
||||
depth_handlers.try_emplace(sec_desc, sec_id, listener);
|
||||
bts_assert(is_new);
|
||||
return it->second;
|
||||
}
|
||||
|
||||
TickerHandler &mk_ticker_handler(const string &sec_desc) {
|
||||
SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
|
||||
TickerHandler &mk_ticker_handler(
|
||||
SecurityType sec_type, const string &sec_desc) {
|
||||
SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
|
||||
auto [it, is_new] =
|
||||
ticker_handlers.try_emplace(sec_desc, sec_id, listener);
|
||||
bts_assert_msgf(
|
||||
@@ -222,14 +233,15 @@ struct CoinbaseSubscription::PImpl {
|
||||
CCApiSession::BatchRequest stats_reqs;
|
||||
|
||||
vector<string> curr_stats_sec_descs;
|
||||
for (const auto &sec_desc : securities) {
|
||||
for (const auto &[sec_type, sec_desc] : securities) {
|
||||
depth_subs.push_back(
|
||||
{mk_depth_sub(sec_desc, depth_opts), &mk_depth_handler(sec_desc)});
|
||||
{mk_depth_sub(sec_desc, depth_opts),
|
||||
&mk_depth_handler(sec_type, sec_desc)});
|
||||
ticker_subs.push_back(
|
||||
{mk_ticker_sub(sec_desc, ticker_opts),
|
||||
&mk_ticker_handler(sec_desc)});
|
||||
&mk_ticker_handler(sec_type, sec_desc)});
|
||||
stats_reqs.push_back(
|
||||
{mk_stats_req(sec_desc), &mk_stats_handler(sec_desc)});
|
||||
{mk_stats_req(sec_desc), &mk_stats_handler(sec_type, sec_desc)});
|
||||
}
|
||||
|
||||
session.subscribe(
|
||||
@@ -247,7 +259,7 @@ struct CoinbaseSubscription::PImpl {
|
||||
void on_securities(std::vector<Security> secs) {
|
||||
LOG(md, Info, "on sec {}", secs.size());
|
||||
// 'sec_descs' is used to make depth subscriptions
|
||||
vector<string> sec_descs;
|
||||
vector<pair<SecurityType, string>> sec_descs;
|
||||
sec_descs.reserve(secs.size());
|
||||
// 'sec_ptrs' is passed to IMarketDataListeners
|
||||
vector<const Security *> sec_ptrs;
|
||||
@@ -260,7 +272,7 @@ struct CoinbaseSubscription::PImpl {
|
||||
id, std::make_unique<Security>(std::move(s)));
|
||||
bts_assert_msgf(
|
||||
is_new, "Received duplicate Coinbase security '{}'", desc);
|
||||
sec_descs.push_back(std::move(desc));
|
||||
sec_descs.push_back(std::make_pair(s.type(), std::move(desc)));
|
||||
// add ptr to entry in 'securities' (not the loop var, we moved that)
|
||||
sec_ptrs.push_back(it->second.get());
|
||||
}
|
||||
|
||||
@@ -25,18 +25,36 @@
|
||||
#include <databento/live.hpp>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace bts::md::DatabentoHandler {
|
||||
|
||||
std::string remove_spaces(const std::string_view& str) {
|
||||
std::string result;
|
||||
result.reserve(str.size()); // Reserve space for better performance
|
||||
for (char c : str) {
|
||||
if (!isspace(c)) {
|
||||
result += c;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
#define SEC_TYPE(name) bts::api::domain::SECURITY_TYPE_##name
|
||||
|
||||
using bts::domain::SecurityID;
|
||||
using bts::api::domain::CallPut;
|
||||
using bts::api::domain::SecurityType;
|
||||
using bts::api::domain::Venue;
|
||||
|
||||
std::string remove_spaces(const std::string_view &str) {
|
||||
std::string result;
|
||||
result.reserve(str.size()); // Reserve space for better performance
|
||||
for (char c : str) {
|
||||
if (!isspace(c)) {
|
||||
result += c;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
static SecurityType parse_sec_type(std::string_view str) {
|
||||
if (str.find("OPT") != std::string::npos) {
|
||||
return SEC_TYPE(OPTION);
|
||||
}
|
||||
if (str.find("FUT") != std::string::npos) {
|
||||
return SEC_TYPE(FUTURE);
|
||||
}
|
||||
return SEC_TYPE(UNSPECIFIED);
|
||||
}
|
||||
|
||||
void DatabentoDatahandler::run(
|
||||
@@ -49,8 +67,10 @@ void DatabentoDatahandler::run(
|
||||
const std::string &key,
|
||||
const std::string &dataset,
|
||||
const bool filter_securities,
|
||||
const std::chrono::duration<long double> sleep_duration) {
|
||||
const bool load_reference_data,
|
||||
const std::chrono::duration<long double> sleep_duration ) {
|
||||
|
||||
(void)filter_securities;
|
||||
auto sleep_duration_updated = sleep_duration;
|
||||
|
||||
if (!use_file) {
|
||||
@@ -77,22 +97,9 @@ void DatabentoDatahandler::run(
|
||||
bts_assert_msg(false, "Exchange not supported on databento");
|
||||
return databento::KeepGoing::Stop;
|
||||
}
|
||||
if(filter_securities){
|
||||
const auto exp_nanos = msg.expiration;
|
||||
std::string exp_string = databento::ToIso8601(exp_nanos);
|
||||
const auto expiration_time = bts::core::TimeStamp::from_iso_string(exp_string);
|
||||
if(expiration_time.seconds() < expiration_time.now().seconds() || expiration_time.seconds() > (expiration_time.now().seconds() + 60*60*24*365)){
|
||||
return databento::KeepGoing::Continue;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
bts::domain::SecurityID sec_id{sec_id_store->get_or_add(
|
||||
venue, std::to_string(msg.hd.instrument_id))};
|
||||
bts::domain::Security sec;
|
||||
sec.security_common = domain::SecurityCommon();
|
||||
sec.security_common.venue = venue;
|
||||
sec.security_common.id = sec_id;
|
||||
sec.security_common.exchange_sec_id = msg.hd.instrument_id;
|
||||
sec.security_common.desc = msg.RawSymbol();
|
||||
sec.security_common.symbol = msg.Asset();
|
||||
@@ -104,38 +111,55 @@ void DatabentoDatahandler::run(
|
||||
sec.security_common.currency = {msg.currency.data(), msg.currency.size()};
|
||||
// sec.security_common.cpp = msg.hd.
|
||||
|
||||
const std::string_view sec_type = {msg.security_type.data(), msg.security_type.size()};
|
||||
const SecurityType sec_type =
|
||||
parse_sec_type({msg.security_type.data(), msg.security_type.size()});
|
||||
const auto exp_nanos = msg.expiration;
|
||||
std::string exp_string = databento::ToIso8601(exp_nanos);
|
||||
const auto expiration_time = bts::core::TimeStamp::from_iso_string(exp_string);
|
||||
|
||||
if (sec_type.find("OPT") != std::string::npos) {
|
||||
auto call_put = api::domain::CallPut::PUT;
|
||||
if(msg.instrument_class == 'C' ){
|
||||
call_put=api::domain::CallPut::CALL;
|
||||
}
|
||||
else{
|
||||
bts_assert(msg.instrument_class == 'P');
|
||||
}
|
||||
sec.security.emplace<bts::domain::OptionSecurity>(
|
||||
expiration_time, bts::domain::Price(msg.strike_price, -9),call_put);
|
||||
} else if (sec_type.find("FUT") != std::string::npos) {
|
||||
sec.security.emplace<bts::domain::FutureSecurity>(expiration_time);
|
||||
} else {
|
||||
bts_assert_msg(false, "Only futures and options supported on databento");
|
||||
return databento::KeepGoing::Stop;
|
||||
}
|
||||
switch (sec_type) {
|
||||
case SEC_TYPE(FUTURE):
|
||||
sec.security.emplace<bts::domain::FutureSecurity>(expiration_time);
|
||||
break;
|
||||
case SEC_TYPE(OPTION): {
|
||||
auto call_put = CallPut::PUT;
|
||||
if (msg.instrument_class == 'C') {
|
||||
call_put = CallPut::CALL;
|
||||
} else {
|
||||
bts_assert(msg.instrument_class == 'P');
|
||||
}
|
||||
sec.security.emplace<bts::domain::OptionSecurity>(
|
||||
expiration_time, bts::domain::Price(msg.strike_price, -9),
|
||||
call_put);
|
||||
break;
|
||||
}
|
||||
default:
|
||||
bts_assert_msg(
|
||||
false, "Only futures and options supported on databento");
|
||||
return databento::KeepGoing::Stop;
|
||||
}
|
||||
|
||||
*sec.security_common.id = sec_id_store->get_or_add(
|
||||
venue, sec_type, std::to_string(msg.hd.instrument_id));
|
||||
listener->on_securities({&sec});
|
||||
return databento::KeepGoing::Continue;
|
||||
};
|
||||
|
||||
auto process_bbo = [&](const databento::Record &record) {
|
||||
const auto &msg = record.Get<databento::Mbp1Msg>();
|
||||
auto sec_id_opt = sec_id_store->get_id(
|
||||
venue, std::to_string(msg.hd.instrument_id), false);
|
||||
if (!sec_id_opt.has_value())
|
||||
return databento::KeepGoing::Continue;
|
||||
// The security type to is needed to get the id. Since the sec type isn't known,
|
||||
// check for an id using both types (OPTION and FUTURE). This is safe to do since
|
||||
// options and futures never have common names (instrument ids).
|
||||
const auto name = std::to_string(msg.hd.instrument_id);
|
||||
auto sec_id_opt =
|
||||
sec_id_store->get_id(venue, SEC_TYPE(OPTION), name, false);
|
||||
if (!sec_id_opt.has_value()) {
|
||||
sec_id_opt =
|
||||
sec_id_store->get_id(venue, SEC_TYPE(FUTURE), name, false);
|
||||
if (!sec_id_opt.has_value()) {
|
||||
return databento::KeepGoing::Continue;
|
||||
}
|
||||
}
|
||||
|
||||
sleep_for(sleep_duration_updated);
|
||||
bts::domain::SecurityID sec_id = bts::domain::SecurityID(sec_id_opt.value());
|
||||
@@ -181,19 +205,16 @@ void DatabentoDatahandler::run(
|
||||
{}
|
||||
};
|
||||
//hardcoding for now will update later
|
||||
std::vector<std::string> symbols;
|
||||
switch (venue) {
|
||||
case api::domain::VENUE_CBOE:
|
||||
client.Subscribe({"SPX.OPT","SPXW.OPT"}, databento::Schema::Definition, databento::SType::Parent,databento::UnixNanos{});
|
||||
client.Subscribe({"SPX.OPT","SPXW.OPT"}, databento::Schema::Mbp1, databento::SType::Parent);
|
||||
break;
|
||||
case api::domain::VENUE_GLBX:
|
||||
client.Subscribe({"ES.FUT"}, databento::Schema::Definition, databento::SType::Parent);
|
||||
client.Subscribe({"ES.FUT"}, databento::Schema::Mbp1, databento::SType::Parent);
|
||||
break;
|
||||
default:
|
||||
bts_assert_msg(false, "ONLY GLBX AND CBOE SUPPORTED FOR NOW");
|
||||
|
||||
case api::domain::VENUE_CBOE: symbols= {"SPX.OPT","SPXW.OPT"}; break;
|
||||
case api::domain::VENUE_GLBX: symbols= {"ES.FUT"}; break;
|
||||
default:bts_assert_msg(false, "ONLY GLBX AND CBOE SUPPORTED FOR NOW");
|
||||
}
|
||||
if(load_reference_data)
|
||||
client.Subscribe(symbols, databento::Schema::Definition, databento::SType::Parent,databento::UnixNanos{});
|
||||
else client.Subscribe(symbols, databento::Schema::Definition, databento::SType::Parent);
|
||||
client.Subscribe(symbols, databento::Schema::Mbp1, databento::SType::Parent);
|
||||
client.Start();
|
||||
while (true) {
|
||||
try {
|
||||
|
||||
@@ -15,6 +15,7 @@ class DatabentoDatahandler {
|
||||
const std::string &dataset="",
|
||||
const std::string& key = "db-Q4eCbTQfrAHaG6sAcuFUYB9YqkF6j",
|
||||
const bool filter_securities=false,
|
||||
const bool load_reference_data = true,
|
||||
const std::chrono::duration<long double> sleep_duration = 0.04s);
|
||||
};
|
||||
} // namespace bts::md::databento
|
||||
@@ -19,6 +19,7 @@
|
||||
namespace bts::md::deribit {
|
||||
|
||||
using bts::api::domain::CallPut;
|
||||
using bts::api::domain::SecurityType;
|
||||
using bts::domain::FutureSecurity;
|
||||
using bts::domain::OptionSecurity;
|
||||
using bts::domain::Price;
|
||||
@@ -106,8 +107,8 @@ struct DeribitInstrumentStore::PImpl {
|
||||
return &it->second;
|
||||
}
|
||||
|
||||
Security *find(const string &name) {
|
||||
auto id = sec_id_store_->get_id(VENUE, name);
|
||||
Security *find(SecurityType type, const string &name) {
|
||||
auto id = sec_id_store_->get_id(VENUE, type, name);
|
||||
if (!id) {
|
||||
return nullptr;
|
||||
}
|
||||
@@ -147,7 +148,7 @@ struct DeribitInstrumentStore::PImpl {
|
||||
continue;
|
||||
}
|
||||
const SecurityID id{
|
||||
sec_id_store_->get_or_add(VENUE, di.instrument_name)};
|
||||
sec_id_store_->get_or_add(VENUE, sec.type(), di.instrument_name)};
|
||||
sec.security_common.id = id;
|
||||
auto [it, is_new] = securities_.try_emplace(id, std::move(sec));
|
||||
if (!is_new) {
|
||||
@@ -188,8 +189,8 @@ struct DeribitInstrumentStore::PImpl {
|
||||
// DeribitCombo object, which is required to resolve the leg definitions.
|
||||
unordered_map<string, Security> unresolved_combos_;
|
||||
|
||||
const Security *find_security_(const string &name) {
|
||||
const auto id = sec_id_store_->get_id(VENUE, name);
|
||||
const Security *find_security_(SecurityType type, const string &name) {
|
||||
const auto id = sec_id_store_->get_id(VENUE, type, name);
|
||||
if (!id) {
|
||||
return nullptr;
|
||||
}
|
||||
@@ -202,7 +203,8 @@ struct DeribitInstrumentStore::PImpl {
|
||||
|
||||
bool resolve_combo_(const DeribitCombo &combo, SpreadSecurity &spd) {
|
||||
for (const auto &leg : combo.legs) {
|
||||
const auto *leg_sec = find_security_(leg.instrument_name);
|
||||
const auto *leg_sec =
|
||||
find_security_(SEC_TYPE(SPREAD), leg.instrument_name);
|
||||
if (leg_sec == nullptr) {
|
||||
spd.legs.clear();
|
||||
return false;
|
||||
@@ -259,7 +261,8 @@ struct DeribitInstrumentStore::PImpl {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
SecurityID id{sec_id_store_->get_or_add(VENUE, sec.desc())};
|
||||
SecurityID id{
|
||||
sec_id_store_->get_or_add(VENUE, sec.type(), sec.desc())};
|
||||
sec.security_common.id = id;
|
||||
auto [sec_it, is_new] = securities_.try_emplace(id, std::move(sec));
|
||||
if (!is_new) {
|
||||
@@ -292,8 +295,8 @@ const Security *DeribitInstrumentStore::find(SecurityID id) const {
|
||||
return me_->find(id);
|
||||
}
|
||||
|
||||
Security *DeribitInstrumentStore::find(const string &id) {
|
||||
return me_->find(id);
|
||||
Security *DeribitInstrumentStore::find(SecurityType type, const string &name) {
|
||||
return me_->find(type, name);
|
||||
}
|
||||
|
||||
void DeribitInstrumentStore::on_instruments(
|
||||
|
||||
@@ -45,11 +45,12 @@ class DeribitInstrumentStore {
|
||||
const domain::Security *find(domain::SecurityID id) const;
|
||||
|
||||
/**
|
||||
* @brief Finds the security with the specified \p name
|
||||
* @brief Finds the security with the specified \p name and \p type
|
||||
* @details No 'const' overload bc the fn that converts name to numeric id
|
||||
* isn't const
|
||||
*/
|
||||
domain::Security *find(const std::string &name);
|
||||
domain::Security *find(
|
||||
api::domain::SecurityType type, const std::string &name);
|
||||
|
||||
void on_instruments(
|
||||
const std::vector<DeribitInstrument> &instruments,
|
||||
|
||||
@@ -41,6 +41,7 @@
|
||||
|
||||
namespace bts::md::deribit {
|
||||
|
||||
using bts::api::domain::SecurityType;
|
||||
using bts::api::md::DeribitMarketDataConfig;
|
||||
using bts::domain::Security;
|
||||
using bts::domain::SecurityCriteria;
|
||||
@@ -58,6 +59,7 @@ using std::unordered_map;
|
||||
using std::unordered_set;
|
||||
using std::vector;
|
||||
|
||||
#define VENUE bts::api::domain::VENUE_DERIBIT
|
||||
#define VENUE_NAME \
|
||||
(bts::api::domain::Venue_Name(bts::api::domain::VENUE_DERIBIT))
|
||||
#define CCAPI_VENUE_NAME CCAPI_EXCHANGE_NAME_DERIBIT
|
||||
@@ -166,8 +168,9 @@ struct DeribitSubscription::PImpl {
|
||||
subscribe_instrument_state();
|
||||
}
|
||||
|
||||
SecurityData &add_security_data(const string sec_desc) {
|
||||
const SecurityID sid{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
|
||||
SecurityData &add_security_data(
|
||||
SecurityType sec_type, const string &sec_desc) {
|
||||
const SecurityID sid{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
|
||||
auto [it, is_new] = security_data_map.try_emplace(
|
||||
sec_desc, std::make_unique<SecurityData>(sid, listener));
|
||||
_ASSERT_MSGF(is_new, "add_security_data. Already added '{}'", sec_desc);
|
||||
@@ -199,27 +202,31 @@ struct DeribitSubscription::PImpl {
|
||||
void subscribe_md(SecurityNames security_names) {
|
||||
// Populate list of 'channels', and prepare their corresponding handlers
|
||||
vector<string> channels;
|
||||
for (const string &sec : security_names) {
|
||||
auto &sec_data = add_security_data(sec);
|
||||
for (auto [type, name] : security_names) {
|
||||
auto &sec_data = add_security_data(type, name);
|
||||
// Depth
|
||||
if (subscribe_depth) {
|
||||
const auto &ch = channels.emplace_back(
|
||||
FMT("book.{}.none.{}.100ms", sec, cfg.num_depth_levels()));
|
||||
FMT("book.{}.none.{}.100ms", name, cfg.num_depth_levels()));
|
||||
channel_demuxer.add_handler(ch)
|
||||
.bind<&DeribitDepthAdaptor::on_depths>(sec_data.depth_hnd);
|
||||
.template bind<&DeribitDepthAdaptor::on_depths>(
|
||||
sec_data.depth_hnd);
|
||||
}
|
||||
// Ticker
|
||||
if (subscribe_ticker) {
|
||||
const auto &ch =
|
||||
channels.emplace_back(FMT("incremental_ticker.{}", sec));
|
||||
channels.emplace_back(FMT("incremental_ticker.{}", name));
|
||||
channel_demuxer.add_handler(ch)
|
||||
.bind<&DeribitTickerAdaptor::on_tickers>(sec_data.ticker_hnd);
|
||||
.template bind<&DeribitTickerAdaptor::on_tickers>(
|
||||
sec_data.ticker_hnd);
|
||||
}
|
||||
// Trades
|
||||
if (subscribe_trades) {
|
||||
const auto &ch = channels.emplace_back(FMT("trades.{}.100ms", sec));
|
||||
const auto &ch =
|
||||
channels.emplace_back(FMT("trades.{}.100ms", name));
|
||||
channel_demuxer.add_handler(ch)
|
||||
.bind<&DeribitTradeAdaptor::on_trades>(sec_data.trade_hnd);
|
||||
.template bind<&DeribitTradeAdaptor::on_trades>(
|
||||
sec_data.trade_hnd);
|
||||
}
|
||||
}
|
||||
// If 'channels' is empty, it's a no-op
|
||||
@@ -253,7 +260,9 @@ struct DeribitSubscription::PImpl {
|
||||
bts_assert(!CONTAINS_KEY(security_data_map, sec->desc()));
|
||||
}
|
||||
listener->on_securities(secs);
|
||||
auto sec_names = TRANSFORM(secs, [](const auto *s) { return s->desc(); });
|
||||
auto sec_names = TRANSFORM(secs, [](const auto *s) {
|
||||
return std::make_pair(s->type(), s->desc());
|
||||
});
|
||||
batch_subscribe_md(sec_names);
|
||||
}
|
||||
|
||||
|
||||
@@ -18,10 +18,11 @@ struct EmumbaDeribitData {
|
||||
|
||||
EmumbaDeribitData()
|
||||
: sec_index(), recorder(
|
||||
{utils::Dispatcher::current(),
|
||||
[=, this](ExchangeKey k) {
|
||||
return SecurityKey(sec_index.get_key(k.value()));
|
||||
}}),
|
||||
nullptr,
|
||||
[=, this](ExchangeKey k) {
|
||||
return SecurityKey(sec_index.get_key(k.value()));
|
||||
},
|
||||
&utils::Dispatcher::current()),
|
||||
md_producer(recorder), listener(sec_index, md_producer) {}
|
||||
};
|
||||
} // namespace bts::md
|
||||
@@ -35,6 +35,7 @@ using std::string;
|
||||
using std::unordered_set;
|
||||
using std::vector;
|
||||
|
||||
#define VENUE Venue::VENUE_FTX
|
||||
#define VENUE_NAME (Venue_Name(Venue::VENUE_FTX))
|
||||
#define SEC_TYPE(x) bts::api::domain::SECURITY_TYPE_##x
|
||||
|
||||
@@ -172,7 +173,8 @@ struct FTXSecurityRequestor::PImpl {
|
||||
for (auto &sec : added_secs) {
|
||||
// Since the ids haven't been set yet, we should expect them to be 0
|
||||
bts_assert(sec.id().value() == 0);
|
||||
SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec.desc())};
|
||||
SecurityID sec_id{
|
||||
sec_id_store->get_or_add(VENUE, sec.type(), sec.desc())};
|
||||
sec.security_common.id = sec_id;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,6 +28,7 @@
|
||||
|
||||
namespace bts::md::ftx {
|
||||
|
||||
using bts::api::domain::SecurityType;
|
||||
using bts::api::domain::Venue;
|
||||
using bts::api::domain::Venue_Name;
|
||||
using bts::api::md::FTXMarketDataConfig;
|
||||
@@ -36,11 +37,13 @@ using bts::domain::SecurityCriteria;
|
||||
using bts::domain::SecurityID;
|
||||
using bts::io::ProtoPrinter;
|
||||
using bts::persistence::IDGenerator;
|
||||
using std::pair;
|
||||
using std::string;
|
||||
using std::unique_ptr;
|
||||
using std::unordered_map;
|
||||
using std::vector;
|
||||
|
||||
#define VENUE Venue::VENUE_FTX
|
||||
#define VENUE_NAME (Venue_Name(Venue::VENUE_FTX))
|
||||
|
||||
#define FTX_MSG_TYPE_NAME(x) \
|
||||
@@ -130,16 +133,18 @@ struct FTXSubscription::PImpl {
|
||||
sec_requestor.request();
|
||||
}
|
||||
|
||||
DepthHandler &create_depth_handler(string sec_desc) {
|
||||
SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
|
||||
DepthHandler &create_depth_handler(
|
||||
SecurityType sec_type, const string &sec_desc) {
|
||||
SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
|
||||
auto [it, is_new] =
|
||||
depth_handlers.try_emplace(sec_desc, sec_id, listener);
|
||||
bts_assert(is_new);
|
||||
return it->second;
|
||||
}
|
||||
|
||||
TickerHandler &create_ticker_handler(string sec_desc) {
|
||||
SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
|
||||
TickerHandler &create_ticker_handler(
|
||||
SecurityType sec_type, const string &sec_desc) {
|
||||
SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
|
||||
auto [it, is_new] =
|
||||
ticker_handlers.try_emplace(sec_desc, sec_id, listener);
|
||||
bts_assert_msgf(
|
||||
@@ -147,8 +152,9 @@ struct FTXSubscription::PImpl {
|
||||
return it->second;
|
||||
}
|
||||
|
||||
StatsHandler &create_stats_handler(string sec_desc) {
|
||||
SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
|
||||
StatsHandler &create_stats_handler(
|
||||
SecurityType sec_type, const string &sec_desc) {
|
||||
SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
|
||||
auto [it, is_new] =
|
||||
stats_handlers.try_emplace(sec_desc, sec_id, listener);
|
||||
bts_assert_msgf(
|
||||
@@ -171,15 +177,16 @@ struct FTXSubscription::PImpl {
|
||||
CCApiSession::BatchSubscription ticker_subs;
|
||||
CCApiSession::BatchRequest stats_reqs;
|
||||
|
||||
for (const auto &sec_desc : securities) {
|
||||
for (const auto &[sec_type, sec_desc] : securities) {
|
||||
depth_subs.push_back(
|
||||
{create_depth_sub(sec_desc, depth_opts),
|
||||
&create_depth_handler(sec_desc)});
|
||||
&create_depth_handler(sec_type, sec_desc)});
|
||||
ticker_subs.push_back(
|
||||
{create_ticker_sub(sec_desc, ticker_opts),
|
||||
&create_ticker_handler(sec_desc)});
|
||||
&create_ticker_handler(sec_type, sec_desc)});
|
||||
stats_reqs.push_back(
|
||||
{create_stats_req(sec_desc), &create_stats_handler(sec_desc)});
|
||||
{create_stats_req(sec_desc),
|
||||
&create_stats_handler(sec_type, sec_desc)});
|
||||
}
|
||||
|
||||
session.subscribe(
|
||||
@@ -196,7 +203,7 @@ struct FTXSubscription::PImpl {
|
||||
*/
|
||||
void on_securities(vector<domain::Security> secs) {
|
||||
// 'sec_descs' is used to make depth subscriptions
|
||||
vector<string> sec_descs;
|
||||
vector<pair<SecurityType, string>> sec_descs;
|
||||
sec_descs.reserve(secs.size());
|
||||
// 'sec_ptrs' is passed to IMarketDataListeners
|
||||
vector<const domain::Security *> sec_ptrs;
|
||||
@@ -208,7 +215,7 @@ struct FTXSubscription::PImpl {
|
||||
auto [it, is_new] = securities.try_emplace(
|
||||
id, std::make_unique<domain::Security>(std::move(s)));
|
||||
bts_assert_msgf(is_new, "Received duplicate FTX security '{}'", desc);
|
||||
sec_descs.push_back(std::move(desc));
|
||||
sec_descs.push_back(std::make_pair(s.type(), std::move(desc)));
|
||||
// add ptr to entry in 'securities' (not the loop var, we moved that)
|
||||
sec_ptrs.push_back(it->second.get());
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
namespace bts::md::okx {
|
||||
|
||||
using bts::api::domain::CallPut;
|
||||
using bts::api::domain::SecurityType;
|
||||
using bts::domain::FutureSecurity;
|
||||
using bts::domain::OptionSecurity;
|
||||
using bts::domain::Price;
|
||||
@@ -94,8 +95,8 @@ struct OkxInstrumentStore::PImpl {
|
||||
return &it->second;
|
||||
}
|
||||
|
||||
Security *find(const string &name) {
|
||||
auto id = sec_id_store_->get_id(VENUE, name);
|
||||
Security *find(SecurityType type, const string &name) {
|
||||
auto id = sec_id_store_->get_id(VENUE, type, name);
|
||||
if (!id) {
|
||||
return nullptr;
|
||||
}
|
||||
@@ -124,7 +125,7 @@ struct OkxInstrumentStore::PImpl {
|
||||
if (!filter_(sec)) {
|
||||
continue;
|
||||
}
|
||||
SecurityID id{sec_id_store_->get_or_add(VENUE, oi.instId)};
|
||||
SecurityID id{sec_id_store_->get_or_add(VENUE, sec.type(), oi.instId)};
|
||||
sec.security_common.id = id;
|
||||
auto [it, is_new] = securities_.try_emplace(id, std::move(sec));
|
||||
if (!is_new) {
|
||||
@@ -152,7 +153,9 @@ const Security *OkxInstrumentStore::find(SecurityID id) const {
|
||||
return me_->find(id);
|
||||
}
|
||||
|
||||
Security *OkxInstrumentStore::find(const string &id) { return me_->find(id); }
|
||||
Security *OkxInstrumentStore::find(SecurityType type, const string &name) {
|
||||
return me_->find(type, name);
|
||||
}
|
||||
|
||||
void OkxInstrumentStore::on_instruments(
|
||||
const vector<OkxInstrument> &instruments,
|
||||
|
||||
@@ -50,7 +50,8 @@ class OkxInstrumentStore {
|
||||
* @details No 'const' overload bc the fn that converts name to numeric id
|
||||
* isn't const
|
||||
*/
|
||||
domain::Security *find(const std::string &name);
|
||||
domain::Security *find(
|
||||
api::domain::SecurityType type, const std::string &name);
|
||||
|
||||
void on_instruments(
|
||||
const std::vector<OkxInstrument> &instruments,
|
||||
|
||||
@@ -38,6 +38,7 @@
|
||||
|
||||
namespace bts::md::okx {
|
||||
|
||||
using bts::api::domain::SecurityType;
|
||||
using bts::api::md::OkxMarketDataConfig;
|
||||
using bts::domain::Security;
|
||||
using bts::domain::SecurityCriteria;
|
||||
@@ -60,6 +61,7 @@ using std::vector;
|
||||
#define _WARN(...) _LOG(Warn, __VA_ARGS__)
|
||||
#define _ERROR(...) _LOG(Error, __VA_ARGS__)
|
||||
|
||||
#define VENUE bts::api::domain::VENUE_OKX
|
||||
#define VENUE_NAME (bts::api::domain::Venue_Name(bts::api::domain::VENUE_OKX))
|
||||
#define CCAPI_VENUE_NAME CCAPI_EXCHANGE_NAME_OKX
|
||||
#define SEC_TYPE(t) bts::api::domain::SECURITY_TYPE_##t
|
||||
@@ -164,8 +166,9 @@ struct OkxSubscription::PImpl {
|
||||
[this](const auto &secs) { on_securities(secs); });
|
||||
}
|
||||
|
||||
SecurityData &add_security_data(const string sec_desc) {
|
||||
const SecurityID sid{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
|
||||
SecurityData &add_security_data(
|
||||
SecurityType sec_type, const string &sec_desc) {
|
||||
const SecurityID sid{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
|
||||
auto [it, is_new] = security_data_map.try_emplace(
|
||||
sec_desc, std::make_unique<SecurityData>(sid, listener));
|
||||
_ASSERT_MSGF(is_new, "add_security_data. Already added '{}'", sec_desc);
|
||||
@@ -182,13 +185,13 @@ struct OkxSubscription::PImpl {
|
||||
void subscribe_md(SecurityNames security_names) {
|
||||
// Populate list of 'channels', and prepare their corresponding handlers
|
||||
vector<OkxChannel> channels;
|
||||
for (const string &sec : security_names) {
|
||||
auto &sec_data = add_security_data(sec);
|
||||
for (const auto &[type, name] : security_names) {
|
||||
auto &sec_data = add_security_data(type, name);
|
||||
// Depth
|
||||
if (subscribe_depth) {
|
||||
auto &ch = channels.emplace_back();
|
||||
ch.channel = "books5";
|
||||
ch.instId = sec;
|
||||
ch.instId = name;
|
||||
channel_demuxer.add_handler(ch).bind<&OkxDepthAdaptor::on_depths>(
|
||||
sec_data.depth_hnd);
|
||||
}
|
||||
@@ -196,7 +199,7 @@ struct OkxSubscription::PImpl {
|
||||
if (subscribe_ticker) {
|
||||
auto &ch = channels.emplace_back();
|
||||
ch.channel = "tickers";
|
||||
ch.instId = sec;
|
||||
ch.instId = name;
|
||||
channel_demuxer.add_handler(ch).bind<&OkxTickerAdaptor::on_tickers>(
|
||||
sec_data.ticker_hnd);
|
||||
}
|
||||
@@ -204,7 +207,7 @@ struct OkxSubscription::PImpl {
|
||||
if (subscribe_trades) {
|
||||
auto &ch = channels.emplace_back();
|
||||
ch.channel = "trades";
|
||||
ch.instId = sec;
|
||||
ch.instId = name;
|
||||
channel_demuxer.add_handler(ch).bind<&OkxTradeAdaptor::on_trades>(
|
||||
sec_data.trade_hnd);
|
||||
}
|
||||
@@ -240,7 +243,9 @@ struct OkxSubscription::PImpl {
|
||||
bts_assert(!CONTAINS_KEY(security_data_map, sec->desc()));
|
||||
}
|
||||
listener->on_securities(secs);
|
||||
auto sec_names = TRANSFORM(secs, [](const auto *s) { return s->desc(); });
|
||||
auto sec_names = TRANSFORM(secs, [](const auto *s) {
|
||||
return std::make_pair(s->type(), s->desc());
|
||||
});
|
||||
batch_subscribe_md(sec_names);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -59,6 +59,10 @@ using namespace bts::orderman;
|
||||
|
||||
namespace bts::md {
|
||||
|
||||
#define SEC_TYPE(x) bts::api::domain::SECURITY_TYPE_##x
|
||||
#define TRADING_STATUS(x) bts::co2::TRADINGSTATUS_##x
|
||||
#define VENUE bts::api::domain::VENUE_DERIBIT
|
||||
|
||||
std::atomic<orderman::IEMOrderMuxer *> EmMDListener::_order_mux = nullptr;
|
||||
using MDArray = std::array<PriceSize, COCONUT_MAX_DEPTH>;
|
||||
using EMOrderKey = uint64_t;
|
||||
@@ -1144,16 +1148,32 @@ DescSearchResult EmMDListener::_find_em_sec(
|
||||
}
|
||||
}
|
||||
|
||||
static api::domain::SecurityType convert_sec_type(
|
||||
emumba::market_data::security_type emumba_sec_type) {
|
||||
switch (emumba_sec_type) {
|
||||
case emumba::market_data::security_type::index:
|
||||
return SEC_TYPE(SPOT);
|
||||
case emumba::market_data::security_type::future:
|
||||
return SEC_TYPE(FUTURE);
|
||||
case emumba::market_data::security_type::spread:
|
||||
return SEC_TYPE(SPREAD);
|
||||
case emumba::market_data::security_type::option:
|
||||
return SEC_TYPE(OPTION);
|
||||
default:
|
||||
bts_assert_msg(false, "Unknown sec type");
|
||||
return SEC_TYPE(UNSPECIFIED);
|
||||
}
|
||||
}
|
||||
|
||||
security_key EmMDListener::_add_sec_entry(
|
||||
const emumba::market_data::security &sec) {
|
||||
const std::string &desc = extract_security_desc(sec);
|
||||
// id_gen either adds a key and returns a new id, or finds one and returns
|
||||
// the previously added id
|
||||
bts_assert(me->sec_emdesc_map.find(desc) == me->sec_emdesc_map.end());
|
||||
auto new_key_opt = me->id_gen.get_id("exchange_key", desc, true);
|
||||
bts_assert(new_key_opt.has_value());
|
||||
const security_key new_key = static_cast<security_key>(
|
||||
me->id_gen.get_or_add(VENUE, convert_sec_type(sec.sec_type()), desc));
|
||||
// TODO (64 -> 32 conversion)
|
||||
security_key new_key = new_key_opt.value();
|
||||
me->sec_emdesc_map.insert({desc, {new_key, sec}});
|
||||
bts_assert(me->sec_btsid_map.find(new_key) == me->sec_btsid_map.end());
|
||||
me->sec_btsid_map.insert({new_key, desc});
|
||||
|
||||
@@ -43,10 +43,11 @@ struct EmumbaMain::PImpl {
|
||||
|
||||
PImpl()
|
||||
: sec_index(), recorder(
|
||||
{utils::Dispatcher::current(),
|
||||
[=, this](ExchangeKey k) {
|
||||
return SecurityKey(sec_index.get_key(k.value()));
|
||||
}}),
|
||||
nullptr,
|
||||
[=, this](ExchangeKey k) {
|
||||
return SecurityKey(sec_index.get_key(k.value()));
|
||||
},
|
||||
&utils::Dispatcher::current()),
|
||||
producer(recorder), listener(sec_index, producer) {
|
||||
|
||||
const auto config_path = Path{config::get<SystemConfig>().config_path()} /
|
||||
|
||||
@@ -1,70 +1,66 @@
|
||||
#include "ICEMain.h"
|
||||
|
||||
#include "lib/config/config.h"
|
||||
#include "lib/md/SecurityDB.h"
|
||||
#include "lib/md/coconut2.h"
|
||||
#include "lib/md/ice/ICEUtils.h"
|
||||
#include "lib/md/protos/co2.pb.h"
|
||||
#include "lib/md/vendor/OnixsUtils.h"
|
||||
#include "lib/os/os.h"
|
||||
#include "lib/utils/Dispatcher.h"
|
||||
#include "lib/utils/bts_chrono.h"
|
||||
#include "lib/utils/helper_macros.h"
|
||||
#include "lib/utils/proto_helper.h"
|
||||
|
||||
#include "lib/config/protos/deployment_type.pb.h"
|
||||
#include "lib/config/protos/market_data.pb.h"
|
||||
#include "lib/config/protos/system_config.pb.h"
|
||||
#include "lib/utils/thread_manager.h"
|
||||
#include "lib/md/ice/ICEUtils.h"
|
||||
#include "lib/md/vendor/OnixsUtils.h"
|
||||
|
||||
#include <OnixS/ICE/iMpact/MarketData.h>
|
||||
#include <boost/filesystem.hpp>
|
||||
#include <memory>
|
||||
#include <set>
|
||||
|
||||
void recursive_copy(
|
||||
const boost::filesystem::path &src, const boost::filesystem::path &dst) {
|
||||
if (boost::filesystem::exists(dst))
|
||||
throw std::runtime_error(dst.generic_string() + " exists");
|
||||
if (boost::filesystem::is_directory(src)) {
|
||||
boost::filesystem::create_directories(dst);
|
||||
for (boost::filesystem::directory_entry &item :
|
||||
boost::filesystem::directory_iterator(src))
|
||||
recursive_copy(item.path(), dst / item.path().filename());
|
||||
} else if (boost::filesystem::is_regular_file(src))
|
||||
boost::filesystem::copy(src, dst);
|
||||
else
|
||||
throw std::runtime_error(dst.generic_string() + " not dir or file");
|
||||
}
|
||||
|
||||
using namespace std;
|
||||
using namespace bts::co2;
|
||||
using std::size_t;
|
||||
|
||||
namespace bts {
|
||||
namespace md {
|
||||
namespace ice {
|
||||
namespace bts::md::ice {
|
||||
|
||||
using bts::config::SystemConfig;
|
||||
using Path = boost::filesystem::path;
|
||||
|
||||
ICEMain::ICEMain()
|
||||
: _index(), _recorder(
|
||||
{utils::Dispatcher::current(),
|
||||
[=, this](ExchangeKey k) {
|
||||
return SecurityKey(_index.get_key(k.value()));
|
||||
}}),
|
||||
_producer(_recorder), _handlers(), _listeners() {}
|
||||
struct ICEMain::PImpl {
|
||||
ICEOptions options;
|
||||
std::string config_file;
|
||||
ICESecurityIndex index;
|
||||
SecurityIDRecorder recorder;
|
||||
MDProducer producer;
|
||||
std::vector<std::pair<ICEChannel, std::unique_ptr<onixs_ice::Handler>>>
|
||||
handlers;
|
||||
std::vector<std::unique_ptr<ICEListener>> listeners;
|
||||
|
||||
ICEMain::~ICEMain() {}
|
||||
PImpl(ICEOptions options)
|
||||
: options(options),
|
||||
config_file((Path{config::get<SystemConfig>().data_path()} /
|
||||
"onixs_ice" / this->options.ice_config_file())
|
||||
.string()),
|
||||
index(), recorder(
|
||||
nullptr,
|
||||
[=, this](ExchangeKey k) {
|
||||
return SecurityKey(index.get_key(k.value()));
|
||||
},
|
||||
&utils::Dispatcher::current()),
|
||||
producer(recorder), handlers(), listeners() {}
|
||||
|
||||
const IceChannelConfig channel_settings_for(const ICEChannel &channel) {
|
||||
const auto &config = config::get<ICEOptions>();
|
||||
for (const auto &ch : config.channel()) {
|
||||
if (ch.name() == channel.name)
|
||||
return ch;
|
||||
IceChannelConfig channel_settings_for(const ICEChannel &channel) {
|
||||
for (const auto &ch : options.channel()) {
|
||||
if (ch.name() == channel.name)
|
||||
return ch;
|
||||
}
|
||||
return {};
|
||||
}
|
||||
return {};
|
||||
};
|
||||
|
||||
ICEMain::ICEMain() : ICEMain(config::get<ICEOptions>()) {}
|
||||
|
||||
ICEMain::ICEMain(ICEOptions options)
|
||||
: me_(std::make_unique<PImpl>(std::move(options))) {
|
||||
me_->options = std::move(options);
|
||||
}
|
||||
|
||||
ICEMain::~ICEMain() = default;
|
||||
|
||||
onixs_ice::MarketSubscriptions market_subcription_for_channel(
|
||||
const ICEChannel &chan) {
|
||||
onixs_ice::MarketSubscriptions subs;
|
||||
@@ -90,23 +86,30 @@ onixs_ice::MarketSubscriptions market_subcription_for_channel(
|
||||
return subs;
|
||||
}
|
||||
|
||||
void ICEMain::start(const std::vector<UserSymbol> &symbols) {
|
||||
auto ice_conf_location =
|
||||
Path{config::get<SystemConfig>().data_path()} / "onixs_ice/";
|
||||
if (!boost::filesystem::is_directory(ice_conf_location)) {
|
||||
auto conf_path = Path{os::sparkd_wd()} / "data_files" / "onixs_ice/";
|
||||
recursive_copy(Path{conf_path}, ice_conf_location);
|
||||
static void validate_config_file(const std::string &config_file) {
|
||||
if (!boost::filesystem::exists(config_file)) {
|
||||
const std::string config_url =
|
||||
config::get<config::DeploymentType>().type() == "dev"
|
||||
? "https://community.theice.com/docs/DOC-19989"
|
||||
: "https://drive.google.com/"
|
||||
"open?id=13_eyF2pKtn3F7chgsa_75ftUdf7emMbp";
|
||||
bts_assert_msgf(
|
||||
false, "Ice config file not found '{}'. Please download from {}",
|
||||
config_file, config_url);
|
||||
}
|
||||
}
|
||||
|
||||
void ICEMain::start(const std::vector<UserSymbol> &symbols) {
|
||||
await_ice_config();
|
||||
validate_config_file(me_->config_file);
|
||||
auto license = onixs_create_temporary_license();
|
||||
|
||||
const auto &channels = parse_ice_channels(symbols);
|
||||
for (const auto &channel : channels) {
|
||||
unique_ptr<onixs_ice::Handler> phandler =
|
||||
prepare_handler(channel, license.path.string());
|
||||
_listeners.push_back(prepare_listener(*phandler, channel));
|
||||
_handlers.push_back(std::make_pair(channel, std::move(phandler)));
|
||||
me_->listeners.push_back(prepare_listener(*phandler, channel));
|
||||
me_->handlers.push_back(std::make_pair(channel, std::move(phandler)));
|
||||
|
||||
std::stringstream symstr;
|
||||
for (const auto &sym : channel.symbols)
|
||||
@@ -115,7 +118,7 @@ void ICEMain::start(const std::vector<UserSymbol> &symbols) {
|
||||
symstr.str());
|
||||
}
|
||||
|
||||
for (auto &pair : _handlers) {
|
||||
for (auto &pair : me_->handlers) {
|
||||
LOG(md, Info, "ICE: Init Handler {}", pair.first.to_string());
|
||||
sleep_for(16s);
|
||||
onixs_ice::MarketSubscriptions subs =
|
||||
@@ -132,7 +135,7 @@ void ICEMain::start(const std::vector<UserSymbol> &symbols) {
|
||||
}
|
||||
|
||||
void ICEMain::send_heartbeats() {
|
||||
for (const auto &listener : _listeners)
|
||||
for (const auto &listener : me_->listeners)
|
||||
listener->heartbeat();
|
||||
}
|
||||
|
||||
@@ -140,20 +143,18 @@ void ICEMain::send_heartbeats() {
|
||||
// Initialize HandlerSettings and allocate the channel Handler
|
||||
unique_ptr<onixs_ice::Handler> ICEMain::prepare_handler(
|
||||
const ICEChannel &channel, const std::string &license) {
|
||||
const auto &options = channel_settings_for(channel);
|
||||
const auto &options = me_->channel_settings_for(channel);
|
||||
LOG(md, Info, "Loaded channel options for channel {}:\n{}",
|
||||
channel.to_string(), options.DebugString());
|
||||
|
||||
onixs_ice::HandlerSettings settings;
|
||||
settings.licenseDirectory = license;
|
||||
//
|
||||
// Logging
|
||||
settings.logLevel = (onixs_ice::LogLevels::Enum)options.onixs_log_mode();
|
||||
settings.logDirectory = ice_log_path();
|
||||
|
||||
//
|
||||
// Config file
|
||||
settings.connectivityConfiguration = ice_config_path();
|
||||
settings.logDirectory =
|
||||
(boost::filesystem::path{config::get<SystemConfig>().data_path()} /
|
||||
"logging" / "onixs_ice/")
|
||||
.string();
|
||||
settings.connectivityConfiguration = me_->config_file;
|
||||
|
||||
//
|
||||
// Network settings
|
||||
@@ -231,12 +232,12 @@ unique_ptr<ICEListener> ICEMain::prepare_listener(
|
||||
// heartbeats
|
||||
vector<MDProducerChannel> producer_channels;
|
||||
for (auto &symbol : channel.symbols) {
|
||||
producer_channels.push_back(_producer.get_channel(
|
||||
producer_channels.push_back(me_->producer.get_channel(
|
||||
symbol.symbol().c_str(), symbol.type(), symbol.exchange()));
|
||||
}
|
||||
|
||||
unique_ptr<ICEListener> listener(new ICEListener(
|
||||
_index, _producer, /*_settlement_recorder,*/ producer_channels));
|
||||
me_->index, me_->producer, /*_settlement_recorder,*/ producer_channels));
|
||||
for (auto &symbol : channel.symbols)
|
||||
listener->add_filter_symbol(symbol.symbol());
|
||||
|
||||
@@ -254,6 +255,4 @@ unique_ptr<ICEListener> ICEMain::prepare_listener(
|
||||
return listener;
|
||||
}
|
||||
|
||||
} // namespace ice
|
||||
} // namespace md
|
||||
} // namespace bts
|
||||
} // namespace bts::md::ice
|
||||
@@ -1,21 +1,12 @@
|
||||
#pragma once
|
||||
|
||||
#include "ICEListener.h"
|
||||
#include "ICESecurityIndex.h"
|
||||
#include "lib/md/coconut2.h"
|
||||
#include "lib/utils/bts_no_copy.h"
|
||||
#include "onixs.h"
|
||||
#include "server/sparkd/md/platforms/IMDPlatform.h"
|
||||
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
namespace bts {
|
||||
namespace md {
|
||||
namespace ice {
|
||||
|
||||
// void run_onixs_ice_server(UserSymbol* sym, size_t symcount);
|
||||
namespace bts::md::ice {
|
||||
|
||||
struct ICEChannel;
|
||||
|
||||
@@ -28,27 +19,22 @@ struct ICEChannel;
|
||||
class ICEMain : public IMDPlatform {
|
||||
public:
|
||||
ICEMain();
|
||||
explicit ICEMain(ICEOptions options);
|
||||
ICEMain(const ICEMain &) = delete;
|
||||
ICEMain &operator=(const ICEMain &) = delete;
|
||||
~ICEMain();
|
||||
|
||||
void start(const std::vector<UserSymbol> &symbols);
|
||||
void send_heartbeats() override;
|
||||
|
||||
std::unique_ptr<onixs_ice::Handler>
|
||||
prepare_handler(const ICEChannel &channel, const std::string &license);
|
||||
std::unique_ptr<ICEListener>
|
||||
prepare_listener(onixs_ice::Handler &handler, const ICEChannel &channel);
|
||||
std::unique_ptr<onixs_ice::Handler> prepare_handler(
|
||||
const ICEChannel &channel, const std::string &license);
|
||||
std::unique_ptr<ICEListener> prepare_listener(
|
||||
onixs_ice::Handler &handler, const ICEChannel &channel);
|
||||
|
||||
private:
|
||||
BTS_NO_COPY(ICEMain);
|
||||
|
||||
ICESecurityIndex _index;
|
||||
SecurityIDRecorder _recorder;
|
||||
MDProducer _producer;
|
||||
std::vector<std::pair<ICEChannel, std::unique_ptr<onixs_ice::Handler>>>
|
||||
_handlers;
|
||||
std::vector<std::unique_ptr<ICEListener>> _listeners;
|
||||
struct PImpl;
|
||||
std::unique_ptr<PImpl> me_;
|
||||
};
|
||||
|
||||
} // namespace ice
|
||||
} // namespace md
|
||||
} // namespace bts
|
||||
} // namespace bts::md::ice
|
||||
@@ -1,24 +1,18 @@
|
||||
#include "MDP3Main.h"
|
||||
|
||||
#include "lib/api/md/GlbxMarketDataConfig.pb.h"
|
||||
#include "lib/config/config.h"
|
||||
#include "lib/config/protos/deployment_type.pb.h"
|
||||
#include "lib/logging/logging.h"
|
||||
#include "lib/md/MBOProcessor.h"
|
||||
#include "lib/md/SecurityDB.h"
|
||||
#include "lib/md/cme/MDP3Utils.h"
|
||||
#include "lib/md/coconut2.h"
|
||||
#include "lib/md/protos/co2.pb.h"
|
||||
#include "lib/md/cme/mdp3_data_directory.h"
|
||||
#include "lib/md/vendor/OnixsUtils.h"
|
||||
#include "lib/os/os.h"
|
||||
#include "lib/utils/Dispatcher.h"
|
||||
#include "lib/utils/proto_helper.h"
|
||||
|
||||
#include "lib/config/protos/market_data.pb.h"
|
||||
#include "lib/config/protos/system_config.pb.h"
|
||||
#include "lib/os/spark_data_directory.h"
|
||||
#include "lib/utils/thread_manager.h"
|
||||
|
||||
#include <boost/filesystem.hpp>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <set>
|
||||
#include <sstream>
|
||||
|
||||
@@ -29,16 +23,33 @@ using std::size_t;
|
||||
namespace bts {
|
||||
namespace md {
|
||||
|
||||
MDP3Main::MDP3Main()
|
||||
: _index(), _recorder(
|
||||
{utils::Dispatcher::current(),
|
||||
[=, this](ExchangeKey k) {
|
||||
return SecurityKey(_index.get_key(k.value()));
|
||||
}}),
|
||||
_producer(_recorder), _engines(), _handlers(), _listeners(), _recovery() {
|
||||
}
|
||||
#define TRADING_STATUS(x) bts::domain::protobufs::TRADINGSTATUS_##x
|
||||
|
||||
MDP3Main::~MDP3Main() {}
|
||||
using Path = boost::filesystem::path;
|
||||
|
||||
struct MDP3Main::PImpl {
|
||||
Mdp3DataDirectory data_dir;
|
||||
CMESecurityIndex index;
|
||||
SecurityIDRecorder recorder;
|
||||
MDProducer producer;
|
||||
std::map<std::string, std::unique_ptr<onixs::FeedEngine>> engines;
|
||||
std::map<std::string, std::unique_ptr<onixs::Handler>> handlers;
|
||||
std::vector<std::unique_ptr<MDP3Listener>> listeners;
|
||||
std::vector<std::unique_ptr<onixs::TcpRecoveryService>> recovery;
|
||||
PImpl()
|
||||
: index(), recorder(
|
||||
nullptr,
|
||||
[=, this](ExchangeKey k) {
|
||||
return SecurityKey(index.get_key(k.value()));
|
||||
},
|
||||
&utils::Dispatcher::current()),
|
||||
producer(recorder), engines(), handlers(), listeners(), recovery() {}
|
||||
};
|
||||
|
||||
MDP3Main::MDP3Main()
|
||||
: me_(std::make_unique<PImpl>()) {}
|
||||
|
||||
MDP3Main::~MDP3Main() = default;
|
||||
|
||||
const CmeChannelConfig channel_settings_for(const string &id) {
|
||||
const auto &config = config::get<CMEOptions>();
|
||||
@@ -173,11 +184,11 @@ void MDP3Main::start(
|
||||
auto license = onixs_create_temporary_license();
|
||||
initialize(license.path.string());
|
||||
const auto &channels = parse_channels(symbols);
|
||||
_engines = create_feed_engines(channels);
|
||||
me_->engines = create_feed_engines(channels);
|
||||
for (const auto &channel : channels) {
|
||||
unique_ptr<onixs::Handler> phandler = prepare_handler(channel.id, mbo);
|
||||
_listeners.push_back(prepare_listener(*phandler, channel, mbo, tssd));
|
||||
_handlers.insert(std::make_pair(channel.id, std::move(phandler)));
|
||||
me_->listeners.push_back(prepare_listener(*phandler, channel, mbo, tssd));
|
||||
me_->handlers.insert(std::make_pair(channel.id, std::move(phandler)));
|
||||
std::stringstream symstr;
|
||||
for (const auto &sym : channel.symbols) {
|
||||
symstr << sym.symbol() << "-" << SecurityType_Name(sym.type()) << " ";
|
||||
@@ -186,7 +197,7 @@ void MDP3Main::start(
|
||||
symstr.str());
|
||||
}
|
||||
|
||||
for (auto &pair : _handlers) {
|
||||
for (auto &pair : me_->handlers) {
|
||||
onixs::SessionSettings settings =
|
||||
session_settings_for_channel(pair.first, mbo);
|
||||
pair.second->start(settings);
|
||||
@@ -196,7 +207,7 @@ void MDP3Main::start(
|
||||
}
|
||||
|
||||
void MDP3Main::send_heartbeats() {
|
||||
for (const auto &listener : _listeners) {
|
||||
for (const auto &listener : me_->listeners) {
|
||||
listener->heartbeat();
|
||||
}
|
||||
}
|
||||
@@ -219,7 +230,8 @@ unique_ptr<onixs::Handler> MDP3Main::prepare_handler(
|
||||
|
||||
onixs::HandlerSettings settings;
|
||||
settings.channel(std::stoul(channel_id));
|
||||
settings.feeds().connectivityConfigurationFile(config_path());
|
||||
settings.feeds().connectivityConfigurationFile(
|
||||
(me_->data_dir.config_file()).string());
|
||||
|
||||
// Market data options
|
||||
settings.bookManagement().directBooks().maintain(true);
|
||||
@@ -306,7 +318,8 @@ unique_ptr<onixs::Handler> MDP3Main::prepare_handler(
|
||||
logger = new onixs::NullLogger();
|
||||
} else {
|
||||
onixs::FileLoggerSettings loggerSettings;
|
||||
loggerSettings.filename(log_path() + "/log_" + channel_id);
|
||||
loggerSettings.filename(
|
||||
(me_->data_dir.log_dir() / ("log_" + channel_id)).string());
|
||||
loggerSettings.severityLevel(
|
||||
(onixs::LogSeverity::Enum)options.onixs_log_mode());
|
||||
logger = new onixs::FileLogger(loggerSettings);
|
||||
@@ -338,19 +351,19 @@ unique_ptr<onixs::Handler> MDP3Main::prepare_handler(
|
||||
|
||||
try {
|
||||
auto handler = unique_ptr<onixs::Handler>(new onixs::Handler(settings));
|
||||
auto iter = _engines.find(options.feed_engine());
|
||||
if (iter == _engines.end()) {
|
||||
auto iter = me_->engines.find(options.feed_engine());
|
||||
if (iter == me_->engines.end()) {
|
||||
LOG(md, Fatal,
|
||||
"Missing feed engine definition '{}' for channel id '{}'. Add a "
|
||||
"feed_engine entry with name:'{}' to the CMEOptions config key.",
|
||||
options.feed_engine(), channel_id, options.feed_engine());
|
||||
bts_assert_msg(
|
||||
iter != _engines.end(), "Missing feed engine definition.");
|
||||
iter != me_->engines.end(), "Missing feed engine definition.");
|
||||
}
|
||||
handler->bindFeedEngine(*(iter->second));
|
||||
if (recovery) {
|
||||
handler->bindTcpRecovery(*recovery);
|
||||
_recovery.push_back(std::move(recovery));
|
||||
me_->recovery.push_back(std::move(recovery));
|
||||
}
|
||||
if (logger) {
|
||||
handler->bindLogger(*logger);
|
||||
@@ -375,12 +388,13 @@ unique_ptr<MDP3Listener> MDP3Main::prepare_listener(
|
||||
// heartbeats
|
||||
vector<MDProducerChannel> producer_channels;
|
||||
for (auto &symbol : channel.symbols) {
|
||||
producer_channels.push_back(_producer.get_channel(
|
||||
producer_channels.push_back(me_->producer.get_channel(
|
||||
symbol.symbol().c_str(), symbol.type(), symbol.exchange()));
|
||||
}
|
||||
|
||||
unique_ptr<MDP3Listener> listener(new MDP3Listener(
|
||||
_index, _producer, mbo, producer_channels, trade_summary_dispatcher));
|
||||
me_->index, me_->producer, mbo, producer_channels,
|
||||
trade_summary_dispatcher));
|
||||
for (auto &symbol : channel.symbols) {
|
||||
listener->add_filter_symbol(symbol.symbol());
|
||||
}
|
||||
|
||||
@@ -29,7 +29,9 @@ struct CMEChannel;
|
||||
*/
|
||||
class MDP3Main : public IMDPlatform {
|
||||
public:
|
||||
MDP3Main();
|
||||
explicit MDP3Main();
|
||||
MDP3Main(const MDP3Main &) = delete;
|
||||
MDP3Main &operator=(const MDP3Main &) = delete;
|
||||
~MDP3Main();
|
||||
|
||||
void start(
|
||||
@@ -39,8 +41,8 @@ class MDP3Main : public IMDPlatform {
|
||||
void send_heartbeats() override;
|
||||
|
||||
void initialize(const std::string &license_store);
|
||||
std::unique_ptr<onixs::Handler>
|
||||
prepare_handler(const std::string &channel, MBOWriter *);
|
||||
std::unique_ptr<onixs::Handler> prepare_handler(
|
||||
const std::string &channel, MBOWriter *);
|
||||
std::unique_ptr<MDP3Listener> prepare_listener(
|
||||
onixs::Handler &handler,
|
||||
const CMEChannel &channel,
|
||||
@@ -48,16 +50,8 @@ class MDP3Main : public IMDPlatform {
|
||||
TradeSummarySignalDispatcher *);
|
||||
|
||||
private:
|
||||
BTS_NO_COPY(MDP3Main);
|
||||
|
||||
CMESecurityIndex _index;
|
||||
SecurityIDRecorder _recorder;
|
||||
MDProducer _producer;
|
||||
// co2::SettlementRecorder _settlement_recorder;
|
||||
std::map<std::string, std::unique_ptr<onixs::FeedEngine>> _engines;
|
||||
std::map<std::string, std::unique_ptr<onixs::Handler>> _handlers;
|
||||
std::vector<std::unique_ptr<MDP3Listener>> _listeners;
|
||||
std::vector<std::unique_ptr<onixs::TcpRecoveryService>> _recovery;
|
||||
struct PImpl;
|
||||
std::unique_ptr<PImpl> me_;
|
||||
};
|
||||
|
||||
} // namespace md
|
||||
|
||||
@@ -201,8 +201,9 @@ class TaifexDataProcessors {
|
||||
const auto id = TaifexUtils::remove_spaces(msg.prod_id);
|
||||
const auto symbol = id.substr(0, 3);
|
||||
|
||||
const auto sec_type = convert_sec_type(msg.parsedProductInfo.productType);
|
||||
bts::domain::SecurityID sec_id{
|
||||
sec_id_store->get_or_add(api::domain::VENUE_TAIFEX, id)};
|
||||
sec_id_store->get_or_add(api::domain::VENUE_TAIFEX, sec_type, id)};
|
||||
auto check =
|
||||
security_details->add_i010_mapping(id, sec_id, msg.decimal_locator);
|
||||
if (!check) {
|
||||
@@ -286,6 +287,20 @@ class TaifexDataProcessors {
|
||||
security_details->add_i011_mapping(
|
||||
id, 1.0, msg.getCurrencyString(), msg.contract_size);
|
||||
}
|
||||
|
||||
private:
|
||||
static api::domain::SecurityType convert_sec_type(
|
||||
TaifexMessages::I010Data::ProductInfo::ProductType type) {
|
||||
switch (type) {
|
||||
case TaifexMessages::I010Data::ProductInfo::ProductType::FUTURES:
|
||||
return api::domain::SecurityType::SECURITY_TYPE_FUTURE;
|
||||
case TaifexMessages::I010Data::ProductInfo::ProductType::OPTIONS:
|
||||
return api::domain::SecurityType::SECURITY_TYPE_OPTION;
|
||||
default:
|
||||
bts_assert_msg(
|
||||
false, "Unexpected security type (must be FUTURES or OPTIONS)");
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
} //namespace bts::md::taifex
|
||||
} //namespace bts::md::taifex
|
||||
@@ -293,9 +293,9 @@ void PositionServiceServer::update_manual_fill(
|
||||
auto *this_resp = resp.add_response();
|
||||
auto id = std::to_string(mf.record_id());
|
||||
|
||||
auto p = me->manual_fills_store->retrieve_one(manual_fills_ns, id);
|
||||
auto found = p.first;
|
||||
if (!found) {
|
||||
auto [exists, rowid, val] =
|
||||
me->manual_fills_store->retrieve_one(manual_fills_ns, id);
|
||||
if (!exists) {
|
||||
this_resp->set_ok(false);
|
||||
this_resp->set_msg("Could not find record");
|
||||
} else {
|
||||
@@ -326,9 +326,9 @@ void PositionServiceServer::remove_manual_fill(
|
||||
auto *this_resp = resp.add_response();
|
||||
auto id = std::to_string(mf.record_id());
|
||||
|
||||
auto p = me->manual_fills_store->retrieve_one(manual_fills_ns, id);
|
||||
auto found = p.first;
|
||||
if (!found) {
|
||||
auto [exists, rowid, val] =
|
||||
me->manual_fills_store->retrieve_one(manual_fills_ns, id);
|
||||
if (!exists) {
|
||||
this_resp->set_ok(false);
|
||||
this_resp->set_msg("Could not find record");
|
||||
} else {
|
||||
@@ -367,10 +367,9 @@ void PositionServiceServer::update_security_parameters(
|
||||
req.op_params_size());
|
||||
for (const auto &p : req.op_params()) {
|
||||
std::string key_str = std::to_string(p.security_key());
|
||||
auto db_p =
|
||||
auto [exists, rowid, val] =
|
||||
me->security_params_store->retrieve_one(security_params_ns, key_str);
|
||||
auto found = db_p.first;
|
||||
if (!found) {
|
||||
if (!exists) {
|
||||
me->security_params_store->insert(
|
||||
security_params_ns, key_str, utils::pb::to_string(p));
|
||||
} else {
|
||||
|
||||
@@ -21,7 +21,8 @@ AdminServiceDB::AdminServiceDB() : me(new PImpl()) {}
|
||||
AdminServiceDB::~AdminServiceDB() {}
|
||||
|
||||
std::optional<api::admin::AdminData> AdminServiceDB::retrieve() {
|
||||
auto [has_table, data] = me->store.retrieve_one(table_name, unique_key);
|
||||
auto [has_table, rowid, data] =
|
||||
me->store.retrieve_one(table_name, unique_key);
|
||||
if (!has_table)
|
||||
return {};
|
||||
|
||||
@@ -33,7 +34,8 @@ std::optional<api::admin::AdminData> AdminServiceDB::retrieve() {
|
||||
}
|
||||
|
||||
bool AdminServiceDB::update(api::admin::AdminData const &state) {
|
||||
auto [has_table, data] = me->store.retrieve_one(table_name, unique_key);
|
||||
auto [has_table, rowid, data] =
|
||||
me->store.retrieve_one(table_name, unique_key);
|
||||
UNUSED(data);
|
||||
|
||||
std::string ser_state;
|
||||
|
||||
Reference in new issue
Block a user