Merge branch 'development' of bitbucket.org:bluetradesys/spark into development

This commit is contained in:
Taha committed 2024-11-26 22:33:31 -05:00
commit fecbcbb6b6
86 files changed
+2592 -1080

No files matched your search

@@ -138,7 +138,7 @@
_dispose = Disposable.Create(resources.ToArray());
static string fmt_sec(TimeAndSalesRowVM row) => row.security.ToString();
static string fmt_time(TimeAndSalesRowVM row) => row.time.ToString("h:mm:ss tt");
static string fmt_time(TimeAndSalesRowVM row) => row.time.ToString("h:mm:ss.ffffff");
static string fmt_price(TimeAndSalesRowVM row) => row.security.spec_or_default().format(row.price);
static string fmt_aggressor(TimeAndSalesRowVM row) => row.aggressor_side.ToString();
static string fmt_volume(TimeAndSalesRowVM row) => new TickFactors(1, 100).format(row.volume);
@@ -46,7 +46,7 @@
Filled = 56,
Symbol = 60,
Time = 112,
TimeOfDay = 64,
TimeOfDay = 84,
TimeInForce = 47,
User = 53,
Aggressor = 80,
@@ -69,8 +69,16 @@ namespace mbts {
public static bool is_valid(this RegionalTickFactors regional_tick) =>
regional_tick.low_tick.is_valid() && regional_tick.high_tick.is_valid();
public static bool is_integer(this TickFactors tick) =>
tick.tick_unit == tick.tick_base;
public static bool is_integer(this TickFactors tick) {
if (tick.tick_unit == tick.tick_base) {
return true;
}
if (tick.tick_unit > tick.tick_base && tick.tick_base != 0) {
var units_per_base = (int)(tick.tick_unit / tick.tick_base);
return units_per_base * tick.tick_base == tick.tick_unit;
}
return false;
}
public static TickFactors tick_factors(this RegionalTickFactors regional_tick, double value) =>
value > regional_tick.low_tick_inclusive_end ?
@@ -295,6 +295,9 @@ namespace mbts.spark {
var ticks_per_dollar = (uint)(tf.low_tick.tick_base / tf.low_tick.tick_unit);
// if ticks_per_row is greater than ticks_per_dollar, round to a multiple of ticks_per_dollar
if (ticks_per_row > ticks_per_dollar) {
if (ticks_per_dollar == 0) {
return null;
}
return (uint)(ticks_per_dollar * Math.Floor((double)(ticks_per_row / ticks_per_dollar)));
} else if (ticks_per_row == ticks_per_dollar) {
return ticks_per_row;
@@ -14,4 +14,5 @@ message DatabentoMarketDataConfig {
string key = 60;
string dataset = 70;
bool filter_securities_one_year= 80;
bool load_reference_data = 90;
}
@@ -156,6 +156,8 @@ AdminServiceUtils::resolve_connection(
const bts::api::admin::ExchangeConnection *rv = nullptr;
switch (sec.exchange()) {
case api::Venue::VENUE_GLBX: {
//CME EXECUTION NEEDS TO BE UPDATED, probably merge with VENUE_BINANCE_USDS_FUTURES
auto deployment_type =
boost::to_lower_copy(config::get<config::DeploymentType>().type());
rv = resolve_cme_connection(data, account, sec);
@@ -227,6 +229,7 @@ void AdminServiceUtils::fill_connections_from_config(
connection.set_name(c.name());
switch (c.venue()) {
//CME EXECUTION NEEDS TO BE UPDATED, probably merge default case
case bts::api::VENUE_GLBX:
if (c.cme_market_segment_id() == 0)
connection.mutable_cme_connection()->mutable_cgw_connection();
@@ -236,10 +239,7 @@ void AdminServiceUtils::fill_connections_from_config(
->set_market_segment_id(c.cme_market_segment_id());
add_connection(admin_data, connection);
break;
case bts::api::VENUE_CBOE:
break;
case bts::api::VENUE_TAIFEX:
break;
case bts::api::VENUE_SIM:
case bts::api::VENUE_SIMICE:
connection.mutable_sim_connection();
@@ -250,6 +250,8 @@ void AdminServiceUtils::fill_connections_from_config(
c.sender_compid());
add_connection(admin_data, connection);
break;
case bts::api::VENUE_TAIFEX:
case bts::api::VENUE_CBOE:
default:
connection.mutable_general_connection()->set_exchange(c.venue());
add_connection(admin_data, connection);
@@ -571,6 +573,13 @@ bts::api::orderman::RoutingRecord AdminServiceUtils::get_routing_record(
return rv;
}
switch (sec.exchange()) {
case api::Venue::VENUE_CBOE:{//subaccount not added
rv.set_session("cboe_1");
rv.set_account(account_ptr->name());
rv.set_operator_id(user_ptr->name());
out_result.set_is_success(true);
return rv;
}
case api::Venue::VENUE_DERIBIT: {
rv.set_session("deribit_1");
rv.set_account(account_ptr->name());
@@ -733,6 +742,13 @@ bts::api::orderman::RoutingRecord AdminServiceUtils::get_routing_record(
rv.set_customer_or_firm(sim_account_data->origin());
return rv;
} break;
case api::Venue::VENUE_TAIFEX:{
rv.set_session("taifex_1");
rv.set_account(account_ptr->name());
rv.set_operator_id(user_ptr->name());
out_result.set_is_success(true);
return rv;
}
case api::VENUE_FTX: // TODO
case api::VENUE_BINANCE: // TODO
case api::VENUE_BINANCE_US:
@@ -741,8 +757,6 @@ bts::api::orderman::RoutingRecord AdminServiceUtils::get_routing_record(
case api::VENUE_OKX:
case api::Venue::VENUE_UNSET:
case api::Venue::VENUE_BTS:
case api::Venue::VENUE_CBOE:
case api::Venue::VENUE_TAIFEX:
case api::Venue::VENUE_CBOECFE:
case api::Venue::Venue_INT_MIN_SENTINEL_DO_NOT_USE_:
case api::Venue::Venue_INT_MAX_SENTINEL_DO_NOT_USE_:
@@ -0,0 +1,22 @@
MarexFixCfg {
value{
customer_or_firm :0
handl_inst: 2
rule80a: 1
heart_bt_int : 30
counterparty_port :22754
name: "trafix"
msg_type: "Order_Single"
protocol_version: "FIX_42"
sender_sub_id: "BTS"
target_sub_id: "TRFXMRX"
client_id: "BTS"
counterparty_host : "74.217.55.230"
open_close :"O"
max_retries: 500
retry_interval: 600
ex_destination : "W"
security_exchange : "CBOE"
currency : "USD"
}
}
@@ -6,8 +6,8 @@ tickConfig
tick_entry {key:"TXF",value:1},
tick_entry {key:"MXF",value:1},
tick_entry {key:"TMF",value:1},
tick_entry {key:"CDF",value:1},
tick_entry {key:"QFF",value:1},
tick_entry {key:"CDF",value:5},
tick_entry {key:"QFF",value:5},
}
}
+33
View File
@@ -0,0 +1,33 @@
syntax = "proto3";
package bts.config;
message MarexFixCfg {
enum HandlInst {
UNDEFINED = 0;
DMA = 1;
ALGO = 2;
CARE = 3;
}
enum Rule80A {
UNUSED = 0;
A = 1;
P = 2;
}
string msg_type = 1;
string protocol_version = 2;
HandlInst handl_inst = 3;
string sender_sub_id = 4;
string target_sub_id = 5;
string client_id = 6;
Rule80A rule80a = 7;
string counterparty_host = 8;
int32 heart_bt_int = 9;
int32 counterparty_port = 10;
uint32 customer_or_firm =11;
string open_close =12;
string name = 13;
int32 max_retries = 14;
uint32 retry_interval = 15;
string ex_destination = 16;
string security_exchange = 17;
string currency = 18;
}
+10
View File
@@ -50,7 +50,9 @@ class TimeSpan {
constexpr bool operator==(TimeSpan lhs, TimeSpan rhs) noexcept;
constexpr bool operator!=(TimeSpan lhs, TimeSpan rhs) noexcept;
constexpr bool operator<(TimeSpan lhs, TimeSpan rhs) noexcept;
constexpr bool operator<=(TimeSpan lhs, TimeSpan rhs) noexcept;
constexpr bool operator>(TimeSpan lhs, TimeSpan rhs) noexcept;
constexpr bool operator>=(TimeSpan lhs, TimeSpan rhs) noexcept;
std::ostream &operator<<(std::ostream &out, const TimeSpan &ts) noexcept;
@@ -116,10 +118,18 @@ constexpr bool operator<(TimeSpan lhs, TimeSpan rhs) noexcept {
return lhs.nanos() < rhs.nanos();
}
constexpr bool operator<=(TimeSpan lhs, TimeSpan rhs) noexcept {
return lhs.nanos() <= rhs.nanos();
}
constexpr bool operator>(TimeSpan lhs, TimeSpan rhs) noexcept {
return lhs.nanos() > rhs.nanos();
}
constexpr bool operator>=(TimeSpan lhs, TimeSpan rhs) noexcept {
return lhs.nanos() >= rhs.nanos();
}
} // namespace bts::core
FMT_IMPL(bts::core::TimeSpan, t, "{}", t.to_iso_string());
+94 -53
View File
@@ -3,6 +3,7 @@
#include "lib/config/config.h"
#include "lib/config/protos/orders_fills_cleanup.pb.h"
#include "lib/md/common.h"
#include "lib/utils/datetime.h"
#include <boost/date_time/posix_time/posix_time.hpp>
@@ -11,51 +12,15 @@
namespace bts {
namespace securities {
std::string to_str(api::Security const &sec) {
std::string serialized;
google::protobuf::TextFormat::PrintToString(sec, &serialized);
return serialized;
}
std::pair<bool, api::Security> from_str(std::string const &str) {
api::Security result;
auto success = google::protobuf::TextFormat::ParseFromString(str, &result);
return std::make_pair(success, result);
}
std::string descriptor(api::Security const &sec) {
switch (sec.ExchangeData_case()) {
case api::Security::kTaifex:
return sec.taifex().security_desc();
case api::Security::kGlbx:
return sec.glbx().security_desc();
case api::Security::kCboe:
return sec.cboe().security_desc();
case api::Security::kCfe:
return sec.cfe().security_desc();
case api::Security::kIce:
return sec.ice().security_desc();
case api::Security::kDeribit:
return sec.deribit().security_desc();
case api::Security::kFtx:
return sec.ftx().security_desc();
case api::Security::kBinance:
return sec.binance().security_desc();
case api::Security::kBinanceUsdsFutures:
return sec.binance_usds_futures().security_desc();
case api::Security::kBinanceUs:
return sec.binance_us().security_desc();
case api::Security::kCoinbase:
return sec.coinbase().security_desc();
case api::Security::kOkx:
return sec.okx().security_desc();
default:
return "No descriptor for non glbx / ice exchange";
}
}
using bts::api::Security;
using bts::api::Venue;
api::SecurityType security_type(api::Security const &security) {
switch (security.Contract_case()) {
return security_type(security.Contract_case());
}
api::SecurityType security_type(api::Security::ContractCase contract_case) {
switch (contract_case) {
case api::Security::ContractCase::kFuture:
return api::SecurityType::SECURITYTYPE_FUTURE;
case api::Security::ContractCase::kOption:
@@ -70,16 +35,6 @@ api::SecurityType security_type(api::Security const &security) {
return api::SecurityType::SECURITYTYPE_UNSET;
}
return api::SecurityType::SECURITYTYPE_UNSET;
// if (sec.has_future())
// return api::SecurityType::SECURITYTYPE_FUTURE;
// else if (sec.has_option())
// return api::SecurityType::SECURITYTYPE_OPTION;
// else if (sec.has_spread())
// return api::SecurityType::SECURITYTYPE_SPREAD;
// else if (sec.has_synthetic())
// return api::SecurityType::SECURITYTYPE_SYNTHETIC;
// else
// return api::SecurityType::SECURITYTYPE_UNSET;
}
boost::gregorian::date smallest_leg(
@@ -140,5 +95,91 @@ bool is_expired(api::Security const &security) {
return is_expired(security, datetime::local_now());
}
void write_date_string(
uint32_t year, uint32_t month, uint32_t day, std::ostringstream &result) {
result << std::setw(2) << std::setfill('0') << month << "/" << std::setw(2)
<< std::setfill('0') << day << "/" << year - 2000;
}
std::ostringstream &write_leg_descriptor(
std::ostringstream &result, char separator, const api::ContractLeg &leg) {
bts_assert(leg.security().id() != md::invalid_security_key);
// Keep the string small, yet readable for diagnostic purposes.
result << (leg.side() == api::SIDE_BUY ? '+' : '-') << separator
<< leg.quantity() << "*(i=" << leg.security().id()
<< ",m=" << leg.price_multiplier() << ",d=" << leg.leg_delta() << ")";
return result;
}
std::ostringstream &write_future_descriptor(
std::ostringstream &result, const api::FutureContract &future) {
write_date_string(
future.expiration_year(), future.expiration_month(),
future.expiration_day(), result);
return result;
}
std::ostringstream &write_option_descriptor(
std::ostringstream &result,
char separator,
const api::OptionContract &option) {
write_date_string(
option.expiration_year(), option.expiration_month(),
option.expiration_day(), result);
result << separator << option.strike()
<< (option.call_put() == api::PUTORCALL_CALL ? 'C' : 'P');
return result;
}
std::ostringstream &write_legs_descriptor(
std::ostringstream &result,
char separator,
const google::protobuf::RepeatedPtrField<api::ContractLeg> &legs) {
bts_assert(legs.size() > 0);
auto it = legs.begin();
write_leg_descriptor(result, separator, *it);
while (++it != legs.end()) {
result << separator;
write_leg_descriptor(result, separator, *it);
}
return result;
}
std::string unique_descriptor(const api::Security &security) {
const char separator = ' ';
std::ostringstream result;
result << api::Venue_Name(security.exchange()).substr(6) << separator
<< api::SecurityType_Name(security_type(security)).substr(12)
<< descriptor(security);
switch (security.Contract_case()) {
case Security::ContractCase::kOption:
result << separator;
write_option_descriptor(result, separator, security.option());
break;
case Security::ContractCase::kFuture:
result << separator;
write_future_descriptor(result, security.future());
break;
case Security::ContractCase::kSpot:
break;
case Security::ContractCase::kSpread:
result << separator;
write_legs_descriptor(result, separator, security.spread().legs());
break;
case Security::ContractCase::kSynthetic: {
result << "pc="
<< api::SyntheticContract::PriceCalc_Name(
security.synthetic().price_calc());
write_legs_descriptor(result, separator, security.synthetic().legs());
break;
}
default:
bts_assert_msg(
false, "unique_descriptor failed due to invalid contract case");
break;
}
return result.str();
}
} // namespace securities
} // namespace bts
+58 -4
View File
@@ -2,6 +2,7 @@
#include "lib/core/FMT_IMPL.h"
#include "lib/domain_model/Security.pb.h"
#include "lib/domain_model/SecurityType.h"
#include "lib/domain_model/Side.h"
#include "lib/domain_model/Venue.h"
#include "lib/domain_model/common.h"
@@ -25,11 +26,27 @@ class ICEExchangeData;
} // namespace api
namespace securities {
SecurityKey security_key(api::Security const &);
std::string to_str(api::Security const &);
std::pair<bool, api::Security> from_str(std::string const &);
std::string descriptor(api::Security const &);
/**
* Gets a descriptor string for a security (should be unique within a particular exchange).
*/
const std::string &descriptor(
const api::Security &sec, const std::string &default_rv = "UNDEFINED");
/**
* Gets a descriptor string for a security that is unique within the entire
* system (not just exchange).
*/
std::string unique_descriptor(const api::Security &security);
api::SecurityType security_type(api::Security const &);
api::SecurityType security_type(api::Security::ContractCase contract_case);
void write_date_string(
std::uint32_t year,
std::uint32_t month,
std::uint32_t day,
std::ostringstream &result);
inline std::string const &side_name(api::Side side) {
return api::Side_Name(side);
@@ -234,6 +251,41 @@ inline Price tick_size(api::Security const &sec) {
return Price(sec.contract_spec().low_tick());
}
inline const std::string &descriptor(
const api::Security &sec, const std::string &default_rv) {
switch (sec.ExchangeData_case()) {
case api::Security::kGlbx:
return sec.glbx().security_desc();
case api::Security::kCboe:
return sec.cboe().security_desc();
case api::Security::kCfe:
return sec.cfe().security_desc();
case api::Security::kIce:
return sec.ice().security_desc();
case api::Security::kDeribit:
return sec.deribit().security_desc();
case api::Security::kFtx:
return sec.ftx().security_desc();
case api::Security::kBinance:
return sec.binance().security_desc();
case api::Security::kBinanceUsdsFutures:
return sec.binance_usds_futures().security_desc();
case api::Security::kBinanceUs:
return sec.binance_us().security_desc();
case api::Security::kCoinbase:
return sec.coinbase().security_desc();
case api::Security::kOkx:
return sec.okx().security_desc();
case api::Security::kTaifex:
return sec.taifex().security_desc();
default: {
if (sec.has_synthetic()) {
return sec.synthetic().name();
}
return default_rv;
}
}
}
// Looking to round md::IntPrice? Look in IntPrice.h
// computes price_2 - price_1 in ticks
@@ -314,3 +366,5 @@ inline bool is_spread_of_strips(const bts::api::Security &sec) {
} // namespace securities
} // namespace bts
FMT_IMPL(bts::api::Security, s, "{}", bts::securities::descriptor(s));
+30
View File
@@ -0,0 +1,30 @@
#pragma once
#include "lib/domain_model/Security.pb.h"
#include "lib/core/FMT_IMPL.h"
#include "lib/core/EnumMacros.h"
#include "lib/io/JsonReader.h"
#include "lib/logging/logging.h"
#include <nlohmann/json.hpp>
namespace bts::api {
PB_ENUM_FROM_JSON(SecurityType)
PB_ENUM_TO_JSON(SecurityType)
std::ostream &operator<<(std::ostream &out, const SecurityType &type) noexcept;
//////////////////////////////////////////////////
// Definition
//////////////////////////////////////////////////
inline std::ostream &operator<<(
std::ostream &out, const SecurityType &type) noexcept {
out << SecurityType_Name(type);
return out;
}
} // namespace bts::api
FMT_IMPL(bts::api::SecurityType, t, "{}", bts::api::SecurityType_Name(t));
+11
View File
@@ -13,6 +13,17 @@ namespace bts::api {
PB_ENUM_FROM_JSON(Venue)
PB_ENUM_TO_JSON(Venue)
std::ostream &operator<<(std::ostream &out, Venue venue) noexcept;
//////////////////////////////////////////////////
// Definition
//////////////////////////////////////////////////
inline std::ostream &operator<<(std::ostream &out, Venue venue) noexcept {
out << Venue_Name(venue);
return out;
}
} // namespace bts::api
FMT_IMPL(bts::api::Venue, v, "{}", bts::api::Venue_Name(v));
+2 -78
View File
@@ -3,6 +3,7 @@
#include "ShmDumper.h"
#include "../Shm/MDStructure.h"
#include "lib/domain_model/Security.h"
#include "lib/io/JsonDumper.h"
#include "lib/logging/logging.h"
#include "lib/utils/SharedList.h"
@@ -51,85 +52,8 @@ string dump_header(const ColdHeader &header) {
return JsonDumper::dump(header);
}
void write_date_string(
uint32_t year, uint32_t month, uint32_t day, std::ostringstream &result);
string ShmDumper::dump_security(const FixedSecurity &security) {
ostringstream res;
res << "{";
res << "'key': " << security.key << ", ";
res << "'symbol': '" << security.symbol() << "', ";
res << "'type': '" << api::SecurityType_Name(security.type) << "', ";
res << "'exchange': '" << api::Venue_Name(security.exchange) << "', ";
res << "'alt_symbol': '" << security.alt_symbol() << "', ";
res << "'cfi_code': '" << security.cfi_code() << "', ";
res << "'security_desc': '" << security.security_desc() << "', ";
res << "'platform_id': '" << security.platform_id() << "', ";
res << "'display_factor': " << (int)security.display_factor << ", ";
res << "'numeric_id': " << security.numeric_id << ", ";
res << "'cboe_class_key': " << security.cboe_class_key << ", ";
res << "'main_frac': " << (int)security.cme_main_fraction << ", ";
res << "'sub_fract': " << (int)security.cme_sub_fraction << ", ";
res << "'price_format': " << (int)security.cme_price_display_format << ", ";
res << "'market_segment': " << (int)security.cme_market_segment_id << ", ";
res << "'price_break': " << security.price_break << ", ";
res << "'low_tick': " << security.low_tick << ", ";
res << "'high_tick': " << security.high_tick << ", ";
res << "'cpp': " << security.cash_per_point << ", ";
res << "'market_description ': " << security.ice_market_description()
<< ", ";
res << "'cleared_alias ': " << security.ice_cleared_alias() << ", ";
res << "'product_description ': " << security.ice_product_name() << ", ";
res << "'hub_alias ': " << security.ice_hub_alias() << ", ";
res << "'strip_name': " << security.ice_strip_name() << ", ";
res << "'mic_code': " << security.mic_code() << ", ";
switch (security.type) {
case SECURITYTYPE_SPOT:
break;
case SECURITYTYPE_FUTURE: {
auto *psec = static_cast<const FixedFutureSecurity *>(&security);
res << "'expiration': '";
write_date_string(
psec->expiration_year, psec->expiration_month, psec->expiration_day,
res);
res << "' ";
break;
}
case SECURITYTYPE_OPTION: {
auto *psec = static_cast<const FixedOptionSecurity *>(&security);
res << "'expiration': '";
write_date_string(
psec->expiration_year, psec->expiration_month, psec->expiration_day,
res);
res << "', ";
res << "'strike': " << psec->strike << ", ";
res << "'callput': '" << api::PutOrCall_Name(psec->call_put) << "' ";
break;
}
case SECURITYTYPE_SPREAD: {
auto *psec = static_cast<const FixedSpreadSecurityXL *>(&security);
res << "'legs': [";
for (size_t x = 0; x < psec->leg_count(); ++x) {
auto &leg = psec->legs[x];
if (x > 0)
res << ",";
res << "{ 'key': " << leg.key << ", 'exkey': " << leg.exchange_key
<< ", 'quantity': " << leg.leg_quantity
<< ", 'leg_price': " << leg.leg_price
<< ", 'delta': " << leg.leg_delta << ", 'side': '"
<< api::Side_Name(leg.leg_side) << "' }";
}
res << "]";
break;
}
default:
break;
}
res << "}";
return res.str();
return JsonDumper::dump(security);
}
string dump(datetime::Date date) {
+13 -14
View File
@@ -23,12 +23,12 @@
#include "lib/domain_model/Security.pb.h"
#include "lib/domain_model/common.h"
#include "lib/logging/logging.h"
#include "lib/md/GeneralSecurityIndex.h"
#include "lib/md/IMarketDataListener.h"
#include "lib/md/IntPrice.h"
#include "lib/md/MDProducer.h"
#include "lib/md/MDProducerHandle.h"
#include "lib/md/PriceSize.h"
#include "lib/md/SecurityIDRecorder.h"
#include "lib/md/SecurityIndex.h"
#include "lib/md/Shm/FixedSecurity.h"
#include "lib/md/Shm/MDStructure.h"
@@ -274,10 +274,11 @@ bool update_incremental(const IMarketDataListener::BookOp &op) {
};
struct GeneralMarketDataListener::PImpl {
const api::domain::Venue venue;
// if this is set to true, then the security key passed on in the
// on_securities function is used. If this is not set to true, then a new key
// is generated
const bool use_given_sec_key = true;
const bool use_given_sec_key = false;
const std::string cc_name_prefix = "cc_security";
size_t n_bk_updates = 0;
bts::persistence::IDGenerator &id_gen =
@@ -296,12 +297,15 @@ struct GeneralMarketDataListener::PImpl {
MDArray asks = {};
IncrementalBookData inc_book_data = {};
explicit PImpl(
api::domain::Venue venue,
GeneralSecurityIndex &index,
MDProducer &md_producer,
std::shared_ptr<bts::orderman::SimExecutionGateway> sim_ex_gwy)
: index(index), md_producer(md_producer), sim_ex_gwy(sim_ex_gwy) {
if (sim_ex_gwy != nullptr) {
sim_ex_gwy->set_producers(&index);
: venue(venue), index(index),
md_producer(md_producer),
sim_ex_gwy(std::move(sim_ex_gwy)) {
if (this->sim_ex_gwy != nullptr) {
this->sim_ex_gwy->set_producers(&this->index);
}
}
};
@@ -363,6 +367,7 @@ void copy_books(
GeneralMarketDataListener::GeneralMarketDataListener(
GeneralSecurityIndex &index, MDProducer &md_producer, api::Venue venue)
: me(std::make_unique<PImpl>(
domain::DomainSecurityUtils::convert(venue),
index,
md_producer,
bts::orderman::SimExecutionGatewayFactory::get()->get_getayeway(
@@ -379,7 +384,7 @@ void GeneralMarketDataListener::on_securities(
LOG(md, Info, "GeneralMarketDataListener::on_securities n_secs: {}",
complete_security_list.size());
for (auto s_ptr : complete_security_list) {
domain::Security sec = *s_ptr;
const domain::Security &sec = *s_ptr;
const auto &s_comm = sec.security_common;
CCSecSearchResult search_result = _find_sec(sec, true);
LOG(md, Info,
@@ -503,11 +508,7 @@ CCSecSearchResult GeneralMarketDataListener::_find_sec(
if (add_if_absent) {
uint64_t bts_id = cc_id.value();
if (!me->use_given_sec_key) {
auto logical_key = get_logical_key(sec);
auto new_key_opt =
me->id_gen.get_id(me->cc_name_prefix, logical_key, true);
bts_assert(new_key_opt.has_value());
bts_id = new_key_opt.value();
bts_id = me->id_gen.get_or_add(me->venue, sec.type(), sec.desc());
}
me->cc_sec_id_map.insert({cc_id, {bts_id}});
LOG(md, Info,
@@ -654,9 +655,7 @@ uint64_t GeneralMarketDataListener::_generate_security_key(
auto logical_key = get_logical_key(sec);
// if (me->use_given_sec_key)
// return sec.security_common.id.value();
auto rv = me->id_gen.get_id(me->cc_name_prefix, logical_key, true);
bts_assert(rv.has_value());
return rv.value();
return me->id_gen.get_or_add(me->venue, sec.type(), sec.desc());
}
void GeneralMarketDataListener::_possibly_match_with_sim_orders(
-22
View File
@@ -61,26 +61,6 @@ datetime::Date sec_expiration_cut_off() {
return datetime::Date(pt.date().year(), pt.date().month(), pt.date().day());
}
void apply_compatibilit_mutations(Security &security) {
// Migrate from old tick stuff
if (security.has_glbx()) {
if (security.has_glbx()) {
if (security.contract_spec().main_fraction() == 0) {
security.mutable_contract_spec()->set_main_fraction(
security.glbx().do_not_use_main_fraction());
}
if (security.contract_spec().sub_fraction() == 0) {
security.mutable_contract_spec()->set_sub_fraction(
security.glbx().do_not_use_sub_fraction());
}
if (security.contract_spec().price_display_format() == 0) {
security.mutable_contract_spec()->set_price_display_format(
security.glbx().do_not_use_price_display_format());
}
}
}
}
size_t load_securities(
std::function<bool(const UserSymbol &symbo)> const &sym_filter,
const std::function<void(const api::Security &)> &callback) {
@@ -104,7 +84,6 @@ size_t load_securities(
if (sym_filter(us)) {
auto blob = select_stmt.getColumn("security_blob");
if (security.ParseFromArray(blob, blob.getBytes())) {
apply_compatibilit_mutations(security);
bts_assert(security_key == security.id());
count++;
callback(security);
@@ -146,7 +125,6 @@ SecurityPtr find_security(SecurityKey key) {
auto result = std::make_shared<Security>();
auto blob = select.getColumn(0);
if (result->ParseFromArray(blob, blob.getBytes())) {
apply_compatibilit_mutations(*result);
return result;
}
}
+148 -161
View File
@@ -1,18 +1,25 @@
#include "lib/domain_model/Security.h"
#ifndef _WIN32
#include "SecurityIDRecorder.h"
#include "lib/domain_model/Security.h"
#include "lib/persistence/IStore.h"
#include "lib/persistence/SqliteStore.h"
#include <boost/pool/object_pool.hpp>
#include "lib/config/config.h"
#include "lib/logging/logging.h"
#include "lib/utils/Sqlite_utils.h"
#include "lib/utils/checked_arithmetic.h"
#include <boost/filesystem.hpp>
#include <tbb/concurrent_queue.h>
#include "lib/config/protos/market_data.pb.h"
#include "lib/config/protos/system_config.pb.h"
#include <memory>
#include <sqlite/sqlite3.h>
#include <stdexcept>
#include <string>
#include <tbb/concurrent_queue.h>
using namespace std;
using namespace bts::utils;
@@ -21,22 +28,55 @@ using std::size_t;
namespace bts {
namespace md {
using bts::api::Security;
using bts::api::SecurityType;
using bts::api::Venue;
using bts::config::SystemConfig;
using bts::persistence::IStore;
using bts::persistence::SqliteStore;
uint64_t exchange_id(const api::Security &security);
struct SecurityIDEntry {
api::Security security;
SecurityIDRecorder::Callback callback;
static std::unique_ptr<IStore> get_default_store() {
return std::make_unique<SqliteStore>("key_ids");
}
static persistence::StoreRow get_row(
Venue venue,
SecurityType type,
const std::string &name,
const std::string &prefix = "") {
return persistence::StoreRow{
fmt::format(
"{}{}.{}", std::string{prefix}, Venue_Name(venue).substr(6),
SecurityType_Name(type).substr(13)),
name, ""};
}
static persistence::StoreRow get_row(
const api::Security &s, const std::string &prefix = "") {
return get_row(s.exchange(), bts::securities::security_type(s), prefix);
}
struct SecurityIdRequest {
Venue venue;
SecurityType type;
std::string name;
std::function<void(Venue, SecurityType, const std::string &, std::uint32_t)>
callback;
};
struct SecurityIDRecorderState {
SecurityIDRecorderState(SecurityIDRecorderParams &p)
: work(), pending_handle(), thread_id(), db(nullptr, sqlite3_close),
timer(), lookup_key(p.lookup_key), unresolved_legs() {}
SecurityIDRecorderState(
std::unique_ptr<IStore> store, KeyLookupOption lookup_key)
: store(store == nullptr ? get_default_store() : std::move(store)),
pending_handle(), thread_id(), timer(), lookup_key(lookup_key),
unresolved_legs() {}
tbb::concurrent_queue<std::shared_ptr<SecurityIDEntry>> work;
std::unique_ptr<IStore> store{nullptr};
tbb::concurrent_queue<std::shared_ptr<SecurityIdRequest>> id_request_queue;
tbb::concurrent_queue<MDProducerHandle> pending_handle;
std::thread::id thread_id;
sqlite3_ptr db;
unique_ptr<DispatcherTimer> timer;
KeyLookupOption lookup_key;
@@ -59,51 +99,41 @@ struct SecurityIDRecorderState {
void work_handles();
void work_db();
void check_access() { bts_assert(thread_id == std::this_thread::get_id()); }
void assign_key(
const api::Security &security,
const SecurityIDRecorder::Callback &key_assigned);
void assign_key(
api::Venue venue,
api::SecurityType type,
const std::string &name,
std::function<void(
api::Venue, api::SecurityType, const std::string &, std::uint32_t)>
key_assigned);
void check_access() { bts_assert(thread_id == std::this_thread::get_id()); }
void add_unresolved_spread(MDProducerHandle &handle);
security_key search(const std::string &descriptor);
security_key insert(const std::string &descriptor, SecurityType);
security_key search(const persistence::StoreRow &row);
security_key insert(const persistence::StoreRow &row);
};
SecurityIDRecorder::SecurityIDRecorder(SecurityIDRecorderParams params)
: _self(new SecurityIDRecorderState(params)) {
if (params.db) {
_self->db = std::move(params.db);
}
SecurityIDRecorder::SecurityIDRecorder(
std::unique_ptr<persistence::IStore> store,
KeyLookupOption lookup_key,
utils::Dispatcher *dispatcher)
: _self(std::make_unique<SecurityIDRecorderState>(
store == nullptr
? std::make_unique<persistence::SqliteStore>("key_ids")
: std::move(store),
lookup_key)) {
init();
if (params.dispatcher) {
_self->run(*params.dispatcher);
if (dispatcher != nullptr) {
_self->run(*dispatcher);
}
}
SecurityIDRecorder::~SecurityIDRecorder() {}
void SecurityIDRecorder::init() {
LOG(md, Info, "SecurityIDRecorder::init");
string db_dir =
config::get<config::SystemConfig>().data_path() + "databases/md/";
string db_path = db_dir + "/security_keys.sqlite";
if (!boost::filesystem::exists(db_dir)) {
boost::filesystem::create_directories(db_dir);
}
if (!_self->db) {
_self->db = sqlite::open_db(db_path.c_str());
bts_assert(_self->db);
}
// Set a long busy timeout
sqlite3_busy_timeout(_self->db.get(), 9999);
sqlite::exec_nofail(_self->db, "PRAGMA journal_mode=WAL");
sqlite::exec_nofail(_self->db, "PRAGMA syncronous=0");
sqlite::exec_nofail(
_self->db, "CREATE TABLE IF NOT EXISTS security_keys (descriptor STRING "
"NOT NULL PRIMARY KEY, type int);");
}
void SecurityIDRecorder::init() {}
void SecurityIDRecorder::ensure_key(MDProducerHandle handle) {
LOG(md, Info, "SecurityIDRecorder::ensure_key desc: {} key: {} num_id: {}",
@@ -113,37 +143,42 @@ void SecurityIDRecorder::ensure_key(MDProducerHandle handle) {
}
void SecurityIDRecorder::assign_key(
const api::Security &security,
const SecurityIDRecorder::Callback &callback) {
LOG(md, Info, "SecurityIDRecorder::assign_key sec: {}",
security.DebugString());
if (security.Contract_case() == Security::ContractCase::kSpread) {
for (const auto &leg : security.spread().legs()) {
if (leg.security().id() == invalid_security_key) {
LOG(md, Error, "Security {} has legs which will never be resolved.",
security.has_glbx() ? security.glbx().security_desc()
: security.symbol());
// No point in trying to add this to the queue
return;
}
}
}
_self->assign_key(security, callback);
const api::Security &security, const Callback &callback) {
assign_key(
security.exchange(), bts::securities::security_type(security),
bts::securities::descriptor(security),
[callback, msec = security](
api::Venue, api::SecurityType, const std::string &,
uint32_t id) mutable {
msec.set_id(id);
callback(msec);
});
}
void SecurityIDRecorderState::assign_key(
const api::Security &security,
const SecurityIDRecorder::Callback &callback) {
LOG(md, Info, "SecurityIDRecorder::assign_key sec: {}",
bts::securities::descriptor(security));
if (security.id() == invalid_security_key) {
auto work = std::make_shared<SecurityIDEntry>();
work->security = security;
work->callback = callback;
LOG(md, Info, "SecurityIDRecorderState::assign_key work.push d: {}",
bts::securities::descriptor(work->security));
this->work.push(work);
}
api::Venue venue,
api::SecurityType type,
const std::string &name,
std::function<
void(api::Venue, api::SecurityType, const std::string &, std::uint32_t)>
callback) {
LOG(md, Info, "SecurityIDRecorder::assign_key {} {} {}", venue, type, name);
auto id_req = std::make_shared<SecurityIdRequest>();
id_req->venue = venue;
id_req->type = type;
id_req->name = name;
id_req->callback = callback;
id_request_queue.push(id_req);
}
void SecurityIDRecorder::assign_key(
api::Venue venue,
api::SecurityType type,
const std::string &name,
std::function<
void(api::Venue, api::SecurityType, const std::string &, std::uint32_t)>
callback) {
_self->assign_key(venue, type, name, std::move(callback));
}
void SecurityIDRecorder::do_work() { _self->do_work(); }
@@ -220,9 +255,12 @@ void SecurityIDRecorderState::work_handles() {
retry.push_back(handle);
} else {
convert_security(handle.security(), &security);
assign_key(security, [handle](const api::Security &security) mutable {
handle.set_key(security.id());
});
assign_key(
security.exchange(), bts::securities::security_type(security),
bts::securities::descriptor(security),
[handle](
api::Venue, api::SecurityType, const std::string &,
uint32_t id) mutable { handle.set_key(id); });
}
}
@@ -239,33 +277,30 @@ void SecurityIDRecorderState::work_db() {
// Then persist that list
// Once a security has a key it is public so record it in the security
// database
std::shared_ptr<SecurityIDEntry> temp;
vector<std::shared_ptr<SecurityIDEntry>> new_entries;
std::shared_ptr<SecurityIdRequest> temp;
vector<std::pair<uint32_t, std::shared_ptr<SecurityIdRequest>>> new_entries;
// SPARK-1446 Need to try to add existing securities to the db in case we
// deleted them by hand
vector<std::shared_ptr<SecurityIDEntry>> existing_securities;
vector<std::shared_ptr<SecurityIDEntry>> defer;
vector<string> new_descriptors;
vector<std::shared_ptr<SecurityIdRequest>> existing_securities;
vector<persistence::StoreRow> new_rows;
size_t count = 0;
string descriptor;
while (work.try_pop(temp)) {
descriptor = security_descriptor(temp->security);
// If we're unable to create a descriptor try again later
if (descriptor == "") {
defer.push_back(std::move(temp));
continue;
while (id_request_queue.try_pop(temp)) {
security_key key = invalid_security_key;
auto row = get_row(temp->venue, temp->type, temp->name);
try {
key = search(row);
} catch (const std::overflow_error &ex) {
LOG(persist, Error,
"SecurityIDRecorderState::work_db. Row search threw ex: {}",
std::string{ex.what()});
}
security_key key = search(descriptor);
if (key != invalid_security_key) {
temp->security.set_id(key);
temp->callback(temp->security);
temp->callback(temp->venue, temp->type, temp->name, key);
existing_securities.push_back(std::move(temp));
} else {
// always push in parallel
new_entries.push_back(std::move(temp));
new_descriptors.push_back(descriptor);
new_entries.push_back(std::make_pair(key, std::move(temp)));
new_rows.push_back(std::move(row));
++count;
// Limit the amount of time we spend writing to sqlite during startup
// to avoid blocking the heartbeat
@@ -275,95 +310,47 @@ void SecurityIDRecorderState::work_db() {
}
}
for (auto &d : defer) {
LOG(md, Info, "SecurityIDRecorderState::work_db work.push d: {}",
bts::securities::descriptor(d->security));
work.push(std::move(d));
}
if (!new_entries.empty()) {
sqlite::exec_nofail(db.get(), "BEGIN TRANSACTION");
store->insert_many_resolve_ids(new_rows);
for (size_t x = 0; x < new_entries.size(); ++x) {
Security &security = new_entries[x]->security;
security_key key =
insert(new_descriptors[x], securitytype_of(security));
bts_assert_d(key);
security.set_id(key);
auto &entry = new_entries[x];
auto &row = new_rows[x];
entry.first = row.id;
bts_assert_d(entry.first != invalid_security_key);
}
sqlite::exec_nofail(db.get(), "COMMIT TRANSACTION");
}
if (!new_entries.empty()) {
for (size_t x = 0; x < new_entries.size(); ++x) {
Security &security = new_entries[x]->security;
new_entries[x]->callback(security);
auto &[id, req] = new_entries[x];
req->callback(req->venue, req->type, req->name, id);
}
}
}
void SecurityIDRecorder::bulk_insert(
const std::vector<const api::Security *> &data, const char *prefix) {
LOG(md, Info, "SecurityIDRecorder::bulk_insert size: {}", data.size());
if (data.size() == 0)
return;
sqlite::exec_nofail(_self->db.get(), "BEGIN TRANSACTION");
std::string descriptor;
{
auto insert_stmt = sqlite::prepare_query(
_self->db, "INSERT INTO security_keys (rowid, descriptor, type) "
"values (?, ?, ?)");
for (const api::Security *sec : data) {
descriptor = prefix + security_descriptor(*sec);
auto type = securitytype_of(*sec);
sqlite3_bind_int(insert_stmt.get(), 1, sec->id());
sqlite3_bind_text(
insert_stmt.get(), 2, descriptor.c_str(), descriptor.size(),
SQLITE_STATIC);
sqlite3_bind_int(insert_stmt.get(), 3, (int)type);
sqlite3_step(insert_stmt.get());
sqlite3_reset(insert_stmt.get());
}
std::vector<persistence::StoreRow> rows;
rows.reserve(data.size());
for (const auto *s : data) {
rows.push_back(get_row(*s, prefix));
}
sqlite::exec_nofail(_self->db.get(), "COMMIT TRANSACTION");
_self->store->insert_many_resolve_ids(rows);
}
security_key SecurityIDRecorderState::search(const std::string &descriptor) {
LOG(md, Info, "SecurityIDRecorderState::search descriptor: {}", descriptor);
auto select_stmt = sqlite::prepare_query(
db, "SELECT rowid from security_keys where descriptor=? LIMIT 1;");
sqlite3_bind_text(
select_stmt.get(), 1, descriptor.c_str(), descriptor.size(),
SQLITE_STATIC);
int result = sqlite3_step(select_stmt.get());
if (result == SQLITE_DONE || result == SQLITE_ROW) {
return sqlite3_column_int(select_stmt.get(), 0);
security_key SecurityIDRecorderState::search(const persistence::StoreRow &row) {
auto [exists, rowid, val] = store->retrieve_one(row.ns, row.key);
if (exists) {
if (!val.empty()) {
bts_assert(stol(val) == rowid);
}
return system::math::checked_ltou(rowid);
}
return invalid_security_key;
}
security_key SecurityIDRecorderState::insert(
const std::string &descriptor, SecurityType type) {
LOG(md, Info, "SecurityIDRecorderState::insert descriptor: {}", descriptor);
auto insert_stmt = sqlite::prepare_query(
db, "INSERT INTO security_keys (descriptor, type) values (?, ?)");
sqlite3_bind_text(
insert_stmt.get(), 1, descriptor.c_str(), descriptor.size(),
SQLITE_STATIC);
sqlite3_bind_int(insert_stmt.get(), 2, (int)type);
int result = sqlite3_step(insert_stmt.get());
if (result == SQLITE_DONE || result == SQLITE_ROW) {
return sqlite3_last_insert_rowid(db.get());
}
// Just in case our insert failed due to duplicate row
auto key = search(descriptor);
if (key == invalid_security_key) {
LOG(md, Error, "Failed to assign security key. SQLite return code: {}",
result);
}
return key;
security_key SecurityIDRecorderState::insert(const persistence::StoreRow &row) {
return store->insert(row.ns, row.key, row.value);
}
} // namespace md
+13 -17
View File
@@ -2,7 +2,7 @@
#ifndef _WIN32
#include "MDProducerHandle.h"
#include "common.h"
#include "lib/persistence/IStore.h"
#include "lib/utils/Sqlite_utils.h"
#include <atomic>
#include <functional>
@@ -22,26 +22,15 @@ namespace md {
typedef const std::function<SecurityKey(ExchangeKey)> KeyLookup;
typedef std::optional<KeyLookup> KeyLookupOption;
struct SecurityIDRecorderParams {
SecurityIDRecorderParams()
: dispatcher(), db(sqlite::null_db()), lookup_key() {}
SecurityIDRecorderParams(utils::Dispatcher &d)
: dispatcher(&d), db(sqlite::null_db()), lookup_key() {}
SecurityIDRecorderParams(utils::Dispatcher &d, KeyLookup &get_key)
: dispatcher(&d), db(sqlite::null_db()), lookup_key(get_key) {}
SecurityIDRecorderParams(sqlite3_ptr &db)
: dispatcher(), db(std::move(db)), lookup_key() {}
SecurityIDRecorderParams(sqlite3_ptr &db, KeyLookup &get_key)
: dispatcher(), db(std::move(db)), lookup_key(get_key) {}
utils::Dispatcher *dispatcher;
sqlite3_ptr db;
KeyLookupOption lookup_key;
};
struct SecurityIDRecorderState;
class SecurityIDRecorder {
public:
SecurityIDRecorder(SecurityIDRecorderParams);
SecurityIDRecorder(
std::unique_ptr<persistence::IStore> store = nullptr,
KeyLookupOption lookup_key = std::nullopt,
utils::Dispatcher *dispatcher = nullptr);
~SecurityIDRecorder();
typedef std::function<void(const api::Security &)> Callback;
@@ -49,6 +38,13 @@ class SecurityIDRecorder {
void ensure_key(MDProducerHandle handle);
// Slow (copies security) - for use by services
void assign_key(const api::Security &security, const Callback &key_assigned);
void assign_key(
api::Venue venue,
api::SecurityType type,
const std::string &name,
std::function<void(
api::Venue, api::SecurityType, const std::string &, std::uint32_t)>
key_assigned);
// If a dispatcher is not provided you must call work periodically to do
// lookups and flush the reserved ids to disk.
+8 -97
View File
@@ -1,4 +1,4 @@
#include "lib/domain_model/Security.pb.h"
#include "lib/domain_model/Security.h"
#include "lib/utils/bts_assert.h"
#ifndef NO_CPP0X
@@ -59,81 +59,6 @@ FixedSpreadLeg::FixedSpreadLeg()
: key(invalid_security_key), leg_side(Side_MIN), leg_quantity(0),
leg_price(0), leg_delta(1) {}
void write_date_string(
uint32_t year, uint32_t month, uint32_t day, std::ostringstream &result) {
result << std::setw(2) << std::setfill('0') << month << "/" << std::setw(2)
<< std::setfill('0') << day << "/" << year - 2000;
}
void write_leg_descriptor(
std::ostringstream &result, char separator, const api::ContractLeg &leg) {
result << separator << leg.security().id() << "-q" << leg.quantity() << "p"
<< leg.price_multiplier() << "d" << leg.leg_delta()
<< (leg.side() == SIDE_BUY ? "BUY" : "SELL");
}
std::string security_descriptor(const api::Security &security) {
const char separator = ' ';
std::ostringstream result;
if (security.exchange() == Venue::VENUE_ICE ||
security.exchange() == Venue::VENUE_SIMICE)
result << security.ice().security_desc() << separator
<< Venue_Name(security.exchange());
else
result << security.symbol() << separator
<< Venue_Name(security.exchange());
switch (security.Contract_case()) {
case Security::ContractCase::kOption: {
result << separator;
const auto &o = security.option();
write_date_string(
o.expiration_year(), o.expiration_month(), o.expiration_day(),
result);
result << separator << o.strike()
<< (o.call_put() == PUTORCALL_CALL ? 'C' : 'P');
break;
}
case Security::ContractCase::kFuture: {
result << separator;
const auto &f = security.future();
write_date_string(
f.expiration_year(), f.expiration_month(), f.expiration_day(),
result);
break;
}
case Security::ContractCase::kSpot: {
break;
}
case Security::ContractCase::kSpread: {
// If the security key of a leg is invalid return an error
const auto &s = security.spread();
bts_assert(s.legs_size() > 0);
for (const auto &leg : s.legs()) {
if (leg.security().id() == invalid_security_key)
return "";
write_leg_descriptor(result, separator, leg);
}
break;
}
case Security::ContractCase::kSynthetic: {
const auto &s = security.synthetic();
bts_assert(s.legs_size() > 0);
result << separator << s.name()
<< " pc=" << SyntheticContract::PriceCalc_Name(s.price_calc());
for (const auto &leg : s.legs()) {
if (leg.security().id() == invalid_security_key)
return ""; // unlikey with synthetics
write_leg_descriptor(result, separator, leg);
}
break;
}
case Security::ContractCase::CONTRACT_NOT_SET:
bts_assert(
security.Contract_case() != Security::ContractCase::CONTRACT_NOT_SET);
}
return result.str();
}
void get_contract_spec(
const FixedSecurity &source, /*out*/ api::ContractSpecification &spec) {
spec.set_price_break(source.price_break);
@@ -292,26 +217,12 @@ void convert_security(const FixedSecurity &source, Security *presult) {
get_contract_spec(source, *dest.mutable_contract_spec());
}
SecurityType securitytype_of_contractcase(Security::ContractCase cc) {
switch (cc) {
case Security::ContractCase::kFuture:
return SECURITYTYPE_FUTURE;
case Security::ContractCase::kOption:
return SECURITYTYPE_OPTION;
case Security::ContractCase::kSpot:
return api::SECURITYTYPE_SPOT;
case Security::ContractCase::kSpread:
return SECURITYTYPE_SPREAD;
case Security::ContractCase::kSynthetic:
return SECURITYTYPE_SYNTHETIC;
case Security::ContractCase::CONTRACT_NOT_SET:
bts_assert(cc != Security::ContractCase::CONTRACT_NOT_SET);
}
bts_assert(false);
}
SecurityType securitytype_of(const Security &s) {
return securitytype_of_contractcase(s.Contract_case());
auto type = bts::securities::security_type(s.Contract_case());
bts_assert(type != api::SecurityType::SECURITYTYPE_UNSET);
return type;
}
size_t num_legs(const Security &sec) {
if (sec.Contract_case() == api::Security::ContractCase::kSpread) {
return sec.spread().legs_size();
@@ -359,7 +270,7 @@ uint64_t exchange_id(const Security &security) {
void convert_security(
const Security &src, FixedSecurity &dest, bool set_security_key) {
dest.key = set_security_key ? src.id() : invalid_security_key;
dest.type = securitytype_of_contractcase(src.Contract_case());
dest.type = bts::securities::security_type(src.Contract_case());
dest.symbol(src.symbol());
dest.currency(src.currency());
dest.exchange = src.exchange();
@@ -374,8 +285,8 @@ void convert_security(
dest.cme_market_segment_id = src.glbx().market_segment_id();
break;
case Security::ExchangeDataCase::kCboe:
dest.numeric_id = src.deribit().exchange_id();
dest.security_desc(src.deribit().security_desc());
dest.numeric_id = src.cboe().exchange_id();
dest.security_desc(src.cboe().security_desc());
dest.cboe_class_key = src.cboe().class_key();
break;
case Security::ExchangeDataCase::kTaifex:
+3 -6
View File
@@ -208,8 +208,6 @@ typedef FixedSpreadSecurity<16> FixedSpreadSecurityS;
typedef FixedSpreadSecurity<128> FixedSpreadSecurityL;
typedef FixedSpreadSecurity<512> FixedSpreadSecurityXL;
std::string security_descriptor(const api::Security &);
// Convert a fixedsecurity to a normal Security - caveat: spreads will be
// normalized
Security convert_security(const FixedSecurity &);
@@ -217,10 +215,9 @@ void convert_security(const FixedSecurity &, Security *);
void convert_security(
const Security &src, FixedSecurity &dest, bool set_security_key = false);
SecurityType securitytype_of_contractcase(Security::ContractCase cc);
SecurityType securitytype_of(const Security &);
size_t num_legs(const Security &);
uint64_t exchange_id(const Security &security);
SecurityType securitytype_of(const api::Security &);
size_t num_legs(const api::Security &);
uint64_t exchange_id(const api::Security &security);
void to_json(nlohmann::json &j, const FixedSpreadLeg &f);
+3 -2
View File
@@ -35,8 +35,9 @@ typedef SyntheticSecurityServiceBase::MetadataStatefulPublisher
struct SyntheticSecurityService::Pimpl {
Pimpl(const MetadataStatefulPublisher &fn)
: recorder(utils::Dispatcher::background()), synthetics(), on_security(),
meta(), on_metadata(), metadata_publisher(fn) {}
: recorder(nullptr, std::nullopt, &utils::Dispatcher::background()),
synthetics(), on_security(), meta(), on_metadata(),
metadata_publisher(fn) {}
SecurityIDRecorder recorder;
std::unordered_map<SecurityKey, Security> synthetics;
utils::Event<Security> on_security;
+32
View File
@@ -0,0 +1,32 @@
#pragma once
#include "lib/os/spark_data_directory.h"
#include <boost/filesystem/operations.hpp>
#include <boost/filesystem/path.hpp>
namespace bts::md {
struct Mdp3DataDirectory {
boost::filesystem::path log_dir() const { return root.log_dir() / "onix"; }
boost::filesystem::path data_dir() const { return root.root() / "cme_mdp3"; }
boost::filesystem::path config_file() const {
return data_dir() / "config.xml";
}
boost::filesystem::path templates_file() const {
return data_dir() / "templates.xml";
}
void create_dirs() {
boost::filesystem::create_directories(data_dir());
boost::filesystem::create_directories(log_dir());
}
private:
spark::SparkDataDirectory root{};
};
} // namespace bts::md
+14 -15
View File
@@ -9,6 +9,7 @@
#include "lib/config/config.h"
#include "lib/metrics/metrics.h"
#include "lib/os/os.h"
#include "lib/persistence/SqliteStore.h"
#include "lib/utils/Dispatcher.h"
#include "lib/utils/Shuffle.h"
#include "lib/utils/bts_chrono.h"
@@ -160,12 +161,10 @@ class Co2Tests {
#ifndef _WIN32
void test_security_id_recorder(const std::string &db_name) {
// Verify callbacks happen after work complete
auto db1 = sqlite::open_db(db_name.c_str());
REQUIRE(db1);
auto db2 = sqlite::open_db(db_name.c_str());
REQUIRE(db2);
SecurityIDRecorder idsvc{{db1}}, idsvc2{{db2}};
SecurityIDRecorder idsvc{
std::make_unique<persistence::SqliteStore>(db_name)};
SecurityIDRecorder idsvc2{
std::make_unique<persistence::SqliteStore>(db_name)};
security_key result = invalid_security_key;
Security s1 = mkoption("FOOX", invalid_security_key, 2018, 1, 20, 1850);
Security f1 = mkfuture("FOO", invalid_security_key, 2018, 1, 20);
@@ -215,18 +214,18 @@ class Co2Tests {
std::make_unique<Co2TestInstrument>(s1, cold_header, hot_header);
// Add spread to SecurityIDRecorder and work it
auto db = sqlite::open_db(":memory:");
auto db = std::make_unique<persistence::SqliteStore>(":memory:");
REQUIRE(db);
SecurityIDRecorder idsvc{
{db, [&](ExchangeKey k) -> SecurityKey {
for (const auto &fut :
{f1handle->producer(), f2handle->producer()}) {
if (k.value() == fut.security().numeric_id)
return SecurityKey(fut.key());
}
return SecurityKey(invalid_security_key);
}}};
std::move(db), [&](ExchangeKey k) -> SecurityKey {
for (const auto &fut :
{f1handle->producer(), f2handle->producer()}) {
if (k.value() == fut.security().numeric_id)
return SecurityKey(fut.key());
}
return SecurityKey(invalid_security_key);
}};
auto workwork = [&]() {
idsvc.do_work();
@@ -4,9 +4,12 @@
#include "lib/orderman/gateways/CCAPIExecutionGateway.h"
// #include "lib/orderman/gateways/EMExecutionGateway.h"
#include "lib/orderman/gateways/FakeExecutionGateway.h"
#include "lib/orderman/gateways/GeneralExecutionGateway.h"
#include "lib/orderman/gateways/GLBXExecutionGateway.h"
#include "lib/orderman/gateways/GLBXiLink3ExecutionGateway.h"
#include "lib/orderman/gateways/ICEExecutionGateway.h"
#include "lib/orderman/gateways/IFIXGenericAdapter.h"
#include "lib/orderman/gateways/OnixFIXAdapter.h"
#include "lib/orderman/gateways/sim/SimExecutionGatewayFactory.h"
#include "lib/orderman/gateways/sim/SimExecutionGatewayUniqueWrapper.h"
#include "lib/tracing/PerfLog.h"
@@ -21,23 +24,30 @@ std::unique_ptr<IExecutionGateway> ExecutionGatewayFactory::create_gateway(
const config::JetFIX::Connection &conn,
trace::PerfLog &pl,
md::SecurityOracle o) {
LOG(OrderMan, Info, "ExecutionGatewayFactory::create_gateway conn: {}",
utils::pb::to_string(conn));
if (SimExecutionGatewayFactory::is_sim_execution_on(conn.venue())) {
return std::make_unique<SimExecutionGatewayUniqueWrapper>(conn.venue());
}
LOG(OrderMan, Info, "ExecutionGatewayFactory::create_gateway conn: {}",
utils::pb::to_string(conn));
if (conn.venue() == bts::api::VENUE_ICE ||
conn.venue() == bts::api::VENUE_SIMICE)
return std::make_unique<ICEExecutionGateway>(conn, pl, o);
else if (conn.venue() == bts::api::VENUE_GLBX) {
if (conn.use_ilink3())
return std::make_unique<ilink3::GLBXiLink3ExecutionGateway>(
conn, pl, o);
return std::make_unique<GLBXExecutionGateway>(conn, pl, o);
} else if (conn.venue() == bts::api::VENUE_DERIBIT)
// return std::make_unique<EMExecutionGateway>(o);
return std::make_unique<CCAPIExecutionGateway>(conn.venue(), o);
return std::make_unique<FakeExecutionGateway>(o);
switch(conn.venue()){
case bts::api::VENUE_ICE:
case bts::api::VENUE_SIMICE:
return std::make_unique<ICEExecutionGateway>(conn, pl, o);
case bts::api::VENUE_GLBX:{
if (conn.use_ilink3())
return std::make_unique<ilink3::GLBXiLink3ExecutionGateway>( conn, pl, o);
return std::make_unique<GLBXExecutionGateway>(conn, pl, o);
}
case bts::api::VENUE_DERIBIT:
return std::make_unique<CCAPIExecutionGateway>(conn.venue(), o);
case bts::api::VENUE_CBOE:
{
std::unique_ptr<IFIXGenericAdapter> adapter = std::make_unique<OnixFIXAdapter>(o);
return std::make_unique<GeneralExecutionGateway>(conn.venue(),std::move(adapter),o);
}
default:
return std::make_unique<FakeExecutionGateway>(o);
}
}
} // namespace bts::orderman
@@ -0,0 +1,86 @@
#include "GeneralExecutionGateway.h"
#include "lib/domain_model/Security.pb.h"
#include "lib/logging/logging.h"
#include "lib/orderman/CreateOrderMsg.h"
#include "lib/orderman/OrderMsg.h"
#include "lib/orderman/gateways/IExecutionGateway.h"
#include <memory>
#include <type_traits>
//Acts as a gateway, On One side is the ordermanager and on the other onixfix listener and onixfix adapter
namespace bts::orderman {
struct GeneralExecutionGateway::PImpl {
PImpl(bts::api::Venue venue,
std::unique_ptr<IFIXGenericAdapter> adapter,
md::SecurityOracle o)
: venue(venue), _adapter(std::move(adapter)), sec_oracle(o) {}
bts::api::Venue venue;
std::unique_ptr<IFIXGenericAdapter> _adapter;
md::SecurityOracle sec_oracle;
OnExeReport on_exe_report;
OnGwStatusChange on_gw_status_change;
OnCancelReject on_cancel_reject;
OnExchangePositionState on_exchange_position_state;
OnExchangeOpenOrdersState on_exchange_open_orders;
};
OnGwStatusChange &GeneralExecutionGateway::on_gw_status_change() {
return me->on_gw_status_change;
}
// Modified constructor to accept the adapter from outside
GeneralExecutionGateway::GeneralExecutionGateway(bts::api::Venue venue,
std::unique_ptr<IFIXGenericAdapter> adapter, md::SecurityOracle o)
: IExecutionGateway(), me(new PImpl(venue, std::move(adapter),o)) {
LOG(OrderMan, Info, "ExecutionGateway::ExecutionGateway constructor");
}
GeneralExecutionGateway::~GeneralExecutionGateway() {}
void GeneralExecutionGateway::start() {
me->on_gw_status_change(true);
me->_adapter->start();
me->_adapter->register_callback(me->on_exe_report);
LOG(OrderMan,Info,"ExecutionGateway::start started" );
}
void GeneralExecutionGateway::send_msgs(
std::vector<OrderMsg> const &msgs, trace::CorID) {
LOG(OrderMan,Info,"ExecutionGateway::send_msgs" );
for (const auto &msg : msgs) {
std::visit(
[this](auto &&order) {
using T = std::decay_t<decltype(order)>;
if constexpr (std::is_same_v<T, CreateOrderMsg>) {
LOG(OrderMan,Warn,"ExecutionGateway::CreateOrderMsg" );
me->_adapter->launch_new_order(order);
} else if constexpr (std::is_same_v<T, CancelOrderMsg>) {
LOG(OrderMan,Warn,"ExecutionGateway::CancelOrderMsg" );
me->_adapter->cancel_order(order);
} else if constexpr (std::is_same_v<T, CancelReplaceOrderMsg>) {
LOG(OrderMan,Warn,"ExecutionGateway::CancelReplaceOrderMsg" );
me->_adapter->modify_order(order);
} else if constexpr (std::is_same_v<T, RFQMsg>) {
// Handle RFQMsg if necessary
// adapter->handle_rfq(order); // Example placeholder
}
},
msg);
}
}
void GeneralExecutionGateway::dispatch_incoming_fix_messages() {}
void GeneralExecutionGateway::on_securities(
const mercury::Update<NewSecuritiesEvent> &) {}
void GeneralExecutionGateway::on_admin_data(
const mercury::Update<NewAdminDataEvent> &) {}
void GeneralExecutionGateway::send_fix(const jetfix::OutFIXPtr &, trace::CorID) {}
OnExeReport &GeneralExecutionGateway::on_exe_report() { return me->on_exe_report; }
OnCancelReject &GeneralExecutionGateway::on_cancel_reject() {
static OnCancelReject default_cancel_reject;
return default_cancel_reject;
}
OnExchangePositionState &GeneralExecutionGateway::on_exchange_position_state() {
static OnExchangePositionState default_exchange_position_state;
return default_exchange_position_state;
}
OnExchangeOpenOrdersState &GeneralExecutionGateway::on_exchange_open_orders() {
static OnExchangeOpenOrdersState default_exchange_open_orders;
return default_exchange_open_orders;
}
} //namespace bts::orderman
@@ -0,0 +1,29 @@
#pragma once
#include "lib/orderman/gateways/IExecutionGateway.h"
#include "lib/orderman/gateways/IFIXGenericAdapter.h"
namespace bts {
namespace orderman {
class GeneralExecutionGateway : public IExecutionGateway {
public:
GeneralExecutionGateway(bts::api::Venue venue,std::unique_ptr<IFIXGenericAdapter> adapter,
md::SecurityOracle o);
virtual ~GeneralExecutionGateway();
void start() override;
void send_msgs(std::vector<OrderMsg> const & msgs, trace::CorID) override;
void dispatch_incoming_fix_messages() override;
void on_securities(const mercury::Update<NewSecuritiesEvent> &) override;
void on_admin_data(const mercury::Update<NewAdminDataEvent> &) override;
void send_fix(const jetfix::OutFIXPtr &, trace::CorID) override;
OnExeReport &on_exe_report() override;
OnGwStatusChange &on_gw_status_change() override;
OnCancelReject &on_cancel_reject() override;
OnExchangePositionState &on_exchange_position_state() override;
OnExchangeOpenOrdersState &on_exchange_open_orders() override;
private:
struct PImpl;
std::unique_ptr<PImpl> me;
};
} //namespace bts
} //namespace orderman
@@ -1,3 +1,6 @@
#include "IFIXGenericAdapter.h"
namespace bts::orderman {}
namespace bts::orderman {
IFIXGenericAdapter::~IFIXGenericAdapter(){}
}
+18 -32
View File
@@ -1,22 +1,13 @@
#pragma once
#include "lib/api/services/OrderService.pb.h"
#include "lib/config/protos/jetfix.pb.h"
#include "lib/domain_model/Security.pb.h"
#include "lib/jetfix/codec/MessageDefinitions.h"
#include "lib/jetfix/common/OutFIXPtr.h"
#include "lib/md/SecurityOracle.h"
#include "lib/mercury/events/Update.h"
#include "lib/orderman/CancelOrderMsg.h"
#include "lib/orderman/CancelReplaceOrderMsg.h"
#include "lib/orderman/CreateOrderMsg.h"
#include "lib/orderman/OrderMsg.h"
#include "lib/orderman/orderman_utils/ExchangeOpenOrders.h"
#include "lib/orderman/orderman_utils/ExchangePosition.h"
#include "lib/orderman/orderman_utils/orderman_common.h"
#include "lib/utils/bts_no_copy.h"
#include "lib/tracing/PerfLog.h"
#include <functional>
#include <memory>
#include <variant>
namespace bts::orderman {
@@ -29,33 +20,28 @@ class NewAdminDataEvent;
class ExecutionReport;
class CancelReject;
struct OrderStateWithFills {
api::orderman::OrderState ordr_state;
std::vector<api::orderman::Fill> fills = {};
};
using OnOrderUpdate = std::function<void(OrderStateWithFills const &)>;
using OnExecutionReport = std::function<void(ExecutionReport const &, const trace::CorID &)>;
// fix execution gateway for a single FIX session
class IFIXGenericAdapter {
public:
IFIXGenericAdapter() = default;
virtual ~IFIXGenericAdapter() {}
virtual ~IFIXGenericAdapter();
virtual void start() = 0;
// this callback should be added in a list, and on receving an update from the other side, it should be passed here
virtual void register_callback(const OnOrderUpdate &);
virtual void launch_new_order(const CreateOrderMsg &);
virtual void modify_order(const CancelReplaceOrderMsg &);
virtual void cancel_order(const CancelOrderMsg &);
// // this callback should be added in a list, and on receving an update from the other side, it should be passed here
virtual void register_callback(const OnExecutionReport &) =0 ;
virtual void launch_new_order(const CreateOrderMsg &) =0;
virtual void modify_order(const CancelReplaceOrderMsg &) = 0;
virtual void cancel_order(const CancelOrderMsg &) =0;
// as new securities are added
virtual void update_securities(
const mercury::Update<NewSecuritiesEvent> &) = 0;
virtual void update_admin_data(
const mercury::Update<NewAdminDataEvent> &) = 0;
// virtual void update_securities(
// const mercury::Update<NewSecuritiesEvent> &) = 0;
// virtual void update_admin_data(
// const mercury::Update<NewAdminDataEvent> &) = 0;
private:
BTS_NO_COPY(IFIXGenericAdapter);
// private:
// BTS_NO_COPY(IFIXGenericAdapter);
};
} // namespace bts::orderman
@@ -0,0 +1,377 @@
#include "OnixFIXAdapter.h"
#include "OnixS/FIXEngine/FIX/Engine.h"
#include "lib/config/protos/marex_fix_cfg.pb.h"
#include "lib/domain_model/Security.pb.h"
#include "lib/logging/logging.h"
#include "lib/orderman/CancelOrderMsg.h"
#include "lib/orderman/CancelReplaceOrderMsg.h"
#include "lib/orderman/CreateOrderMsg.h"
#include "lib/orderman/gateways/OnixFIXListener.h"
#include "lib/orderman/gateways/TrafixOrderStateHandler.h"
#include "lib/orderman/orderman_utils/orderman_common.h"
#include "lib/utils/bts_assert.h"
#include "lib/utils/helper_macros.h"
#include <OnixS/FIXEngine/FIX/Exception.h>
#include <OnixS/FIXEngine/FIX/FIX42.h>
#include <OnixS/FIXEngine/FIX/Message.h>
#include <OnixS/FIXEngine/FIX/Numeric.h>
#include <OnixS/FIXEngine/FIX/ProtocolVersion.h>
#include <cassert>
#include <chrono>
#include <memory>
#include <quickfix/FixFields.h>
#include <ratio>
#include <string>
#include <unistd.h>
namespace bts::orderman {
void handleException(
const std::string &context, const std::string &error) {
bts_assert_msgf(false, "{} Exception: {}", context, error);
}
OnixFIXAdapter::OnixFIXAdapter(md::SecurityOracle sec_oracle)
: state_handler(std::make_shared<TrafixOrderStateHandler>(sec_oracle)),
listener(state_handler) {
try {
auto onix_cfg = config::get<config::MarexFixCfg>();
LOG(OrderMan, Warn, "Initializing OnixFIXAdapter");
OnixS::FIX::Engine::
init(); // OnixS::FIX::Engine::init() needs to run before initialising session
LOG(OrderMan, Diag, "Initializing OnixFIXAdapter engine initailes trying login");
session = std::make_unique<Session>(
onix_cfg.sender_sub_id(), onix_cfg.target_sub_id(),
OnixS::FIX::ProtocolVersion::FIX_42, &listener);
} catch (const OnixS::FIX::Exception &ex) {
const std::string err = ex.what();
LOG(OrderMan, Error, "Initializing OnixFIXAdapter ...{}",err);
Engine::shutdown();
handleException("OnixFIXAdapter::Construcotr init", err);
}
}
OnixFIXAdapter::~OnixFIXAdapter() {
try {
if (session)
session->logout();
Engine::instance()->unregisterListener();
Engine::shutdown();
} catch (const OnixS::FIX::Exception &ex) {
const std::string err = ex.what();
handleException("OnixFIXAdapter::Destructor ", err);
}
}
void OnixFIXAdapter::start() {
auto cfg = config::get<config::MarexFixCfg>();
const std::string counterpartyHost = cfg.counterparty_host();
const int heartBtInt = cfg.heart_bt_int();
const int counterpartyPort = cfg.counterparty_port();
const int retryInterval = cfg.retry_interval(); // Add this field in MarexFixCfg
const int maxRetries = cfg.max_retries(); // Optional: Set a limit on retries
int attempt = 0;
while (true) {
try {
LOG(OrderMan, Warn, "OnixFIXAdapter::start Attempting login (Attempt: {})", attempt + 1);
session->logonAsInitiator(counterpartyHost, counterpartyPort, heartBtInt);
LOG(OrderMan, Warn, "OnixFIXAdapter::start Login successful");
break; // Exit loop on success
} catch (const OnixS::FIX::Exception &ex) {
const std::string err = ex.what();
LOG(OrderMan, Error, "OnixFIXAdapter::start Login failed: {}. Retrying in {} seconds.", err, retryInterval);
if (maxRetries > 0 && ++attempt >= maxRetries) {
Engine::shutdown();
LOG(OrderMan, Error, "OnixFIXAdapter::start Exceeded max retries ({}). Aborting.", maxRetries);
handleException("OnixFIXAdapter::start", err);
break; // Exit if retries are exhausted
}
std::this_thread::sleep_for(std::chrono::seconds(std::max(60,retryInterval)));
}
}
}
template <typename T>
uint translateEnum(T enum_value, const std::map<int, int> &translationMap) {
auto it = translationMap.find(enum_value);
bts_assert(it != translationMap.end()); // Using assert instead of bts_assert
return it->second;
}
uint translateTimeInForce(api::orderman::TimeInForce time_in_force) {
//0 = Day 1 = GTC 2 = At the Open 3 = IOC 4 = FOK 5 = GTX
static const std::map<int, int> tifMap = {
{api::orderman::TimeInForce::TIME_IN_FORCE_DAY, 0},
{api::orderman::TimeInForce::TIME_IN_FORCE_GTC, 1},
{api::orderman::TimeInForce::TIME_IN_FORCE_IOC, 3}};
return translateEnum(time_in_force, tifMap);
}
uint translateOrderType(api::orderman::NativeOrderType order_type) {
//1 = Market 2 = Limit 3 = Stop 4 = Stop Limit 5 = Market on close A = On Close B = Limit on Close
static const std::map<int, int> orderTypeMap = {
{api::orderman::NativeOrderType::NATIVE_ORDER_TYPE_MARKET, 1},
{api::orderman::NativeOrderType::NATIVE_ORDER_TYPE_LIMIT, 2},
{api::orderman::NativeOrderType::NATIVE_ORDER_TYPE_STOP, 3},
{api::orderman::NativeOrderType::NATIVE_ORDER_TYPE_STOP_LIMIT, 4}};
return translateEnum(order_type, orderTypeMap);
}
uint translateSide(api::Side side) {
//1 = Buy 2 = Sell 5 = Sell short 6 = Sell short Exempt
static const std::map<int, int> sideMap = {
{api::Side::SIDE_BUY, 1}, {api::Side::SIDE_SELL, 2}};
return translateEnum(side, sideMap);
}
std::string pad_string(
const std::string &str, size_t length, char padding = ' ') {
if (str.size() > length) {
return str.substr(0, length);
}
return str + std::string(length - str.size(), padding);
}
// Function to create the OSI Symbol
static std::string create_osi_symbol(
const std::string &symbol,
const uint32_t &year,
const uint32_t &month,
const uint32_t &day,
const char &call_put_indicator,
const double &strike_price) {
if (symbol.size() > 6) {
throw std::invalid_argument("Symbol length cannot exceed 6 characters.");
}
std::ostringstream osi_stream;
// Symbol: 6 bytes, padded with spaces
osi_stream << pad_string(symbol, 6);
// Year: last 2 digits of the year
osi_stream << std::setw(2) << std::setfill('0') << (year % 100);
// Month: 2 digits
osi_stream << std::setw(2) << std::setfill('0') << month;
// Day: 2 digits
osi_stream << std::setw(2) << std::setfill('0') << day;
// Call/Put Indicator: 1 byte
osi_stream << call_put_indicator;
// Strike Price: 5 bytes for dollars, 3 bytes for decimal part
int strike_dollars = static_cast<int>(strike_price);
int strike_decimals =
static_cast<int>((strike_price - strike_dollars) * 1000);
osi_stream << std::setw(5) << std::setfill('0') << strike_dollars;
osi_stream << std::setw(3) << std::setfill('0') << strike_decimals;
return osi_stream.str();
}
void parse_option(Message &order, const api::Security &sec) {
auto osi = create_osi_symbol(
sec.symbol(), sec.option().expiration_year(),
sec.option().expiration_month(), sec.option().expiration_day(),
sec.option().call_put() == bts::api::PutOrCall::PUTORCALL_CALL ? 'c'
: 'p',
sec.option().strike());
std::ostringstream month_stream;
month_stream << std::setw(2) << std::setfill('0')
<< sec.option().expiration_month();
std::string month = month_stream.str();
LOG(OrderMan, Warn, "OnixFIXAdapter::parse_option parsing osi:{}", osi);
order
.set(
OnixS::FIX::FIX42::Tags::PutOrCall,
sec.option().call_put() == bts::api::PutOrCall::PUTORCALL_CALL ? "1"
: "0")
.set(
OnixS::FIX::FIX42::Tags::StrikePrice,
std::to_string(sec.option().strike()))
.set(
OnixS::FIX::FIX42::Tags::MaturityMonthYear,
std::to_string(sec.option().expiration_year()) + month)
.set(
OnixS::FIX::FIX42::Tags::MaturityDay,
std::to_string(sec.option().expiration_day()))
.set(OnixS::FIX::FIX42::Tags::SecurityType, "OPT")
.set(OnixS::FIX::FIX42::Tags::Symbol, sec.symbol());
}
void add_cfg_fields(Message &order, const config::MarexFixCfg &cfg) {
order
.set(
OnixS::FIX::FIX42::Tags::HandlInst,
cfg.handl_inst()) // 1: DMA 2: ALGO 3:CARE
.set(
OnixS::FIX::FIX42::Tags::Rule80A,
cfg.Rule80A_Name(cfg.rule80a())) //A = AGENCY P = PRINCIPAL
.set(OnixS::FIX::FIX42::Tags::SenderSubID, cfg.sender_sub_id())
.set(OnixS::FIX::FIX42::Tags::TargetSubID, cfg.target_sub_id())
.set(OnixS::FIX::FIX42::Tags::ClientID, cfg.client_id())
.set(OnixS::FIX::FIX42::Tags::CustomerOrFirm, cfg.customer_or_firm())
.set(OnixS::FIX::FIX42::Tags::OpenClose, cfg.open_close());
order.set(FIX42::Tags::Currency,cfg.currency());
order.set(FIX42::Tags::SecurityExchange,cfg.security_exchange());
order.set(FIX42::Tags::ExDestination,cfg.ex_destination());
}
void populate_common_order_fields(Message &order, const CreateOrderMsg &msg) {
order.set(OnixS::FIX::FIX42::Tags::Account, msg.routing_record->account())
.set(
OnixS::FIX::FIX42::Tags::TimeInForce,
translateTimeInForce(msg.time_in_force))
.set(
OnixS::FIX::FIX42::Tags::OrdType, translateOrderType(msg.order_type))
.set(OnixS::FIX::FIX42::Tags::Side, translateSide(msg.side))
.set(
OnixS::FIX::FIX42::Tags::Price,
static_cast<float>(msg.price.value()))
.set(OnixS::FIX::FIX42::Tags::OrderQty, msg.quantity.value());
if (msg.security->has_option()) {
parse_option(order, *msg.security);
} else {
order.set(OnixS::FIX::FIX42::Tags::Symbol, msg.security->symbol())
.set(OnixS::FIX::FIX42::Tags::SecurityType, "FUT") //need to fix this
.set(OnixS::FIX::FIX42::Tags::Currency, msg.security->currency());
}
}
void OnixFIXAdapter::modify_order(
const CancelReplaceOrderMsg &msg) { //confirm will have everything
try {
//use this fileOnixS.FixEngineCpp-Ubuntu2204-gcc1210-c++17-x64-4.10.1/OnixS.FixEngineCpp-Ubuntu2204-gcc1210-c++17-x64-4.10.1/doc/html/group__double-decimal-conversion.html
LOG(OrderMan, Warn, "OnixFIXAdapter::modify_order modifying order{}",
msg);
auto order_cfg = config::get<config::MarexFixCfg>();
auto id =
state_handler->get_marex_key(OrderKey(msg.create_fields.key.value()));
if (id == "") {
LOG(OrderMan, Error,
"OnixFIXAdapter::modify_order could not find clorid");
return;
}
auto new_id = state_handler->new_id(id);
Message order(
OnixS::FIX::FIX42::Values::MsgType::Order_Cancel_Replace_Request,
OnixS::FIX::ProtocolVersion::FIX_42);
order.set(OnixS::FIX::FIX42::Tags::OrigClOrdID, id)
.set(OnixS::FIX::FIX42::Tags::ClOrdID, new_id) // ID for order
.set(
OnixS::FIX::FIX42::Tags::TransactTime,
OnixS::FIX::Timestamp::utc(),
OnixS::FIX::TimestampFormat::YYYYMMDDHHMMSSMsec);
add_cfg_fields(order, order_cfg);
populate_common_order_fields(order, msg.create_fields);
order.validate();
LOG(OrderMan, Warn, "order Validated");
order.set(OnixFIXAdapter::FIX_TAG_SMART_ROUTING, "SMART");
std::string order_string = order.toString('|');
state_handler->modifyOrder(msg, new_id);
session->send(&order);
LOG(OrderMan, Warn, "Order update sent :{}", order_string);
} catch (const OnixS::FIX::Exception &ex) {
const std::string err = ex.what();
handleException("Modify order", err);
}
}
void OnixFIXAdapter::launch_new_order(const CreateOrderMsg &msg) {
LOG(OrderMan, Warn, "launch order {}", msg);
try {
LOG(OrderMan, Warn, "OnixFIXAdapter::launch_new_order creating order");
auto order_cfg = config::get<config::MarexFixCfg>();
auto new_id =
state_handler->new_id(":" + std::to_string(msg.key.value()));
LOG(OrderMan, Warn, "OnixFIXAdapter::launch_new_order new id:'{}'",
new_id);
std::string symbol = msg.security->symbol();
Message order(
OnixS::FIX::FIX42::Values::MsgType::Order_Single,
OnixS::FIX::ProtocolVersion::FIX_42);
order
.set(OnixS::FIX::FIX42::Tags::ClOrdID, new_id) // ID for order
.set(
OnixS::FIX::FIX42::Tags::TransactTime,
OnixS::FIX::Timestamp::utc(),
OnixS::FIX::TimestampFormat::YYYYMMDDHHMMSSMsec);
add_cfg_fields(order, order_cfg);
populate_common_order_fields(order, msg);
std::string order_string = order.toString('|');
std::string order_json = order.toJson();
order.validate();
LOG(OrderMan, Warn, "order Validated");
order.set(OnixFIXAdapter::FIX_TAG_SMART_ROUTING, "SMART");
state_handler->launchOrder(msg, new_id);
// listener.fire_dummy_report(msg);
session->send(&order);
LOG(OrderMan, Warn, "Order sent :{} json:{}", order_string, order_json);
}
// Shut down all reactors if they exist.
catch (const OnixS::FIX::Exception &ex) {
const std::string err = ex.what();
handleException("Launch order", err);
}
}
void OnixFIXAdapter::register_callback(const OnExecutionReport &cb) {
LOG(OrderMan, Warn, "OnixFIXAdapter::register_callback callback registered");
listener.setCallback(cb);
}
void OnixFIXAdapter::cancel_order(const CancelOrderMsg &msg) {
LOG(OrderMan, Warn, "cancel order {}", msg);
try {
auto id = state_handler->get_marex_key(OrderKey(msg.key.value()));
auto order_cfg = config::get<config::MarexFixCfg>();
if (id == "") {
LOG(OrderMan, Error,
"OnixFIXAdapter::modify_order could not find clorid");
return;
}
auto new_id = state_handler->new_id(id);
Message order_cancel(
OnixS::FIX::FIX42::Values::MsgType::Order_Cancel_Request,
OnixS::FIX::ProtocolVersion::FIX_42);
order_cancel.set(
OnixS::FIX::FIX42::Tags::HandlInst, order_cfg.handl_inst());
order_cancel.set(OnixS::FIX::FIX42::Tags::ClOrdID, new_id);
order_cancel.set(OnixS::FIX::FIX42::Tags::OrigClOrdID, id)
.set(OnixS::FIX::FIX42::Tags::Side, translateSide(msg.side))
.set(
OnixS::FIX::FIX42::Tags::TransactTime,
OnixS::FIX::Timestamp::utc(),
OnixS::FIX::TimestampFormat::YYYYMMDDHHMMSSMsec);
if (msg.security->has_option()) {
parse_option(order_cancel, *msg.security);
} else {
order_cancel
.set(OnixS::FIX::FIX42::Tags::Symbol, msg.security->symbol())
.set(OnixS::FIX::FIX42::Tags::Currency, msg.security->currency());
}
std::string order_str = order_cancel.toString('|');
LOG(OrderMan, Warn, "order_cancel Created: {}", order_str);
// order_cancel.validate(); fails becasuse of HandlInst
LOG(OrderMan, Warn, "order Validated");
order_cancel.set(OnixFIXAdapter::FIX_TAG_SMART_ROUTING, "SMART");
state_handler->cancelOrder(msg, new_id);
session->send(&order_cancel);
LOG(OrderMan, Warn, "order cancel sent");
}
// Shut down all reactors if they exist.
catch (const OnixS::FIX::Exception &ex) {
const std::string err = ex.what();
handleException("Cacnel order", err);
}
}
} // namespace bts::orderman
@@ -0,0 +1,27 @@
#pragma once
#include "IFIXGenericAdapter.h"
#include "lib/orderman/gateways/OnixFIXListener.h"
#include "lib/orderman/gateways/TrafixOrderStateHandler.h"
#include <memory>
namespace bts::orderman {
class OnixFIXAdapter : public IFIXGenericAdapter {
static constexpr int FIX_TAG_SMART_ROUTING = 847;
public:
OnixFIXAdapter(md::SecurityOracle sec_oracle);
~OnixFIXAdapter() override;
void register_callback(const OnExecutionReport &report) override;
void launch_new_order(const CreateOrderMsg &msg) override;
void cancel_order(const CancelOrderMsg &msg) override;
void modify_order(const CancelReplaceOrderMsg &) override;
void start() override;
private:
std::shared_ptr<TrafixOrderStateHandler> state_handler;
OnixFixListener listener;
std::unique_ptr<Session> session;
};
} // namespace bts::orderman
@@ -0,0 +1,264 @@
#include "OnixFIXListener.h"
#include "lib/api/services/OrderService.pb.h"
#include "lib/domain_model/Security.pb.h"
#include "lib/domain_model/common.h"
#include "lib/logging/logging.h"
#include "lib/orderman/gateways/IExecutionGateway.h"
#include "lib/orderman/gateways/IFIXGenericAdapter.h"
#include "lib/orderman/orderman_utils/ExecutionReport.h"
#include "lib/orderman/orderman_utils/orderman_common.h"
#include "lib/utils/Uuid.h"
#include "lib/utils/bts_assert.h"
#include <OnixS/FIXEngine/FIX/FIX42.h>
#include <OnixS/FIXEngine/FIX/Tag.h>
#include <string>
namespace bts::orderman {
ExecutionReport OnixFixListener::fire_dummy_report(CreateOrderMsg msg) {
ExecutionReport *er_p = exe_report_pool.construct();
ExecutionReport &er = *er_p;
std::string account_name = "BTSIM";
std::array<char, 12> an_array = {};
size_t i = 0;
for (const char c : account_name) {
an_array[i++] = c;
}
er.account_name = AccountName(an_array);
to_array(msg.routing_record->account(), &er.account_name.value());
to_array(
msg.routing_record->operator_id(), &er.operator_id.value());
er.order_key = msg.key;
er.msg_seq_num = 0;
er.security = msg.security;
er.leg_security = nullptr;
er.side = msg.side;
er.time_in_force = msg.time_in_force;
er.order_type = msg.order_type;
er.exec_type = api::orderman::EXEC_TYPE_NEW;
er.order_status = api::orderman::NATIVE_ORDER_STATUS_NEW;
er.manual_order_indicator = api::orderman::MANUAL_ORDER_INDICATOR_NO;
auto s = std::to_string(msg.key.value());
std::array<char, 17> idAsArray{};
std::copy_n(s.begin(), std::min(s.size()-1, idAsArray.size() - 1), idAsArray.begin());
er.exchange_order_id = ExchangeOrderID{idAsArray};
er.price = msg.price;
er.quantity = msg.quantity;
er.filled_quantity = Quantity(0);
er.last_shares = Quantity(0);
er.msg = "Message Placed";
er.exchange = msg.security->exchange();
// Set transaction time
ExchangeTimeStamp64 now = ExchangeTimeStamp64(datetime::unix_nano_now());
er.transact_time = now;
cb(er,{});
return er;
}
namespace {
ExecutionReport createDefaultExecutionReport() {
ExecutionReport er;
er.exchange = api::VENUE_CBOE;
std::string account_name = "BTSIM";
std::string ops_id = "op_id_bts";
std::array<char, 12> account_array = {};
std::copy_n(account_name.begin(), std::min(account_name.size(), account_array.size()), account_array.begin());
er.account_name = AccountName(account_array);
to_array(ops_id, &er.operator_id.value());
return er;
}
void populateExecTypeAndOrderStatus(ExecutionReport &er, int exec_type_value) {
switch (exec_type_value) {
case 0:
er.exec_type = api::orderman::ExecType::EXEC_TYPE_NEW;
er.order_status = api::orderman::NATIVE_ORDER_STATUS_NEW;
break;
case 1:
er.exec_type = api::orderman::ExecType::EXEC_TYPE_PARTIAL_FILL;
er.order_status = api::orderman::NATIVE_ORDER_STATUS_PARTIALLY_FILLED;
break;
case 2:
er.exec_type = api::orderman::ExecType::EXEC_TYPE_FILL;
er.order_status = api::orderman::NATIVE_ORDER_STATUS_FILLED;
break;
case 4:
er.exec_type = api::orderman::ExecType::EXEC_TYPE_CANCELED;
er.order_status = api::orderman::NATIVE_ORDER_STATUS_CANCELED;
break;
case 5:
er.exec_type = api::orderman::ExecType::EXEC_TYPE_REPLACED;
er.order_status = api::orderman::NATIVE_ORDER_STATUS_REPLACED;
break;
case 8:
er.exec_type = api::orderman::ExecType::EXEC_TYPE_REJECTED;
er.order_status = api::orderman::NATIVE_ORDER_STATUS_REJECTED;
break;
default:
er.exec_type = api::orderman::ExecType::EXEC_TYPE_UNDEFINED;
er.order_status = api::orderman::NATIVE_ORDER_STATUS_UNDEFINED;
break;
}
}
void populateCreateOrderFields(ExecutionReport &er, const CreateOrderMsg &create_order_msg) {
er.security = create_order_msg.security;
er.order_type = create_order_msg.order_type;
er.time_in_force = create_order_msg.time_in_force;
er.manual_order_indicator = create_order_msg.manual_order_indicator;
to_array(create_order_msg.routing_record->account(), &er.account_name.value());
to_array(create_order_msg.routing_record->operator_id(), &er.operator_id.value());
er.exchange = create_order_msg.security->exchange();
}
std::array<char, 17> stringToFixedArray(const std::string &s) {
std::array<char, 17> idAsArray{};
std::copy_n(s.begin(), std::min(s.size(), idAsArray.size() - 1), idAsArray.begin());
return idAsArray;
}
}
ExecutionReport OnixFixListener::get_exe_report(Message msg) {
ExecutionReport *er_p = exe_report_pool.construct();
ExecutionReport &er = *er_p;
auto msg_str = msg.toString();
LOG(OrderMan,Info, "OnixFixListener::get_exe_report {}", msg_str);
// Extract order key and create order message
er.order_key = state_handler->getOrderKey(msg.get(FIX42::Tags::ClOrdID).toString()); //OrderKey(extractUInt(msg.get(FIX42::Tags::ClOrdID).toString()));
auto m = state_handler->getCreateOrderMsg(er.order_key);
if(m.has_value()){
populateCreateOrderFields(er, m.value());
}else{ //handle this later
LOG(OrderMan,Error, "OnixFixListener::get_exe_report {} could not find create order message", msg_str);
er = createDefaultExecutionReport();
}
// Populate fields from the message
int tmp;
msg.get(FIX42::Tags::MsgSeqNum).toNumber(tmp);
er.msg_seq_num = tmp;
er.leg_security = nullptr;
er.side = msg.get(FIX42::Tags::Side).toString()=="1"?api::Side::SIDE_BUY:api::Side::SIDE_SELL;
int exec_type_value = 9; // Default to undefined
msg.get(FIX42::Tags::ExecType).toNumber(exec_type_value);
populateExecTypeAndOrderStatus(er, exec_type_value);
auto price_str = msg.get(FIX42::Tags::Price);
er.price = Price(price_str.valid() ? std::stod(price_str.toString()) : 0);
auto order_qty_str = msg.get(FIX42::Tags::OrderQty);
er.quantity = Quantity(order_qty_str.valid() ? std::stoi(order_qty_str.toString()) : 0);
auto leaves_qty_str = msg.get(FIX42::Tags::LeavesQty);
er.filled_quantity = leaves_qty_str.valid() ? er.quantity - Quantity(std::stoi(leaves_qty_str.toString())) : er.quantity;
auto last_shares_str = msg.get(FIX42::Tags::LastShares);
er.last_shares = Quantity(last_shares_str.valid() ? std::stoi(last_shares_str.toString()) : 0);
auto avg_px_str = msg.get(FIX42::Tags::AvgPx);
er.avg_price = Price(avg_px_str.valid() ? std::stod(avg_px_str.toString()) : 0);
auto text_str = msg.get(FIX42::Tags::Text).toString();
er.msg = text_str.substr(0, std::min(text_str.length(), size_t(15)));
auto exec_id_str = msg.get(FIX42::Tags::ExecID);
std::string fill_id = exec_id_str.valid() ? exec_id_str.toString() : bts::utils::Uuid().to_text_string();
to_array(fill_id, &er.exchange_fill_id.value());
auto last_px_str = msg.get(FIX42::Tags::LastPx);
er.last_price = Price(last_px_str.valid() ? std::stod(last_px_str.toString()) : 0);
er.exchange_order_id = ExchangeOrderID{stringToFixedArray(msg.get(FIX42::Tags::OrderID).toString())};
// er.transact_time = ExchangeTimeStamp64(datetime::unix_nano_now());
//exchgane gives this "TransactTime": "20241111-16:36:08.243",
// er.transact_time =
// Set transaction time
ExchangeTimeStamp64 now = ExchangeTimeStamp64(datetime::unix_nano_now());
er.transact_time = now;
return er;
}
void OnixFixListener::onInboundApplicationMsg(Message &msg, Session * /*sn*/) {
std::string msg_str = msg.toString();
std::string msg_json = msg.toJson();
FieldValueRef t = msg.type();
char message_type;
if(!t.toChar(message_type)){
LOG(OrderMan, Warn,
"no message type message: {} As json {}",
msg_str, msg_json);
return;
}
LOG(OrderMan, Warn,
"Incoming application-level message: {} As json {}, type: {}",
msg_str, msg_json,message_type);
switch (message_type) {
case '8':
{
//exection report
auto er = get_exe_report(msg);
if(er.security==nullptr){
LOG(OrderMan, Error,"DID not have sec");
return;
}
LOG(OrderMan, Error,"msg {}",er.msg);
cb(er,{});
}
}
// LOG(OrderMan, Warn,
// "onInboundApplicationMsg::inbound firing er: {}",
// er->to_string());
// cb(*er, {});
}
void OnixFixListener::onInboundSessionMsg(Message & msg, Session * /*sn*/)
{
std::string msg_str = msg.toString();
std::string msg_json = msg.toJson();
LOG( OrderMan, Warn,
"Incoming session-level message:\n {} As json {}",
msg_str,
msg_json);
}
void OnixFixListener::onStateChange(SessionState::Enum newState, SessionState::Enum prevState,
Session * /*sn*/)
{
std::string prevst = SessionState::toString(prevState);
std::string newst = SessionState::toString(newState);
LOG(OrderMan, Warn,
"Session's state is changed, prevState = {} , newState= {}",
prevst,
newst);
}
void OnixFixListener::onError(ErrorReason::Enum /*reason*/, const std::string & description, Session * /*sn*/)
{
LOG( OrderMan, Error, "Session-level error: {}", description);
}
void OnixFixListener::onWarning(WarningReason::Enum /*reason*/, const std::string & description,
Session * /*sn*/)
{
LOG( OrderMan, Error, "Session-level warning: {}", description);
}
} // namespace bts::orderman
@@ -0,0 +1,52 @@
#pragma once
#include "lib/orderman/CreateOrderMsg.h"
#include "lib/orderman/gateways/IExecutionGateway.h"
#include "lib/orderman/gateways/TrafixOrderStateHandler.h"
#include "lib/orderman/orderman_utils/ExecutionReport.h"
#include <OnixS/FIXEngine.h>
#include <OnixS/FIXEngine/FIX/Message.h>
#include <boost/pool/object_pool.hpp>
namespace bts::orderman {
class TrafixOrderStateHandler;
using namespace OnixS::FIX;
class OnixFixListener : public OnixS::FIX::ISessionListener {
public:
OnixFixListener( std::shared_ptr<TrafixOrderStateHandler> state_handler) : state_handler(state_handler),cb(dummy) {
LOG(OrderMan, Info, "OnixFixListener::cb set callback registered");
}
std::shared_ptr<TrafixOrderStateHandler> state_handler;
unsigned long exchange_oid = 0;
/// Is called when the application-level message is received from the counterparty.
void onInboundApplicationMsg(Message &msg, Session *sn)
ONIXS_FIXENGINE_FINAL;
// Set the real callback function
void setCallback(const OnExeReport &callback) {
LOG(OrderMan, Info, "OnixFixListener::setCallback callback registered");
cb = callback;
}
/// Is called when the session-level message is received from the counterparty.
void onInboundSessionMsg(Message &msg, Session *sn) ONIXS_FIXENGINE_FINAL;
void onStateChange(
SessionState::Enum newState,
SessionState::Enum prevState,
Session *sn) ONIXS_FIXENGINE_FINAL;
OnExeReport cb;
ExecutionReport fire_dummy_report(CreateOrderMsg msg);
ExecutionReport get_exe_report(Message msg);
static void dummy(const ExecutionReport &, const trace::CorID &) {
LOG(OrderMan, Error, "OnixFixListener::Dummy fired");
}
void onError(
ErrorReason::Enum reason,
const std::string &description,
Session *sn) ONIXS_FIXENGINE_FINAL;
void onWarning(
WarningReason::Enum reason,
const std::string &description,
Session *sn) ONIXS_FIXENGINE_FINAL;
boost::object_pool<ExecutionReport> exe_report_pool;
};
} // namespace bts::orderman
@@ -0,0 +1,88 @@
#include "TrafixOrderStateHandler.h"
#include "lib/logging/logging.h"
#include "lib/orderman/CreateOrderMsg.h"
#include "lib/orderman/order_factory/IOrderFactory.h"
#include "lib/orderman/orderman_utils/orderman_common.h"
#include <cstddef>
#include <memory>
#include <optional>
#include <string>
namespace bts::orderman {
TrafixOrderStateHandler::TrafixOrderStateHandler(md::SecurityOracle sec_oracle)
: pimpl_(std::make_unique<PImpl>(sec_oracle)) {}
TrafixOrderStateHandler::~TrafixOrderStateHandler() = default;
TrafixOrderStateHandler::OrderInfo* TrafixOrderStateHandler::get_order_info(const OrderKey&key) const {
auto it = pimpl_->orders.find(key);
if (it != pimpl_->orders.end()) {
return &it->second;
}
return nullptr;
}
OrderKey TrafixOrderStateHandler::launchOrder(const CreateOrderMsg& msg, const std::string& clor_id) {
std::lock_guard<std::mutex> lock(pimpl_->orders_mutex);
pimpl_->orders.emplace(msg.key, OrderInfo(msg, clor_id));
return msg.key;
}
OrderKey TrafixOrderStateHandler::modifyOrder(const CancelReplaceOrderMsg& msg, const std::string& clor_id) {
std::lock_guard<std::mutex> lock(pimpl_->orders_mutex);
LOG(md,Info,"Modifying Order: {}, id:{}",msg,clor_id);
auto v = get_order_info(msg.create_fields.key);
if(v==nullptr) return null_order_key();
LOG(md,Info,"Modifying found order info Order: {}, id:{}",msg,clor_id);
v->update(msg, clor_id);
return msg.create_fields.key;
}
OrderKey TrafixOrderStateHandler::cancelOrder(const CancelOrderMsg& msg, const std::string& clor_id) {
std::lock_guard<std::mutex> lock(pimpl_->orders_mutex);
LOG(md,Info,"canceling Order: {}, id:{}",msg,clor_id);
auto v = get_order_info(msg.key);
if(v==nullptr) return null_order_key();
std::string cancel_id = clor_id;
v->cancel(msg, cancel_id);
return msg.key;
}
//static
std::string TrafixOrderStateHandler::new_id(const std::string& current_id) {
LOG(OrderMan, Warn, "new_id {}", current_id);
int update_number = getUpdateNumber(current_id)+1;
LOG(OrderMan, Warn, "new_id {}, update number {}", current_id,update_number);
return std::to_string(update_number) + ":" + std::to_string(getOrderKey(current_id).value());
}
OrderKey TrafixOrderStateHandler::getOrderKey(const std::string& id) {
std::size_t pos = id.find(':');
if (pos == std::string::npos) return null_order_key();
return OrderKey(std::stoi(id.substr(pos + 1)));
}
int TrafixOrderStateHandler::getUpdateNumber(const std::string& id) {
std::size_t pos = id.find(':');
if(pos==std::string::npos || pos ==0){
return 0;
}
std::string prefix = id.substr(0, pos);
bts_assert_msgf(std::all_of(prefix.begin(), prefix.end(), ::isdigit), "Failed to get update number id:{}",id);
return std::stoi(prefix);
}
std::optional<CreateOrderMsg> TrafixOrderStateHandler::getCreateOrderMsg(const OrderKey& key) const {
std::lock_guard<std::mutex> lock(pimpl_->orders_mutex);
auto v = get_order_info(key);
if(v!=nullptr) return v->order_msg;
LOG(md, Error, "Order with key {} not found.", key);
return std::nullopt;
}
std::string TrafixOrderStateHandler::get_marex_key(const OrderKey &key) {
std::lock_guard<std::mutex> lock(pimpl_->orders_mutex);
auto v = get_order_info(key);
if(v!=nullptr) return v->current_id;
return "";
}
} // namespace b
@@ -0,0 +1,61 @@
#pragma once
#include "IFIXGenericAdapter.h"
#include "lib/orderman/CancelOrderMsg.h"
#include "lib/orderman/CreateOrderMsg.h"
#include "lib/orderman/gateways/OnixFIXListener.h"
#include "lib/orderman/order_factory/IOrderFactory.h"
#include "lib/orderman/orderman_utils/orderman_common.h"
#include "lib/utils/bts_assert.h"
#include <memory>
#include <optional>
#include <string>
#include <unordered_map>
namespace bts::orderman {
class TrafixOrderStateHandler {
public:
explicit TrafixOrderStateHandler(md::SecurityOracle sec_oracle);
~TrafixOrderStateHandler();
OrderKey launchOrder(const CreateOrderMsg& msg, const std::string& clor_id);
std::optional<CreateOrderMsg> getCreateOrderMsg(const OrderKey& key) const;
std::string get_marex_key(const OrderKey &key) ;
OrderKey modifyOrder(const CancelReplaceOrderMsg& msg, const std::string& clor_id);
OrderKey cancelOrder(const CancelOrderMsg& msg, const std::string& clor_id);
static std::string new_id(const std::string& current_id);
static OrderKey getOrderKey(const std::string& id);
static int getUpdateNumber(const std::string& id);
private:
struct OrderInfo {
explicit OrderInfo(const CreateOrderMsg& msg, const std::string& id)
: current_id(id), order_msg(msg) {}
void update(const CancelReplaceOrderMsg& msg, const std::string& new_id) {
LOG(md,Info,"OrderInfo:update Order: {}, id:{}",msg,new_id);
upd_msg = msg;
current_id = new_id;
}
void cancel(const CancelOrderMsg& msg, const std::string& new_id) {
cancel_msg = msg;
current_id = new_id;
}
std::string current_id;
CreateOrderMsg order_msg;
CancelReplaceOrderMsg upd_msg;
CancelOrderMsg cancel_msg;
};
OrderInfo* get_order_info(const OrderKey&key) const;
struct PImpl {
explicit PImpl(md::SecurityOracle sec_oracle) : sec_oracle(sec_oracle) {}
mutable std::mutex orders_mutex;//need to use this every time orders are accesed
//can be improved later on
std::unordered_map<OrderKey, OrderInfo> orders;
md::SecurityOracle sec_oracle;
};
std::unique_ptr<PImpl> pimpl_;
};
}
@@ -129,7 +129,7 @@ NativeOrderBase *OrderFactory::make_order(const api::Venue exchange) {
return new ICENativeOrder();
else if (exchange == api::VENUE_GLBX || exchange == api::VENUE_SIM)
return new GLBXNativeOrder();
return new EMNativeOrder();
return new EMNativeOrder(exchange);
}
NativeOrderBase *OrderFactory::make_native_order(
+22 -6
View File
@@ -5,6 +5,7 @@
#include "lib/config/protos/toggles.pb.h"
#include "lib/domain/OrderStatus.h"
#include "lib/domain_model/Security.h"
#include "lib/domain_model/Security.pb.h"
#include "lib/domain_model/common.h"
#include "lib/jetfix/codec/FIXFragment.h"
#include "lib/jetfix/codec/FieldDefinitions.h"
@@ -52,10 +53,10 @@ struct InflightUpdateState {
};
struct EMNativeOrder::PImpl {
PImpl()
PImpl(const api::Venue exchange)
: instance_id(EMNativeOrder::count++), services(nullptr),
session_services(), sec(nullptr), previous_working_position(0),
definition(), state(), exchange_order_id() {}
definition(), state(), exchange_order_id(), exchange(exchange) {}
const int instance_id;
bool inited = false;
OrderServices *services;
@@ -67,7 +68,7 @@ struct EMNativeOrder::PImpl {
OrderDefinition definition;
OrderState state;
std::optional<ExchangeOrderIDVar> exchange_order_id;
const api::Venue exchange;
OrderServices::ExeReportSubID exe_report_sid;
OrderServices::CancelRejectSubID cancel_reject_sid;
// If a cancel is sent before the order placement is ack'ed this is set to
@@ -82,7 +83,7 @@ struct EMNativeOrder::PImpl {
RoutingRecord routing_record;
};
EMNativeOrder::EMNativeOrder() : me(new PImpl()) {
EMNativeOrder::EMNativeOrder(const api::Venue exchange) : me(new PImpl(exchange)) {
LOG(OrderMan, Info,
"EMNativeOrder::EMNativeOrder instance_id: {} t_id: {} t_name: {}",
me->instance_id, os::get_thread_id(), os::get_application_name());
@@ -342,6 +343,10 @@ OrderOpResult EMNativeOrder::on_cancel() {
case ORDER_STATUS_WORKING: {
// TODO, CancelOrderMsg is not completely filled
CancelOrderMsg cxl_msg = {};
cxl_msg.side = me->definition.side();
cxl_msg.security = me->sec;
// cxl_msg.exchange_order_id=me->exchange_order_id;//handle this
cxl_msg.routing_record = &me->routing_record;
cxl_msg.key = key();
LOG(OrderMan, Info, "EMNativeOrder queue_order_msg for cxl");
me->services->queue_order_msg(*me->session_services, cxl_msg);
@@ -501,11 +506,22 @@ OrderOpResult EMNativeOrder::_on_activate(
me->state.set_filled_quantity(0);
me->state.set_last_price(0);
me->state.set_avg_price(0);
me->session_services = &me->services->get_session_services(
get_native_state(me->state).routing_record().session());
// TODO: just to get going
me->session_services = &me->services->get_session_services("deribit_1");
std::string session_name;
switch (me->exchange) {
case api::VENUE_CBOE:
session_name = "cboe_1";
break;
case api::VENUE_TAIFEX:
session_name="taifex_1";
break;
default:
session_name ="deribit_1";
}
me->session_services = &me->services->get_session_services(session_name);
// Checks for the following:
// 1. Gateway connection
+2 -1
View File
@@ -1,6 +1,7 @@
#pragma once
#include "lib/api/services/OrderService.pb.h"
#include "lib/domain_model/Security.pb.h"
#include "lib/domain_model/common.h"
#include "lib/orderman/CreateOrderMsg.h"
#include "lib/orderman/orders/NativeOrderBase.h"
@@ -19,7 +20,7 @@ class SessionServices;
// Wrapper around Emumba's order
class EMNativeOrder : public NativeOrderBase {
public:
EMNativeOrder();
EMNativeOrder(const api::Venue exchange = api::VENUE_DERIBIT);
~EMNativeOrder() override;
api::orderman::OrderDefinition const &definition() const override;
void on_init_new(
+4 -4
View File
@@ -478,7 +478,7 @@ OrderOpResult GLBXNativeOrder::on_activate(
now.to_uint64(), (now - corid.app_arrival).nanoseconds());
// in sim mode, we dont need this,
me->encoder->send_create(*me->services, *me->session_services);
if (me->in_sim_mode) {
if (me->in_sim_mode) { //CME EXECUTION NEEDS TO BE UPDATED
CreateOrderMsg c_msg = {};
_create_create_msg(c_msg);
me->services->queue_order_msg(*me->session_services, c_msg);
@@ -579,7 +579,7 @@ OrderOpResult GLBXNativeOrder::on_update(api::orderman::Update const &update) {
auto p = Price(update.price());
auto q = Quantity(update.quantity());
auto send_update_in_sim_mode = [&]() {
if (me->in_sim_mode) {
if (me->in_sim_mode) { //CME EXECUTION NEEDS TO BE UPDATED
CancelReplaceOrderMsg cxl_repl_msg = {};
_create_create_msg(cxl_repl_msg.create_fields);
Price new_p = Price(update.price());
@@ -799,7 +799,7 @@ OrderOpResult GLBXNativeOrder::on_cancel() {
me->encoder->update_cancel_encoding(*me->services);
me->encoder->send_cancel(*me->services, *me->session_services);
// TODO: check for sim mode here
if (me->in_sim_mode) {
if (me->in_sim_mode) { //CME EXECUTION NEEDS TO BE UPDATED
CancelOrderMsg cxl_msg = {};
cxl_msg.key = key();
LOG(OrderMan, Info, "GLBXNativeOrder queue_order_msg for cxl");
@@ -1381,7 +1381,7 @@ void GLBXNativeOrder::_create_create_msg(CreateOrderMsg &c) {
bts::orderman::get_quantity(me->state, StateQuantityType::OrderQuantity);
c.security = me->sec;
c.side = me->definition.side();
c.order_type = get_glbx_native_ord_type(me->definition);
c.order_type = get_glbx_native_ord_type(me->definition); //CME EXECUTION NEEDS TO BE UPDATED
c.time_in_force = me->definition.time_in_force();
c.order_type = api::orderman::NATIVE_ORDER_TYPE_LIMIT;
c.manual_order_indicator = me->definition.manual_order_indicator();
@@ -277,6 +277,7 @@ void iLink2OrderEncoder::update_exchange_order_id(ExchangeOrderIDVar const &v) {
auto oid = to_trim_str(std::get<ExchangeOrderID>(v));
me->update_encoding.add<jetfix::fix::OrderID>(oid);
me->cancel_encoding.add<jetfix::fix::OrderID>(oid);
//CME EXECUTION NEEDS TO BE UPDATED, can be commented out
}
void iLink2OrderEncoder::update_create_order_encoding(
+46
View File
@@ -0,0 +1,46 @@
#include "spark_data_directory.h"
#include "lib/config/config.h"
#include "lib/config/protos/system_config.pb.h"
#include "lib/os/os.h"
#include <boost/filesystem/path.hpp>
#include <fmt/format.h>
#include <string>
namespace bts::spark {
SparkDataDirectory::SparkDataDirectory()
: SparkDataDirectory(config::get<config::SystemConfig>().data_path()) {}
SparkDataDirectory::SparkDataDirectory(const std::string &root) : root_(root) {}
boost::filesystem::path SparkDataDirectory::root() const { return root_; }
boost::filesystem::path SparkDataDirectory::log_dir() const {
return root_ / fmt::format("logging.{}/", os::get_application_name());
}
boost::filesystem::path SparkDataDirectory::log_file() const {
return log_dir() / "log";
}
boost::filesystem::path SparkDataDirectory::log_cache_file() const {
return log_dir() / "log_cache";
}
boost::filesystem::path SparkDataDirectory::database_dir(
const std::string &name) const {
return root_ / "databases" / (name + "/");
}
boost::filesystem::path SparkDataDirectory::database_file(
const std::string &name) const {
return root_ / "databases" / name / (name + ".sqlite");
}
boost::filesystem::path SparkDataDirectory::database_file(
const std::string &dir_name, const std::string &db_name) const {
return root_ / "databases" / dir_name / (db_name + ".sqlite");
}
} // namespace bts::spark
+28
View File
@@ -0,0 +1,28 @@
#pragma once
#include <boost/filesystem/path.hpp>
#include <string>
namespace bts::spark {
class SparkDataDirectory {
public:
SparkDataDirectory();
explicit SparkDataDirectory(const std::string &root);
boost::filesystem::path root() const;
boost::filesystem::path log_dir() const;
boost::filesystem::path log_file() const;
boost::filesystem::path log_cache_file() const;
boost::filesystem::path database_dir(const std::string &name) const;
boost::filesystem::path database_file(const std::string &name) const;
boost::filesystem::path database_file(
const std::string &dir_name, const std::string &db_name) const;
private:
boost::filesystem::path root_;
};
} // namespace bts::spark
+18 -17
View File
@@ -9,6 +9,8 @@
namespace bts::persistence {
using bts::api::domain::SecurityType;
using bts::api::domain::SecurityType_Name;
using bts::api::domain::Venue;
using bts::api::domain::Venue_Name;
using std::string;
@@ -52,24 +54,27 @@ uint64_t get_next_id(
}
std::optional<uint64_t> IDGenerator::get_id(
const Venue &venue, const string &name, bool add_if_new) {
return get_id(Venue_Name(venue), name, add_if_new);
const Venue &venue,
SecurityType type,
const string &name,
bool add_if_new) {
std::string ns = fmt::format(
"{}.{}", Venue_Name(venue).substr(6),
SecurityType_Name(type).substr(14));
return get_id(ns, name, add_if_new);
}
std::optional<uint64_t> IDGenerator::get_id(
const std::string &ns, const std::string &key, bool add_if_new) {
std::lock_guard<std::mutex> myLock(_instance_mx);
auto [exists, val] = me->kv_store->retrieve_one(ns, key);
auto [exists, rowid, val] = me->kv_store->retrieve_one(ns, key);
if (exists) {
uint64_t rv = str_to_id(val);
LOG(Gen, Diag, "IDGenerator::get_id ns: {} key:{} id: {}", ns, key, rv);
return std::optional<uint64_t>(rv);
LOG(Gen, Diag, "IDGenerator::get_id ns: {} key: id: {}", ns, key, rowid);
return std::optional<uint64_t>(rowid);
} else {
if (add_if_new) {
auto all = me->kv_store->retrieve_all();
uint64_t new_id = get_next_id(all);
me->kv_store->insert(ns, key, std::to_string(new_id));
auto new_id = me->kv_store->insert(ns, key, "");
LOG(Gen, Diag, "IDGenerator::get_id ns: {} key: id: {}", ns, key,
new_id);
return std::optional<uint64_t>(new_id);
@@ -80,15 +85,11 @@ std::optional<uint64_t> IDGenerator::get_id(
}
}
std::uint64_t IDGenerator::get_or_add(const Venue &venue, const string &name) {
return get_or_add(Venue_Name(venue), name);
}
std::uint64_t IDGenerator::get_or_add(
const std::string &ns, const std::string &key) {
auto id = get_id(ns, key, true);
bts_assert(id.has_value());
return *id;
api::domain::Venue venue,
api::domain::SecurityType type,
const std::string &name) {
return *get_id(venue, type, name, true);
}
IDGenerator::~IDGenerator() {}
+10 -3
View File
@@ -1,6 +1,8 @@
#pragma once
#include "lib/api/domain/SecurityType.pb.h"
#include "lib/api/domain/Venue.pb.h"
#include <OnixS/CME/MDH/Integral.h>
#include <cstdint>
#include <memory>
@@ -21,6 +23,7 @@ class IDGenerator {
std::optional<uint64_t> get_id(
const api::domain::Venue &venue,
api::domain::SecurityType type,
const std::string &name,
bool add_if_new = false);
@@ -28,9 +31,9 @@ class IDGenerator {
const std::string &ns, const std::string &key, bool add_if_new);
std::uint64_t get_or_add(
const api::domain::Venue &venue, const std::string &name);
std::uint64_t get_or_add(const std::string &ns, const std::string &key);
api::domain::Venue venue,
api::domain::SecurityType type,
const std::string &name);
private:
static std::shared_ptr<IDGenerator> instance_ptr_;
@@ -38,6 +41,10 @@ class IDGenerator {
struct PImpl;
std::unique_ptr<PImpl> me;
// These methods are private to force callers to use the [public] overloads with venue, type and
// name params, to ensure consistent formatting.
std::uint64_t get_or_add(const std::string &ns, const std::string &key);
};
} // namespace bts::persistence
+36 -3
View File
@@ -1,8 +1,13 @@
#pragma once
#include "lib/persistence/store_row.h"
#include <cstdint>
#include <map>
#include <set>
#include <string>
#include <tuple>
#include <vector>
namespace bts {
namespace persistence {
@@ -11,17 +16,45 @@ class IStore {
public:
virtual std::map<std::string, std::map<std::string, std::string>>
retrieve_all() = 0;
virtual std::pair<bool, std::string>
retrieve_one(std::string const &ns, std::string const &k) = 0;
/**
* Finds the row with the specified namespace and key.
*
* @return std::tuple<bool, std::int64, std::string> The first item indicates
* whether or not a row was found. For successful queries, the second and third
* items are the rowid and value, respectively.
*/
virtual std::tuple<bool, std::int64_t, std::string> retrieve_one(
std::string const &ns, std::string const &k) = 0;
virtual void
remove(std::string const &ns, std::set<std::string> const &keys) = 0;
virtual void remove_all(std::string const &ns) = 0;
virtual void insert(
virtual std::int64_t insert(
std::string const &ns, std::string const &k, std::string const &v) = 0;
void insert_many(const std::vector<StoreRow> &rows);
void insert_many_resolve_ids(std::vector<StoreRow> &rows);
virtual void update(
std::string const &ns, std::string const &k, std::string const &v) = 0;
virtual ~IStore();
};
//////////////////////////////////////////////////
// Definition
//////////////////////////////////////////////////
inline void IStore::insert_many(const std::vector<StoreRow> &rows) {
for (const auto &row : rows) {
insert(row.ns, row.key, row.value);
}
}
inline void IStore::insert_many_resolve_ids(std::vector<StoreRow> &rows) {
for (auto &row : rows) {
row.id = insert(row.ns, row.key, row.value);
}
}
} // namespace persistence
} // namespace bts
+248 -122
View File
@@ -18,82 +18,112 @@
namespace bts {
namespace persistence {
using bts::config::SystemConfig;
static boost::filesystem::path get_db_path(const std::string &str) {
for (char c : str) {
switch (c) {
case '.':
case '/':
case '\\':
// Looks like 'str' is already a path
return {str};
}
}
return boost::filesystem::path{config::get<SystemConfig>().data_path()} /
"databases" / str / (str + ".sqlite");
}
struct SqliteStore::PImpl {
PImpl(std::string const &store_root, std::string const &db_name)
: db(db_name), db_path(persistence::storage_location(db, store_root)),
conn(nullptr, sqlite3_close), insert(nullptr, sqlite3_finalize),
remove(nullptr, sqlite3_finalize),
remove_all(nullptr, sqlite3_finalize),
update(nullptr, sqlite3_finalize) {
std::string const db;
boost::filesystem::path db_path;
sqlite3_ptr conn{nullptr, sqlite3_close};
sqlite3_stmt_ptr insert{nullptr, sqlite3_finalize};
sqlite3_stmt_ptr insert_many{nullptr, sqlite3_finalize};
sqlite3_stmt_ptr insert_with_id{nullptr, sqlite3_finalize};
sqlite3_stmt_ptr remove{nullptr, sqlite3_finalize};
sqlite3_stmt_ptr remove_all{nullptr, sqlite3_finalize};
sqlite3_stmt_ptr update{nullptr, sqlite3_finalize};
PImpl(const boost::filesystem::path &db_path)
: db(db_path.stem().string()), db_path(db_path) {
bts_assert(db.size());
}
std::string const db;
boost::filesystem::path db_path;
sqlite3_ptr conn;
sqlite3_stmt_ptr insert;
sqlite3_stmt_ptr remove;
sqlite3_stmt_ptr remove_all;
sqlite3_stmt_ptr update;
BTS_NO_COPY(PImpl);
PImpl(std::string const &store_root, std::string const &db_name)
: PImpl(boost::filesystem::path(store_root) / (db_name + ".sqlite")) {}
explicit PImpl(sqlite3_ptr conn) : db(), db_path(), conn(std::move(conn)) {}
PImpl(const PImpl &) = delete;
PImpl &operator=(const PImpl &) = delete;
};
SqliteStore::SqliteStore(std::string const &db)
: SqliteStore(config::get<config::SystemConfig>().data_path(), db) {}
: me(std::make_unique<PImpl>(get_db_path(db))) {
LOG(persist, Info, "SqliteStore ctr. db: {}; db_path: {}", db,
me->db_path.string());
init_db();
}
SqliteStore::SqliteStore(std::string const &root_path, std::string const &db)
: me(new PImpl(root_path, db)) {
SqliteStore::SqliteStore(sqlite3_ptr db)
: me(std::make_unique<PImpl>(std::move(db))) {
LOG(persist, Info, "SqliteStore ctr. db: <imported ptr>");
init_db();
}
void SqliteStore::init_db() {
auto path = me->db_path;
auto parent = path.parent_path();
const auto path = me->db_path;
if (!boost::filesystem::is_directory(parent))
bts_assert(boost::filesystem::create_directories(parent));
if (!boost::filesystem::exists(path)) {
LOG(persist, Warn, "{} database does not exist yet. Creating it at {}",
me->db, path.string());
me->conn = sqlite::open_db(path.c_str());
bts_assert_msg(me->conn, path.c_str());
const char *create_schema_sql =
"create table store ( ns text, key text, value text)";
sqlite::exec_nofail(me->conn, create_schema_sql);
const char *key_index_sql =
"create unique index key_index on store(ns,key)";
sqlite::exec_nofail(me->conn, key_index_sql);
const char *ns_index_sql = "create index ns_index on store(ns)";
sqlite::exec_nofail(me->conn, ns_index_sql);
} else {
me->conn = sqlite::open_db(path.c_str());
if (!path.empty()) {
if (!boost::filesystem::is_directory(path.parent_path())) {
bts_assert(boost::filesystem::create_directories(path.parent_path()));
}
if (!boost::filesystem::exists(path)) {
LOG(persist, Warn, "{} database does not exist yet. Creating it at {}",
me->db, path.string());
}
}
auto &conn = me->conn;
if (conn == nullptr) {
conn = sqlite::open_db(path.c_str());
bts_assert(conn != nullptr);
}
sqlite::exec_nofail(
conn,
"CREATE TABLE IF NOT EXISTS store (ns TEXT, key TEXT, value TEXT)");
sqlite::exec_nofail(
conn, "CREATE UNIQUE INDEX IF NOT EXISTS key_index on store(ns, key)");
sqlite::exec_nofail(
conn, "CREATE INDEX IF NOT EXISTS ns_index on store(ns)");
conn = sqlite::open_db(path.c_str());
// Asynchronous for speed
sqlite::exec_nofail(me->conn, "PRAGMA synchronous = 0;");
sqlite::exec_nofail(me->conn, "PRAGMA journal_mode=WAL");
sqlite::exec_nofail(conn, "PRAGMA synchronous = 0;");
sqlite::exec_nofail(conn, "PRAGMA journal_mode=WAL");
// prepare insert statement
const char *insert_sql = "insert into store (ns, key, value) values(?,?,?)";
me->insert = sqlite::prepare_query(me->conn, insert_sql);
me->insert = sqlite::prepare_query(conn, insert_sql);
// prepare insert_with_id statement
const char *insert_with_id_sql =
"insert into store (rowid, ns, key, value) values (?,?,?,?)";
me->insert_with_id = sqlite::prepare_query(conn, insert_with_id_sql);
// prepare update statement
const char *update_sql =
"update store set value = ? where ns = ? and key = ?";
me->update = sqlite::prepare_query(me->conn, update_sql);
me->update = sqlite::prepare_query(conn, update_sql);
// prepare remove statement
const char *remove_all_sql = "delete from store where ns = ?";
me->remove_all = sqlite::prepare_query(me->conn, remove_all_sql);
me->remove_all = sqlite::prepare_query(conn, remove_all_sql);
// prepare remove statement
const char *remove_sql = "delete from store where ns = ? and key = ?";
me->remove = sqlite::prepare_query(me->conn, remove_sql);
me->remove = sqlite::prepare_query(conn, remove_sql);
}
SqliteStore::~SqliteStore() {}
@@ -122,25 +152,27 @@ SqliteStore::Data SqliteStore::retrieve_all() {
return result;
}
std::pair<bool, std::string>
SqliteStore::retrieve_one(std::string const &ns, std::string const &k) {
std::tuple<bool, std::int64_t, std::string> SqliteStore::retrieve_one(
std::string const &ns, std::string const &k) {
auto query_str = fmt::format(
"select ns,key,value from store where ns='{}' and key='{}'", ns, k);
"select rowid,ns,key,value from store where ns='{}' and key='{}'", ns,
k);
sqlite3_stmt_ptr query = sqlite::prepare_query(me->conn, query_str.c_str());
std::pair<bool, std::string> result{false, ""};
auto result = std::make_tuple<bool, std::int64_t, std::string>(false, 0, "");
auto pq = query.get();
auto sqlite_result = sqlite3_step(pq);
if (sqlite_result == SQLITE_ROW) {
std::int64_t rowid = sqlite3_column_int64(pq, 0);
std::string ns(
(char *)sqlite3_column_text(pq, 0), sqlite3_column_bytes(pq, 0));
std::string key(
(char *)sqlite3_column_text(pq, 1), sqlite3_column_bytes(pq, 1));
std::string key(
(char *)sqlite3_column_text(pq, 2), sqlite3_column_bytes(pq, 2));
std::string val(
(char *)sqlite3_column_blob(pq, 2), sqlite3_column_bytes(pq, 2));
result.first = true;
result.second = std::move(val);
(char *)sqlite3_column_blob(pq, 3), sqlite3_column_bytes(pq, 3));
std::get<0>(result) = true;
std::get<1>(result) = rowid;
std::get<2>(result) = std::move(val);
}
return result;
}
@@ -149,91 +181,185 @@ void SqliteStore::remove(
std::string const &ns, std::set<std::string> const &keys) {
bts_assert(ns.size());
for (auto &k : keys) {
SQLITE_CHK(
sqlite3_bind_text(
me->remove.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT),
me->conn);
SQLITE_CHK(
sqlite3_bind_text(
me->remove.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT),
me->conn);
auto r = sqlite3_step(me->remove.get());
if (r != SQLITE_DONE) {
LOG(persist, Error,
"Could not delete row in store. Will retry once. error_code: {} "
"ns:{}, k: {}",
r, ns, k);
sleep_for(2s);
r = sqlite3_step(me->remove.get());
try {
for (auto &k : keys) {
SQLITE_CHK(
sqlite3_bind_text(
me->remove.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT),
me->conn);
SQLITE_CHK(
sqlite3_bind_text(
me->remove.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT),
me->conn);
auto r = sqlite3_step(me->remove.get());
if (r != SQLITE_DONE) {
LOG(persist, Error,
"Failed a second time to delete row. Bailing out. error_code: "
"{}",
r);
bts_assert(r == SQLITE_DONE);
"Could not delete row in store. Will retry once. error_code: "
"{} "
"ns:{}, k: {}",
r, ns, k);
sleep_for(2s);
r = sqlite3_step(me->remove.get());
if (r != SQLITE_DONE) {
LOG(persist, Error,
"Failed a second time to delete row. Bailing out. "
"error_code: "
"{}",
r);
bts_assert(r == SQLITE_DONE);
}
}
SQLITE_CHK(sqlite3_reset(me->remove.get()), me->conn);
}
SQLITE_CHK(sqlite3_reset(me->remove.get()), me->conn);
} catch (const std::exception &ex) {
LOG(persist, Error, "SqliteStore::remove threw. what: {}",
std::string{ex.what()});
}
}
void SqliteStore::remove_all(std::string const &ns) {
bts_assert(ns.size());
SQLITE_CHK(
sqlite3_bind_text(
me->remove_all.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT),
me->conn);
auto r = sqlite3_step(me->remove_all.get());
if (r != SQLITE_DONE) {
LOG(persist, Error,
"Could not delete all rows in store. Will retry once. error_code: {} "
"ns:{}",
r, ns);
sleep_for(2s);
r = sqlite3_step(me->remove_all.get());
try {
SQLITE_CHK(
sqlite3_bind_text(
me->remove_all.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT),
me->conn);
auto r = sqlite3_step(me->remove_all.get());
if (r != SQLITE_DONE) {
LOG(persist, Error,
"Failed a second time to delete all rows. Bailing out. "
"error_code: "
"{}",
r);
bts_assert(r == SQLITE_DONE);
"Could not delete all rows in store. Will retry once. error_code: "
"{} "
"ns:{}",
r, ns);
sleep_for(2s);
r = sqlite3_step(me->remove_all.get());
if (r != SQLITE_DONE) {
LOG(persist, Error,
"Failed a second time to delete all rows. Bailing out. "
"error_code: "
"{}",
r);
bts_assert(r == SQLITE_DONE);
}
}
SQLITE_CHK(sqlite3_reset(me->remove_all.get()), me->conn);
} catch (const std::exception &ex) {
LOG(persist, Error, "SqliteStore::remove_all threw. what: {}",
std::string{ex.what()});
}
SQLITE_CHK(sqlite3_reset(me->remove_all.get()), me->conn);
}
void SqliteStore::insert(
std::int64_t SqliteStore::insert(
std::string const &ns, std::string const &k, std::string const &v) {
SQLITE_CHK(
sqlite3_bind_text(
me->insert.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT),
me->conn);
SQLITE_CHK(
sqlite3_bind_text(
me->insert.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT),
me->conn);
SQLITE_CHK(
sqlite3_bind_text(
me->insert.get(), 3, v.c_str(), v.size(), SQLITE_TRANSIENT),
me->conn);
auto r = sqlite3_step(me->insert.get());
if (r != SQLITE_DONE) {
LOG(persist, Error,
"Could not insert row in store. Will retry once. error_code: {} ns: "
"{}, k: {}, v: {}",
r, ns, k, v);
sleep_for(2s);
r = sqlite3_step(me->insert.get());
try {
SQLITE_CHK(
sqlite3_bind_text(
me->insert.get(), 1, ns.c_str(), ns.size(), SQLITE_TRANSIENT),
me->conn);
SQLITE_CHK(
sqlite3_bind_text(
me->insert.get(), 2, k.c_str(), k.size(), SQLITE_TRANSIENT),
me->conn);
SQLITE_CHK(
sqlite3_bind_text(
me->insert.get(), 3, v.c_str(), v.size(), SQLITE_TRANSIENT),
me->conn);
auto r = sqlite3_step(me->insert.get());
if (r != SQLITE_DONE) {
LOG(persist, Error,
"Failed a second time to insert row. Bailing out. error_code: {}",
r);
bts_assert(r == SQLITE_DONE);
"Could not insert row in store. Will retry once. error_code: {} "
"ns: "
"{}, k: {}, v: {}",
r, ns, k, v);
sleep_for(2s);
r = sqlite3_step(me->insert.get());
if (r != SQLITE_DONE) {
LOG(persist, Error,
"Failed a second time to insert row. Bailing out. error_code: "
"{}",
r);
bts_assert(r == SQLITE_DONE);
}
}
const auto rowid = sqlite3_last_insert_rowid(me->conn.get());
SQLITE_CHK(sqlite3_reset(me->insert.get()), me->conn);
return rowid;
} catch (const std::exception &ex) {
LOG(persist, Error, "SqliteStore::insert threw. what: {}",
std::string{ex.what()});
}
SQLITE_CHK(sqlite3_reset(me->insert.get()), me->conn);
return 0;
}
void SqliteStore::insert_many(const std::vector<StoreRow> &rows) {
LOG(md, Info, "SqliteStore::insert_many size: {}", rows.size());
if (rows.empty()) {
return;
}
try {
sqlite::exec_nofail(me->conn.get(), "BEGIN TRANSACTION");
sqlite3 *conn_rawp = me->conn.get();
sqlite3_stmt *insert_rawp = me->insert.get();
sqlite3_stmt *insert_with_id_rawp = me->insert_with_id.get();
for (const auto &row : rows) {
int col_idx = 1;
sqlite3_stmt *stmt;
if (row.id > 0) {
stmt = insert_with_id_rawp;
sqlite3_bind_int(stmt, col_idx++, row.id);
} else {
stmt = insert_rawp;
}
sqlite3_bind_text(
stmt, col_idx++, row.ns.c_str(), row.ns.size(), SQLITE_STATIC);
sqlite3_bind_text(
stmt, col_idx, row.key.c_str(), row.key.size(), SQLITE_STATIC);
int result = sqlite3_step(stmt);
if (result == SQLITE_DONE || result == SQLITE_ROW) {
sqlite3_last_insert_rowid(conn_rawp);
}
sqlite3_reset(stmt);
}
sqlite::exec_nofail(me->conn.get(), "COMMIT TRANSACTION");
} catch (const std::exception &ex) {
LOG(persist, Error, "SqliteStore::insert_many threw. what: {}",
std::string{ex.what()});
}
}
void SqliteStore::insert_many_resolve_ids(std::vector<StoreRow> &rows) {
LOG(md, Info, "SqliteStore::insert_many_resolve_ids size: {}", rows.size());
if (rows.empty()) {
return;
}
sqlite::exec_nofail(me->conn.get(), "BEGIN TRANSACTION");
sqlite3 *conn_rawp = me->conn.get();
sqlite3_stmt *insert_rawp = me->insert.get();
sqlite3_stmt *insert_with_id_rawp = me->insert_with_id.get();
for (auto &row : rows) {
int col_idx = 1;
sqlite3_stmt *stmt;
if (row.id > 0) {
stmt = insert_with_id_rawp;
sqlite3_bind_int(stmt, col_idx++, row.id);
} else {
stmt = insert_rawp;
}
sqlite3_bind_text(
stmt, col_idx++, row.ns.c_str(), row.ns.size(), SQLITE_STATIC);
sqlite3_bind_text(
stmt, col_idx, row.key.c_str(), row.key.size(), SQLITE_STATIC);
int result = sqlite3_step(stmt);
if (result == SQLITE_DONE || result == SQLITE_ROW) {
if (row.id > 0) {
bts_assert(sqlite3_last_insert_rowid(conn_rawp) == row.id);
} else {
row.id = sqlite3_last_insert_rowid(conn_rawp);
}
}
sqlite3_reset(stmt);
}
sqlite::exec_nofail(me->conn.get(), "COMMIT TRANSACTION");
}
void SqliteStore::update(
+18 -6
View File
@@ -1,29 +1,39 @@
#pragma once
#include "IStore.h"
#include "lib/persistence/store_row.h"
#include "lib/utils/Sqlite_utils.h"
#include <cstdint>
#include <memory>
#include <vector>
namespace bts {
namespace persistence {
class SqliteStore : public IStore {
public:
SqliteStore(std::string const &storage_root, std::string const &db);
SqliteStore(std::string const &db);
explicit SqliteStore(sqlite3_ptr db);
~SqliteStore();
static SqliteStore from_path(const std::string &full_path);
typedef std::map<std::string, std::map<std::string, std::string>> Data;
Data retrieve_all() override;
std::pair<bool, std::string>
retrieve_one(std::string const &ns, std::string const &k) override;
void
remove(std::string const &ns, std::set<std::string> const &keys) override;
std::tuple<bool, std::int64_t, std::string> retrieve_one(
std::string const &ns, std::string const &k) override;
void remove(
std::string const &ns, std::set<std::string> const &keys) override;
void remove_all(std::string const &ns) override;
void insert(
std::int64_t insert(
std::string const &ns,
std::string const &k,
std::string const &v) override;
void insert_many(const std::vector<StoreRow> &rows);
void insert_many_resolve_ids(std::vector<StoreRow> &rows);
void update(
std::string const &ns,
std::string const &k,
@@ -33,6 +43,8 @@ class SqliteStore : public IStore {
static std::string storage_location(std::string const &db);
private:
SqliteStore();
struct PImpl;
std::unique_ptr<PImpl> me;
void init_db();
+20
View File
@@ -0,0 +1,20 @@
#pragma once
#include <cstdint>
#include <string>
namespace bts::persistence {
struct StoreRow {
std::string ns;
std::string key;
std::string value;
/**
* Represents the unique 'rowid' in the sqlite db. A value of 0 indicates that
* the rowid isn't known, and should either be generated by the db (for inserts),
* or resolved by finding the row with matching ns, key, value (for retrieves).
*/
std::int64_t id{0};
};
} // namespace bts::persistence
@@ -16,7 +16,8 @@ namespace persistence {
TEST_CASE("PersistenceService Sqlite storage", "[persistence]") {
utils::TempDir db_dir;
SqliteStore st(db_dir.name(), "test_db");
SqliteStore st(
(boost::filesystem::path{db_dir.name()} / "test_db.sqlite").string());
SECTION("initial_db_is_empty") { REQUIRE(st.retrieve_all().empty()); }
+37
View File
@@ -0,0 +1,37 @@
#pragma once
#include <cstdint>
#include <limits>
#include <stdexcept>
namespace bts::system::math {
static std::uint32_t checked_ltou(std::int64_t l);
static std::uint64_t checked_ltoul(std::int64_t l);
//////////////////////////////////////////////////
// Definition
//////////////////////////////////////////////////
inline std::uint32_t checked_ltou(std::int64_t l) {
if (l < 0) {
throw std::overflow_error{
"Failed to convert int64 to uint32 (value < 0)"};
}
if (l >
static_cast<std::int64_t>(std::numeric_limits<std::uint32_t>::max())) {
throw std::overflow_error{
"Failed to convert int64 to uint32. value > max)"};
}
return static_cast<std::uint32_t>(l);
}
inline std::uint64_t checked_ltoul(std::int64_t l) {
if (l < 0) {
throw std::overflow_error{
"Failed to convert int64 to uint64 (value < 0)"};
}
return static_cast<std::uint64_t>(l);
}
} // namespace bts::system::math
@@ -52,7 +52,7 @@ void add_or_update_to_table(
const std::string &ns,
const std::string &key,
const std::string &val) {
auto [exists, existing_val] = store->retrieve_one(ns, key);
auto [exists, rowid, existing_val] = store->retrieve_one(ns, key);
LOG(Gen, Info,
"add_or_update ns: {} key: {} val: {} exists: {} existing_val: {}", ns,
key, val, exists, existing_val);
@@ -386,7 +386,8 @@ void AnalyticsServiceServer::save_report(
const api::Response<::bts::api::ResultMessage> &resp) {
Report report = report_arg;
std::string key = std::to_string(report.id());
auto [present, val] = me->reports_store->retrieve_one(me->reports_ns, key);
auto [present, rowid, val] =
me->reports_store->retrieve_one(me->reports_ns, key);
if (!present) {
int new_id = get_max_id(me->reports_store.get(), me->reports_ns) + 1;
report.set_id(new_id);
@@ -408,7 +409,8 @@ void AnalyticsServiceServer::delete_report(
const bts::api::analytics::IntWrapper &id,
const api::Response<::bts::api::ResultMessage> &resp) {
std::string key = std::to_string(id.data());
auto [present, val] = me->reports_store->retrieve_one(me->reports_ns, key);
auto [present, rowid, val] =
me->reports_store->retrieve_one(me->reports_ns, key);
api::ResultMessage res;
res.set_success(false);
if (!present) {
@@ -581,10 +581,11 @@ void negate_id(api::Security *sec) {
}
void ExchangeSimApplication::persist_book(OrderBook const &b) {
auto existing = me->store->retrieve_one("books", b.sec_desc());
auto [exists, rowid, oldval] =
me->store->retrieve_one("books", b.sec_desc());
auto val = utils::pb::to_string(b);
if (existing.first)
if (exists)
me->store->update("books", b.sec_desc(), val);
else
me->store->insert("books", b.sec_desc(), val);
+36 -28
View File
@@ -246,21 +246,23 @@ void MarketDataGateway::unsubscribe(MarketDataListenerID id) {
//////////////////////////////////////////////////////////////////////////////
void MarketDataGateway::start_glbx(const bts::api::md::MarketDataConfig &cfg){
for (const auto &glbx : cfg.glbx()) {
if(glbx.use_databento()){
if(!glbx.use_databento()){
_start_onix_glbx(glbx);
return;
}
auto p = me_->venue_listeners.emplace_back(
std::make_shared<VenueListenerData>(bts::api::VENUE_GLBX));
// Capture `p->listener` and `me_->sec_id_store` by value
auto listener = p->listener;
auto sec_id_store = me_->sec_id_store;
std::string tid_name = "glbx";
bts::threading::make_thread(tid_name, [&]() {
bts::threading::make_thread(tid_name, [listener, sec_id_store, glbx](){
api::md::glbx::GlbxMarketDatahandler::run(
p->listener,
me_->sec_id_store,
listener,
sec_id_store,
glbx);
}).detach();
}
else{
_start_onix_glbx(glbx);
}
}
}
}
static void run_onixs() {
LOG(md, Info, "MarketDataGateway::run_onixs t_id:{} t_name",
@@ -295,7 +297,10 @@ void MarketDataGateway::_start_onix_glbx(
"MarketDataGateway::_start_onix_glbx starting t_id:{} t_name:{}. "
"cfg:\n{}",
os::get_thread_id(), os::get_thread_name(), cfg.DebugString());
bts::md::run_onixs();
bts::threading::make_thread("Onix CME", [&]() {
bts::md::run_onixs();
}).detach();
}
@@ -374,35 +379,38 @@ void MarketDataGateway::_start_replays(
void MarketDataGateway::start_taifex(const bts::api::md::MarketDataConfig &cfg) {
// taifex
int counter = 0;
for (const auto &taifex : cfg.taifex()) {
counter++;
auto p = me_->venue_listeners.emplace_back(
std::make_shared<VenueListenerData>(bts::api::VENUE_TAIFEX));
std::string tid_name = "taifex" +std::to_string(counter);
bts::threading::make_thread(tid_name, [&]() {
// Capture `p->listener` and `me_->sec_id_store` by value
auto listener = p->listener;
auto sec_id_store = me_->sec_id_store;
std::string tid_name = "taifex";
bts::threading::make_thread(tid_name, [listener, sec_id_store, taifex]() {
bts::md::taifex::TaifexDataGateway::run(
taifex, p->listener, me_->sec_id_store);
taifex, listener, sec_id_store);
}).detach();
}
}
void MarketDataGateway::start_cboe(const bts::api::md::MarketDataConfig &cfg){
// cboe
void MarketDataGateway::start_cboe(const bts::api::md::MarketDataConfig &cfg) {
for (const api::md::CboeMarketDataConfig &cboe : cfg.cboe()) {
auto p = me_->venue_listeners.emplace_back(
std::make_shared<VenueListenerData>(bts::api::VENUE_CBOE));
std::string tid_name = "CBOE";
bts::threading::make_thread(tid_name, [&]() {
api::md::cboe::CboeMarketDatahandler::run(
p->listener,
me_->sec_id_store,
cboe);
}).detach();
auto p = me_->venue_listeners.emplace_back(
std::make_shared<VenueListenerData>(bts::api::VENUE_CBOE));
std::string tid_name = "CBOE";
// Capture `p->listener` and `me_->sec_id_store` by value
auto listener = p->listener;
auto sec_id_store = me_->sec_id_store;
bts::threading::make_thread(tid_name, [listener, sec_id_store, cboe]() {
api::md::cboe::CboeMarketDatahandler::run(
listener,
sec_id_store,
cboe);
}).detach();
UNUSED(cboe);
}
}
} // namespace bts::md
@@ -10,20 +10,21 @@
namespace bts::md::binance {
using bts::api::domain::SecurityType;
using bts::domain::SecurityID;
using bts::domain::SecurityMarketDataStatistics;
using std::pair;
using std::string;
using std::tuple;
using std::vector;
#define VENUE_NAME (Venue_Name(Venue::BINANCE))
struct BinancePriceStatisticsAdaptor::PImpl {
vector<pair<string, SecurityID>> securities;
vector<tuple<SecurityType, string, SecurityID>> securities;
IMarketDataListener *listener;
SecurityID get_sec_id(const string &sec_desc) const {
for (const auto &[desc, id] : securities) {
for (const auto &[type, desc, id] : securities) {
if (desc == sec_desc) {
return id;
}
@@ -36,7 +37,8 @@ struct BinancePriceStatisticsAdaptor::PImpl {
};
BinancePriceStatisticsAdaptor::BinancePriceStatisticsAdaptor(
vector<pair<string, SecurityID>> securities, IMarketDataListener *listener)
vector<tuple<SecurityType, string, SecurityID>> securities,
IMarketDataListener *listener)
: me_(std::make_unique<PImpl>()) {
bts_assert(listener != nullptr);
me_->securities = std::move(securities);
@@ -1,9 +1,11 @@
#pragma once
#include "lib/api/domain/SecurityType.pb.h"
#include "lib/domain/SecurityID.h"
#include "server/sparkd/md/ccapi/ICCApiMessageHandler.h"
#include <string>
#include <tuple>
#include <utility>
#include <vector>
@@ -20,7 +22,10 @@ namespace bts::md::binance {
class BinancePriceStatisticsAdaptor : public ICCApiMessageHandler {
public:
BinancePriceStatisticsAdaptor(
std::vector<std::pair<std::string, domain::SecurityID>> securities,
std::vector<std::tuple<
api::domain::SecurityType,
std::string,
domain::SecurityID>> securities,
IMarketDataListener *listener);
~BinancePriceStatisticsAdaptor();
@@ -49,6 +49,7 @@ using std::vector;
#define _WARN(...) _LOG(Warn, __VA_ARGS__)
#define _ERROR(...) _LOG(Error, __VA_ARGS__)
#define VENUE Venue::VENUE_BINANCE
#define VENUE_NAME (Venue_Name(Venue::VENUE_BINANCE))
#define CCAPI_VENUE_NAME CCAPI_EXCHANGE_NAME_BINANCE
#define BINANCE_MSG_TYPE_NAME(x) \
@@ -156,7 +157,8 @@ struct BinanceSecurityRequestor::PImpl {
for (auto &sec : added_secs) {
// Since the ids haven't been set yet, we should expect them to be 0
bts_assert(sec.id().value() == 0);
SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec.desc())};
SecurityID sec_id{
sec_id_store->get_or_add(VENUE, sec.type(), sec.desc())};
sec.security_common.id = sec_id;
}
}
@@ -34,6 +34,7 @@
namespace bts::md::binance {
using bts::api::domain::SecurityType;
using bts::api::domain::Venue;
using bts::api::domain::Venue_Name;
using bts::api::md::BinanceMarketDataConfig;
@@ -50,6 +51,7 @@ using std::unique_ptr;
using std::unordered_map;
using std::vector;
#define VENUE Venue::VENUE_BINANCE
#define VENUE_NAME (Venue_Name(Venue::VENUE_BINANCE))
#define CCAPI_VENUE_NAME CCAPI_EXCHANGE_NAME_BINANCE
@@ -123,14 +125,15 @@ static CCApiCorrelationID mk_stats_corr_id(const string &desc) {
}
static ccapi::Request mk_stats_req(
const vector<string> &sec_descs, uint32_t batch_id) {
const vector<std::pair<SecurityType, string>> &sec_descs,
uint32_t batch_id) {
const auto corr_id = mk_stats_corr_id(FMT("BATCH_{}", batch_id));
ccapi::Request req{
ccapi::Request::Operation::GENERIC_PUBLIC_REQUEST, CCAPI_VENUE_NAME};
req.setCorrelationId(corr_id.to_string());
auto quoted_sec_descs =
sec_descs |
ranges::views::transform([](const auto &s) { return "\"" + s + "\""; });
sec_descs | ranges::views::transform(
[](const auto &s) { return "\"" + s.second + "\""; });
const string symbols_query_arg = FMT_JOIN(quoted_sec_descs, ",");
req.appendParam(
{{CCAPI_HTTP_PATH, BINANCE_PRICE_STATS_PATH},
@@ -209,16 +212,18 @@ struct BinanceSubscription::PImpl {
sec_requestor.request();
}
DepthHandler &mk_depth_handler(const string &sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
DepthHandler &mk_depth_handler(
SecurityType sec_type, const string &sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
auto [it, is_new] =
depth_handlers.try_emplace(sec_desc, sec_id, listener);
bts_assert(is_new);
return it->second;
}
TickerHandler &mk_ticker_handler(const string &sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
TickerHandler &mk_ticker_handler(
SecurityType sec_type, const string &sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
auto [it, is_new] =
ticker_handlers.try_emplace(sec_desc, sec_id, listener);
bts_assert_msgf(
@@ -227,23 +232,25 @@ struct BinanceSubscription::PImpl {
}
StatsHandler &mk_stats_handler(
const vector<string> &sec_descs, uint32_t hnd_id) {
vector<pair<string, SecurityID>> sec_desc_id_pairs;
sec_desc_id_pairs.reserve(sec_descs.size());
for (const auto &sec_desc : sec_descs) {
sec_desc_id_pairs.emplace_back(
sec_desc,
SecurityID{sec_id_store->get_or_add(VENUE_NAME, sec_desc)});
const vector<std::pair<SecurityType, string>> &sec_descs,
uint32_t hnd_id) {
vector<std::tuple<SecurityType, string, SecurityID>> sec_desc_id_tuples;
sec_desc_id_tuples.reserve(sec_descs.size());
for (auto [type, name] : sec_descs) {
SecurityID id{sec_id_store->get_or_add(VENUE, type, name)};
sec_desc_id_tuples.emplace_back(type, std::move(name), id);
}
auto [it, is_new] = stats_handlers.try_emplace(
hnd_id, std::make_unique<StatsHandler>(sec_desc_id_pairs, listener));
hnd_id,
std::make_unique<StatsHandler>(sec_desc_id_tuples, listener));
bts_assert_msgf(
is_new, "StatsHandler already created with id '{}'", hnd_id);
return *it->second;
}
void add_stats_handler(
const vector<string> &sec_descs, CCApiSession::BatchRequest &reqs) {
const vector<std::pair<SecurityType, string>> &sec_descs,
CCApiSession::BatchRequest &reqs) {
if (sec_descs.empty()) {
return;
}
@@ -268,14 +275,14 @@ struct BinanceSubscription::PImpl {
CCApiSession::BatchRequest stats_reqs;
const int N_SECS_PER_STATS_REQ = 16;
vector<string> curr_stats_sec_descs;
for (const auto &sec_desc : securities) {
vector<std::pair<SecurityType, string>> curr_stats_sec_descs;
for (const auto &[type, desc] : securities) {
depth_subs.push_back(
{mk_depth_sub(sec_desc, depth_opts), &mk_depth_handler(sec_desc)});
{mk_depth_sub(desc, depth_opts), &mk_depth_handler(type, desc)});
ticker_subs.push_back(
{mk_ticker_sub(sec_desc, ticker_opts),
&mk_ticker_handler(sec_desc)});
curr_stats_sec_descs.push_back(sec_desc);
{mk_ticker_sub(desc, ticker_opts),
&mk_ticker_handler(type, desc)});
curr_stats_sec_descs.push_back(std::make_pair(type, desc));
if (curr_stats_sec_descs.size() == N_SECS_PER_STATS_REQ) {
add_stats_handler(curr_stats_sec_descs, stats_reqs);
curr_stats_sec_descs.clear();
@@ -299,7 +306,7 @@ struct BinanceSubscription::PImpl {
*/
void on_securities(vector<Security> secs) {
// 'sec_descs' is used to make depth subscriptions
vector<string> sec_descs;
vector<pair<SecurityType, string>> sec_descs;
sec_descs.reserve(secs.size());
// 'sec_ptrs' is passed to IMarketDataListeners
vector<const Security *> sec_ptrs;
@@ -312,7 +319,8 @@ struct BinanceSubscription::PImpl {
id, std::make_unique<Security>(std::move(s)));
bts_assert_msgf(
is_new, "Received duplicate Binance security '{}'", desc);
sec_descs.push_back(std::move(desc));
sec_descs.push_back(
std::make_pair(it->second->type(), std::move(desc)));
// add ptr to entry in 'securities' (not the loop var, we moved that)
sec_ptrs.push_back(it->second.get());
}
@@ -10,20 +10,21 @@
namespace bts::md::binance_us {
using bts::api::domain::SecurityType;
using bts::domain::SecurityID;
using bts::domain::SecurityMarketDataStatistics;
using std::pair;
using std::string;
using std::tuple;
using std::vector;
#define VENUE_NAME (Venue_Name(Venue::BINANCE))
struct BinanceUsPriceStatisticsAdaptor::PImpl {
vector<pair<string, SecurityID>> securities;
vector<tuple<SecurityType, string, SecurityID>> securities;
IMarketDataListener *listener;
SecurityID get_sec_id(const string &sec_desc) const {
for (const auto &[desc, id] : securities) {
for (const auto &[type, desc, id] : securities) {
if (desc == sec_desc) {
return id;
}
@@ -36,7 +37,8 @@ struct BinanceUsPriceStatisticsAdaptor::PImpl {
};
BinanceUsPriceStatisticsAdaptor::BinanceUsPriceStatisticsAdaptor(
vector<pair<string, SecurityID>> securities, IMarketDataListener *listener)
vector<tuple<SecurityType, string, SecurityID>> securities,
IMarketDataListener *listener)
: me_(std::make_unique<PImpl>()) {
bts_assert(listener != nullptr);
me_->securities = std::move(securities);
@@ -1,9 +1,11 @@
#pragma once
#include "lib/api/domain/SecurityType.pb.h"
#include "lib/domain/SecurityID.h"
#include "server/sparkd/md/ccapi/ICCApiMessageHandler.h"
#include <string>
#include <tuple>
#include <utility>
#include <vector>
@@ -25,7 +27,10 @@ namespace bts::md::binance_us {
class BinanceUsPriceStatisticsAdaptor : public ICCApiMessageHandler {
public:
BinanceUsPriceStatisticsAdaptor(
std::vector<std::pair<std::string, domain::SecurityID>> securities,
std::vector<std::tuple<
api::domain::SecurityType,
std::string,
domain::SecurityID>> securities,
IMarketDataListener *listener);
~BinanceUsPriceStatisticsAdaptor();
@@ -156,7 +156,8 @@ struct BinanceUsSecurityRequestor::PImpl {
for (auto &sec : added_secs) {
// Since the ids haven't been set yet, we should expect them to be 0
bts_assert(sec.id().value() == 0);
SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec.desc())};
SecurityID sec_id{
sec_id_store->get_or_add(VENUE, sec.type(), sec.desc())};
sec.security_common.id = sec_id;
}
}
@@ -36,6 +36,7 @@
namespace bts::md::binance_us {
using bts::api::domain::SecurityType;
using bts::api::domain::Venue;
using bts::api::domain::Venue_Name;
using bts::api::md::BinanceUsMarketDataConfig;
@@ -47,11 +48,13 @@ using bts::io::QueryParams;
using bts::persistence::IDGenerator;
using std::pair;
using std::string;
using std::tuple;
using std::uint32_t;
using std::unique_ptr;
using std::unordered_map;
using std::vector;
#define VENUE Venue::VENUE_BINANCE_US
#define VENUE_NAME (Venue_Name(Venue::VENUE_BINANCE_US))
#define CCAPI_VENUE_NAME CCAPI_EXCHANGE_NAME_BINANCE_US
@@ -126,14 +129,14 @@ static CCApiCorrelationID mk_stats_corr_id(const string &desc) {
}
static ccapi::Request mk_stats_req(
const vector<string> &sec_descs, uint32_t batch_id) {
const vector<pair<SecurityType, string>> &sec_descs, uint32_t batch_id) {
const auto corr_id = mk_stats_corr_id(FMT("BATCH_{}", batch_id));
ccapi::Request req{
ccapi::Request::Operation::GENERIC_PUBLIC_REQUEST, CCAPI_VENUE_NAME};
req.setCorrelationId(corr_id.to_string());
auto quoted_sec_descs =
sec_descs |
ranges::views::transform([](const auto &s) { return "\"" + s + "\""; });
sec_descs | ranges::views::transform(
[](const auto &s) { return "\"" + s.second + "\""; });
const string symbols_query_arg = FMT_JOIN(quoted_sec_descs, ",");
req.appendParam(
{{CCAPI_HTTP_PATH, BINANCE_US_PRICE_STATS_PATH},
@@ -212,16 +215,18 @@ struct BinanceUsSubscription::PImpl {
sec_requestor.request();
}
DepthHandler &mk_depth_handler(const string &sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
DepthHandler &mk_depth_handler(
SecurityType sec_type, const string &sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
auto [it, is_new] =
depth_handlers.try_emplace(sec_desc, sec_id, listener);
bts_assert(is_new);
return it->second;
}
TickerHandler &mk_ticker_handler(const string &sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
TickerHandler &mk_ticker_handler(
SecurityType sec_type, const string &sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
auto [it, is_new] =
ticker_handlers.try_emplace(sec_desc, sec_id, listener);
bts_assert_msgf(
@@ -230,23 +235,25 @@ struct BinanceUsSubscription::PImpl {
}
StatsHandler &mk_stats_handler(
const vector<string> &sec_descs, uint32_t hnd_id) {
vector<pair<string, SecurityID>> sec_desc_id_pairs;
sec_desc_id_pairs.reserve(sec_descs.size());
for (const auto &sec_desc : sec_descs) {
sec_desc_id_pairs.emplace_back(
sec_desc,
SecurityID{sec_id_store->get_or_add(VENUE_NAME, sec_desc)});
const vector<std::pair<SecurityType, string>> &sec_descs,
uint32_t hnd_id) {
vector<tuple<SecurityType, string, SecurityID>> sec_desc_id_tuples;
sec_desc_id_tuples.reserve(sec_descs.size());
for (auto [type, name] : sec_descs) {
SecurityID id{sec_id_store->get_or_add(VENUE, type, name)};
sec_desc_id_tuples.emplace_back(type, std::move(name), id);
}
auto [it, is_new] = stats_handlers.try_emplace(
hnd_id, std::make_unique<StatsHandler>(sec_desc_id_pairs, listener));
hnd_id,
std::make_unique<StatsHandler>(sec_desc_id_tuples, listener));
bts_assert_msgf(
is_new, "StatsHandler already created with id '{}'", hnd_id);
return *it->second;
}
void add_stats_handler(
const vector<string> &sec_descs, CCApiSession::BatchRequest &reqs) {
const vector<pair<SecurityType, string>> &sec_descs,
CCApiSession::BatchRequest &reqs) {
if (sec_descs.empty()) {
return;
}
@@ -271,14 +278,14 @@ struct BinanceUsSubscription::PImpl {
CCApiSession::BatchRequest stats_reqs;
const int N_SECS_PER_STATS_REQ = 16;
vector<string> curr_stats_sec_descs;
for (const auto &sec_desc : securities) {
vector<pair<SecurityType, string>> curr_stats_sec_descs;
for (const auto &[type, name] : securities) {
depth_subs.push_back(
{mk_depth_sub(sec_desc, depth_opts), &mk_depth_handler(sec_desc)});
{mk_depth_sub(name, depth_opts), &mk_depth_handler(type, name)});
ticker_subs.push_back(
{mk_ticker_sub(sec_desc, ticker_opts),
&mk_ticker_handler(sec_desc)});
curr_stats_sec_descs.push_back(sec_desc);
{mk_ticker_sub(name, ticker_opts),
&mk_ticker_handler(type, name)});
curr_stats_sec_descs.push_back(std::make_pair(type, name));
if (curr_stats_sec_descs.size() == N_SECS_PER_STATS_REQ) {
add_stats_handler(curr_stats_sec_descs, stats_reqs);
curr_stats_sec_descs.clear();
@@ -302,7 +309,7 @@ struct BinanceUsSubscription::PImpl {
*/
void on_securities(vector<Security> secs) {
// 'sec_descs' is used to make depth subscriptions
vector<string> sec_descs;
vector<pair<SecurityType, string>> sec_descs;
sec_descs.reserve(secs.size());
// 'sec_ptrs' is passed to IMarketDataListeners
vector<const Security *> sec_ptrs;
@@ -315,7 +322,7 @@ struct BinanceUsSubscription::PImpl {
id, std::make_unique<Security>(std::move(s)));
bts_assert_msgf(
is_new, "Received duplicate Binance US security '{}'", desc);
sec_descs.push_back(std::move(desc));
sec_descs.push_back(std::make_pair(s.type(), std::move(desc)));
// add ptr to entry in 'securities' (not the loop var, we moved that)
sec_ptrs.push_back(it->second.get());
}
@@ -48,6 +48,7 @@ using std::vector;
#define _WARN(...) _LOG(Warn, __VA_ARGS__)
#define _ERROR(...) _LOG(Error, __VA_ARGS__)
#define VENUE Venue::VENUE_BINANCE_USDS_FUTURES
#define VENUE_NAME (Venue_Name(Venue::VENUE_BINANCE_USDS_FUTURES))
#define CCAPI_VENUE_NAME CCAPI_EXCHANGE_NAME_BINANCE_USDS_FUTURES
#define VENUE_PRICE_STATS_PATH BINANCE_USDS_FUTURES_PRICE_STATS_PATH
@@ -157,7 +158,8 @@ struct BinanceUsdsFuturesSecurityRequestor::PImpl {
// the ids, if it does (for whatever reason), better to assert/crash
// than quietly let a potential bug slip through the cracks.
bts_assert(sec.id().value() == 0);
SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec.desc())};
SecurityID sec_id{
sec_id_store->get_or_add(VENUE, sec.type(), sec.desc())};
sec.security_common.id = sec_id;
}
}
@@ -30,6 +30,7 @@
namespace bts::md::binance_usds_futures {
using bts::api::domain::SecurityType;
using bts::api::domain::Venue;
using bts::api::domain::Venue_Name;
using bts::api::md::BinanceUsdsFuturesMarketDataConfig;
@@ -39,12 +40,14 @@ using bts::domain::SecurityID;
using bts::io::ProtoPrinter;
using bts::io::QueryParams;
using bts::persistence::IDGenerator;
using std::pair;
using std::string;
using std::uint32_t;
using std::unique_ptr;
using std::unordered_map;
using std::vector;
#define VENUE Venue::VENUE_BINANCE_USDS_FUTURES
#define VENUE_NAME (Venue_Name(Venue::VENUE_BINANCE_USDS_FUTURES))
#define CCAPI_VENUE_NAME CCAPI_EXCHANGE_NAME_BINANCE_USDS_FUTURES
#define VENUE_PRICE_STATS_PATH BINANCE_USDS_FUTURES_PRICE_STATS_PATH
@@ -192,8 +195,8 @@ struct BinanceUsdsFuturesSubscription::PImpl {
sec_requestor.request();
}
SecurityID get_sec_id(const string &sec_desc) {
return SecurityID{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
SecurityID get_sec_id(SecurityType sec_type, const string &sec_desc) {
return SecurityID{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
}
/**
@@ -211,8 +214,9 @@ struct BinanceUsdsFuturesSubscription::PImpl {
return sec_desc;
}
DepthHandler &add_depth_handler(const string &sec_desc) {
const auto sec_id = get_sec_id(sec_desc);
DepthHandler &add_depth_handler(
SecurityType sec_type, const string &sec_desc) {
const auto sec_id = get_sec_id(sec_type, sec_desc);
auto [it, is_new] =
depth_handlers.try_emplace(sec_desc, sec_id, listener);
bts_assert_msgf(
@@ -220,8 +224,9 @@ struct BinanceUsdsFuturesSubscription::PImpl {
return it->second;
}
TickerHandler &add_ticker_handler(const string &sec_desc) {
const auto sec_id = get_sec_id(sec_desc);
TickerHandler &add_ticker_handler(
SecurityType sec_type, const string &sec_desc) {
const auto sec_id = get_sec_id(sec_type, sec_desc);
auto [it, is_new] =
ticker_handlers.try_emplace(sec_desc, sec_id, listener);
bts_assert_msgf(
@@ -229,8 +234,9 @@ struct BinanceUsdsFuturesSubscription::PImpl {
return it->second;
}
StatsHandler &add_stats_handler(const string &sec_desc) {
const auto sec_id = get_sec_id(sec_desc);
StatsHandler &add_stats_handler(
SecurityType sec_type, const string &sec_desc) {
const auto sec_id = get_sec_id(sec_type, sec_desc);
auto [it, is_new] =
stats_handlers.try_emplace(sec_desc, sec_id, listener);
bts_assert_msgf(
@@ -252,16 +258,16 @@ struct BinanceUsdsFuturesSubscription::PImpl {
CCApiSession::BatchSubscription ticker_subs;
CCApiSession::BatchRequest stats_reqs;
for (const auto &sec_desc : securities) {
const auto sec_desc_for_reqs = get_sec_desc_for_reqs(sec_desc);
for (auto [type, name] : securities) {
const auto sec_desc_for_reqs = get_sec_desc_for_reqs(name);
depth_subs.push_back(
{mk_depth_sub(sec_desc_for_reqs, depth_opts),
&add_depth_handler(sec_desc)});
&add_depth_handler(type, name)});
ticker_subs.push_back(
{mk_ticker_sub(sec_desc_for_reqs, ticker_opts),
&add_ticker_handler(sec_desc)});
&add_ticker_handler(type, name)});
stats_reqs.push_back(
{mk_stats_req(sec_desc), &add_stats_handler(sec_desc)});
{mk_stats_req(name), &add_stats_handler(type, name)});
}
session.subscribe(
@@ -278,7 +284,7 @@ struct BinanceUsdsFuturesSubscription::PImpl {
*/
void on_securities(vector<Security> secs) {
// 'sec_descs' is used to make depth subscriptions
vector<string> sec_descs;
vector<pair<SecurityType, string>> sec_descs;
sec_descs.reserve(secs.size());
// 'sec_ptrs' is passed to IMarketDataListeners
vector<const Security *> sec_ptrs;
@@ -291,7 +297,7 @@ struct BinanceUsdsFuturesSubscription::PImpl {
id, std::make_unique<Security>(std::move(s)));
bts_assert_msgf(
is_new, "Received duplicate Binance USDS Future '{}'", desc);
sec_descs.push_back(std::move(desc));
sec_descs.push_back(std::make_pair(s.type(), std::move(desc)));
// add ptr to entry in 'securities' (not the loop var, we moved that)
sec_ptrs.push_back(it->second.get());
}
@@ -30,7 +30,8 @@ void CboeMarketDatahandler::run(
base_path + config.databento_settings().bbo_file_name(),
config.databento_settings().key(),
config.databento_settings().dataset(),
config.databento_settings().filter_securities_one_year());
config.databento_settings().filter_securities_one_year(),
config.databento_settings().load_reference_data());
}
} // namespace bts::md::cboe
@@ -147,7 +147,8 @@ struct CoinbaseSecurityRequestor::PImpl {
for (auto &sec : added_secs) {
// Since the ids haven't been set yet, we should expect them to be 0
bts_assert(sec.id().value() == 0);
SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec.desc())};
SecurityID sec_id{
sec_id_store->get_or_add(VENUE, sec.type(), sec.desc())};
sec.security_common.id = sec_id;
}
}
@@ -25,6 +25,9 @@
#include <unordered_map>
namespace bts::md::coinbase {
using bts::api::domain::SecurityType;
using bts::api::domain::Venue;
using bts::api::md::CoinbaseMarketDataConfig;
using bts::domain::Security;
using bts::domain::SecurityCriteria;
@@ -32,10 +35,13 @@ using bts::domain::SecurityID;
using bts::io::ProtoPrinter;
using bts::io::QueryParams;
using bts::persistence::IDGenerator;
using std::pair;
using std::string;
using std::unique_ptr;
using std::unordered_map;
using std::vector;
#define VENUE Venue::VENUE_COINBASE
#define VENUE_NAME (Venue_Name(Venue::VENUE_COINBASE))
#define CCAPI_VENUE_NAME CCAPI_EXCHANGE_NAME_COINBASE
@@ -183,24 +189,29 @@ struct CoinbaseSubscription::PImpl {
sec_requestor.callback().bind<&PImpl::on_securities>(*this);
sec_requestor.request();
}
StatsHandler &mk_stats_handler(const string &sec_desc) {
auto sec_id = SecurityID{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
StatsHandler &mk_stats_handler(
SecurityType sec_type, const string &sec_desc) {
auto sec_id =
SecurityID{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
auto [it, is_new] =
stats_handlers.try_emplace(sec_desc, sec_id, listener);
bts_assert_msgf(
is_new, "StatsHandler already created with id '{}'", sec_id);
return it->second;
}
DepthHandler &mk_depth_handler(const string &sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
DepthHandler &mk_depth_handler(
SecurityType sec_type, const string &sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
auto [it, is_new] =
depth_handlers.try_emplace(sec_desc, sec_id, listener);
bts_assert(is_new);
return it->second;
}
TickerHandler &mk_ticker_handler(const string &sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
TickerHandler &mk_ticker_handler(
SecurityType sec_type, const string &sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
auto [it, is_new] =
ticker_handlers.try_emplace(sec_desc, sec_id, listener);
bts_assert_msgf(
@@ -222,14 +233,15 @@ struct CoinbaseSubscription::PImpl {
CCApiSession::BatchRequest stats_reqs;
vector<string> curr_stats_sec_descs;
for (const auto &sec_desc : securities) {
for (const auto &[sec_type, sec_desc] : securities) {
depth_subs.push_back(
{mk_depth_sub(sec_desc, depth_opts), &mk_depth_handler(sec_desc)});
{mk_depth_sub(sec_desc, depth_opts),
&mk_depth_handler(sec_type, sec_desc)});
ticker_subs.push_back(
{mk_ticker_sub(sec_desc, ticker_opts),
&mk_ticker_handler(sec_desc)});
&mk_ticker_handler(sec_type, sec_desc)});
stats_reqs.push_back(
{mk_stats_req(sec_desc), &mk_stats_handler(sec_desc)});
{mk_stats_req(sec_desc), &mk_stats_handler(sec_type, sec_desc)});
}
session.subscribe(
@@ -247,7 +259,7 @@ struct CoinbaseSubscription::PImpl {
void on_securities(std::vector<Security> secs) {
LOG(md, Info, "on sec {}", secs.size());
// 'sec_descs' is used to make depth subscriptions
vector<string> sec_descs;
vector<pair<SecurityType, string>> sec_descs;
sec_descs.reserve(secs.size());
// 'sec_ptrs' is passed to IMarketDataListeners
vector<const Security *> sec_ptrs;
@@ -260,7 +272,7 @@ struct CoinbaseSubscription::PImpl {
id, std::make_unique<Security>(std::move(s)));
bts_assert_msgf(
is_new, "Received duplicate Coinbase security '{}'", desc);
sec_descs.push_back(std::move(desc));
sec_descs.push_back(std::make_pair(s.type(), std::move(desc)));
// add ptr to entry in 'securities' (not the loop var, we moved that)
sec_ptrs.push_back(it->second.get());
}
@@ -25,18 +25,36 @@
#include <databento/live.hpp>
#include <optional>
#include <string>
#include <vector>
namespace bts::md::DatabentoHandler {
std::string remove_spaces(const std::string_view& str) {
std::string result;
result.reserve(str.size()); // Reserve space for better performance
for (char c : str) {
if (!isspace(c)) {
result += c;
}
}
return result;
#define SEC_TYPE(name) bts::api::domain::SECURITY_TYPE_##name
using bts::domain::SecurityID;
using bts::api::domain::CallPut;
using bts::api::domain::SecurityType;
using bts::api::domain::Venue;
std::string remove_spaces(const std::string_view &str) {
std::string result;
result.reserve(str.size()); // Reserve space for better performance
for (char c : str) {
if (!isspace(c)) {
result += c;
}
}
return result;
}
static SecurityType parse_sec_type(std::string_view str) {
if (str.find("OPT") != std::string::npos) {
return SEC_TYPE(OPTION);
}
if (str.find("FUT") != std::string::npos) {
return SEC_TYPE(FUTURE);
}
return SEC_TYPE(UNSPECIFIED);
}
void DatabentoDatahandler::run(
@@ -49,8 +67,10 @@ void DatabentoDatahandler::run(
const std::string &key,
const std::string &dataset,
const bool filter_securities,
const std::chrono::duration<long double> sleep_duration) {
const bool load_reference_data,
const std::chrono::duration<long double> sleep_duration ) {
(void)filter_securities;
auto sleep_duration_updated = sleep_duration;
if (!use_file) {
@@ -77,22 +97,9 @@ void DatabentoDatahandler::run(
bts_assert_msg(false, "Exchange not supported on databento");
return databento::KeepGoing::Stop;
}
if(filter_securities){
const auto exp_nanos = msg.expiration;
std::string exp_string = databento::ToIso8601(exp_nanos);
const auto expiration_time = bts::core::TimeStamp::from_iso_string(exp_string);
if(expiration_time.seconds() < expiration_time.now().seconds() || expiration_time.seconds() > (expiration_time.now().seconds() + 60*60*24*365)){
return databento::KeepGoing::Continue;
}
}
bts::domain::SecurityID sec_id{sec_id_store->get_or_add(
venue, std::to_string(msg.hd.instrument_id))};
bts::domain::Security sec;
sec.security_common = domain::SecurityCommon();
sec.security_common.venue = venue;
sec.security_common.id = sec_id;
sec.security_common.exchange_sec_id = msg.hd.instrument_id;
sec.security_common.desc = msg.RawSymbol();
sec.security_common.symbol = msg.Asset();
@@ -104,38 +111,55 @@ void DatabentoDatahandler::run(
sec.security_common.currency = {msg.currency.data(), msg.currency.size()};
// sec.security_common.cpp = msg.hd.
const std::string_view sec_type = {msg.security_type.data(), msg.security_type.size()};
const SecurityType sec_type =
parse_sec_type({msg.security_type.data(), msg.security_type.size()});
const auto exp_nanos = msg.expiration;
std::string exp_string = databento::ToIso8601(exp_nanos);
const auto expiration_time = bts::core::TimeStamp::from_iso_string(exp_string);
if (sec_type.find("OPT") != std::string::npos) {
auto call_put = api::domain::CallPut::PUT;
if(msg.instrument_class == 'C' ){
call_put=api::domain::CallPut::CALL;
}
else{
bts_assert(msg.instrument_class == 'P');
}
sec.security.emplace<bts::domain::OptionSecurity>(
expiration_time, bts::domain::Price(msg.strike_price, -9),call_put);
} else if (sec_type.find("FUT") != std::string::npos) {
sec.security.emplace<bts::domain::FutureSecurity>(expiration_time);
} else {
bts_assert_msg(false, "Only futures and options supported on databento");
return databento::KeepGoing::Stop;
}
switch (sec_type) {
case SEC_TYPE(FUTURE):
sec.security.emplace<bts::domain::FutureSecurity>(expiration_time);
break;
case SEC_TYPE(OPTION): {
auto call_put = CallPut::PUT;
if (msg.instrument_class == 'C') {
call_put = CallPut::CALL;
} else {
bts_assert(msg.instrument_class == 'P');
}
sec.security.emplace<bts::domain::OptionSecurity>(
expiration_time, bts::domain::Price(msg.strike_price, -9),
call_put);
break;
}
default:
bts_assert_msg(
false, "Only futures and options supported on databento");
return databento::KeepGoing::Stop;
}
*sec.security_common.id = sec_id_store->get_or_add(
venue, sec_type, std::to_string(msg.hd.instrument_id));
listener->on_securities({&sec});
return databento::KeepGoing::Continue;
};
auto process_bbo = [&](const databento::Record &record) {
const auto &msg = record.Get<databento::Mbp1Msg>();
auto sec_id_opt = sec_id_store->get_id(
venue, std::to_string(msg.hd.instrument_id), false);
if (!sec_id_opt.has_value())
return databento::KeepGoing::Continue;
// The security type to is needed to get the id. Since the sec type isn't known,
// check for an id using both types (OPTION and FUTURE). This is safe to do since
// options and futures never have common names (instrument ids).
const auto name = std::to_string(msg.hd.instrument_id);
auto sec_id_opt =
sec_id_store->get_id(venue, SEC_TYPE(OPTION), name, false);
if (!sec_id_opt.has_value()) {
sec_id_opt =
sec_id_store->get_id(venue, SEC_TYPE(FUTURE), name, false);
if (!sec_id_opt.has_value()) {
return databento::KeepGoing::Continue;
}
}
sleep_for(sleep_duration_updated);
bts::domain::SecurityID sec_id = bts::domain::SecurityID(sec_id_opt.value());
@@ -181,19 +205,16 @@ void DatabentoDatahandler::run(
{}
};
//hardcoding for now will update later
std::vector<std::string> symbols;
switch (venue) {
case api::domain::VENUE_CBOE:
client.Subscribe({"SPX.OPT","SPXW.OPT"}, databento::Schema::Definition, databento::SType::Parent,databento::UnixNanos{});
client.Subscribe({"SPX.OPT","SPXW.OPT"}, databento::Schema::Mbp1, databento::SType::Parent);
break;
case api::domain::VENUE_GLBX:
client.Subscribe({"ES.FUT"}, databento::Schema::Definition, databento::SType::Parent);
client.Subscribe({"ES.FUT"}, databento::Schema::Mbp1, databento::SType::Parent);
break;
default:
bts_assert_msg(false, "ONLY GLBX AND CBOE SUPPORTED FOR NOW");
case api::domain::VENUE_CBOE: symbols= {"SPX.OPT","SPXW.OPT"}; break;
case api::domain::VENUE_GLBX: symbols= {"ES.FUT"}; break;
default:bts_assert_msg(false, "ONLY GLBX AND CBOE SUPPORTED FOR NOW");
}
if(load_reference_data)
client.Subscribe(symbols, databento::Schema::Definition, databento::SType::Parent,databento::UnixNanos{});
else client.Subscribe(symbols, databento::Schema::Definition, databento::SType::Parent);
client.Subscribe(symbols, databento::Schema::Mbp1, databento::SType::Parent);
client.Start();
while (true) {
try {
@@ -15,6 +15,7 @@ class DatabentoDatahandler {
const std::string &dataset="",
const std::string& key = "db-Q4eCbTQfrAHaG6sAcuFUYB9YqkF6j",
const bool filter_securities=false,
const bool load_reference_data = true,
const std::chrono::duration<long double> sleep_duration = 0.04s);
};
} // namespace bts::md::databento
@@ -19,6 +19,7 @@
namespace bts::md::deribit {
using bts::api::domain::CallPut;
using bts::api::domain::SecurityType;
using bts::domain::FutureSecurity;
using bts::domain::OptionSecurity;
using bts::domain::Price;
@@ -106,8 +107,8 @@ struct DeribitInstrumentStore::PImpl {
return &it->second;
}
Security *find(const string &name) {
auto id = sec_id_store_->get_id(VENUE, name);
Security *find(SecurityType type, const string &name) {
auto id = sec_id_store_->get_id(VENUE, type, name);
if (!id) {
return nullptr;
}
@@ -147,7 +148,7 @@ struct DeribitInstrumentStore::PImpl {
continue;
}
const SecurityID id{
sec_id_store_->get_or_add(VENUE, di.instrument_name)};
sec_id_store_->get_or_add(VENUE, sec.type(), di.instrument_name)};
sec.security_common.id = id;
auto [it, is_new] = securities_.try_emplace(id, std::move(sec));
if (!is_new) {
@@ -188,8 +189,8 @@ struct DeribitInstrumentStore::PImpl {
// DeribitCombo object, which is required to resolve the leg definitions.
unordered_map<string, Security> unresolved_combos_;
const Security *find_security_(const string &name) {
const auto id = sec_id_store_->get_id(VENUE, name);
const Security *find_security_(SecurityType type, const string &name) {
const auto id = sec_id_store_->get_id(VENUE, type, name);
if (!id) {
return nullptr;
}
@@ -202,7 +203,8 @@ struct DeribitInstrumentStore::PImpl {
bool resolve_combo_(const DeribitCombo &combo, SpreadSecurity &spd) {
for (const auto &leg : combo.legs) {
const auto *leg_sec = find_security_(leg.instrument_name);
const auto *leg_sec =
find_security_(SEC_TYPE(SPREAD), leg.instrument_name);
if (leg_sec == nullptr) {
spd.legs.clear();
return false;
@@ -259,7 +261,8 @@ struct DeribitInstrumentStore::PImpl {
continue;
}
}
SecurityID id{sec_id_store_->get_or_add(VENUE, sec.desc())};
SecurityID id{
sec_id_store_->get_or_add(VENUE, sec.type(), sec.desc())};
sec.security_common.id = id;
auto [sec_it, is_new] = securities_.try_emplace(id, std::move(sec));
if (!is_new) {
@@ -292,8 +295,8 @@ const Security *DeribitInstrumentStore::find(SecurityID id) const {
return me_->find(id);
}
Security *DeribitInstrumentStore::find(const string &id) {
return me_->find(id);
Security *DeribitInstrumentStore::find(SecurityType type, const string &name) {
return me_->find(type, name);
}
void DeribitInstrumentStore::on_instruments(
@@ -45,11 +45,12 @@ class DeribitInstrumentStore {
const domain::Security *find(domain::SecurityID id) const;
/**
* @brief Finds the security with the specified \p name
* @brief Finds the security with the specified \p name and \p type
* @details No 'const' overload bc the fn that converts name to numeric id
* isn't const
*/
domain::Security *find(const std::string &name);
domain::Security *find(
api::domain::SecurityType type, const std::string &name);
void on_instruments(
const std::vector<DeribitInstrument> &instruments,
@@ -41,6 +41,7 @@
namespace bts::md::deribit {
using bts::api::domain::SecurityType;
using bts::api::md::DeribitMarketDataConfig;
using bts::domain::Security;
using bts::domain::SecurityCriteria;
@@ -58,6 +59,7 @@ using std::unordered_map;
using std::unordered_set;
using std::vector;
#define VENUE bts::api::domain::VENUE_DERIBIT
#define VENUE_NAME \
(bts::api::domain::Venue_Name(bts::api::domain::VENUE_DERIBIT))
#define CCAPI_VENUE_NAME CCAPI_EXCHANGE_NAME_DERIBIT
@@ -166,8 +168,9 @@ struct DeribitSubscription::PImpl {
subscribe_instrument_state();
}
SecurityData &add_security_data(const string sec_desc) {
const SecurityID sid{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
SecurityData &add_security_data(
SecurityType sec_type, const string &sec_desc) {
const SecurityID sid{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
auto [it, is_new] = security_data_map.try_emplace(
sec_desc, std::make_unique<SecurityData>(sid, listener));
_ASSERT_MSGF(is_new, "add_security_data. Already added '{}'", sec_desc);
@@ -199,27 +202,31 @@ struct DeribitSubscription::PImpl {
void subscribe_md(SecurityNames security_names) {
// Populate list of 'channels', and prepare their corresponding handlers
vector<string> channels;
for (const string &sec : security_names) {
auto &sec_data = add_security_data(sec);
for (auto [type, name] : security_names) {
auto &sec_data = add_security_data(type, name);
// Depth
if (subscribe_depth) {
const auto &ch = channels.emplace_back(
FMT("book.{}.none.{}.100ms", sec, cfg.num_depth_levels()));
FMT("book.{}.none.{}.100ms", name, cfg.num_depth_levels()));
channel_demuxer.add_handler(ch)
.bind<&DeribitDepthAdaptor::on_depths>(sec_data.depth_hnd);
.template bind<&DeribitDepthAdaptor::on_depths>(
sec_data.depth_hnd);
}
// Ticker
if (subscribe_ticker) {
const auto &ch =
channels.emplace_back(FMT("incremental_ticker.{}", sec));
channels.emplace_back(FMT("incremental_ticker.{}", name));
channel_demuxer.add_handler(ch)
.bind<&DeribitTickerAdaptor::on_tickers>(sec_data.ticker_hnd);
.template bind<&DeribitTickerAdaptor::on_tickers>(
sec_data.ticker_hnd);
}
// Trades
if (subscribe_trades) {
const auto &ch = channels.emplace_back(FMT("trades.{}.100ms", sec));
const auto &ch =
channels.emplace_back(FMT("trades.{}.100ms", name));
channel_demuxer.add_handler(ch)
.bind<&DeribitTradeAdaptor::on_trades>(sec_data.trade_hnd);
.template bind<&DeribitTradeAdaptor::on_trades>(
sec_data.trade_hnd);
}
}
// If 'channels' is empty, it's a no-op
@@ -253,7 +260,9 @@ struct DeribitSubscription::PImpl {
bts_assert(!CONTAINS_KEY(security_data_map, sec->desc()));
}
listener->on_securities(secs);
auto sec_names = TRANSFORM(secs, [](const auto *s) { return s->desc(); });
auto sec_names = TRANSFORM(secs, [](const auto *s) {
return std::make_pair(s->type(), s->desc());
});
batch_subscribe_md(sec_names);
}
@@ -18,10 +18,11 @@ struct EmumbaDeribitData {
EmumbaDeribitData()
: sec_index(), recorder(
{utils::Dispatcher::current(),
[=, this](ExchangeKey k) {
return SecurityKey(sec_index.get_key(k.value()));
}}),
nullptr,
[=, this](ExchangeKey k) {
return SecurityKey(sec_index.get_key(k.value()));
},
&utils::Dispatcher::current()),
md_producer(recorder), listener(sec_index, md_producer) {}
};
} // namespace bts::md
@@ -35,6 +35,7 @@ using std::string;
using std::unordered_set;
using std::vector;
#define VENUE Venue::VENUE_FTX
#define VENUE_NAME (Venue_Name(Venue::VENUE_FTX))
#define SEC_TYPE(x) bts::api::domain::SECURITY_TYPE_##x
@@ -172,7 +173,8 @@ struct FTXSecurityRequestor::PImpl {
for (auto &sec : added_secs) {
// Since the ids haven't been set yet, we should expect them to be 0
bts_assert(sec.id().value() == 0);
SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec.desc())};
SecurityID sec_id{
sec_id_store->get_or_add(VENUE, sec.type(), sec.desc())};
sec.security_common.id = sec_id;
}
}
+19 -12
View File
@@ -28,6 +28,7 @@
namespace bts::md::ftx {
using bts::api::domain::SecurityType;
using bts::api::domain::Venue;
using bts::api::domain::Venue_Name;
using bts::api::md::FTXMarketDataConfig;
@@ -36,11 +37,13 @@ using bts::domain::SecurityCriteria;
using bts::domain::SecurityID;
using bts::io::ProtoPrinter;
using bts::persistence::IDGenerator;
using std::pair;
using std::string;
using std::unique_ptr;
using std::unordered_map;
using std::vector;
#define VENUE Venue::VENUE_FTX
#define VENUE_NAME (Venue_Name(Venue::VENUE_FTX))
#define FTX_MSG_TYPE_NAME(x) \
@@ -130,16 +133,18 @@ struct FTXSubscription::PImpl {
sec_requestor.request();
}
DepthHandler &create_depth_handler(string sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
DepthHandler &create_depth_handler(
SecurityType sec_type, const string &sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
auto [it, is_new] =
depth_handlers.try_emplace(sec_desc, sec_id, listener);
bts_assert(is_new);
return it->second;
}
TickerHandler &create_ticker_handler(string sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
TickerHandler &create_ticker_handler(
SecurityType sec_type, const string &sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
auto [it, is_new] =
ticker_handlers.try_emplace(sec_desc, sec_id, listener);
bts_assert_msgf(
@@ -147,8 +152,9 @@ struct FTXSubscription::PImpl {
return it->second;
}
StatsHandler &create_stats_handler(string sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
StatsHandler &create_stats_handler(
SecurityType sec_type, const string &sec_desc) {
SecurityID sec_id{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
auto [it, is_new] =
stats_handlers.try_emplace(sec_desc, sec_id, listener);
bts_assert_msgf(
@@ -171,15 +177,16 @@ struct FTXSubscription::PImpl {
CCApiSession::BatchSubscription ticker_subs;
CCApiSession::BatchRequest stats_reqs;
for (const auto &sec_desc : securities) {
for (const auto &[sec_type, sec_desc] : securities) {
depth_subs.push_back(
{create_depth_sub(sec_desc, depth_opts),
&create_depth_handler(sec_desc)});
&create_depth_handler(sec_type, sec_desc)});
ticker_subs.push_back(
{create_ticker_sub(sec_desc, ticker_opts),
&create_ticker_handler(sec_desc)});
&create_ticker_handler(sec_type, sec_desc)});
stats_reqs.push_back(
{create_stats_req(sec_desc), &create_stats_handler(sec_desc)});
{create_stats_req(sec_desc),
&create_stats_handler(sec_type, sec_desc)});
}
session.subscribe(
@@ -196,7 +203,7 @@ struct FTXSubscription::PImpl {
*/
void on_securities(vector<domain::Security> secs) {
// 'sec_descs' is used to make depth subscriptions
vector<string> sec_descs;
vector<pair<SecurityType, string>> sec_descs;
sec_descs.reserve(secs.size());
// 'sec_ptrs' is passed to IMarketDataListeners
vector<const domain::Security *> sec_ptrs;
@@ -208,7 +215,7 @@ struct FTXSubscription::PImpl {
auto [it, is_new] = securities.try_emplace(
id, std::make_unique<domain::Security>(std::move(s)));
bts_assert_msgf(is_new, "Received duplicate FTX security '{}'", desc);
sec_descs.push_back(std::move(desc));
sec_descs.push_back(std::make_pair(s.type(), std::move(desc)));
// add ptr to entry in 'securities' (not the loop var, we moved that)
sec_ptrs.push_back(it->second.get());
}
@@ -14,6 +14,7 @@
namespace bts::md::okx {
using bts::api::domain::CallPut;
using bts::api::domain::SecurityType;
using bts::domain::FutureSecurity;
using bts::domain::OptionSecurity;
using bts::domain::Price;
@@ -94,8 +95,8 @@ struct OkxInstrumentStore::PImpl {
return &it->second;
}
Security *find(const string &name) {
auto id = sec_id_store_->get_id(VENUE, name);
Security *find(SecurityType type, const string &name) {
auto id = sec_id_store_->get_id(VENUE, type, name);
if (!id) {
return nullptr;
}
@@ -124,7 +125,7 @@ struct OkxInstrumentStore::PImpl {
if (!filter_(sec)) {
continue;
}
SecurityID id{sec_id_store_->get_or_add(VENUE, oi.instId)};
SecurityID id{sec_id_store_->get_or_add(VENUE, sec.type(), oi.instId)};
sec.security_common.id = id;
auto [it, is_new] = securities_.try_emplace(id, std::move(sec));
if (!is_new) {
@@ -152,7 +153,9 @@ const Security *OkxInstrumentStore::find(SecurityID id) const {
return me_->find(id);
}
Security *OkxInstrumentStore::find(const string &id) { return me_->find(id); }
Security *OkxInstrumentStore::find(SecurityType type, const string &name) {
return me_->find(type, name);
}
void OkxInstrumentStore::on_instruments(
const vector<OkxInstrument> &instruments,
@@ -50,7 +50,8 @@ class OkxInstrumentStore {
* @details No 'const' overload bc the fn that converts name to numeric id
* isn't const
*/
domain::Security *find(const std::string &name);
domain::Security *find(
api::domain::SecurityType type, const std::string &name);
void on_instruments(
const std::vector<OkxInstrument> &instruments,
+13 -8
View File
@@ -38,6 +38,7 @@
namespace bts::md::okx {
using bts::api::domain::SecurityType;
using bts::api::md::OkxMarketDataConfig;
using bts::domain::Security;
using bts::domain::SecurityCriteria;
@@ -60,6 +61,7 @@ using std::vector;
#define _WARN(...) _LOG(Warn, __VA_ARGS__)
#define _ERROR(...) _LOG(Error, __VA_ARGS__)
#define VENUE bts::api::domain::VENUE_OKX
#define VENUE_NAME (bts::api::domain::Venue_Name(bts::api::domain::VENUE_OKX))
#define CCAPI_VENUE_NAME CCAPI_EXCHANGE_NAME_OKX
#define SEC_TYPE(t) bts::api::domain::SECURITY_TYPE_##t
@@ -164,8 +166,9 @@ struct OkxSubscription::PImpl {
[this](const auto &secs) { on_securities(secs); });
}
SecurityData &add_security_data(const string sec_desc) {
const SecurityID sid{sec_id_store->get_or_add(VENUE_NAME, sec_desc)};
SecurityData &add_security_data(
SecurityType sec_type, const string &sec_desc) {
const SecurityID sid{sec_id_store->get_or_add(VENUE, sec_type, sec_desc)};
auto [it, is_new] = security_data_map.try_emplace(
sec_desc, std::make_unique<SecurityData>(sid, listener));
_ASSERT_MSGF(is_new, "add_security_data. Already added '{}'", sec_desc);
@@ -182,13 +185,13 @@ struct OkxSubscription::PImpl {
void subscribe_md(SecurityNames security_names) {
// Populate list of 'channels', and prepare their corresponding handlers
vector<OkxChannel> channels;
for (const string &sec : security_names) {
auto &sec_data = add_security_data(sec);
for (const auto &[type, name] : security_names) {
auto &sec_data = add_security_data(type, name);
// Depth
if (subscribe_depth) {
auto &ch = channels.emplace_back();
ch.channel = "books5";
ch.instId = sec;
ch.instId = name;
channel_demuxer.add_handler(ch).bind<&OkxDepthAdaptor::on_depths>(
sec_data.depth_hnd);
}
@@ -196,7 +199,7 @@ struct OkxSubscription::PImpl {
if (subscribe_ticker) {
auto &ch = channels.emplace_back();
ch.channel = "tickers";
ch.instId = sec;
ch.instId = name;
channel_demuxer.add_handler(ch).bind<&OkxTickerAdaptor::on_tickers>(
sec_data.ticker_hnd);
}
@@ -204,7 +207,7 @@ struct OkxSubscription::PImpl {
if (subscribe_trades) {
auto &ch = channels.emplace_back();
ch.channel = "trades";
ch.instId = sec;
ch.instId = name;
channel_demuxer.add_handler(ch).bind<&OkxTradeAdaptor::on_trades>(
sec_data.trade_hnd);
}
@@ -240,7 +243,9 @@ struct OkxSubscription::PImpl {
bts_assert(!CONTAINS_KEY(security_data_map, sec->desc()));
}
listener->on_securities(secs);
auto sec_names = TRANSFORM(secs, [](const auto *s) { return s->desc(); });
auto sec_names = TRANSFORM(secs, [](const auto *s) {
return std::make_pair(s->type(), s->desc());
});
batch_subscribe_md(sec_names);
}
};
@@ -59,6 +59,10 @@ using namespace bts::orderman;
namespace bts::md {
#define SEC_TYPE(x) bts::api::domain::SECURITY_TYPE_##x
#define TRADING_STATUS(x) bts::co2::TRADINGSTATUS_##x
#define VENUE bts::api::domain::VENUE_DERIBIT
std::atomic<orderman::IEMOrderMuxer *> EmMDListener::_order_mux = nullptr;
using MDArray = std::array<PriceSize, COCONUT_MAX_DEPTH>;
using EMOrderKey = uint64_t;
@@ -1144,16 +1148,32 @@ DescSearchResult EmMDListener::_find_em_sec(
}
}
static api::domain::SecurityType convert_sec_type(
emumba::market_data::security_type emumba_sec_type) {
switch (emumba_sec_type) {
case emumba::market_data::security_type::index:
return SEC_TYPE(SPOT);
case emumba::market_data::security_type::future:
return SEC_TYPE(FUTURE);
case emumba::market_data::security_type::spread:
return SEC_TYPE(SPREAD);
case emumba::market_data::security_type::option:
return SEC_TYPE(OPTION);
default:
bts_assert_msg(false, "Unknown sec type");
return SEC_TYPE(UNSPECIFIED);
}
}
security_key EmMDListener::_add_sec_entry(
const emumba::market_data::security &sec) {
const std::string &desc = extract_security_desc(sec);
// id_gen either adds a key and returns a new id, or finds one and returns
// the previously added id
bts_assert(me->sec_emdesc_map.find(desc) == me->sec_emdesc_map.end());
auto new_key_opt = me->id_gen.get_id("exchange_key", desc, true);
bts_assert(new_key_opt.has_value());
const security_key new_key = static_cast<security_key>(
me->id_gen.get_or_add(VENUE, convert_sec_type(sec.sec_type()), desc));
// TODO (64 -> 32 conversion)
security_key new_key = new_key_opt.value();
me->sec_emdesc_map.insert({desc, {new_key, sec}});
bts_assert(me->sec_btsid_map.find(new_key) == me->sec_btsid_map.end());
me->sec_btsid_map.insert({new_key, desc});
@@ -43,10 +43,11 @@ struct EmumbaMain::PImpl {
PImpl()
: sec_index(), recorder(
{utils::Dispatcher::current(),
[=, this](ExchangeKey k) {
return SecurityKey(sec_index.get_key(k.value()));
}}),
nullptr,
[=, this](ExchangeKey k) {
return SecurityKey(sec_index.get_key(k.value()));
},
&utils::Dispatcher::current()),
producer(recorder), listener(sec_index, producer) {
const auto config_path = Path{config::get<SystemConfig>().config_path()} /
@@ -1,70 +1,66 @@
#include "ICEMain.h"
#include "lib/config/config.h"
#include "lib/md/SecurityDB.h"
#include "lib/md/coconut2.h"
#include "lib/md/ice/ICEUtils.h"
#include "lib/md/protos/co2.pb.h"
#include "lib/md/vendor/OnixsUtils.h"
#include "lib/os/os.h"
#include "lib/utils/Dispatcher.h"
#include "lib/utils/bts_chrono.h"
#include "lib/utils/helper_macros.h"
#include "lib/utils/proto_helper.h"
#include "lib/config/protos/deployment_type.pb.h"
#include "lib/config/protos/market_data.pb.h"
#include "lib/config/protos/system_config.pb.h"
#include "lib/utils/thread_manager.h"
#include "lib/md/ice/ICEUtils.h"
#include "lib/md/vendor/OnixsUtils.h"
#include <OnixS/ICE/iMpact/MarketData.h>
#include <boost/filesystem.hpp>
#include <memory>
#include <set>
void recursive_copy(
const boost::filesystem::path &src, const boost::filesystem::path &dst) {
if (boost::filesystem::exists(dst))
throw std::runtime_error(dst.generic_string() + " exists");
if (boost::filesystem::is_directory(src)) {
boost::filesystem::create_directories(dst);
for (boost::filesystem::directory_entry &item :
boost::filesystem::directory_iterator(src))
recursive_copy(item.path(), dst / item.path().filename());
} else if (boost::filesystem::is_regular_file(src))
boost::filesystem::copy(src, dst);
else
throw std::runtime_error(dst.generic_string() + " not dir or file");
}
using namespace std;
using namespace bts::co2;
using std::size_t;
namespace bts {
namespace md {
namespace ice {
namespace bts::md::ice {
using bts::config::SystemConfig;
using Path = boost::filesystem::path;
ICEMain::ICEMain()
: _index(), _recorder(
{utils::Dispatcher::current(),
[=, this](ExchangeKey k) {
return SecurityKey(_index.get_key(k.value()));
}}),
_producer(_recorder), _handlers(), _listeners() {}
struct ICEMain::PImpl {
ICEOptions options;
std::string config_file;
ICESecurityIndex index;
SecurityIDRecorder recorder;
MDProducer producer;
std::vector<std::pair<ICEChannel, std::unique_ptr<onixs_ice::Handler>>>
handlers;
std::vector<std::unique_ptr<ICEListener>> listeners;
ICEMain::~ICEMain() {}
PImpl(ICEOptions options)
: options(options),
config_file((Path{config::get<SystemConfig>().data_path()} /
"onixs_ice" / this->options.ice_config_file())
.string()),
index(), recorder(
nullptr,
[=, this](ExchangeKey k) {
return SecurityKey(index.get_key(k.value()));
},
&utils::Dispatcher::current()),
producer(recorder), handlers(), listeners() {}
const IceChannelConfig channel_settings_for(const ICEChannel &channel) {
const auto &config = config::get<ICEOptions>();
for (const auto &ch : config.channel()) {
if (ch.name() == channel.name)
return ch;
IceChannelConfig channel_settings_for(const ICEChannel &channel) {
for (const auto &ch : options.channel()) {
if (ch.name() == channel.name)
return ch;
}
return {};
}
return {};
};
ICEMain::ICEMain() : ICEMain(config::get<ICEOptions>()) {}
ICEMain::ICEMain(ICEOptions options)
: me_(std::make_unique<PImpl>(std::move(options))) {
me_->options = std::move(options);
}
ICEMain::~ICEMain() = default;
onixs_ice::MarketSubscriptions market_subcription_for_channel(
const ICEChannel &chan) {
onixs_ice::MarketSubscriptions subs;
@@ -90,23 +86,30 @@ onixs_ice::MarketSubscriptions market_subcription_for_channel(
return subs;
}
void ICEMain::start(const std::vector<UserSymbol> &symbols) {
auto ice_conf_location =
Path{config::get<SystemConfig>().data_path()} / "onixs_ice/";
if (!boost::filesystem::is_directory(ice_conf_location)) {
auto conf_path = Path{os::sparkd_wd()} / "data_files" / "onixs_ice/";
recursive_copy(Path{conf_path}, ice_conf_location);
static void validate_config_file(const std::string &config_file) {
if (!boost::filesystem::exists(config_file)) {
const std::string config_url =
config::get<config::DeploymentType>().type() == "dev"
? "https://community.theice.com/docs/DOC-19989"
: "https://drive.google.com/"
"open?id=13_eyF2pKtn3F7chgsa_75ftUdf7emMbp";
bts_assert_msgf(
false, "Ice config file not found '{}'. Please download from {}",
config_file, config_url);
}
}
void ICEMain::start(const std::vector<UserSymbol> &symbols) {
await_ice_config();
validate_config_file(me_->config_file);
auto license = onixs_create_temporary_license();
const auto &channels = parse_ice_channels(symbols);
for (const auto &channel : channels) {
unique_ptr<onixs_ice::Handler> phandler =
prepare_handler(channel, license.path.string());
_listeners.push_back(prepare_listener(*phandler, channel));
_handlers.push_back(std::make_pair(channel, std::move(phandler)));
me_->listeners.push_back(prepare_listener(*phandler, channel));
me_->handlers.push_back(std::make_pair(channel, std::move(phandler)));
std::stringstream symstr;
for (const auto &sym : channel.symbols)
@@ -115,7 +118,7 @@ void ICEMain::start(const std::vector<UserSymbol> &symbols) {
symstr.str());
}
for (auto &pair : _handlers) {
for (auto &pair : me_->handlers) {
LOG(md, Info, "ICE: Init Handler {}", pair.first.to_string());
sleep_for(16s);
onixs_ice::MarketSubscriptions subs =
@@ -132,7 +135,7 @@ void ICEMain::start(const std::vector<UserSymbol> &symbols) {
}
void ICEMain::send_heartbeats() {
for (const auto &listener : _listeners)
for (const auto &listener : me_->listeners)
listener->heartbeat();
}
@@ -140,20 +143,18 @@ void ICEMain::send_heartbeats() {
// Initialize HandlerSettings and allocate the channel Handler
unique_ptr<onixs_ice::Handler> ICEMain::prepare_handler(
const ICEChannel &channel, const std::string &license) {
const auto &options = channel_settings_for(channel);
const auto &options = me_->channel_settings_for(channel);
LOG(md, Info, "Loaded channel options for channel {}:\n{}",
channel.to_string(), options.DebugString());
onixs_ice::HandlerSettings settings;
settings.licenseDirectory = license;
//
// Logging
settings.logLevel = (onixs_ice::LogLevels::Enum)options.onixs_log_mode();
settings.logDirectory = ice_log_path();
//
// Config file
settings.connectivityConfiguration = ice_config_path();
settings.logDirectory =
(boost::filesystem::path{config::get<SystemConfig>().data_path()} /
"logging" / "onixs_ice/")
.string();
settings.connectivityConfiguration = me_->config_file;
//
// Network settings
@@ -231,12 +232,12 @@ unique_ptr<ICEListener> ICEMain::prepare_listener(
// heartbeats
vector<MDProducerChannel> producer_channels;
for (auto &symbol : channel.symbols) {
producer_channels.push_back(_producer.get_channel(
producer_channels.push_back(me_->producer.get_channel(
symbol.symbol().c_str(), symbol.type(), symbol.exchange()));
}
unique_ptr<ICEListener> listener(new ICEListener(
_index, _producer, /*_settlement_recorder,*/ producer_channels));
me_->index, me_->producer, /*_settlement_recorder,*/ producer_channels));
for (auto &symbol : channel.symbols)
listener->add_filter_symbol(symbol.symbol());
@@ -254,6 +255,4 @@ unique_ptr<ICEListener> ICEMain::prepare_listener(
return listener;
}
} // namespace ice
} // namespace md
} // namespace bts
} // namespace bts::md::ice
@@ -1,21 +1,12 @@
#pragma once
#include "ICEListener.h"
#include "ICESecurityIndex.h"
#include "lib/md/coconut2.h"
#include "lib/utils/bts_no_copy.h"
#include "onixs.h"
#include "server/sparkd/md/platforms/IMDPlatform.h"
#include <map>
#include <memory>
#include <vector>
namespace bts {
namespace md {
namespace ice {
// void run_onixs_ice_server(UserSymbol* sym, size_t symcount);
namespace bts::md::ice {
struct ICEChannel;
@@ -28,27 +19,22 @@ struct ICEChannel;
class ICEMain : public IMDPlatform {
public:
ICEMain();
explicit ICEMain(ICEOptions options);
ICEMain(const ICEMain &) = delete;
ICEMain &operator=(const ICEMain &) = delete;
~ICEMain();
void start(const std::vector<UserSymbol> &symbols);
void send_heartbeats() override;
std::unique_ptr<onixs_ice::Handler>
prepare_handler(const ICEChannel &channel, const std::string &license);
std::unique_ptr<ICEListener>
prepare_listener(onixs_ice::Handler &handler, const ICEChannel &channel);
std::unique_ptr<onixs_ice::Handler> prepare_handler(
const ICEChannel &channel, const std::string &license);
std::unique_ptr<ICEListener> prepare_listener(
onixs_ice::Handler &handler, const ICEChannel &channel);
private:
BTS_NO_COPY(ICEMain);
ICESecurityIndex _index;
SecurityIDRecorder _recorder;
MDProducer _producer;
std::vector<std::pair<ICEChannel, std::unique_ptr<onixs_ice::Handler>>>
_handlers;
std::vector<std::unique_ptr<ICEListener>> _listeners;
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace ice
} // namespace md
} // namespace bts
} // namespace bts::md::ice
@@ -1,24 +1,18 @@
#include "MDP3Main.h"
#include "lib/api/md/GlbxMarketDataConfig.pb.h"
#include "lib/config/config.h"
#include "lib/config/protos/deployment_type.pb.h"
#include "lib/logging/logging.h"
#include "lib/md/MBOProcessor.h"
#include "lib/md/SecurityDB.h"
#include "lib/md/cme/MDP3Utils.h"
#include "lib/md/coconut2.h"
#include "lib/md/protos/co2.pb.h"
#include "lib/md/cme/mdp3_data_directory.h"
#include "lib/md/vendor/OnixsUtils.h"
#include "lib/os/os.h"
#include "lib/utils/Dispatcher.h"
#include "lib/utils/proto_helper.h"
#include "lib/config/protos/market_data.pb.h"
#include "lib/config/protos/system_config.pb.h"
#include "lib/os/spark_data_directory.h"
#include "lib/utils/thread_manager.h"
#include <boost/filesystem.hpp>
#include <memory>
#include <optional>
#include <set>
#include <sstream>
@@ -29,16 +23,33 @@ using std::size_t;
namespace bts {
namespace md {
MDP3Main::MDP3Main()
: _index(), _recorder(
{utils::Dispatcher::current(),
[=, this](ExchangeKey k) {
return SecurityKey(_index.get_key(k.value()));
}}),
_producer(_recorder), _engines(), _handlers(), _listeners(), _recovery() {
}
#define TRADING_STATUS(x) bts::domain::protobufs::TRADINGSTATUS_##x
MDP3Main::~MDP3Main() {}
using Path = boost::filesystem::path;
struct MDP3Main::PImpl {
Mdp3DataDirectory data_dir;
CMESecurityIndex index;
SecurityIDRecorder recorder;
MDProducer producer;
std::map<std::string, std::unique_ptr<onixs::FeedEngine>> engines;
std::map<std::string, std::unique_ptr<onixs::Handler>> handlers;
std::vector<std::unique_ptr<MDP3Listener>> listeners;
std::vector<std::unique_ptr<onixs::TcpRecoveryService>> recovery;
PImpl()
: index(), recorder(
nullptr,
[=, this](ExchangeKey k) {
return SecurityKey(index.get_key(k.value()));
},
&utils::Dispatcher::current()),
producer(recorder), engines(), handlers(), listeners(), recovery() {}
};
MDP3Main::MDP3Main()
: me_(std::make_unique<PImpl>()) {}
MDP3Main::~MDP3Main() = default;
const CmeChannelConfig channel_settings_for(const string &id) {
const auto &config = config::get<CMEOptions>();
@@ -173,11 +184,11 @@ void MDP3Main::start(
auto license = onixs_create_temporary_license();
initialize(license.path.string());
const auto &channels = parse_channels(symbols);
_engines = create_feed_engines(channels);
me_->engines = create_feed_engines(channels);
for (const auto &channel : channels) {
unique_ptr<onixs::Handler> phandler = prepare_handler(channel.id, mbo);
_listeners.push_back(prepare_listener(*phandler, channel, mbo, tssd));
_handlers.insert(std::make_pair(channel.id, std::move(phandler)));
me_->listeners.push_back(prepare_listener(*phandler, channel, mbo, tssd));
me_->handlers.insert(std::make_pair(channel.id, std::move(phandler)));
std::stringstream symstr;
for (const auto &sym : channel.symbols) {
symstr << sym.symbol() << "-" << SecurityType_Name(sym.type()) << " ";
@@ -186,7 +197,7 @@ void MDP3Main::start(
symstr.str());
}
for (auto &pair : _handlers) {
for (auto &pair : me_->handlers) {
onixs::SessionSettings settings =
session_settings_for_channel(pair.first, mbo);
pair.second->start(settings);
@@ -196,7 +207,7 @@ void MDP3Main::start(
}
void MDP3Main::send_heartbeats() {
for (const auto &listener : _listeners) {
for (const auto &listener : me_->listeners) {
listener->heartbeat();
}
}
@@ -219,7 +230,8 @@ unique_ptr<onixs::Handler> MDP3Main::prepare_handler(
onixs::HandlerSettings settings;
settings.channel(std::stoul(channel_id));
settings.feeds().connectivityConfigurationFile(config_path());
settings.feeds().connectivityConfigurationFile(
(me_->data_dir.config_file()).string());
// Market data options
settings.bookManagement().directBooks().maintain(true);
@@ -306,7 +318,8 @@ unique_ptr<onixs::Handler> MDP3Main::prepare_handler(
logger = new onixs::NullLogger();
} else {
onixs::FileLoggerSettings loggerSettings;
loggerSettings.filename(log_path() + "/log_" + channel_id);
loggerSettings.filename(
(me_->data_dir.log_dir() / ("log_" + channel_id)).string());
loggerSettings.severityLevel(
(onixs::LogSeverity::Enum)options.onixs_log_mode());
logger = new onixs::FileLogger(loggerSettings);
@@ -338,19 +351,19 @@ unique_ptr<onixs::Handler> MDP3Main::prepare_handler(
try {
auto handler = unique_ptr<onixs::Handler>(new onixs::Handler(settings));
auto iter = _engines.find(options.feed_engine());
if (iter == _engines.end()) {
auto iter = me_->engines.find(options.feed_engine());
if (iter == me_->engines.end()) {
LOG(md, Fatal,
"Missing feed engine definition '{}' for channel id '{}'. Add a "
"feed_engine entry with name:'{}' to the CMEOptions config key.",
options.feed_engine(), channel_id, options.feed_engine());
bts_assert_msg(
iter != _engines.end(), "Missing feed engine definition.");
iter != me_->engines.end(), "Missing feed engine definition.");
}
handler->bindFeedEngine(*(iter->second));
if (recovery) {
handler->bindTcpRecovery(*recovery);
_recovery.push_back(std::move(recovery));
me_->recovery.push_back(std::move(recovery));
}
if (logger) {
handler->bindLogger(*logger);
@@ -375,12 +388,13 @@ unique_ptr<MDP3Listener> MDP3Main::prepare_listener(
// heartbeats
vector<MDProducerChannel> producer_channels;
for (auto &symbol : channel.symbols) {
producer_channels.push_back(_producer.get_channel(
producer_channels.push_back(me_->producer.get_channel(
symbol.symbol().c_str(), symbol.type(), symbol.exchange()));
}
unique_ptr<MDP3Listener> listener(new MDP3Listener(
_index, _producer, mbo, producer_channels, trade_summary_dispatcher));
me_->index, me_->producer, mbo, producer_channels,
trade_summary_dispatcher));
for (auto &symbol : channel.symbols) {
listener->add_filter_symbol(symbol.symbol());
}
@@ -29,7 +29,9 @@ struct CMEChannel;
*/
class MDP3Main : public IMDPlatform {
public:
MDP3Main();
explicit MDP3Main();
MDP3Main(const MDP3Main &) = delete;
MDP3Main &operator=(const MDP3Main &) = delete;
~MDP3Main();
void start(
@@ -39,8 +41,8 @@ class MDP3Main : public IMDPlatform {
void send_heartbeats() override;
void initialize(const std::string &license_store);
std::unique_ptr<onixs::Handler>
prepare_handler(const std::string &channel, MBOWriter *);
std::unique_ptr<onixs::Handler> prepare_handler(
const std::string &channel, MBOWriter *);
std::unique_ptr<MDP3Listener> prepare_listener(
onixs::Handler &handler,
const CMEChannel &channel,
@@ -48,16 +50,8 @@ class MDP3Main : public IMDPlatform {
TradeSummarySignalDispatcher *);
private:
BTS_NO_COPY(MDP3Main);
CMESecurityIndex _index;
SecurityIDRecorder _recorder;
MDProducer _producer;
// co2::SettlementRecorder _settlement_recorder;
std::map<std::string, std::unique_ptr<onixs::FeedEngine>> _engines;
std::map<std::string, std::unique_ptr<onixs::Handler>> _handlers;
std::vector<std::unique_ptr<MDP3Listener>> _listeners;
std::vector<std::unique_ptr<onixs::TcpRecoveryService>> _recovery;
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace md
@@ -201,8 +201,9 @@ class TaifexDataProcessors {
const auto id = TaifexUtils::remove_spaces(msg.prod_id);
const auto symbol = id.substr(0, 3);
const auto sec_type = convert_sec_type(msg.parsedProductInfo.productType);
bts::domain::SecurityID sec_id{
sec_id_store->get_or_add(api::domain::VENUE_TAIFEX, id)};
sec_id_store->get_or_add(api::domain::VENUE_TAIFEX, sec_type, id)};
auto check =
security_details->add_i010_mapping(id, sec_id, msg.decimal_locator);
if (!check) {
@@ -286,6 +287,20 @@ class TaifexDataProcessors {
security_details->add_i011_mapping(
id, 1.0, msg.getCurrencyString(), msg.contract_size);
}
private:
static api::domain::SecurityType convert_sec_type(
TaifexMessages::I010Data::ProductInfo::ProductType type) {
switch (type) {
case TaifexMessages::I010Data::ProductInfo::ProductType::FUTURES:
return api::domain::SecurityType::SECURITY_TYPE_FUTURE;
case TaifexMessages::I010Data::ProductInfo::ProductType::OPTIONS:
return api::domain::SecurityType::SECURITY_TYPE_OPTION;
default:
bts_assert_msg(
false, "Unexpected security type (must be FUTURES or OPTIONS)");
}
}
};
} //namespace bts::md::taifex
} //namespace bts::md::taifex
@@ -293,9 +293,9 @@ void PositionServiceServer::update_manual_fill(
auto *this_resp = resp.add_response();
auto id = std::to_string(mf.record_id());
auto p = me->manual_fills_store->retrieve_one(manual_fills_ns, id);
auto found = p.first;
if (!found) {
auto [exists, rowid, val] =
me->manual_fills_store->retrieve_one(manual_fills_ns, id);
if (!exists) {
this_resp->set_ok(false);
this_resp->set_msg("Could not find record");
} else {
@@ -326,9 +326,9 @@ void PositionServiceServer::remove_manual_fill(
auto *this_resp = resp.add_response();
auto id = std::to_string(mf.record_id());
auto p = me->manual_fills_store->retrieve_one(manual_fills_ns, id);
auto found = p.first;
if (!found) {
auto [exists, rowid, val] =
me->manual_fills_store->retrieve_one(manual_fills_ns, id);
if (!exists) {
this_resp->set_ok(false);
this_resp->set_msg("Could not find record");
} else {
@@ -367,10 +367,9 @@ void PositionServiceServer::update_security_parameters(
req.op_params_size());
for (const auto &p : req.op_params()) {
std::string key_str = std::to_string(p.security_key());
auto db_p =
auto [exists, rowid, val] =
me->security_params_store->retrieve_one(security_params_ns, key_str);
auto found = db_p.first;
if (!found) {
if (!exists) {
me->security_params_store->insert(
security_params_ns, key_str, utils::pb::to_string(p));
} else {
+4 -2
View File
@@ -21,7 +21,8 @@ AdminServiceDB::AdminServiceDB() : me(new PImpl()) {}
AdminServiceDB::~AdminServiceDB() {}
std::optional<api::admin::AdminData> AdminServiceDB::retrieve() {
auto [has_table, data] = me->store.retrieve_one(table_name, unique_key);
auto [has_table, rowid, data] =
me->store.retrieve_one(table_name, unique_key);
if (!has_table)
return {};
@@ -33,7 +34,8 @@ std::optional<api::admin::AdminData> AdminServiceDB::retrieve() {
}
bool AdminServiceDB::update(api::admin::AdminData const &state) {
auto [has_table, data] = me->store.retrieve_one(table_name, unique_key);
auto [has_table, rowid, data] =
me->store.retrieve_one(table_name, unique_key);
UNUSED(data);
std::string ser_state;