Changing replay mechanis in sim to allow for cert and sim md.

This commit is contained in:
Pedro Pinto committed 2018-07-16 21:33:01 +00:00
1 parent 94c5a1bcd1
commit 6960ee1a12
6 files changed
+156 -97

No files matched your search

+1 -1
View File
@@ -56,7 +56,7 @@ message MDRequest {
TRADE_LOG = 0x8;
RFQ = 0x10;
CUSTOMER_BBO = 0x20;
MISC = 0x40;
MISC = 0x40;
}
string host = 2;
}
+1
View File
@@ -29,6 +29,7 @@ void print_recording(const std::string&);
void unprint_recording(const std::string& src, const std::string& dest);
void trace_recording(const std::string& file, security_key key);
void replay_recording(const std::string&, int32_t delay_ms);
void replay_url(const std::string& url, IRelayWriter& writer, int32_t naggle_ms);
}}//namespace
+53 -2
View File
@@ -15,6 +15,11 @@
#include <quickfix/FileLog.h>
#include "lib/utils/bts_chrono.h"
#include "lib/md/relay/RelayClient.h"
#include "lib/domain_model/Security.pb.h"
#include "lib/md/relay/RelayRecorder.h"
#include <set>
namespace bts {
namespace sim {
@@ -36,6 +41,48 @@ auto cfg =
"ValidateUserDefinedFields=N\n"
"ValidateFieldsOutOfOrder=N\n";
struct MDPlayer : md::IRelayWriter
{
MDPlayer (IMDPlayerObserver &obs,std::map<uint32_t,md::Security> &known_securities) :
obs (obs),
known_securities(known_securities)
{}
void write(const co2::MDReply& data, bool* ) override {
for (auto const &ch : data.channels())
for (auto const &cr : ch.updates()) {
auto it = known_securities.find(cr.key());
api::Security const *sec_ptr (nullptr);
if (it == known_securities.end()) {
bts_assert(cr.has_security());
sec_ptr = &cr.security();
known_securities[cr.key()] = *sec_ptr;
obs.on_new_security(*sec_ptr);
}
else
sec_ptr = &it->second;
auto const &book = cr.book();
md::PriceSize bids[md::COCONUT_MAX_DEPTH], asks[md::COCONUT_MAX_DEPTH];
size_t i = 0;
for (auto &b : book.bids())
bids[i++] = md::PriceSize(b.mantissa(),b.exponent(),b.size());
size_t n = 0;
for (auto &a : book.asks())
asks[n++] = md::PriceSize(a.mantissa(),a.exponent(),a.size());
obs.on_depth(*sec_ptr, bids, asks);
}
}
IMDPlayerObserver &obs;
std::map<uint32_t, md::Security> &known_securities;
};
int start_exchange_sim()
{
try
@@ -55,9 +102,13 @@ int start_exchange_sim()
FIX::SocketAcceptor acceptor( application, storeFactory, settings, logFactory );
acceptor.start();
std::map<uint32_t,md::Security> known_securities;
while ( true ) {
std::this_thread::sleep_for(10ms) ;
application.merge_with_md_book();
MDPlayer player(application,known_securities);
md::replay_url("http://mdreplay.bluetradesys.local/spark/futures.coconut",
player,
config::get<md::RelayOptions>().naggle_ms());
}
acceptor.stop();
return 0;
@@ -6,7 +6,6 @@
#include "lib/config/protos/ExchangeSimCfg.pb.h"
#include "lib/domain_model/Security.h"
#include "lib/logging/logging.h"
#include "lib/md/MDConsumer.h"
#include "lib/md/MDProducer.h"
#include "lib/md/Shm/FixedSecurity.h"
#include "lib/md/Shm/MDStructure.h"
@@ -93,7 +92,6 @@ convert(const FIX::ManualOrderIndicator &moi) {
}
struct ExchangeSimApplication::MDConsumerRec {
md::MDConsumerHandle handle;
api::Security security;
utils::TimeStamp last_read;
Book previous_md_book;
@@ -102,7 +100,6 @@ struct ExchangeSimApplication::MDConsumerRec {
struct ExchangeSimApplication::PImpl {
PImpl()
: producer(co2::PLATFORM_SIM),
consumer(api::VENUE_GLBX, co2::PLATFORM_MDP3ONIXS),
timer(utils::Dispatcher::background(), 200,
[=]() { producer.heartbeat(); }),
do_merge_with_md(config::get<config::ExchangeSim>().merge_with_md()),
@@ -110,13 +107,11 @@ struct ExchangeSimApplication::PImpl {
IDGenerator m_generator;
md::MDProducer producer;
md::MDConsumer consumer;
std::map<std::string, MDConsumerRec> md_consumer_recs;
std::map<std::string, md::MDProducerHandle> md_producer_handles;
utils::DispatcherTimer timer;
std::map<std::string, ex_sim::OrderMatcher> matchers;
bool do_merge_with_md;
TimeStamp last_scan;
std::unique_ptr<persistence::IStore> store;
std::mutex lock;
};
@@ -138,10 +133,6 @@ ExchangeSimApplication::ExchangeSimApplication() : me(new PImpl()) {
}
me->timer.start();
me->consumer.on_security(
[this](api::Security const &s, md::MDConsumerHandle h) {
on_new_security(s, h);
});
}
ExchangeSimApplication::~ExchangeSimApplication() {}
@@ -243,6 +234,7 @@ void ExchangeSimApplication::onMessage(const FIX42::NewOrderSingle &message,
order.set_executed_quantity(0);
order.set_avg_executed_price(0);
order.set_last_executed_price(0);
order.set_last_executed_quantity(0);
*order.mutable_create_time() = utils::pb::pb_now();
*order.mutable_last_update_time() = utils::pb::pb_now();
@@ -440,6 +432,8 @@ void ExchangeSimApplication::rejectOrder(
}
}
// for the passed sec_desc, generate the book from the order matcher
// and used the associated producer to publish the MD to the world
void ExchangeSimApplication::update_md(std::string const &sec_desc) {
auto it = me->md_producer_handles.find(sec_desc);
if (it != me->md_producer_handles.end()) {
@@ -484,86 +478,86 @@ void ExchangeSimApplication::update_md(std::string const &sec_desc) {
}
}
std::string get_sim_sec_desc(api::Security const&src_sec)
{
std::string get_sim_sec_desc(api::Security const &src_sec) {
return std::string("SIM:") + src_sec.glbx().security_desc();
}
void negate_id(api::Security *sec)
{
void negate_id(api::Security *sec) {
sec->set_id(-sec->id());
if (sec->has_spread()) {
for (auto &l : *sec->mutable_spread()->mutable_legs())
for (auto &l : *sec->mutable_spread()->mutable_legs())
negate_id(l.mutable_security());
}
}
void ExchangeSimApplication::on_new_security(
api::Security const &s, md::MDConsumerHandle consumer_handle) {
md::with_security_like(s, [this, s, consumer_handle](auto &fs) {
auto s2 = s;
*s2.mutable_glbx()->mutable_security_desc() = get_sim_sec_desc(s);
s2.set_exchange(api::VENUE_SIM);
negate_id(&s2);
md::convert_security(s2, fs, true);
md::MDProducerHandle handle = me->producer.add_security(fs);
auto sec_desc = securities::descriptor(s2);
me->md_producer_handles.insert(make_pair(sec_desc, handle));
me->md_consumer_recs.insert(
make_pair(sec_desc, MDConsumerRec{consumer_handle, s2,
utils::TimeStamp::now(), Book()}));
});
void ExchangeSimApplication::persist_book(OrderBook const &b) {
auto existing = me->store->retrieve_one("books", b.sec_desc());
auto val = utils::pb::to_string(b);
if (existing.first)
me->store->update("books", b.sec_desc(), val);
else
me->store->insert("books", b.sec_desc(), val);
}
void ExchangeSimApplication::merge_with_md_book() {
void ExchangeSimApplication::on_new_security(api::Security const &s) {
std::lock_guard<std::mutex> l(me->lock);
if (!s.has_synthetic()) {
md::with_security_like(s, [this, s](auto &fs) {
auto s2 = s;
*s2.mutable_glbx()->mutable_security_desc() = get_sim_sec_desc(s);
s2.set_exchange(api::VENUE_SIM);
negate_id(&s2);
md::convert_security(s2, fs, true);
md::MDProducerHandle handle = me->producer.add_security(fs);
auto sec_desc = securities::descriptor(s2);
me->md_producer_handles.insert(make_pair(sec_desc, handle));
me->md_consumer_recs.insert(
make_pair(s.glbx().security_desc(),
MDConsumerRec{s2, utils::TimeStamp::now(), Book()}));
});
}
}
void ExchangeSimApplication::on_depth(
md::Security const &sec, md::PriceSize bids[md::COCONUT_MAX_DEPTH],
md::PriceSize asks[md::COCONUT_MAX_DEPTH]) {
std::lock_guard<std::mutex> l(me->lock);
auto now = Timer::now();
if (sec_diff(now, me->last_scan) > 5) {
me->last_scan = now;
me->consumer.scan_securities();
if (sec.has_synthetic())
return ;
auto it = me->md_consumer_recs.find(sec.glbx().security_desc());
bts_assert(it != me->md_consumer_recs.end());
// cache previous Book so we can update it place
Book previous_md_book = it->second.previous_md_book;
// populate Book structure with new bids, asks coming from outside
Book current_md_book;
for (size_t i = 0; i < md::COCONUT_MAX_DEPTH; i++) {
auto b = securities::encode_exchange_price(sec, Price(bids[i].price()));
if (!os::is_nan(b.value()))
current_md_book.bids[b] = Quantity(bids[i].size());
auto a = securities::encode_exchange_price(sec, Price(asks[i].price()));
if (!os::is_nan(a.value()))
current_md_book.asks[a] = Quantity(asks[i].size());
}
for (auto &kvp : me->md_consumer_recs) {
auto &handle = kvp.second.handle;
auto &sec = kvp.second.security;
auto &ts = kvp.second.last_read;
// update the indexed Book structure with the latest values
it->second.previous_md_book = current_md_book;
if (sec.has_synthetic()) {
//
} else if (me->do_merge_with_md) {
md::PriceSize bids[md::COCONUT_MAX_DEPTH], asks[md::COCONUT_MAX_DEPTH];
if (handle.get_depth(ts, md::COCONUT_MAX_DEPTH, bids, asks) ==
md::HandleReadResult::SUCCESS) {
Book previous_md_book = kvp.second.previous_md_book;
Book current_md_book;
for (size_t i = 0; i < md::COCONUT_MAX_DEPTH; i++) {
auto b = securities::encode_exchange_price(
sec, Price(bids[i].price()));
if (!os::is_nan(b.value()))
current_md_book.bids[b] = Quantity(bids[i].size());
auto a = securities::encode_exchange_price(
sec, Price(asks[i].price()));
if (!os::is_nan(a.value()))
current_md_book.asks[a] = Quantity(asks[i].size());
}
kvp.second.previous_md_book = current_md_book;
LOG(ExSim, Diag, "Moving {} book \nfrom: {}\nto: {}", kvp.first,
previous_md_book, current_md_book);
merge_with_md_book(kvp.first, previous_md_book, current_md_book);
}
}
}
// from the differences between current_md_book and previous_md_book
// we generate a set of orders that we apply to our order book
merge_with_order_book(it->second.security.glbx().security_desc(),
previous_md_book, current_md_book);
}
void ExchangeSimApplication::merge_with_md_book(std::string const &sec_desc,
Book const &src,
Book const &dest) {
void ExchangeSimApplication::merge_with_order_book(std::string const &sec_desc,
Book const &src,
Book const &dest) {
auto &matcher = me->matchers[sec_desc];
auto src_book = matcher.get_book();
auto transform = compute_book_transform(src, dest);
@@ -629,16 +623,6 @@ void ExchangeSimApplication::merge_with_md_book(std::string const &sec_desc,
std::get<1>(sz), std::get<2>(sz));
}
void ExchangeSimApplication::persist_book(OrderBook const &b) {
auto existing = me->store->retrieve_one("books", b.sec_desc());
auto val = utils::pb::to_string(b);
if (existing.first)
me->store->update("books", b.sec_desc(), val);
else
me->store->insert("books", b.sec_desc(), val);
}
GCC_DIAG_RESTORE()
GCC_DIAG_RESTORE()
} // namespace ex_sim
@@ -39,6 +39,7 @@ GCC_DIAG_OFF("-Wignored-qualifiers")
#include <quickfix/fix42/NewOrderSingle.h>
#include <quickfix/fix42/OrderCancelRequest.h>
#include "server/sparkd/exchange_sim/IMDPlayerObserver.h"
namespace bts {
namespace api {class Security;}
@@ -48,7 +49,8 @@ class Order;
struct Book;
class OrderBook;
class ExchangeSimApplication
: public FIX::Application,
: public sim::IMDPlayerObserver,
public FIX::Application,
public FIX::MessageCracker
{
public:
@@ -58,21 +60,21 @@ public:
private:
// Application overloads
void onCreate( const FIX::SessionID& ) ;
void onLogon( const FIX::SessionID& sessionID );
void onLogout( const FIX::SessionID& sessionID );
void toAdmin( FIX::Message&, const FIX::SessionID& );
void onCreate( const FIX::SessionID& ) override;
void onLogon( const FIX::SessionID& sessionID )override;
void onLogout( const FIX::SessionID& sessionID )override;
void toAdmin( FIX::Message&, const FIX::SessionID& )override;
void toApp( FIX::Message&, const FIX::SessionID& )
throw( FIX::DoNotSend ) ;
throw( FIX::DoNotSend ) override;
void fromAdmin( const FIX::Message&, const FIX::SessionID& )
throw( FIX::FieldNotFound, FIX::IncorrectDataFormat, FIX::IncorrectTagValue, FIX::RejectLogon ) ;
throw( FIX::FieldNotFound, FIX::IncorrectDataFormat, FIX::IncorrectTagValue, FIX::RejectLogon ) override;
void fromApp( const FIX::Message& message, const FIX::SessionID& sessionID )
throw( FIX::FieldNotFound, FIX::IncorrectDataFormat, FIX::IncorrectTagValue, FIX::UnsupportedMessageType );
throw( FIX::FieldNotFound, FIX::IncorrectDataFormat, FIX::IncorrectTagValue, FIX::UnsupportedMessageType )override;
// MessageCracker overloads
void onMessage( const FIX42::NewOrderSingle&, const FIX::SessionID& );
void onMessage( const FIX42::OrderCancelReplaceRequest&, const FIX::SessionID& );
void onMessage( const FIX42::OrderCancelRequest&, const FIX::SessionID& );
void onMessage( const FIX42::NewOrderSingle&, const FIX::SessionID& )override;
void onMessage( const FIX42::OrderCancelReplaceRequest&, const FIX::SessionID& )override;
void onMessage( const FIX42::OrderCancelRequest&, const FIX::SessionID& )override;
// Order functionality
void processOrder( const Order& );
@@ -93,10 +95,14 @@ public:
struct PImpl;
std::unique_ptr<PImpl> me;
struct MDConsumerRec;
void on_new_security(api::Security const&sec, md::MDConsumerHandle handle);
void merge_with_md_book(std::string const &sec_desc, Book const&previous,Book const&current);
void update_md(std::string const&sec_desc);
void persist_book(OrderBook const &b);
void on_new_security(md::Security const&) override;
void on_depth(md::Security const&s,
md::PriceSize bids[md::COCONUT_MAX_DEPTH],
md::PriceSize asks[md::COCONUT_MAX_DEPTH]) override;
void merge_with_order_book(std::string const &sec_desc, Book const&previous,Book const&current);
};
}}
@@ -0,0 +1,17 @@
#pragma once
#include "lib/md/PriceSize.h"
#include "lib/md/Shm/MDStructure.h"
namespace bts {
namespace api {class Security;}
namespace sim {
struct IMDPlayerObserver {
virtual void on_new_security(md::Security const&) = 0;
virtual void on_depth(md::Security const&s,
md::PriceSize bids[md::COCONUT_MAX_DEPTH],
md::PriceSize asks[md::COCONUT_MAX_DEPTH]) = 0;
};
}}