From 6960ee1a1243ddd3dfd16230c51110b4ce73a67f Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Mon, 16 Jul 2018 20:04:37 +0000 Subject: [PATCH] Changing replay mechanis in sim to allow for cert and sim md. --- src/lib/md/protos/co2.proto | 2 +- src/lib/md/relay/RelayRecorder.h | 1 + .../sparkd/exchange_sim/ExchangeSim.cpp | 55 ++++++- .../exchange_sim/ExchangeSimApplication.cpp | 146 ++++++++---------- .../exchange_sim/ExchangeSimApplication.h | 32 ++-- .../sparkd/exchange_sim/IMDPlayerObserver.h | 17 ++ 6 files changed, 156 insertions(+), 97 deletions(-) create mode 100644 src/server/sparkd/exchange_sim/IMDPlayerObserver.h diff --git a/src/lib/md/protos/co2.proto b/src/lib/md/protos/co2.proto index a173cf257..f6f99a56d 100644 --- a/src/lib/md/protos/co2.proto +++ b/src/lib/md/protos/co2.proto @@ -56,7 +56,7 @@ message MDRequest { TRADE_LOG = 0x8; RFQ = 0x10; CUSTOMER_BBO = 0x20; - MISC = 0x40; + MISC = 0x40; } string host = 2; } diff --git a/src/lib/md/relay/RelayRecorder.h b/src/lib/md/relay/RelayRecorder.h index e3768dd2c..6faab5b5d 100644 --- a/src/lib/md/relay/RelayRecorder.h +++ b/src/lib/md/relay/RelayRecorder.h @@ -29,6 +29,7 @@ void print_recording(const std::string&); void unprint_recording(const std::string& src, const std::string& dest); void trace_recording(const std::string& file, security_key key); void replay_recording(const std::string&, int32_t delay_ms); +void replay_url(const std::string& url, IRelayWriter& writer, int32_t naggle_ms); }}//namespace diff --git a/src/server/sparkd/exchange_sim/ExchangeSim.cpp b/src/server/sparkd/exchange_sim/ExchangeSim.cpp index c3a6c5028..948c3273a 100644 --- a/src/server/sparkd/exchange_sim/ExchangeSim.cpp +++ b/src/server/sparkd/exchange_sim/ExchangeSim.cpp @@ -15,6 +15,11 @@ #include #include "lib/utils/bts_chrono.h" +#include "lib/md/relay/RelayClient.h" +#include "lib/domain_model/Security.pb.h" +#include "lib/md/relay/RelayRecorder.h" + +#include namespace bts { namespace sim { @@ -36,6 +41,48 @@ auto cfg = "ValidateUserDefinedFields=N\n" "ValidateFieldsOutOfOrder=N\n"; + +struct MDPlayer : md::IRelayWriter +{ + MDPlayer (IMDPlayerObserver &obs,std::map &known_securities) : + obs (obs), + known_securities(known_securities) + {} + + + void write(const co2::MDReply& data, bool* ) override { + for (auto const &ch : data.channels()) + for (auto const &cr : ch.updates()) { + auto it = known_securities.find(cr.key()); + + api::Security const *sec_ptr (nullptr); + if (it == known_securities.end()) { + bts_assert(cr.has_security()); + sec_ptr = &cr.security(); + known_securities[cr.key()] = *sec_ptr; + obs.on_new_security(*sec_ptr); + } + else + sec_ptr = &it->second; + + auto const &book = cr.book(); + md::PriceSize bids[md::COCONUT_MAX_DEPTH], asks[md::COCONUT_MAX_DEPTH]; + + size_t i = 0; + for (auto &b : book.bids()) + bids[i++] = md::PriceSize(b.mantissa(),b.exponent(),b.size()); + + size_t n = 0; + for (auto &a : book.asks()) + asks[n++] = md::PriceSize(a.mantissa(),a.exponent(),a.size()); + obs.on_depth(*sec_ptr, bids, asks); + } + } + IMDPlayerObserver &obs; + std::map &known_securities; +}; + + int start_exchange_sim() { try @@ -55,9 +102,13 @@ int start_exchange_sim() FIX::SocketAcceptor acceptor( application, storeFactory, settings, logFactory ); acceptor.start(); + + std::map known_securities; while ( true ) { - std::this_thread::sleep_for(10ms) ; - application.merge_with_md_book(); + MDPlayer player(application,known_securities); + md::replay_url("http://mdreplay.bluetradesys.local/spark/futures.coconut", + player, + config::get().naggle_ms()); } acceptor.stop(); return 0; diff --git a/src/server/sparkd/exchange_sim/ExchangeSimApplication.cpp b/src/server/sparkd/exchange_sim/ExchangeSimApplication.cpp index 068bb3299..b757e23ba 100644 --- a/src/server/sparkd/exchange_sim/ExchangeSimApplication.cpp +++ b/src/server/sparkd/exchange_sim/ExchangeSimApplication.cpp @@ -6,7 +6,6 @@ #include "lib/config/protos/ExchangeSimCfg.pb.h" #include "lib/domain_model/Security.h" #include "lib/logging/logging.h" -#include "lib/md/MDConsumer.h" #include "lib/md/MDProducer.h" #include "lib/md/Shm/FixedSecurity.h" #include "lib/md/Shm/MDStructure.h" @@ -93,7 +92,6 @@ convert(const FIX::ManualOrderIndicator &moi) { } struct ExchangeSimApplication::MDConsumerRec { - md::MDConsumerHandle handle; api::Security security; utils::TimeStamp last_read; Book previous_md_book; @@ -102,7 +100,6 @@ struct ExchangeSimApplication::MDConsumerRec { struct ExchangeSimApplication::PImpl { PImpl() : producer(co2::PLATFORM_SIM), - consumer(api::VENUE_GLBX, co2::PLATFORM_MDP3ONIXS), timer(utils::Dispatcher::background(), 200, [=]() { producer.heartbeat(); }), do_merge_with_md(config::get().merge_with_md()), @@ -110,13 +107,11 @@ struct ExchangeSimApplication::PImpl { IDGenerator m_generator; md::MDProducer producer; - md::MDConsumer consumer; std::map md_consumer_recs; std::map md_producer_handles; utils::DispatcherTimer timer; std::map matchers; bool do_merge_with_md; - TimeStamp last_scan; std::unique_ptr store; std::mutex lock; }; @@ -138,10 +133,6 @@ ExchangeSimApplication::ExchangeSimApplication() : me(new PImpl()) { } me->timer.start(); - me->consumer.on_security( - [this](api::Security const &s, md::MDConsumerHandle h) { - on_new_security(s, h); - }); } ExchangeSimApplication::~ExchangeSimApplication() {} @@ -243,6 +234,7 @@ void ExchangeSimApplication::onMessage(const FIX42::NewOrderSingle &message, order.set_executed_quantity(0); order.set_avg_executed_price(0); order.set_last_executed_price(0); + order.set_last_executed_quantity(0); *order.mutable_create_time() = utils::pb::pb_now(); *order.mutable_last_update_time() = utils::pb::pb_now(); @@ -440,6 +432,8 @@ void ExchangeSimApplication::rejectOrder( } } +// for the passed sec_desc, generate the book from the order matcher +// and used the associated producer to publish the MD to the world void ExchangeSimApplication::update_md(std::string const &sec_desc) { auto it = me->md_producer_handles.find(sec_desc); if (it != me->md_producer_handles.end()) { @@ -484,86 +478,86 @@ void ExchangeSimApplication::update_md(std::string const &sec_desc) { } } -std::string get_sim_sec_desc(api::Security const&src_sec) -{ +std::string get_sim_sec_desc(api::Security const &src_sec) { return std::string("SIM:") + src_sec.glbx().security_desc(); } -void negate_id(api::Security *sec) -{ +void negate_id(api::Security *sec) { sec->set_id(-sec->id()); if (sec->has_spread()) { - for (auto &l : *sec->mutable_spread()->mutable_legs()) + for (auto &l : *sec->mutable_spread()->mutable_legs()) negate_id(l.mutable_security()); } } -void ExchangeSimApplication::on_new_security( - api::Security const &s, md::MDConsumerHandle consumer_handle) { - md::with_security_like(s, [this, s, consumer_handle](auto &fs) { - auto s2 = s; - *s2.mutable_glbx()->mutable_security_desc() = get_sim_sec_desc(s); - s2.set_exchange(api::VENUE_SIM); - negate_id(&s2); - md::convert_security(s2, fs, true); - md::MDProducerHandle handle = me->producer.add_security(fs); - auto sec_desc = securities::descriptor(s2); - me->md_producer_handles.insert(make_pair(sec_desc, handle)); - me->md_consumer_recs.insert( - make_pair(sec_desc, MDConsumerRec{consumer_handle, s2, - utils::TimeStamp::now(), Book()})); - }); +void ExchangeSimApplication::persist_book(OrderBook const &b) { + auto existing = me->store->retrieve_one("books", b.sec_desc()); + auto val = utils::pb::to_string(b); + + if (existing.first) + me->store->update("books", b.sec_desc(), val); + else + me->store->insert("books", b.sec_desc(), val); } -void ExchangeSimApplication::merge_with_md_book() { +void ExchangeSimApplication::on_new_security(api::Security const &s) { + std::lock_guard l(me->lock); + if (!s.has_synthetic()) { + md::with_security_like(s, [this, s](auto &fs) { + auto s2 = s; + *s2.mutable_glbx()->mutable_security_desc() = get_sim_sec_desc(s); + s2.set_exchange(api::VENUE_SIM); + negate_id(&s2); + md::convert_security(s2, fs, true); + md::MDProducerHandle handle = me->producer.add_security(fs); + auto sec_desc = securities::descriptor(s2); + me->md_producer_handles.insert(make_pair(sec_desc, handle)); + me->md_consumer_recs.insert( + make_pair(s.glbx().security_desc(), + MDConsumerRec{s2, utils::TimeStamp::now(), Book()})); + }); + } +} + +void ExchangeSimApplication::on_depth( + md::Security const &sec, md::PriceSize bids[md::COCONUT_MAX_DEPTH], + md::PriceSize asks[md::COCONUT_MAX_DEPTH]) { std::lock_guard l(me->lock); - auto now = Timer::now(); - if (sec_diff(now, me->last_scan) > 5) { - me->last_scan = now; - me->consumer.scan_securities(); + if (sec.has_synthetic()) + return ; + + auto it = me->md_consumer_recs.find(sec.glbx().security_desc()); + bts_assert(it != me->md_consumer_recs.end()); + + // cache previous Book so we can update it place + Book previous_md_book = it->second.previous_md_book; + + // populate Book structure with new bids, asks coming from outside + Book current_md_book; + for (size_t i = 0; i < md::COCONUT_MAX_DEPTH; i++) { + + auto b = securities::encode_exchange_price(sec, Price(bids[i].price())); + if (!os::is_nan(b.value())) + current_md_book.bids[b] = Quantity(bids[i].size()); + + auto a = securities::encode_exchange_price(sec, Price(asks[i].price())); + if (!os::is_nan(a.value())) + current_md_book.asks[a] = Quantity(asks[i].size()); } - for (auto &kvp : me->md_consumer_recs) { - auto &handle = kvp.second.handle; - auto &sec = kvp.second.security; - auto &ts = kvp.second.last_read; + // update the indexed Book structure with the latest values + it->second.previous_md_book = current_md_book; - if (sec.has_synthetic()) { - // - } else if (me->do_merge_with_md) { - - md::PriceSize bids[md::COCONUT_MAX_DEPTH], asks[md::COCONUT_MAX_DEPTH]; - - if (handle.get_depth(ts, md::COCONUT_MAX_DEPTH, bids, asks) == - md::HandleReadResult::SUCCESS) { - Book previous_md_book = kvp.second.previous_md_book; - Book current_md_book; - for (size_t i = 0; i < md::COCONUT_MAX_DEPTH; i++) { - - auto b = securities::encode_exchange_price( - sec, Price(bids[i].price())); - if (!os::is_nan(b.value())) - current_md_book.bids[b] = Quantity(bids[i].size()); - - auto a = securities::encode_exchange_price( - sec, Price(asks[i].price())); - if (!os::is_nan(a.value())) - current_md_book.asks[a] = Quantity(asks[i].size()); - } - - kvp.second.previous_md_book = current_md_book; - LOG(ExSim, Diag, "Moving {} book \nfrom: {}\nto: {}", kvp.first, - previous_md_book, current_md_book); - merge_with_md_book(kvp.first, previous_md_book, current_md_book); - } - } - } + // from the differences between current_md_book and previous_md_book + // we generate a set of orders that we apply to our order book + merge_with_order_book(it->second.security.glbx().security_desc(), + previous_md_book, current_md_book); } -void ExchangeSimApplication::merge_with_md_book(std::string const &sec_desc, - Book const &src, - Book const &dest) { +void ExchangeSimApplication::merge_with_order_book(std::string const &sec_desc, + Book const &src, + Book const &dest) { auto &matcher = me->matchers[sec_desc]; auto src_book = matcher.get_book(); auto transform = compute_book_transform(src, dest); @@ -629,16 +623,6 @@ void ExchangeSimApplication::merge_with_md_book(std::string const &sec_desc, std::get<1>(sz), std::get<2>(sz)); } -void ExchangeSimApplication::persist_book(OrderBook const &b) { - auto existing = me->store->retrieve_one("books", b.sec_desc()); - auto val = utils::pb::to_string(b); - - if (existing.first) - me->store->update("books", b.sec_desc(), val); - else - me->store->insert("books", b.sec_desc(), val); -} - GCC_DIAG_RESTORE() GCC_DIAG_RESTORE() } // namespace ex_sim diff --git a/src/server/sparkd/exchange_sim/ExchangeSimApplication.h b/src/server/sparkd/exchange_sim/ExchangeSimApplication.h index 6dfd21c1a..8f7cdc54e 100644 --- a/src/server/sparkd/exchange_sim/ExchangeSimApplication.h +++ b/src/server/sparkd/exchange_sim/ExchangeSimApplication.h @@ -39,6 +39,7 @@ GCC_DIAG_OFF("-Wignored-qualifiers") #include #include +#include "server/sparkd/exchange_sim/IMDPlayerObserver.h" namespace bts { namespace api {class Security;} @@ -48,7 +49,8 @@ class Order; struct Book; class OrderBook; class ExchangeSimApplication - : public FIX::Application, + : public sim::IMDPlayerObserver, + public FIX::Application, public FIX::MessageCracker { public: @@ -58,21 +60,21 @@ public: private: // Application overloads - void onCreate( const FIX::SessionID& ) ; - void onLogon( const FIX::SessionID& sessionID ); - void onLogout( const FIX::SessionID& sessionID ); - void toAdmin( FIX::Message&, const FIX::SessionID& ); + void onCreate( const FIX::SessionID& ) override; + void onLogon( const FIX::SessionID& sessionID )override; + void onLogout( const FIX::SessionID& sessionID )override; + void toAdmin( FIX::Message&, const FIX::SessionID& )override; void toApp( FIX::Message&, const FIX::SessionID& ) - throw( FIX::DoNotSend ) ; + throw( FIX::DoNotSend ) override; void fromAdmin( const FIX::Message&, const FIX::SessionID& ) - throw( FIX::FieldNotFound, FIX::IncorrectDataFormat, FIX::IncorrectTagValue, FIX::RejectLogon ) ; + throw( FIX::FieldNotFound, FIX::IncorrectDataFormat, FIX::IncorrectTagValue, FIX::RejectLogon ) override; void fromApp( const FIX::Message& message, const FIX::SessionID& sessionID ) - throw( FIX::FieldNotFound, FIX::IncorrectDataFormat, FIX::IncorrectTagValue, FIX::UnsupportedMessageType ); + throw( FIX::FieldNotFound, FIX::IncorrectDataFormat, FIX::IncorrectTagValue, FIX::UnsupportedMessageType )override; // MessageCracker overloads - void onMessage( const FIX42::NewOrderSingle&, const FIX::SessionID& ); - void onMessage( const FIX42::OrderCancelReplaceRequest&, const FIX::SessionID& ); - void onMessage( const FIX42::OrderCancelRequest&, const FIX::SessionID& ); + void onMessage( const FIX42::NewOrderSingle&, const FIX::SessionID& )override; + void onMessage( const FIX42::OrderCancelReplaceRequest&, const FIX::SessionID& )override; + void onMessage( const FIX42::OrderCancelRequest&, const FIX::SessionID& )override; // Order functionality void processOrder( const Order& ); @@ -93,10 +95,14 @@ public: struct PImpl; std::unique_ptr me; struct MDConsumerRec; - void on_new_security(api::Security const&sec, md::MDConsumerHandle handle); - void merge_with_md_book(std::string const &sec_desc, Book const&previous,Book const¤t); void update_md(std::string const&sec_desc); void persist_book(OrderBook const &b); + + void on_new_security(md::Security const&) override; + void on_depth(md::Security const&s, + md::PriceSize bids[md::COCONUT_MAX_DEPTH], + md::PriceSize asks[md::COCONUT_MAX_DEPTH]) override; + void merge_with_order_book(std::string const &sec_desc, Book const&previous,Book const¤t); }; }} diff --git a/src/server/sparkd/exchange_sim/IMDPlayerObserver.h b/src/server/sparkd/exchange_sim/IMDPlayerObserver.h new file mode 100644 index 000000000..6ba748133 --- /dev/null +++ b/src/server/sparkd/exchange_sim/IMDPlayerObserver.h @@ -0,0 +1,17 @@ +#pragma once + +#include "lib/md/PriceSize.h" +#include "lib/md/Shm/MDStructure.h" + +namespace bts { +namespace api {class Security;} +namespace sim { + +struct IMDPlayerObserver { + virtual void on_new_security(md::Security const&) = 0; + virtual void on_depth(md::Security const&s, + md::PriceSize bids[md::COCONUT_MAX_DEPTH], + md::PriceSize asks[md::COCONUT_MAX_DEPTH]) = 0; +}; + +}}