refactoring onix options

This reverts commit 4f05b95bd0.

notes

@@ -0,0 +1,6 @@
+cd ~/spark/build/clang/debug/; cmake --build . --target install; rm -rf ~/spark/install/Debug/bin.zip; tar -zcvf  ~/spark/install/Debug/bin.zip ~/spark/install/Debug/bin; scp ~/spark/install/Debug/bin.zip bts@10.1.10.31:~/13000/; cd ~/spark/
+cd ~/13000/; tar -xf bin.zip; mv home/bts/spark/install/Debug/bin/* sparkbin/; rm -rf ~/13000/home/; docker container stop spark_13000; docker container start spark_13000
+cd ~/13000/; tar -xf bin.zip; mv home/bts/spark/install/Debug/bin/* sparkbin/; rm -rf ~/13000/home/; cp -r sparkbin ~/.sparkd-dev/mounts/sparkd_build/; 
+ES SECURITYTYPE_FUTURE at VENUE_GLBX
+./sparkd md dump --symbol ES --type future --venue glbx --platform mdp3onixsex
+BT5u53r

src/lib/api/md/GlbxMarketDataConfig.proto

@@ -5,18 +5,12 @@ import "lib/api/domain/SecurityCriteria.proto";
 package bts.api.md;
 
 option csharp_namespace = "bts.api.md";
-
+message OnixsOptions {
+  string onixs_license = 1;
+}
 
 message GlbxMarketDataConfig {
   uint32 unique_check_value = 1;
-  //uint32 num_depth_levels = 1;
-  //uint32 depth_throttle_ms = 2;
   bts.api.domain.SecurityCriteria security_criteria = 3;
-  // Market data isn't subscribed for securities listed here (even if they are
+  OnixsOptions onixs_options = 7;
-  // included by 'security_criteria'). There are some securities that don't
-  // fit into our system just yet, such as 'BTCBIDR', which trades in prices
-  // larger than our cpp 'domain::Price' class supports.
-  //repeated string excluded_securities = 4;
-  //string url_rest_base = 5;
-  //string url_ws_base = 6;
 }

src/lib/api/md/MarketDataRecordingConfig.proto

@@ -12,4 +12,5 @@ message MarketDataRecordingConfig {
    uint32 num_of_seconds_to_run = 10;
    bts.api.Venue venue = 20;
    uint32 n_book_levels = 30;
+   bool record_books = 40; //should be true
 }

src/lib/api/md/MarketDataReplayConfig.proto

@@ -13,4 +13,7 @@ option cc_enable_arenas = true;
 message MarketDataReplayConfig {
    bts.api.Venue venue = 10;
    double replay_speed = 20;
+   bool generate_book_on_trade = 30;
+   uint32 generated_book_levels = 31;
+   uint32 generated_book_quantity = 32;
 }

src/lib/config/protos/market_data.proto

@@ -186,7 +186,7 @@ message ICEOptions {
   string ice_config_file = 2;
   repeated IceChannelConfig channel = 3;
 }
-
+//need to remoce 
 message OnixsOptions {
   string onixs_license = 1;
 }

src/lib/md/MarketDataGateway.cpp

@@ -98,7 +98,7 @@ struct MDReplayData {
    MDReplayData(const bts::api::md::MarketDataReplayConfig &replay_conf)
        : replay_conf(replay_conf), venue(replay_conf.venue()), sec_index(),
          md_producer(bts::co2::PLATFORM_SIM),
-         md_replayer(replay_conf.venue(), sec_index, md_producer) {}
+         md_replayer(replay_conf.venue(), sec_index, md_producer, true) {}
 };
 
 struct CCMDSourceData {

src/lib/md/recording/MDRecorder.cpp

@@ -110,6 +110,7 @@ struct MDRecordingSettings {
    const uint64_t total_runtime_ns = 300000000000;      // 5 mins
    const uint64_t thread_sleep_time_ns = 100000;        // 100 usecs
    const uint64_t security_scan_period_ns = 1000000000; // 1 sec
+   const bool record_books = true;
 };
 
 // So we dont have to create multiple bids/asks
@@ -146,7 +147,7 @@ struct MDRecorder::PImpl {
              recording_config.n_book_levels(),
              static_cast<uint64_t>(recording_config.num_of_seconds_to_run()) *
                  1000 * 1000 * 1000,
-             100000, 1000000000}),
+             100000, 1000000000, recording_config.record_books()}),
          md_client(md_client) {}
 };
 
@@ -184,7 +185,7 @@ void MDRecorder::_poll_md() {
       uint64_t now_ns = core::TimeStamp::now_ns();
       for (auto &kv : me->handles) {
          MDRecordingSecEntry &md_entry = kv.second;
-         if(me->settings.n_book_levels > 0)
+         if(me->settings.n_book_levels > 0 && me->settings.record_books)
             _poll_books(md_entry);
          _poll_trades(md_entry);
          _poll_misc(md_entry);

src/lib/md/recording/MDReplayer.cpp

@@ -1,3 +1,6 @@
+#include "lib/domain/BookSide.h"
+#include "lib/md/GeneralSecurityIndex.h"
+#include "lib/orderman/orderman_utils/Book.h"
 #if !defined(_WIN32) && defined(SPARK_ENABLE_CCAPI)
 #include "MDReplayer.h"
 #include "lib/config/config.h"
@@ -53,19 +56,26 @@ void copy_books(const bts::co2::BookMessage &from, bts::domain::Book &to) {
 struct MDReplayer ::PImpl {
    bts::api::Venue venue;
    std::set<uint32_t> sec_keys = {};
+   std::map<uint32_t, double> sec_keys_to_ticks;
    uint64_t start_time = bts::core::TimeStamp::now_ns();
    GeneralMarketDataListener listener;
+   bool generate_books = false;
+   double generated_book_level = 3;
+   uint generated_book_quantity = 6;
    PImpl(
        bts::api::Venue venue,
        bts::md::GeneralSecurityIndex &index,
-       bts::md::MDProducer &md_producer)
+       bts::md::MDProducer &md_producer,
-       : venue(venue), listener(index, md_producer, venue) {}
+       bool generate_books)
+       : venue(venue), listener(index, md_producer, venue),
+         generate_books(generate_books) {}
 };
 MDReplayer::MDReplayer(
     bts::api::Venue venue,
     bts::md::GeneralSecurityIndex &index,
-    bts::md::MDProducer &md_producer)
+    bts::md::MDProducer &md_producer,
-    : me(new PImpl(venue, index, md_producer)) {
+    bool generate_books)
+    : me(new PImpl(venue, index, md_producer, generate_books)) {
    LOG(md, Info, "MDReplayer::MDReplayer venue: {}", Venue_Name(venue));
 }
 
@@ -119,7 +129,41 @@ void MDReplayer::_replay_file_once(const std::string &file_name) {
    LOG(md, Info, "MDReplayer::_replay_file_once time: {} ns", end_ns - beg_ns);
    in.close();
 }
+co2::PriceSizeMessage get_psm(const double &quantity, const bts::domain::Price & trade_price, double ticks, uint i ){
+   auto price = trade_price + bts::domain::Price::from_double(ticks * i);
+   co2::PriceSizeMessage pm;
+   pm.set_size(quantity);
+   pm.set_mantissa(price.mantissa());
+   pm.set_exponent(price.exponent());
+   return pm;
+}
+void trade_to_book(
+    const bts::domain::TradeRecord &from, bts::co2::RecordedMDMessage &to,const double & ticks, const uint &levels, const double &quantity) {
+   to.set_sec_key(from.sec_id.value());
+   to.set_timestamp(
+       from.time_stamp.millis()); // Assuming that time_stamp is in milliseconds
 
+   co2::TotalBook *tb =
+       to.mutable_total_book(); 
+
+   co2::BookMessage *direct_book =
+       tb->mutable_direct_book();
+
+   for (uint i = 0; i < levels; i++) {
+      direct_book->add_asks()->CopyFrom(get_psm(quantity,from.trade_price,ticks,i));
+      direct_book->add_bids()->CopyFrom(get_psm(quantity,from.trade_price,-ticks,i+1));
+   }
+}
+void MDReplayer::_trades_to_books(
+    std::vector<domain::TradeRecord const *> trades) {
+   for (auto &tr : trades) {
+      auto it = me->sec_keys_to_ticks.find(tr->sec_id.value());
+      if (it == me->sec_keys_to_ticks.end()) {continue;}
+      bts::co2::RecordedMDMessage rm;
+      trade_to_book(*tr, rm, it->second, me->generated_book_level, me->generated_book_quantity);
+      _replay_message(rm);
+   }
+}
 void MDReplayer::_replay_message(const bts::co2::RecordedMDMessage &m) {
    switch (m.MDRecord_case()) {
    case (co2::RecordedMDMessage::MDRecordCase::kSecurity): {
@@ -127,6 +171,7 @@ void MDReplayer::_replay_message(const bts::co2::RecordedMDMessage &m) {
           utils::pb::to_string(m));
       bts::domain::Security new_sec = m.security();
       me->sec_keys.insert(m.security().id());
+      me->sec_keys_to_ticks[m.security().id()] = m.security().contract_spec().low_tick();
       std::vector<domain::Security const *> sec_vec = {};
       sec_vec.push_back(&new_sec);
       me->listener.on_securities(sec_vec);
@@ -153,10 +198,13 @@ void MDReplayer::_replay_message(const bts::co2::RecordedMDMessage &m) {
       break;
    }
    case (co2::RecordedMDMessage::MDRecordCase::kTrade): {
+
       bts::domain::TradeRecord tr(m.trade());
       std::vector<domain::TradeRecord const *> trades;
       trades.push_back(&tr);
       me->listener.on_trades(trades);
+      if (me->generate_books)
+         MDReplayer::_trades_to_books(trades);
       break;
    }
    default: {

src/lib/md/recording/MDReplayer.h

@@ -25,7 +25,8 @@ class MDReplayer {
    MDReplayer(
        bts::api::Venue venue,
        bts::md::GeneralSecurityIndex &index,
-       bts::md::MDProducer &md_producer);
+       bts::md::MDProducer &md_producer,
+       bool generate_books);
    ~MDReplayer();
    void run();
 
@@ -34,6 +35,7 @@ class MDReplayer {
    std::unique_ptr<PImpl> me;
    void _replay_file_once(const std::string &file_name);
    void _replay_message(const bts::co2::RecordedMDMessage &);
+   void _trades_to_books(std::vector<domain::TradeRecord const *> trades);
 };
 } // namespace bts::md::recording
 #endif