file restructuring for sdk

WARNING: This breaks the sparkd clang build (only works with gcc)

src/client/spark_api_server/Program.cs

@@ -38,9 +38,9 @@
          var gwy = new bts.websocket_core.WebsocketConnectionGateway();
          var prms = new WebsocketServerConfig {
             DefaultEndPoint = "/",
-            Host = "0.0.0.0",
+            Host = "titan",
             Scheme = "http",
-            Port = "3000",
+            Port = "8080",
             KeepAliveIntervalSecs = 0.1,
          };
          gwy.init(sync);

src/client/spark_websocket/Core/WebsocketConnectionGateway.cs

@@ -424,7 +424,6 @@ namespace bts.websocket_core {
       /// </summary>
       /// <param name="server_params">The parameters for the websocket server includind the URL and endpoint,</param>
       public Task start_listening(WebsocketServerConfig server_params) {
-         LoggerContext logger = new(Log.Gen, $"WebsocketGateway.start_listening ", "");
          // start listening, assert that init has been called already
          bts.Log.notnull(_sync);
          bts.Log.isnull(_server_params);
@@ -434,7 +433,6 @@ namespace bts.websocket_core {
          _server_params = server_params;
          var builder = WebApplication.CreateBuilder();
          builder.WebHost.UseUrls(_server_params.Uri());
-         logger.warn(_server_params.Uri());
          // var MyAllowSpecificOrigins = "_myAllowSpecificOrigins";
          builder.Services.AddLogging(builder => {
             builder.AddFilter("Microsoft", Microsoft.Extensions.Logging.LogLevel.Warning)

src/lib/orderman/actors/MDSource.h

@@ -5,7 +5,7 @@
 #include "lib/md/IntPrice.h"
 #include "lib/md/MDConsumer.h"
 #include "lib/md/MDConsumerHandle.h"
-#include "lib/md/cme/MDP3Utils.h"
+#include "server/sparkd/md/cme/MDP3Utils.h"
 #include "lib/mercury/actors/Actor.h"
 #include "lib/mercury/actors/IPollableActor.h"
 #include "lib/mercury/events/TimerEvent.h"

src/sdk/examples/md_publisher/md_publisher_example.cpp

@@ -1,6 +1,6 @@
 #include "spark/lib/logging/logging.h"
 #include "spark/lib/md/MDProducer.h"
-#include "spark/lib/md/cme/MDP3Utils.h"
+#include "server/sparkd/md/cme/MDP3Utils.h"
 #include "spark/lib/tracing/bts_trace.h"
 #include "spark/lib/utils/PBUtils.h"
 #include "spark/lib/utils/bts_chrono.h"

src/server/sparkd/cmds/md/cmds/Channels.cpp

@@ -1,5 +1,5 @@
 #include "Channels.h"
-#include "lib/md/cme/MDP3Utils.h"
+#include "server/sparkd/md/cme/MDP3Utils.h"
 
 namespace bts::cmds::md::cmds {
 

src/server/sparkd/main.cpp

@@ -1,3 +1,4 @@
+#include "server/sparkd/md/cme/CMEActivSession.h"
 #include "lib/testing/catch_test_runner.h"
 #include "lib/utils/init_spark_lib.h"
 #include "server/sparkd/cmds/AdminReport.h"
@@ -8,9 +9,8 @@
 #include "server/sparkd/cmds/Run.h"
 #include "server/sparkd/cmds/Web.h"
 #include "server/sparkd/cmds/crypto/CryptoCmd.h"
-#include "server/sparkd/md/cme/CMEActivSession.h"
-
 #include <CLI11/CLI11.hpp>
+
 #include <iostream>
 #include <string>
 

src/server/sparkd/md/MarketDataGateway.cpp

@@ -1,6 +1,5 @@
 #include "MarketDataGateway.h"
 #include "lib/api/md/EmumbaDeribitMarketDataConfig.pb.h"
-#include "lib/api/md/GlbxMarketDataConfig.pb.h"
 #include "lib/api/md/MarketDataConfig.pb.h"
 #include "lib/api/md/MarketDataConfigUtils.h"
 #include "lib/api/md/MarketDataReplayConfig.pb.h"
@@ -26,7 +25,6 @@
 #include "server/sparkd/md/binance/BinanceSubscription.h"
 #include "server/sparkd/md/binance_us/BinanceUsSubscription.h"
 #include "server/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.h"
-#include "server/sparkd/md/cme/CMEActivSession.h"
 #include "server/sparkd/md/coinbase/CoinbaseSubscription.h"
 #include "server/sparkd/md/deribit/DeribitSubscription.h"
 #include "server/sparkd/md/emumba/EmumbaDeribitData.h"
@@ -34,6 +32,7 @@
 #include "server/sparkd/md/okx/OkxSubscription.h"
 #include "server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h"
 #include "server/sparkd/md/recording/MDReplayer.h"
+#include "server/sparkd/md/cme/CMEActivSession.h"
 
 #include <bits/stdint-uintn.h>
 #include <boost/filesystem.hpp>
@@ -128,15 +127,19 @@ void MarketDataGateway::run() {
    }
    LOG(md, Info, "MarketDataGateway::run starting t_id:{} t_name:{}",
        os::get_thread_id(), os::get_thread_name());
+
    _start_cc(me_->md_config);
+   
    if (!me_->md_config.emumba_deribit().empty()) {
       bts_assert(me_->md_config.emumba_deribit().size() == 1);
       _start_emumba_deribit(me_->md_config.emumba_deribit().at(0));
    }
+
    if (!me_->md_config.glbx().empty()) {
       bts_assert(me_->md_config.glbx().size() == 1);
       _start_glbx(me_->md_config.glbx().at(0));
    }
+
    _start_replays(me_->md_config);
    while (true) {
       std::this_thread::sleep_for(std::chrono::seconds(1));
@@ -232,33 +235,26 @@ void MarketDataGateway::unsubscribe(MarketDataListenerID id) {
    LOG(md, Info, "Unsubscribing listener {}", id);
    me_->listeners.erase(id);
 }
-
-/// onix active //////////////////////////////////////////////////////////////
+/// onix active //////////////////////////////////////////////////////////////////////
 //////////////////////////////////////////////////////////////////////////////
-static void run_active(
-    bts::api::md::GlbxMarketDataConfig cfg,
-    persistence::IDGenerator *sec_id_store) {
-   LOG(md, Info, "MarketDataGateway::run_active t_id:{} t_name",
-       os::get_thread_id(), os::get_thread_name());
-   bts::md::GeneralSecurityIndex sec_index;
-   bts::md::MDProducer md_producer;
-   std::shared_ptr<GeneralMarketDataListener> listener =
-       std::make_shared<bts::md::GeneralMarketDataListener>(
-           sec_index, md_producer, bts::api::VENUE_GLBX);
-   bts::md::cme::CMEActivSession::run(
-       cfg.glbx_active_platform().host(), cfg.glbx_active_platform().user(),
-       cfg.glbx_active_platform().password(), listener, sec_id_store);
+static void run_active(bts::api::md::GlbxMarketDataConfig cfg, persistence::IDGenerator *sec_id_store) {
+   LOG(md, Info, "MarketDataGateway::run_active t_id:{} t_name", os::get_thread_id(),
+       os::get_thread_name());
+      bts::md::GeneralSecurityIndex sec_index;
+      bts::md::MDProducer md_producer;
+      std::shared_ptr<GeneralMarketDataListener> listener =std::make_shared<bts::md::GeneralMarketDataListener>(sec_index,md_producer,bts::api::VENUE_GLBX);
+      bts::md::cme::CMEActivSession::run(cfg.glbx_active_platform().host(), cfg.glbx_active_platform().user(), cfg.glbx_active_platform().password(), listener, sec_id_store);
    // Timers
-   // auto &disp = utils::Dispatcher::current();
-   // utils::DispatcherTimer heartbeat_timer(disp, 250, [&]() {
-   //    for (auto &p : platforms)
-   //       p->send_heartbeats();
-   // });
-   // heartbeat_timer.start();
-   // disp.run();
-}
+//    auto &disp = utils::Dispatcher::current();
+//    utils::DispatcherTimer heartbeat_timer(disp, 250, [&]() {
+//       for (auto &p : platforms)
+//          p->send_heartbeats();
+//    });
+//    heartbeat_timer.start();
 
-/// onix glbx ////////////////////////////////////////////////////////////////
+//    disp.run();
+}
+/// onix glbx //////////////////////////////////////////////////////////////////////
 //////////////////////////////////////////////////////////////////////////////
 static void run_onixs() {
    LOG(md, Info, "MarketDataGateway::run_onixs t_id:{} t_name",
@@ -287,29 +283,29 @@ static void run_onixs() {
    disp.run();
 }
 
-void MarketDataGateway::_start_active_glbx(
-    bts::api::md::GlbxMarketDataConfig cfg) {
+void MarketDataGateway::_start_active_glbx(bts::api::md::GlbxMarketDataConfig cfg){
    LOG(md, Info,
        "MarketDataGateway::_start_active_glbx starting t_id:{} t_name:{}. "
        "cfg:\n{}",
        os::get_thread_id(), os::get_thread_name(), cfg.DebugString());
-   LOG(md, Info, "MarketDataGateway::_start_active_glbx");
+LOG(md, Info,
+       "MarketDataGateway::_start_active_glbx");
    bts::md::run_active(cfg, &me_->sec_id_store);
 }
-
-void MarketDataGateway::_start_glbx(bts::api::md::GlbxMarketDataConfig cfg) {
+void MarketDataGateway::_start_glbx( bts::api::md::GlbxMarketDataConfig cfg){
    LOG(md, Info,
-       "MarketDataGateway::_start_glbx starting t_id:{} t_name:{}. cfg:\n{}",
+       "MarketDataGateway::_start_glbx starting t_id:{} t_name:{}. "
+       "cfg:\n{}",
        os::get_thread_id(), os::get_thread_name(), cfg.DebugString());
-   if (cfg.has_glbx_active_platform()) {
-      _start_active_glbx(cfg);
-   } else if (cfg.has_glbx_onix_platform()) {
-      _start_onix_glbx(cfg);
-   }
-}
+       if(cfg.has_glbx_active_platform()){
+            _start_active_glbx(cfg);
+       }
+       else if(cfg.has_glbx_onix_platform()){
+            _start_onix_glbx(cfg);
+       }
 
-void MarketDataGateway::_start_onix_glbx(
-    bts::api::md::GlbxMarketDataConfig cfg) {
+}
+void MarketDataGateway::_start_onix_glbx( bts::api::md::GlbxMarketDataConfig cfg){
    LOG(md, Info,
        "MarketDataGateway::_start_onix_glbx starting t_id:{} t_name:{}. "
        "cfg:\n{}",

src/server/sparkd/md/cme/CMEActivSession.cpp

@@ -1,18 +1,17 @@
 #include "CMEActivSession.h"
 #include "lib/logging/logging.h"
 #include "lib/md/GeneralMarketDataListener.h"
-#include "server/sparkd/md/cme/CMEActivQueryHandler.h"
 #include "server/sparkd/md/cme/CMEActivSessionHandler.h"
-#include "server/sparkd/md/cme/CMEActivSnapshotHandler.h"
 #include "server/sparkd/md/cme/CMEActivSubscriptionHandler.h"
-
+#include "server/sparkd/md/cme/CMEActiveQueryHandler.h"
+#include "server/sparkd/md/cme/CMEActiveSnapshotHandler.h"
 #include <ActivMiddleware/ActivBase/Exception.h>
 #include <ActivMiddleware/ActivBase/FieldTypes/ActivFieldTypes.h>
 #include <ActivMiddleware/ActivBase/FieldTypes/FieldMap.h>
-#include <ActivOneApi/Constants/DataSourceIds.h>
 #include <ActivOneApi/Constants/FieldIds.h>
 #include <ActivOneApi/Session.h>
 #include <exception>
+#include <ActivOneApi/Constants/DataSourceIds.h>
 
 namespace bts::md::cme {
 

src/server/sparkd/md/cme/CMEActivSession.h

@@ -2,7 +2,6 @@
 
 #include "lib/md/GeneralMarketDataListener.h"
 #include "lib/persistence/IDGenerator.h"
-
 #include <string>
 
 namespace bts::md::cme {

src/server/sparkd/md/cme/CMEActivSessionHandler.cpp

@@ -1,8 +1,7 @@
 #include "CMEActivSessionHandler.h"
-#include "server/sparkd/md/cme/CMEActivSubscriptionHandler.h"
-
 #include <ActivMiddleware/ActivBase/ActivBase.h>
 #include <ActivOneApi/Session.h>
+#include "CMEActivSubscriptionHandler.h"
 
 namespace bts::md::cme {
 

src/server/sparkd/md/cme/CMEActivSessionHandler.h

@@ -1,5 +1,6 @@
 #pragma once
 
+
 #include <ActivMiddleware/ActivBase/ActivBase.h>
 #include <ActivOneApi/ISessionHandler.h>
 

src/server/sparkd/md/cme/CMEActivSubscriptionHandler.h

@@ -1,5 +1,4 @@
 #pragma once
-
 #include "lib/core/TimeStamp.h"
 #include "lib/domain/AggressorSide.h"
 #include "lib/domain/Book.h"
@@ -10,32 +9,28 @@
 #include "lib/domain/Side.h"
 #include "lib/domain/TradeRecord.h"
 #include "lib/md/GeneralMarketDataListener.h"
+#include "server/sparkd/md/MarketDataGateway.h"
 #include "lib/orderman/orderman_utils/Book.h"
 #include "lib/persistence/IDGenerator.h"
 #include "lib/utils/bts_assert.h"
-#include "server/sparkd/md/MarketDataGateway.h"
-
-#include <ActivMiddleware/ActivBase/ActivBase.h>
-#include <ActivOneApi/ISessionHandler.h>
-#include <ActivOneApi/ISubscriptionHandler.h>
 #include <ActivOneApi/MarketData/EventTypes.h>
 #include <ActivOneApi/MarketData/FieldIds.h>
-#include <ActivOneApi/MarketData/MarketData.h>
 #include <ActivOneApi/MarketData/Separators.h>
-#include <ActivOneApi/Messages/RefreshMessage.h>
-#include <ActivOneApi/Messages/UpdateMessage.h>
-#include <ActivOneApi/OneApi.h>
 #include <cassert>
 #include <iostream>
 #include <string>
+#define UNUSED(x) (void)(x)
+#include "ActivOneApi/MarketData/MarketData.h"
+#include <ActivMiddleware/ActivBase/ActivBase.h>
 
+#include <ActivOneApi/OneApi.h>
+#include <ActivOneApi/ISessionHandler.h>
+#include <ActivOneApi/ISubscriptionHandler.h>
+#include <ActivOneApi/Messages/RefreshMessage.h>
+#include <ActivOneApi/Messages/UpdateMessage.h>
 using namespace Activ;
 using namespace Activ::OneApi;
-
-#define UNUSED(x) ((void)x)
-
 namespace bts::md::cme {
-
 class CMEActivSubscriptionHandler : public Activ::OneApi::ISubscriptionHandler {
 
    FieldListValidator m_fieldListValidator;
@@ -368,5 +363,3 @@ class CMEActivSubscriptionHandler : public Activ::OneApi::ISubscriptionHandler {
 };
 
 } // namespace bts::md::cme
-
-#undef UNUSED

src/server/sparkd/md/cme/CMEActiveQueryHandler.h

@@ -1,15 +1,9 @@
 #pragma once
-
-#include "lib/md/GeneralMarketDataListener.h"
 #include "server/sparkd/md/cme/CMEActivSubscriptionHandler.h"
-
+#include "lib/md/GeneralMarketDataListener.h"
 #include <ActivMiddleware/CxxYZ/unordered_map.h>
-#include <ActivOneApi/OneApi.h>
-
-// Be sure to undef this at the end of the file. As a general rule, headers
-// shouldn't export macros.
 #define UNUSED(x) (void)(x)
-
+#include "ActivOneApi/OneApi.h"
 namespace bts::md::cme {
 class CMEActivQueryHandler : public Activ::OneApi::ITopicQueryHandler {
    class TopicDetails {
@@ -111,6 +105,4 @@ class CMEActivQueryHandler : public Activ::OneApi::ITopicQueryHandler {
       UNUSED(context);
    }
 };
-} // namespace bts::md::cme
-
-#undef UNUSED
+} // namespace bts::md::cme

src/server/sparkd/md/cme/CMEActiveSnapshotHandler.h

@@ -3,12 +3,14 @@
  *
  *	@brief	SnapshotHandler definition.
  */
-
+#pragma once
 #if (!defined ACTIV_SAMPLE_SNAPSHOT_HANDLER_H)
 #define ACTIV_SAMPLE_SNAPSHOT_HANDLER_H
 
-#include <ActivOneApi/MarketData/MarketData.h>
-#include <ActivOneApi/OneApi.h>
+#include "ActivOneApi/OneApi.h"
+#include "ActivOneApi/MarketData/MarketData.h"
+
+
 #include <ostream>
 
 using namespace Activ;
@@ -95,4 +97,4 @@ inline void SnapshotHandler::ProcessField(const FieldId fieldId, const String &f
 }
 
 }
-#endif // (!defined ACTIV_SAMPLE_SNAPSHOT_HANDLER_H)
+#endif // (!defined ACTIV_SAMPLE_SNAPSHOT_HANDLER_H)

src/server/sparkd/md/cme/CMEChannel.h

@@ -2,7 +2,7 @@
 #include "lib/core/FMT.h"
 #include "lib/core/FMT_IMPL.h"
 #include "lib/md/UserSymbol.h"
-#include "lib/md/cme/CMEConnection.h"
+#include "server/sparkd/md/cme/CMEConnection.h"
 
 #include <map>
 #include <ostream>

src/server/sparkd/md/cme/MDP3Utils.cpp

@@ -15,7 +15,7 @@
 #include <boost/filesystem.hpp>
 #include <iostream>
 #include <mutex>
-#include <pugixml/pugixml.hpp>
+// #include <pugixml/pugixml.hpp>
 #include <sstream>
 
 using namespace std;
@@ -132,62 +132,63 @@ std::string to_str(const std::vector<T> &vec) {
    return ss.str();
 }
 const vector<CMEChannel> parse_channels(const vector<UserSymbol> &symbols) {
-   LOG(md, Info, "parse_channels symbols: {}", FMT_JOIN(symbols, ", "));
-   await_cme_config();
+   // LOG(md, Info, "parse_channels symbols: {}", FMT_JOIN(symbols, ", "));
+   // await_cme_config();
    vector<CMEChannel> result;
-   bool skip_options = config::get<PublisherOptions>().skip_options();
-
-   pugi::xml_document doc;
-   bts_assert_msg(
-       doc.load_file(config_path().c_str()), "Error parsing config.xml as xml");
-
-   auto configuration = doc.root().child("configuration");
-   bts_assert(configuration);
-
-   LOG(md, Info, "CME: Loaded '{}' config. Last updated: {}",
-       std::string(configuration.attribute("environment").as_string("??")),
-       std::string(configuration.attribute("updated").as_string("??")));
-   auto channels = configuration.children("channel");
-   for (const auto &chan : channels) {
-      CMEChannel channel;
-      channel.id = chan.attribute("id").as_string();
-      auto channel_type =
-          infer_security_type(chan.attribute("label").as_string());
-      if (channel_type == SECURITYTYPE_OPTION && skip_options)
-         continue;
-      if (!(channel.id.empty())) {
-         auto product_list = chan.child("products").children("product");
-         for (const auto &product : product_list) {
-            auto symbol = product.attribute("code").as_string();
-            for (auto s : symbols) {
-               if ((s.type() == channel_type ||
-                    s.type() == SECURITYTYPE_SPREAD) &&
-                   s.symbol() == symbol) {
-                  channel.symbols.insert(s);
-               }
-            }
-         }
-         if (!channel.symbols.empty()) {
-            auto connections = chan.child("connections").children("connection");
-            for (const auto &srccon : connections) {
-               std::string id = srccon.attribute("id").as_string();
-               if (!id.empty()) {
-                  CMEConnection connection;
-                  connection.feed_type =
-                      srccon.child("type").attribute("feed-type").as_string();
-                  connection.ip = srccon.child_value("ip");
-                  connection.port = srccon.child("port").text().as_uint();
-                  connection.feed = srccon.child_value("feed")[0];
-                  channel.connections.insert(std::make_pair(id, connection));
-               }
-            }
-
-            result.push_back(channel);
-         }
-      }
-   }
-   LOG(md, Info, "parse_channels channels: {}", to_str(result));
+   (void)(symbols);
+   // bool skip_options = config::get<PublisherOptions>().skip_options();
    return result;
+   // pugi::xml_document doc;
+   // bts_assert_msg(
+   //     doc.load_file(config_path().c_str()), "Error parsing config.xml as xml");
+
+   // auto configuration = doc.root().child("configuration");
+   // bts_assert(configuration);
+
+   // LOG(md, Info, "CME: Loaded '{}' config. Last updated: {}",
+   //     std::string(configuration.attribute("environment").as_string("??")),
+   //     std::string(configuration.attribute("updated").as_string("??")));
+   // auto channels = configuration.children("channel");
+   // for (const auto &chan : channels) {
+   //    CMEChannel channel;
+   //    channel.id = chan.attribute("id").as_string();
+   //    auto channel_type =
+   //        infer_security_type(chan.attribute("label").as_string());
+   //    if (channel_type == SECURITYTYPE_OPTION && skip_options)
+   //       continue;
+   //    if (!(channel.id.empty())) {
+   //       auto product_list = chan.child("products").children("product");
+   //       for (const auto &product : product_list) {
+   //          auto symbol = product.attribute("code").as_string();
+   //          for (auto s : symbols) {
+   //             if ((s.type() == channel_type ||
+   //                  s.type() == SECURITYTYPE_SPREAD) &&
+   //                 s.symbol() == symbol) {
+   //                channel.symbols.insert(s);
+   //             }
+   //          }
+   //       }
+   //       if (!channel.symbols.empty()) {
+   //          auto connections = chan.child("connections").children("connection");
+   //          for (const auto &srccon : connections) {
+   //             std::string id = srccon.attribute("id").as_string();
+   //             if (!id.empty()) {
+   //                CMEConnection connection;
+   //                connection.feed_type =
+   //                    srccon.child("type").attribute("feed-type").as_string();
+   //                connection.ip = srccon.child_value("ip");
+   //                connection.port = srccon.child("port").text().as_uint();
+   //                connection.feed = srccon.child_value("feed")[0];
+   //                channel.connections.insert(std::make_pair(id, connection));
+   //             }
+   //          }
+
+   //          result.push_back(channel);
+   //       }
+   //    }
+   // }
+   // LOG(md, Info, "parse_channels channels: {}", to_str(result));
+   // return result;
 }
 
 } // namespace md

src/server/sparkd/md/cme/MDP3Utils.h

@@ -1,8 +1,8 @@
 #pragma once
 
 #include "lib/core/FMT_IMPL.h"
-#include "lib/md/cme/CMEChannel.h"
-#include "lib/md/cme/CMEConnection.h"
+#include "server/sparkd/md/cme/CMEChannel.h"
+#include "server/sparkd/md/cme/CMEConnection.h"
 #include "lib/md/coconut2.h"
 #include <set>
 #include <string>

src/server/sparkd/md/platforms/mdp3_onixs/MDP3ListenerHelpers.h

@@ -1,4 +1,5 @@
-#include "lib/md/cme/MDP3Utils.h"
+#pragma once
+#include "server/sparkd/md/cme/MDP3Utils.h"
 #include "lib/md/coconut2.h"
 #include "lib/utils/bts_assert.h"
 

src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp

@@ -4,7 +4,7 @@
 #include "lib/logging/logging.h"
 #include "lib/md/MBOProcessor.h"
 #include "lib/md/SecurityDB.h"
-#include "lib/md/cme/MDP3Utils.h"
+#include "server/sparkd/md/cme/MDP3Utils.h"
 #include "lib/md/coconut2.h"
 #include "lib/md/protos/co2.pb.h"
 #include "lib/md/vendor/OnixsUtils.h"