For SIM exchanges, securities with expired expirations are not filtered out

src/client/spark/Daemons/MergedMDDaemon.cs

@@ -65,18 +65,21 @@ namespace bts {
          _merge_and_post();
       }
 
-      private void _merge_and_post() {
-         Func<SecurityMD, bool> is_active = (sec_md) => {
-            if (sec_md.security.HasFuture || sec_md.security.HasOption) {
-               var exp = sec_md.security.security_expiration();
-               if (exp.HasValue) {
-                  return exp.Value.Date >= DateTime.Now.Date.AddDays(-1);
-               }
-               return true;
+      private static bool _is_active(SecurityMD sec_md) {
+         if (sec_md.security.Exchange == Venue.Sim)
+            return true;
+         if (sec_md.security.HasFuture || sec_md.security.HasOption) {
+            var exp = sec_md.security.security_expiration();
+            if (exp.HasValue) {
+               return exp.Value.Date >= DateTime.Now.Date.AddDays(-1);
             }
             return true;
-         };
-         var to_pub = _server_sec_map.Where(kv => is_active(kv.Value)).ToDictionary(kv => kv.Key, kv => kv.Value);
+         }
+         return true;
+      }
+
+      private void _merge_and_post() {
+         var to_pub = _server_sec_map.Where(kv => _is_active(kv.Value)).ToDictionary(kv => kv.Key, kv => kv.Value);
          foreach (var kv in _fake_sec_map) {
             to_pub[kv.Key] = kv.Value;
          }

src/lib/CMakeDeps.cmake

@@ -1 +1 @@
-set(sources api/analytics/ExpirationDriver.proto;api/analytics/TimeWeightedInterpolatedDriver.proto;api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/domain/Currency.proto;api/domain/SecurityCriteria.proto;api/domain/SecurityCriteriaClause.proto;api/domain/SecurityExpiration.proto;api/domain/SecurityMetadata.proto;api/domain/SecurityMetadataCollection.proto;api/domain/SecurityMetadataItem.proto;api/domain/SecurityType.proto;api/domain/TickDisplaySettings.proto;api/domain/TickFactors.proto;api/domain/Venue.proto;api/io/URL.proto;api/md/MarketDataSubscriptionParams.proto;api/md/binance/BinanceDepthStreamParams.proto;api/md/binance/BinanceMarketDataSubscriptionParams.proto;api/md/deribit/DeribitMarketDataSubscriptionParams.proto;api/md/ftx/FTXDepthStreamParams.proto;api/md/ftx/FTXMarketDataSubscriptionParams.proto;api/md/glbx/GlbxMarketDataSubscriptionParams.proto;api/md/huobi/HuobiMarketDataSubscriptionParams.proto;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/service_helpers/PositionUtilities.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/AnalyticsService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/BasePositionService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MBOService.proto;api/services/MarginService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/MarketDataService.proto;api/services/MoneyFileService.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;communincation/websockets/tests/WebsocketSyncClientTests.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/SecurityMetadataCollection.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/keycode.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/position_cfg.conf;config/defaults/span.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/algo_types.proto;config/protos/cme_reference_data.proto;config/protos/exchange_sim_cfg.proto;config/protos/jetfix.proto;config/protos/keycode.proto;config/protos/logging.proto;config/protos/market_data.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/ports.proto;config/protos/position_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/span.proto;config/protos/spartan.proto;config/protos/spreader.proto;config/protos/subprocess.proto;config/protos/symbols.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/protos/ui.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain/AggressorSide.cpp;domain/NewOrder.cpp;domain/OrderDefinition.cpp;domain/OrderStatus.cpp;domain/OrderUpdate.cpp;domain/Side.cpp;domain/TimeInForce.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/sod.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;io/QueryParams.cpp;io/URL.cpp;io/http/CurlHandle.cpp;io/http/HTTPClient.cpp;io/http/HTTPHeaders.cpp;io/http/HTTPRequest.cpp;io/http/HTTPResponse.cpp;io/websocket/WebsocketClient.cpp;io/websocket/WebsocketClientConfiguration.cpp;io/websocket/WebsocketError.cpp;io/websocket/WebsocketMessage.cpp;io/websocket/WebsocketSSLClientInternal.cpp;io/websocket/tests/WebsocketClientTests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGatLine truncated
+set(sources api/analytics/ExpirationDriver.proto;api/analytics/TimeWeightedInterpolatedDriver.proto;api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/domain/Currency.proto;api/domain/SecurityCriteria.proto;api/domain/SecurityCriteriaClause.proto;api/domain/SecurityExpiration.proto;api/domain/SecurityMetadata.proto;api/domain/SecurityMetadataCollection.proto;api/domain/SecurityMetadataItem.proto;api/domain/SecurityType.proto;api/domain/TickDisplaySettings.proto;api/domain/TickFactors.proto;api/domain/Venue.proto;api/io/URL.proto;api/md/MarketDataSubscriptionParams.proto;api/md/binance/BinanceDepthStreamParams.proto;api/md/binance/BinanceMarketDataSubscriptionParams.proto;api/md/deribit/DeribitMarketDataSubscriptionParams.proto;api/md/ftx/FTXDepthStreamParams.proto;api/md/ftx/FTXMarketDataSubscriptionParams.proto;api/md/glbx/GlbxMarketDataSubscriptionParams.proto;api/md/huobi/HuobiMarketDataSubscriptionParams.proto;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/service_helpers/PositionUtilities.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/AnalyticsService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/BasePositionService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MBOService.proto;api/services/MarginService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/MarketDataService.proto;api/services/MoneyFileService.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;communincation/websockets/tests/WebsocketSyncClientTests.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/SecurityMetadataCollection.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/keycode.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/position_cfg.conf;config/defaults/span.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/algo_types.proto;config/protos/cme_reference_data.proto;config/protos/exchange_sim_cfg.proto;config/protos/jetfix.proto;config/protos/keycode.proto;config/protos/logging.proto;config/protos/market_data.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/ports.proto;config/protos/position_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/span.proto;config/protos/spartan.proto;config/protos/spreader.proto;config/protos/subprocess.proto;config/protos/symbols.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/protos/ui.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain/AggressorSide.cpp;domain/NewOrder.cpp;domain/OrderDefinition.cpp;domain/OrderStatus.cpp;domain/OrderUpdate.cpp;domain/Side.cpp;domain/TimeInForce.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/sod.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;io/QueryParams.cpp;io/URL.cpp;io/http/CurlHandle.cpp;io/http/HTTPClient.cpp;io/http/HTTPHeaders.cpp;io/http/HTTPRequest.cpp;io/http/HTTPResponse.cpp;io/websocket/WebsocketClient.cpp;io/websocket/WebsocketClientConfiguration.cpp;io/websocket/WebsocketError.cpp;io/websocket/WebsocketMessage.cpp;io/websocket/WebsocketSSLClientInternal.cpp;io/websocket/tests/WebsocketClientTests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGatLine truncated

src/lib/api/service_helpers/AdminServiceUtils.cpp

@@ -5,7 +5,9 @@
 #include "lib/config/protos/jetfix.pb.h"
 #include "lib/config/protos/system.pb.h"
 #include "lib/domain_model/Security.pb.h"
+#include "lib/logging/logging.h"
 #include "lib/md/get_symbols.h"
+#include "lib/utils/PBUtils.h"
 
 #include <boost/algorithm/string/case_conv.hpp>
 #include <boost/algorithm/string/predicate.hpp>
@@ -186,7 +188,12 @@ void AdminServiceUtils::fill_connections_from_config(
     bts::api::admin::AdminData &admin_data) {
    admin_data.clear_connections();
    auto &all_connections = config::get<config::JetFIX>().connection();
+   LOG(Gen, Info,
+       "AdminServiceUtils::fill_connections_from_config, n_conns: {}",
+       all_connections.size());
    for (auto const &c : all_connections) {
+      LOG(Gen, Info, "AdminServiceUtils::fill_connections_from_config conn: {}",
+          utils::pb::to_string(c));
       bts::api::admin::ExchangeConnection connection;
       uint32_t new_id = find_max_connection_id(admin_data) + 1;
       connection.set_id(new_id);
@@ -214,6 +221,10 @@ void AdminServiceUtils::fill_connections_from_config(
              c.sender_compid());
          add_connection(admin_data, connection);
          break;
+      case config::JetFIX::Connection::DERIBIT:
+         connection.mutable_sim_connection();
+         add_connection(admin_data, connection);
+         break;
       }
    }
    for (int i = 0; i < admin_data.accounts_size(); i++)
@@ -525,7 +536,7 @@ bts::api::orderman::RoutingRecord AdminServiceUtils::get_routing_record(
    }
    switch (sec.exchange()) {
    case api::Venue::VENUE_DERIBIT: {
-      rv.set_session("sim");
+      rv.set_session("deribit_em");
       rv.set_account(account_ptr->name());
       rv.set_operator_id(user_ptr->name());
       AdminServiceUtils::ExchangeSubAccountResolver resolver;

src/lib/config/defaults/MarketData.conf

@@ -5,7 +5,6 @@
 MarketDataConfig {
    rule: "DeploymentType.type == 'PROD'"
    value {
-      use_sim: false
    }
 }
 
@@ -45,12 +44,12 @@ CMEOptions {
     }
 }
 
-ICEPublisherOptions {
-   value { 
-      network_interface: "eth1"
-   }
-}
-
+ICEPublisherOptions {
+   value { 
+      network_interface: "eth1"
+   }
+}
+
 #to make this work in dev clone the operations repo
 #go here: c:\\code\\bts\\operations\\ice\\ICE_iMpactMulticastFeed_GettingStarted-1_1_36\\bin\\easyTunnel>
 #modify the easyTunnelProxy.properties file to replace 

src/lib/config/defaults/Symbols.conf

@@ -3,65 +3,64 @@ SymbolList {
     value {
         # This key is here for backwards compatability ONLY.
         # If you need to update symbols you should update SymbolLists (plural)
-        # symbol: "6B"
-        # symbol: "6E"
-        # symbol: "6J"
-        # symbol: "6P"
-        # symbol: "ARE"
-        # symbol: "BK"
-        # symbol: "BOB"
-        # symbol: "BUB"
-        # symbol: "BZ"
-        # symbol: "BZT"
-        # symbol: "CL"
-        # symbol: "CLT"
-        # symbol: "EH"
-        # symbol: "ES"
-        # symbol: "FYT"
-        # symbol: "FOB"
-        # symbol: "GC"
-        # symbol: "GE"
-        # symbol: "HE"
-        # symbol: "HG"
-        # symbol: "HO"
-        # symbol: "HOT"
-        # symbol: "KE"
-        # symbol: "LE"
-        # symbol: "NB"
-        # symbol: "NBY"
-        # symbol: "NOB"
-        # symbol: "NON"
-        # symbol: "NG"
-        # symbol: "NGT"
-        # symbol: "NQ"
-        # symbol: "NUB"
-        # symbol: "RB"
-        # symbol: "RBT"
-        # symbol: "SI"
-        # symbol: "SOM"
-        # symbol: "SR1"
-        # symbol: "SR3"
-        # symbol: "TN"
-        # symbol: "TUF"
-        # symbol: "TUT"
-        # symbol: "UB"
-        # symbol: "XC"
-        # symbol: "XK"
-        # symbol: "YM"
-        # symbol: "ZB"
-        # symbol: "ZC"
-        # symbol: "ZCW"
-        # symbol: "ZF"
-        # symbol: "ZL"
-        # symbol: "ZM"
-        # symbol: "ZN"
-        # symbol: "ZO"
-        # symbol: "ZQ"
-        # symbol: "ZR"
-        # symbol: "ZS"
-        # symbol: "ZT"
-        # symbol: "ZW"
+        symbol: "6B"
+        symbol: "6E"
+        symbol: "6J"
+        symbol: "6P"
+        symbol: "ARE"
+        symbol: "BK"
+        symbol: "BOB"
+        symbol: "BUB"
+        symbol: "BZ"
+        symbol: "BZT"
+        symbol: "CL"
+        symbol: "CLT"
+        symbol: "EH"
         symbol: "ES"
+        symbol: "FYT"
+        symbol: "FOB"
+        symbol: "GC"
+        symbol: "GE"
+        symbol: "HE"
+        symbol: "HG"
+        symbol: "HO"
+        symbol: "HOT"
+        symbol: "KE"
+        symbol: "LE"
+        symbol: "NB"
+        symbol: "NBY"
+        symbol: "NOB"
+        symbol: "NON"
+        symbol: "NG"
+        symbol: "NGT"
+        symbol: "NQ"
+        symbol: "NUB"
+        symbol: "RB"
+        symbol: "RBT"
+        symbol: "SI"
+        symbol: "SOM"
+        symbol: "SR1"
+        symbol: "SR3"
+        symbol: "TN"
+        symbol: "TUF"
+        symbol: "TUT"
+        symbol: "UB"
+        symbol: "XC"
+        symbol: "XK"
+        symbol: "YM"
+        symbol: "ZB"
+        symbol: "ZC"
+        symbol: "ZCW"
+        symbol: "ZF"
+        symbol: "ZL"
+        symbol: "ZM"
+        symbol: "ZN"
+        symbol: "ZO"
+        symbol: "ZQ"
+        symbol: "ZR"
+        symbol: "ZS"
+        symbol: "ZT"
+        symbol: "ZW"
         symbol: "BTC"
         symbol: "ETH"
     }

src/lib/config/protos/jetfix.proto

@@ -12,6 +12,7 @@ message JetFIX {
       SIM = 2;
       ICE = 3;
       SIMICE = 4;
+      DERIBIT = 5;
     }
     optional JetFIXPlatform platform = 15 [ default = GLBX ];
 

src/lib/config/protos/market_data.proto

@@ -1,14 +1,24 @@
 syntax = 'proto2';
 package bts.md;
+import "lib/domain_model/Security.proto";
+
 
 message MarketDataConfig {
-  // Allows you to turn off in-proc market data entirely
+  enum MarketDataMode {
+    Live = 0;
+    Replay = 10;
+  }
+  // Allows you to turn off in-proc market data entirely, is use_md == false 
+  // then none of the market data machinery is loaded (except cleanup)
   optional bool use_md = 1 [ default = true ];
-  optional bool use_onixs = 2 [ default = true ];
-  optional bool use_replay = 3 [ default = false ];
-  optional bool use_sim = 4 [ default = true ];
+  // Replay will assert right now, 07/29/2022 (replay != sim, it runs only for CME
+  // and needs access to a Chicago machine)
+  optional MarketDataMode mode = 2 [ default = Live ];
+  // All the exchanges for which the market data will run
+  repeated bts.api.Venue md_venues = 3;
 }
 
+
 message PlatformMap {
   // PLATFORM_MDP3ONIXS
   optional string glbx_platform = 1 [ default = "PLATFORM_SIM" ];

src/lib/md/PlatformLoader.cpp

@@ -1,6 +1,7 @@
 #include "lib/domain_model/Security.pb.h"
 #include "lib/logging/logging.h"
 #include "lib/md/UserSymbol.h"
+#include "lib/md/protos/co2.pb.h"
 #ifndef NO_CPP0X
 
 #include "PlatformLoader.h"
@@ -263,6 +264,8 @@ std::set<UserSymbol> get_system_md_symbols() {
 }
 
 extern "C" BTS_EXPORT Platform platform_for_exchange(Venue v) {
+   if (v == bts::api::VENUE_DERIBIT)
+      return co2::PLATFORM_DERIBIT;
    const auto &pmap = config::get<PlatformMap>();
    co2::Platform platform;
    if (v == VENUE_SIM)

src/lib/orderman/actors/order_manager/OrderServicesData.h

@@ -58,6 +58,8 @@ struct SessionServicesData {
        : session(cfg.name()), gw_connected(false), cfg(cfg),
          governor(cfg.max_order_msgs_per_window(), cfg.window_duration_msecs()),
          gw(ExecutionGatewayFactory::create_gateway(cfg, log, o)) {
+      LOG(OrderMan, Info, "SessionServicesData::cons\nconn: {}",
+          utils::pb::to_string(cfg));
       buffered_order_msgs.reserve(1000);
    }
 

src/lib/orderman/gateways/ExecutionGatewayFactory.cpp

@@ -1,4 +1,5 @@
 #include "ExecutionGatewayFactory.h"
+#include "lib/logging/logging.h"
 #include "lib/md/SecurityOracle.h"
 #include "lib/orderman/gateways/EMExecutionGateway.h"
 #include "lib/orderman/gateways/FakeExecutionGateway.h"
@@ -6,6 +7,7 @@
 #include "lib/orderman/gateways/GLBXiLink3ExecutionGateway.h"
 #include "lib/orderman/gateways/ICEExecutionGateway.h"
 #include "lib/tracing/PerfLog.h"
+#include "lib/utils/PBUtils.h"
 #include "lib/utils/ValueWrapper.h"
 #include <variant>
 
@@ -15,15 +17,17 @@ std::unique_ptr<IExecutionGateway> ExecutionGatewayFactory::create_gateway(
     const config::JetFIX::Connection &conn,
     trace::PerfLog &pl,
     md::SecurityOracle o) {
+   LOG(OrderMan, Info, "ExecutionGatewayFactory::create_gateway\nconn: {}",
+       utils::pb::to_string(conn));
    if (conn.platform() == bts::config::JetFIX_Connection_JetFIXPlatform_ICE ||
        conn.platform() == bts::config::JetFIX_Connection_JetFIXPlatform_SIMICE)
       return std::make_unique<ICEExecutionGateway>(conn, pl, o);
    else if (conn.use_ilink3())
       return std::make_unique<ilink3::GLBXiLink3ExecutionGateway>(conn, pl, o);
-   else
+   else if (conn.platform() == bts::config::JetFIX_Connection::DERIBIT)
       return std::make_unique<EMExecutionGateway>(o);
-   // return std::make_unique<FakeExecutionGateway>(o);
-   // return std::make_unique<GLBXExecutionGateway>(conn, pl, o);
+   else
+      return std::make_unique<GLBXExecutionGateway>(conn, pl, o);
 }
 
 } // namespace bts::orderman

src/lib/orderman/order_factory/OrderFactory.cpp

@@ -124,17 +124,17 @@ NativeOrderBase *OrderFactory::make_order(const api::Venue exchange) {
    if (exchange == api::Venue::VENUE_ICE ||
        exchange == api::Venue::VENUE_SIMICE)
       return new ICENativeOrder();
-   else
-      // return new GLBXNativeOrder();
+   else if (exchange == api::VENUE_DERIBIT)
       return new EMNativeOrder();
+   else
+      return new GLBXNativeOrder();
 }
 
 NativeOrderBase *OrderFactory::make_native_order(
     const api::orderman::OrderDefinition &def) {
    auto secptr = me->sec_db.resolve(SecurityKey(def.security_key()));
    if (secptr == nullptr)
-      // return new GLBXNativeOrder(); // GLBX Native can handle unresolved
-      return new EMNativeOrder();
+      return new GLBXNativeOrder(); // GLBX Native can handle unresolved
    // securities
    else
       return make_order(secptr->exchange());

src/lib/orderman/orders/EMNativeOrder.cpp

@@ -474,7 +474,7 @@ OrderOpResult EMNativeOrder::_on_activate(
    me->session_services = &me->services->get_session_services(
        get_native_state(me->state).routing_record().session());
    // TODO: just to get going
-   me->session_services = &me->services->get_session_services("sim");
+   me->session_services = &me->services->get_session_services("deribit_em");
 
    // Checks for the following:
    // 1. Gateway connection

src/lib/utils/datetime.cpp

@@ -362,6 +362,7 @@ struct UtcTimeStamp {
    int month;
    int year;
 };
+bool is_digit_or_empty(char c) { return isdigit(c) || c == ' '; }
 
 struct UtcTimeStampConvertor {
    static UtcTimeStamp convert(const std::string &value) {
@@ -401,7 +402,7 @@ struct UtcTimeStampConvertor {
          if (value[i++] != '.')
             bts_assert(false);
          for (c = 0; c < 3; ++c)
-            if (!isdigit(value[i++]))
+            if (!is_digit_or_empty(value[i++]))
                bts_assert(false);
       }
 
@@ -450,9 +451,13 @@ struct UtcTimeStampConvertor {
          bts_assert(false);
 
       if (haveMilliseconds) {
+         auto ms_c = [&value](auto i) {
+            auto c = value[i];
+            return c == ' ' ? '0' : c;
+         };
          millis =
-             (100 * (value[i + 1] - '0') + 10 * (value[i + 2] - '0') +
-              (value[i + 3] - '0'));
+             (100 * (ms_c(i + 1) - '0') + 10 * (ms_c(i + 2) - '0') +
+              (ms_c(i + 3) - '0'));
       } else
          millis = 0;
 

src/server/sparkd/cmds/md/cmds/Cleanup.cpp

@@ -38,10 +38,11 @@ void clean_dir(std::string base_path) {
 
 void Cleanup::execute() {
    clean_dir("/dev/shm");
+   clean_dir("/run/shm");
 }
 
-Cleanup::Cleanup() :
-    CLI::App("Remove all persisted market data shared memory", "cleanup") {
+Cleanup::Cleanup()
+    : CLI::App("Remove all persisted market data shared memory", "cleanup") {
    callback(execute);
 }
 

src/server/sparkd/cmds/md/cmds/Run.cpp

@@ -1,10 +1,15 @@
 #include "Run.h"
 #include "Cleanup.h"
 #include "lib/config/protos/market_data.pb.h"
+#include "lib/domain_model/Security.pb.h"
 #include "lib/logging/logging.h"
 #include "lib/md/UserSymbol.h"
+#include "lib/md/cme/MDP3Utils.h"
 #include "lib/md/relay/RelayRecorder.h"
 #include "lib/os/os.h"
+#include "lib/utils/PBUtils.h"
+#include "lib/utils/Puid.h"
+#include "lib/utils/bts_assert.h"
 #include "lib/utils/thread_manager.h"
 #include "server/sparkd/exchange_sim/ExchangeSim.h"
 #include "server/sparkd/md/platforms/IMDPlatform.h"
@@ -55,30 +60,50 @@ static void run_replay() {
 }
 
 static void run_onixs(
+    const std::set<bts::api::Venue> &venue_map,
     bts::md::MBOWriter *mbo_writer,
     bts::md::TradeSummarySignalDispatcher *tss_dispatcher) {
+   auto has_venue = [&venue_map](auto venue) {
+      return venue_map.find(venue) != venue_map.end();
+   };
    LOG(md, Info, "Run::run_onixs t_id:{} t_name", os::get_thread_id(),
        get_thread_name());
    std::vector<std::shared_ptr<bts::md::IMDPlatform>> platforms;
-   // get all glbx symbols
-   std::vector<bts::md::UserSymbol> cme_symbols;
-   get_system_md_symbols(
-       api::VENUE_GLBX, api::SECURITYTYPE_UNSET, &cme_symbols);
-   if (cme_symbols.size() > 0) {
-      auto *main = new bts::md::MDP3Main();
-      main->start(cme_symbols, mbo_writer, tss_dispatcher);
-      platforms.emplace_back(main);
+   // glbx
+   if (has_venue(bts::api::VENUE_GLBX)) {
+      std::vector<bts::md::UserSymbol> cme_symbols;
+      get_system_md_symbols(
+          api::VENUE_GLBX, api::SECURITYTYPE_UNSET, &cme_symbols);
+      if (cme_symbols.size() > 0) {
+         LOG(md, Info,
+             "Run::run_onixs starting CME OnyxS market data, n_symbols: {}",
+             cme_symbols.size());
+         auto *main = new bts::md::MDP3Main();
+         main->start(cme_symbols, mbo_writer, tss_dispatcher);
+         platforms.emplace_back(main);
+      }
    }
    // ice
-   std::vector<bts::md::UserSymbol> ice_symbols;
-   get_system_md_symbols(api::VENUE_ICE, api::SECURITYTYPE_UNSET, &ice_symbols);
-   if (ice_symbols.size() > 0) {
-      auto *imain = new bts::md::ice::ICEMain();
-      imain->start(ice_symbols);
-      platforms.emplace_back(imain);
+   if (has_venue(bts::api::VENUE_ICE)) {
+      std::vector<bts::md::UserSymbol> ice_symbols;
+      get_system_md_symbols(
+          api::VENUE_ICE, api::SECURITYTYPE_UNSET, &ice_symbols);
+      if (ice_symbols.size() > 0) {
+         LOG(md, Info,
+             "Run::run_onixs starting ICE OnyxS market data, n_symbols: {}",
+             ice_symbols.size());
+         auto *imain = new bts::md::ice::ICEMain();
+         imain->start(ice_symbols);
+         platforms.emplace_back(imain);
+      }
+   }
+   if (platforms.empty()) {
+      LOG(md, Info,
+          "Run::run_onixs exited because no OnyxS exchange was enabled (or "
+          "their "
+          "symbols were not)");
+      return;
    }
-   // deribit
-   // auto *d_platform = new bts::md::EmMDListener()
    // Timers
    auto &disp = utils::Dispatcher::current();
    utils::DispatcherTimer heartbeat_timer(disp, 250, [&]() {
@@ -90,9 +115,29 @@ static void run_onixs(
    disp.run();
 }
 
-static void run_emumba() {
+static void run_sim(
+    const std::set<bts::api::Venue> &venue_map,
+    bts::md::MBOWriter *mbo_writer) {
+   LOG(md, Info, "Run::run_sim t_id:{} t_name", os::get_thread_id(),
+       get_thread_name());
+   if (venue_map.find(api::VENUE_SIM) == venue_map.end()) {
+      LOG(md, Info, "Run::run_sim exited because no sim exchanges was enabled");
+      return;
+   }
+   LOG(md, Info, "Run::run_sim starting sim market data");
+   sim::start_exchange_sim(api::Venue::VENUE_SIM, mbo_writer);
+}
+
+static void run_emumba(const std::set<bts::api::Venue> &venue_map) {
    LOG(md, Info, "Run::run_emumba t_id:{} t_name:{}", os::get_thread_id(),
        get_thread_name());
+   if (venue_map.find(api::VENUE_DERIBIT) == venue_map.end()) {
+      LOG(md, Info,
+          "Run::run_emumba exited because no Emymba driven exchanges were "
+          "enabled");
+      return;
+   }
+   LOG(md, Info, "Run::run_emumba starting Emumba market data");
    bts::md::EmumbaMain em_main = {};
    em_main.run();
 }
@@ -100,61 +145,45 @@ static void run_emumba() {
 void Run::run(
     bts::md::MBOWriter *mbo_writer,
     bts::md::TradeSummarySignalDispatcher *tss_dispatcher,
-    Run::MDType type,
+    Run::MDType,
     bool cleanup,
     bool run_on_md_thread) {
-   LOG(md, Info, "Run::run cleanup:{} run_on_md_thread:{} t_id:{} t_name:{}",
-       cleanup, run_on_md_thread, os::get_thread_id(), get_thread_name());
-   if (type == Run::MDType::Configured &&
-       !config::get<bts::md::MarketDataConfig>().use_md()) {
+   // bts::md::MarketDataConfig test_conf = {};
+   // test_conf.add_md_venues(::bts::api::Venue::VENUE_DERIBIT);
+   // test_conf.add_md_venues(::bts::api::Venue::VENUE_SIM);
+   // LOG(md, Info, "Ye lo: {}", utils::pb::to_string(test_conf));
+   auto conf = config::get<bts::md::MarketDataConfig>();
+   LOG(md, Info,
+       "Run::run conf: {} cleanup: {} run_on_md_thread: {} t_id: {} t_name: {}",
+       utils::pb::to_string(conf), cleanup, run_on_md_thread,
+       os::get_thread_id(), get_thread_name());
+   if (!config::get<bts::md::MarketDataConfig>().use_md()) {
       LOG(md, Info, "Ignoring request to run md: 'use_md' is configured false");
       return;
    }
-
-   auto impl = [&mbo_writer, &tss_dispatcher, &type, &cleanup]() {
-      LOG(md, Info, "Run::impl lambda t_id:{} t_name", os::get_thread_id(),
-          get_thread_name());
-      if (cleanup)
-         md::cmds::Cleanup::execute();
-      if (type == Run::MDType::Configured) {
-         const auto &conf = config::get<bts::md::MarketDataConfig>();
-         if (conf.use_replay())
-            type = Run::MDType::Replay;
-         else if (conf.use_onixs())
-            type = Run::MDType::Onixs;
-      }
-      switch (type) {
-      case Run::MDType::Replay:
-         run_replay();
-         break;
-      case Run::MDType::Onixs:
-         run_onixs(mbo_writer, tss_dispatcher);
-         break;
-      default:
-         bts_assert_msg(false, "Unexpected md type");
-      }
-      return;
-   };
-
-   if (run_on_md_thread) {
-      bts_assert(_md_thread == std::nullopt);
-
-      // start a thread for the md
-      // _md_thread =
-      //     std::optional(bts::threading::make_thread("MDPublisher", impl));
-      bool is_run_emumba = true;
-      if (is_run_emumba) {
-         threading::make_thread("Emumba", []() { run_emumba(); }).detach();
-      }
-      if (bts::config::get<bts::md::MarketDataConfig>().use_sim()) {
-         threading::make_thread("ExchangeSimGLBX", [&mbo_writer]() {
-            sim::start_exchange_sim(api::Venue::VENUE_SIM, mbo_writer);
-         }).detach();
-      }
-      boot_sleep();
-   } else {
-      impl();
+   if (cleanup) {
+      md::cmds::Cleanup::execute();
    }
+   // bts_assert_msg(false, "run_replay is not enabled");
+   std::set<bts::api::Venue> venue_map;
+   for (auto i = 0; i < conf.md_venues_size(); i++)
+      venue_map.insert(conf.md_venues(i));
+   // EMumba
+   threading::make_thread("Emumba", [&venue_map]() {
+      run_emumba(venue_map);
+   }).detach();
+   // OnixS
+   threading::make_thread(
+       "Onixs",
+       [&venue_map, &mbo_writer, &tss_dispatcher]() {
+          run_onixs(venue_map, mbo_writer, tss_dispatcher);
+       })
+       .detach();
+   // Sim
+   threading::make_thread("Sim", [&venue_map, &mbo_writer]() {
+      run_sim(venue_map, mbo_writer);
+   }).detach();
+   boot_sleep();
 }
 
 Run::Run()
@@ -165,7 +194,7 @@ Run::Run()
           "run") {
    add_subcommand("replay", "Run replay market data")->callback(run_replay);
    add_subcommand("onixs", "Run onixs market data")->callback([] {
-      run_onixs(nullptr, nullptr);
+      run_onixs({}, nullptr, nullptr);
    });
    callback([] { run(nullptr, nullptr, MDType::Configured, true, true); });
 }

src/server/sparkd/cmds/md/common/PlatformOption.cpp

@@ -1,4 +1,5 @@
 #include "PlatformOption.h"
+#include "lib/md/protos/co2.pb.h"
 
 namespace bts::cmds::md {
 
@@ -7,9 +8,9 @@ namespace bts::cmds::md {
 
 EnumOptionNames<bts::co2::Platform> PlatformOption::names() {
    return {
-       NAME_PAIR(co2::PLATFORM_CELOXICA), NAME_PAIR(co2::PLATFORM_ICEONIXS),
+       NAME_PAIR(co2::PLATFORM_CELOXICA),   NAME_PAIR(co2::PLATFORM_ICEONIXS),
        NAME_PAIR(co2::PLATFORM_MDP3EMUMBA), NAME_PAIR(co2::PLATFORM_MDP3ONIXS),
-       NAME_PAIR(co2::PLATFORM_SIM)};
+       NAME_PAIR(co2::PLATFORM_SIM),        NAME_PAIR(co2::PLATFORM_DERIBIT)};
 }
 
 std::string PlatformOption::name(bts::co2::Platform platform) {
@@ -24,6 +25,8 @@ std::string PlatformOption::name(bts::co2::Platform platform) {
       return "mdp3onixs";
    case co2::PLATFORM_SIM:
       return "sim";
+   case co2::PLATFORM_DERIBIT:
+      return "deribit";
    default:
       return std::to_string(static_cast<int32_t>(platform));
    }

src/server/sparkd/exchange_sim/ExchangeSim.cpp

@@ -157,7 +157,8 @@ int start_exchange_sim(bts::api::Venue venue, md::MBOWriter *mbo) {
          bts_assert_msg(false, "Not a valid sim venue");
       }
       boost::replace_all(str_settings, "$EXCHANGE_SIM_FIX_PORT", port_string);
-
+      LOG(md, Info, "start_exchange_sim venue: {} port: {} settings: {}",
+          venue_string, port_string, str_settings);
       std::istringstream str{str_settings};
       FIX::SessionSettings settings(str);
 

src/server/sparkd/md/platforms/emumba/EmumbaMain.cpp

@@ -43,7 +43,7 @@ struct EmumbaMain::PImpl {
                            [=](ExchangeKey k) {
                               return SecurityKey(sec_index.get_key(k.value()));
                            }}),
-         producer(bts::co2::PLATFORM_SIM, recorder),
+         producer(bts::co2::PLATFORM_DERIBIT, recorder),
          listener(sec_index, producer) {
       auto config_path =
           os::sparkdata_path("conf") + "deribit_demo_config.json";

src/web/CMakeDeps.cmake

@@ -1 +1 @@
-set(sources ../../externals/linux/include/google/protobuf/descriptor.proto;../../externals/linux/include/google/protobuf/timestamp.proto;../lib/api/analytics/ExpirationDriver.proto;../lib/api/analytics/TimeWeightedInterpolatedDriver.proto;../lib/api/apicore.proto;../lib/api/apigateway.proto;../lib/api/apioptions.proto;../lib/api/domain/Currency.proto;../lib/api/domain/SecurityCriteria.proto;../lib/api/domain/SecurityCriteriaClause.proto;../lib/api/domain/SecurityExpiration.proto;../lib/api/domain/SecurityMetadata.proto;../lib/api/domain/SecurityMetadataCollection.proto;../lib/api/domain/SecurityMetadataItem.proto;../lib/api/domain/SecurityType.proto;../lib/api/domain/TickDisplaySettings.proto;../lib/api/domain/TickFactors.proto;../lib/api/domain/Venue.proto;../lib/api/io/URL.proto;../lib/api/md/MarketDataSubscriptionParams.proto;../lib/api/md/binance/BinanceMarketDataSubscriptionParams.proto;../lib/api/md/deribit/DeribitMarketDataSubscriptionParams.proto;../lib/api/md/ftx/FTXMarketDataSubscriptionParams.proto;../lib/api/md/glbx/GlbxMarketDataSubscriptionParams.proto;../lib/api/md/huobi/HuobiMarketDataSubscriptionParams.proto;../lib/api/services/AdminService.proto;../lib/api/services/AlgoDefinitionService.proto;../lib/api/services/AlgoService.proto;../lib/api/services/AnalyticsService.proto;../lib/api/services/ApiUnitTestService.proto;../lib/api/services/AuthService.proto;../lib/api/services/BasePositionService.proto;../lib/api/services/ErrorReportService.proto;../lib/api/services/HelloService.proto;../lib/api/services/MBOService.proto;../lib/api/services/MarginService.proto;../lib/api/services/MarketDataAuxiliaryService.proto;../lib/api/services/MarketDataRelay.proto;../lib/api/services/MarketDataService.proto;../lib/api/services/MoneyFileService.proto;../lib/api/services/OrderService.proto;../lib/api/services/OutOfProcessURLListener.proto;../lib/api/services/PersistenceService.proto;../lib/api/services/PositionService.proto;../lib/api/services/ShellDiagnosticService.proto;../lib/api/services/ShellLatencyService.proto;../lib/api/services/SyntheticSecurityService.proto;../lib/domain_model/Security.proto;../lib/md/protos/co2.proto;../lib/sparkproto/SparkProtos.proto;api/build.sh;api/quiet_build.sh;api/src/DynamicService.ts;api/src/ServiceContext.ts;api/src/bts_assert.ts;ui/src/Access.fs;ui/src/Accounts/ClipLimits.fs;ui/src/Accounts/ProductLimits.fs;ui/src/Accounts/SecurityLimits.fs;ui/src/Accounts/State.fs;ui/src/Accounts/Types.fs;ui/src/Accounts/View.fs;ui/src/App.fs;ui/src/Cmd.fs;ui/src/Controls/Button.fs;ui/src/Controls/DataGrid.fs;ui/src/Controls/Fields.fs;ui/src/Controls/Modal.fs;ui/src/Controls/ReactGrid.fs;ui/src/Controls/Table.fs;ui/src/Controls/nbsp.fs;ui/src/Global.fs;ui/src/Imports.fs;ui/src/Lens.fs;ui/src/Login/Login.fs;ui/src/Nav.fs;ui/src/State.fs;ui/src/Types.fs;ui/src/Users/State.fs;ui/src/Users/Types.fs;ui/src/Users/View.fs;ui/src/Util/EnumUtils.fs;ui/src/Util/JSMap.fs;ui/src/Util/StreamUtils.fs;ui/src/obj/Debug/netstandard2.0/ui.AssemblyInfo.fs)
+set(sources ../../externals/linux/include/google/protobuf/descriptor.proto;../../externals/linux/include/google/protobuf/timestamp.proto;../lib/api/analytics/ExpirationDriver.proto;../lib/api/analytics/TimeWeightedInterpolatedDriver.proto;../lib/api/apicore.proto;../lib/api/apigateway.proto;../lib/api/apioptions.proto;../lib/api/domain/Currency.proto;../lib/api/domain/SecurityCriteria.proto;../lib/api/domain/SecurityCriteriaClause.proto;../lib/api/domain/SecurityExpiration.proto;../lib/api/domain/SecurityMetadata.proto;../lib/api/domain/SecurityMetadataCollection.proto;../lib/api/domain/SecurityMetadataItem.proto;../lib/api/domain/SecurityType.proto;../lib/api/domain/TickDisplaySettings.proto;../lib/api/domain/TickFactors.proto;../lib/api/domain/Venue.proto;../lib/api/io/URL.proto;../lib/api/md/MarketDataSubscriptionParams.proto;../lib/api/md/binance/BinanceDepthStreamParams.proto;../lib/api/md/binance/BinanceMarketDataSubscriptionParams.proto;../lib/api/md/deribit/DeribitMarketDataSubscriptionParams.proto;../lib/api/md/ftx/FTXDepthStreamParams.proto;../lib/api/md/ftx/FTXMarketDataSubscriptionParams.proto;../lib/api/md/glbx/GlbxMarketDataSubscriptionParams.proto;../lib/api/md/huobi/HuobiMarketDataSubscriptionParams.proto;../lib/api/services/AdminService.proto;../lib/api/services/AlgoDefinitionService.proto;../lib/api/services/AlgoService.proto;../lib/api/services/AnalyticsService.proto;../lib/api/services/ApiUnitTestService.proto;../lib/api/services/AuthService.proto;../lib/api/services/BasePositionService.proto;../lib/api/services/ErrorReportService.proto;../lib/api/services/HelloService.proto;../lib/api/services/MBOService.proto;../lib/api/services/MarginService.proto;../lib/api/services/MarketDataAuxiliaryService.proto;../lib/api/services/MarketDataRelay.proto;../lib/api/services/MarketDataService.proto;../lib/api/services/MoneyFileService.proto;../lib/api/services/OrderService.proto;../lib/api/services/OutOfProcessURLListener.proto;../lib/api/services/PersistenceService.proto;../lib/api/services/PositionService.proto;../lib/api/services/ShellDiagnosticService.proto;../lib/api/services/ShellLatencyService.proto;../lib/api/services/SyntheticSecurityService.proto;../lib/domain_model/Security.proto;../lib/md/protos/co2.proto;../lib/sparkproto/SparkProtos.proto;api/build.sh;api/quiet_build.sh;api/src/DynamicService.ts;api/src/ServiceContext.ts;api/src/bts_assert.ts;ui/src/Access.fs;ui/src/Accounts/ClipLimits.fs;ui/src/Accounts/ProductLimits.fs;ui/src/Accounts/SecurityLimits.fs;ui/src/Accounts/State.fs;ui/src/Accounts/Types.fs;ui/src/Accounts/View.fs;ui/src/App.fs;ui/src/Cmd.fs;ui/src/Controls/Button.fs;ui/src/Controls/DataGrid.fs;ui/src/Controls/Fields.fs;ui/src/Controls/Modal.fs;ui/src/Controls/ReactGrid.fs;ui/src/Controls/Table.fs;ui/src/Controls/nbsp.fs;ui/src/Global.fs;ui/src/Imports.fs;ui/src/Lens.fs;ui/src/Login/Login.fs;ui/src/Nav.fs;ui/src/State.fs;ui/src/Types.fs;ui/src/Users/State.fs;ui/src/Users/Types.fs;ui/src/Users/View.fs;ui/src/Util/EnumUtils.fs;ui/src/Util/JSMap.fs;ui/src/Util/StreamUtils.fs;ui/src/obj/Debug/netstandard2.0/ui.AssemblyInfo.fs)