webapp check pt?

Also moved a few logging deps from lib_spark -> system:
* bts_chrono.h
* bts_no_copy.h
* defer.h
* dispatcher.h/cpp
* file_utils.h/cpp
* type_name.h

.devcontainer/Dockerfile

@@ -87,6 +87,10 @@ RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
 # Pcap
 RUN apt-get update && apt-get install -y libpcap0.8-dev
 
+# Databento (deps)
+RUN apt-get update && apt-get -y install libzstd-dev
+
+# Docker in Docker
 RUN apt-get update && apt-get install -y \
     docker.io \
     docker-compose \

CMakeLists.txt

@@ -1,5 +1,5 @@
 cmake_minimum_required(VERSION 3.12)
-project(spark_root)
+project(spark)
 set(CMAKE_CXX_STANDARD 20)
 
 if(WIN32)
@@ -107,6 +107,7 @@ else()
     add_subdirectory(src/lib_system)
     add_subdirectory(src/lib_domain)
     add_subdirectory(src/lib_python_md)
+    add_subdirectory(src/lib_logging)
     add_subdirectory(src/app_sparkd)
     add_subdirectory(src/data_files)
   endif()

cmake/InstallBasic.cmake

@@ -1,7 +1,7 @@
 include(${CMAKE_SOURCE_DIR}/cmake/PackBasic.cmake)
 include(GNUInstallDirs)
 
-function(install_basic_config)
+function(install_basic)
   file(
     GLOB_RECURSE ${PROJECT_NAME}_HEADERS
     RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}

src/DataStreamSuite/DataStream/Primitives/FileStreams/LoaderStream.cs

@@ -1,6 +1,33 @@
 using BCG;
 
 namespace bts {
+   //[DataStreamUserRespresentation("saver")]
+   // public class SaverStream : ViewStream {
+   //    private readonly ViewStream _vs;
+   //    private readonly string _file_name;
+   //    public SaverStream(ViewStream vs, string file_name) {
+   //       _vs = vs;
+   //       _file_name = file_name;
+   //    }
+
+   //    protected override void myOnFirstSubscriber() {
+   //       _vs.onChange += _handle_dp;
+   //    }
+
+   //    protected override void myOnLastSubscriber() {
+
+   //    }
+
+   //    private void _handle_dp(object dp_obj) {
+   //       DataPacket dp = dp_obj as DataPacket;
+   //       byte[] serialized_update = null;
+   //       var _serializer = new UniversalStreamIOSerializer();
+   //       serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.CreateMessage(dp.newRows)));
+   //       StreamSerializationUtils.write_to_file(serialized_update, _file_name);
+   //       myFireEvent(dp);
+   //    }
+   // }
+
    [DataStreamUserRespresentation("loader")]
    public class LoaderStream : ViewStream {
 

src/DataStreamSuite/DataStream/Primitives/FileStreams/SaverStream.cs

@@ -31,7 +31,6 @@ namespace bts {
          StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
          byte[] str_bytes = _serializer.serialize(new StreamIOMessage(cr_msg));
          StreamSerializationUtils.write_to_file(str_bytes, _filename);
-
       }
    }
 }

src/DataStreamSuite/DataStream/Primitives/Misc/PubsubEchoTestStream.cs

@@ -10,22 +10,17 @@ using System.Runtime.CompilerServices;
 using bts.api;
 using Bts.PubSub.Routing;
 
-namespace bts
-{
+namespace bts {
    [DataStreamUserRespresentation("pss_echo")]
-   public class PubsubEchoTestStream : ViewStream, IWorker
-   {
-      public class PayloadMessage
-      {
+   public class PubsubEchoTestStream : ViewStream, IWorker {
+      public class PayloadMessage {
          public readonly string msg_id;
          public readonly Int64 timestamp = 0;
          public readonly byte[] payload = null;
 
-         public static byte[] create_data(int payload_size, byte payload_byte)
-         {
+         public static byte[] create_data(int payload_size, byte payload_byte) {
             var rv = new byte[payload_size];
-            for (int i = 0; i < payload_size; i++)
-            {
+            for (int i = 0; i < payload_size; i++) {
                rv[i] = payload_byte;
             }
             return rv;
@@ -41,16 +36,14 @@ namespace bts
          //       payload[i] = payload_byte;
          //    }
          // }
-         public PayloadMessage(string guid_, Int64 timestamp_, byte[] payload_)
-         {
+         public PayloadMessage(string guid_, Int64 timestamp_, byte[] payload_) {
             msg_id = guid_;
             timestamp = timestamp_;
             payload = payload_;
          }
 
          private PayloadMessage() { }
-         public byte[] serialize()
-         {
+         public byte[] serialize() {
             var str = new MemoryStream();
             BinaryWriter bw = new BinaryWriter(str);
             bw.Write(msg_id);
@@ -62,8 +55,7 @@ namespace bts
             return bytes;
          }
 
-         public static PayloadMessage deserialize(byte[] ser_bytes)
-         {
+         public static PayloadMessage deserialize(byte[] ser_bytes) {
             var str = new MemoryStream(ser_bytes);
             BinaryReader br = new BinaryReader(str);
             string guid = br.ReadString();
@@ -89,8 +81,7 @@ namespace bts
       private readonly string _tag_name;
       private readonly int _payload_sz;
       private readonly IPubSubRouter _router;
-      private enum IOState
-      {
+      private enum IOState {
          SEND = 0,
          RECV = 1
       };
@@ -119,8 +110,7 @@ namespace bts
       public PubsubEchoTestStream(string server_url, int publish_period_msecs, int payload_size, string tag_name)
       : this(server_url, TimeSpan.FromMilliseconds(publish_period_msecs), payload_size, tag_name, ViewStreamDatabase.instance) { }
 
-      protected override void myOnFirstSubscriber()
-      {
+      protected override void myOnFirstSubscriber() {
          _pub_core = new PublisherCore(_router);
          _sub_core = new SubscriberCore(_router);
          _sub_core.subscribe(_tag_name);
@@ -128,14 +118,12 @@ namespace bts
 
       }
 
-      protected override void myOnLastSubscriber()
-      {
+      protected override void myOnLastSubscriber() {
          _vsd.removeWorker(this);
          _sub_core.unsubscribe(_tag_name);
       }
 
-      private void _fire_dp_after_send()
-      {
+      private void _fire_dp_after_send() {
          // as soon as the data is published, its reflected in the stream state
          Log.assert(_last_sent_msg != null);
          BlueRow r = new BlueRow(_table.schema, _table.Count, _last_sent_msg.msg_id, _last_sent_msg.timestamp, DBNull.Value, DBNull.Value,
@@ -145,8 +133,7 @@ namespace bts
          myFireEvent(dp_out);
       }
 
-      private void _fire_dp_after_recv()
-      {
+      private void _fire_dp_after_recv() {
          Log.assert(_last_recvd_msg != null);
          // update the last publish row
          int id = _table.Count - 1;
@@ -160,13 +147,11 @@ namespace bts
          myFireEvent(dp_out);
       }
 
-      private string _get_next_id()
-      {
+      private string _get_next_id() {
          return _table.Count.ToString();
       }
 
-      private void _publish_next_packet()
-      {
+      private void _publish_next_packet() {
          var bytes = PayloadMessage.create_data(_payload_sz, _payload_byte);
          _last_sent_msg = new PayloadMessage(_get_next_id(), epoch_now(), bytes);
          var ser_bytes = _last_sent_msg.serialize();
@@ -175,25 +160,19 @@ namespace bts
          _last_publish_time = DateTime.Now;
       }
 
-      public void doWork()
-      {
-         Func<IEnumerable<PubSubServerMessage>, PayloadMessage> find_packet = (updates) =>
-         {
-            foreach (var u in updates)
-            {
+      public void doWork() {
+         Func<IEnumerable<PubSubServerMessage>, PayloadMessage> find_packet = (updates) => {
+            foreach (var u in updates) {
                var recv_msg = PayloadMessage.deserialize(u.data);
-               if (recv_msg.msg_id == _last_sent_msg.msg_id)
-               {
+               if (recv_msg.msg_id == _last_sent_msg.msg_id) {
                   return recv_msg;
                }
             }
             return null;
          };
-         switch (_state)
-         {
+         switch (_state) {
             case IOState.SEND:
-               if (DateTime.Now - _last_publish_time >= _publish_period)
-               {
+               if (DateTime.Now - _last_publish_time >= _publish_period) {
                   _publish_next_packet();
                   _last_publish_time = DateTime.Now;
                   _fire_dp_after_send();
@@ -205,8 +184,7 @@ namespace bts
                var updates = _sub_core.get_updates(TimeSpan.Zero);
                updates = updates.Where(u => u.tag == _tag_name);
                var recv_msg = find_packet(updates);
-               if (recv_msg != null)
-               {
+               if (recv_msg != null) {
                   _last_recvd_msg = recv_msg;
                   _fire_dp_after_recv();
                   _state = IOState.SEND;
@@ -216,8 +194,7 @@ namespace bts
          }
       }
 
-      public static long epoch_now()
-      {
+      public static long epoch_now() {
          return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
       }
 

src/DataStreamSuite/DataStreamPublisher/DataStreamPublisherConfig.cs

@@ -16,6 +16,14 @@ public class DataStreamPublisherConfig {
 
    public List<string>? CsvDirs { get; set; }
 
+   public string? MONGO_DB_URL { get; set; }
+
+   public string? MONGO_DB_NAME { get; set; }
+
+   public string? MONGO_DB_COLLECTION { get; set; }
+
+   public string? BENZINGA_NEWS_TTL { get; set; }
+
    [Option(
       "config",
       MetaValue = "PATH",

src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_APM.cs

@@ -419,7 +419,6 @@ namespace bts {
          // var rcv_message = ExceptionAssert.Cast(res.AsyncState).To<AsyncReceiveMessage>();
          // var buff_txt = AsyncTCPConnection_AsyncWrapper.to_hex(rcv_message.buffer, 0, rcv_message.buffer.Length);
          // Log.Gen.info($"BeginReceive_callback_function _log_incoming_msg buff: {buff_txt}");
-
       }
 
       // this is the function passed to BeginReceive, we 

src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_AsyncWrapper.cs

@@ -1,5 +1,4 @@
-namespace bts
-{
+namespace bts {
    using BCG;
    using BCG.Utilities.General;
    using System;
@@ -12,19 +11,15 @@ namespace bts
 
    // this ITCPConnection uses the TCPClient class instead of a raw socket (in the hope, that it would configure the socket correctly)
 
-   public class AsyncTCPConnection_AsyncWrapper : ITCPConnection, IWorker, IDisposable
-   {
-      public class ConnectionID
-      {
+   public class AsyncTCPConnection_AsyncWrapper : ITCPConnection, IWorker, IDisposable {
+      public class ConnectionID {
          public readonly RemoteConnectionInfo self;
          public readonly RemoteConnectionInfo peer;
          public readonly Type conn_type;
-         public override string ToString()
-         {
+         public override string ToString() {
             return $"{conn_type} self: {self} peer: {peer}";
          }
-         public ConnectionID(RemoteConnectionInfo self_, RemoteConnectionInfo peer_, Type conn_type_)
-         {
+         public ConnectionID(RemoteConnectionInfo self_, RemoteConnectionInfo peer_, Type conn_type_) {
             self = self_;
             peer = peer_;
             conn_type = conn_type_;
@@ -51,21 +46,17 @@ namespace bts
       //    return hex_builder.ToString().Trim();
       // }
 
-      public AsyncTCPConnection_AsyncWrapper(TcpClient tcp_client)
-      {
+      public AsyncTCPConnection_AsyncWrapper(TcpClient tcp_client) {
          _tcp_client = tcp_client;
          _nw_stream = _tcp_client.GetStream();
          _connection_id = new ConnectionID(TCPSocketUtils.get_self_address(_tcp_client.Client), TCPSocketUtils.get_peer_address(_tcp_client.Client), GetType());
          Log.Gen.info($"AsyncTCPConnection_AsyncWrapper.cons connection: {_connection_id}");
       }
 
-      private void _detect_connection_status_change()
-      {
-         Action testing_func = () =>
-         {
+      private void _detect_connection_status_change() {
+         Action testing_func = () => {
             bool connected = _tcp_client.Connected;
-            while (true)
-            {
+            while (true) {
                if (connected != _tcp_client.Connected)
                   Log.Gen.error($"Connection status changed this: {ToString()} {connected} -> {_tcp_client.Connected}");
                connected = _tcp_client.Connected;
@@ -78,17 +69,14 @@ namespace bts
       }
 
       #region Sending 
-      public void _send_fixed_size_msg(byte[] buffer)
-      {
-         // string buff_txt = to_hex(buffer, 0, buffer.Length);
-         // Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._send_fixed_size_msg buffer: {buff_txt}");
+      public void _send_fixed_size_msg(byte[] buffer) {
+         //Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._send_fixed_size_msg buffer: {to_hex(buffer, 0, buffer.Length)}");
 
          int total_bytes_sent = 0;
          int n_bytes = buffer.Length;
 
          int n_transfers = 0;
-         while (_is_alive && total_bytes_sent < n_bytes)
-         {
+         while (_is_alive && total_bytes_sent < n_bytes) {
             int remaining_bytes = n_bytes - total_bytes_sent;
             _nw_stream.Write(buffer, total_bytes_sent, remaining_bytes);
             total_bytes_sent += remaining_bytes;
@@ -97,10 +85,8 @@ namespace bts
          Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._send_fixed_size_msg sent {buffer.Length} bytes in {n_transfers} transfers");
       }
 
-      private void _send_one_msg(byte[] msg)
-      {
-         try
-         {
+      private void _send_one_msg(byte[] msg) {
+         try {
             // the total msg = [msg_sz, msg_type, data] 
             int sz = msg.Length + 4;
             MemoryStream str = new MemoryStream();
@@ -111,20 +97,15 @@ namespace bts
             str.Position = 0;
             byte[] buff = str.ToArray();
             _send_fixed_size_msg(buff);
-         }
-         catch (Exception ex)
-         {
+         } catch (Exception ex) {
             _handle_error(new ErrorMessage(ex, $"_send_one_msg call_stacK: {ex.StackTrace}"));
          }
       }
 
-      private void _send_forever()
-      {
-         while (_is_alive)
-         {
+      private void _send_forever() {
+         while (_is_alive) {
             byte[] data = _send_queue.dequeue();
-            if (data != null)
-            {
+            if (data != null) {
                _send_one_msg(data);
             }
          }
@@ -137,47 +118,39 @@ namespace bts
       }
       #endregion
 
-      public void register(ITCPMessageHandler message_handler)
-      {
+      public void register(ITCPMessageHandler message_handler) {
          _msg_handler = message_handler;
          // starting thread pool threads to keep reading and sending
          ThreadPool.QueueUserWorkItem(_ => _read_msgs_forever());
          ThreadPool.QueueUserWorkItem(_ => _send_forever());
       }
 
-      public RemoteConnectionInfo get_peer_address()
-      {
+      public RemoteConnectionInfo get_peer_address() {
          return _connection_id.peer;
       }
 
-      public RemoteConnectionInfo get_self_address()
-      {
+      public RemoteConnectionInfo get_self_address() {
          return _connection_id.self;
       }
 
       public bool isDisposed { get { return !_is_alive; } }
 
-      public void doWork()
-      {
+      public void doWork() {
          // send hb
       }
 
       #region Reading
       // This makes the blocking IO call, the callstack is _read_msgs_forever -> _read_one_message -> _read_fixed_size_msg
-      private byte[] _read_fixed_size_msg(int n_bytes)
-      {
-         if (n_bytes <= 0)
-         {
+      private byte[] _read_fixed_size_msg(int n_bytes) {
+         if (n_bytes <= 0) {
             throw new ArgumentException("Number of bytes to read must be greater than zero.", nameof(n_bytes));
          }
          byte[] buffer = new byte[n_bytes];
          int total_bytes_read = 0;
          int n_transfers = 0;
-         while (_is_alive && total_bytes_read < n_bytes)
-         {
+         while (_is_alive && total_bytes_read < n_bytes) {
             int bytes_read = _nw_stream.Read(buffer, total_bytes_read, n_bytes - total_bytes_read);
-            if (bytes_read <= 0)
-            {
+            if (bytes_read <= 0) {
                throw new IOException("Read 0 bytes, the connection was closed before all data could be read.");
             }
             n_transfers++;
@@ -188,44 +161,37 @@ namespace bts
          return buffer;
       }
 
-      private class OneRecvMessage
-      {
+      private class OneRecvMessage {
          public int msg_type;
          public byte[] data;
-         public OneRecvMessage(int msg_type_, byte[] data_)
-         {
+         public OneRecvMessage(int msg_type_, byte[] data_) {
             msg_type = msg_type_;
             data = data_;
          }
-         public OneRecvMessage(byte[] msg)
-         {
+         public OneRecvMessage(byte[] msg) {
             MemoryStream str = new MemoryStream(msg);
             BinaryReader br = new BinaryReader(str);
             int msg_type = br.ReadInt32();
-            if (msg_type == DATA_MSG_SIGNAL)
-            {
+            if (msg_type == DATA_MSG_SIGNAL) {
                // TODO, polish it, too much memory bneing allocated
                data = br.ReadBytes(msg.Length - 4);
             }
          }
 
-         public override string ToString()
-         {
+         public override string ToString() {
             string rv = $"OneMessage msg_type: {msg_type} data_len: {data.Length}";
             return rv;
          }
       }
 
-      private OneRecvMessage _read_one_message()
-      {
+      private OneRecvMessage _read_one_message() {
          // read size first
          byte[] sz_arr = _read_fixed_size_msg(4);
          BinaryReader br = new BinaryReader(new MemoryStream(sz_arr));
          br.BaseStream.Position = 0;
          int msg_sz = br.ReadInt32();
          Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._read_one_message read msg_sz: {msg_sz}");
-         if (msg_sz < 0)
-         {
+         if (msg_sz < 0) {
             throw new IOException("Read 0 bytes, the connection was closed before all data could be read.");
          }
          byte[] msg = _read_fixed_size_msg(msg_sz);
@@ -235,29 +201,19 @@ namespace bts
          return rv;
       }
 
-      private void _read_msgs_forever()
-      {
-         while (_is_alive)
-         {
-            try
-            {
+      private void _read_msgs_forever() {
+         while (_is_alive) {
+            try {
                OneRecvMessage one_msg = _read_one_message();
-               if (one_msg.msg_type == DATA_MSG_SIGNAL)
-               {
+               if (one_msg.msg_type == DATA_MSG_SIGNAL) {
                   _msg_handler.onData(new RemoteMessage(one_msg.data, RemoteMessageType.Data, _connection_id.peer));
-               }
-               else if (one_msg.msg_type == HB_MSG_SIGNAL)
-               {
+               } else if (one_msg.msg_type == HB_MSG_SIGNAL) {
                   // keep a last hb time an dudpate it here atomically
                   _msg_handler.onData(new RemoteMessage(null, RemoteMessageType.Heartbeat, _connection_id.peer));
-               }
-               else
-               {
+               } else {
                   throw new ApplicationException($"Unknown message type: {one_msg.msg_type}");
                }
-            }
-            catch (Exception ex)
-            {
+            } catch (Exception ex) {
                _handle_error(new ErrorMessage(ex, $"_read_msgs_forever, call_stack: {ex.StackTrace}"));
             }
          }
@@ -267,19 +223,16 @@ namespace bts
 
       public bool is_disposed => !_is_alive;
 
-      public class ErrorMessage
-      {
+      public class ErrorMessage {
          public readonly SocketError? socket_error = null;
          public readonly Exception exception = null;
          public readonly bool is_exception = false;
          public readonly string context = String.Empty;
-         public ErrorMessage(SocketError socket_error_, string context_)
-         {
+         public ErrorMessage(SocketError socket_error_, string context_) {
             socket_error = socket_error_;
             context = context_;
          }
-         public ErrorMessage(Exception exception_, string context_)
-         {
+         public ErrorMessage(Exception exception_, string context_) {
             is_exception = true;
             exception = exception_;
             context = context_;
@@ -287,30 +240,23 @@ namespace bts
 
          public bool oom => exception != null && exception is OutOfMemoryException;
 
-         public override string ToString()
-         {
+         public override string ToString() {
             if (is_exception)
                return $"ErrorMessage exception: {exception} context: {context}";
             return $"socket_error: {socket_error} context: {context}";
          }
       }
 
-      public void Dispose()
-      {
+      public void Dispose() {
          // socket is closed and disposed here
-         lock (this)
-         {
-            if (_is_alive)
-            {
+         lock (this) {
+            if (_is_alive) {
                //_master_worker.removeWorker(this);
                _is_alive = false;
-               try
-               {
+               try {
                   bts.Log.Gen.info($"AsyncTCPConnection_APM.Dispose this: {this} closing tcp_client nin an orderly fashion now");
                   _tcp_client.Close();
-               }
-               catch (Exception ex)
-               {
+               } catch (Exception ex) {
                   String.Format("AsyncTCPConnection_APM.Dispose exception: {ex} thrown when closing socket");
                   if (ex is OutOfMemoryException)
                      throw ex;
@@ -320,10 +266,8 @@ namespace bts
          }
       }
 
-      private void _handle_error(ErrorMessage err_msg)
-      {
-         lock (this)
-         {
+      private void _handle_error(ErrorMessage err_msg) {
+         lock (this) {
             Log.Gen.error($"AsyncTCPConnection_APM._handle_error this: {this} error_msg: {err_msg}");
             if (err_msg.oom)
                throw err_msg.exception;
@@ -331,9 +275,8 @@ namespace bts
          }
       }
 
-      public override string ToString()
-      {
+      public override string ToString() {
          return _connection_id.ToString();
       }
    }
-}
+}

src/DataStreamSuite/PubSub/Server/Communication/ManagedTCP/ManagedTCPMessageHub.cs

@@ -286,4 +286,4 @@ namespace BCG {
 
 
 
-}
+}

src/app_sparkd/CMakeLists.txt

@@ -1,6 +1,7 @@
-cmake_minimum_required(VERSION 3.5)
+cmake_minimum_required(VERSION 3.14)
 project(sparkd)
 
+include(FetchContent)
 include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
 include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateCpp.cmake)
 include(GNUInstallDirs)
@@ -57,6 +58,14 @@ target_include_directories(
 )
 add_dependencies(${PROJECT_NAME} rapidjson_project)
 
+# databento
+FetchContent_Declare(
+  databento
+  GIT_REPOSITORY https://github.com/databento/databento-cpp
+  GIT_TAG HEAD
+)
+FetchContent_MakeAvailable(databento)
+
 # deribit-demo log level
 if(NOT DEFINED DERIBIT_DEMO_LOG_LEVEL)
   add_definitions(-DSPDLOG_ACTIVE_LEVEL=0)
@@ -130,6 +139,7 @@ target_link_libraries(
   python_md
   python_md_module
   spark
+  databento::databento
   ${emumba-deribit-connector-library_LIBRARIES}
   ssl
   cryptopp
@@ -148,8 +158,8 @@ target_link_libraries(
   -static-libstdc++
   -pthread)
 
-include(${CMAKE_SOURCE_DIR}/cmake/InstallBasicConfig.cmake)
-install_basic_config()
+include(${CMAKE_SOURCE_DIR}/cmake/InstallBasic.cmake)
+install_basic()
 install(
   FILES
     "${EXTERNALS_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"

src/app_sparkd/sparkd/algo/AlgoDefinitionServiceServer.h

@@ -1,22 +1,22 @@
 #pragma once
 
+#include "logging/logging.h"
 #include "spark/api/public/ServiceContext.h"
 #include "spark/api/services/AlgoDefinitionService.server.pb.h"
 #include "spark/api/services/OrderService.client.pb.h"
 #include "spark/api/services/OrderService.pb.h"
-#include "spark/logging/logging.h"
 #include "spark/persistence/IStore.h"
 #include "spark/persistence/SqliteStore.h"
 #include "spark/utils/PBUtils.h"
 #include "spark/utils/TimeStamp.h"
 #include "spark/utils/bts_assert.h"
 #include "spark/utils/bts_no_copy.h"
-#include <memory>
 
 #include <boost/tokenizer.hpp>
 #include <fmt/format.h>
 #include <google/protobuf/util/time_util.h>
 #include <map>
+#include <memory>
 #include <set>
 #include <string>
 #include <utility>
@@ -69,4 +69,4 @@ class AlgoDefinitionServiceServer : public AlgoDefinitionServiceBase {
 } // namespace algo
 } // namespace orderman
 } // namespace api
-} // namespace bts
+} // namespace bts

src/app_sparkd/sparkd/analytics/AnalyticsServiceServer.cpp

@@ -1,8 +1,8 @@
 #include "AnalyticsServiceServer.h"
+#include "logging/logging.h"
 #include "spark/api/apicore.pb.h"
 #include "spark/api/services/AnalyticsService.pb.h"
 #include "spark/api/services/AnalyticsService.server.pb.h"
-#include "spark/logging/logging.h"
 #include "spark/persistence/IStore.h"
 #include "spark/persistence/SqliteStore.h"
 #include "spark/utils/PBUtils.h"

src/app_sparkd/sparkd/cmds/AdminReport.cpp

@@ -1,9 +1,9 @@
 #include "AdminReport.h"
 
+#include "logging/logging.h"
+#include "logging/protobufs/logging.pb.h"
 #include "spark/api/public/ServiceContext.h"
 #include "spark/config/config.h"
-#include "spark/config/protos/logging.pb.h"
-#include "spark/logging/logging.h"
 #include "spark/quickbus/quickbus.h"
 
 #include "spark/api/service_helpers/AdminServiceUtils.h"
@@ -17,9 +17,9 @@
 namespace bts::cmds {
 
 void md_report() {
-   auto cfg = config::get<config::Logging>();
+   auto cfg = config::get<logging::protobufs::Logging>();
    cfg.mutable_console()->set_default_level("Fatal");
-   log::configure_logging(cfg);
+   logging::configure_logging(cfg);
 
    std::map<api::CMEMics, std::string> mic_map;
    mic_map[api::CMEMics::CME] = "XCME";

src/app_sparkd/sparkd/cmds/CMECert.cpp

@@ -1,5 +1,5 @@
 #include "CMECert.h"
-#include "spark/logging/logging.h"
+#include "logging/logging.h"
 #include "spark/orderman/ilink3/IL3Dispatcher.h"
 #include "spark/orderman/ilink3/IL3GW.h"
 #include "spark/orderman/ilink3/IL3Reactor.h"

src/app_sparkd/sparkd/cmds/KeyCmd.cpp

@@ -1,8 +1,8 @@
 #include "KeyCmd.h"
 
+#include "logging/logging.h"
 #include "spark/config/config.h"
 #include "spark/config/protos/keycode.pb.h"
-#include "spark/logging/logging.h"
 #include "spark/os/os.h"
 #include "spark/utils/AuthToken.h"
 #include "spark/utils/FileUtils.h"

src/app_sparkd/sparkd/cmds/QuickTest.cpp

@@ -1,5 +1,5 @@
 #include "QuickTest.h"
-#include "spark/logging/logging.h"
+#include "logging/logging.h"
 
 #include <thread>
 

src/app_sparkd/sparkd/cmds/Run.cpp

@@ -1,9 +1,9 @@
 #include "Run.h"
+#include "logging/logging.h"
 #include "spark/api/public/ServiceContext.h"
 #include "spark/config/config.h"
 #include "spark/config/protos/ports.pb.h"
 #include "spark/config/protos/quickbus_cfg.pb.h"
-#include "spark/logging/logging.h"
 #include "spark/md/MBOProcessor.h"
 #include "spark/md/MBOServiceServer.h"
 #include "spark/md/MDClient.h"

src/app_sparkd/sparkd/cmds/TestPythonMdProviderCmd.cpp

@@ -1,7 +1,7 @@
 #include "TestPythonMdProviderCmd.h"
+#include "logging/logging.h"
 #include "python_md/md_sink_impl.h"
 #include "python_md/py_md_executor.h"
-#include "spark/logging/logging.h"
 
 #include <pybind11/embed.h>
 #include <pybind11/pybind11.h>

src/app_sparkd/sparkd/cmds/Web.cpp

@@ -1,30 +1,22 @@
 #include "Web.h"
+#include "md/cmds/Run.h"
 #include "spark/api/public/ServiceContext.h"
-#include "spark/api/services/MarketDataService.pb.h"
-#include "spark/api/services/SyntheticSecurityService.pb.h"
 #include "spark/config/config.h"
-#include "spark/config/protos/market_data.pb.h"
 #include "spark/config/protos/ports.pb.h"
 #include "spark/config/protos/quickbus_cfg.pb.h"
-#include "spark/md/MBOProcessor.h"
 #include "spark/md/MDClient.h"
 #include "spark/md/SyntheticSecurityService.h"
 #include "spark/md/relay/QueryServer.h"
 #include "spark/metrics/metrics.h"
 #include "spark/metrics/system_metrics.h"
-#include "spark/quickbus/quickbus.h"
+#include "spark/quickbus/core/Quickbus.h"
 #include "spark/tracing/PerfLog.h"
 #include "spark/utils/Dispatcher.h"
-#include "spark/utils/thread_manager.h"
-#include "md/cmds/Cleanup.h"
-#include "md/cmds/Run.h"
-#include "sparkd/exchange_sim/ExchangeSim.h"
 #include "sparkd/http/WebServer.h"
 #include "sparkd/user/AdminServiceServer.h"
 #include "sparkd/user/AuthServiceServer.h"
-#include <optional>
+
 #include <string>
-#include <vector>
 
 namespace bts::cmds {
 

src/app_sparkd/sparkd/cmds/crypto/DeribitDemo.cpp

@@ -1,5 +1,5 @@
 #include "DeribitDemo.h"
-#include "spark/logging/logging.h"
+#include "logging/logging.h"
 #include "sparkd/cmds/crypto/DeribitDemoListener.h"
 
 #include <csignal>

src/app_sparkd/sparkd/cmds/crypto/DeribitDemoConfiguration.cpp

@@ -1,6 +1,6 @@
 #include "DeribitDemoConfiguration.h"
 #include "RapidJsonReader.h"
-#include "spark/logging/logging.h"
+#include "logging/logging.h"
 #include "rapidjson/istreamwrapper.h"
 #include <emumba-deribit-connector/deribit_utils.h>
 #include <fstream>

src/app_sparkd/sparkd/cmds/crypto/LiveLadderDemoCmd.cpp

@@ -1,9 +1,9 @@
 #ifdef SPARK_ENABLE_CCAPI
 #include "LiveLadderDemoCmd.h"
 #include "LiveLadderDemo.h"
+#include "logging/logging.h"
 #include "spark/api/md/MarketDataConfig.pb.h"
 #include "spark/io/ProtoReader.h"
-#include "spark/logging/logging.h"
 #include "sparkd/cmds/common/Validators.h"
 #include "sparkd/cmds/crypto/LiveLadderDemo.h"
 #include <CLI11/CLI11.hpp>

src/app_sparkd/sparkd/cmds/md/cmds/Cleanup.cpp

@@ -1,7 +1,7 @@
 #include "Cleanup.h"
+#include "logging/logging.h"
 #include "spark/config/config.h"
 #include "spark/config/protos/market_data.pb.h"
-#include "spark/logging/logging.h"
 #include "spark/md/ShmSegmentName.h"
 #include "spark/md/protos/co2.pb.h"
 #include <boost/filesystem.hpp>

src/app_sparkd/sparkd/cmds/md/cmds/Run.cpp

@@ -1,8 +1,8 @@
 #include "Run.h"
 #include "Cleanup.h"
 #include "domain/security_criteria.h"
+#include "logging/logging.h"
 #include "spark/config/protos/market_data.pb.h"
-#include "spark/logging/logging.h"
 #include "spark/md/SymbolLoader.h"
 #include "spark/md/UserSymbol.h"
 #include "spark/md/relay/RelayRecorder.h"

src/app_sparkd/sparkd/cmds/md/common/VenueOption.cpp

@@ -18,7 +18,9 @@ EnumOptionNames<api::Venue> VenueOption::names() {
        NAME_PAIR(api::VENUE_BINANCE),
        NAME_PAIR(api::VENUE_BINANCE_US),
        NAME_PAIR(api::VENUE_COINBASE),
-       NAME_PAIR(api::VENUE_BINANCE_USDS_FUTURES)};
+       NAME_PAIR(api::VENUE_BINANCE_USDS_FUTURES),
+       NAME_PAIR(api::VENUE_CBOE),
+       NAME_PAIR(api::VENUE_TAIFEX)};
 }
 
 std::string VenueOption::name(api::Venue venue) {
@@ -45,6 +47,10 @@ std::string VenueOption::name(api::Venue venue) {
       return "coinbase";
    case api::VENUE_BINANCE_USDS_FUTURES:
       return "binance_usds_futures";
+   case api::VENUE_CBOE:
+      return "cboe";
+   case api::VENUE_TAIFEX:
+      return "taifex";
    default:
       return std::to_string(static_cast<int32_t>(venue));
    }

src/app_sparkd/sparkd/exchange_sim/ExchangeSim.cpp

@@ -10,14 +10,14 @@
 #include <iostream>
 #include <string>
 
+#include "logging/logging.h"
 #include "spark/config/config.h"
 #include "spark/config/protos/exchange_sim_cfg.pb.h"
 #include "spark/config/protos/ports.pb.h"
-#include "spark/config/protos/system_config.pb.h"
 #include "spark/domain_model/Security.pb.h"
-#include "spark/logging/logging.h"
 #include "spark/md/PriceSize.h"
 #include "spark/os/os.h"
+#include "system/protobufs/system_config.pb.h"
 
 #include "spark/domain_model/Security.pb.h"
 #include "spark/md/IntPrice.h"
@@ -31,6 +31,9 @@
 
 namespace bts {
 namespace sim {
+
+using bts::system::protobufs::SystemConfig;
+
 auto cfg = "[DEFAULT]\n"
            "ConnectionType=acceptor\n"
            "SocketAcceptPort=$EXCHANGE_SIM_FIX_PORT\n"
@@ -144,7 +147,7 @@ int start_exchange_sim(bts::api::Venue venue, md::MBOWriter *mbo) {
       auto venue_string = boost::to_lower_copy(api::Venue_Name(venue));
       boost::replace_all(
           str_settings, "$BTS_DATA_PATH",
-          config::get<config::SystemConfig>().data_path());
+          config::get<SystemConfig>().data_path());
       boost::replace_all(str_settings, "$SIM_VENUE", venue_string);
       std::string port_string;
       switch (venue) {

src/app_sparkd/sparkd/exchange_sim/ExchangeSimApplication.cpp

@@ -2,10 +2,10 @@
 #include "spark/md/SecurityDB.h"
 
 #include "IDGenerator.h"
+#include "logging/logging.h"
 #include "spark/config/config.h"
 #include "spark/config/protos/exchange_sim_cfg.pb.h"
 #include "spark/domain_model/Security.h"
-#include "spark/logging/logging.h"
 #include "spark/md/MBOProcessor.h"
 #include "spark/md/MDProducer.h"
 #include "spark/md/Shm/FixedSecurity.h"

src/app_sparkd/sparkd/execution/EMExecutionGateway.cpp

@@ -1,11 +1,11 @@
 #include "EMExecutionGateway.h"
+#include "logging/logging.h"
 #include "spark/api/service_helpers/AdminServiceUtils.h"
 #include "spark/api/services/AdminService.pb.h"
 #include "spark/api/services/OrderService.pb.h"
 #include "spark/domain_model/Security.pb.h"
 #include "spark/domain_model/SecurityKey.h"
 #include "spark/domain_model/common.h"
-#include "spark/logging/logging.h"
 #include "spark/md/SecurityOracle.h"
 #include "spark/orderman/CancelOrderMsg.h"
 #include "spark/orderman/CancelReplaceOrderMsg.h"

src/app_sparkd/sparkd/http/ApiGateway.cpp

@@ -1,10 +1,10 @@
 #include "ApiGateway.h"
 
+#include "logging/logging.h"
 #include "spark/api/core/ApiClientInterop.h"
 #include "spark/api/public/ApiUtils.h"
 #include "spark/api/services/AdminService.client.pb.h"
 #include "spark/api/services/AuthService.client.pb.h"
-#include "spark/logging/logging.h"
 #include "spark/utils/bts_assert.h"
 #include <mutex>
 #include <unordered_map>

src/app_sparkd/sparkd/http/MultipartFormDecoder.cpp

@@ -1,5 +1,6 @@
 #include "MultipartFormDecoder.h"
-#include "spark/logging/logging.h"
+#include "logging/logging.h"
+
 #include <boost/algorithm/string.hpp>
 #include <boost/algorithm/string/iter_find.hpp>
 

src/app_sparkd/sparkd/http/WebServer.cpp

@@ -3,9 +3,9 @@
 #include "WebServer.h"
 #include "ApiGateway.h"
 #include "MultipartFormDecoder.h"
+#include "logging/logging.h"
 #include "spark/api/apigateway.pb.h"
 #include "spark/api/core/ApiClientInterop.h"
-#include "spark/logging/logging.h"
 #include "spark/os/os.h"
 #include "spark/tracing/PerfLog.h"
 #include "spark/utils/Dispatcher.h"

src/app_sparkd/sparkd/md/MarketDataAuxiliaryServiceServer.cpp

@@ -1,11 +1,11 @@
 #include "MarketDataAuxiliaryServiceServer.h"
 #include "VolumeAtPriceState.h"
+#include "logging/logging.h"
 #include "spark/api/public/ServiceContext.h"
 #include "spark/api/services/MarketDataRelay.client.pb.h"
 #include "spark/api/services/MarketDataRelay.pb.h"
 #include "spark/config/config.h"
 #include "spark/config/protos/toggles.pb.h"
-#include "spark/logging/logging.h"
 #include "spark/utils/Dispatcher.h"
 #include "spark/utils/PBUtils.h"
 #include "spark/utils/bts_chrono.h"

src/app_sparkd/sparkd/md/MarketDataGateway.cpp

@@ -1,10 +1,10 @@
 #include "MarketDataGateway.h"
 #include "domain/security_id.h"
+#include "logging/logging.h"
 #include "spark/api/md/MarketDataConfig.pb.h"
 #include "spark/api/md/MarketDataConfigUtils.h"
 #include "spark/config/config.h"
 #include "spark/domain/DomainSecurityUtils.h"
-#include "spark/logging/logging.h"
 #include "spark/md/GeneralMarketDataListener.h"
 #include "spark/md/GeneralSecurityIndex.h"
 #include "spark/md/IMarketDataListener.h"

src/app_sparkd/sparkd/md/MarketDataListener.h

@@ -3,9 +3,9 @@
 #include "domain/book.h"
 #include "domain/rfq.h"
 #include "domain/security.h"
+#include "domain/security_market_data_statistics.h"
 #include "domain/security_state.h"
 #include "domain/trade_record.h"
-#include "spark/domain/SecurityMarketDataStatistics.h"
 #include "spark/md/IMarketDataListener.h"
 #include "sparkd/md/MarketDataError.h"
 
@@ -19,7 +19,7 @@ class MarketDataListener : public IMarketDataListener {
    void on_security_statistics(
        const domain::SecurityMarketDataStatistics &security_stats) override;
 
-   void on_book(const domain::Book &book) override;
+   void on_book(const domain::Book &book, std::optional<utils::TimeStamp>) override;
    void on_trades(
        const std::vector<const domain::TradeRecord *> &trades) override;
    void on_rfqs(const std::vector<const domain::RFQ *> &rfqs) override;
@@ -39,7 +39,7 @@ inline void MarketDataListener::on_security_state(
 inline void MarketDataListener::on_security_statistics(
     const domain::SecurityMarketDataStatistics & /*security_stats*/) {}
 
-inline void MarketDataListener::on_book(const domain::Book & /*book*/) {}
+inline void MarketDataListener::on_book(const domain::Book & /*book*/, std::optional<utils::TimeStamp>) {}
 
 inline void MarketDataListener::on_trades(
     const std::vector<const domain::TradeRecord *> & /*trades*/) {}

src/app_sparkd/sparkd/md/MarketDataListenerID.h

@@ -1,13 +1,13 @@
 #pragma once
 #ifndef _WIN32
 
-#include "spark/core/ValueWrapper.h"
 #include "system/FMT_IMPL.h"
+#include "system/value_wrapper.h"
 
 namespace bts::md {
 
 using MarketDataListenerID =
-    core::ValueWrapper<struct MarketDataListenedIDTag, std::uint32_t>;
+    utils::ValueWrapper<struct MarketDataListenedIDTag, std::uint32_t>;
 
 } // namespace bts::md
 

src/app_sparkd/sparkd/md/PyMarketDataSubscription.cpp

@@ -7,13 +7,13 @@
 #include "domain/security_id.h"
 #include "domain/spot_security.h"
 #include "domain/trade_record.h"
+#include "logging/logging.h"
 #include "python_md/md_sink.h"
 #include "python_md/py_book.h"
 #include "python_md/py_md_executor.h"
 #include "python_md/py_security.h"
 #include "python_md/py_security_list.h"
 #include "python_md/py_trade.h"
-#include "spark/logging/logging.h"
 #include "spark/md/IMarketDataListener.h"
 #include "spark/os/os.h"
 #include "spark/persistence/IDGenerator.h"

src/app_sparkd/sparkd/md/ReplayMarketDataSubscription.cpp

@@ -1,6 +1,6 @@
 #include "ReplayMarketDataSubscription.h"
+#include "logging/logging.h"
 #include "spark/api/md/MarketDataReplayConfig.pb.h"
-#include "spark/logging/logging.h"
 #include "spark/md/GeneralMarketDataListener.h"
 #include "spark/md/IMarketDataListener.h"
 #include "spark/md/MDProducer.h"

src/app_sparkd/sparkd/md/binance/BinanceIcebergPartsFilter.h

@@ -1,7 +1,7 @@
 #pragma once
 
-#include "spark/io/JsonMacros.h"
 #include "sparkd/md/binance/BinanceFilterType.h"
+#include "system/json/json_macros.h"
 #include "system/json/json_reader.h"
 
 #include <iostream>

src/app_sparkd/sparkd/md/binance/BinanceLotSizeFilter.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "spark/logging/logging.h"
+#include "logging/logging.h"
 #include "sparkd/md/binance/BinanceFilterType.h"
 #include "system/json/json_reader.h"
 

src/app_sparkd/sparkd/md/binance/BinanceMinNotionalFilter.h

@@ -1,7 +1,7 @@
 #pragma once
 
-#include "spark/io/JsonMacros.h"
 #include "sparkd/md/binance/BinanceFilterType.h"
+#include "system/json/json_macros.h"
 #include "system/json/json_reader.h"
 
 #include <iostream>

src/app_sparkd/sparkd/md/binance/BinanceMiniTickerUpdate.cpp

@@ -2,7 +2,7 @@
 #include "BinanceMiniTickerUpdate.h"
 #include "domain/price.h"
 #include "domain/quantity.h"
-#include "spark/domain/SecurityMarketDataStatistics.h"
+#include "domain/security_market_data_statistics.h"
 #include "sparkd/md/ccapi/CCApiReader.h"
 #include "system/json/json_dumper.h"
 #include "system/json/json_reader.h"

src/app_sparkd/sparkd/md/binance/BinanceMiniTickerUpdate.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "spark/domain/SecurityMarketDataStatistics.h"
+#include "domain/security_market_data_statistics.h"
 #include "sparkd/md/binance/BinanceMiniTickerEventType.h"
 #include "system/time_stamp.h"
 

src/app_sparkd/sparkd/md/binance/BinanceNotionalFilter.h

@@ -1,7 +1,7 @@
 #pragma once
 
-#include "spark/io/JsonMacros.h"
 #include "sparkd/md/binance/BinanceFilterType.h"
+#include "system/json/json_macros.h"
 #include "system/json/json_reader.h"
 
 #include <nlohmann/json.hpp>

src/app_sparkd/sparkd/md/binance/BinancePriceStatistics.cpp

@@ -1,7 +1,7 @@
 #include "BinancePriceStatistics.h"
 #include "domain/price.h"
 #include "domain/quantity.h"
-#include "spark/domain/SecurityMarketDataStatistics.h"
+#include "domain/security_market_data_statistics.h"
 #include "sparkd/md/ccapi/CCApiReader.h"
 #include "system/json/json_dumper.h"
 #include "system/json/json_reader.h"

src/app_sparkd/sparkd/md/binance/BinancePriceStatisticsAdaptor.cpp

@@ -1,6 +1,6 @@
 #include "BinancePriceStatisticsAdaptor.h"
 #include "domain/security_id.h"
-#include "spark/domain/SecurityMarketDataStatistics.h"
+#include "domain/security_market_data_statistics.h"
 #include "spark/md/IMarketDataListener.h"
 #include "spark/utils/bts_fatal.h"
 #include "sparkd/md/binance/BinancePriceStatistics.h"

src/app_sparkd/sparkd/md/binance/BinanceSecurityRequestor.cpp

@@ -3,7 +3,7 @@
 #include "domain/protobufs/venue.pb.h"
 #include "domain/security.h"
 #include "domain/security_criteria.h"
-#include "spark/logging/logging.h"
+#include "logging/logging.h"
 #include "spark/persistence/IDGenerator.h"
 #include "spark/utils/bts_fatal.h"
 #include "sparkd/md/binance/BinanceMessageType.h"

src/app_sparkd/sparkd/md/binance/BinanceSpot.cpp

@@ -2,7 +2,7 @@
 #include "domain/protobufs/venue.pb.h"
 #include "domain/quantity.h"
 #include "domain/security.h"
-#include "spark/logging/logging.h"
+#include "logging/logging.h"
 #include "system/json/json_reader.h"
 
 #include <ccapi_cpp/ccapi_message.h>

src/app_sparkd/sparkd/md/binance/BinanceSpot.h

@@ -1,13 +1,13 @@
 #pragma once
 
 #include "domain/protobufs/venue.h"
-#include "spark/io/JsonMacros.h"
 #include "sparkd/md/binance/BinanceFilterTuple.h"
 #include "sparkd/md/binance/BinanceInstrumentStatus.h"
 #include "sparkd/md/binance/BinanceOrderType.h"
 #include "sparkd/md/binance/BinancePermission.h"
 #include "sparkd/md/binance/BinanceQuoteAsset.h"
 #include "system/FMT_IMPL.h"
+#include "system/json/json_macros.h"
 
 #include <nlohmann/json.hpp>
 #include <ostream>

src/app_sparkd/sparkd/md/binance/BinanceSubscription.cpp

@@ -2,10 +2,10 @@
 #include "domain/protobufs/venue.pb.h"
 #include "domain/security.h"
 #include "domain/security_criteria.h"
+#include "logging/logging.h"
 #include "spark/api/md/BinanceMarketDataConfig.pb.h"
 #include "spark/io/ProtoPrinter.h"
 #include "spark/io/QueryParams.h"
-#include "spark/logging/logging.h"
 #include "spark/md/IMarketDataListener.h"
 #include "spark/persistence/IDGenerator.h"
 #include "sparkd/md/binance/BinanceMessageType.h"

src/app_sparkd/sparkd/md/binance_us/BinanceUsPriceStatistics.cpp

@@ -1,7 +1,7 @@
 #include "BinanceUsPriceStatistics.h"
 #include "domain/price.h"
 #include "domain/quantity.h"
-#include "spark/domain/SecurityMarketDataStatistics.h"
+#include "domain/security_market_data_statistics.h"
 #include "sparkd/md/ccapi/CCApiReader.h"
 #include "system/json/json_dumper.h"
 #include "system/json/json_reader.h"

src/app_sparkd/sparkd/md/binance_us/BinanceUsPriceStatisticsAdaptor.cpp

@@ -1,6 +1,6 @@
 #include "BinanceUsPriceStatisticsAdaptor.h"
 #include "domain/security_id.h"
-#include "spark/domain/SecurityMarketDataStatistics.h"
+#include "domain/security_market_data_statistics.h"
 #include "spark/md/IMarketDataListener.h"
 #include "spark/utils/bts_fatal.h"
 #include "sparkd/md/binance_us/BinanceUsPriceStatistics.h"

src/app_sparkd/sparkd/md/binance_us/BinanceUsSecurityRequestor.cpp

@@ -3,7 +3,7 @@
 #include "domain/protobufs/venue.pb.h"
 #include "domain/security.h"
 #include "domain/security_criteria.h"
-#include "spark/logging/logging.h"
+#include "logging/logging.h"
 #include "spark/persistence/IDGenerator.h"
 #include "sparkd/md/binance_us/BinanceUsMessageType.h"
 #include "sparkd/md/binance_us/BinanceUsSpot.h"

src/app_sparkd/sparkd/md/binance_us/BinanceUsSubscription.cpp

@@ -2,10 +2,10 @@
 #include "domain/protobufs/venue.pb.h"
 #include "domain/security.h"
 #include "domain/security_criteria.h"
+#include "logging/logging.h"
 #include "spark/api/md/BinanceUsMarketDataConfig.pb.h"
 #include "spark/io/ProtoPrinter.h"
 #include "spark/io/QueryParams.h"
-#include "spark/logging/logging.h"
 #include "spark/md/IMarketDataListener.h"
 #include "spark/persistence/IDGenerator.h"
 #include "sparkd/md/binance/BinanceSpot.h"

src/app_sparkd/sparkd/md/binance_usds_futures/BinanceUsdsFuture.cpp

@@ -2,7 +2,7 @@
 #include "domain/price.h"
 #include "domain/quantity.h"
 #include "domain/security.h"
-#include "spark/logging/logging.h"
+#include "logging/logging.h"
 #include "system/json/json_dumper.h"
 #include "system/json/json_reader.h"
 

src/app_sparkd/sparkd/md/binance_usds_futures/BinanceUsdsFuturesPriceStatistics.cpp

@@ -1,7 +1,7 @@
 #include "BinanceUsdsFuturesPriceStatistics.h"
 #include "domain/price.h"
 #include "domain/quantity.h"
-#include "spark/domain/SecurityMarketDataStatistics.h"
+#include "domain/security_market_data_statistics.h"
 #include "sparkd/md/ccapi/CCApiReader.h"
 #include "system/json/json_reader.h"
 

src/app_sparkd/sparkd/md/binance_usds_futures/BinanceUsdsFuturesPriceStatistics.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "spark/io/JsonMacros.h"
+#include "system/json/json_macros.h"
 #include "system/json/json_reader.h"
 
 #include <nlohmann/json.hpp>

src/app_sparkd/sparkd/md/binance_usds_futures/BinanceUsdsFuturesPriceStatisticsAdaptor.cpp

@@ -1,6 +1,6 @@
 #include "BinanceUsdsFuturesPriceStatisticsAdaptor.h"
 #include "domain/security_id.h"
-#include "spark/domain/SecurityMarketDataStatistics.h"
+#include "domain/security_market_data_statistics.h"
 #include "spark/md/IMarketDataListener.h"
 #include "spark/utils/bts_fatal.h"
 #include "sparkd/md/binance_usds_futures/BinanceUsdsFuturesPriceStatistics.h"

src/app_sparkd/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSecurityRequestor.cpp

@@ -2,8 +2,8 @@
 #include "domain/security.h"
 #include "domain/security_criteria.h"
 #include "domain/security_id.h"
+#include "logging/logging.h"
 #include "spark/api/md/BinanceUsdsFuturesMarketDataConfig.pb.h"
-#include "spark/logging/logging.h"
 #include "spark/persistence/IDGenerator.h"
 #include "spark/utils/bts_fatal.h"
 #include "sparkd/md/binance_usds_futures/BinanceUsdsFuture.h"

src/app_sparkd/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.cpp

@@ -3,10 +3,10 @@
 #include "domain/security.h"
 #include "domain/security_criteria.h"
 #include "domain/security_id.h"
+#include "logging/logging.h"
 #include "spark/api/md/BinanceUsdsFuturesMarketDataConfig.pb.h"
 #include "spark/io/ProtoPrinter.h"
 #include "spark/io/QueryParams.h"
-#include "spark/logging/logging.h"
 #include "spark/md/IMarketDataListener.h"
 #include "spark/persistence/IDGenerator.h"
 #include "sparkd/md/binance_usds_futures/BinanceUsdsFuturesMessageType.h"

src/app_sparkd/sparkd/md/ccapi/CCApiCorrelationID.h

@@ -1,9 +1,9 @@
 #pragma once
 
-#include "spark/core/StringFinder.h"
-#include "spark/core/StringJoiner.h"
 #include "spark/utils/bts_assert.h"
 #include "system/FMT_IMPL.h"
+#include "system/string_finder.h"
+#include "system/string_joiner.h"
 
 #include <boost/algorithm/string/split.hpp>
 #include <cstddef>

src/app_sparkd/sparkd/md/ccapi/CCApiEventLogger.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "spark/logging/logging.h"
+#include "logging/logging.h"
 
 #include <ccapi_cpp/ccapi_event_handler.h>
 

src/app_sparkd/sparkd/md/ccapi/CCApiMarketDataProvider.cpp

@@ -1,9 +1,9 @@
 #include "CCApiMarketDataProvider.h"
 #include "domain/protobufs/venue.pb.h"
+#include "logging/logging.h"
 #include "spark/api/md/MarketDataConfig.pb.h"
 #include "spark/config/config.h"
 #include "spark/io/ProtoPrinter.h"
-#include "spark/logging/logging.h"
 #include "spark/md/GeneralMarketDataListener.h"
 #include "spark/md/GeneralSecurityIndex.h"
 #include "spark/md/MDProducer.h"

src/app_sparkd/sparkd/md/ccapi/CCApiMarketDepthAdaptor.cpp

@@ -9,6 +9,7 @@
 #include <ccapi_cpp/ccapi_macro.h>
 #include <ccapi_cpp/ccapi_message.h>
 #include <ccapi_cpp/ccapi_session.h>
+#include <exception>
 #include <vector>
 
 namespace bts::md {
@@ -32,8 +33,20 @@ static bool parse(const ccapi::Message &msg, domain::Book &book) {
       //    ["ASK_PRICE", "ASK_SIZE"]
       const auto &props = level.getNameValueMap();
       for (const auto &[name, value] : props) {
-         bid_walker.parse_field(name, value) ||
-             ask_walker.parse_field(name, value);
+         if (value.empty()) {
+            LOG(md, Error,
+                "Received depth item with empty '{}' value. full_msg:\n{}",
+                name, msg.toStringPretty());
+            continue;
+         }
+         try {
+            bid_walker.parse_field(name, value) ||
+                ask_walker.parse_field(name, value);
+         } catch (const std::exception &ex) {
+            LOG(md, Fatal, "Error parsing depth update: {}\nmsg:\n{}",
+                std::string{ex.what()}, msg.toStringPretty());
+            bts_assert_msg(false, "Fatal error parsing depth update.");
+         }
       }
    }
 

src/app_sparkd/sparkd/md/ccapi/CCApiMessageLogger.cpp

@@ -1,6 +1,5 @@
 #include "CCApiMessageLogger.h"
-#include "spark/core/StringJoiner.h"
-#include "system/time_stamp.h"
+#include "system/string_joiner.h"
 
 #include <fmt/format.h>
 #include <range/v3/view.hpp>

src/app_sparkd/sparkd/md/ccapi/CCApiSession.cpp

@@ -1,5 +1,5 @@
 #include "CCApiSession.h"
-#include "spark/logging/logging.h"
+#include "logging/logging.h"
 #include "sparkd/md/ccapi/CCApiCorrelationID.h"
 #include "sparkd/md/ccapi/CCApiEventDemuxer.h"
 #include "sparkd/md/ccapi/ICCApiMessageHandler.h"

src/app_sparkd/sparkd/md/ccapi/CCApiTradeRecord.h

@@ -1,8 +1,7 @@
 #pragma once
 
-#include "spark/core/NaN.h"
-
 #include <cstdint>
+#include <limits>
 #include <nlohmann/json.hpp>
 #include <vector>
 
@@ -24,8 +23,8 @@ namespace bts::md {
 class CCApiTradeRecord {
  public:
    bool is_buyer_maker = false;
-   double last_price = core::NaN::value();
-   double last_size = core::NaN::value();
+   double last_price = std::numeric_limits<double>::quiet_NaN();
+   double last_size = std::numeric_limits<double>::quiet_NaN();
    std::uint64_t trade_id = 0;
 
    static std::vector<CCApiTradeRecord> parse(const ccapi::Message &source);

src/app_sparkd/sparkd/md/ccapi/CCDemoListener.cpp

@@ -1,5 +1,5 @@
 #include "CCDemoListener.h"
-#include "spark/logging/logging.h"
+#include "logging/logging.h"
 #include "sparkd/md/MarketDataError.h"
 
 #include <functional>
@@ -25,7 +25,7 @@ void CCDemoListener::on_security_state(const domain::SecurityState &) {}
 void CCDemoListener::on_security_statistics(
     const domain::SecurityMarketDataStatistics &) {}
 
-void CCDemoListener::on_book(const domain::Book &book) {
+void CCDemoListener::on_book(const domain::Book &book, std::optional<utils::TimeStamp> ) {
    auto bids = book[domain::Side::Buy];
    auto asks = book[domain::Side::Sell];
    size_t idx = 0;

src/app_sparkd/sparkd/md/ccapi/CCDemoListener.h

@@ -11,7 +11,7 @@ class CCDemoListener : public IMarketDataListener {
    void on_security_state(const domain::SecurityState &security_state) override;
    void on_security_statistics(
        const domain::SecurityMarketDataStatistics &security_stats) override;
-   void on_book(const domain::Book &book) override;
+   void on_book(const domain::Book &book, std::optional<utils::TimeStamp> ) override;
    void on_trades(
        const std::vector<domain::TradeRecord const *> &trades) override;
    void on_rfqs(const std::vector<domain::RFQ const *> &rfqs) override;

src/app_sparkd/sparkd/md/coinbase/CoinbasePriceStatistics.cpp

@@ -1,7 +1,7 @@
 #include "CoinbasePriceStatistics.h"
 #include "domain/price.h"
 #include "domain/quantity.h"
-#include "spark/domain/SecurityMarketDataStatistics.h"
+#include "domain/security_market_data_statistics.h"
 #include "sparkd/md/ccapi/CCApiReader.h"
 #include "system/json/json_dumper.h"
 #include "system/json/json_reader.h"

src/app_sparkd/sparkd/md/coinbase/CoinbasePriceStatisticsAdaptor.cpp

@@ -1,6 +1,6 @@
 #include "CoinbasePriceStatisticsAdaptor.h"
 #include "domain/security_id.h"
-#include "spark/domain/SecurityMarketDataStatistics.h"
+#include "domain/security_market_data_statistics.h"
 #include "spark/md/IMarketDataListener.h"
 #include "sparkd/md/coinbase/CoinbasePriceStatistics.h"
 

src/app_sparkd/sparkd/md/coinbase/CoinbaseSecurityRequestor.cpp

@@ -3,7 +3,7 @@
 #include "domain/protobufs/venue.pb.h"
 #include "domain/security.h"
 #include "domain/security_criteria.h"
-#include "spark/logging/logging.h"
+#include "logging/logging.h"
 #include "spark/persistence/IDGenerator.h"
 #include "sparkd/md/ccapi/CCApiCorrelationID.h"
 #include "sparkd/md/ccapi/CCApiMessageHandler.h"

src/app_sparkd/sparkd/md/coinbase/CoinbaseSpot.cpp

@@ -2,7 +2,7 @@
 #include "domain/protobufs/currency.pb.h"
 #include "domain/quantity.h"
 #include "domain/security.h"
-#include "spark/logging/logging.h"
+#include "logging/logging.h"
 #include "system/json/json_reader.h"
 
 #include <ccapi_cpp/ccapi_message.h>

src/app_sparkd/sparkd/md/deribit/DeribitDepthAdaptor.cpp

@@ -2,7 +2,7 @@
 #include "domain/book.h"
 #include "domain/book_level_comp.h"
 #include "domain/security_id.h"
-#include "spark/logging/logging.h"
+#include "logging/logging.h"
 #include "spark/md/IMarketDataListener.h"
 #include "spark/utils/bts_assert.h"
 

src/app_sparkd/sparkd/md/deribit/DeribitInstrumentStore.cpp

@@ -1,7 +1,7 @@
 #include "DeribitInstrumentStore.h"
 #include "domain/protobufs/security_type.h"
 #include "domain/security.h"
-#include "spark/logging/logging.h"
+#include "logging/logging.h"
 #include "spark/persistence/IDGenerator.h"
 #include "spark/utils/bts_fatal.h"
 #include "sparkd/md/deribit/DeribitCombo.h"

src/app_sparkd/sparkd/md/deribit/DeribitSecurityRequestor.cpp

@@ -60,10 +60,10 @@
 #include "domain/protobufs/venue.pb.h"
 #include "domain/security.h"
 #include "domain/security_criteria.h"
+#include "logging/logging.h"
 #include "spark/io/websocket/JsonRpcError.h"
 #include "spark/io/websocket/JsonRpcHandler.h"
 #include "spark/io/websocket/JsonRpcRequest.h"
-#include "spark/logging/logging.h"
 #include "sparkd/md/ccapi/CCApiCorrelationID.h"
 #include "sparkd/md/ccapi/CCApiJsonRpcRequest.h"
 #include "sparkd/md/ccapi/CCApiJsonRpcRouter.h"

src/app_sparkd/sparkd/md/deribit/DeribitSubscription.cpp

@@ -2,12 +2,12 @@
 #include "domain/protobufs/venue.pb.h"
 #include "domain/security.h"
 #include "domain/security_criteria.h"
+#include "logging/logging.h"
 #include "spark/api/md/DeribitMarketDataConfig.pb.h"
 #include "spark/domain_model/Security.pb.h"
 #include "spark/io/ProtoPrinter.h"
 #include "spark/io/websocket/JsonRpcError.h"
 #include "spark/io/websocket/JsonRpcRequest.h"
-#include "spark/logging/logging.h"
 #include "spark/md/IMarketDataListener.h"
 #include "spark/persistence/IDGenerator.h"
 #include "sparkd/md/ccapi/CCApiCorrelationID.h"

src/app_sparkd/sparkd/md/deribit/DeribitTickerAdaptor.cpp

@@ -1,7 +1,8 @@
 #include "DeribitTickerAdaptor.h"
 #include "domain/security_id.h"
-#include "spark/domain/SecurityMarketDataStatistics.h"
+#include "domain/security_market_data_statistics.h"
 #include "spark/md/IMarketDataListener.h"
+#include "spark/utils/bts_assert.h"
 #include "system/json/json_reader.h"
 
 #include <nlohmann/json.hpp>

src/app_sparkd/sparkd/md/emumba/EmumbaDeribitSubscription.cpp

@@ -1,11 +1,12 @@
 #include "EmumbaDeribitSubscription.h"
 #include "spark/api/md/EmumbaDeribitMarketDataConfig.pb.h"
-#include "spark/config/protos/system_config.pb.h"
 #include "spark/utils/thread_manager.h"
 #include "sparkd/md/emumba/EmumbaDeribitData.h"
+#include "system/protobufs/system_config.pb.h"
 
 namespace bts::md {
-using bts::config::SystemConfig;
+
+using bts::system::protobufs::SystemConfig;
 
 using Path = boost::filesystem::path;
 

src/app_sparkd/sparkd/md/ftx/FTXFuture.cpp

@@ -1,11 +1,10 @@
 #include "FTXFuture.h"
 #include "domain/future_security.h"
-#include "domain/protobufs/currency.pb.h"
 #include "domain/protobufs/venue.pb.h"
 #include "domain/security.h"
-#include "spark/core/TableFormatter.h"
-#include "spark/logging/logging.h"
+#include "logging/logging.h"
 #include "system/json/json_reader.h"
+#include "system/table_formatter.h"
 
 #include <ccapi_cpp/ccapi_message.h>
 #include <nlohmann/json.hpp>
@@ -18,7 +17,6 @@ namespace bts::md::ftx {
 using bts::core::TableFormatter;
 using bts::domain::FutureSecurity;
 using bts::domain::Security;
-using bts::domain::protobufs::Currency;
 using bts::io::JsonReader;
 using std::string;
 using std::stringstream;

src/app_sparkd/sparkd/md/ftx/FTXHistoricalPriceAdaptor.cpp

@@ -1,5 +1,5 @@
 #include "FTXHistoricalPriceAdaptor.h"
-#include "spark/domain/SecurityMarketDataStatistics.h"
+#include "domain/security_market_data_statistics.h"
 #include "spark/md/IMarketDataListener.h"
 #include "sparkd/md/ftx/FTXHistoricalPriceRecord.h"
 

src/app_sparkd/sparkd/md/ftx/FTXHistoricalPriceRecord.cpp

@@ -1,6 +1,6 @@
 #include "FTXHistoricalPriceRecord.h"
 #include "domain/price.h"
-#include "spark/domain/SecurityMarketDataStatistics.h"
+#include "domain/security_market_data_statistics.h"
 #include "sparkd/md/ccapi/CCApiReader.h"
 #include "system/json/json_reader.h"
 

src/app_sparkd/sparkd/md/ftx/FTXMarket.cpp

@@ -1,16 +1,17 @@
 #include "FTXMarket.h"
 #include "domain/protobufs/venue.pb.h"
 #include "domain/security.h"
-#include "spark/core/NaN.h"
-#include "spark/core/TableFormatter.h"
-#include "spark/logging/logging.h"
+#include "logging/logging.h"
 #include "spark/utils/bts_fatal.h"
 #include "sparkd/md/ftx/FTXSecurityType.h"
 #include "system/json/json_reader.h"
+#include "system/nan.h"
+#include "system/table_formatter.h"
 
 #include <ccapi_cpp/ccapi_element.h>
 #include <ccapi_cpp/ccapi_macro.h>
 #include <ccapi_cpp/ccapi_message.h>
+#include <limits>
 #include <nlohmann/json.hpp>
 #include <regex>
 
@@ -133,11 +134,11 @@ void from_json(const nlohmann::json &j, FTXMarket &mkt) {
    jr.read("minProvideSize", mkt.min_provide_size);
 
    jr.read("enabled", mkt.enabled);
-   jr.read_or("ask", mkt.ask, NaN::value());
-   jr.read_or("bid", mkt.bid, NaN::value());
-   jr.read_or("last", mkt.last, NaN::value());
+   jr.read_or("ask", mkt.ask, std::numeric_limits<double>::quiet_NaN());
+   jr.read_or("bid", mkt.bid, std::numeric_limits<double>::quiet_NaN());
+   jr.read_or("last", mkt.last, std::numeric_limits<double>::quiet_NaN());
    jr.read("postOnly", mkt.post_only);
-   jr.read_or("price", mkt.price, NaN::value());
+   jr.read_or("price", mkt.price, std::numeric_limits<double>::quiet_NaN());
    jr.read("priceIncrement", mkt.price_increment);
    jr.read("sizeIncrement", mkt.size_increment);
    jr.read("restricted", mkt.restricted);

src/app_sparkd/sparkd/md/ftx/FTXSecurityRequestor.cpp

@@ -3,7 +3,7 @@
 #include "domain/protobufs/venue.pb.h"
 #include "domain/security.h"
 #include "domain/security_criteria.h"
-#include "spark/logging/logging.h"
+#include "logging/logging.h"
 #include "spark/persistence/IDGenerator.h"
 #include "sparkd/md/ccapi/CCApiCorrelationID.h"
 #include "sparkd/md/ccapi/CCApiMessageHandler.h"

src/app_sparkd/sparkd/md/ftx/FTXSubscription.cpp

@@ -2,9 +2,9 @@
 #include "domain/protobufs/venue.pb.h"
 #include "domain/security.h"
 #include "domain/security_criteria.h"
+#include "logging/logging.h"
 #include "spark/api/md/FTXMarketDataConfig.pb.h"
 #include "spark/io/ProtoPrinter.h"
-#include "spark/logging/logging.h"
 #include "spark/md/IMarketDataListener.h"
 #include "spark/persistence/IDGenerator.h"
 #include "sparkd/md/ccapi/CCApiCorrelationID.h"

src/app_sparkd/sparkd/md/glbx/OnixsGlbxSubscription.cpp

@@ -1,8 +1,8 @@
 #include "OnixsGlbxSubscription.h"
+#include "logging/logging.h"
 #include "spark/api/md/MarketDataConfig.pb.h"
 #include "spark/api/md/MarketDataConfigUtils.h"
 #include "spark/config/config.h"
-#include "spark/logging/logging.h"
 #include "spark/md/IMarketDataListener.h"
 #include "spark/md/MBOProcessor.h"
 #include "spark/md/SymbolLoader.h"

src/app_sparkd/sparkd/md/listeners/LiveLadderPrinter.cpp

@@ -49,7 +49,7 @@ static inline std::ostream &fmt_ladder(
 
 LiveLadderPrinter::LiveLadderPrinter(std::ostream *out) : out_ptr_(out) {}
 
-void LiveLadderPrinter::on_book(const domain::Book &book) {
+void LiveLadderPrinter::on_book(const domain::Book &book, std::optional<utils::TimeStamp>) {
    auto bids = book[domain::Side::Buy];
    auto asks = book[domain::Side::Sell];
    if (bids.is_populated() || asks.is_populated()) {

src/app_sparkd/sparkd/md/listeners/LiveLadderPrinter.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include "spark/utils/bts_assert.h"
 #include "sparkd/md/MarketDataError.h"
 #include "sparkd/md/MarketDataListener.h"
 
@@ -12,7 +13,7 @@ class LiveLadderPrinter : public MarketDataListener {
  public:
    explicit LiveLadderPrinter(std::ostream *out = &std::cout);
 
-   void on_book(const domain::Book &book) override;
+   void on_book(const domain::Book &book, std::optional<utils::TimeStamp>) override;
    void on_error(const MarketDataError &error) override;
 
  private:

src/app_sparkd/sparkd/md/okx/OkxDepthAdaptor.cpp

@@ -2,7 +2,7 @@
 #include "domain/book.h"
 #include "domain/book_level_comp.h"
 #include "domain/security_id.h"
-#include "spark/logging/logging.h"
+#include "logging/logging.h"
 #include "spark/md/IMarketDataListener.h"
 #include "sparkd/md/ccapi/CCApiCorrelationID.h"
 #include "system/json/json_reader.h"

src/app_sparkd/sparkd/md/okx/OkxInstrumentStore.cpp

@@ -1,7 +1,7 @@
 #include "OkxInstrumentStore.h"
 #include "domain/protobufs/security_type.h"
 #include "domain/security.h"
-#include "spark/logging/logging.h"
+#include "logging/logging.h"
 #include "spark/persistence/IDGenerator.h"
 #include "spark/utils/bts_fatal.h"
 #include "sparkd/md/okx/OkxInstrument.h"

src/app_sparkd/sparkd/md/okx/OkxSubscription.cpp

@@ -2,9 +2,9 @@
 #include "domain/protobufs/venue.pb.h"
 #include "domain/security.h"
 #include "domain/security_criteria.h"
+#include "logging/logging.h"
 #include "spark/api/md/OkxMarketDataConfig.pb.h"
 #include "spark/io/ProtoPrinter.h"
-#include "spark/logging/logging.h"
 #include "spark/md/IMarketDataListener.h"
 #include "spark/persistence/IDGenerator.h"
 #include "sparkd/md/ccapi/CCApiSession.h"

src/app_sparkd/sparkd/md/okx/OkxTickerAdaptor.cpp

@@ -1,7 +1,7 @@
 #include "OkxTickerAdaptor.h"
 #include "domain/price.h"
 #include "domain/security_id.h"
-#include "spark/domain/SecurityMarketDataStatistics.h"
+#include "domain/security_market_data_statistics.h"
 #include "spark/md/IMarketDataListener.h"
 #include "sparkd/md/ccapi/CCApiCorrelationID.h"
 #include "sparkd/md/okx/OkxTicker.h"

src/app_sparkd/sparkd/md/platforms/emumba/EmMDListener.cpp

@@ -1,8 +1,8 @@
 #include "EmMDListener.h"
 #include "domain/protobufs/trading_status.pb.h"
+#include "logging/logging.h"
 #include "spark/api/services/ShellDiagnosticService.server.pb.h"
 #include "spark/domain_model/Security.pb.h"
-#include "spark/logging/logging.h"
 #include "spark/md/GeneralSecurityIndex.h"
 #include "spark/md/IntPrice.h"
 #include "spark/md/MDProducer.h"

src/app_sparkd/sparkd/md/platforms/emumba/EmumbaMain.cpp

@@ -1,9 +1,8 @@
 #include "EmumbaMain.h"
 #include "EMTradingSession.h"
 #include "EmMDListener.h"
+#include "logging/logging.h"
 #include "spark/config/config.h"
-#include "spark/config/protos/system_config.pb.h"
-#include "spark/logging/logging.h"
 #include "spark/md/GeneralSecurityIndex.h"
 #include "spark/md/MDProducer.h"
 #include "spark/md/SecurityDB.h"
@@ -17,6 +16,7 @@
 #include "spark/utils/proto_helper.h"
 #include "spark/utils/thread_manager.h"
 #include "sparkd/cmds/crypto/DeribitDemoConfiguration.h"
+#include "system/protobufs/system_config.pb.h"
 
 #include <boost/filesystem.hpp>
 #include <emumba-deribit-connector/deribit_listener.h>
@@ -30,7 +30,7 @@ using namespace bts::orderman;
 
 namespace bts::md {
 
-using bts::config::SystemConfig;
+using bts::system::protobufs::SystemConfig;
 using Path = boost::filesystem::path;
 
 struct EmumbaMain::PImpl {

src/app_sparkd/sparkd/md/platforms/emumba/IEMOrderMuxer.h

@@ -1,9 +1,11 @@
 #pragma once
+
+#include "logging/logging.h"
 #include "spark/domain_model/Security.pb.h"
-#include "spark/logging/logging.h"
 #include "spark/md/common.h"
 #include "spark/orderman/RFQMsg.h"
 #include "spark/os/os.h"
+
 #include <boost/container/small_vector.hpp>
 #include <emumba-deribit-connector/deribit_handler.h>
 #include <emumba-deribit-connector/deribit_listener.h>

src/app_sparkd/sparkd/md/platforms/ice_onixs/ICEMain.cpp

@@ -13,8 +13,8 @@
 #include "spark/utils/proto_helper.h"
 
 #include "spark/config/protos/market_data.pb.h"
-#include "spark/config/protos/system_config.pb.h"
 #include "spark/utils/thread_manager.h"
+#include "system/protobufs/system_config.pb.h"
 
 #include <boost/filesystem.hpp>
 #include <memory>
@@ -43,7 +43,7 @@ namespace bts {
 namespace md {
 namespace ice {
 
-using bts::config::SystemConfig;
+using bts::system::protobufs::SystemConfig;
 using Path = boost::filesystem::path;
 
 ICEMain::ICEMain()

src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp

@@ -1,8 +1,8 @@
 #include "MDP3Main.h"
 
 #include "domain/protobufs/trading_status.pb.h"
+#include "logging/logging.h"
 #include "spark/config/config.h"
-#include "spark/logging/logging.h"
 #include "spark/md/MBOProcessor.h"
 #include "spark/md/SecurityDB.h"
 #include "spark/md/cme/MDP3Utils.h"
@@ -14,7 +14,6 @@
 #include "spark/utils/proto_helper.h"
 
 #include "spark/config/protos/market_data.pb.h"
-#include "spark/config/protos/system_config.pb.h"
 #include "spark/utils/thread_manager.h"
 
 #include <boost/filesystem.hpp>

src/app_sparkd/sparkd/md/recording/MDRecorder.cpp

@@ -1,12 +1,11 @@
 #include "MDRecorder.h"
+#include "logging/logging.h"
 #include "spark/api/md/MarketDataConfig.pb.h"
 #include "spark/api/md/MarketDataConfigUtils.h"
 #include "spark/api/services/MarketDataRelay.client.pb.h"
 #include "spark/api/services/MarketDataRelay.pb.h"
 #include "spark/config/config.h"
-#include "spark/config/protos/system_config.pb.h"
 #include "spark/domain_model/Security.pb.h"
-#include "spark/logging/logging.h"
 #include "spark/md/MDClient.h"
 #include "spark/md/MDConsumer.h"
 #include "spark/md/MDConsumerHandle.h"
@@ -18,6 +17,7 @@
 #include "spark/utils/bts_assert.h"
 #include "spark/utils/datetime.h"
 #include "spark/utils/thread_manager.h"
+#include "system/protobufs/system_config.pb.h"
 #include "system/time_stamp.h"
 
 #include <array>
@@ -30,7 +30,7 @@
 
 namespace bts::md::recording {
 
-using bts::config::SystemConfig;
+using bts::system::protobufs::SystemConfig;
 using Path = boost::filesystem::path;
 
 std::string MDRecorder::file_start_seq = "marketdata";

src/app_sparkd/sparkd/md/recording/MDReplayer.cpp

@@ -5,12 +5,12 @@
 #include "domain/security.h"
 #include "domain/security_common.h"
 #include "domain/security_id.h"
+#include "domain/security_market_data_statistics.h"
 #include "domain/side.h"
 #include "domain/trade_record.h"
+#include "logging/logging.h"
 #include "spark/domain/DomainSecurityUtils.h"
-#include "spark/domain/SecurityMarketDataStatistics.h"
 #include "spark/domain_model/Security.pb.h"
-#include "spark/logging/logging.h"
 #include "spark/md/GeneralMarketDataListener.h"
 #include "spark/md/IMarketDataListener.h"
 #include "spark/md/MDProducer.h"
@@ -22,6 +22,7 @@
 #include "sparkd/md/recording/MDRecorder.h"
 #include "system/time_stamp.h"
 
+#include <cstdint>
 #include <fstream>
 #include <string>
 
@@ -56,10 +57,14 @@ struct MDReplayer ::PImpl {
    std::set<uint32_t> sec_keys = {};
    uint64_t start_time = 0;
    std::shared_ptr<IMarketDataListener> listener{nullptr};
-   PImpl(bts::api::Venue venue) : venue(venue) {}
+   double replay_speed;
+
+   PImpl(bts::api::Venue venue, double replay_speed)
+       : venue(venue), replay_speed(replay_speed) {}
 };
 
-MDReplayer::MDReplayer(bts::api::Venue venue) : me(new PImpl(venue)) {
+MDReplayer::MDReplayer(bts::api::Venue venue, double replay_speed)
+    : me(new PImpl(venue, replay_speed)) {
    LOG(md, Info, "MDReplayer::MDReplayer venue: {}", Venue_Name(venue));
 }
 
@@ -98,6 +103,7 @@ void MDReplayer::_replay_file_once(const std::string &file_name) {
    int i = 0;
    uint64_t beg_ns = bts::core::TimeStamp::now_ns();
    uint64_t last_msg_ts = 0;
+   double replay_speed = me->replay_speed > 0 ? me->replay_speed: 1;
    while (!in.eof() && utils::pb::readbuf(in, &rec_msg)) {
       if (i++ == 0) {
          last_msg_ts = rec_msg.timestamp();
@@ -107,8 +113,9 @@ void MDReplayer::_replay_file_once(const std::string &file_name) {
          uint64_t this_msg_ts = rec_msg.timestamp();
          bts_assert(this_msg_ts >= last_msg_ts);
          uint64_t ts_diff = this_msg_ts - last_msg_ts;
+         uint64_t adjusted_ts = ts_diff/replay_speed;
          last_msg_ts = this_msg_ts;
-         std::this_thread::sleep_for(std::chrono::nanoseconds(ts_diff));
+         std::this_thread::sleep_for(std::chrono::nanoseconds(adjusted_ts));
          _replay_message(rec_msg);
       }
       log_progress(i);
@@ -147,7 +154,7 @@ void MDReplayer::_replay_message(const bts::co2::RecordedMDMessage &m) {
       break;
    }
    case (co2::RecordedMDMessage::MDRecordCase::kMisc): {
-      bts::domain::SecurityMarketDataStatistics stats(m.sec_key(), m.misc());
+      auto stats = domain::DomainSecurityUtils::convert(m.sec_key(), m.misc());
       me->listener->on_security_statistics(stats);
       break;
    }