build: Clean up client CMake

These were used to make code migration smoother (to the new lib_system and lib_domain)

* spark/algorithm/TryCatch.h -> system/algorithm/try_catch.h
* spark/core/BoolName.h -> system/bool_name.h
* spark/core/EnumMacros.h -> system/enum_macros.h
* spark/core/FMT.h -> system/FMT.h
* spark/core/FMT_IMPL.h -> system/FMT_IMPL.h
* spark/core/TimeSpan.h -> system/time_span.h
* spark/core/TimeStamp.h -> system/time_stamp.h
* spark/domain/Price.h -> domain/price.h
* spark/io/JsonReader.h -> system/json/json_reader.h
* spark/io/JsonDumper.h -> system/json/json_dumper.h

.devcontainer/Dockerfile

@@ -87,10 +87,6 @@ RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
 # Pcap
 RUN apt-get update && apt-get install -y libpcap0.8-dev
 
-# Databento (deps)
-RUN apt-get update && apt-get -y install libzstd-dev
-
-# Docker in Docker
 RUN apt-get update && apt-get install -y \
     docker.io \
     docker-compose \

CMakeLists.txt

@@ -1,5 +1,5 @@
 cmake_minimum_required(VERSION 3.12)
-project(spark)
+project(spark_root)
 set(CMAKE_CXX_STANDARD 20)
 
 if(WIN32)
@@ -107,7 +107,6 @@ else()
     add_subdirectory(src/lib_system)
     add_subdirectory(src/lib_domain)
     add_subdirectory(src/lib_python_md)
-    add_subdirectory(src/lib_logging)
     add_subdirectory(src/app_sparkd)
     add_subdirectory(src/data_files)
   endif()

cmake/InstallBasicConfig.cmake

@@ -1,7 +1,7 @@
 include(${CMAKE_SOURCE_DIR}/cmake/PackBasic.cmake)
 include(GNUInstallDirs)
 
-function(install_basic)
+function(install_basic_config)
   file(
     GLOB_RECURSE ${PROJECT_NAME}_HEADERS
     RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}

src/DataStreamSuite/DataStream/Primitives/FileStreams/LoaderStream.cs

@@ -1,33 +1,6 @@
 using BCG;
 
 namespace bts {
-   //[DataStreamUserRespresentation("saver")]
-   // public class SaverStream : ViewStream {
-   //    private readonly ViewStream _vs;
-   //    private readonly string _file_name;
-   //    public SaverStream(ViewStream vs, string file_name) {
-   //       _vs = vs;
-   //       _file_name = file_name;
-   //    }
-
-   //    protected override void myOnFirstSubscriber() {
-   //       _vs.onChange += _handle_dp;
-   //    }
-
-   //    protected override void myOnLastSubscriber() {
-
-   //    }
-
-   //    private void _handle_dp(object dp_obj) {
-   //       DataPacket dp = dp_obj as DataPacket;
-   //       byte[] serialized_update = null;
-   //       var _serializer = new UniversalStreamIOSerializer();
-   //       serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.CreateMessage(dp.newRows)));
-   //       StreamSerializationUtils.write_to_file(serialized_update, _file_name);
-   //       myFireEvent(dp);
-   //    }
-   // }
-
    [DataStreamUserRespresentation("loader")]
    public class LoaderStream : ViewStream {
 

src/DataStreamSuite/DataStream/Primitives/FileStreams/SaverStream.cs

@@ -31,6 +31,7 @@ namespace bts {
          StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
          byte[] str_bytes = _serializer.serialize(new StreamIOMessage(cr_msg));
          StreamSerializationUtils.write_to_file(str_bytes, _filename);
+
       }
    }
 }

src/DataStreamSuite/DataStream/Primitives/Misc/PubsubEchoTestStream.cs

@@ -10,17 +10,22 @@ using System.Runtime.CompilerServices;
 using bts.api;
 using Bts.PubSub.Routing;
 
-namespace bts {
+namespace bts
+{
    [DataStreamUserRespresentation("pss_echo")]
-   public class PubsubEchoTestStream : ViewStream, IWorker {
-      public class PayloadMessage {
+   public class PubsubEchoTestStream : ViewStream, IWorker
+   {
+      public class PayloadMessage
+      {
          public readonly string msg_id;
          public readonly Int64 timestamp = 0;
          public readonly byte[] payload = null;
 
-         public static byte[] create_data(int payload_size, byte payload_byte) {
+         public static byte[] create_data(int payload_size, byte payload_byte)
+         {
             var rv = new byte[payload_size];
-            for (int i = 0; i < payload_size; i++) {
+            for (int i = 0; i < payload_size; i++)
+            {
                rv[i] = payload_byte;
             }
             return rv;
@@ -36,14 +41,16 @@ namespace bts {
          //       payload[i] = payload_byte;
          //    }
          // }
-         public PayloadMessage(string guid_, Int64 timestamp_, byte[] payload_) {
+         public PayloadMessage(string guid_, Int64 timestamp_, byte[] payload_)
+         {
             msg_id = guid_;
             timestamp = timestamp_;
             payload = payload_;
          }
 
          private PayloadMessage() { }
-         public byte[] serialize() {
+         public byte[] serialize()
+         {
             var str = new MemoryStream();
             BinaryWriter bw = new BinaryWriter(str);
             bw.Write(msg_id);
@@ -55,7 +62,8 @@ namespace bts {
             return bytes;
          }
 
-         public static PayloadMessage deserialize(byte[] ser_bytes) {
+         public static PayloadMessage deserialize(byte[] ser_bytes)
+         {
             var str = new MemoryStream(ser_bytes);
             BinaryReader br = new BinaryReader(str);
             string guid = br.ReadString();
@@ -81,7 +89,8 @@ namespace bts {
       private readonly string _tag_name;
       private readonly int _payload_sz;
       private readonly IPubSubRouter _router;
-      private enum IOState {
+      private enum IOState
+      {
          SEND = 0,
          RECV = 1
       };
@@ -110,7 +119,8 @@ namespace bts {
       public PubsubEchoTestStream(string server_url, int publish_period_msecs, int payload_size, string tag_name)
       : this(server_url, TimeSpan.FromMilliseconds(publish_period_msecs), payload_size, tag_name, ViewStreamDatabase.instance) { }
 
-      protected override void myOnFirstSubscriber() {
+      protected override void myOnFirstSubscriber()
+      {
          _pub_core = new PublisherCore(_router);
          _sub_core = new SubscriberCore(_router);
          _sub_core.subscribe(_tag_name);
@@ -118,12 +128,14 @@ namespace bts {
 
       }
 
-      protected override void myOnLastSubscriber() {
+      protected override void myOnLastSubscriber()
+      {
          _vsd.removeWorker(this);
          _sub_core.unsubscribe(_tag_name);
       }
 
-      private void _fire_dp_after_send() {
+      private void _fire_dp_after_send()
+      {
          // as soon as the data is published, its reflected in the stream state
          Log.assert(_last_sent_msg != null);
          BlueRow r = new BlueRow(_table.schema, _table.Count, _last_sent_msg.msg_id, _last_sent_msg.timestamp, DBNull.Value, DBNull.Value,
@@ -133,7 +145,8 @@ namespace bts {
          myFireEvent(dp_out);
       }
 
-      private void _fire_dp_after_recv() {
+      private void _fire_dp_after_recv()
+      {
          Log.assert(_last_recvd_msg != null);
          // update the last publish row
          int id = _table.Count - 1;
@@ -147,11 +160,13 @@ namespace bts {
          myFireEvent(dp_out);
       }
 
-      private string _get_next_id() {
+      private string _get_next_id()
+      {
          return _table.Count.ToString();
       }
 
-      private void _publish_next_packet() {
+      private void _publish_next_packet()
+      {
          var bytes = PayloadMessage.create_data(_payload_sz, _payload_byte);
          _last_sent_msg = new PayloadMessage(_get_next_id(), epoch_now(), bytes);
          var ser_bytes = _last_sent_msg.serialize();
@@ -160,19 +175,25 @@ namespace bts {
          _last_publish_time = DateTime.Now;
       }
 
-      public void doWork() {
-         Func<IEnumerable<PubSubServerMessage>, PayloadMessage> find_packet = (updates) => {
-            foreach (var u in updates) {
+      public void doWork()
+      {
+         Func<IEnumerable<PubSubServerMessage>, PayloadMessage> find_packet = (updates) =>
+         {
+            foreach (var u in updates)
+            {
                var recv_msg = PayloadMessage.deserialize(u.data);
-               if (recv_msg.msg_id == _last_sent_msg.msg_id) {
+               if (recv_msg.msg_id == _last_sent_msg.msg_id)
+               {
                   return recv_msg;
                }
             }
             return null;
          };
-         switch (_state) {
+         switch (_state)
+         {
             case IOState.SEND:
-               if (DateTime.Now - _last_publish_time >= _publish_period) {
+               if (DateTime.Now - _last_publish_time >= _publish_period)
+               {
                   _publish_next_packet();
                   _last_publish_time = DateTime.Now;
                   _fire_dp_after_send();
@@ -184,7 +205,8 @@ namespace bts {
                var updates = _sub_core.get_updates(TimeSpan.Zero);
                updates = updates.Where(u => u.tag == _tag_name);
                var recv_msg = find_packet(updates);
-               if (recv_msg != null) {
+               if (recv_msg != null)
+               {
                   _last_recvd_msg = recv_msg;
                   _fire_dp_after_recv();
                   _state = IOState.SEND;
@@ -194,7 +216,8 @@ namespace bts {
          }
       }
 
-      public static long epoch_now() {
+      public static long epoch_now()
+      {
          return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
       }
 

src/DataStreamSuite/DataStreamPublisher/DataStreamPublisherConfig.cs

@@ -16,14 +16,6 @@ public class DataStreamPublisherConfig {
 
    public List<string>? CsvDirs { get; set; }
 
-   public string? MONGO_DB_URL { get; set; }
-
-   public string? MONGO_DB_NAME { get; set; }
-
-   public string? MONGO_DB_COLLECTION { get; set; }
-
-   public string? BENZINGA_NEWS_TTL { get; set; }
-
    [Option(
       "config",
       MetaValue = "PATH",

src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_APM.cs

@@ -419,6 +419,7 @@ namespace bts {
          // var rcv_message = ExceptionAssert.Cast(res.AsyncState).To<AsyncReceiveMessage>();
          // var buff_txt = AsyncTCPConnection_AsyncWrapper.to_hex(rcv_message.buffer, 0, rcv_message.buffer.Length);
          // Log.Gen.info($"BeginReceive_callback_function _log_incoming_msg buff: {buff_txt}");
+
       }
 
       // this is the function passed to BeginReceive, we 

src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_AsyncWrapper.cs

@@ -1,4 +1,5 @@
-namespace bts {
+namespace bts
+{
    using BCG;
    using BCG.Utilities.General;
    using System;
@@ -11,15 +12,19 @@ namespace bts {
 
    // this ITCPConnection uses the TCPClient class instead of a raw socket (in the hope, that it would configure the socket correctly)
 
-   public class AsyncTCPConnection_AsyncWrapper : ITCPConnection, IWorker, IDisposable {
-      public class ConnectionID {
+   public class AsyncTCPConnection_AsyncWrapper : ITCPConnection, IWorker, IDisposable
+   {
+      public class ConnectionID
+      {
          public readonly RemoteConnectionInfo self;
          public readonly RemoteConnectionInfo peer;
          public readonly Type conn_type;
-         public override string ToString() {
+         public override string ToString()
+         {
             return $"{conn_type} self: {self} peer: {peer}";
          }
-         public ConnectionID(RemoteConnectionInfo self_, RemoteConnectionInfo peer_, Type conn_type_) {
+         public ConnectionID(RemoteConnectionInfo self_, RemoteConnectionInfo peer_, Type conn_type_)
+         {
             self = self_;
             peer = peer_;
             conn_type = conn_type_;
@@ -46,17 +51,21 @@ namespace bts {
       //    return hex_builder.ToString().Trim();
       // }
 
-      public AsyncTCPConnection_AsyncWrapper(TcpClient tcp_client) {
+      public AsyncTCPConnection_AsyncWrapper(TcpClient tcp_client)
+      {
          _tcp_client = tcp_client;
          _nw_stream = _tcp_client.GetStream();
          _connection_id = new ConnectionID(TCPSocketUtils.get_self_address(_tcp_client.Client), TCPSocketUtils.get_peer_address(_tcp_client.Client), GetType());
          Log.Gen.info($"AsyncTCPConnection_AsyncWrapper.cons connection: {_connection_id}");
       }
 
-      private void _detect_connection_status_change() {
-         Action testing_func = () => {
+      private void _detect_connection_status_change()
+      {
+         Action testing_func = () =>
+         {
             bool connected = _tcp_client.Connected;
-            while (true) {
+            while (true)
+            {
                if (connected != _tcp_client.Connected)
                   Log.Gen.error($"Connection status changed this: {ToString()} {connected} -> {_tcp_client.Connected}");
                connected = _tcp_client.Connected;
@@ -69,14 +78,17 @@ namespace bts {
       }
 
       #region Sending 
-      public void _send_fixed_size_msg(byte[] buffer) {
-         //Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._send_fixed_size_msg buffer: {to_hex(buffer, 0, buffer.Length)}");
+      public void _send_fixed_size_msg(byte[] buffer)
+      {
+         // string buff_txt = to_hex(buffer, 0, buffer.Length);
+         // Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._send_fixed_size_msg buffer: {buff_txt}");
 
          int total_bytes_sent = 0;
          int n_bytes = buffer.Length;
 
          int n_transfers = 0;
-         while (_is_alive && total_bytes_sent < n_bytes) {
+         while (_is_alive && total_bytes_sent < n_bytes)
+         {
             int remaining_bytes = n_bytes - total_bytes_sent;
             _nw_stream.Write(buffer, total_bytes_sent, remaining_bytes);
             total_bytes_sent += remaining_bytes;
@@ -85,8 +97,10 @@ namespace bts {
          Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._send_fixed_size_msg sent {buffer.Length} bytes in {n_transfers} transfers");
       }
 
-      private void _send_one_msg(byte[] msg) {
-         try {
+      private void _send_one_msg(byte[] msg)
+      {
+         try
+         {
             // the total msg = [msg_sz, msg_type, data] 
             int sz = msg.Length + 4;
             MemoryStream str = new MemoryStream();
@@ -97,15 +111,20 @@ namespace bts {
             str.Position = 0;
             byte[] buff = str.ToArray();
             _send_fixed_size_msg(buff);
-         } catch (Exception ex) {
+         }
+         catch (Exception ex)
+         {
             _handle_error(new ErrorMessage(ex, $"_send_one_msg call_stacK: {ex.StackTrace}"));
          }
       }
 
-      private void _send_forever() {
-         while (_is_alive) {
+      private void _send_forever()
+      {
+         while (_is_alive)
+         {
             byte[] data = _send_queue.dequeue();
-            if (data != null) {
+            if (data != null)
+            {
                _send_one_msg(data);
             }
          }
@@ -118,39 +137,47 @@ namespace bts {
       }
       #endregion
 
-      public void register(ITCPMessageHandler message_handler) {
+      public void register(ITCPMessageHandler message_handler)
+      {
          _msg_handler = message_handler;
          // starting thread pool threads to keep reading and sending
          ThreadPool.QueueUserWorkItem(_ => _read_msgs_forever());
          ThreadPool.QueueUserWorkItem(_ => _send_forever());
       }
 
-      public RemoteConnectionInfo get_peer_address() {
+      public RemoteConnectionInfo get_peer_address()
+      {
          return _connection_id.peer;
       }
 
-      public RemoteConnectionInfo get_self_address() {
+      public RemoteConnectionInfo get_self_address()
+      {
          return _connection_id.self;
       }
 
       public bool isDisposed { get { return !_is_alive; } }
 
-      public void doWork() {
+      public void doWork()
+      {
          // send hb
       }
 
       #region Reading
       // This makes the blocking IO call, the callstack is _read_msgs_forever -> _read_one_message -> _read_fixed_size_msg
-      private byte[] _read_fixed_size_msg(int n_bytes) {
-         if (n_bytes <= 0) {
+      private byte[] _read_fixed_size_msg(int n_bytes)
+      {
+         if (n_bytes <= 0)
+         {
             throw new ArgumentException("Number of bytes to read must be greater than zero.", nameof(n_bytes));
          }
          byte[] buffer = new byte[n_bytes];
          int total_bytes_read = 0;
          int n_transfers = 0;
-         while (_is_alive && total_bytes_read < n_bytes) {
+         while (_is_alive && total_bytes_read < n_bytes)
+         {
             int bytes_read = _nw_stream.Read(buffer, total_bytes_read, n_bytes - total_bytes_read);
-            if (bytes_read <= 0) {
+            if (bytes_read <= 0)
+            {
                throw new IOException("Read 0 bytes, the connection was closed before all data could be read.");
             }
             n_transfers++;
@@ -161,37 +188,44 @@ namespace bts {
          return buffer;
       }
 
-      private class OneRecvMessage {
+      private class OneRecvMessage
+      {
          public int msg_type;
          public byte[] data;
-         public OneRecvMessage(int msg_type_, byte[] data_) {
+         public OneRecvMessage(int msg_type_, byte[] data_)
+         {
             msg_type = msg_type_;
             data = data_;
          }
-         public OneRecvMessage(byte[] msg) {
+         public OneRecvMessage(byte[] msg)
+         {
             MemoryStream str = new MemoryStream(msg);
             BinaryReader br = new BinaryReader(str);
             int msg_type = br.ReadInt32();
-            if (msg_type == DATA_MSG_SIGNAL) {
+            if (msg_type == DATA_MSG_SIGNAL)
+            {
                // TODO, polish it, too much memory bneing allocated
                data = br.ReadBytes(msg.Length - 4);
             }
          }
 
-         public override string ToString() {
+         public override string ToString()
+         {
             string rv = $"OneMessage msg_type: {msg_type} data_len: {data.Length}";
             return rv;
          }
       }
 
-      private OneRecvMessage _read_one_message() {
+      private OneRecvMessage _read_one_message()
+      {
          // read size first
          byte[] sz_arr = _read_fixed_size_msg(4);
          BinaryReader br = new BinaryReader(new MemoryStream(sz_arr));
          br.BaseStream.Position = 0;
          int msg_sz = br.ReadInt32();
          Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._read_one_message read msg_sz: {msg_sz}");
-         if (msg_sz < 0) {
+         if (msg_sz < 0)
+         {
             throw new IOException("Read 0 bytes, the connection was closed before all data could be read.");
          }
          byte[] msg = _read_fixed_size_msg(msg_sz);
@@ -201,19 +235,29 @@ namespace bts {
          return rv;
       }
 
-      private void _read_msgs_forever() {
-         while (_is_alive) {
-            try {
+      private void _read_msgs_forever()
+      {
+         while (_is_alive)
+         {
+            try
+            {
                OneRecvMessage one_msg = _read_one_message();
-               if (one_msg.msg_type == DATA_MSG_SIGNAL) {
+               if (one_msg.msg_type == DATA_MSG_SIGNAL)
+               {
                   _msg_handler.onData(new RemoteMessage(one_msg.data, RemoteMessageType.Data, _connection_id.peer));
-               } else if (one_msg.msg_type == HB_MSG_SIGNAL) {
+               }
+               else if (one_msg.msg_type == HB_MSG_SIGNAL)
+               {
                   // keep a last hb time an dudpate it here atomically
                   _msg_handler.onData(new RemoteMessage(null, RemoteMessageType.Heartbeat, _connection_id.peer));
-               } else {
+               }
+               else
+               {
                   throw new ApplicationException($"Unknown message type: {one_msg.msg_type}");
                }
-            } catch (Exception ex) {
+            }
+            catch (Exception ex)
+            {
                _handle_error(new ErrorMessage(ex, $"_read_msgs_forever, call_stack: {ex.StackTrace}"));
             }
          }
@@ -223,16 +267,19 @@ namespace bts {
 
       public bool is_disposed => !_is_alive;
 
-      public class ErrorMessage {
+      public class ErrorMessage
+      {
          public readonly SocketError? socket_error = null;
          public readonly Exception exception = null;
          public readonly bool is_exception = false;
          public readonly string context = String.Empty;
-         public ErrorMessage(SocketError socket_error_, string context_) {
+         public ErrorMessage(SocketError socket_error_, string context_)
+         {
             socket_error = socket_error_;
             context = context_;
          }
-         public ErrorMessage(Exception exception_, string context_) {
+         public ErrorMessage(Exception exception_, string context_)
+         {
             is_exception = true;
             exception = exception_;
             context = context_;
@@ -240,23 +287,30 @@ namespace bts {
 
          public bool oom => exception != null && exception is OutOfMemoryException;
 
-         public override string ToString() {
+         public override string ToString()
+         {
             if (is_exception)
                return $"ErrorMessage exception: {exception} context: {context}";
             return $"socket_error: {socket_error} context: {context}";
          }
       }
 
-      public void Dispose() {
+      public void Dispose()
+      {
          // socket is closed and disposed here
-         lock (this) {
-            if (_is_alive) {
+         lock (this)
+         {
+            if (_is_alive)
+            {
                //_master_worker.removeWorker(this);
                _is_alive = false;
-               try {
+               try
+               {
                   bts.Log.Gen.info($"AsyncTCPConnection_APM.Dispose this: {this} closing tcp_client nin an orderly fashion now");
                   _tcp_client.Close();
-               } catch (Exception ex) {
+               }
+               catch (Exception ex)
+               {
                   String.Format("AsyncTCPConnection_APM.Dispose exception: {ex} thrown when closing socket");
                   if (ex is OutOfMemoryException)
                      throw ex;
@@ -266,8 +320,10 @@ namespace bts {
          }
       }
 
-      private void _handle_error(ErrorMessage err_msg) {
-         lock (this) {
+      private void _handle_error(ErrorMessage err_msg)
+      {
+         lock (this)
+         {
             Log.Gen.error($"AsyncTCPConnection_APM._handle_error this: {this} error_msg: {err_msg}");
             if (err_msg.oom)
                throw err_msg.exception;
@@ -275,8 +331,9 @@ namespace bts {
          }
       }
 
-      public override string ToString() {
+      public override string ToString()
+      {
          return _connection_id.ToString();
       }
    }
-}
+}

src/DataStreamSuite/PubSub/Server/Communication/ManagedTCP/ManagedTCPMessageHub.cs

@@ -286,4 +286,4 @@ namespace BCG {
 
 
 
-}
+}

src/app_sparkd/CMakeLists.txt

@@ -1,7 +1,6 @@
-cmake_minimum_required(VERSION 3.14)
+cmake_minimum_required(VERSION 3.5)
 project(sparkd)
 
-include(FetchContent)
 include(${CMAKE_SOURCE_DIR}/cmake/safe_dep_glob.cmake)
 include(${CMAKE_SOURCE_DIR}/cmake/ProtobufGenerateCpp.cmake)
 include(GNUInstallDirs)
@@ -58,14 +57,6 @@ target_include_directories(
 )
 add_dependencies(${PROJECT_NAME} rapidjson_project)
 
-# databento
-FetchContent_Declare(
-  databento
-  GIT_REPOSITORY https://github.com/databento/databento-cpp
-  GIT_TAG HEAD
-)
-FetchContent_MakeAvailable(databento)
-
 # deribit-demo log level
 if(NOT DEFINED DERIBIT_DEMO_LOG_LEVEL)
   add_definitions(-DSPDLOG_ACTIVE_LEVEL=0)
@@ -139,7 +130,6 @@ target_link_libraries(
   python_md
   python_md_module
   spark
-  databento::databento
   ${emumba-deribit-connector-library_LIBRARIES}
   ssl
   cryptopp
@@ -158,8 +148,8 @@ target_link_libraries(
   -static-libstdc++
   -pthread)
 
-include(${CMAKE_SOURCE_DIR}/cmake/InstallBasic.cmake)
-install_basic()
+include(${CMAKE_SOURCE_DIR}/cmake/InstallBasicConfig.cmake)
+install_basic_config()
 install(
   FILES
     "${EXTERNALS_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"

src/app_sparkd/sparkd/algo/AlgoDefinitionServiceServer.h

@@ -1,22 +1,22 @@
 #pragma once
 
-#include "logging/logging.h"
 #include "spark/api/public/ServiceContext.h"
 #include "spark/api/services/AlgoDefinitionService.server.pb.h"
 #include "spark/api/services/OrderService.client.pb.h"
 #include "spark/api/services/OrderService.pb.h"
+#include "spark/logging/logging.h"
 #include "spark/persistence/IStore.h"
 #include "spark/persistence/SqliteStore.h"
 #include "spark/utils/PBUtils.h"
 #include "spark/utils/TimeStamp.h"
 #include "spark/utils/bts_assert.h"
 #include "spark/utils/bts_no_copy.h"
+#include <memory>
 
 #include <boost/tokenizer.hpp>
 #include <fmt/format.h>
 #include <google/protobuf/util/time_util.h>
 #include <map>
-#include <memory>
 #include <set>
 #include <string>
 #include <utility>
@@ -69,4 +69,4 @@ class AlgoDefinitionServiceServer : public AlgoDefinitionServiceBase {
 } // namespace algo
 } // namespace orderman
 } // namespace api
-} // namespace bts
+} // namespace bts

src/app_sparkd/sparkd/analytics/AnalyticsServiceServer.cpp

@@ -1,8 +1,8 @@
 #include "AnalyticsServiceServer.h"
-#include "logging/logging.h"
 #include "spark/api/apicore.pb.h"
 #include "spark/api/services/AnalyticsService.pb.h"
 #include "spark/api/services/AnalyticsService.server.pb.h"
+#include "spark/logging/logging.h"
 #include "spark/persistence/IStore.h"
 #include "spark/persistence/SqliteStore.h"
 #include "spark/utils/PBUtils.h"

src/app_sparkd/sparkd/cmds/AdminReport.cpp

@@ -1,9 +1,9 @@
 #include "AdminReport.h"
 
-#include "logging/logging.h"
-#include "logging/protobufs/logging.pb.h"
 #include "spark/api/public/ServiceContext.h"
 #include "spark/config/config.h"
+#include "spark/config/protos/logging.pb.h"
+#include "spark/logging/logging.h"
 #include "spark/quickbus/quickbus.h"
 
 #include "spark/api/service_helpers/AdminServiceUtils.h"
@@ -17,9 +17,9 @@
 namespace bts::cmds {
 
 void md_report() {
-   auto cfg = config::get<logging::protobufs::Logging>();
+   auto cfg = config::get<config::Logging>();
    cfg.mutable_console()->set_default_level("Fatal");
-   logging::configure_logging(cfg);
+   log::configure_logging(cfg);
 
    std::map<api::CMEMics, std::string> mic_map;
    mic_map[api::CMEMics::CME] = "XCME";

src/app_sparkd/sparkd/cmds/CMECert.cpp

@@ -1,5 +1,5 @@
 #include "CMECert.h"
-#include "logging/logging.h"
+#include "spark/logging/logging.h"
 #include "spark/orderman/ilink3/IL3Dispatcher.h"
 #include "spark/orderman/ilink3/IL3GW.h"
 #include "spark/orderman/ilink3/IL3Reactor.h"

src/app_sparkd/sparkd/cmds/KeyCmd.cpp

@@ -1,8 +1,8 @@
 #include "KeyCmd.h"
 
-#include "logging/logging.h"
 #include "spark/config/config.h"
 #include "spark/config/protos/keycode.pb.h"
+#include "spark/logging/logging.h"
 #include "spark/os/os.h"
 #include "spark/utils/AuthToken.h"
 #include "spark/utils/FileUtils.h"

src/app_sparkd/sparkd/cmds/QuickTest.cpp

@@ -1,5 +1,5 @@
 #include "QuickTest.h"
-#include "logging/logging.h"
+#include "spark/logging/logging.h"
 
 #include <thread>
 

src/app_sparkd/sparkd/cmds/Run.cpp

@@ -1,9 +1,9 @@
 #include "Run.h"
-#include "logging/logging.h"
 #include "spark/api/public/ServiceContext.h"
 #include "spark/config/config.h"
 #include "spark/config/protos/ports.pb.h"
 #include "spark/config/protos/quickbus_cfg.pb.h"
+#include "spark/logging/logging.h"
 #include "spark/md/MBOProcessor.h"
 #include "spark/md/MBOServiceServer.h"
 #include "spark/md/MDClient.h"

src/app_sparkd/sparkd/cmds/TestPythonMdProviderCmd.cpp

@@ -1,7 +1,7 @@
 #include "TestPythonMdProviderCmd.h"
-#include "logging/logging.h"
 #include "python_md/md_sink_impl.h"
 #include "python_md/py_md_executor.h"
+#include "spark/logging/logging.h"
 
 #include <pybind11/embed.h>
 #include <pybind11/pybind11.h>

src/app_sparkd/sparkd/cmds/Web.cpp

@@ -1,22 +1,30 @@
 #include "Web.h"
-#include "md/cmds/Run.h"
 #include "spark/api/public/ServiceContext.h"
+#include "spark/api/services/MarketDataService.pb.h"
+#include "spark/api/services/SyntheticSecurityService.pb.h"
 #include "spark/config/config.h"
+#include "spark/config/protos/market_data.pb.h"
 #include "spark/config/protos/ports.pb.h"
 #include "spark/config/protos/quickbus_cfg.pb.h"
+#include "spark/md/MBOProcessor.h"
 #include "spark/md/MDClient.h"
 #include "spark/md/SyntheticSecurityService.h"
 #include "spark/md/relay/QueryServer.h"
 #include "spark/metrics/metrics.h"
 #include "spark/metrics/system_metrics.h"
-#include "spark/quickbus/core/Quickbus.h"
+#include "spark/quickbus/quickbus.h"
 #include "spark/tracing/PerfLog.h"
 #include "spark/utils/Dispatcher.h"
+#include "spark/utils/thread_manager.h"
+#include "md/cmds/Cleanup.h"
+#include "md/cmds/Run.h"
+#include "sparkd/exchange_sim/ExchangeSim.h"
 #include "sparkd/http/WebServer.h"
 #include "sparkd/user/AdminServiceServer.h"
 #include "sparkd/user/AuthServiceServer.h"
-
+#include <optional>
 #include <string>
+#include <vector>
 
 namespace bts::cmds {
 

src/app_sparkd/sparkd/cmds/crypto/DeribitDemo.cpp

@@ -1,5 +1,5 @@
 #include "DeribitDemo.h"
-#include "logging/logging.h"
+#include "spark/logging/logging.h"
 #include "sparkd/cmds/crypto/DeribitDemoListener.h"
 
 #include <csignal>

src/app_sparkd/sparkd/cmds/crypto/DeribitDemoConfiguration.cpp

@@ -1,6 +1,6 @@
 #include "DeribitDemoConfiguration.h"
 #include "RapidJsonReader.h"
-#include "logging/logging.h"
+#include "spark/logging/logging.h"
 #include "rapidjson/istreamwrapper.h"
 #include <emumba-deribit-connector/deribit_utils.h>
 #include <fstream>

src/app_sparkd/sparkd/cmds/crypto/LiveLadderDemoCmd.cpp

@@ -1,9 +1,9 @@
 #ifdef SPARK_ENABLE_CCAPI
 #include "LiveLadderDemoCmd.h"
 #include "LiveLadderDemo.h"
-#include "logging/logging.h"
 #include "spark/api/md/MarketDataConfig.pb.h"
 #include "spark/io/ProtoReader.h"
+#include "spark/logging/logging.h"
 #include "sparkd/cmds/common/Validators.h"
 #include "sparkd/cmds/crypto/LiveLadderDemo.h"
 #include <CLI11/CLI11.hpp>

src/app_sparkd/sparkd/cmds/md/cmds/Cleanup.cpp

@@ -1,7 +1,7 @@
 #include "Cleanup.h"
-#include "logging/logging.h"
 #include "spark/config/config.h"
 #include "spark/config/protos/market_data.pb.h"
+#include "spark/logging/logging.h"
 #include "spark/md/ShmSegmentName.h"
 #include "spark/md/protos/co2.pb.h"
 #include <boost/filesystem.hpp>

src/app_sparkd/sparkd/cmds/md/cmds/Run.cpp

@@ -1,8 +1,8 @@
 #include "Run.h"
 #include "Cleanup.h"
 #include "domain/security_criteria.h"
-#include "logging/logging.h"
 #include "spark/config/protos/market_data.pb.h"
+#include "spark/logging/logging.h"
 #include "spark/md/SymbolLoader.h"
 #include "spark/md/UserSymbol.h"
 #include "spark/md/relay/RelayRecorder.h"

src/app_sparkd/sparkd/cmds/md/common/VenueOption.cpp

@@ -18,9 +18,7 @@ EnumOptionNames<api::Venue> VenueOption::names() {
        NAME_PAIR(api::VENUE_BINANCE),
        NAME_PAIR(api::VENUE_BINANCE_US),
        NAME_PAIR(api::VENUE_COINBASE),
-       NAME_PAIR(api::VENUE_BINANCE_USDS_FUTURES),
-       NAME_PAIR(api::VENUE_CBOE),
-       NAME_PAIR(api::VENUE_TAIFEX)};
+       NAME_PAIR(api::VENUE_BINANCE_USDS_FUTURES)};
 }
 
 std::string VenueOption::name(api::Venue venue) {
@@ -47,10 +45,6 @@ std::string VenueOption::name(api::Venue venue) {
       return "coinbase";
    case api::VENUE_BINANCE_USDS_FUTURES:
       return "binance_usds_futures";
-   case api::VENUE_CBOE:
-      return "cboe";
-   case api::VENUE_TAIFEX:
-      return "taifex";
    default:
       return std::to_string(static_cast<int32_t>(venue));
    }

src/app_sparkd/sparkd/exchange_sim/ExchangeSim.cpp

@@ -10,14 +10,14 @@
 #include <iostream>
 #include <string>
 
-#include "logging/logging.h"
 #include "spark/config/config.h"
 #include "spark/config/protos/exchange_sim_cfg.pb.h"
 #include "spark/config/protos/ports.pb.h"
+#include "spark/config/protos/system_config.pb.h"
 #include "spark/domain_model/Security.pb.h"
+#include "spark/logging/logging.h"
 #include "spark/md/PriceSize.h"
 #include "spark/os/os.h"
-#include "system/protobufs/system_config.pb.h"
 
 #include "spark/domain_model/Security.pb.h"
 #include "spark/md/IntPrice.h"
@@ -31,9 +31,6 @@
 
 namespace bts {
 namespace sim {
-
-using bts::system::protobufs::SystemConfig;
-
 auto cfg = "[DEFAULT]\n"
            "ConnectionType=acceptor\n"
            "SocketAcceptPort=$EXCHANGE_SIM_FIX_PORT\n"
@@ -147,7 +144,7 @@ int start_exchange_sim(bts::api::Venue venue, md::MBOWriter *mbo) {
       auto venue_string = boost::to_lower_copy(api::Venue_Name(venue));
       boost::replace_all(
           str_settings, "$BTS_DATA_PATH",
-          config::get<SystemConfig>().data_path());
+          config::get<config::SystemConfig>().data_path());
       boost::replace_all(str_settings, "$SIM_VENUE", venue_string);
       std::string port_string;
       switch (venue) {

src/app_sparkd/sparkd/exchange_sim/ExchangeSimApplication.cpp

@@ -2,10 +2,10 @@
 #include "spark/md/SecurityDB.h"
 
 #include "IDGenerator.h"
-#include "logging/logging.h"
 #include "spark/config/config.h"
 #include "spark/config/protos/exchange_sim_cfg.pb.h"
 #include "spark/domain_model/Security.h"
+#include "spark/logging/logging.h"
 #include "spark/md/MBOProcessor.h"
 #include "spark/md/MDProducer.h"
 #include "spark/md/Shm/FixedSecurity.h"

src/app_sparkd/sparkd/execution/EMExecutionGateway.cpp

@@ -1,11 +1,11 @@
 #include "EMExecutionGateway.h"
-#include "logging/logging.h"
 #include "spark/api/service_helpers/AdminServiceUtils.h"
 #include "spark/api/services/AdminService.pb.h"
 #include "spark/api/services/OrderService.pb.h"
 #include "spark/domain_model/Security.pb.h"
 #include "spark/domain_model/SecurityKey.h"
 #include "spark/domain_model/common.h"
+#include "spark/logging/logging.h"
 #include "spark/md/SecurityOracle.h"
 #include "spark/orderman/CancelOrderMsg.h"
 #include "spark/orderman/CancelReplaceOrderMsg.h"

src/app_sparkd/sparkd/http/ApiGateway.cpp

@@ -1,10 +1,10 @@
 #include "ApiGateway.h"
 
-#include "logging/logging.h"
 #include "spark/api/core/ApiClientInterop.h"
 #include "spark/api/public/ApiUtils.h"
 #include "spark/api/services/AdminService.client.pb.h"
 #include "spark/api/services/AuthService.client.pb.h"
+#include "spark/logging/logging.h"
 #include "spark/utils/bts_assert.h"
 #include <mutex>
 #include <unordered_map>

src/app_sparkd/sparkd/http/MultipartFormDecoder.cpp

@@ -1,6 +1,5 @@
 #include "MultipartFormDecoder.h"
-#include "logging/logging.h"
-
+#include "spark/logging/logging.h"
 #include <boost/algorithm/string.hpp>
 #include <boost/algorithm/string/iter_find.hpp>
 

src/app_sparkd/sparkd/http/WebServer.cpp

@@ -3,9 +3,9 @@
 #include "WebServer.h"
 #include "ApiGateway.h"
 #include "MultipartFormDecoder.h"
-#include "logging/logging.h"
 #include "spark/api/apigateway.pb.h"
 #include "spark/api/core/ApiClientInterop.h"
+#include "spark/logging/logging.h"
 #include "spark/os/os.h"
 #include "spark/tracing/PerfLog.h"
 #include "spark/utils/Dispatcher.h"

src/app_sparkd/sparkd/md/MarketDataAuxiliaryServiceServer.cpp

@@ -1,11 +1,11 @@
 #include "MarketDataAuxiliaryServiceServer.h"
 #include "VolumeAtPriceState.h"
-#include "logging/logging.h"
 #include "spark/api/public/ServiceContext.h"
 #include "spark/api/services/MarketDataRelay.client.pb.h"
 #include "spark/api/services/MarketDataRelay.pb.h"
 #include "spark/config/config.h"
 #include "spark/config/protos/toggles.pb.h"
+#include "spark/logging/logging.h"
 #include "spark/utils/Dispatcher.h"
 #include "spark/utils/PBUtils.h"
 #include "spark/utils/bts_chrono.h"

src/app_sparkd/sparkd/md/MarketDataGateway.cpp

@@ -1,10 +1,10 @@
 #include "MarketDataGateway.h"
 #include "domain/security_id.h"
-#include "logging/logging.h"
 #include "spark/api/md/MarketDataConfig.pb.h"
 #include "spark/api/md/MarketDataConfigUtils.h"
 #include "spark/config/config.h"
 #include "spark/domain/DomainSecurityUtils.h"
+#include "spark/logging/logging.h"
 #include "spark/md/GeneralMarketDataListener.h"
 #include "spark/md/GeneralSecurityIndex.h"
 #include "spark/md/IMarketDataListener.h"

src/app_sparkd/sparkd/md/MarketDataListener.h

@@ -3,9 +3,9 @@
 #include "domain/book.h"
 #include "domain/rfq.h"
 #include "domain/security.h"
-#include "domain/security_market_data_statistics.h"
 #include "domain/security_state.h"
 #include "domain/trade_record.h"
+#include "spark/domain/SecurityMarketDataStatistics.h"
 #include "spark/md/IMarketDataListener.h"
 #include "sparkd/md/MarketDataError.h"
 
@@ -19,7 +19,7 @@ class MarketDataListener : public IMarketDataListener {
    void on_security_statistics(
        const domain::SecurityMarketDataStatistics &security_stats) override;
 
-   void on_book(const domain::Book &book, std::optional<utils::TimeStamp>) override;
+   void on_book(const domain::Book &book) override;
    void on_trades(
        const std::vector<const domain::TradeRecord *> &trades) override;
    void on_rfqs(const std::vector<const domain::RFQ *> &rfqs) override;
@@ -39,7 +39,7 @@ inline void MarketDataListener::on_security_state(
 inline void MarketDataListener::on_security_statistics(
     const domain::SecurityMarketDataStatistics & /*security_stats*/) {}
 
-inline void MarketDataListener::on_book(const domain::Book & /*book*/, std::optional<utils::TimeStamp>) {}
+inline void MarketDataListener::on_book(const domain::Book & /*book*/) {}
 
 inline void MarketDataListener::on_trades(
     const std::vector<const domain::TradeRecord *> & /*trades*/) {}

src/app_sparkd/sparkd/md/MarketDataListenerID.h

@@ -1,13 +1,13 @@
 #pragma once
 #ifndef _WIN32
 
+#include "spark/core/ValueWrapper.h"
 #include "system/FMT_IMPL.h"
-#include "system/value_wrapper.h"
 
 namespace bts::md {
 
 using MarketDataListenerID =
-    utils::ValueWrapper<struct MarketDataListenedIDTag, std::uint32_t>;
+    core::ValueWrapper<struct MarketDataListenedIDTag, std::uint32_t>;
 
 } // namespace bts::md
 

src/app_sparkd/sparkd/md/PyMarketDataSubscription.cpp

@@ -7,13 +7,13 @@
 #include "domain/security_id.h"
 #include "domain/spot_security.h"
 #include "domain/trade_record.h"
-#include "logging/logging.h"
 #include "python_md/md_sink.h"
 #include "python_md/py_book.h"
 #include "python_md/py_md_executor.h"
 #include "python_md/py_security.h"
 #include "python_md/py_security_list.h"
 #include "python_md/py_trade.h"
+#include "spark/logging/logging.h"
 #include "spark/md/IMarketDataListener.h"
 #include "spark/os/os.h"
 #include "spark/persistence/IDGenerator.h"

src/app_sparkd/sparkd/md/ReplayMarketDataSubscription.cpp

@@ -1,6 +1,6 @@
 #include "ReplayMarketDataSubscription.h"
-#include "logging/logging.h"
 #include "spark/api/md/MarketDataReplayConfig.pb.h"
+#include "spark/logging/logging.h"
 #include "spark/md/GeneralMarketDataListener.h"
 #include "spark/md/IMarketDataListener.h"
 #include "spark/md/MDProducer.h"

src/app_sparkd/sparkd/md/binance/BinanceIcebergPartsFilter.h

@@ -1,7 +1,7 @@
 #pragma once
 
+#include "spark/io/JsonMacros.h"
 #include "sparkd/md/binance/BinanceFilterType.h"
-#include "system/json/json_macros.h"
 #include "system/json/json_reader.h"
 
 #include <iostream>

src/app_sparkd/sparkd/md/binance/BinanceLotSizeFilter.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "logging/logging.h"
+#include "spark/logging/logging.h"
 #include "sparkd/md/binance/BinanceFilterType.h"
 #include "system/json/json_reader.h"
 

src/app_sparkd/sparkd/md/binance/BinanceMinNotionalFilter.h

@@ -1,7 +1,7 @@
 #pragma once
 
+#include "spark/io/JsonMacros.h"
 #include "sparkd/md/binance/BinanceFilterType.h"
-#include "system/json/json_macros.h"
 #include "system/json/json_reader.h"
 
 #include <iostream>

src/app_sparkd/sparkd/md/binance/BinanceMiniTickerUpdate.cpp

@@ -2,7 +2,7 @@
 #include "BinanceMiniTickerUpdate.h"
 #include "domain/price.h"
 #include "domain/quantity.h"
-#include "domain/security_market_data_statistics.h"
+#include "spark/domain/SecurityMarketDataStatistics.h"
 #include "sparkd/md/ccapi/CCApiReader.h"
 #include "system/json/json_dumper.h"
 #include "system/json/json_reader.h"

src/app_sparkd/sparkd/md/binance/BinanceMiniTickerUpdate.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "domain/security_market_data_statistics.h"
+#include "spark/domain/SecurityMarketDataStatistics.h"
 #include "sparkd/md/binance/BinanceMiniTickerEventType.h"
 #include "system/time_stamp.h"
 

src/app_sparkd/sparkd/md/binance/BinanceNotionalFilter.h

@@ -1,7 +1,7 @@
 #pragma once
 
+#include "spark/io/JsonMacros.h"
 #include "sparkd/md/binance/BinanceFilterType.h"
-#include "system/json/json_macros.h"
 #include "system/json/json_reader.h"
 
 #include <nlohmann/json.hpp>

src/app_sparkd/sparkd/md/binance/BinancePriceStatistics.cpp

@@ -1,7 +1,7 @@
 #include "BinancePriceStatistics.h"
 #include "domain/price.h"
 #include "domain/quantity.h"
-#include "domain/security_market_data_statistics.h"
+#include "spark/domain/SecurityMarketDataStatistics.h"
 #include "sparkd/md/ccapi/CCApiReader.h"
 #include "system/json/json_dumper.h"
 #include "system/json/json_reader.h"

src/app_sparkd/sparkd/md/binance/BinancePriceStatisticsAdaptor.cpp

@@ -1,6 +1,6 @@
 #include "BinancePriceStatisticsAdaptor.h"
 #include "domain/security_id.h"
-#include "domain/security_market_data_statistics.h"
+#include "spark/domain/SecurityMarketDataStatistics.h"
 #include "spark/md/IMarketDataListener.h"
 #include "spark/utils/bts_fatal.h"
 #include "sparkd/md/binance/BinancePriceStatistics.h"

src/app_sparkd/sparkd/md/binance/BinanceSecurityRequestor.cpp

@@ -3,7 +3,7 @@
 #include "domain/protobufs/venue.pb.h"
 #include "domain/security.h"
 #include "domain/security_criteria.h"
-#include "logging/logging.h"
+#include "spark/logging/logging.h"
 #include "spark/persistence/IDGenerator.h"
 #include "spark/utils/bts_fatal.h"
 #include "sparkd/md/binance/BinanceMessageType.h"

src/app_sparkd/sparkd/md/binance/BinanceSpot.cpp

@@ -2,7 +2,7 @@
 #include "domain/protobufs/venue.pb.h"
 #include "domain/quantity.h"
 #include "domain/security.h"
-#include "logging/logging.h"
+#include "spark/logging/logging.h"
 #include "system/json/json_reader.h"
 
 #include <ccapi_cpp/ccapi_message.h>

src/app_sparkd/sparkd/md/binance/BinanceSpot.h

@@ -1,13 +1,13 @@
 #pragma once
 
 #include "domain/protobufs/venue.h"
+#include "spark/io/JsonMacros.h"
 #include "sparkd/md/binance/BinanceFilterTuple.h"
 #include "sparkd/md/binance/BinanceInstrumentStatus.h"
 #include "sparkd/md/binance/BinanceOrderType.h"
 #include "sparkd/md/binance/BinancePermission.h"
 #include "sparkd/md/binance/BinanceQuoteAsset.h"
 #include "system/FMT_IMPL.h"
-#include "system/json/json_macros.h"
 
 #include <nlohmann/json.hpp>
 #include <ostream>

src/app_sparkd/sparkd/md/binance/BinanceSubscription.cpp

@@ -2,10 +2,10 @@
 #include "domain/protobufs/venue.pb.h"
 #include "domain/security.h"
 #include "domain/security_criteria.h"
-#include "logging/logging.h"
 #include "spark/api/md/BinanceMarketDataConfig.pb.h"
 #include "spark/io/ProtoPrinter.h"
 #include "spark/io/QueryParams.h"
+#include "spark/logging/logging.h"
 #include "spark/md/IMarketDataListener.h"
 #include "spark/persistence/IDGenerator.h"
 #include "sparkd/md/binance/BinanceMessageType.h"

src/app_sparkd/sparkd/md/binance_us/BinanceUsPriceStatistics.cpp

@@ -1,7 +1,7 @@
 #include "BinanceUsPriceStatistics.h"
 #include "domain/price.h"
 #include "domain/quantity.h"
-#include "domain/security_market_data_statistics.h"
+#include "spark/domain/SecurityMarketDataStatistics.h"
 #include "sparkd/md/ccapi/CCApiReader.h"
 #include "system/json/json_dumper.h"
 #include "system/json/json_reader.h"

src/app_sparkd/sparkd/md/binance_us/BinanceUsPriceStatisticsAdaptor.cpp

@@ -1,6 +1,6 @@
 #include "BinanceUsPriceStatisticsAdaptor.h"
 #include "domain/security_id.h"
-#include "domain/security_market_data_statistics.h"
+#include "spark/domain/SecurityMarketDataStatistics.h"
 #include "spark/md/IMarketDataListener.h"
 #include "spark/utils/bts_fatal.h"
 #include "sparkd/md/binance_us/BinanceUsPriceStatistics.h"

src/app_sparkd/sparkd/md/binance_us/BinanceUsSecurityRequestor.cpp

@@ -3,7 +3,7 @@
 #include "domain/protobufs/venue.pb.h"
 #include "domain/security.h"
 #include "domain/security_criteria.h"
-#include "logging/logging.h"
+#include "spark/logging/logging.h"
 #include "spark/persistence/IDGenerator.h"
 #include "sparkd/md/binance_us/BinanceUsMessageType.h"
 #include "sparkd/md/binance_us/BinanceUsSpot.h"

src/app_sparkd/sparkd/md/binance_us/BinanceUsSubscription.cpp

@@ -2,10 +2,10 @@
 #include "domain/protobufs/venue.pb.h"
 #include "domain/security.h"
 #include "domain/security_criteria.h"
-#include "logging/logging.h"
 #include "spark/api/md/BinanceUsMarketDataConfig.pb.h"
 #include "spark/io/ProtoPrinter.h"
 #include "spark/io/QueryParams.h"
+#include "spark/logging/logging.h"
 #include "spark/md/IMarketDataListener.h"
 #include "spark/persistence/IDGenerator.h"
 #include "sparkd/md/binance/BinanceSpot.h"

src/app_sparkd/sparkd/md/binance_usds_futures/BinanceUsdsFuture.cpp

@@ -2,7 +2,7 @@
 #include "domain/price.h"
 #include "domain/quantity.h"
 #include "domain/security.h"
-#include "logging/logging.h"
+#include "spark/logging/logging.h"
 #include "system/json/json_dumper.h"
 #include "system/json/json_reader.h"
 

src/app_sparkd/sparkd/md/binance_usds_futures/BinanceUsdsFuturesPriceStatistics.cpp

@@ -1,7 +1,7 @@
 #include "BinanceUsdsFuturesPriceStatistics.h"
 #include "domain/price.h"
 #include "domain/quantity.h"
-#include "domain/security_market_data_statistics.h"
+#include "spark/domain/SecurityMarketDataStatistics.h"
 #include "sparkd/md/ccapi/CCApiReader.h"
 #include "system/json/json_reader.h"
 

src/app_sparkd/sparkd/md/binance_usds_futures/BinanceUsdsFuturesPriceStatistics.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "system/json/json_macros.h"
+#include "spark/io/JsonMacros.h"
 #include "system/json/json_reader.h"
 
 #include <nlohmann/json.hpp>

src/app_sparkd/sparkd/md/binance_usds_futures/BinanceUsdsFuturesPriceStatisticsAdaptor.cpp

@@ -1,6 +1,6 @@
 #include "BinanceUsdsFuturesPriceStatisticsAdaptor.h"
 #include "domain/security_id.h"
-#include "domain/security_market_data_statistics.h"
+#include "spark/domain/SecurityMarketDataStatistics.h"
 #include "spark/md/IMarketDataListener.h"
 #include "spark/utils/bts_fatal.h"
 #include "sparkd/md/binance_usds_futures/BinanceUsdsFuturesPriceStatistics.h"

src/app_sparkd/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSecurityRequestor.cpp

@@ -2,8 +2,8 @@
 #include "domain/security.h"
 #include "domain/security_criteria.h"
 #include "domain/security_id.h"
-#include "logging/logging.h"
 #include "spark/api/md/BinanceUsdsFuturesMarketDataConfig.pb.h"
+#include "spark/logging/logging.h"
 #include "spark/persistence/IDGenerator.h"
 #include "spark/utils/bts_fatal.h"
 #include "sparkd/md/binance_usds_futures/BinanceUsdsFuture.h"

src/app_sparkd/sparkd/md/binance_usds_futures/BinanceUsdsFuturesSubscription.cpp

@@ -3,10 +3,10 @@
 #include "domain/security.h"
 #include "domain/security_criteria.h"
 #include "domain/security_id.h"
-#include "logging/logging.h"
 #include "spark/api/md/BinanceUsdsFuturesMarketDataConfig.pb.h"
 #include "spark/io/ProtoPrinter.h"
 #include "spark/io/QueryParams.h"
+#include "spark/logging/logging.h"
 #include "spark/md/IMarketDataListener.h"
 #include "spark/persistence/IDGenerator.h"
 #include "sparkd/md/binance_usds_futures/BinanceUsdsFuturesMessageType.h"

src/app_sparkd/sparkd/md/ccapi/CCApiCorrelationID.h

@@ -1,9 +1,9 @@
 #pragma once
 
+#include "spark/core/StringFinder.h"
+#include "spark/core/StringJoiner.h"
 #include "spark/utils/bts_assert.h"
 #include "system/FMT_IMPL.h"
-#include "system/string_finder.h"
-#include "system/string_joiner.h"
 
 #include <boost/algorithm/string/split.hpp>
 #include <cstddef>

src/app_sparkd/sparkd/md/ccapi/CCApiEventLogger.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "logging/logging.h"
+#include "spark/logging/logging.h"
 
 #include <ccapi_cpp/ccapi_event_handler.h>
 

src/app_sparkd/sparkd/md/ccapi/CCApiMarketDataProvider.cpp

@@ -1,9 +1,9 @@
 #include "CCApiMarketDataProvider.h"
 #include "domain/protobufs/venue.pb.h"
-#include "logging/logging.h"
 #include "spark/api/md/MarketDataConfig.pb.h"
 #include "spark/config/config.h"
 #include "spark/io/ProtoPrinter.h"
+#include "spark/logging/logging.h"
 #include "spark/md/GeneralMarketDataListener.h"
 #include "spark/md/GeneralSecurityIndex.h"
 #include "spark/md/MDProducer.h"

src/app_sparkd/sparkd/md/ccapi/CCApiMarketDepthAdaptor.cpp

@@ -9,7 +9,6 @@
 #include <ccapi_cpp/ccapi_macro.h>
 #include <ccapi_cpp/ccapi_message.h>
 #include <ccapi_cpp/ccapi_session.h>
-#include <exception>
 #include <vector>
 
 namespace bts::md {
@@ -33,20 +32,8 @@ static bool parse(const ccapi::Message &msg, domain::Book &book) {
       //    ["ASK_PRICE", "ASK_SIZE"]
       const auto &props = level.getNameValueMap();
       for (const auto &[name, value] : props) {
-         if (value.empty()) {
-            LOG(md, Error,
-                "Received depth item with empty '{}' value. full_msg:\n{}",
-                name, msg.toStringPretty());
-            continue;
-         }
-         try {
-            bid_walker.parse_field(name, value) ||
-                ask_walker.parse_field(name, value);
-         } catch (const std::exception &ex) {
-            LOG(md, Fatal, "Error parsing depth update: {}\nmsg:\n{}",
-                std::string{ex.what()}, msg.toStringPretty());
-            bts_assert_msg(false, "Fatal error parsing depth update.");
-         }
+         bid_walker.parse_field(name, value) ||
+             ask_walker.parse_field(name, value);
       }
    }
 

src/app_sparkd/sparkd/md/ccapi/CCApiMessageLogger.cpp

@@ -1,5 +1,6 @@
 #include "CCApiMessageLogger.h"
-#include "system/string_joiner.h"
+#include "spark/core/StringJoiner.h"
+#include "system/time_stamp.h"
 
 #include <fmt/format.h>
 #include <range/v3/view.hpp>

src/app_sparkd/sparkd/md/ccapi/CCApiSession.cpp

@@ -1,5 +1,5 @@
 #include "CCApiSession.h"
-#include "logging/logging.h"
+#include "spark/logging/logging.h"
 #include "sparkd/md/ccapi/CCApiCorrelationID.h"
 #include "sparkd/md/ccapi/CCApiEventDemuxer.h"
 #include "sparkd/md/ccapi/ICCApiMessageHandler.h"

src/app_sparkd/sparkd/md/ccapi/CCApiTradeRecord.h

@@ -1,7 +1,8 @@
 #pragma once
 
+#include "spark/core/NaN.h"
+
 #include <cstdint>
-#include <limits>
 #include <nlohmann/json.hpp>
 #include <vector>
 
@@ -23,8 +24,8 @@ namespace bts::md {
 class CCApiTradeRecord {
  public:
    bool is_buyer_maker = false;
-   double last_price = std::numeric_limits<double>::quiet_NaN();
-   double last_size = std::numeric_limits<double>::quiet_NaN();
+   double last_price = core::NaN::value();
+   double last_size = core::NaN::value();
    std::uint64_t trade_id = 0;
 
    static std::vector<CCApiTradeRecord> parse(const ccapi::Message &source);

src/app_sparkd/sparkd/md/ccapi/CCDemoListener.cpp

@@ -1,5 +1,5 @@
 #include "CCDemoListener.h"
-#include "logging/logging.h"
+#include "spark/logging/logging.h"
 #include "sparkd/md/MarketDataError.h"
 
 #include <functional>
@@ -25,7 +25,7 @@ void CCDemoListener::on_security_state(const domain::SecurityState &) {}
 void CCDemoListener::on_security_statistics(
     const domain::SecurityMarketDataStatistics &) {}
 
-void CCDemoListener::on_book(const domain::Book &book, std::optional<utils::TimeStamp> ) {
+void CCDemoListener::on_book(const domain::Book &book) {
    auto bids = book[domain::Side::Buy];
    auto asks = book[domain::Side::Sell];
    size_t idx = 0;

src/app_sparkd/sparkd/md/ccapi/CCDemoListener.h

@@ -11,7 +11,7 @@ class CCDemoListener : public IMarketDataListener {
    void on_security_state(const domain::SecurityState &security_state) override;
    void on_security_statistics(
        const domain::SecurityMarketDataStatistics &security_stats) override;
-   void on_book(const domain::Book &book, std::optional<utils::TimeStamp> ) override;
+   void on_book(const domain::Book &book) override;
    void on_trades(
        const std::vector<domain::TradeRecord const *> &trades) override;
    void on_rfqs(const std::vector<domain::RFQ const *> &rfqs) override;

src/app_sparkd/sparkd/md/coinbase/CoinbasePriceStatistics.cpp

@@ -1,7 +1,7 @@
 #include "CoinbasePriceStatistics.h"
 #include "domain/price.h"
 #include "domain/quantity.h"
-#include "domain/security_market_data_statistics.h"
+#include "spark/domain/SecurityMarketDataStatistics.h"
 #include "sparkd/md/ccapi/CCApiReader.h"
 #include "system/json/json_dumper.h"
 #include "system/json/json_reader.h"

src/app_sparkd/sparkd/md/coinbase/CoinbasePriceStatisticsAdaptor.cpp

@@ -1,6 +1,6 @@
 #include "CoinbasePriceStatisticsAdaptor.h"
 #include "domain/security_id.h"
-#include "domain/security_market_data_statistics.h"
+#include "spark/domain/SecurityMarketDataStatistics.h"
 #include "spark/md/IMarketDataListener.h"
 #include "sparkd/md/coinbase/CoinbasePriceStatistics.h"
 

src/app_sparkd/sparkd/md/coinbase/CoinbaseSecurityRequestor.cpp

@@ -3,7 +3,7 @@
 #include "domain/protobufs/venue.pb.h"
 #include "domain/security.h"
 #include "domain/security_criteria.h"
-#include "logging/logging.h"
+#include "spark/logging/logging.h"
 #include "spark/persistence/IDGenerator.h"
 #include "sparkd/md/ccapi/CCApiCorrelationID.h"
 #include "sparkd/md/ccapi/CCApiMessageHandler.h"

src/app_sparkd/sparkd/md/coinbase/CoinbaseSpot.cpp

@@ -2,7 +2,7 @@
 #include "domain/protobufs/currency.pb.h"
 #include "domain/quantity.h"
 #include "domain/security.h"
-#include "logging/logging.h"
+#include "spark/logging/logging.h"
 #include "system/json/json_reader.h"
 
 #include <ccapi_cpp/ccapi_message.h>

src/app_sparkd/sparkd/md/deribit/DeribitDepthAdaptor.cpp

@@ -2,7 +2,7 @@
 #include "domain/book.h"
 #include "domain/book_level_comp.h"
 #include "domain/security_id.h"
-#include "logging/logging.h"
+#include "spark/logging/logging.h"
 #include "spark/md/IMarketDataListener.h"
 #include "spark/utils/bts_assert.h"
 

src/app_sparkd/sparkd/md/deribit/DeribitInstrumentStore.cpp

@@ -1,7 +1,7 @@
 #include "DeribitInstrumentStore.h"
 #include "domain/protobufs/security_type.h"
 #include "domain/security.h"
-#include "logging/logging.h"
+#include "spark/logging/logging.h"
 #include "spark/persistence/IDGenerator.h"
 #include "spark/utils/bts_fatal.h"
 #include "sparkd/md/deribit/DeribitCombo.h"

src/app_sparkd/sparkd/md/deribit/DeribitSecurityRequestor.cpp

@@ -60,10 +60,10 @@
 #include "domain/protobufs/venue.pb.h"
 #include "domain/security.h"
 #include "domain/security_criteria.h"
-#include "logging/logging.h"
 #include "spark/io/websocket/JsonRpcError.h"
 #include "spark/io/websocket/JsonRpcHandler.h"
 #include "spark/io/websocket/JsonRpcRequest.h"
+#include "spark/logging/logging.h"
 #include "sparkd/md/ccapi/CCApiCorrelationID.h"
 #include "sparkd/md/ccapi/CCApiJsonRpcRequest.h"
 #include "sparkd/md/ccapi/CCApiJsonRpcRouter.h"

src/app_sparkd/sparkd/md/deribit/DeribitSubscription.cpp

@@ -2,12 +2,12 @@
 #include "domain/protobufs/venue.pb.h"
 #include "domain/security.h"
 #include "domain/security_criteria.h"
-#include "logging/logging.h"
 #include "spark/api/md/DeribitMarketDataConfig.pb.h"
 #include "spark/domain_model/Security.pb.h"
 #include "spark/io/ProtoPrinter.h"
 #include "spark/io/websocket/JsonRpcError.h"
 #include "spark/io/websocket/JsonRpcRequest.h"
+#include "spark/logging/logging.h"
 #include "spark/md/IMarketDataListener.h"
 #include "spark/persistence/IDGenerator.h"
 #include "sparkd/md/ccapi/CCApiCorrelationID.h"

src/app_sparkd/sparkd/md/deribit/DeribitTickerAdaptor.cpp

@@ -1,8 +1,7 @@
 #include "DeribitTickerAdaptor.h"
 #include "domain/security_id.h"
-#include "domain/security_market_data_statistics.h"
+#include "spark/domain/SecurityMarketDataStatistics.h"
 #include "spark/md/IMarketDataListener.h"
-#include "spark/utils/bts_assert.h"
 #include "system/json/json_reader.h"
 
 #include <nlohmann/json.hpp>

src/app_sparkd/sparkd/md/emumba/EmumbaDeribitSubscription.cpp

@@ -1,12 +1,11 @@
 #include "EmumbaDeribitSubscription.h"
 #include "spark/api/md/EmumbaDeribitMarketDataConfig.pb.h"
+#include "spark/config/protos/system_config.pb.h"
 #include "spark/utils/thread_manager.h"
 #include "sparkd/md/emumba/EmumbaDeribitData.h"
-#include "system/protobufs/system_config.pb.h"
 
 namespace bts::md {
-
-using bts::system::protobufs::SystemConfig;
+using bts::config::SystemConfig;
 
 using Path = boost::filesystem::path;
 

src/app_sparkd/sparkd/md/ftx/FTXFuture.cpp

@@ -1,10 +1,11 @@
 #include "FTXFuture.h"
 #include "domain/future_security.h"
+#include "domain/protobufs/currency.pb.h"
 #include "domain/protobufs/venue.pb.h"
 #include "domain/security.h"
-#include "logging/logging.h"
+#include "spark/core/TableFormatter.h"
+#include "spark/logging/logging.h"
 #include "system/json/json_reader.h"
-#include "system/table_formatter.h"
 
 #include <ccapi_cpp/ccapi_message.h>
 #include <nlohmann/json.hpp>
@@ -17,6 +18,7 @@ namespace bts::md::ftx {
 using bts::core::TableFormatter;
 using bts::domain::FutureSecurity;
 using bts::domain::Security;
+using bts::domain::protobufs::Currency;
 using bts::io::JsonReader;
 using std::string;
 using std::stringstream;

src/app_sparkd/sparkd/md/ftx/FTXHistoricalPriceAdaptor.cpp

@@ -1,5 +1,5 @@
 #include "FTXHistoricalPriceAdaptor.h"
-#include "domain/security_market_data_statistics.h"
+#include "spark/domain/SecurityMarketDataStatistics.h"
 #include "spark/md/IMarketDataListener.h"
 #include "sparkd/md/ftx/FTXHistoricalPriceRecord.h"
 

src/app_sparkd/sparkd/md/ftx/FTXHistoricalPriceRecord.cpp

@@ -1,6 +1,6 @@
 #include "FTXHistoricalPriceRecord.h"
 #include "domain/price.h"
-#include "domain/security_market_data_statistics.h"
+#include "spark/domain/SecurityMarketDataStatistics.h"
 #include "sparkd/md/ccapi/CCApiReader.h"
 #include "system/json/json_reader.h"
 

src/app_sparkd/sparkd/md/ftx/FTXMarket.cpp

@@ -1,17 +1,16 @@
 #include "FTXMarket.h"
 #include "domain/protobufs/venue.pb.h"
 #include "domain/security.h"
-#include "logging/logging.h"
+#include "spark/core/NaN.h"
+#include "spark/core/TableFormatter.h"
+#include "spark/logging/logging.h"
 #include "spark/utils/bts_fatal.h"
 #include "sparkd/md/ftx/FTXSecurityType.h"
 #include "system/json/json_reader.h"
-#include "system/nan.h"
-#include "system/table_formatter.h"
 
 #include <ccapi_cpp/ccapi_element.h>
 #include <ccapi_cpp/ccapi_macro.h>
 #include <ccapi_cpp/ccapi_message.h>
-#include <limits>
 #include <nlohmann/json.hpp>
 #include <regex>
 
@@ -134,11 +133,11 @@ void from_json(const nlohmann::json &j, FTXMarket &mkt) {
    jr.read("minProvideSize", mkt.min_provide_size);
 
    jr.read("enabled", mkt.enabled);
-   jr.read_or("ask", mkt.ask, std::numeric_limits<double>::quiet_NaN());
-   jr.read_or("bid", mkt.bid, std::numeric_limits<double>::quiet_NaN());
-   jr.read_or("last", mkt.last, std::numeric_limits<double>::quiet_NaN());
+   jr.read_or("ask", mkt.ask, NaN::value());
+   jr.read_or("bid", mkt.bid, NaN::value());
+   jr.read_or("last", mkt.last, NaN::value());
    jr.read("postOnly", mkt.post_only);
-   jr.read_or("price", mkt.price, std::numeric_limits<double>::quiet_NaN());
+   jr.read_or("price", mkt.price, NaN::value());
    jr.read("priceIncrement", mkt.price_increment);
    jr.read("sizeIncrement", mkt.size_increment);
    jr.read("restricted", mkt.restricted);

src/app_sparkd/sparkd/md/ftx/FTXSecurityRequestor.cpp

@@ -3,7 +3,7 @@
 #include "domain/protobufs/venue.pb.h"
 #include "domain/security.h"
 #include "domain/security_criteria.h"
-#include "logging/logging.h"
+#include "spark/logging/logging.h"
 #include "spark/persistence/IDGenerator.h"
 #include "sparkd/md/ccapi/CCApiCorrelationID.h"
 #include "sparkd/md/ccapi/CCApiMessageHandler.h"

src/app_sparkd/sparkd/md/ftx/FTXSubscription.cpp

@@ -2,9 +2,9 @@
 #include "domain/protobufs/venue.pb.h"
 #include "domain/security.h"
 #include "domain/security_criteria.h"
-#include "logging/logging.h"
 #include "spark/api/md/FTXMarketDataConfig.pb.h"
 #include "spark/io/ProtoPrinter.h"
+#include "spark/logging/logging.h"
 #include "spark/md/IMarketDataListener.h"
 #include "spark/persistence/IDGenerator.h"
 #include "sparkd/md/ccapi/CCApiCorrelationID.h"

src/app_sparkd/sparkd/md/glbx/OnixsGlbxSubscription.cpp

@@ -1,8 +1,8 @@
 #include "OnixsGlbxSubscription.h"
-#include "logging/logging.h"
 #include "spark/api/md/MarketDataConfig.pb.h"
 #include "spark/api/md/MarketDataConfigUtils.h"
 #include "spark/config/config.h"
+#include "spark/logging/logging.h"
 #include "spark/md/IMarketDataListener.h"
 #include "spark/md/MBOProcessor.h"
 #include "spark/md/SymbolLoader.h"

src/app_sparkd/sparkd/md/listeners/LiveLadderPrinter.cpp

@@ -49,7 +49,7 @@ static inline std::ostream &fmt_ladder(
 
 LiveLadderPrinter::LiveLadderPrinter(std::ostream *out) : out_ptr_(out) {}
 
-void LiveLadderPrinter::on_book(const domain::Book &book, std::optional<utils::TimeStamp>) {
+void LiveLadderPrinter::on_book(const domain::Book &book) {
    auto bids = book[domain::Side::Buy];
    auto asks = book[domain::Side::Sell];
    if (bids.is_populated() || asks.is_populated()) {

src/app_sparkd/sparkd/md/listeners/LiveLadderPrinter.h

@@ -1,6 +1,5 @@
 #pragma once
 
-#include "spark/utils/bts_assert.h"
 #include "sparkd/md/MarketDataError.h"
 #include "sparkd/md/MarketDataListener.h"
 
@@ -13,7 +12,7 @@ class LiveLadderPrinter : public MarketDataListener {
  public:
    explicit LiveLadderPrinter(std::ostream *out = &std::cout);
 
-   void on_book(const domain::Book &book, std::optional<utils::TimeStamp>) override;
+   void on_book(const domain::Book &book) override;
    void on_error(const MarketDataError &error) override;
 
  private:

src/app_sparkd/sparkd/md/okx/OkxDepthAdaptor.cpp

@@ -2,7 +2,7 @@
 #include "domain/book.h"
 #include "domain/book_level_comp.h"
 #include "domain/security_id.h"
-#include "logging/logging.h"
+#include "spark/logging/logging.h"
 #include "spark/md/IMarketDataListener.h"
 #include "sparkd/md/ccapi/CCApiCorrelationID.h"
 #include "system/json/json_reader.h"

src/app_sparkd/sparkd/md/okx/OkxInstrumentStore.cpp

@@ -1,7 +1,7 @@
 #include "OkxInstrumentStore.h"
 #include "domain/protobufs/security_type.h"
 #include "domain/security.h"
-#include "logging/logging.h"
+#include "spark/logging/logging.h"
 #include "spark/persistence/IDGenerator.h"
 #include "spark/utils/bts_fatal.h"
 #include "sparkd/md/okx/OkxInstrument.h"

src/app_sparkd/sparkd/md/okx/OkxSubscription.cpp

@@ -2,9 +2,9 @@
 #include "domain/protobufs/venue.pb.h"
 #include "domain/security.h"
 #include "domain/security_criteria.h"
-#include "logging/logging.h"
 #include "spark/api/md/OkxMarketDataConfig.pb.h"
 #include "spark/io/ProtoPrinter.h"
+#include "spark/logging/logging.h"
 #include "spark/md/IMarketDataListener.h"
 #include "spark/persistence/IDGenerator.h"
 #include "sparkd/md/ccapi/CCApiSession.h"

src/app_sparkd/sparkd/md/okx/OkxTickerAdaptor.cpp

@@ -1,7 +1,7 @@
 #include "OkxTickerAdaptor.h"
 #include "domain/price.h"
 #include "domain/security_id.h"
-#include "domain/security_market_data_statistics.h"
+#include "spark/domain/SecurityMarketDataStatistics.h"
 #include "spark/md/IMarketDataListener.h"
 #include "sparkd/md/ccapi/CCApiCorrelationID.h"
 #include "sparkd/md/okx/OkxTicker.h"

src/app_sparkd/sparkd/md/platforms/emumba/EmMDListener.cpp

@@ -1,8 +1,8 @@
 #include "EmMDListener.h"
 #include "domain/protobufs/trading_status.pb.h"
-#include "logging/logging.h"
 #include "spark/api/services/ShellDiagnosticService.server.pb.h"
 #include "spark/domain_model/Security.pb.h"
+#include "spark/logging/logging.h"
 #include "spark/md/GeneralSecurityIndex.h"
 #include "spark/md/IntPrice.h"
 #include "spark/md/MDProducer.h"

src/app_sparkd/sparkd/md/platforms/emumba/EmumbaMain.cpp

@@ -1,8 +1,9 @@
 #include "EmumbaMain.h"
 #include "EMTradingSession.h"
 #include "EmMDListener.h"
-#include "logging/logging.h"
 #include "spark/config/config.h"
+#include "spark/config/protos/system_config.pb.h"
+#include "spark/logging/logging.h"
 #include "spark/md/GeneralSecurityIndex.h"
 #include "spark/md/MDProducer.h"
 #include "spark/md/SecurityDB.h"
@@ -16,7 +17,6 @@
 #include "spark/utils/proto_helper.h"
 #include "spark/utils/thread_manager.h"
 #include "sparkd/cmds/crypto/DeribitDemoConfiguration.h"
-#include "system/protobufs/system_config.pb.h"
 
 #include <boost/filesystem.hpp>
 #include <emumba-deribit-connector/deribit_listener.h>
@@ -30,7 +30,7 @@ using namespace bts::orderman;
 
 namespace bts::md {
 
-using bts::system::protobufs::SystemConfig;
+using bts::config::SystemConfig;
 using Path = boost::filesystem::path;
 
 struct EmumbaMain::PImpl {

src/app_sparkd/sparkd/md/platforms/emumba/IEMOrderMuxer.h

@@ -1,11 +1,9 @@
 #pragma once
-
-#include "logging/logging.h"
 #include "spark/domain_model/Security.pb.h"
+#include "spark/logging/logging.h"
 #include "spark/md/common.h"
 #include "spark/orderman/RFQMsg.h"
 #include "spark/os/os.h"
-
 #include <boost/container/small_vector.hpp>
 #include <emumba-deribit-connector/deribit_handler.h>
 #include <emumba-deribit-connector/deribit_listener.h>

src/app_sparkd/sparkd/md/platforms/ice_onixs/ICEMain.cpp

@@ -13,8 +13,8 @@
 #include "spark/utils/proto_helper.h"
 
 #include "spark/config/protos/market_data.pb.h"
+#include "spark/config/protos/system_config.pb.h"
 #include "spark/utils/thread_manager.h"
-#include "system/protobufs/system_config.pb.h"
 
 #include <boost/filesystem.hpp>
 #include <memory>
@@ -43,7 +43,7 @@ namespace bts {
 namespace md {
 namespace ice {
 
-using bts::system::protobufs::SystemConfig;
+using bts::config::SystemConfig;
 using Path = boost::filesystem::path;
 
 ICEMain::ICEMain()

src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp

@@ -1,8 +1,8 @@
 #include "MDP3Main.h"
 
 #include "domain/protobufs/trading_status.pb.h"
-#include "logging/logging.h"
 #include "spark/config/config.h"
+#include "spark/logging/logging.h"
 #include "spark/md/MBOProcessor.h"
 #include "spark/md/SecurityDB.h"
 #include "spark/md/cme/MDP3Utils.h"
@@ -14,6 +14,7 @@
 #include "spark/utils/proto_helper.h"
 
 #include "spark/config/protos/market_data.pb.h"
+#include "spark/config/protos/system_config.pb.h"
 #include "spark/utils/thread_manager.h"
 
 #include <boost/filesystem.hpp>

src/app_sparkd/sparkd/md/recording/MDRecorder.cpp

@@ -1,11 +1,12 @@
 #include "MDRecorder.h"
-#include "logging/logging.h"
 #include "spark/api/md/MarketDataConfig.pb.h"
 #include "spark/api/md/MarketDataConfigUtils.h"
 #include "spark/api/services/MarketDataRelay.client.pb.h"
 #include "spark/api/services/MarketDataRelay.pb.h"
 #include "spark/config/config.h"
+#include "spark/config/protos/system_config.pb.h"
 #include "spark/domain_model/Security.pb.h"
+#include "spark/logging/logging.h"
 #include "spark/md/MDClient.h"
 #include "spark/md/MDConsumer.h"
 #include "spark/md/MDConsumerHandle.h"
@@ -17,7 +18,6 @@
 #include "spark/utils/bts_assert.h"
 #include "spark/utils/datetime.h"
 #include "spark/utils/thread_manager.h"
-#include "system/protobufs/system_config.pb.h"
 #include "system/time_stamp.h"
 
 #include <array>
@@ -30,7 +30,7 @@
 
 namespace bts::md::recording {
 
-using bts::system::protobufs::SystemConfig;
+using bts::config::SystemConfig;
 using Path = boost::filesystem::path;
 
 std::string MDRecorder::file_start_seq = "marketdata";

src/app_sparkd/sparkd/md/recording/MDReplayer.cpp

@@ -5,12 +5,12 @@
 #include "domain/security.h"
 #include "domain/security_common.h"
 #include "domain/security_id.h"
-#include "domain/security_market_data_statistics.h"
 #include "domain/side.h"
 #include "domain/trade_record.h"
-#include "logging/logging.h"
 #include "spark/domain/DomainSecurityUtils.h"
+#include "spark/domain/SecurityMarketDataStatistics.h"
 #include "spark/domain_model/Security.pb.h"
+#include "spark/logging/logging.h"
 #include "spark/md/GeneralMarketDataListener.h"
 #include "spark/md/IMarketDataListener.h"
 #include "spark/md/MDProducer.h"
@@ -22,7 +22,6 @@
 #include "sparkd/md/recording/MDRecorder.h"
 #include "system/time_stamp.h"
 
-#include <cstdint>
 #include <fstream>
 #include <string>
 
@@ -57,14 +56,10 @@ struct MDReplayer ::PImpl {
    std::set<uint32_t> sec_keys = {};
    uint64_t start_time = 0;
    std::shared_ptr<IMarketDataListener> listener{nullptr};
-   double replay_speed;
-
-   PImpl(bts::api::Venue venue, double replay_speed)
-       : venue(venue), replay_speed(replay_speed) {}
+   PImpl(bts::api::Venue venue) : venue(venue) {}
 };
 
-MDReplayer::MDReplayer(bts::api::Venue venue, double replay_speed)
-    : me(new PImpl(venue, replay_speed)) {
+MDReplayer::MDReplayer(bts::api::Venue venue) : me(new PImpl(venue)) {
    LOG(md, Info, "MDReplayer::MDReplayer venue: {}", Venue_Name(venue));
 }
 
@@ -103,7 +98,6 @@ void MDReplayer::_replay_file_once(const std::string &file_name) {
    int i = 0;
    uint64_t beg_ns = bts::core::TimeStamp::now_ns();
    uint64_t last_msg_ts = 0;
-   double replay_speed = me->replay_speed > 0 ? me->replay_speed: 1;
    while (!in.eof() && utils::pb::readbuf(in, &rec_msg)) {
       if (i++ == 0) {
          last_msg_ts = rec_msg.timestamp();
@@ -113,9 +107,8 @@ void MDReplayer::_replay_file_once(const std::string &file_name) {
          uint64_t this_msg_ts = rec_msg.timestamp();
          bts_assert(this_msg_ts >= last_msg_ts);
          uint64_t ts_diff = this_msg_ts - last_msg_ts;
-         uint64_t adjusted_ts = ts_diff/replay_speed;
          last_msg_ts = this_msg_ts;
-         std::this_thread::sleep_for(std::chrono::nanoseconds(adjusted_ts));
+         std::this_thread::sleep_for(std::chrono::nanoseconds(ts_diff));
          _replay_message(rec_msg);
       }
       log_progress(i);
@@ -154,7 +147,7 @@ void MDReplayer::_replay_message(const bts::co2::RecordedMDMessage &m) {
       break;
    }
    case (co2::RecordedMDMessage::MDRecordCase::kMisc): {
-      auto stats = domain::DomainSecurityUtils::convert(m.sec_key(), m.misc());
+      bts::domain::SecurityMarketDataStatistics stats(m.sec_key(), m.misc());
       me->listener->on_security_statistics(stats);
       break;
    }