file restructuring for sdk

WARNING: This breaks the sparkd clang build (only works with gcc)

.devcontainer/Dockerfile

@@ -0,0 +1,122 @@
+FROM ubuntu:22.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+# Note installations that take a while (more than 10s)
+
+RUN apt-get update
+RUN apt-get install -y ca-certificates
+RUN apt-get install -y ccache
+RUN apt-get install -y curl
+RUN apt-get install -y fish
+RUN apt-get install -y gdb
+# git-all: ~250s
+RUN apt-get install -y git-all
+RUN apt-get install -y gnupg
+# liblttng-ust-dev: ~30s
+RUN apt-get install -y liblttng-ust-dev
+RUN apt-get install -y libssh2-1-dev
+RUN apt-get install -y locales
+RUN apt-get install -y ninja-build
+# openssh-server: ~40s
+RUN apt-get install -y openssh-server
+RUN apt-get install -y pkg-config
+RUN apt-get install -y software-properties-common
+RUN apt-get install -y sudo
+RUN apt-get install -y tzdata
+RUN apt-get install -y vim
+RUN apt-get install -y zip
+
+# Python: ~30s
+RUN apt-get install -y \
+   python3-pip \
+   python3-venv \
+   && python3 -m pip install --upgrade pip
+
+# GCC
+ARG gcc_major=13
+RUN \
+   add-apt-repository ppa:ubuntu-toolchain-r/test \
+   && apt-get update \
+   && apt-get install -y g++-$gcc_major
+
+# Clang
+ARG clang_major=17
+RUN \
+   curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key 2> /dev/null \
+   | gpg --dearmor - \
+   | tee /usr/share/keyrings/llvm-snapshot.gpg > /dev/null \
+   && echo "deb [signed-by=/usr/share/keyrings/llvm-snapshot.gpg] http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-$clang_major main" \
+   | tee -a "/etc/apt/sources.list.d/llvm.list" \
+   && apt-get update \
+   && apt-get install -y \
+   clang-$clang_major \
+   clang-format-$clang_major \
+   clangd-$clang_major
+
+# CMake
+RUN \
+   curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
+   | gpg --dearmor - \
+   | tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null \
+   && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
+   | tee /etc/apt/sources.list.d/kitware.list > /dev/null \
+   && apt-get update \
+   && apt-get install -y cmake
+
+# Dotnet
+RUN apt-get update && apt-get install -y dotnet7
+
+# Node
+ARG node_major=18
+RUN mkdir -p /etc/apt/keyrings \
+   && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \
+   | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
+   && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$node_major.x nodistro main" \
+   | tee /etc/apt/sources.list.d/nodesource.list \
+   && apt-get update \
+   && apt-get install -y nodejs
+
+# Emumba (deps)
+RUN apt-get update && apt-get -y install rapidjson-dev
+# Emumba
+COPY emumba /emumba
+RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
+    && rm -r /emumba
+
+# Pcap
+RUN apt-get update && apt-get install -y libpcap0.8-dev
+
+# Set up default executables (update-alternatives)
+RUN \
+   update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \
+   update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-$clang_major 100; \
+   update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-$clang_major 100; \
+   update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-$clang_major 100; \
+   update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-$gcc_major 100; \
+   update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-$gcc_major 100
+
+# Add compiler symlinks for ccache 
+RUN \
+   ln -s /usr/bin/ccache /usr/local/bin/clang; \
+   ln -s /usr/bin/ccache /usr/local/bin/clang++; \
+   ln -s /usr/bin/ccache /usr/local/bin/clang-$clang_major; \
+   ln -s /usr/bin/ccache /usr/local/bin/clang++-$clang_major; \
+   ln -s /usr/bin/ccache /usr/local/bin/gcc; \
+   ln -s /usr/bin/ccache /usr/local/bin/g++
+
+RUN locale-gen en_US.UTF-8
+
+ARG docker_user="bts"
+ARG docker_uid="1000"
+ARG docker_pass="bts"
+
+RUN \
+   groupadd -g "$docker_uid" "$docker_user" \
+   && useradd \
+   -m \
+   -u "$docker_uid" \
+   -g "$docker_uid" \
+   -G sudo \
+   --password "$(openssl passwd -1 "$docker_pass")" \
+   "$docker_user"

.devcontainer/devcontainer.json

@@ -0,0 +1,48 @@
+{
+  "dockerFile": "Dockerfile",
+  // Creates local dir to mount to HOME in docker 
+  "initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
+  "hostRequirements": {
+    "memory": "4gb",
+    "storage": "6gb"
+  },
+  "postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
+  "containerUser": "bts",
+  "workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
+  "workspaceFolder": "/home/bts/spark",
+  "updateRemoteUserUID": true,
+  "shutdownAction": "none",
+  // Note: If running multiple containers on the same host, each must use a
+  // unique set of ports. Usually we just add 1000 to each port as needed. For
+  // example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
+  "runArgs": [
+    "--shm-size",
+    "2gb",
+    "-p",
+    "8000:8000",
+    "-p",
+    "8001:8001",
+    "-p",
+    "8002:8002",
+    "-p",
+    "8003:8003"
+  ],
+  "mounts": [
+    "source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind"
+  ],
+  "customizations": {
+    "vscode": {
+      "settings": {
+        "clangd.arguments": [
+          "--compile-commands-dir=${workspaceFolder}/build/clang/debug"
+        ]
+      },
+      "extensions": [
+        "eamodio.gitlens",
+        "llvm-vs-code-extensions.vscode-clangd",
+        "ms-vscode.cmake-tools",
+        "ms-vscode.cpptools"
+      ]
+    }
+  }
+}

.devcontainer/emumba/install_emumba-deribit-connector.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+this_dir="$(realpath "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")")"
+sudo dpkg -i "$this_dir/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb"

.gitignore

@@ -54,6 +54,6 @@ trace_data/
 /src/Spark.Admin.WebApp/src/bin/
 **/CMakeDeps.cmake
 **/CMakeDeps_win.cmake
-**/.devcontainer
+.devcontainer/
 *.Generated.cs
 *.lst

CMakePresets.json

@@ -113,8 +113,8 @@
       "description": "Debug build compiled w/ gcc",
       "binaryDir": "${sourceDir}/build/gcc/debug",
       "cacheVariables": {
-        "CMAKE_C_COMPILER": "gcc",
-        "CMAKE_CXX_COMPILER": "g++",
+        "CMAKE_C_COMPILER": "gcc-12",
+        "CMAKE_CXX_COMPILER": "g++-12",
         "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -DDEBUG",
         "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/debug/sparkd"
       }
@@ -143,7 +143,7 @@
       "cacheVariables": {
         "CMAKE_C_COMPILER": "gcc",
         "CMAKE_CXX_COMPILER": "g++",
-        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3",
+        "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers -Wno-stringop-overflow --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3",
         "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/release/sparkd"
       }
     },

playground-1.mongodb.js

@@ -1,45 +0,0 @@
-/* global use, db */
-// MongoDB Playground
-// To disable this template go to Settings | MongoDB | Use Default Template For Playground.
-// Make sure you are connected to enable completions and to be able to run a playground.
-// Use Ctrl+Space inside a snippet or a string literal to trigger completions.
-// The result of the last command run in a playground is shown on the results panel.
-// By default the first 20 documents will be returned with a cursor.
-// Use 'console.log()' to print to the debug output.
-// For more documentation on playgrounds please refer to
-// https://www.mongodb.com/docs/mongodb-vscode/playgrounds/
-
-// Select the database to use.
-use('mongodbVSCodePlaygroundDB');
-
-// Insert a few documents into the sales collection.
-db.getCollection('sales').insertMany([
-    { 'item': 'abc', 'price': 10, 'quantity': 2, 'date': new Date('2014-03-01T08:00:00Z') },
-    { 'item': 'jkl', 'price': 20, 'quantity': 1, 'date': new Date('2014-03-01T09:00:00Z') },
-    { 'item': 'xyz', 'price': 5, 'quantity': 10, 'date': new Date('2014-03-15T09:00:00Z') },
-    { 'item': 'xyz', 'price': 5, 'quantity': 20, 'date': new Date('2014-04-04T11:21:39.736Z') },
-    { 'item': 'abc', 'price': 10, 'quantity': 10, 'date': new Date('2014-04-04T21:23:13.331Z') },
-    { 'item': 'def', 'price': 7.5, 'quantity': 5, 'date': new Date('2015-06-04T05:08:13Z') },
-    { 'item': 'def', 'price': 7.5, 'quantity': 10, 'date': new Date('2015-09-10T08:43:00Z') },
-    { 'item': 'abc', 'price': 10, 'quantity': 5, 'date': new Date('2016-02-06T20:20:13Z') },
-]);
-
-// Run a find command to view items sold on April 4th, 2014.
-const salesOnApril4th = db.getCollection('sales').find({
-    date: { $gte: new Date('2014-04-04'), $lt: new Date('2014-04-05') }
-}).count();
-
-// Print a message to the output window.
-console.log(`${salesOnApril4th} sales occurred in 2014.`);
-
-// Here we run an aggregation and open a cursor to the results.
-// Use '.toArray()' to exhaust the cursor to return the whole result set.
-// You can use '.hasNext()/.next()' to iterate through the cursor page by page.
-db.getCollection('sales').aggregate([
-    // Find all of the sales that occurred in 2014.
-    { $match: { date: { $gte: new Date('2014-01-01'), $lt: new Date('2015-01-01') } } },
-    // Group the total sales for each product.
-    { $group: { _id: '$item', totalSaleAmount: { $sum: { $multiply: ['$price', '$quantity'] } } } }
-]);
-
-

src/DataStreamSuite/DataStream/Misc/PandasStream.cs

@@ -0,0 +1,62 @@
+// using System;
+// using System.Diagnostics;
+
+// using BCG;
+// using BCG.Utilities.General;
+
+
+// namespace bts {
+
+//     public class PandasStream : ViewStream, IWorker {
+//         //C:\Users\Owner\AppData\Local\Microsoft\WindowsApps\python.exe
+
+//         public PandasStream(ViewStream vs, ViewStreamDatabase vsd, string pandas_code) {
+
+//         }
+
+//         public PandasStream(ViewStream vs, string pandas_code) :
+//         this(vs, ViewStreamDatabase.instance, pandas_code) {
+//         }
+
+
+//         protected override void myOnFirstSubscriber() {
+
+//             // string python_code = @"
+//             //    import winsound
+//             //    frequency = 2500  # Set frequency (in Hz)
+//             //    duration = 1000   # Set duration (in milliseconds)
+//             //    winsound.Beep(frequency, duration)
+//             // ";
+//             string python_code = @"
+//             file_path = \"C:\\Temp\\test2.txt\\"  # r prefix for raw string to avoid escape characters
+//             with open(file_path, \"w\") as f:
+//                 f.write(\"Hello, world!\n\")
+//             ";
+
+//             execute_python_code(python_code);
+//         }
+
+//         static void execute_python_code(string python_code) {
+//             ProcessStartInfo start = new ProcessStartInfo();
+//             start.FileName = "python"; // Python executable
+//             start.Arguments = "-c \"" + python_code.Replace("\"", "\\\"") + "\""; // Pass code as argument
+//             start.UseShellExecute = false;
+//             start.RedirectStandardOutput = true;
+
+//             using (Process process = Process.Start(start)) {
+//                 using (System.IO.StreamReader reader = process.StandardOutput) {
+//                     string result = reader.ReadToEnd();
+//                     Console.Write(result);
+//                 }
+//             }
+//         }
+
+//         protected override void myOnLastSubscriber() {
+//         }
+
+//         public void doWork() {
+
+//         }
+
+//     }
+// }

src/DataStreamSuite/DataStream/Misc/test.py

@@ -0,0 +1,3 @@
+file_path = r"C:\Temp\test2.txt"  # r prefix for raw string to avoid escape characters
+with open(file_path, "w") as f:
+    f.write("Hello, world!\n")

src/DataStreamSuite/DataStream/Primitives/AddColumns/AddConstantColumnRow.cs

@@ -0,0 +1,12 @@
+namespace bts {
+    using BCG;
+
+    using System;
+    using System.Collections;
+
+    public class AddConstantColumnRow {
+        AddConstantColumnRow(IBlueRow r, Tuple<string, Type, object>[] constant_cols) {
+
+        }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/FileStreams/PersistorStream.cs

@@ -15,6 +15,11 @@ namespace BCG {
    // data:
    // epochtime, size, serialized data
    [DataStreamUserRespresentation("pers")]
+   // Saves each udpate of a stream into a file. The token is the the unique logical name of the stream,
+   // which maps the name of the directory where the streams are saved. The directory (3rd arg) is the 
+   // parent directory where the token folder is stored. Each file goes up to rolloevr_size after which a new
+   // file is created. The file names are token_epoch_datetime
+   // the first entry in a file is the complete state
    public class PersistorStream : ViewStream {
       private ViewStream _source_stream;
       private string _token;

src/DataStreamSuite/DataStream/Primitives/FileStreams/SnapshotStream.cs

@@ -11,13 +11,27 @@ using System.Reflection.Metadata;
 
 namespace BCG {
    public class SnapshotStream : ViewStream {
-
+      public enum SpecificSnapshot {
+         FirstOfDay,
+         LastOfDay
+      };
       private BlueTable _current_table;
       private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
       public DateTime _snapshot_time;
       public string _snapshot_file;
       public IEnumerable<PlaybackPacket> _packets;
-
+      public SnapshotStream(string snapshot_date, string specific_snapshot, string directory_name, string token) {
+         DateTime dt;
+         var parsed = DateTime.TryParse(snapshot_date, out dt);
+         dt = dt.Date;
+         bts.Log.assert(parsed);
+         SpecificSnapshot specific_snapshot_enum;
+         parsed = Enum.TryParse<SpecificSnapshot>(specific_snapshot, out specific_snapshot_enum);
+         bts.Log.assert(parsed);
+         string snapshot_dt = StreamPersistorUtils.find_snapshot_file($"{directory_name}\\{token}", dt, specific_snapshot_enum);
+         ViewStream rs = new SnapshotStream(snapshot_dt, directory_name, token);
+         mySetRealSource(rs);
+      }
 
       public SnapshotStream(string snapshot_dt, string directory_name, string token) {
          _snapshot_time = PersistorStream.parse_datetime(snapshot_dt);

src/DataStreamSuite/DataStream/Primitives/FileStreams/StreamPersistorUtils.cs

@@ -37,6 +37,20 @@ namespace BCG {
       public IEnumerable<PlaybackPacket> packets = new List<PlaybackPacket>();
    }
    public static class StreamPersistorUtils {
+      public static string find_snapshot_file(string directory_name, DateTime snapshot_date, SnapshotStream.SpecificSnapshot specific_snapshot) {
+         DateTime max_dt = DateTime.MinValue;
+         string rv = String.Empty;
+         string[] file_names = Directory.GetFiles(directory_name);
+         for (int i = 0; i < file_names.Count() - 1; i++) {
+            string fn = file_names[i];
+            var dt = parse_datetime_from_filename(fn);
+            if (dt > max_dt) {
+               max_dt = dt;
+               rv = fn;
+            }
+         }
+         return rv;
+      }
       public static string find_snapshot_file(string directory_name, DateTime snapshot_time) {
          string[] file_names = Directory.GetFiles(directory_name);
          for (int i = 0; i < file_names.Count() - 1; i++) {

src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxGroupByEvaluator.cs

@@ -1,47 +1,42 @@
-// using System;
-// using System.Collections;
-// using BCG.Utilities;
+using System;
+using System.Collections;
 
-// namespace BCG
-// {
-// 	/// <summary>
-// 	/// Computes the max for a given column
-// 	/// </summary>
-// 	public class MaxGroupByEvaluator : GroupByEvaluator
-// 	{
+using BCG.Utilities;
 
-//       public MaxGroupByEvaluator (string outputCol, string[] groupByCols) : base (outputCol, groupByCols)
-//       {}
+namespace BCG {
+    /// <summary>
+    /// Computes the max for a given column
+    /// </summary>
+    public class MaxGroupByEvaluator : GroupByEvaluator {
 
-//       public MaxGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols)
-//          : base (colMapEpression, outputCol, groupByCols)   {}
+        public MaxGroupByEvaluator(string outputCol, string[] groupByCols) : base(outputCol, groupByCols) { }
 
-//       public override IBlueRow evaluate (DataPacket dp)  
-//       {
-//          myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) :
-//             myOutputTable;
-//          string hashString = BlueDataUtility.rowToString (GroupByStream.getRepRow (dp), myGroupByCols);
-//          IBlueRow currMaxRow = myOutputTable.getRow (hashString);
-//          IBlueRow maxRow = BlueDataUtility.computeMaxRow (dp.deletedRows, myOutputColumn);
-//          if (maxRow != null)
-//          {
-//             BlueEventLog.logAssert (currMaxRow != null);
-//             BlueEventLog.logAssert (BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) <= 0);
-//             if (BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) == 0)
-//             {
-//                maxRow = BlueDataUtility.computeMaxRow (dp.newRows, myOutputColumn);
-//                if (maxRow == null)
-//                   myOutputTable.removeRow (hashString);
-//                else
-//                   myOutputTable.addRow (maxRow, hashString);
-//             }
-//          }
-//          maxRow = BlueDataUtility.computeMaxRow (dp.addedRows, myOutputColumn);
-//          // if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true,
-//          // otherwise if maxRow > currMaxRow then its replaced
-//          if (maxRow != null && (currMaxRow == null || BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) > 0))
-//             myOutputTable.addRow (maxRow, hashString);
-//          return myProject (myOutputTable.getRow (hashString));
-//       }
-// 	}
-// }
+        public MaxGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
+           : base(colMapEpression, outputCol, groupByCols) { }
+
+        public override IBlueRow evaluate(DataPacket dp) {
+            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) :
+               myOutputTable;
+            string hashString = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
+            IBlueRow currMaxRow = myOutputTable.getRow(hashString);
+            IBlueRow maxRow = BlueDataUtility.computeMaxRow(dp.deletedRows, myOutputColumn);
+            if (maxRow != null) {
+                bts.Log.assert(currMaxRow != null);
+                bts.Log.assert(BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) <= 0);
+                if (BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) == 0) {
+                    maxRow = BlueDataUtility.computeMaxRow(dp.newRows, myOutputColumn);
+                    if (maxRow == null)
+                        myOutputTable.removeRow(hashString);
+                    else
+                        myOutputTable.addRow(maxRow, hashString);
+                }
+            }
+            maxRow = BlueDataUtility.computeMaxRow(dp.addedRows, myOutputColumn);
+            // if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true,
+            // otherwise if maxRow > currMaxRow then its replaced
+            if (maxRow != null && (currMaxRow == null || BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) > 0))
+                myOutputTable.addRow(maxRow, hashString);
+            return myProject(myOutputTable.getRow(hashString));
+        }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxStream.cs

@@ -1,20 +1,15 @@
-// using System;
+using System;
 
-// namespace BCG
-// {
-// 	public class MaxStream : ViewStream
-// 	{
-//       public MaxStream(ViewStream stream, string outputCol, string inputCols) : this(stream, outputCol, inputCols.Split(' '))
-//       {
-//       }
-// 		public MaxStream (ViewStream stream, string outputCol, string[] inputCols)
-// 		{
-//          mySetRealSource (new GroupByStream (stream, 
-//                                              new MaxGroupByEvaluator (outputCol, inputCols)));
-// 		}
+namespace BCG {
+    public class MaxStream : ViewStream {
+        public MaxStream(ViewStream stream, string outputCol, string inputCols) : this(stream, outputCol, inputCols.Split(' ')) {
+        }
+        public MaxStream(ViewStream stream, string outputCol, string[] inputCols) {
+            mySetRealSource(new GroupByStream(stream,
+                                                new MaxGroupByEvaluator(outputCol, inputCols)));
+        }
 
-//       public MaxStream (ViewStream stream, string outputCol)
-//          : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, new string[]{"fakeCol"})
-//       {}
-// 	}
-// }
+        public MaxStream(ViewStream stream, string outputCol)
+           : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/MinGroupByEvaluator.cs

@@ -1,40 +1,36 @@
-// using System;
-// using BCG.Utilities;
+using System;
 
-// namespace BCG
-// {
-//    public class MinGroupByEvaluator : MaxGroupByEvaluator
-//    {
-//       public MinGroupByEvaluator (string outputCol, string[] groupByCols) 
-//          : base (outputCol, groupByCols)  {}
+using BCG.Utilities;
 
-//       public MinGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols) :
-//          base (colMapEpression, outputCol, groupByCols)  {}
+namespace BCG {
+    public class MinGroupByEvaluator : MaxGroupByEvaluator {
+        public MinGroupByEvaluator(string outputCol, string[] groupByCols)
+           : base(outputCol, groupByCols) { }
 
-//       public override IBlueRow evaluate (DataPacket dp)  
-//       {
-//          myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) : myOutputTable;
-//          string stringRep = BlueDataUtility.rowToString (GroupByStream.getRepRow (dp), myGroupByCols);
-//          IBlueRow currMinRow = myOutputTable.getRow (stringRep);
-//          IBlueRow minRow = BlueDataUtility.computeMinRow (dp.deletedRows, myOutputColumn);
-//          if (minRow != null)
-//          {
-//             // BlueEventLog.logAssert (currMinRow != null);
-//             // BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0);
-//             if (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) == 0)
-//             {
-//                minRow = BlueDataUtility.computeMinRow (dp.newRows, myOutputColumn);
-//                if (minRow == null)
-//                   myOutputTable.removeRow (stringRep);
-//                else
-//                   myOutputTable.addRow (minRow, stringRep);
-//             }
-//          }
-//          minRow = BlueDataUtility.computeMinRow (dp.addedRows, myOutputColumn);
-//          if (minRow != null && (currMinRow == null || BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) < 0))
-//             myOutputTable.addRow (minRow, stringRep);
-//          return myOutputTable.getRow (stringRep);
-//       }
+        public MinGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols) :
+           base(colMapEpression, outputCol, groupByCols) { }
 
-//    }
-// }
+        public override IBlueRow evaluate(DataPacket dp) {
+            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
+            string stringRep = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
+            IBlueRow currMinRow = myOutputTable.getRow(stringRep);
+            IBlueRow minRow = BlueDataUtility.computeMinRow(dp.deletedRows, myOutputColumn);
+            if (minRow != null) {
+                // BlueEventLog.logAssert (currMinRow != null);
+                // BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0);
+                if (BlueDataUtility.compareRows(minRow, currMinRow, myOutputColumn) == 0) {
+                    minRow = BlueDataUtility.computeMinRow(dp.newRows, myOutputColumn);
+                    if (minRow == null)
+                        myOutputTable.removeRow(stringRep);
+                    else
+                        myOutputTable.addRow(minRow, stringRep);
+                }
+            }
+            minRow = BlueDataUtility.computeMinRow(dp.addedRows, myOutputColumn);
+            if (minRow != null && (currMinRow == null || BlueDataUtility.compareRows(minRow, currMinRow, myOutputColumn) < 0))
+                myOutputTable.addRow(minRow, stringRep);
+            return myOutputTable.getRow(stringRep);
+        }
+
+    }
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/MinStream.cs

@@ -1,17 +1,13 @@
-// using System;
+using System;
 
-// namespace BCG
-// {
-//    public class MinStream : ViewStream
-//    {
-//       public MinStream (ViewStream stream, string outputCol, string[] inputCol)
-//       {
-//          mySetRealSource (new GroupByStream (stream, 
-//                            new MinGroupByEvaluator (outputCol, inputCol)));
-//       }
+namespace BCG {
+    public class MinStream : ViewStream {
+        public MinStream(ViewStream stream, string outputCol, string[] inputCol) {
+            mySetRealSource(new GroupByStream(stream,
+                              new MinGroupByEvaluator(outputCol, inputCol)));
+        }
 
-//       public MinStream (ViewStream stream, string outputCol)
-//                      : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, new string[]{"fakeCol"})
-//       {}
-//    }
-// }
+        public MinStream(ViewStream stream, string outputCol)
+                       : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/SumGroupByEvaluator.cs

@@ -1,94 +1,82 @@
-// using System;
+using System;
 
-// namespace BCG
-// {
-//    /// <summary>
-//    /// Computes the sum for a given column
-//    /// </summary>
-//    public class SumGroupByEvaluator : GroupByEvaluator
-//    {
-//       public SumGroupByEvaluator (string outputCol, string[] groupByCols) : base (outputCol, groupByCols)
-//       {}
+namespace BCG {
+    /// <summary>
+    /// Computes the sum for a given column
+    /// </summary>
+    public class SumGroupByEvaluator : GroupByEvaluator {
+        public SumGroupByEvaluator(string outputCol, string[] groupByCols) : base(outputCol, groupByCols) { }
 
-//       public SumGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols)
-//          : base (colMapEpression, outputCol, groupByCols)   {}
+        public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
+           : base(colMapEpression, outputCol, groupByCols) { }
 
-//       public override IBlueRow evaluate (DataPacket dp)  
-//       {
-//          myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) : myOutputTable;
-         
-//          IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn);
-//          return sumRow;
-//       }
-//    }
+        public override IBlueRow evaluate(DataPacket dp) {
+            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
+
+            IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn);
+            return sumRow;
+        }
+    }
 
 
-//    public class MultiColumnSumGroupByEvaluator : IGroupByEvaluator
-//    {
-//       private string[] myOutputCols, myGroupByCols;
-//       public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols)
-//       {
-//          myOutputCols = spaceSeperatedOutputCols.Split(' ');
-//          myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
-//       }
+    public class MultiColumnSumGroupByEvaluator : IGroupByEvaluator {
+        private string[] myOutputCols, myGroupByCols;
+        public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols) {
+            myOutputCols = spaceSeperatedOutputCols.Split(' ');
+            myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
+        }
 
-//       public MultiColumnSumGroupByEvaluator(string[] output_columns, string[] groupby_columns)
-//       {
-//          myOutputCols = output_columns;
-//          myGroupByCols = groupby_columns;
-//       }
+        public MultiColumnSumGroupByEvaluator(string[] output_columns, string[] groupby_columns) {
+            myOutputCols = output_columns;
+            myGroupByCols = groupby_columns;
+        }
 
-//       #region IGroupByEvaluator Members
+        #region IGroupByEvaluator Members
 
-//       public IBlueRow evaluate(DataPacket packet)  {return BlueDataUtility.computeSum(packet.newRows, myOutputCols);}
+        public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum(packet.newRows, myOutputCols); }
 
-//       public string[] groupByColumns { get { return myGroupByCols; } }
+        public string[] groupByColumns { get { return myGroupByCols; } }
 
-//       public void clear()  {/*stateless*/}
+        public void clear() {/*stateless*/}
 
-//       public IBlueTableSchema getOutputSchema(IBlueTableSchema schema)  {return schema;}
+        public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return schema; }
 
-//       #endregion
-//    }
+        #endregion
+    }
 
-//    public class SumGroupByEvaluator<T> : GroupByEvaluator
-//    {
-//       public SumGroupByEvaluator(string outputCol, string[] groupByCols)
-//          : base(outputCol, groupByCols)
-//       { }
+    public class SumGroupByEvaluator<T> : GroupByEvaluator {
+        public SumGroupByEvaluator(string outputCol, string[] groupByCols)
+           : base(outputCol, groupByCols) { }
 
-//       public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
-//          : base(colMapEpression, outputCol, groupByCols) { }
+        public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
+           : base(colMapEpression, outputCol, groupByCols) { }
 
-//       public override IBlueRow evaluate(DataPacket dp)
-//       {
-//          myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
+        public override IBlueRow evaluate(DataPacket dp) {
+            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
 
-//          IBlueRow sumRow = BlueDataUtility.computeSum<T>(dp.newRows, myOutputColumn);
-//          return sumRow;
-//       }
-//    }
+            IBlueRow sumRow = BlueDataUtility.computeSum<T>(dp.newRows, myOutputColumn);
+            return sumRow;
+        }
+    }
 
-//    public class MultiColumnSumGroupByEvaluator<T> : IGroupByEvaluator
-//    {
-//       private string[] myOutputCols, myGroupByCols;
-//       public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols)
-//       {
-//          myOutputCols = spaceSeperatedOutputCols.Split(' ');
-//          myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
-//       }
+    public class MultiColumnSumGroupByEvaluator<T> : IGroupByEvaluator {
+        private string[] myOutputCols, myGroupByCols;
+        public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols) {
+            myOutputCols = spaceSeperatedOutputCols.Split(' ');
+            myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
+        }
 
-//       #region IGroupByEvaluator Members
+        #region IGroupByEvaluator Members
 
-//       public IBlueRow evaluate(DataPacket packet)  {return BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols);}
+        public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols); }
 
-//       public string[] groupByColumns { get { return myGroupByCols; } }
+        public string[] groupByColumns { get { return myGroupByCols; } }
 
-//       public void clear()  {/*stateless*/}
+        public void clear() {/*stateless*/}
 
-//       public IBlueTableSchema getOutputSchema(IBlueTableSchema schema)  {return schema;}
+        public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return schema; }
 
-//       #endregion
-//    }
+        #endregion
+    }
 
-// }
+}

src/DataStreamSuite/DataStream/Primitives/GroupBy/SumStream.cs

@@ -1,107 +1,89 @@
-// using System;
-// using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 
-// namespace BCG
-// {
-//    public class NullMapper : IStreamMapper
-//    {
-//       private Dictionary<string, bool> _valid_cols = new Dictionary<string, bool>();
+namespace BCG {
+    public class NullMapper : IStreamMapper {
+        private Dictionary<string, bool> _valid_cols = new Dictionary<string, bool>();
 
-//       public NullMapper(string[] valid_cols)
-//       {
-//          foreach (string c in valid_cols)
-//             _valid_cols[c] = true;
-//          _valid_cols["id"] = true;
-//       }
+        public NullMapper(string[] valid_cols) {
+            foreach (string c in valid_cols)
+                _valid_cols[c] = true;
+            _valid_cols["id"] = true;
+        }
 
-//       public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) { return input_schema; }
+        public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) { return input_schema; }
 
-//       public IBlueRow map(IBlueRow row)
-//       {
-//          MutableBlueRow out_row = new MutableBlueRow(row);
-//          foreach (string c in row.schema.columnNames)
-//             if (!_valid_cols.ContainsKey(c))
-//                out_row.setValue(c, DBNull.Value);
-//          return out_row;
-//       }
-//    }
+        public IBlueRow map(IBlueRow row) {
+            MutableBlueRow out_row = new MutableBlueRow(row);
+            foreach (string c in row.schema.columnNames)
+                if (!_valid_cols.ContainsKey(c))
+                    out_row.setValue(c, DBNull.Value);
+            return out_row;
+        }
+    }
 
-//    public class SumStream : ViewStream
-//    {
-//       public SumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols)
-//       {
-//          string[] outputCols = spaceSeparatedOutputCols.Split(' ');
-//          string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
-//          mySetRealSource(new GroupByStream(stream,
-//                                            outputCols.Length == 1 ? new SumGroupByEvaluator(outputCols[0], groupByCols) as IGroupByEvaluator :
-//                                            new MultiColumnSumGroupByEvaluator(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
-//       }
+    public class SumStream : ViewStream {
+        public SumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols) {
+            string[] outputCols = spaceSeparatedOutputCols.Split(' ');
+            string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
+            mySetRealSource(new GroupByStream(stream,
+                                              outputCols.Length == 1 ? new SumGroupByEvaluator(outputCols[0], groupByCols) as IGroupByEvaluator :
+                                              new MultiColumnSumGroupByEvaluator(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
+        }
 
-//       public SumStream(ViewStream stream, string[] output_columns)
-//       {
-//          ViewStream rs = new AddColumnsStream(stream, "fake_col=0");
-//          rs = new SumStream(rs, output_columns, new string[] { "fake_col" }, false);
-//          rs = new ProjectStream(rs, "!fake_col");
-//          mySetRealSource(rs);
-//       }
+        public SumStream(ViewStream stream, string[] output_columns) {
+            ViewStream rs = new AddColumnsStream(stream, "fake_col=0");
+            rs = new SumStream(rs, output_columns, new string[] { "fake_col" }, false);
+            rs = new ProjectStream(rs, "!fake_col");
+            mySetRealSource(rs);
+        }
 
-//       public SumStream(ViewStream stream, string space_seperated_output_columns, string space_seperated_groupby_columns, bool force_null_into_other_columns)
-//       {
-//          string[] output_columns = space_seperated_output_columns.Split(' ');
-//          string[] groupby_columns = String.IsNullOrEmpty (space_seperated_groupby_columns) ? new string[]{} : space_seperated_groupby_columns.Split (' ');
-//          mySetRealSource (new SumStream (stream, output_columns, groupby_columns, force_null_into_other_columns));
-//       }
+        public SumStream(ViewStream stream, string space_seperated_output_columns, string space_seperated_groupby_columns, bool force_null_into_other_columns) {
+            string[] output_columns = space_seperated_output_columns.Split(' ');
+            string[] groupby_columns = String.IsNullOrEmpty(space_seperated_groupby_columns) ? new string[] { } : space_seperated_groupby_columns.Split(' ');
+            mySetRealSource(new SumStream(stream, output_columns, groupby_columns, force_null_into_other_columns));
+        }
 
-//       public SumStream(ViewStream stream, string[] output_columns, string[] groupby_columns, bool force_null_into_other_columns)
-//       {
-//          ViewStream rs = null;
-//          groupby_columns = groupby_columns ?? new string[] {};
-//          if (groupby_columns.Length == 0)
-//          {
-//             rs = new AddColumnsStream(stream, "fake_col=0");
-//             rs = new GroupByStream(rs, new MultiColumnSumGroupByEvaluator(output_columns, new string[] {"fake_col"}));
-//             rs = new ProjectStream(rs, "!fake_col");
-//          }
-//          else
-//          {
-//             rs = new GroupByStream(stream, new MultiColumnSumGroupByEvaluator(output_columns, groupby_columns));
-//          }
-//          if (force_null_into_other_columns)
-//          {
-//             List<string> valid_cols = new List<string>(output_columns);
-//             valid_cols.AddRange(groupby_columns);
-//             rs = new MapStream(rs, new NullMapper(valid_cols.ToArray()));
-//          }
-//          mySetRealSource(rs);
-//       }
+        public SumStream(ViewStream stream, string[] output_columns, string[] groupby_columns, bool force_null_into_other_columns) {
+            ViewStream rs = null;
+            groupby_columns = groupby_columns ?? new string[] { };
+            if (groupby_columns.Length == 0) {
+                // rs = new AddColumnsStream(stream, "fake_col=0");
+                // rs = new GroupByStream(rs, new MultiColumnSumGroupByEvaluator(output_columns, new string[] { "fake_col" }));
+                // rs = new ProjectStream(rs, "!fake_col");
+            } else {
+                rs = new GroupByStream(stream, new MultiColumnSumGroupByEvaluator(output_columns, groupby_columns));
+            }
+            if (force_null_into_other_columns) {
+                List<string> valid_cols = new List<string>(output_columns);
+                valid_cols.AddRange(groupby_columns);
+                rs = new MapStream(rs, new NullMapper(valid_cols.ToArray()));
+            }
+            mySetRealSource(rs);
+        }
 
-//    public SumStream (ViewStream stream, string outputCol)
-//    : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, "fakeCol")
-//    {
-//    }
-// }
+        public SumStream(ViewStream stream, string outputCol)
+        : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, "fakeCol") {
+        }
+    }
 
-// public class IntSumStream : ViewStream
-// {
-//    public IntSumStream (ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols)
-//    {
-//       string[] outputCols = spaceSeparatedOutputCols.Split(' ');
-//       string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
-//       mySetRealSource(new GroupByStream(stream,
-//                                         outputCols.Length == 1 ? new SumGroupByEvaluator<int>(outputCols[0], groupByCols) as IGroupByEvaluator:
-//                                         new MultiColumnSumGroupByEvaluator<int>(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
-//    }
+    public class IntSumStream : ViewStream {
+        public IntSumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols) {
+            string[] outputCols = spaceSeparatedOutputCols.Split(' ');
+            string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
+            mySetRealSource(new GroupByStream(stream,
+                                              outputCols.Length == 1 ? new SumGroupByEvaluator<int>(outputCols[0], groupByCols) as IGroupByEvaluator :
+                                              new MultiColumnSumGroupByEvaluator<int>(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
+        }
 
-// }
+    }
 
 
-//    public class SumStream<T> : ViewStream
-//    {
-//       public SumStream(ViewStream stream, string outputCol, string spaceSeparatedInputCols)
-//       {
-//          string[] inputCols = spaceSeparatedInputCols.Split(' ');
-//          mySetRealSource(new GroupByStream(stream,
-//                                              new SumGroupByEvaluator<T>(outputCol, inputCols)));
-//       }
-//    }
-// }
+    public class SumStream<T> : ViewStream {
+        public SumStream(ViewStream stream, string outputCol, string spaceSeparatedInputCols) {
+            string[] inputCols = spaceSeparatedInputCols.Split(' ');
+            mySetRealSource(new GroupByStream(stream,
+                                                new SumGroupByEvaluator<T>(outputCol, inputCols)));
+        }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/InterRow/RowDiffStream.cs

@@ -0,0 +1,79 @@
+using BCG;
+
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+
+using ZstdSharp.Unsafe;
+
+namespace bts {
+    public class RowDiffStream : ViewStream {
+        private ViewStream _vs;
+        private readonly string _order_col, _col_1, _col_2;
+        private readonly int _diff;
+        private IBlueTableSchema _out_schema;
+        private readonly string _diff_col_name;
+        public RowDiffStream(ViewStream vs, string order_col, string col_1, string col_2, int diff, string diff_col_name) {
+            _vs = vs;
+            _order_col = order_col;
+            _col_1 = col_1;
+            _col_2 = col_2;
+            _diff = diff;
+            _diff_col_name = diff_col_name;
+        }
+
+        private string _diff_col_name_perc {
+            get {
+                return $"{_diff_col_name}_percentage";
+            }
+        }
+
+        protected override void myOnFirstSubscriber() {
+            _vs.onChange += _handle_vs;
+        }
+
+        protected override void myOnLastSubscriber() {
+            _vs.onChange -= _handle_vs;
+        }
+
+        private void _handle_vs(object dp_obj) {
+            var dp = dp_obj as DataPacket;
+            IBlueRowCollection nrows = dp.newRows;
+            _out_schema = BlueDataUtility.add_column(nrows.schema, _diff_col_name, typeof(double));
+            _out_schema = BlueDataUtility.add_column(_out_schema, _diff_col_name_perc, typeof(double));
+            List<IBlueRow> row_list = new List<IBlueRow>();
+            foreach (IBlueRow r in nrows)
+                row_list.Add(r);
+            //OrderBy sorts in ascedning order
+            var sorted_rows = row_list.OrderBy(r => r[_order_col]).ToList();
+            var sorted_row_array = sorted_rows.ToArray();
+            // reverse it to make it descending    
+            sorted_rows.Reverse();
+            //sort rows so we can do the diff
+            OrderedBlueTable rv = new OrderedBlueTable(_out_schema);
+            int i = 0;
+            foreach (IBlueRow r in sorted_rows) {
+                MutableBlueRow nr = new MutableBlueRow(_out_schema, r);
+                //So, in the expression "x - y," where y is subtracted from x, x is the minuend, and y is the subtrahend.
+                // we assume x - y, and name variables like that
+                double x = r.get<double>(_col_1);
+                int y_row_idx = i + _diff;
+                double diff = System.Double.NaN;
+                double perc = System.Double.NaN;
+                if (y_row_idx < sorted_rows.Count()) {
+                    var y = sorted_rows[y_row_idx].get<double>(_col_2);
+                    diff = x - y;
+                    perc = 100.0 * diff / y;
+                }
+                nr.setValue(_diff_col_name, diff);
+                nr.setValue(_diff_col_name_perc, perc);
+                rv.addRow(nr);
+                i++;
+            }
+
+            var dp_out = BlueDataUtility.build_dp_inc(new_rows: rv, myLastValue as DataPacket, true);
+            myFireEvent(dp_out);
+
+        }
+    }
+}

src/DataStreamSuite/DataStream/Primitives/SideEffectStreams/SchemaGuesserStream.cs

@@ -0,0 +1,34 @@
+using BCG;
+
+namespace bts {
+    public class SchemaGuesserStream : ViewStream {
+
+        private IBlueRowCollection _rows = null;
+        private ViewStream _vs;
+        public SchemaGuesserStream(ViewStream vs) {
+            _vs = vs;
+        }
+
+        protected override void myOnFirstSubscriber() {
+            _vs.onChange += _on_data;
+        }
+        protected override void myOnLastSubscriber() {
+            _vs.onChange -= _on_data;
+        }
+
+        private void _on_data(object obj) {
+            DataPacket dp = obj as DataPacket;
+            if (_rows == null) {
+                _rows = BlueDataUtility.get_typed_table(dp.newRows);
+            }
+            var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket);
+            myFireEvent(dp_out);
+
+
+        }
+
+
+
+
+    }
+}

src/DataStreamSuite/DataStream/Primitives/Sort/SortStream.cs

@@ -1,99 +1,99 @@
-using Bts.ColumnSortExpressions;
-
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-
-namespace BCG;
-
-/// <summary>
-/// Defines a Sort stream, with performance of O(n) for all the increments. It uses simple merge sort
-/// to merge the additions to the accumulates table
-/// </summary>
-public class SortStream : ViewStream {
-   ViewStream stream;
-   IList<ColumnSortExpression> sortCols;
-   IComparer? comparer;
-   DataPacket? lastFiredPacket;
-   DataPacket? lastReceivedPacket;
-
-   public SortStream(ViewStream stream, string sortExpression) {
-      this.stream = stream;
-      SortExpression = sortExpression;
-   }
-
-   public SortStream(ViewStream vs, string colName, bool isAsc) {
-      stream = vs;
-      SortExpression = isAsc ? "+" + colName : "-" + colName;
-   }
-
-   public SortStream(ViewStream stream, IComparer comparer) {
-      this.stream = stream;
-      this.comparer = comparer;
-      sortCols = Array.Empty<ColumnSortExpression>();
-   }
-
-   [MemberNotNull(nameof(sortCols))]
-   public string SortExpression {
-      set {
-         comparer = null;
-         lastFiredPacket = null;
-         sortCols = ColumnSortExpressionInterpreter.Evaluate(value);
-         if (lastReceivedPacket is not null) {
-            myChangeHandler(lastReceivedPacket);
-         }
-      }
-   }
-
-   public IComparer Comparer {
-      set {
-         comparer = value;
-         lastFiredPacket = null;
-         if (lastReceivedPacket is not null) {
-            myChangeHandler(lastReceivedPacket);
-         }
-      }
-   }
-
-   public void myChangeHandlerTest(object o) =>
-      myChangeHandler(o);
-
-   [MemberNotNull(nameof(lastFiredPacket))]
-   internal void myChangeHandler(object o) {
-      lastReceivedPacket = (DataPacket)o;
-      // First callback, so we need to sort the whole table, this is the
-      // only time we will do that for all the other sorting operations would
-      // be sorting the increments and inserting them into the whole list
-      if (lastFiredPacket is null) {
-         lastFiredPacket = new() {
-            newRows = comparer is null ?
-               new SortedTable(lastReceivedPacket.newRows, sortCols) :
-               new SortedTable(lastReceivedPacket.newRows, comparer)
-         };
-         lastFiredPacket.addedRows = new OrderedBlueTable(lastFiredPacket.newRows);
-         lastFiredPacket.deletedRows = new BlueTable(lastFiredPacket.newRows.schema);
-         myFireEvent(lastFiredPacket.Clone());
-         return;
-      }
-      // Deletes
-      lastFiredPacket.deletedRows =
-         comparer == null ?
-         new SortedTable(lastReceivedPacket.deletedRows, sortCols) :
-         new SortedTable(lastReceivedPacket.deletedRows, comparer);
-      ((SortedTable)lastFiredPacket.newRows).deleteRows(lastFiredPacket.deletedRows);
-      // Adds
-      lastFiredPacket.addedRows = new BlueTable(lastReceivedPacket.addedRows);
-      ((SortedTable)lastFiredPacket.newRows).addRows(lastFiredPacket.addedRows);
-      myFireEvent(lastFiredPacket.Clone());
-   }
-
-   protected override void myOnFirstSubscriber() =>
-      stream.onChange += myChangeHandler;
-
-   protected override void myOnLastSubscriber() {
-      lastFiredPacket = null;
-      lastReceivedPacket = null;
-      stream.onChange -= myChangeHandler;
-   }
-}
+using Bts.ColumnSortExpressions;
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+
+namespace BCG;
+
+/// <summary>
+/// Defines a Sort stream, with performance of O(n) for all the increments. It uses simple merge sort
+/// to merge the additions to the accumulates table
+/// </summary>
+public class SortStream : ViewStream {
+   ViewStream stream;
+   IList<ColumnSortExpression> sortCols;
+   IComparer? comparer;
+   DataPacket? lastFiredPacket;
+   DataPacket? lastReceivedPacket;
+
+   public SortStream(ViewStream stream, string sortExpression) {
+      this.stream = stream;
+      SortExpression = sortExpression;
+   }
+
+   public SortStream(ViewStream vs, string colName, bool isAsc) {
+      stream = vs;
+      SortExpression = isAsc ? "+" + colName : "-" + colName;
+   }
+
+   public SortStream(ViewStream stream, IComparer comparer) {
+      this.stream = stream;
+      this.comparer = comparer;
+      sortCols = Array.Empty<ColumnSortExpression>();
+   }
+
+   [MemberNotNull(nameof(sortCols))]
+   public string SortExpression {
+      set {
+         comparer = null;
+         lastFiredPacket = null;
+         sortCols = ColumnSortExpressionInterpreter.Evaluate(value);
+         if (lastReceivedPacket is not null) {
+            myChangeHandler(lastReceivedPacket);
+         }
+      }
+   }
+
+   public IComparer Comparer {
+      set {
+         comparer = value;
+         lastFiredPacket = null;
+         if (lastReceivedPacket is not null) {
+            myChangeHandler(lastReceivedPacket);
+         }
+      }
+   }
+
+   public void myChangeHandlerTest(object o) =>
+      myChangeHandler(o);
+
+   [MemberNotNull(nameof(lastFiredPacket))]
+   internal void myChangeHandler(object o) {
+      lastReceivedPacket = (DataPacket)o;
+      // First callback, so we need to sort the whole table, this is the
+      // only time we will do that for all the other sorting operations would
+      // be sorting the increments and inserting them into the whole list
+      if (lastFiredPacket is null) {
+         lastFiredPacket = new() {
+            newRows = comparer is null ?
+               new SortedTable(lastReceivedPacket.newRows, sortCols) :
+               new SortedTable(lastReceivedPacket.newRows, comparer)
+         };
+         lastFiredPacket.addedRows = new OrderedBlueTable(lastFiredPacket.newRows);
+         lastFiredPacket.deletedRows = new BlueTable(lastFiredPacket.newRows.schema);
+         myFireEvent(lastFiredPacket.Clone());
+         return;
+      }
+      // Deletes
+      lastFiredPacket.deletedRows =
+         comparer == null ?
+         new SortedTable(lastReceivedPacket.deletedRows, sortCols) :
+         new SortedTable(lastReceivedPacket.deletedRows, comparer);
+      ((SortedTable)lastFiredPacket.newRows).deleteRows(lastFiredPacket.deletedRows);
+      // Adds
+      lastFiredPacket.addedRows = new BlueTable(lastReceivedPacket.addedRows);
+      ((SortedTable)lastFiredPacket.newRows).addRows(lastFiredPacket.addedRows);
+      myFireEvent(lastFiredPacket.Clone());
+   }
+
+   protected override void myOnFirstSubscriber() =>
+      stream.onChange += myChangeHandler;
+
+   protected override void myOnLastSubscriber() {
+      lastFiredPacket = null;
+      lastReceivedPacket = null;
+      stream.onChange -= myChangeHandler;
+   }
+}

src/DataStreamSuite/DataStream/Primitives/Top/TopStream.cs

@@ -1,49 +1,43 @@
-// using System;
+using System;
 
-// namespace BCG
-// {
-//    public class TopStream : MethodDrivenStream
-//    {
-//       private int myTopCount;
+namespace BCG {
+    public class TopStream : MethodDrivenStream {
+        private int myTopCount;
 
-//       public TopStream (ViewStream stream, int topCount) : base (stream)  {myTopCount = topCount;}
+        public TopStream(ViewStream stream, int topCount) : base(stream) { myTopCount = topCount; }
 
-//       public TopStream (ViewStream stream, 
-//          int topCount, 
-//          string sortExpression) 
-//          :
-//          this (sortExpression == null? 
-//          stream :
-//          new SortStream (stream, sortExpression), topCount)
-//       {}
+        public TopStream(ViewStream stream,
+           int topCount,
+           string sortExpression)
+           :
+           this(sortExpression == null ?
+           stream :
+           new SortStream(stream, sortExpression), topCount) { }
 
-//       protected override void myOnTableClear () 
-//       {
-//          myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
-//          myLastFiredPacket.newRows = new OrderedBlueTable (myLastFiredPacket.deletedRows.schema);
-//          myLastFiredPacket.addedRows = new OrderedBlueTable (myLastFiredPacket.deletedRows.schema);
-//       }
+        protected override void myOnTableClear() {
+            myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
+            myLastFiredPacket.newRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
+            myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
+        }
 
-//       protected override void myHandleFirstCall () 
-//       {
-//          myLastFiredPacket = new DataPacket ();
-//          myLastFiredPacket.newRows = new OrderedBlueTable (myLastArrivedPacket.newRows, myTopCount);
-//          myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
-//          myLastFiredPacket.deletedRows = new OrderedBlueTable (myLastFiredPacket.newRows.schema);
-//       }
+        protected override void myHandleFirstCall() {
+            myLastFiredPacket = new DataPacket();
+            myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
+            myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
+            myLastFiredPacket.deletedRows = new OrderedBlueTable(myLastFiredPacket.newRows.schema);
+        }
 
-//       protected override void myHandleIncrementalAdd ()     {}
+        protected override void myHandleIncrementalAdd() { }
 
-//       protected override void myHandleIncrementalDelete()   
-//       {
-//          IBlueRowCollection oldRows = myLastFiredPacket.newRows;
-//          myLastFiredPacket.newRows = new OrderedBlueTable (myLastArrivedPacket.newRows, myTopCount);
-//          BlueDataUtility.computeDifferences (myLastFiredPacket.newRows,
-//             oldRows,
-//             myLastFiredPacket.schema.columnNames,
-//             out myLastFiredPacket.deletedRows,
-//             out myLastFiredPacket.addedRows);
-//          myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true;
-//       }
-//    }
-// }
+        protected override void myHandleIncrementalDelete() {
+            IBlueRowCollection oldRows = myLastFiredPacket.newRows;
+            myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
+            BlueDataUtility.computeDifferences(myLastFiredPacket.newRows,
+               oldRows,
+               myLastFiredPacket.schema.columnNames,
+               out myLastFiredPacket.deletedRows,
+               out myLastFiredPacket.addedRows);
+            myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true;
+        }
+    }
+}

src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs

@@ -6,6 +6,7 @@ using System.Reflection;
 using System.IO;
 using System.Diagnostics;
 using System.Linq;
+using System.Globalization;
 
 //using BCG.Utilities;
 using BCG.Utilities.General;
@@ -292,21 +293,118 @@ namespace BCG {
          return TypeCode.Empty;
       }
 
-      public static IBlueTableSchema to_typed_schema(IBlueRowCollection rows) {
-         List<ColumnInfo> cis = new List<ColumnInfo>();
-         foreach (var col in rows.schema.columnInfo) {
-            var text_vals = BlueDataUtility.get_colvalues<string>(rows, col.colName);
-            TypeCode type_code = convert_column_type(text_vals);
-            cis.Add(new ColumnInfo(col.colName, cis.Count, _typecode_to_type[type_code]));
+      // public static Type guess_type(string val) {
+      //    //DateTime
+      //    {
+      //       DateTime dt;
+      //       var success = DateTime.TryParseExact(val, "yyyy-MM-dd HH:mm:ss", null, DateTimeStyles.AllowWhiteSpaces, out dt);
+      //       if (success) {
+      //          return typeof(DateTime);
+      //       }
+      //       DateTimeOffset dtoff;
+      //       success = DateTimeOffset.TryParseExact(val, "yyyy-MM-dd HH:mm:sszzz",
+      //                                              null, System.Globalization.DateTimeStyles.None, out dtoff);
+
+      //       if (success) {
+      //          return typeof(DateTime);
+      //       }
+      //    }
+      //    //int
+      //    {
+      //       int i;
+      //       var success = Int32.TryParse(val, out i);
+      //       if (success) {
+      //          return typeof(Int32);
+      //       }
+      //    }
+      //    //double
+      //    {
+      //       double d;
+      //       var success = Double.TryParse(val, out d);
+      //       if (success) {
+      //          return typeof(Double);
+      //       }
+      //    }
+      //    //bool
+      //    {
+      //       bool b;
+      //       var success = Boolean.TryParse(val, out b);
+      //       if (success) {
+      //          return typeof(bool);
+      //       }
+      //    }
+      //    //string
+      //    return typeof(string);
+      // }
+
+      private static Type[] _parsable_types = { typeof(DateTime), typeof(int), typeof(double), typeof(bool) };
+      public static Type guess_type(string val) {
+         foreach (var t in _parsable_types) {
+            var type_val = convert_to_type(t, val);
+            if (type_val != null)
+               return t;
          }
-         return new BlueTableSchema(cis.ToArray(), rows.schema.tableName);
+         return typeof(string);
+
       }
 
-      // public static IBlueRow convert_string_row(IBlueRow r, IBlueTableSchema out_schema) {
-      //    foreach (var ci in out_schema.columnInfo){
+      public static IBlueTableSchema get_typed_schema(IBlueRowCollection rows) {
+         // just use one row to guess
+         List<ColumnInfo> cis = new List<ColumnInfo>();
+         foreach (IBlueRow r in rows) {
+            foreach (var ci in r.schema.columnInfo) {
+               if (ci.colType != typeof(System.String)) {
+                  cis.Add(ci);
+                  continue;
+               }
+               Type type = guess_type(r[ci.colName] as string);
+               cis.Add(new ColumnInfo(ci.colName, ci.colNo, type));
+            }
+            return new BlueTableSchema(cis.ToArray(), rows.schema.tableName);
+         }
+         return null;
+      }
+
+      static object convert_to_type(Type t, string value) {
+         // Attempt to convert the string to the specified type
+         try {
+            return Convert.ChangeType(value, t);
+         } catch (Exception) {
+            if (t == typeof(double)) {
+               // some doubles come with currency and ',' we clean that up here (there might be better ways to do this)
+               string cleaned_val = new string(value.Where(c => c != '$' && c != ',').ToArray());
+               cleaned_val = cleaned_val.Replace('−', '-');
+               double rv_d;
+               if (Double.TryParse(cleaned_val, out rv_d))
+                  return rv_d;
+               else
+                  return null;
+            }
+            return null;
+            // Handle conversion failure gracefully
+            // throw new ArgumentException("Failed to convert the string to the specified type.");
+         }
+      }
+
+      // converts a row where all columns are string types to a schema where the best types are guesed
+      public static IBlueRowCollection get_typed_table(IBlueRowCollection string_rows) {
+         Func<IBlueRow, IBlueTableSchema, IBlueRow> get_typed_row = (IBlueRow string_row, IBlueTableSchema typed_schema) => {
+            MutableBlueRow rv = new MutableBlueRow(typed_schema);
+            foreach (ColumnInfo ci in typed_schema.columnInfo) {
+               var typed_val = convert_to_type(ci.colType, value: string_row[ci.colName].ToString());
+               rv.setValue(ci.colName, typed_val);
+            }
+            return rv;
+         };
+         IBlueTableSchema typed_schema = get_typed_schema(string_rows);
+         var rv = new BlueTable(typed_schema);
+         foreach (IBlueRow r in string_rows) {
+            var typed_row = get_typed_row(r, typed_schema);
+            rv.addRow(typed_row);
+         }
+         return rv;
+      }
 
-      //    }
-      // }
 
 
       /// <summary>
@@ -2112,5 +2210,33 @@ namespace BCG {
       }
 
       #endregion
+
+      public class RowComparer<T> : IComparer<IBlueRow> {
+         private readonly IEnumerable<Tuple<string, bool>> _sort_cols;
+         public RowComparer(IEnumerable<Tuple<string, bool>> sort_cols) {
+            _sort_cols = sort_cols;
+         }
+         public int Compare(IBlueRow? x, IBlueRow? y) {
+            Func<object, object, int> compare_cols = (o1, o2) => {
+               return (o1 as IComparable).CompareTo(o2);
+            };
+            foreach (var c in _sort_cols) {
+               int i = compare_cols(x[c.Item1], y[c.Item1]);
+
+            }
+            return 1;
+         }
+
+      }
+      public static IList<IBlueRow> get_sorted_row(IBlueRowCollection rows, IEnumerable<Tuple<string, bool>> sort_cols) {
+         List<IBlueRow> row_list = new List<IBlueRow>();
+         foreach (IBlueRow r in rows) {
+            row_list.Add(r);
+         }
+         var row_comparer = new RowComparer<IBlueRow>(sort_cols);
+         ParallelSort.QuicksortSequential<IBlueRow>(row_list, row_comparer);
+         return row_list;
+
+      }
    }
 }

src/DataStreamSuite/DataStreamPublisher/Program.cs

@@ -6,7 +6,10 @@ using Bts.DataStreamExpressions;
 using System.Collections.Generic;
 
 using BCG;
-using System.Threading;
+using System.IO;
+using BCG.Utilities.General;
+using bts;
+using System.Linq;
 
 
 bts.Log.Gen.info($"Starting DSPublisher");
@@ -57,6 +60,8 @@ static void start_echo_client() {
 void _publish_streams() {
    PubTableBroker pb_broker = new PubTableBroker();
    CloudStreamManager csm = new CloudStreamManager();
+   string csv_folder = @"C:\Temp\csvs_topublish";
+   CSVPublisher csv_publisher = new CSVPublisher(csv_folder, pb_broker);
    // get it from config
    Dictionary<string, string> streams_to_publish = new Dictionary<string, string>() {
         //{"bbo","bbo()"},
@@ -65,6 +70,7 @@ void _publish_streams() {
         //{"books","books()"},
        {"seconds_1_10","second(1,10)"},
         {"literal_420","literal(\"x\", \"420.0\")"},
+        {"tradebar_30","tradebars(30)"},
         {"mongodb_testcollection", "mongodb(\"mongodb://192.168.1.80:27017\",\"local\",\"test_collection\")"}
         };
    //  {"bbo_ts","timeSeries(bbo())"}
@@ -88,4 +94,37 @@ void _publish_streams() {
 
 void on_change(object obj) {
    bts.Log.Gen.info($"DSPublisher on_change dp: {obj}");
+}
+
+public class CSVPublisher : IWorker {
+   private IDictionary<string, ViewPublisher> _pubs = new Dictionary<string, ViewPublisher>();
+   // read from config
+   private readonly string _csv_directory = @"C:\Temp\csvs_topublish\";
+
+   private PubTableBroker _pub_table_broker;
+   public CSVPublisher(string csv_directory, PubTableBroker pub_table_broker) {
+      _csv_directory = csv_directory;
+      _pub_table_broker = pub_table_broker;
+      MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
+   }
+   public void doWork() {
+      var file_names = Directory.GetFiles(_csv_directory);
+      foreach (var fn in file_names) {
+         if (!_pubs.ContainsKey(fn)) {
+            _handle_new_file(fn);
+         }
+      }
+   }
+
+   private void _handle_new_file(string file_name) {
+      if (file_name.EndsWith(".csv")) {
+         ViewStream vs = new CSVReaderStream(file_name);
+         vs = new SchemaGuesserStream(vs);
+         var toks = file_name.Split('\\');
+         string table_name = toks.Last();
+         table_name = table_name.Remove(table_name.Length - ".csv".Length);
+         ViewPublisher vp = new ViewPublisher(table_name, _pub_table_broker, vs);
+         _pubs.Add(file_name, vp);
+      }
+   }
 }

src/DataStreamSuite/SharedStreams/Business/MarketData/Charting/TradeBarsStream.cs

@@ -0,0 +1,161 @@
+using BCG;
+using BCG.Utilities.General;
+
+using System;
+
+namespace bts {
+    [DataStreamUserRespresentation("tradebars")]
+    public class TradeBarsStream : ViewStream, IWorker {
+
+        public class TradeBar {
+            static double[] calculate_log_returns(double[] prices) {
+                // Calculate logarithmic returns
+                double[] returns = new double[prices.Length - 1];
+                for (int i = 1; i < prices.Length; i++) {
+                    returns[i - 1] = Math.Log(prices[i] / prices[i - 1]);
+                }
+                return returns;
+            }
+
+            static double calculate_volatility(double[] returns) {
+                // Calculate standard deviation of logarithmic returns
+                double mean = returns.Average();
+                double sum_of_squares = returns.Sum(x => Math.Pow(x - mean, 2));
+                double variance = sum_of_squares / (returns.Length - 1);
+                double volatility = Math.Sqrt(variance);
+
+                return volatility;
+            }
+            public void add_trade(IBlueRow r) {
+                double price = r.get<double>("price");
+                //prices.Add(price);
+                double qty = r.get<double>(col_name: "qty");
+                if (n_entries == 0) {
+                    open = price;
+                    low = price < low ? price : low;
+                    high = price > high ? price : high;
+                }
+                n_entries++;
+                volume += qty;
+                vwap_sum += price * qty;
+                close = price;
+                if ("Buy".Equals(r.get<string>("aggressor"))) {
+                    n_buys_aggs++;
+                } else {
+                    n_sell_aggs++;
+                }
+            }
+            private List<double> prices = new List<double>();
+            public readonly string security;
+            public double low = Double.MaxValue, high, open, close, median, std_dev, vwap_sum, volume;
+            public uint n_buys_aggs, n_sell_aggs, n_entries;
+            public DateTime bar_start = DateTime.Now;
+            public TradeBar(string security_) {
+                security = security_;
+            }
+            public double volatility {
+                get {
+                    //if (prices.Count == 1)
+                    return 0.0;
+                    var log_returns = calculate_log_returns(prices.ToArray());
+                    return calculate_volatility(log_returns);
+                }
+            }
+        }
+
+        private readonly int _bar_period_seconds;
+        private readonly ViewStreamDatabase _vsd;
+        private ViewStream _timesales_stream;
+        private ViewStream _bbo_stream;
+        private IDictionary<uint, TradeBar> _sec_bar_map = new Dictionary<uint, TradeBar>();
+        private bool _dirty = false;
+        private TimeSpan _fire_period;
+        private DateTime _last_fire_time = DateTime.Now;
+        private BlueTable _table = new BlueTable(BlueDataUtility.form_schema("TradeBars",
+        Tuple.Create("id", TypeCode.Int32),
+        Tuple.Create("bar_start", TypeCode.DateTime),
+        Tuple.Create("bar_end", TypeCode.DateTime),
+        Tuple.Create("day_of_week", TypeCode.String),
+        Tuple.Create("n_trades", TypeCode.UInt32),
+        Tuple.Create("security", TypeCode.String),
+        Tuple.Create("sec_key", TypeCode.UInt32),
+        Tuple.Create("low", TypeCode.Double),
+        Tuple.Create("high", TypeCode.Double),
+        Tuple.Create("open", TypeCode.Double),
+        Tuple.Create("close", TypeCode.Double),
+        Tuple.Create("vwap_sum", TypeCode.Double),
+        Tuple.Create("volume", TypeCode.Double),
+        Tuple.Create("n_buy_aggs", TypeCode.UInt32),
+        Tuple.Create("n_sell_aggs", TypeCode.UInt32),
+        Tuple.Create("volatility", TypeCode.Double)));
+
+
+        public TradeBarsStream(int bar_period_seconds)
+        : this(bar_period_seconds, ViewStreamDatabase.instance) { }
+        public TradeBarsStream(int bar_period_seconds, ViewStreamDatabase vsd) {
+            _vsd = vsd;
+            _bar_period_seconds = bar_period_seconds;
+            _timesales_stream = new TimeSalesStream(10000, vsd);
+            _fire_period = TimeSpan.FromSeconds(bar_period_seconds);
+            _bbo_stream = new BBOStream(_vsd);
+        }
+
+        protected override void myOnFirstSubscriber() {
+            _timesales_stream.onChange += _on_timesales;
+            _bbo_stream.onChange += _on_bbo;
+            _vsd.addWorker(this);
+        }
+
+        protected override void myOnLastSubscriber() {
+            _vsd.removeWorker(this);
+            _timesales_stream.onChange -= _on_timesales;
+            _bbo_stream.onChange -= _on_bbo;
+        }
+
+        public void doWork() {
+            if (DateTime.Now - _last_fire_time > _fire_period) {
+                if (_dirty) {
+                    _dirty = false;
+                    _last_fire_time = DateTime.Now;
+                    BlueTable added_rows = new BlueTable(_table.schema);
+                    foreach (var kv in _sec_bar_map) {
+                        var sec_key = kv.Key;
+                        var tbar = kv.Value;
+                        IBlueRow new_row = new BlueRow(_table.schema, _table.Count, tbar.bar_start, DateTime.Now,
+                        tbar.bar_start.DayOfWeek.ToString(),
+                                                        tbar.n_entries, tbar.security, sec_key, tbar.low, tbar.high, tbar.open,
+                                                        tbar.close, tbar.vwap_sum, tbar.volume, tbar.n_sell_aggs, tbar.n_sell_aggs,
+                                                        tbar.volatility);
+                        added_rows.addRow(new_row);
+                        _table.addRow(new_row);
+                    }
+                    _sec_bar_map.Clear();
+                    DataPacket dp_out = new DataPacket(_table, added_rows, new BlueTable(_table.schema));
+                    myFireEvent(dp_out);
+                }
+            }
+        }
+
+        private void _on_timesales(object obj) {
+            DataPacket dp = obj as DataPacket;
+            foreach (IBlueRow r in dp.addedRows) {
+                uint sec_key = r.get<UInt32>("sec_key");
+                TradeBar tbar = null;
+                if (!_sec_bar_map.TryGetValue(sec_key, out tbar)) {
+                    tbar = new TradeBar(r.get<string>("sec"));
+                    _sec_bar_map.Add(sec_key, tbar);
+                }
+                _dirty = true;
+                tbar.add_trade(r);
+            }
+
+
+        }
+
+        private void _on_bbo(object obj) {
+
+        }
+    }
+
+
+}

src/DataStreamSuite/SharedStreams/Business/MarketData/PriceStatsStream.cs

@@ -1,127 +1,130 @@
-namespace BCG {
-   using BCG.Utilities.General;
-
-   using bts.api;
-   using bts.api.domain;
-   using bts.api.synthetic;
-   using bts.md;
-   using bts.spark;
-   using bts.utils;
-
-   using mbts;
-   using mbts.co2;
-   using mbts.config;
-   using mbts.spark;
-   using mbts.ui;
-
-   using Spark.Core.Data;
-   using Spark.Core.Disposables;
-   using Spark.Core.Reactive;
-
-   using System;
-   using System.Collections.Concurrent;
-   using System.Collections.Generic;
-   using System.Collections.ObjectModel;
-   using System.Linq;
-   using System.Reactive.Linq;
-   using bts.api.core;
-   using Spark.Api.MarketData;
-
-   using SecurityKey = System.UInt32;
-   using System.Data;
-   using System.Reflection.Metadata;
-
-   [DataStreamUserRespresentation("price_stats")]
-   public class PriceStatsStream : ViewStream, IWorker {
-      private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500);
-      private DateTime _last_poll_time = DateTime.MinValue;
-      private ViewStreamDatabase _vsd;
-      private int _id = 0;
-      IReadOnlyDictionary<SecurityKey, PriceStats> _cached_state = null;
-      MarketDataClient _md_client;
-      private bool _dirty = false;
-      private readonly IBlueTableSchema _pstats_scheam = BlueDataUtility.form_schema("bbo",
-                                                Tuple.Create("id", TypeCode.Int32),
-                                                Tuple.Create("sec_key", TypeCode.Int32),
-                                                Tuple.Create("sec", TypeCode.String),
-                                                Tuple.Create("open", TypeCode.UInt32),
-                                                Tuple.Create("close", TypeCode.Double),
-                                                Tuple.Create("settlement", TypeCode.Single));
-      IDisposable? subscription;
-
-      // TODO: add cons for symbol, exchange etc 
-      public PriceStatsStream()
-      : this(ViewStreamDatabase.instance) {
-      }
-
-      public PriceStatsStream(ViewStreamDatabase vsd) {
-         bts.Log.Gen.info("PriceStatsStream.cons");
-         _vsd = vsd;
-         _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _vsd.addWorker(this);
-         subscription = _md_client.price_stats().Subscribe(_on_price_stats);
-      }
-
-      private void _log_updates(TableUpdate<PriceStats, uint> update) {
-         int i = 0;
-         // foreach (BookTuple bt in update.added) {
-         //    bts.Log.Gen.info($"PriceStatsStream added: idx: {i++} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
-         // }
-         // foreach (Change<BookTuple> c in update.changes) {
-         //    var bt = c.New;
-         //    bts.Log.Gen.info($"PriceStatsStream changes: idx: {i} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
-         // }
-      }
-
-      private void _on_price_stats(TableUpdate<PriceStats, uint> update) {
-         _log_updates(update);
-         // just cacching in here, will populate the rows in _poll_scan and fire the datapacket
-         bts.Log.Gen.info($"PriceStatsStream._on_price_stats state: {update.state.Count()} changes: {update.changes.Count()} added: {update.added.Count()}");
-         _cached_state = update.state;
-         _dirty = true;
-      }
-
-      protected override void myOnLastSubscriber() {
-         _vsd.removeWorker(this);
-         Disposable.dispose(ref subscription);
-      }
-
-      private void _scan() {
-         // this shouldn be made incremental using the TableUpdate
-         bts.Log.Gen.info("PriceStatsStream._scan_bbo");
-         if (_cached_state == null)
-            return;
-         BlueTable new_state = new BlueTable(_pstats_scheam);
-         foreach (var kv in _cached_state) {
-            SecurityKey skey = kv.Key;
-            int id = Convert.ToInt32(skey);
-            PriceStats pstats = kv.Value;
-            IBlueRow r = new BlueRow(_pstats_scheam, id, skey, pstats.security.ToString(), pstats.open, pstats.close, pstats.settlement);
-            new_state.addRow(r);
-         }
-         DataPacket dp = null;
-         if (myLastValue == null) {
-            dp = new DataPacket(new_state, new_state, new BlueTable(_pstats_scheam));
-         } else {
-            DataPacket old_dp = myLastValue as DataPacket;
-            IBlueRowCollection added_rows, del_rows;
-            BlueDataUtility.compute_differences_fast(new_state, old_dp.newRows, out del_rows, out added_rows);
-            dp = new DataPacket(new_state, added_rows, del_rows);
-         }
-         if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
-            return;
-         myFireEvent(dp);
-      }
-
-      public void doWork() {
-         if (_dirty && DateTime.Now - _last_poll_time > _poll_period) {
-            _last_poll_time = DateTime.Now;
-            _dirty = false;
-            _scan();
-         }
-      }
-   }
-}
+namespace BCG {
+   using BCG.Utilities.General;
+
+   using bts.api;
+   using bts.api.domain;
+   using bts.api.synthetic;
+   using bts.md;
+   using bts.spark;
+   using bts.utils;
+
+   using mbts;
+   using mbts.co2;
+   using mbts.config;
+   using mbts.spark;
+   using mbts.ui;
+
+   using Spark.Core.Data;
+   using Spark.Core.Disposables;
+   using Spark.Core.Reactive;
+
+   using System;
+   using System.Collections.Concurrent;
+   using System.Collections.Generic;
+   using System.Collections.ObjectModel;
+   using System.Linq;
+   using System.Reactive.Linq;
+   using bts.api.core;
+   using Spark.Api.MarketData;
+
+   using SecurityKey = System.UInt32;
+   using System.Data;
+   using System.Reflection.Metadata;
+
+   [DataStreamUserRespresentation("price_stats")]
+   public class PriceStatsStream : ViewStream, IWorker {
+      private readonly TimeSpan _poll_period = TimeSpan.FromMilliseconds(500);
+      private DateTime _last_poll_time = DateTime.MinValue;
+      private ViewStreamDatabase _vsd;
+      private int _id = 0;
+      IReadOnlyDictionary<SecurityKey, PriceStats> _cached_state = null;
+      MarketDataClient _md_client;
+      private bool _dirty = false;
+      private IDisposable _sub = null;
+      private readonly IBlueTableSchema _pstats_scheam = BlueDataUtility.form_schema("bbo",
+                                                Tuple.Create("id", TypeCode.Int32),
+                                                Tuple.Create("sec_key", TypeCode.UInt32),
+                                                Tuple.Create("sec", TypeCode.String),
+                                                Tuple.Create("date", TypeCode.DateTime),
+                                                Tuple.Create("open", TypeCode.Double),
+                                                Tuple.Create("close", TypeCode.Double),
+                                                Tuple.Create("settlement", TypeCode.Double));
+      IDisposable? subscription;
+
+      // TODO: add cons for symbol, exchange etc 
+      public PriceStatsStream()
+      : this(ViewStreamDatabase.instance) {
+      }
+
+      public PriceStatsStream(ViewStreamDatabase vsd) {
+         bts.Log.Gen.info("PriceStatsStream.cons");
+         _vsd = vsd;
+         _md_client = new MarketDataClient(new MDSource(TimeSpan.FromSeconds(0.5)));
+      }
+
+      protected override void myOnFirstSubscriber() {
+         _vsd.addWorker(this);
+         subscription = _md_client.price_stats().Subscribe(_on_price_stats);
+      }
+
+      private void _log_updates(TableUpdate<PriceStats, uint> update) {
+         int i = 0;
+         // foreach (BookTuple bt in update.added) {
+         //    bts.Log.Gen.info($"PriceStatsStream added: idx: {i++} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
+         // }
+         // foreach (Change<BookTuple> c in update.changes) {
+         //    var bt = c.New;
+         //    bts.Log.Gen.info($"PriceStatsStream changes: idx: {i} bid: {bt.book.bids[0].price} / {bt.book.bids[0].size} ask: {bt.book.asks[0].price} / {bt.book.asks[0].size}");
+         // }
+      }
+
+      private void _on_price_stats(TableUpdate<PriceStats, uint> update) {
+         _log_updates(update);
+         // just cacching in here, will populate the rows in _poll_scan and fire the datapacket
+         bts.Log.Gen.info($"PriceStatsStream._on_price_stats state: {update.state.Count()} changes: {update.changes.Count()} added: {update.added.Count()}");
+         _cached_state = update.state;
+         _dirty = true;
+      }
+
+      protected override void myOnLastSubscriber() {
+         _vsd.removeWorker(this);
+         Disposable.dispose(ref _sub);
+      }
+
+      private void _scan() {
+         // this shouldn be made incremental using the TableUpdate
+         bts.Log.Gen.info("PriceStatsStream._scan_bbo");
+         if (_cached_state == null)
+            return;
+         BlueTable new_state = new BlueTable(_pstats_scheam);
+         foreach (var kv in _cached_state) {
+            SecurityKey skey = kv.Key;
+            int id = Convert.ToInt32(skey);
+            PriceStats pstats = kv.Value;
+            var today = DateTime.Now.Date;
+            IBlueRow r = new BlueRow(_pstats_scheam, id, skey, pstats.security.ToString(), today, pstats.open, pstats.close, pstats.settlement);
+            new_state.addRow(r);
+         }
+         DataPacket dp = null;
+         if (myLastValue == null) {
+            dp = new DataPacket(new_state, new_state, new BlueTable(_pstats_scheam));
+         } else {
+            DataPacket old_dp = myLastValue as DataPacket;
+            IBlueRowCollection added_rows, del_rows;
+            BlueDataUtility.compute_differences_fast(new_state, old_dp.newRows, out del_rows, out added_rows);
+            dp = new DataPacket(new_state, added_rows, del_rows);
+         }
+         if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
+            return;
+         myFireEvent(dp);
+      }
+
+      public void doWork() {
+         if (_dirty && DateTime.Now - _last_poll_time > _poll_period) {
+            _last_poll_time = DateTime.Now;
+            _dirty = false;
+            _scan();
+         }
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/SharedStreams.csproj

@@ -12,6 +12,7 @@
     <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
     <ProjectReference Include="..\DataStream\DataStream.csproj" />
     <ProjectReference Include="..\PubSub\PubSub.csproj" />
+    <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
   </ItemGroup>
 
   <ItemGroup>

src/Spark.Admin.WebApp/src/Accounts/Types.fs

@@ -17,9 +17,9 @@ type Tab =
   | PositionManagementTab
   | MarginTab
 
-let venues = [| Venue.DERIBIT; Venue.GLBX; Venue.ICE; Venue.SIM; |]
+let venues = [| Venue.BINANCE; Venue.BinanceUs; Venue.BinanceUsdsFutures; Venue.COINBASE; Venue.DERIBIT; Venue.GLBX; Venue.ICE; Venue.SIM; |]
 
-let securityTypes = [| SecurityType.FUTURE; SecurityType.OPTION; SecurityType.SPREAD; |]
+let securityTypes = [| SecurityType.SPOT; SecurityType.FUTURE; SecurityType.OPTION; SecurityType.SPREAD; |]
 
 let emptyAccount = Access.Accounts.empty
 
@@ -35,6 +35,26 @@ type CMEExchangeField =
   | ClearingAccount of string
   | MiscTagMapping of JSMap<string>
 
+type BINANCEExchangeField =
+  | CTI of CTICode
+  | Origin of CustomerOrFirm
+  | ClearingAccount of string
+
+type BinanceUsExchangeField =
+  | CTI of CTICode
+  | Origin of CustomerOrFirm
+  | ClearingAccount of string
+
+type BinanceUsdsFUTURESExchangeField =
+  | CTI of CTICode
+  | Origin of CustomerOrFirm
+  | ClearingAccount of string
+
+type COINBASEExchangeField =
+  | CTI of CTICode
+  | Origin of CustomerOrFirm
+  | ClearingAccount of string
+
 type DERIBITExchangeField =
   | CTI of CTICode
   | Origin of CustomerOrFirm

src/Spark.Admin.WebApp/src/Accounts/View.fs

@@ -797,6 +797,7 @@ let venueToString (venue:bts.api.Venue) =
     | bts.api.Venue.BINANCE -> "BINANCE"
     | bts.api.Venue.BinanceUs -> "BinanceUs"
     | bts.api.Venue.BinanceUsdsFutures -> "BinanceUsdsFutures"
+    | bts.api.Venue.COINBASE -> "COINBASE"
     | bts.api.Venue.FTX -> "FTX"
     | bts.api.Venue.OKX -> "OKX"
 

src/client/spark/Modules/Trading/AlgoStudio/Model/AlgoDataVM.cs

@@ -1,39 +1,42 @@
-using bts.api.orderman;
-using bts.api.orderman.algo;
-using bts.utils;
-
-using mbts.spark.utilities;
-
-using System;
-using System.Windows.Media;
-
-namespace mbts.spark {
-   struct AlgoDataVM : IEquatable<AlgoDataVM> {
-      public static readonly Func<AlgoDataVM, AlgoID> id_fn = vm => AlgoID.from_algo_key(vm.data.key);
-
-      public readonly AlgoData data;
-      public readonly AlgoDefinitionMetadata? meta;
-
-      public AlgoDataVM(AlgoData data, AlgoDefinitionMetadata? meta = null) {
-         this.data = data;
-         this.meta = meta;
-      }
-
-      public string name => data.name;
-      public Brush badge_fill => OrderUtils.badge_fill(meta, OrderDataHelper.get_metadata(data.value.CustomCase));
-      public AlgoDisplayGroup group => data.key.display_group();
-
-      public override string ToString() =>
-         data.name;
-
-      public static AlgoDataVM from_data(AlgoData data, AlgoDefinitionMetadata? meta) =>
-         new AlgoDataVM(data, meta);
-
-      public static AlgoDataVM from_data((AlgoData algo, AlgoDefinitionMetadata? meta) tup) =>
-         new AlgoDataVM(tup.algo, tup.meta);
-
-      public bool Equals(AlgoDataVM other) => data == other.data && meta == other.meta;
-      public override bool Equals(object obj) => obj is AlgoDataVM other && Equals(other);
-      public override int GetHashCode() => EqualityComparer.aggregate_hash(data.GetHashCode(), meta.GetHashCode());
-   }
-}
+using bts.api.orderman;
+using bts.api.orderman.algo;
+using bts.utils;
+
+using mbts.spark.utilities;
+
+using System;
+using System.Windows.Media;
+
+namespace mbts.spark {
+   struct AlgoDataVM : IEquatable<AlgoDataVM> {
+      public static readonly Func<AlgoDataVM, AlgoID> id_fn = vm => AlgoID.from_algo_key(vm.data.key);
+
+      public readonly AlgoData data;
+      public readonly AlgoDefinitionMetadata? meta;
+
+      public AlgoDataVM(AlgoData data, AlgoDefinitionMetadata? meta = null) {
+         this.data = data;
+         this.meta = meta;
+      }
+
+      public string name => data.name;
+
+      public Brush badge_fill => OrderUtils.badge_fill(
+         meta, OrderMetadataStore.get_metadata(data.value.CustomCase));
+
+      public AlgoDisplayGroup group => data.key.display_group();
+
+      public override string ToString() =>
+         data.name;
+
+      public static AlgoDataVM from_data(AlgoData data, AlgoDefinitionMetadata? meta) =>
+         new AlgoDataVM(data, meta);
+
+      public static AlgoDataVM from_data((AlgoData algo, AlgoDefinitionMetadata? meta) tup) =>
+         new AlgoDataVM(tup.algo, tup.meta);
+
+      public bool Equals(AlgoDataVM other) => data == other.data && meta == other.meta;
+      public override bool Equals(object obj) => obj is AlgoDataVM other && Equals(other);
+      public override int GetHashCode() => EqualityComparer.aggregate_hash(data.GetHashCode(), meta.GetHashCode());
+   }
+}

src/client/spark/Modules/Trading/OrderBook/OrderBookWindow.cs

@@ -298,7 +298,7 @@ namespace mbts.spark {
          }
 
          bool has_algo_params(OrderSecurityAlgo? order) =>
-            order.HasValue && !OrderDataHelper.get_metadata(order.Value.algo.value.CustomCase).NoProps;
+            order.HasValue && !OrderMetadataStore.get_metadata(order.Value.algo.value.CustomCase).NoProps;
 
          Window mk_algo_params_dialog(OrderSecurityAlgo? order) {
             if (!has_algo_params(order))

src/client/spark/Orders/OrderFactory.cs

@@ -1,323 +1,323 @@
-#nullable enable
-namespace mbts.spark.utilities {
-   using bts;
-   using bts.api;
-   using bts.api.core;
-   using bts.api.orderman;
-   using bts.collections;
-   using bts.spark;
-
-   using mbts.ui;
-   using mbts.ui.utilities;
-   using mbts.ui.utility;
-   using mbts.utils;
-
-   using Spark.Core;
-   using Spark.Core.Disposables;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Composition;
-   using System.Linq;
-   using System.Reactive.Linq;
-
-   using OrderType = bts.api.orderman.CustomOrderDefinition.CustomOneofCase;
-   using DynamicOrderParamType = bts.api.orderman.DynamicOrderParam.ValueOneofCase;
-
-   [Export(typeof(OrderFactory))]
-   class OrderFactory : IDisposable {
-      static readonly OrderMetadata _default_order_metadata = new() {
-         KeyBinding = "",
-         Color = BTSColors.Orders.default_order
-      };
-      readonly AdminModel _admin_model;
-      readonly IReadOnlyDictionary<OneOf<OrderType, string>, string> _algo_abbreviation_index;
-      readonly IReadOnlyDictionary<string, DynamicOrderDescriptor> _dynamic_descriptors;
-
-      IDisposable? _dispose;
-
-      [ImportingConstructor]
-      public OrderFactory(OrderService service, AdminModel admin_model) {
-         _admin_model = admin_model;
-         _dynamic_descriptors = service.get_registered_dynamic_order_types(new VoidMessage())
-            .DynamicOrderDesc
-            .ToDictionary(d => d.DynamicOrderType);
-         var algo_name_index = new HashSet<OneOf<OrderType, string>>(
-            OneOf<OrderType, string>.ValueEqualityComparer);
-         // add standard types
-         algo_name_index.AddRange(
-            EnumHelper.get_values_except(OrderType.None)
-            .Select(t => new OneOf<OrderType, string>(t)));
-         // add dynamic types
-         algo_name_index.AddRange(_dynamic_descriptors.Keys
-            .Select(dt => new OneOf<OrderType, string>(dt)));
-         // build abbreviation index
-         _algo_abbreviation_index = DisplayNameHelper.get_word_tree_abbreviations(
-            algo_name_index,
-            oneof => oneof.value.ToString(),
-            OneOf<OrderType, string>.ValueEqualityComparer);
-      }
-
-      public static string? fmt_dynamic_param(DynamicOrderParam? dp) {
-         if (dp is null) {
-            return null;
-         }
-         return dp.ValueCase switch {
-            DynamicOrderParamType.AsArray => ", ".join(dp.AsArray.Element.Select(fmt_dynamic_param)),
-            DynamicOrderParamType.AsBool => dp.AsBool.ToString(),
-            DynamicOrderParamType.AsDouble => dp.AsDouble.ToString(),
-            DynamicOrderParamType.AsEnumerated => dp.AsEnumerated.EnumValue,
-            DynamicOrderParamType.AsInt32 => dp.AsInt32.ToString(),
-            DynamicOrderParamType.AsQuantity => dp.AsQuantity.ToString(),
-            DynamicOrderParamType.AsString => dp.AsString,
-            _ => $"({dp.ValueCase})",
-         };
-      }
-
-      public static DynamicOrderParam init_dynamic_param(DynamicOrderParamDescriptor descriptor) {
-         var rv = new DynamicOrderParam();
-         switch (descriptor.ParamType.DescriptorCase) {
-            case DynamicOrderTypeDescriptor.DescriptorOneofCase.Array:
-               rv.AsArray = new DynamicOrderArray();
-               break;
-            case DynamicOrderTypeDescriptor.DescriptorOneofCase.Enumerated:
-               rv.AsEnumerated = new() {
-                  EnumValue = descriptor.ParamType.Enumerated.Element.FirstOrDefault()
-               };
-               break;
-            case DynamicOrderTypeDescriptor.DescriptorOneofCase.Primitive:
-               switch (descriptor.ParamType.Primitive) {
-                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveBool: rv.AsBool = false; break;
-                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveDouble: rv.AsDouble = 0.0; break;
-                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveInt: rv.AsInt32 = 0; break;
-                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveOrderDef: rv.AsOrderDef = new(); break;
-                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveString: rv.AsString = string.Empty; break;
-                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveSecurityKey:
-                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveUint: rv.AsUint32 = 0u; break;
-                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveQuantity: rv.AsQuantity = new(); break;
-               }
-               break;
-            case DynamicOrderTypeDescriptor.DescriptorOneofCase.Structure:
-               rv.AsStruct = new();
-               break;
-         }
-         return rv;
-      }
-
-      public static SpreaderOrderDefinition spreader_def() => new() {
-         ExecutionStrategy = new ExecutionStrategy()
-      };
-
-      public DynamicOrderDescriptor? dynamic_descriptor(string type) =>
-         _dynamic_descriptors.GetValueOrDefault(type);
-
-      public DynamicOrderDefinition? dynamic_def(string type) {
-         if (!_dynamic_descriptors.TryGetValue(type, out var desc)) {
-            return null;
-         }
-         var def = new DynamicOrderDefinition { DynamicOrderType = type };
-         foreach (var p in desc.ParamDesc) {
-            def.OrderParams[p.ParamName] = init_dynamic_param(p);
-         }
-         return def;
-      }
-
-      public OrderType default_order_type(SecurityImmutable security) =>
-         security.ContractCase switch {
-            Security.ContractOneofCase.Synthetic => OrderType.Spreader,
-            _ => OrderType.Limit,
-         };
-
-      public TimeInForce default_time_in_force() =>
-         TimeInForce.Day;
-
-      public TimeInForce default_time_in_force(SecurityImmutable security) =>
-         default_time_in_force();
-
-      public IEnumerable<OrderType> default_order_types() =>
-         OrderDataHelper.exchange_order_types;
-
-      public IEnumerable<TimeInForce> default_time_in_forces(
-         SecurityImmutable security,
-         OrderType order_type,
-         bool is_child = false
-      ) {
-         var tifs = EnumHelper.get_values_except(TimeInForce.Undefined);
-         if (order_type != OrderType.Limit) {
-            tifs = tifs.Except(new[] { TimeInForce.Ioc });
-         }
-         return tifs;
-      }
-
-      public IEnumerable<string> dynamic_types() =>
-         _dynamic_descriptors.Keys;
-
-      public CustomOrderDefinitionImmutable? custom_defi(string type_name) {
-         if (Enum.TryParse(type_name, out OrderType sys_type)) {
-            return _custom_def(sys_type);
-         }
-         var dyn_def = dynamic_def(type_name);
-         if (dyn_def is null) {
-            return null;
-         }
-         return new(new() { Dynamic = dyn_def });
-      }
-
-      public OrderDefinition order(SecurityImmutable security, AccountTuple? account = null) {
-         var rv = new OrderDefinition {
-            SecurityKey = security.Id,
-            Side = Side.Buy,
-            TimeInForce = TimeInForce.Day,
-            CustomDef = OrderDataHelper.Defaults.CustomOrderDefinition(),
-            CtiCode = CTICode.CtiCode1,
-            CustomerOrFirm = CustomerOrFirm.Firm,
-            // this will be the tag 50 on CME orders
-            // current user might be okay, but we might also want to get it
-            // from the store that serves the risk admin gui
-            ManualOrderIndicator = ManualOrderIndicator.Yes,
-            User = _admin_model.user.name,
-            UserId = _admin_model.user.id
-         };
-         if (account.HasValue) {
-            rv.Account = account.Value.name;
-            rv.AccountId = account.Value.id;
-         }
-         return rv;
-      }
-
-      public string abbreviate_algo(OneOf<OrderType, string> id) {
-         if (id is null) {
-            throw new ArgumentNullException(nameof(id));
-         }
-         if (id.value is string) {
-            return _algo_abbreviation_index.GetValueOrDefault(id) ??
-               id.value.ToString() ??
-               string.Empty;
-         }
-         if (id.value is OrderType t) {
-            return OrderDataHelper.abbreviate_algo_type(t);
-         }
-         return id.ToString() ?? string.Empty;
-      }
-
-      public OrderMetadata get_metadata(OrderRecordImmutable order) =>
-         get_metadata(order.Def.CustomDef.CustomCase);
-
-      public OrderMetadata get_metadata(OrderType order_type) =>
-         order_type switch {
-            OrderType.Market => new() { KeyBinding = "M", Color = BTSColors.Orders.market_order, NoProps = true },
-            OrderType.Limit => new() { KeyBinding = "L", Color = BTSColors.Orders.limit_order, NoProps = true },
-            OrderType.Stop => new() { KeyBinding = "S", Color = BTSColors.Orders.stop_order, NoProps = true },
-            OrderType.StopLimit => new() { KeyBinding = "SL", Color = BTSColors.Orders.stop_limit_order },
-            OrderType.GlbxIceberg => new() { KeyBinding = "GB", Color = BTSColors.Orders.glbx_iceberg_order },
-            OrderType.Iceberg => new() { KeyBinding = "I", Color = BTSColors.Orders.iceberg_order },
-            OrderType.TimeBox => new() { KeyBinding = "TB", Color = BTSColors.Orders.time_box_order },
-            OrderType.Tick => new() { KeyBinding = "T", Color = BTSColors.Orders.tick_order },
-            OrderType.Lean => new() { KeyBinding = "L", Color = BTSColors.Orders.lean_order },
-            OrderType.Join => new() { KeyBinding = "J", Color = BTSColors.Orders.join_order },
-            OrderType.Top => new() { KeyBinding = "D", Color = BTSColors.Orders.top_order },
-            OrderType.Cover => new() { KeyBinding = "C", Color = BTSColors.Orders.cover_order },
-            OrderType.Oco => new() { KeyBinding = "C", Color = BTSColors.Orders.oco_order },
-            OrderType.Pricer => new() { KeyBinding = "C", Color = BTSColors.Orders.pricer_order },
-            OrderType.Offset => new() { KeyBinding = "O", Color = BTSColors.Orders.off_order },
-            OrderType.Trigger => new() { KeyBinding = "T", Color = BTSColors.Orders.trigger_order },
-            OrderType.Spreader => new() { KeyBinding = "", Color = BTSColors.Orders.spreader_order },
-            OrderType.Dynamic => new() { KeyBinding = "", Color = BTSColors.Orders.top_order },
-            _ => _default_order_metadata
-         };
-
-      public SpreaderOrderDefinition spreader_order(
-         SecurityImmutable security, ExecutionStrategyImmutable strategy) =>
-         new() {
-            Security = security.CloneToMutable(),
-            ExecutionStrategy = strategy.CloneToMutable()
-         };
-
-      public SpreaderOrderDefinitionImmutable spreader_orderi(
-         SecurityImmutable security, ExecutionStrategyImmutable strategy) =>
-         new(spreader_order(security, strategy));
-
-      public void Dispose() =>
-         Disposable.dispose(ref _dispose);
-
-      static CustomOrderDefinition _custom_def() =>
-         new() { Limit = _limit_def() };
-
-      static CustomOrderDefinition _custom_def(
-         OrderType type) {
-         var rv = new CustomOrderDefinition();
-         switch (type) {
-            case OrderType.Market: rv.Market = _market_def(); break;
-            case OrderType.Limit: rv.Limit = _limit_def(); break;
-            case OrderType.Stop: rv.Stop = _stop_def(); break;
-            case OrderType.StopLimit: rv.StopLimit = _stop_limit_def(); break;
-            case OrderType.Iceberg: rv.Iceberg = _iceberg_def(); break;
-            case OrderType.GlbxIceberg: rv.GlbxIceberg = _glbx_iceberg_def(); break;
-            case OrderType.TimeBox: rv.TimeBox = _time_box_def(); break;
-            case OrderType.Tick: rv.Tick = _tick_def(); break;
-            case OrderType.Lean: rv.Lean = _lean_def(); break;
-            case OrderType.Join: rv.Join = _join_def(); break;
-            case OrderType.Top: rv.Top = _top_def(); break;
-            case OrderType.Cover: rv.Cover = _cover_def(); break;
-            case OrderType.Oco: rv.Oco = _oco_def(); break;
-            case OrderType.Pricer: rv.Pricer = _pricer_def(); break;
-            case OrderType.Offset: rv.Offset = _offset_def(); break;
-            case OrderType.Trigger: rv.Trigger = _trigger_def(); break;
-            case OrderType.Spreader: rv.Spreader = spreader_def(); break;
-            case OrderType.Dynamic: rv.Dynamic = _dynamic_def(); break;
-            default:
-               OrderTypeVMFactory.GenericOrderFactory.setup_generic_custom_order(rv, type);
-               break;
-         }
-         return rv;
-      }
-
-      static MarketOrderDefinition _market_def() => new();
-      static LimitOrderDefinition _limit_def() => new();
-      static StopOrderDefinition _stop_def() => new();
-      static StopLimitOrderDefinition _stop_limit_def() => new();
-      static GLBXIcebergOrderDefinition _glbx_iceberg_def() => new() { MaxShowQuantity = 1 };
-      static IcebergOrderDefinition _iceberg_def() =>
-         new() { Child = _custom_def(), ShowQuantity = 1 };
-      static TimeBoxOrderDefinition _time_box_def() => new() {
-         Child = _custom_def(),
-         Start = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTime(DateTime.UtcNow),
-         Stop = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTime(DateTime.UtcNow)
-      };
-      static TickOrderDefinition _tick_def() => new() {
-         Child = _custom_def(),
-         CrossThreshold = new() { Absolute = 1 }
-      };
-      static LeanOrderDefinition _lean_def() => new() {
-         Child = _custom_def(),
-         CrossThreshold = new() { Absolute = 1 },
-         MaxChaseTicks = 2
-      };
-      static JoinOrderDefinition _join_def() => new() {
-         Child = _custom_def(),
-         JoinThreshold = new() { Absolute = 1 }
-      };
-      static TopOrderDefinition _top_def() => new() {
-         Child = _custom_def(),
-         TopThreshold = new() { Absolute = 1 }
-      };
-      static CoverOrderDefinition _cover_def() => new() {
-         Child = _custom_def(),
-         Hedge = _custom_def(),
-         HedgeTicks = 1
-      };
-      static OCOOrderDefinition _oco_def() => new() {
-         Child = _custom_def(),
-         Other = _custom_def(),
-         OtherSide = RelativeSide.Same
-      };
-      static PricerOrderDefinition _pricer_def() => new() { Child = _custom_def() };
-      static OffsetOrderDefinition _offset_def() => new() { Child = _custom_def(), };
-      static TriggerOrderDefinition _trigger_def() => new() {
-         Child = _custom_def(),
-         TriggerCondition = new TriggerCondition(),
-         TriggerAction = new TriggerAction(),
-      };
-      static DynamicOrderDefinition _dynamic_def() => new();
-   }
-}
+#nullable enable
+namespace mbts.spark.utilities {
+   using bts;
+   using bts.api;
+   using bts.api.core;
+   using bts.api.orderman;
+   using bts.collections;
+   using bts.spark;
+
+   using mbts.ui;
+   using mbts.ui.utilities;
+   using mbts.ui.utility;
+   using mbts.utils;
+
+   using Spark.Core;
+   using Spark.Core.Disposables;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Composition;
+   using System.Linq;
+   using System.Reactive.Linq;
+
+   using OrderType = bts.api.orderman.CustomOrderDefinition.CustomOneofCase;
+   using DynamicOrderParamType = bts.api.orderman.DynamicOrderParam.ValueOneofCase;
+
+   [Export(typeof(OrderFactory))]
+   class OrderFactory : IDisposable {
+      static readonly OrderMetadata _default_order_metadata = new() {
+         KeyBinding = "",
+         Color = BTSColors.Orders.default_order
+      };
+      readonly AdminModel _admin_model;
+      readonly IReadOnlyDictionary<OneOf<OrderType, string>, string> _algo_abbreviation_index;
+      readonly IReadOnlyDictionary<string, DynamicOrderDescriptor> _dynamic_descriptors;
+
+      IDisposable? _dispose;
+
+      [ImportingConstructor]
+      public OrderFactory(OrderService service, AdminModel admin_model) {
+         _admin_model = admin_model;
+         _dynamic_descriptors = service.get_registered_dynamic_order_types(new VoidMessage())
+            .DynamicOrderDesc
+            .ToDictionary(d => d.DynamicOrderType);
+         var algo_name_index = new HashSet<OneOf<OrderType, string>>(
+            OneOf<OrderType, string>.ValueEqualityComparer);
+         // add standard types
+         algo_name_index.AddRange(
+            EnumHelper.get_values_except(OrderType.None)
+            .Select(t => new OneOf<OrderType, string>(t)));
+         // add dynamic types
+         algo_name_index.AddRange(_dynamic_descriptors.Keys
+            .Select(dt => new OneOf<OrderType, string>(dt)));
+         // build abbreviation index
+         _algo_abbreviation_index = DisplayNameHelper.get_word_tree_abbreviations(
+            algo_name_index,
+            oneof => oneof.value.ToString(),
+            OneOf<OrderType, string>.ValueEqualityComparer);
+      }
+
+      public static string? fmt_dynamic_param(DynamicOrderParam? dp) {
+         if (dp is null) {
+            return null;
+         }
+         return dp.ValueCase switch {
+            DynamicOrderParamType.AsArray => ", ".join(dp.AsArray.Element.Select(fmt_dynamic_param)),
+            DynamicOrderParamType.AsBool => dp.AsBool.ToString(),
+            DynamicOrderParamType.AsDouble => dp.AsDouble.ToString(),
+            DynamicOrderParamType.AsEnumerated => dp.AsEnumerated.EnumValue,
+            DynamicOrderParamType.AsInt32 => dp.AsInt32.ToString(),
+            DynamicOrderParamType.AsQuantity => dp.AsQuantity.ToString(),
+            DynamicOrderParamType.AsString => dp.AsString,
+            _ => $"({dp.ValueCase})",
+         };
+      }
+
+      public static DynamicOrderParam init_dynamic_param(DynamicOrderParamDescriptor descriptor) {
+         var rv = new DynamicOrderParam();
+         switch (descriptor.ParamType.DescriptorCase) {
+            case DynamicOrderTypeDescriptor.DescriptorOneofCase.Array:
+               rv.AsArray = new DynamicOrderArray();
+               break;
+            case DynamicOrderTypeDescriptor.DescriptorOneofCase.Enumerated:
+               rv.AsEnumerated = new() {
+                  EnumValue = descriptor.ParamType.Enumerated.Element.FirstOrDefault()
+               };
+               break;
+            case DynamicOrderTypeDescriptor.DescriptorOneofCase.Primitive:
+               switch (descriptor.ParamType.Primitive) {
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveBool: rv.AsBool = false; break;
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveDouble: rv.AsDouble = 0.0; break;
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveInt: rv.AsInt32 = 0; break;
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveOrderDef: rv.AsOrderDef = new(); break;
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveString: rv.AsString = string.Empty; break;
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveSecurityKey:
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveUint: rv.AsUint32 = 0u; break;
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveQuantity: rv.AsQuantity = new(); break;
+               }
+               break;
+            case DynamicOrderTypeDescriptor.DescriptorOneofCase.Structure:
+               rv.AsStruct = new();
+               break;
+         }
+         return rv;
+      }
+
+      public static SpreaderOrderDefinition spreader_def() => new() {
+         ExecutionStrategy = new ExecutionStrategy()
+      };
+
+      public DynamicOrderDescriptor? dynamic_descriptor(string type) =>
+         _dynamic_descriptors.GetValueOrDefault(type);
+
+      public DynamicOrderDefinition? dynamic_def(string type) {
+         if (!_dynamic_descriptors.TryGetValue(type, out var desc)) {
+            return null;
+         }
+         var def = new DynamicOrderDefinition { DynamicOrderType = type };
+         foreach (var p in desc.ParamDesc) {
+            def.OrderParams[p.ParamName] = init_dynamic_param(p);
+         }
+         return def;
+      }
+
+      public OrderType default_order_type(SecurityImmutable security) =>
+         security.ContractCase switch {
+            Security.ContractOneofCase.Synthetic => OrderType.Spreader,
+            _ => OrderType.Limit,
+         };
+
+      public TimeInForce default_time_in_force() =>
+         TimeInForce.Day;
+
+      public TimeInForce default_time_in_force(SecurityImmutable security) =>
+         default_time_in_force();
+
+      public IEnumerable<OrderType> default_order_types() =>
+         OrderDataHelper.exchange_order_types;
+
+      public IEnumerable<TimeInForce> default_time_in_forces(
+         SecurityImmutable security,
+         OrderType order_type,
+         bool is_child = false
+      ) {
+         var tifs = EnumHelper.get_values_except(TimeInForce.Undefined);
+         if (order_type != OrderType.Limit) {
+            tifs = tifs.Except(new[] { TimeInForce.Ioc });
+         }
+         return tifs;
+      }
+
+      public IEnumerable<string> dynamic_types() =>
+         _dynamic_descriptors.Keys;
+
+      public CustomOrderDefinitionImmutable? custom_defi(string type_name) {
+         if (Enum.TryParse(type_name, out OrderType sys_type)) {
+            return _custom_def(sys_type);
+         }
+         var dyn_def = dynamic_def(type_name);
+         if (dyn_def is null) {
+            return null;
+         }
+         return new(new() { Dynamic = dyn_def });
+      }
+
+      public OrderDefinition order(SecurityImmutable security, AccountTuple? account = null) {
+         var rv = new OrderDefinition {
+            SecurityKey = security.Id,
+            Side = Side.Buy,
+            TimeInForce = TimeInForce.Day,
+            CustomDef = OrderDataHelper.Defaults.CustomOrderDefinition(),
+            CtiCode = CTICode.CtiCode1,
+            CustomerOrFirm = CustomerOrFirm.Firm,
+            // this will be the tag 50 on CME orders
+            // current user might be okay, but we might also want to get it
+            // from the store that serves the risk admin gui
+            ManualOrderIndicator = ManualOrderIndicator.Yes,
+            User = _admin_model.user.name,
+            UserId = _admin_model.user.id
+         };
+         if (account.HasValue) {
+            rv.Account = account.Value.name;
+            rv.AccountId = account.Value.id;
+         }
+         return rv;
+      }
+
+      public string abbreviate_algo(OneOf<OrderType, string> id) {
+         if (id is null) {
+            throw new ArgumentNullException(nameof(id));
+         }
+         if (id.value is string) {
+            return _algo_abbreviation_index.GetValueOrDefault(id) ??
+               id.value.ToString() ??
+               string.Empty;
+         }
+         if (id.value is OrderType t) {
+            return OrderDataHelper.abbreviate_algo_type(t);
+         }
+         return id.ToString() ?? string.Empty;
+      }
+
+      public OrderMetadata get_metadata(OrderRecordImmutable order) =>
+         get_metadata(order.Def.CustomDef.CustomCase);
+
+      public OrderMetadata get_metadata(OrderType order_type) =>
+         order_type switch {
+            OrderType.Market => new() { KeyBinding = "M", Color = BTSColors.Orders.market_order, NoProps = true },
+            OrderType.Limit => new() { KeyBinding = "L", Color = BTSColors.Orders.limit_order, NoProps = true },
+            OrderType.Stop => new() { KeyBinding = "S", Color = BTSColors.Orders.stop_order, NoProps = true },
+            OrderType.StopLimit => new() { KeyBinding = "SL", Color = BTSColors.Orders.stop_limit_order },
+            OrderType.GlbxIceberg => new() { KeyBinding = "GB", Color = BTSColors.Orders.glbx_iceberg_order },
+            OrderType.Iceberg => new() { KeyBinding = "I", Color = BTSColors.Orders.iceberg_order },
+            OrderType.TimeBox => new() { KeyBinding = "TB", Color = BTSColors.Orders.time_box_order },
+            OrderType.Tick => new() { KeyBinding = "T", Color = BTSColors.Orders.tick_order },
+            OrderType.Lean => new() { KeyBinding = "L", Color = BTSColors.Orders.lean_order },
+            OrderType.Join => new() { KeyBinding = "J", Color = BTSColors.Orders.join_order },
+            OrderType.Top => new() { KeyBinding = "D", Color = BTSColors.Orders.top_order },
+            OrderType.Cover => new() { KeyBinding = "C", Color = BTSColors.Orders.cover_order },
+            OrderType.Oco => new() { KeyBinding = "C", Color = BTSColors.Orders.oco_order },
+            OrderType.Pricer => new() { KeyBinding = "C", Color = BTSColors.Orders.pricer_order },
+            OrderType.Offset => new() { KeyBinding = "O", Color = BTSColors.Orders.off_order },
+            OrderType.Trigger => new() { KeyBinding = "T", Color = BTSColors.Orders.trigger_order },
+            OrderType.Spreader => new() { KeyBinding = "", Color = BTSColors.Orders.spreader_order },
+            OrderType.Dynamic => new() { KeyBinding = "", Color = BTSColors.Orders.top_order },
+            _ => _default_order_metadata
+         };
+
+      public SpreaderOrderDefinition spreader_order(
+         SecurityImmutable security, ExecutionStrategyImmutable strategy) =>
+         new() {
+            Security = security.CloneToMutable(),
+            ExecutionStrategy = strategy.CloneToMutable()
+         };
+
+      public SpreaderOrderDefinitionImmutable spreader_orderi(
+         SecurityImmutable security, ExecutionStrategyImmutable strategy) =>
+         new(spreader_order(security, strategy));
+
+      public void Dispose() =>
+         Disposable.dispose(ref _dispose);
+
+      static CustomOrderDefinition _custom_def() =>
+         new() { Limit = _limit_def() };
+
+      static CustomOrderDefinition _custom_def(
+         OrderType type) {
+         var rv = new CustomOrderDefinition();
+         switch (type) {
+            case OrderType.Market: rv.Market = _market_def(); break;
+            case OrderType.Limit: rv.Limit = _limit_def(); break;
+            case OrderType.Stop: rv.Stop = _stop_def(); break;
+            case OrderType.StopLimit: rv.StopLimit = _stop_limit_def(); break;
+            case OrderType.Iceberg: rv.Iceberg = _iceberg_def(); break;
+            case OrderType.GlbxIceberg: rv.GlbxIceberg = _glbx_iceberg_def(); break;
+            case OrderType.TimeBox: rv.TimeBox = _time_box_def(); break;
+            case OrderType.Tick: rv.Tick = _tick_def(); break;
+            case OrderType.Lean: rv.Lean = _lean_def(); break;
+            case OrderType.Join: rv.Join = _join_def(); break;
+            case OrderType.Top: rv.Top = _top_def(); break;
+            case OrderType.Cover: rv.Cover = _cover_def(); break;
+            case OrderType.Oco: rv.Oco = _oco_def(); break;
+            case OrderType.Pricer: rv.Pricer = _pricer_def(); break;
+            case OrderType.Offset: rv.Offset = _offset_def(); break;
+            case OrderType.Trigger: rv.Trigger = _trigger_def(); break;
+            case OrderType.Spreader: rv.Spreader = spreader_def(); break;
+            case OrderType.Dynamic: rv.Dynamic = _dynamic_def(); break;
+            default:
+               OrderDataHelper.setup_generic_custom_order(rv, type);
+               break;
+         }
+         return rv;
+      }
+
+      static MarketOrderDefinition _market_def() => new();
+      static LimitOrderDefinition _limit_def() => new();
+      static StopOrderDefinition _stop_def() => new();
+      static StopLimitOrderDefinition _stop_limit_def() => new();
+      static GLBXIcebergOrderDefinition _glbx_iceberg_def() => new() { MaxShowQuantity = 1 };
+      static IcebergOrderDefinition _iceberg_def() =>
+         new() { Child = _custom_def(), ShowQuantity = 1 };
+      static TimeBoxOrderDefinition _time_box_def() => new() {
+         Child = _custom_def(),
+         Start = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTime(DateTime.UtcNow),
+         Stop = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTime(DateTime.UtcNow)
+      };
+      static TickOrderDefinition _tick_def() => new() {
+         Child = _custom_def(),
+         CrossThreshold = new() { Absolute = 1 }
+      };
+      static LeanOrderDefinition _lean_def() => new() {
+         Child = _custom_def(),
+         CrossThreshold = new() { Absolute = 1 },
+         MaxChaseTicks = 2
+      };
+      static JoinOrderDefinition _join_def() => new() {
+         Child = _custom_def(),
+         JoinThreshold = new() { Absolute = 1 }
+      };
+      static TopOrderDefinition _top_def() => new() {
+         Child = _custom_def(),
+         TopThreshold = new() { Absolute = 1 }
+      };
+      static CoverOrderDefinition _cover_def() => new() {
+         Child = _custom_def(),
+         Hedge = _custom_def(),
+         HedgeTicks = 1
+      };
+      static OCOOrderDefinition _oco_def() => new() {
+         Child = _custom_def(),
+         Other = _custom_def(),
+         OtherSide = RelativeSide.Same
+      };
+      static PricerOrderDefinition _pricer_def() => new() { Child = _custom_def() };
+      static OffsetOrderDefinition _offset_def() => new() { Child = _custom_def(), };
+      static TriggerOrderDefinition _trigger_def() => new() {
+         Child = _custom_def(),
+         TriggerCondition = new TriggerCondition(),
+         TriggerAction = new TriggerAction(),
+      };
+      static DynamicOrderDefinition _dynamic_def() => new();
+   }
+}

src/client/spark/Orders/OrderTypeVMs/DefaultAlgoVM.cs

@@ -35,7 +35,7 @@ namespace mbts.spark {
       public IAlgoVM impl { get; private set; }
       public AlgoKey id { get; } = AlgoKey.from_known_type(AlgoType.None);
       public AlgoType order_type => _definition.value.CustomCase;
-      public OrderMetadata meta => OrderDataHelper.get_metadata(order_type);
+      public OrderMetadata meta => OrderMetadataStore.get_metadata(order_type);
       public bool can_expand => impl?.can_expand ?? false;
       public bool is_expanded {
          get => false;

src/client/spark/Orders/OrderTypeVMs/GenericOrderVM.cs

@@ -41,17 +41,6 @@ namespace mbts.spark {
             object vm = Activator.CreateInstance(makevm, key, algo_type, display_name, editing_permissions);
             return (IAlgoVM)vm;
          }
-         static public void setup_generic_custom_order(CustomOrderDefinition custom, CustomOrderDefinition.CustomOneofCase type) {
-            var prop = typeof(CustomOrderDefinition).GetProperty(type.ToString(),
-               BindingFlags.Instance | BindingFlags.Public);
-            var rt = prop.PropertyType;
-            var rtt = Activator.CreateInstance(rt);
-
-            var ci = rt.GetProperties().Where(x => x.PropertyType == typeof(CustomOrderDefinition));
-            foreach (var cid in ci)
-               cid.SetValue(rtt, new CustomOrderDefinition { Limit = new LimitOrderDefinition() });
-            prop.SetValue(custom, rtt);
-         }
       }
 
       class GenericOrderVM<TMutable, TImmutable> : OrderTypeVMBase<TMutable, TImmutable>

src/client/spark/Orders/OrderTypeVMs/OrderMetadataStore.cs

@@ -0,0 +1,32 @@
+namespace mbts.spark;
+
+using bts.api.orderman;
+using bts.spark;
+
+using mbts.ui;
+
+static class OrderMetadataStore {
+   public static OrderMetadata get_metadata(CustomOrderDefinition.CustomOneofCase order_type) =>
+      order_type switch {
+         CustomOrderDefinition.CustomOneofCase.Market => new() { KeyBinding = "M", Color = BTSColors.Orders.market_order, NoProps = true },
+         CustomOrderDefinition.CustomOneofCase.Limit => new() { KeyBinding = "L", Color = BTSColors.Orders.limit_order, NoProps = true },
+         CustomOrderDefinition.CustomOneofCase.Stop => new() { KeyBinding = "S", Color = BTSColors.Orders.stop_order, NoProps = true },
+         CustomOrderDefinition.CustomOneofCase.StopLimit => new() { KeyBinding = "SL", Color = BTSColors.Orders.stop_limit_order },
+         CustomOrderDefinition.CustomOneofCase.GlbxIceberg => new() { KeyBinding = "GB", Color = BTSColors.Orders.glbx_iceberg_order },
+         CustomOrderDefinition.CustomOneofCase.Iceberg => new() { KeyBinding = "I", Color = BTSColors.Orders.iceberg_order },
+         CustomOrderDefinition.CustomOneofCase.TimeBox => new() { KeyBinding = "TB", Color = BTSColors.Orders.time_box_order },
+         CustomOrderDefinition.CustomOneofCase.Tick => new() { KeyBinding = "T", Color = BTSColors.Orders.tick_order },
+         CustomOrderDefinition.CustomOneofCase.Lean => new() { KeyBinding = "L", Color = BTSColors.Orders.lean_order },
+         CustomOrderDefinition.CustomOneofCase.Join => new() { KeyBinding = "J", Color = BTSColors.Orders.join_order },
+         CustomOrderDefinition.CustomOneofCase.Top => new() { KeyBinding = "D", Color = BTSColors.Orders.top_order },
+         CustomOrderDefinition.CustomOneofCase.Cover => new() { KeyBinding = "C", Color = BTSColors.Orders.cover_order },
+         CustomOrderDefinition.CustomOneofCase.Oco => new() { KeyBinding = "C", Color = BTSColors.Orders.oco_order },
+         CustomOrderDefinition.CustomOneofCase.Pricer => new() { KeyBinding = "C", Color = BTSColors.Orders.pricer_order },
+         CustomOrderDefinition.CustomOneofCase.Offset => new() { KeyBinding = "O", Color = BTSColors.Orders.off_order },
+         CustomOrderDefinition.CustomOneofCase.Trigger => new() { KeyBinding = "T", Color = BTSColors.Orders.trigger_order },
+         CustomOrderDefinition.CustomOneofCase.Spreader => new() { KeyBinding = "", Color = BTSColors.Orders.spreader_order },
+         CustomOrderDefinition.CustomOneofCase.Dynamic => new() { KeyBinding = "", Color = BTSColors.Orders.dynamic_order },
+         _ => new OrderMetadata { KeyBinding = "", Color = BTSColors.Orders.default_order },
+      };
+
+}

src/client/spark/Orders/OrderTypeVMs/OrderTypeVMFactory.cs

@@ -201,7 +201,7 @@ namespace mbts.spark {
          public virtual string abbreviation => model == null ? display_name : model.factory.abbreviate_algo(new OneOf<AlgoType, string>(order_type));
          public virtual AlgoDisplayGroup group => order_type.is_bts_algo_type() ? AlgoDisplayGroup.BTS : AlgoDisplayGroup.Exchange;
          public OrdersModel orders_model => model;
-         public virtual OrderMetadata meta => OrderDataHelper.get_metadata(order_type);
+         public virtual OrderMetadata meta => OrderMetadataStore.get_metadata(order_type);
          // child algos
          [EditorBrowsable(EditorBrowsableState.Never)]
          public dynamic children_vm {

src/client/spark_api/MarketData/BookTuple.cs

@@ -36,7 +36,7 @@ namespace Spark.Api.MarketData {
       public override int GetHashCode() =>
          security.Id.GetHashCode();
       public static Object ToJsonObj(BookTuple obj) => new {
-         secuirty = obj.security.ToString(),
+         security = obj.security.ToString(),
          book = obj.book.ToJsonObj()
       };
       public static bool operator ==(BookTuple left, BookTuple right) =>

src/client/spark_api/MarketData/Co2NativeMethods.cs

@@ -67,7 +67,10 @@ namespace mbts.co2 {
       [DllImport("libspark")]
       internal unsafe static extern int md_consumer_read_trades(
          IntPtr consumer, UserSymbol channel, ref long cursor, raw_receive_pb cb);
-
+      [DllImport("libspark")]
+      internal unsafe static extern int md_consumer_read_trades_linux(
+         IntPtr consumer, string symbol, int venue, int type, ref long cursor, raw_receive_pb cb);
+         
       [DllImport("libspark")]
       internal unsafe static extern int md_consumer_read_rfqs(
          IntPtr consumer, UserSymbol channel, ref long cursor, raw_receive_pb cb);

src/client/spark_api/MarketData/MDConsumerChannel.cs

@@ -23,7 +23,7 @@ namespace mbts.co2 {
 
       public unsafe IList<TradeMessage> read_trades(ref long cursor) {
          IList<TradeMessage> trades = Array.Empty<TradeMessage>();
-         Co2NativeMethods.md_consumer_read_trades(_consumer.Handle, _symbol, ref cursor, callback);
+         Co2NativeMethods.md_consumer_read_trades_linux(_consumer.Handle, _symbol.symbol,(int)_symbol.exchange,(int)_symbol.type , ref cursor, callback);
          return trades;
 
          void callback(byte* buffer, ulong size) =>

src/client/spark_api/MarketData/TradeInfo.cs

@@ -28,7 +28,7 @@ namespace mbts.spark {
       public TickDisplaySettingsImmutable qty_display => security_md.quantity_display;
 
       public static System.Object ToJsonObj(TradeInfo obj) => new {
-         secuirty = obj.security.Exchange.ToString() + "." + obj.security.ToString(),
+         security = obj.security.Exchange.ToString() + "." + obj.security.ToString(),
          price = obj.trade.price(),
          timestamp = obj.trade.Time,
          quantity = obj.volume,

src/client/spark_api/Orders/OrderDataHelper.cs

@@ -1,5 +1,4 @@
-using bts;
-using bts.api;
+using bts.api;
 using bts.api.core;
 using bts.api.orderman;
 using bts.collections;
@@ -10,23 +9,19 @@ using Google.Protobuf;
 using Google.Protobuf.Reflection;
 using Google.Protobuf.WellKnownTypes;
 
-using mbts.ui;
-using mbts.ui.utilities;
 using mbts.ui.utility;
 using mbts.utils;
 
-using Spark.Core.Disposables;
-
 using System;
 using System.Collections.Generic;
-using System.Composition;
 using System.Linq;
 using System.Reactive.Linq;
+using System.Reflection;
 
 using OrderState = bts.api.orderman.OrderState;
 
 namespace mbts.spark.utilities {
-   struct ChildAlgoData {
+   public readonly struct ChildAlgoData {
       public readonly string name;
       public readonly Func<IMessage, CustomOrderDefinition> get_child_from_impl;
       public readonly Action<IMessage, CustomOrderDefinition> assign_child_to_impl;
@@ -48,27 +43,29 @@ namespace mbts.spark.utilities {
       }
    }
 
-   static class OrderDataHelper {
-      class AlgoTypeCaseDescriptorData {
-         public readonly IReadOnlyList<ChildAlgoData> child_algo_data;
-         public readonly IReadOnlyList<FieldDescriptor> value_descriptors;
-         public readonly FieldDescriptor type_case_descriptor;
+   public class AlgoTypeCaseDescriptorData {
+      public readonly IReadOnlyList<ChildAlgoData> child_algo_data;
+      public readonly IReadOnlyList<FieldDescriptor> value_descriptors;
+      public readonly FieldDescriptor type_case_descriptor;
 
-         public AlgoTypeCaseDescriptorData(FieldDescriptor type_case_descriptor, IReadOnlyList<ChildAlgoData> child_algo_data, IReadOnlyList<FieldDescriptor> value_descriptors) {
-            this.type_case_descriptor = type_case_descriptor;
-            this.child_algo_data = child_algo_data;
-            this.value_descriptors = value_descriptors;
-         }
+      public AlgoTypeCaseDescriptorData(
+         FieldDescriptor type_case_descriptor,
+         IReadOnlyList<ChildAlgoData> child_algo_data,
+         IReadOnlyList<FieldDescriptor> value_descriptors
+      ) {
+         this.type_case_descriptor = type_case_descriptor;
+         this.child_algo_data = child_algo_data;
+         this.value_descriptors = value_descriptors;
       }
+   }
 
+   public static class OrderDataHelper {
       const string _primary_child_field = "child";
       // algo-type-case maps
       // Note: obv this will break if we change the name of the 'CustomOneofCase' field
       static readonly OneofDescriptor _oneof_desc = CustomOrderDefinition.Descriptor.Oneofs.FirstOrDefault(fd => fd.Name == "custom");
       static readonly IReadOnlyDictionary<System.Type, CustomOrderDefinition.CustomOneofCase> _algo_type_case_index;
       static readonly IReadOnlyDictionary<CustomOrderDefinition.CustomOneofCase, AlgoTypeCaseDescriptorData> _algo_type_case_descriptor_data;
-      // 
-      static readonly OrderMetadata _default_order_metadata = new OrderMetadata { KeyBinding = "", Color = BTSColors.Orders.default_order };
 
       static OrderDataHelper() {
          var algo_type_case_index = new Dictionary<System.Type, CustomOrderDefinition.CustomOneofCase>();
@@ -106,6 +103,12 @@ namespace mbts.spark.utilities {
       }
 
       #region Algo Structure/Hierarchy
+      public static AlgoTypeCaseDescriptorData get_algo_type_descriptor(
+         CustomOrderDefinition.CustomOneofCase algo_type) =>
+         _algo_type_case_descriptor_data.TryGetValue(algo_type, out var d) ? d :
+            throw new ArgumentException(
+               $"Failed to find descriptor data for algo type '{algo_type}'", nameof(algo_type));
+
       public static IEnumerable<(CustomOrderDefinition.CustomOneofCase type, string dyn, uint studio, uint spreader_sec)> enumerate_algo_structure(CustomOrderDefinitionImmutable custom_defi) {
          string dyn = null;
          uint studio = 0, spreader_sec = 0;
@@ -205,29 +208,6 @@ namespace mbts.spark.utilities {
          _algo_type_case_descriptor_data.TryGetValue(algo_type, out AlgoTypeCaseDescriptorData d) ? d.child_algo_data : Array.Empty<ChildAlgoData>();
       #endregion
 
-      public static OrderMetadata get_metadata(CustomOrderDefinition.CustomOneofCase order_type) =>
-         order_type switch {
-            CustomOrderDefinition.CustomOneofCase.Market => new() { KeyBinding = "M", Color = BTSColors.Orders.market_order, NoProps = true },
-            CustomOrderDefinition.CustomOneofCase.Limit => new() { KeyBinding = "L", Color = BTSColors.Orders.limit_order, NoProps = true },
-            CustomOrderDefinition.CustomOneofCase.Stop => new() { KeyBinding = "S", Color = BTSColors.Orders.stop_order, NoProps = true },
-            CustomOrderDefinition.CustomOneofCase.StopLimit => new() { KeyBinding = "SL", Color = BTSColors.Orders.stop_limit_order },
-            CustomOrderDefinition.CustomOneofCase.GlbxIceberg => new() { KeyBinding = "GB", Color = BTSColors.Orders.glbx_iceberg_order },
-            CustomOrderDefinition.CustomOneofCase.Iceberg => new() { KeyBinding = "I", Color = BTSColors.Orders.iceberg_order },
-            CustomOrderDefinition.CustomOneofCase.TimeBox => new() { KeyBinding = "TB", Color = BTSColors.Orders.time_box_order },
-            CustomOrderDefinition.CustomOneofCase.Tick => new() { KeyBinding = "T", Color = BTSColors.Orders.tick_order },
-            CustomOrderDefinition.CustomOneofCase.Lean => new() { KeyBinding = "L", Color = BTSColors.Orders.lean_order },
-            CustomOrderDefinition.CustomOneofCase.Join => new() { KeyBinding = "J", Color = BTSColors.Orders.join_order },
-            CustomOrderDefinition.CustomOneofCase.Top => new() { KeyBinding = "D", Color = BTSColors.Orders.top_order },
-            CustomOrderDefinition.CustomOneofCase.Cover => new() { KeyBinding = "C", Color = BTSColors.Orders.cover_order },
-            CustomOrderDefinition.CustomOneofCase.Oco => new() { KeyBinding = "C", Color = BTSColors.Orders.oco_order },
-            CustomOrderDefinition.CustomOneofCase.Pricer => new() { KeyBinding = "C", Color = BTSColors.Orders.pricer_order },
-            CustomOrderDefinition.CustomOneofCase.Offset => new() { KeyBinding = "O", Color = BTSColors.Orders.off_order },
-            CustomOrderDefinition.CustomOneofCase.Trigger => new() { KeyBinding = "T", Color = BTSColors.Orders.trigger_order },
-            CustomOrderDefinition.CustomOneofCase.Spreader => new() { KeyBinding = "", Color = BTSColors.Orders.spreader_order },
-            CustomOrderDefinition.CustomOneofCase.Dynamic => new() { KeyBinding = "", Color = BTSColors.Orders.dynamic_order },
-            _ => _default_order_metadata,
-         };
-
       public static string abbreviate_algo_type(CustomOrderDefinition.CustomOneofCase algo_type) {
          switch (algo_type) {
             case CustomOrderDefinition.CustomOneofCase.Limit: return "LM";
@@ -331,6 +311,19 @@ namespace mbts.spark.utilities {
          order_type_def != null && _algo_type_case_index.TryGetValue(order_type_def.Descriptor.ClrType, out CustomOrderDefinition.CustomOneofCase algo_case) ?
             algo_case : CustomOrderDefinition.CustomOneofCase.None;
 
+      public static void setup_generic_custom_order(
+         CustomOrderDefinition custom, CustomOrderDefinition.CustomOneofCase type) {
+         var prop = typeof(CustomOrderDefinition).GetProperty(type.ToString(),
+            BindingFlags.Instance | BindingFlags.Public);
+         var rt = prop.PropertyType;
+         var rtt = Activator.CreateInstance(rt);
+
+         var ci = rt.GetProperties().Where(x => x.PropertyType == typeof(CustomOrderDefinition));
+         foreach (var cid in ci)
+            cid.SetValue(rtt, new CustomOrderDefinition { Limit = new LimitOrderDefinition() });
+         prop.SetValue(custom, rtt);
+      }
+
       #region Private Members
       // get child
       static CustomOrderDefinitionImmutable? _get_iceberg_child(IcebergOrderDefinitionImmutable iceberg) => iceberg.HasChild ? iceberg.Child : (CustomOrderDefinitionImmutable?)null;
@@ -655,7 +648,7 @@ namespace mbts.spark.utilities {
                case bts.api.orderman.CustomOrderDefinition.CustomOneofCase.Spreader: rv.Spreader = SpreaderOrderDefinition(); break;
                case bts.api.orderman.CustomOrderDefinition.CustomOneofCase.Dynamic: rv.Dynamic = DynamicOrderDefinition(); break;
                default:
-                  OrderTypeVMFactory.GenericOrderFactory.setup_generic_custom_order(rv, order_type);
+                  setup_generic_custom_order(rv, order_type);
                   break;
             }
             return rv;

src/client/spark_api/Orders/OrdersClient.cs

@@ -1,6 +1,8 @@
 #nullable enable
 namespace Spark.Api.Orders {
    using bts;
+   using bts.api;
+
    using bts.api.core;
    using bts.api.orderman;
    using bts.api.position;
@@ -14,6 +16,8 @@ namespace Spark.Api.Orders {
    using Spark.Core.Reactive;
 
    using System;
+   using System.Collections.Generic;
+
    using System.Reactive.Concurrency;
    using System.Reactive.Linq;
    using System.Threading.Tasks;
@@ -55,7 +59,7 @@ namespace Spark.Api.Orders {
                o => o.order.ParentKey,
                (o, r) => new OrderRecordTupleWithRoot(o, r),
                o => new OrderRecordTupleWithRoot(o),
-               o => o.root.order.Key)
+               o => o.key)
                .Cache()
                .ToObservable());
 
@@ -127,11 +131,29 @@ namespace Spark.Api.Orders {
             });
             return r.value;
          } catch {
-            var r = new Response();
-            r.ErrorCode = OrderError.Undefined;
+            var r = new Response {
+               ErrorCode = OrderError.Undefined
+            };
             return r;
          }
       }
+      public async Task<Response> TrySetPrecomputedTheosAsync(
+         List<PrecomputedTheoPoint> ptp_list
+      ) {
+         bts.api.orderman.PrecomputedTheos server_theos = new();
+         server_theos.Theos.AddRange(ptp_list);
+         var r = await _order_service.set_precomputed_theos_async(server_theos);
+         var r2 = new Response {
+            ErrorCode = OrderError.Ok,
+            Msg = r.ToString()
+         };
+         return r2;
+      }
+
+      public async Task<RegisteredDynamicOrderTypes> GetRegisteredDynamicOrderTypesAsync(){
+         Result<RegisteredDynamicOrderTypes> r = await _order_service.get_registered_dynamic_order_types_async(new bts.api.VoidMessage());
+         return r.value;
+      }
       public async Task<Response> TryUpdateOrderAsync(
          CustomOrderDefinition custom,
          double price, double quantity, uint order_id
@@ -146,8 +168,9 @@ namespace Spark.Api.Orders {
             });
             return r.value;
          } catch {
-            var r = new Response();
-            r.ErrorCode = OrderError.Undefined;
+            var r = new Response {
+               ErrorCode = OrderError.Undefined
+            };
             return r;
          }
       }
@@ -159,8 +182,9 @@ namespace Spark.Api.Orders {
             });
             return r.value;
          } catch {
-            var r = new Response();
-            r.ErrorCode = OrderError.Undefined;
+            var r = new Response {
+               ErrorCode = OrderError.Undefined
+            };
             return r;
          }
       }

src/client/spark_api_server/Program.cs

@@ -44,7 +44,7 @@
             KeepAliveIntervalSecs = 0.1,
          };
          gwy.init(sync);
-         sync.init(gwy);
+         await sync.initAsync(gwy);
          var errors = Errors.None;
          Log.Api.info("Starting websocket server...");
          // try {

src/client/spark_websocket/Core/WebsocketDataSync.cs

@@ -151,6 +151,15 @@ namespace bts.websocket_core {
             _orders_client = value;
          }
       }
+      
+      IReadOnlyDictionary<string, DynamicOrderDescriptor>? _dynamic_descriptors;
+      IReadOnlyDictionary<string, DynamicOrderDescriptor> Dynamic_Descriptors{
+         get => Log.notnull<IReadOnlyDictionary<string, DynamicOrderDescriptor>>(_dynamic_descriptors);
+         set {
+            Log.isnull<IReadOnlyDictionary<string, DynamicOrderDescriptor>>(_dynamic_descriptors);
+            _dynamic_descriptors=value.ToDictionary(k=>k.Key.ToLower(),v => v.Value);
+         }
+      }
       WebsocketConnectionGateway? gateway;
       WebsocketConnectionGateway Gateway {
          get => Log.notnull<WebsocketConnectionGateway>(gateway);
@@ -178,10 +187,10 @@ namespace bts.websocket_core {
       /// Should be called in the start, stores pointer for gtwy and creates dictionary for clients, starts broadcast thread
       /// </summary>
       /// <param name="gtwy"></param>
-      public void init(WebsocketConnectionGateway gtwy) {
+      public async Task initAsync(WebsocketConnectionGateway gtwy) {
          Gateway = gtwy;
          clients = new ClientsStore();
-         TimeSpan md_throttle = new TimeSpan(10000);
+         TimeSpan md_throttle = TimeSpan.FromSeconds(0.1);
          MD_Source = new MDSource(md_throttle);
          Md_Client = new(MD_Source);
          _securities_sub = MD_Source.subscribe_securities().Subscribe(sec_map => _securities = sec_map);
@@ -197,6 +206,10 @@ namespace bts.websocket_core {
          var sec_table_src = MD_Source.subscribe_securities()
             .Table(s => s.security.Id)
             .Select(s => s.security, s => s.Id);
+         var r  = await Orders_Client.GetRegisteredDynamicOrderTypesAsync();
+         Dynamic_Descriptors = r
+         .DynamicOrderDesc
+         .ToDictionary(d => d.DynamicOrderType);
          Synth_Sec_Client = new(synth_sec_service, sec_table_src);
 
 
@@ -270,6 +283,10 @@ namespace bts.websocket_core {
                   handle_update_order(req);
                   break;
                }
+            case WebsocketRequest.REQUEST_TYPES.SET_PRECOMPUTED_THEOS: {
+                  handle_set_theos(req);
+                  break;
+               }
             case WebsocketRequest.REQUEST_TYPES.UNKOWN: {
                   ///send error
                   break;
@@ -284,30 +301,21 @@ namespace bts.websocket_core {
             System.Console.WriteLine("error for req", client_id, ex.ToString());
          }
       }
-      public async void handle_delete_order(WebsocketRequest req) {
-         (bool suc, string status) = await WebsocketRestHandler.DeleteOrderAsync(req, Orders_Client);
-         if (!suc) {
+      public async Task HandleRestRequest(WebsocketRequest req, Func<WebsocketRequest, OrdersClient, Task<(bool success, uint? id, string status)>> handlerAsync) {
+         (bool success, uint? id, string status) = await handlerAsync(req, Orders_Client);
+         if (!success){
             handle_incorrect_request(req, status);
             return;
          }
-         handle_correct_request(req.id, status, null, req.client_id);
-      }
-      public async void handle_enter_order(WebsocketRequest req) {
-         (bool suc, uint? id, string status) = await WebsocketRestHandler.EnterOrderAsync(req, Orders_Client, _securities);
-         if (!suc) {
-            handle_incorrect_request(req, status);
-            return;
-         }
-         handle_correct_request(req.id, status, new { order_key = id }, req.client_id);
-      }
-      public async void handle_update_order(WebsocketRequest req) {
-         (bool suc, string status) = await WebsocketRestHandler.UpdateOrderAsync(req, Orders_Client);
-         if (!suc) {
-            handle_incorrect_request(req, status);
-            return;
-         }
-         handle_correct_request(req.id, status, null, req.client_id);
+         handle_correct_request(req.id, status, id != null ? new { order_key = id } : null, req.client_id);
       }
+      public async void handle_delete_order(WebsocketRequest req) { await HandleRestRequest(req,WebsocketRestHandler.DeleteOrderAsync);} 
+      public async void handle_set_theos(WebsocketRequest req) { await HandleRestRequest(req,(req, Orders_Client) => WebsocketRestHandler.SetTheosAsync(req, Orders_Client, _securities));}
+      public async void handle_enter_order(WebsocketRequest req) { await HandleRestRequest(req,(req, Orders_Client) => WebsocketRestHandler.EnterOrderAsync(
+         req, Orders_Client, _securities,Dynamic_Descriptors  
+      ));} 
+      public async void handle_update_order(WebsocketRequest req) { await HandleRestRequest(req,(req, Orders_Client) => WebsocketRestHandler.UpdateOrderAsync(req,Orders_Client,Dynamic_Descriptors));}
+      
       public void handle_subscription(WebsocketRequest req) {
          WebsocketSubscriptionHandler? hdlr;
          string status;

src/client/spark_websocket/Core/WebsocketJsonParsers.cs

@@ -1,6 +1,10 @@
 namespace bts.websocket_core {
    using bts.api;
    using bts.api.core;
+   using bts.api.orderman;
+
+   using bts.md;
+
 
    using Newtonsoft.Json;
    using Newtonsoft.Json.Linq;
@@ -27,9 +31,67 @@ namespace bts.websocket_core {
       public static Object to_json<T>(IList<T> list)
           where T : IJsonFormatter<T> =>
           to_json(list, T.ToJsonObj);
-      public struct ListWrapper : IEnumerable<string> {
+      public struct PrecomputedTheoPointWrapper {
+         UInt32 SecId{get;init;}
+         public double driver{get;init;}
+         UInt32 DriverSecId{get;init;}
+         public double theo{get;init;}
+         public double delta{get;init;}
+         public double gamma{get;init;}
+         public double vega{get;init;}
+         public double theta{get;init;}
+         public double rho{get;init;}
+         public double vol{get;init;}
+         public Venue venue{get;init;}
+         public String exchange_security_name{get;init;}
+         public String driver_exchange_security_name{get;init;}
+         public static bool GetTheoPoint(PrecomputedTheoPointWrapper inp, [NotNullWhen(true)] out PrecomputedTheoPoint? p, IReadOnlyDictionary<uint, mbts.SecurityMD> _securites){
+            p = null;
+            if (!_securites.Values.TryGetFirst(sc_md =>
+            (
+               sc_md.security.Exchange == inp.venue &&
+               sc_md.security.ToString() == inp.exchange_security_name),
+            out mbts.SecurityMD sec
+            )) {
+            return false;
+            }
+            if (!_securites.Values.TryGetFirst(sc_md =>
+            (
+               sc_md.security.Exchange == inp.venue &&
+               sc_md.security.ToString() == inp.driver_exchange_security_name),
+            out mbts.SecurityMD driver_sec
+            )) {
+            return false;
+            }
+            p = new(){
+               SecId = sec.id,
+               Driver=inp.driver,
+               DriverSecId=driver_sec.id,
+               Theo=inp.theo,
+               Delta=inp.delta,
+               Gamma=inp.gamma,
+               Vega=inp.vega,
+               Theta=inp.theta,
+               Rho=inp.rho,
+               Vol=inp.vol,
+            };
+            return true;
+
+         }
+         public static bool TryParse(string input, [NotNullWhen(true)] out PrecomputedTheoPointWrapper? P) {
+            P = null;
+            try {
+               var temp = JsonConvert.DeserializeObject<PrecomputedTheoPointWrapper>(input);
+               P = temp;
+               return true;
+            } catch {
+               return false;
+            }
+         }
+      }
+      public struct StringListWrapper : IEnumerable<string> {
          public List<string> Lis { get; set; }
-         public ListWrapper(List<string> l) {
+         public StringListWrapper(List<string> l) {
             Lis = l;
          }
 
@@ -49,14 +111,24 @@ namespace bts.websocket_core {
          public string[] getlwrarr() {
             return Lis.Select(v => v.ToLower()).ToArray();
          }
-         public static bool TryParse(string input, [NotNullWhen(true)] out ListWrapper? l) {
+         public static bool TryParse(string input, [NotNullWhen(true)] out StringListWrapper? l) {
             l = null;
             try {
-               var temp = JsonConvert.DeserializeObject<List<string>>(input);
+               var temp = JsonConvert.DeserializeObject<List<Object>>(input);
                if (temp == null) {
                   return false;
                }
-               l = new ListWrapper(temp);
+                List<string> stringList = new List<string>();
+
+               foreach (object obj in temp)
+               {
+                  string? val = obj.ToString();
+                  if(val==null){
+                     return false;
+                  }
+                  stringList.Add(val);
+               }
+               l = new StringListWrapper(stringList);
                return true;
             } catch {
                return false;
@@ -71,26 +143,33 @@ namespace bts.websocket_core {
 
       }
       public static bool TryParse<T>(JObject filter, string key, out T[]? list, out string error) where T : struct {
-         T[] values = new T[] { };
          list = null;
-         ListWrapper? wrap;
-         if (TryParse<ListWrapper>(filter, key, out wrap, out error)) {
+         if (TryParse<StringListWrapper>(filter, key, out StringListWrapper? wrap, out error)) {
             if (wrap != null) {
+                var values = new List<T>();
                foreach (var item in wrap) {
-                  T? obj;
-                  if (TryParse<T>(item, key, out obj, out error) && obj != null) {
-                     values = values.ToList().Append<T>((T)obj).ToArray();
+                  if (TryParse<T>(item, key, out T? obj, out error) && obj != null) {
+                     values.Add(obj.Value);
                   } else {
                      return false;
                   }
                }
-               list = values;
+               list = values.ToArray();
             }
             return true;
          }
          return false;
       }
+      public static bool TryParse<T>(JObject filter, string key, out T? obj, [NotNullWhen(false)] out string error) where T : struct {
+         obj = null;
+         var value = filter.GetValueOrDefault(key)?.ToString();
+         if (value == null) {
+            error = $"{key} not present";
+            return true;
+         }
+         return TryParse<T>(value, key, out obj, out error);
 
+      }
       public static bool TryParse<T>(string value, string key, [NotNullWhen(true)] out T? obj, [NotNull] out string error) where T : struct {
          error = "";
          if (typeof(T).IsEnum && Enum.TryParse<T>(value, true, out var result)) {
@@ -100,7 +179,10 @@ namespace bts.websocket_core {
          if (typeof(T) == typeof(double) && double.TryParse(value, out var d)) {
             obj = (T)(object)d;
             return true;
-
+         }
+         if (typeof(T) == typeof(bool) && bool.TryParse(value, out var b)) {
+            obj = (T)(object)b;
+            return true;
          }
          if (typeof(T) == typeof(uint) && uint.TryParse(value, out var uin)) {
             obj = (T)(object)uin;
@@ -118,10 +200,14 @@ namespace bts.websocket_core {
             obj = (T)(object)new StringWrapper(value);
             return true;
          }
-         if (typeof(T) == typeof(ListWrapper) && ListWrapper.TryParse(value, out var l)) {
+         if (typeof(T) == typeof(StringListWrapper) && StringListWrapper.TryParse(value, out var l)) {
             obj = (T)(object)l;
             return true;
          }
+         if (typeof(T)==typeof(PrecomputedTheoPointWrapper) && PrecomputedTheoPointWrapper.TryParse(value, out var p)){
+            obj=(T)(object)p;
+            return true;
+         }
          if (typeof(T) == typeof(DateTime) && DateTime.TryParseExact(value, "yyyy/MM/dd", CultureInfo.InvariantCulture,
                             DateTimeStyles.None, out var dt)) {
             obj = (T)(object)dt;
@@ -132,16 +218,7 @@ namespace bts.websocket_core {
          obj = null;
          return false;
       }
-      public static bool TryParse<T>(JObject filter, string key, out T? obj, [NotNullWhen(false)] out string error) where T : struct {
-         obj = null;
-         var value = filter.GetValueOrDefault(key)?.ToString();
-         if (value == null) {
-            error = $"{key} not present";
-            return true;
-         }
-         return TryParse<T>(value, key, out obj, out error);
-
-      }
+      
 
    }
 }

src/client/spark_websocket/Core/WebsocketRequest.cs

@@ -12,8 +12,8 @@ namespace bts.websocket_core {
          UNSUBSCRIBE,
          ENTER_ORDER,
          DELETE_ORDER,
-
          UPDATE_ORDER,
+         SET_PRECOMPUTED_THEOS,
          DELETE,
          PATCH,
          UNKOWN = 0

src/client/spark_websocket/Core/WebsocketRestHandler.cs

@@ -6,109 +6,352 @@ namespace bts.websocket_core {
 
    using mbts;
 
+   using Microsoft.VisualBasic;
+
+   using mbts.spark;
    using Newtonsoft.Json.Linq;
 
    using Spark.Api.Orders;
 
    using System;
+   using System.Diagnostics.CodeAnalysis;
+   using bts.spark;
+   using bts.utils;
 
    public class WebsocketRestHandler {
+   public static (bool,DynamicOrderParam?,string) TryGetDynamicParam(JObject custom,string key, DynamicOrderParamDescriptor descriptor, IReadOnlyDictionary<string, DynamicOrderDescriptor> Dynamic_Descriptors) {
+         var rv = new DynamicOrderParam();
+         string error = "";
+         switch (descriptor.ParamType.DescriptorCase) {
+            case DynamicOrderTypeDescriptor.DescriptorOneofCase.Array:
+               if (custom[key] is not JArray arr_obj) {
+                  return (false, null, key + " not correct");
+               }
+               rv.AsArray = new DynamicOrderArray();
+               foreach (var p in arr_obj.ToList()){
+                  var child_key = key+"_child";
+                  DynamicOrderParamDescriptor child_descriptor=new() {
+                     ParamName = child_key,
+                     ParamType = descriptor.ParamType.Array.ElementType
+                  };
+
+                  JObject child = new() {
+                     [child_key] = p
+                  };
+
+                  (bool suc, DynamicOrderParam? param, error) = TryGetDynamicParam(child,child_key,child_descriptor,Dynamic_Descriptors);
+                  if(!suc){
+                     return (suc,null,error);
+                  }
+                  rv.AsArray.Element.Add(param);
+               }
+               return (true,rv,error);
+            case DynamicOrderTypeDescriptor.DescriptorOneofCase.Enumerated:
+               return (false,null,key+"Enumerated DynamicOrderTypeDescriptors not enabled on api");
+            case DynamicOrderTypeDescriptor.DescriptorOneofCase.Primitive:
+               switch (descriptor.ParamType.Primitive) {
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveBool:
+                     if (!WebsocketJsonParsers.TryParse<bool>(custom, key, out bool? b, out error) || b == null) {
+                        return (false, null, error);
+                     }
+                     rv.AsBool = (bool)b; 
+                     return (true,rv,error);
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveDouble:
+                     if (!WebsocketJsonParsers.TryParse<double>(custom, key, out double? d, out error) || d == null) {
+                        return (false, null, error);
+                     }
+                     rv.AsDouble = (double)d; 
+                     return (true,rv,error);
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveInt: 
+                  if (!WebsocketJsonParsers.TryParse<int>(custom, key, out int? i, out error) || i == null) {
+                        return (false, null, error);
+                     }
+                     rv.AsInt32 = (int)i; 
+                     return (true,rv,error);
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveOrderDef: 
+                     if (custom[key] is not JObject child_custom) {
+                        return (false, null, key + " not correct");
+                     }
+                     (bool suc,var child_custom_def,error) = TryGetCustomDef(child_custom,Dynamic_Descriptors);
+                  if(!suc || child_custom_def==null){
+                     return (false,null,error);
+                  }
+                  rv.AsOrderDef = child_custom_def;
+                  return(true,rv,error);
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveString: 
+                  if (!WebsocketJsonParsers.TryParse<WebsocketJsonParsers.StringWrapper>(custom,key,out WebsocketJsonParsers.StringWrapper? s,out error)|| s == null) {
+                     return (false, null, error);
+                  }
+                  rv.AsString = s.Value.Val; 
+                  return (true,rv,error);
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveSecurityKey:
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveUint: 
+                  if (!WebsocketJsonParsers.TryParse<uint>(custom, key, out uint? u, out error) || u == null) {
+                        return (false, null, error);
+                     }
+                     rv.AsUint32 = (uint)u; 
+                     return (true,rv,error);
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveQuantity:
+                  if (WebsocketJsonParsers.TryParse<uint>(custom, key, out uint? q, out error) && q != null) {
+                        rv.AsQuantity = new QuantityMeasure() {
+                           Absolute = (uint)q
+                        };
+                        return (true, rv, error);
+                  } else if (WebsocketJsonParsers.TryParse<double>(custom, key, out double? qd, out error) && qd != null){
+                     rv.AsQuantity = new QuantityMeasure() {
+                           Ratio = (double)qd
+                        };
+                        return (true, rv, error);
+                  }
+                  else{
+                     return (false,null,key+" not correct");
+                  }
+               }
+               return (false,null,key+" not correct");
+            case DynamicOrderTypeDescriptor.DescriptorOneofCase.Structure:
+                if (custom[key] is not JObject obj) {
+                  return (false, null, key + " not correct");
+               }
+               rv.AsStruct = new DynamicOrderStruct();
+               
+               foreach (var prop in obj.Properties()) {
+                  var child_key = prop.Name;
+                  var check = descriptor.ParamType.Structure.Field.TryGetFirst(v => v?.ParamName == child_key,out DynamicOrderParamDescriptor? p);
+                  if(!check || p == null){
+                     return (false,null,key+" "+child_key+" not correct");
+                  }
+                  var child_descriptor = new DynamicOrderParamDescriptor {
+                        ParamName = child_key,
+                        ParamType = p.ParamType
+                  };
+
+                  (bool success, DynamicOrderParam? param, error) = TryGetDynamicParam(obj, child_key, child_descriptor, Dynamic_Descriptors);
+                  if (!success) {
+                        return (success, null, error);
+                  }
+                  rv.AsStruct.Field[child_key] = param;
+               }
+               return (true,rv,error);
+         }
+         return (false,null,key+" not correct");
+      }
       readonly static Lazy<AdminService> _admin = new Lazy<AdminService>(() => Locator.locate<AdminService>(), true);
-      private static (bool, CustomOrderDefinition?, string) TryGetCustomDef(WebsocketRequest req) {
-         string status = "";
+      
+      private static (bool, CustomOrderDefinition?, string) TryGetCustomDef(WebsocketRequest req, IReadOnlyDictionary<string, DynamicOrderDescriptor> Dynamic_Descriptors){
+         if (req.parameters["options"] is not JObject custom) {
+            return (false, null, "options not correct");
+         }
+         return TryGetCustomDef(custom,Dynamic_Descriptors);
+      }
+      private static (bool, CustomOrderDefinition?, string) TryGetCustomDef(JObject custom, IReadOnlyDictionary<string, DynamicOrderDescriptor> Dynamic_Descriptors) {
          CustomOrderDefinition customdef;
-         JObject? custom = req.parameters["custom"] as JObject;
-         if (!WebsocketJsonParsers.TryParse<CustomOrderDefinition.CustomOneofCase>(
-            req.parameters,
-            "order_type",
-            out CustomOrderDefinition.CustomOneofCase? order_type,
-            out status)
-            || order_type == null) {
+
+         if (!WebsocketJsonParsers.TryParse<CustomOrderDefinition.CustomOneofCase>(custom, "order_type", out CustomOrderDefinition.CustomOneofCase? order_type, out string status) || order_type == null) {
             return (false, null, status);
          }
          switch (order_type) {
+            case CustomOrderDefinition.CustomOneofCase.Stop:{
+               customdef = new CustomOrderDefinition() {
+                     Stop = new StopOrderDefinition()
+                  };
+                  break;
+            }
+            case CustomOrderDefinition.CustomOneofCase.StopLimit:{
+               if (!WebsocketJsonParsers.TryParse<double>(custom, "stop_price", out double? sp, out status) || sp == null) {
+                     return (false, null, status);
+                  }
+               customdef = new CustomOrderDefinition() {
+                     StopLimit = new StopLimitOrderDefinition(){
+                        StopPrice=(double)sp
+                     }
+                  };
+                  break;
+            }
+            case CustomOrderDefinition.CustomOneofCase.Iceberg: {
+                  if (!WebsocketJsonParsers.TryParse<double>(custom, "show_quantity", out double? sq, out status) || sq == null) {
+                     return (false, null, status);
+                  }
+                  if (custom["child"] is not JObject child) {
+                     return (false, null, "Iceberg child not correct");
+                  }
+                  (var check, var childdef, status) = TryGetCustomDef(child,Dynamic_Descriptors);
+                  if(!check){
+                     return (false, null, status);
+                  }
+                  // mbts.spark.utilities.OrderDataHelper 
+                  customdef = new CustomOrderDefinition() {
+                     Iceberg = new IcebergOrderDefinition() {
+                        ShowQuantity = (double)sq,
+                        Child = childdef
+                     }
+                  };
+                  break;
+               }
+            case CustomOrderDefinition.CustomOneofCase.TimeBox:{
+               return (false,null,"timebox not enabled on api");
+            }
+            case CustomOrderDefinition.CustomOneofCase.Spreader: {
+                  // if (custom == null) {
+                  //    message = "custom not present";
+                  //    return false;
+                  // }
+                  return (false,null,"spreaders not enabled on api");
+               }
+            case CustomOrderDefinition.CustomOneofCase.Tick:{
+               return (false,null,"Ticks not enabled on api");
+            }
+            case CustomOrderDefinition.CustomOneofCase.Lean:{
+               return (false,null,"Lean not enabled on api");
+               // if (!WebsocketJsonParsers.TryParse<double>(custom, "", out double? sq, out status) || sq == null) {
+               //       return (false, null, status);
+               //    }
+               // if (custom["child"] is not JObject child) {
+               //       return (false, null, "Lean child not correct");
+               //    }
+               //    (var check, var childdef, status) = TryGetCustomDef(child,Dynamic_Descriptors);
+               //    if(!check){
+               //       return (false, null, status);
+               //    }
+               
+               // customdef = new CustomOrderDefinition() {
+               //       Lean = new LeanOrderDefinition(){
+               //          DetachOnToward = false,
+               //          CrossThreshold = 0,
+               //          MaxChaseTicks = 0,
+               //          Child = childdef
+
+               //       }
+               //    };
+
+            }
+            case CustomOrderDefinition.CustomOneofCase.Join:{
+               return (false,null,"join not enabled on api");
+            }
+            case CustomOrderDefinition.CustomOneofCase.Top:{
+               return (false,null,"top not enabled on api");
+            }
+            case CustomOrderDefinition.CustomOneofCase.Trigger:{
+               return (false,null,"trigger not enabled on api");
+            }
+            case CustomOrderDefinition.CustomOneofCase.Cover: {
+               if (
+                  !WebsocketJsonParsers.TryParse<uint>( custom, "hedge_ticks", out uint? ht, out status) || ht == null || 
+                  !WebsocketJsonParsers.TryParse<bool>(custom, "cancel_on_external_hedge_cancel", out bool? coehc, out status) || coehc == null ||
+                  !WebsocketJsonParsers.TryParse<bool>(custom, "cancel_hedges_on_cancel", out bool? choc, out status) || choc == null ||
+                  !WebsocketJsonParsers.TryParse<bool>(custom, "detach_hedges", out bool? dh, out status) || dh == null
+                  ) {
+                  return (false, null, status);
+               }
+               if (custom["child"] is not JObject child) {
+                  return (false, null, "Cover child not correct");
+               }
+               (var check, var childdef, status) = TryGetCustomDef(child,Dynamic_Descriptors);
+               if(!check){
+                  return (false, null, status);
+               }
+               if (custom["hedge"] is not JObject hedge) {
+                  return (false, null, "Cover hedge not correct");
+               }
+               (check, var hedgedef, status) = TryGetCustomDef(hedge,Dynamic_Descriptors);
+               if(!check){
+                  return (false, null, status);
+               }
+               customdef = new CustomOrderDefinition() {
+                  Cover = new CoverOrderDefinition() {
+                     HedgeTicks = (uint)ht,
+                     CancelOnExternalHedgeCancel = (bool)coehc,
+                     CancelHedgesOnCancel = (bool)choc,
+                     DetachHedges = (bool)dh,
+                     Hedge = hedgedef,
+                     Child = childdef
+                  }
+               };
+               break;
+            }
+            case CustomOrderDefinition.CustomOneofCase.Offset:{
+               if (!WebsocketJsonParsers.TryParse<uint>(custom, "tick_offset", out uint? to, out status) || to == null) {
+                     return (false, null, status);
+                  }
+                  if (custom["child"] is not JObject child) {
+                     return (false, null, "offset child not correct");
+                  }
+                  (var check, var childdef, status) = TryGetCustomDef(child,Dynamic_Descriptors);
+                  if(!check){
+                     return (false, null, status);
+                  }
+                  // mbts.spark.utilities.OrderDataHelper 
+                  customdef = new CustomOrderDefinition() {
+                     Offset = new OffsetOrderDefinition() {
+                        TickOffsetOnDeploy = (uint)to,
+                        Child = childdef
+                     }
+                  };
+                  break;
+            }
+            case CustomOrderDefinition.CustomOneofCase.Oco:{
+               return (false,null,"Oco not enabled on api");
+            }
+            case CustomOrderDefinition.CustomOneofCase.Pricer:{
+               return (false,null,"pricer not enabled on api");
+            }
+            case CustomOrderDefinition.CustomOneofCase.Composite:{
+               return (false,null,"Composite not enabled on api");
+            }
+            case CustomOrderDefinition.CustomOneofCase.GlbxIceberg:{
+               if (!WebsocketJsonParsers.TryParse<uint>(custom, "max_show_quantity", out uint? msq, out status) || msq == null) {
+                     return (false, null, status);
+                  }
+                  // mbts.spark.utilities.OrderDataHelper 
+                  customdef = new CustomOrderDefinition() {
+                     GlbxIceberg = new GLBXIcebergOrderDefinition() {
+                        MaxShowQuantity = (uint)msq,
+                     }
+                  };
+                  break;
+            }
+            case CustomOrderDefinition.CustomOneofCase.Splitter:{
+               return (false,null,"splitter not enabled on api");
+            }
+            case CustomOrderDefinition.CustomOneofCase.QueueHolder:{
+               return (false,null,"QueueHolder not enabled on api");
+            }
+            case CustomOrderDefinition.CustomOneofCase.Scratch:{
+               return (false,null,"Scratch not enabled on api");
+            }
+            case CustomOrderDefinition.CustomOneofCase.Prorata:{
+               return (false,null,"Prorata not enabled on api");
+            }
+            case CustomOrderDefinition.CustomOneofCase.DeleteAndCover:{
+               return (false,null,"DeleteAndCover not enabled on api");
+            }
             case CustomOrderDefinition.CustomOneofCase.Limit: {
                   customdef = new CustomOrderDefinition() {
                      Limit = new LimitOrderDefinition()
                   };
                   break;
-               }
+               }       
+            case CustomOrderDefinition.CustomOneofCase.Dynamic: {
+                  if (custom == null) {
+                     return (false, null, "custom not present");
+                  }
+                  if(!WebsocketJsonParsers.TryParse<WebsocketJsonParsers.StringWrapper>(custom, "dynamic_order_type", out WebsocketJsonParsers.StringWrapper? dynamic_order_type, out status) || dynamic_order_type == null ||
+                     !Dynamic_Descriptors.TryGetValue(dynamic_order_type.Value.Val, out var desc)){
+                     return (false, null, status);
+                  }
+                  var def = new DynamicOrderDefinition() {
+                        DynamicOrderType = desc.DynamicOrderType
+                     };
 
-            // case CustomOrderDefinition.CustomOneofCase.Spreader: {
-            //       // if (custom == null) {
-            //       //    message = "custom not present";
-            //       //    return false;
-            //       // }
-            //       //ENTER SPREADER CODE HERE
-            //       break;
-            //    }
-            case CustomOrderDefinition.CustomOneofCase.Iceberg: {
-                  if (custom == null) {
-                     return (false, null, "custom not present");
-                  }
-                  double? sq;
-                  if (!WebsocketJsonParsers.TryParse<double>(custom, "show_quantity", out sq, out status) || sq == null) {
-                     return (false, null, status);
-                  }
-                  customdef = new CustomOrderDefinition() {
-                     Iceberg = new IcebergOrderDefinition() {
-                        ShowQuantity = (double)sq,
-                        Child = new CustomOrderDefinition() {
-                           Limit = new LimitOrderDefinition()
-                        }
+                  foreach (var p in desc.ParamDesc) {
+                     (bool suc, var d, var error)=TryGetDynamicParam(custom,p.ParamName,p,Dynamic_Descriptors);
+                     if(!suc){
+                        return (suc,null,error);
                      }
-                  };
-                  break;
-               }
-            case CustomOrderDefinition.CustomOneofCase.Cover: {
-                  if (custom == null) {
-                     return (false, null, "custom not present");
-                  }
-                  if (!WebsocketJsonParsers.TryParse<uint>(
-                     custom,
-                     "hedge_ticks",
-                     out uint? ht,
-                     out status)
-                     || ht == null) {
-                     return (false, null, status);
-                  }
-                  if (!WebsocketJsonParsers.TryParse<bool>(
-                     custom,
-                     "cancel_on_external_hedge_cancel",
-                     out bool? coehc,
-                     out status)
-                     || coehc == null) {
-                     return (false, null, status);
-                  }
-                  if (!WebsocketJsonParsers.TryParse<bool>(
-                     custom,
-                     "cancel_hedges_on_cancel",
-                     out bool? choc,
-                     out status)
-                     || choc == null) {
-                     return (false, null, status);
-                  }
-                  if (!WebsocketJsonParsers.TryParse<bool>(
-                     custom,
-                     "detach_hedges",
-                     out bool? dh,
-                     out status)
-                     || dh == null) {
-                     return (false, null, status);
+                     def.OrderParams[p.ParamName] = d;
                   }
+
                   customdef = new CustomOrderDefinition() {
-                     Cover = new CoverOrderDefinition() {
-                        Hedge = new CustomOrderDefinition() {
-                           Limit = new LimitOrderDefinition()
-                        },
-                        HedgeTicks = (uint)ht,
-                        CancelOnExternalHedgeCancel = (bool)coehc,
-                        CancelHedgesOnCancel = (bool)choc,
-                        DetachHedges = (bool)dh,
-                        Child = new CustomOrderDefinition() {
-                           Limit = new LimitOrderDefinition()
-                        }
-                     }
+                     Dynamic = def
                   };
                   break;
                }
@@ -124,26 +367,20 @@ namespace bts.websocket_core {
          };
          return (true, customdef, status);
       }
-      private async static Task<(bool, OrderDefinition?, string)> TryGetOrderDefAsync(WebsocketRequest req, IReadOnlyDictionary<uint, SecurityMD> securities) {
+      private async static Task<(bool, OrderDefinition?, string)> TryGetOrderDefAsync(WebsocketRequest req, IReadOnlyDictionary<uint, SecurityMD> securities, IReadOnlyDictionary<string, DynamicOrderDescriptor> Dynamic_Descriptors) {
          if (req.parameters == null) {
             return (false, null, "parameters not present");
          }
-         if (!WebsocketJsonParsers.TryParse<WebsocketJsonParsers.StringWrapper>(
-            req.parameters,
-            "user",
-            out WebsocketJsonParsers.StringWrapper? usr,
-            out string status)
-            || usr == null) {
-            return (false, null, status);
-         }
-         if (!WebsocketJsonParsers.TryParse<WebsocketJsonParsers.StringWrapper>(
-            req.parameters,
-            "account",
-            out WebsocketJsonParsers.StringWrapper? act,
-            out status)
-            || act == null) {
+         if (
+            !WebsocketJsonParsers.TryParse<WebsocketJsonParsers.StringWrapper>(req.parameters,"user",out WebsocketJsonParsers.StringWrapper? usr,out string status) || usr == null ||
+            !WebsocketJsonParsers.TryParse<Venue>(req.parameters, "venue", out Venue? exchange, out status) || exchange == null || 
+            !WebsocketJsonParsers.TryParse<WebsocketJsonParsers.StringWrapper>(req.parameters, "account", out WebsocketJsonParsers.StringWrapper? act, out status) || act == null ||
+            !WebsocketJsonParsers.TryParse<Side>(req.parameters, "side", out Side? side, out status) || side == null || 
+            !WebsocketJsonParsers.TryParse<WebsocketJsonParsers.StringWrapper>(req.parameters, "exchange_security_name", out WebsocketJsonParsers.StringWrapper? exchange_securty_name, out status) || exchange_securty_name == null
+            ) {
             return (false, null, status);
          }
+
          Result<AdminData> adminResult = await _admin.Value.get_admin_data_async(new VoidMessage());
          uint user_id;
          uint account_id;
@@ -160,23 +397,11 @@ namespace bts.websocket_core {
          user_id = (uint)user.Id;
          BTSAccount? account = admin.Accounts.FirstOrDefault(x => x.Name.ToLowerInvariant() == accountname.ToLowerInvariant());
          if (account == null) {
-            return (false, null, "account not corrent");
+            return (false, null, "account not correct");
          }
          account_id = (uint)account.Id;
-         if (!WebsocketJsonParsers.TryParse<Side>(
-            req.parameters,
-            "side",
-            out Side? side,
-            out status)
-            || side == null) {
-            return (false, null, status);
-         }
-         if (!WebsocketJsonParsers.TryParse<WebsocketJsonParsers.StringWrapper>(req.parameters, "exchange_securty_name", out WebsocketJsonParsers.StringWrapper? exchange_securty_name, out status) || exchange_securty_name == null) {
-            return (false, null, status);
-         }
-         if (!WebsocketJsonParsers.TryParse<Venue>(req.parameters, "venue", out Venue? exchange, out status) || exchange == null) {
-            return (false, null, status);
-         }
+         
+
          var _securites = securities;
          if (!_securites.Values.TryGetFirst(sc_md =>
             (
@@ -187,73 +412,100 @@ namespace bts.websocket_core {
             return (false, null, "security not found");
          }
 
-         (var check, var customdef, status) = TryGetCustomDef(req);
+         (var check, var customdef, status) = TryGetCustomDef(req, Dynamic_Descriptors);
          if (check == false) {
             return (false, null, status);
          }
+
+
          return (true, new OrderDefinition {
-            TimeInForce = TimeInForce.Gtc,
+            TimeInForce = TimeInForce.Day,
             SecurityKey = sc_md.id,
             Side = (Side)side,
             UserId = user_id,
             AccountId = account_id,
-            CustomDef = customdef
+            CustomDef = customdef,
+            Account = account.Name,
+            User = user.Name
          }, status);
       }
-      public static async Task<(bool success, string status)> DeleteOrderAsync(WebsocketRequest req, OrdersClient oc) {
-         if (req.parameters == null) {
-            return (false, "parameters not present");
+      static bool TryGetPoints(WebsocketRequest req, [NotNullWhen(true)] out List<PrecomputedTheoPoint>? theos, out string message, IReadOnlyDictionary<uint, SecurityMD> securities) {
+         theos = null;
+         if (!WebsocketJsonParsers.TryParse<WebsocketJsonParsers.PrecomputedTheoPointWrapper>(req.parameters, "points", out WebsocketJsonParsers.PrecomputedTheoPointWrapper[]? wrapper_theos, out message) || wrapper_theos == null) {
+            return false;
          }
-         if (!WebsocketJsonParsers.TryParse<uint>(req.parameters, "order_id", out uint? key, out string status) || key == null) {
-            return (false, status);
+         theos = wrapper_theos.Select(wrapper => {
+            if (WebsocketJsonParsers.PrecomputedTheoPointWrapper.GetTheoPoint(wrapper, out PrecomputedTheoPoint? point, securities)) {
+               return point;
+            }
+            return null;
+         })
+        .Where(p => p != null)
+        .Cast<PrecomputedTheoPoint>()
+        .ToList();
+         return true;
+
+      }
+      public static async Task<(bool success, uint? id, string status)> SetTheosAsync(WebsocketRequest req, OrdersClient oc, IReadOnlyDictionary<uint, SecurityMD> securities) {
+         if (req.parameters == null) {
+            return (false, null, "parameters not present");
+         }
+         var log_txt = $"SetTheosAsync req {req.ToString()}";
+         bts.Log.Gen.warn(log_txt);
+
+         var _securites = securities;
+         if (!TryGetPoints(req, out List<PrecomputedTheoPoint>? theos, out string message, _securites)) {
+            return (false, null, message);
+         }
+         Response res = await oc.TrySetPrecomputedTheosAsync(theos);
+         return (res.ErrorCode != OrderError.Ok, null, res.Msg);
+      }
+      public static async Task<(bool success, uint? id, string status)> DeleteOrderAsync(WebsocketRequest req, OrdersClient oc) {
+         if (req.parameters == null) {
+            return (false, null, "parameters not present");
+         }
+         if (!WebsocketJsonParsers.TryParse<uint>(req.parameters, "order_key", out uint? key, out string status) || key == null) {
+            return (false, null, status);
          }
          Response res = await oc.TryDeleteOrderAsync((uint)key);
          status = res.ErrorCode.ToString();
-         return (res.ErrorCode != OrderError.Ok, status);
+         return (res.ErrorCode != OrderError.Ok, null, status);
       }
-      public static async Task<(bool success, string status)> UpdateOrderAsync(WebsocketRequest request, OrdersClient oc) {
+      public static async Task<(bool success, uint? id, string status)> UpdateOrderAsync(WebsocketRequest request, OrdersClient oc, IReadOnlyDictionary<string, DynamicOrderDescriptor> Dynamic_Descriptors) {
          if (request.parameters == null) {
-            return (false, "parameters not present");
+            return (false, null, "parameters not present");
          }
-         if (!WebsocketJsonParsers.TryParse<uint>(request.parameters, "order_id", out uint? key, out string status) || key == null) {
-            return (false, status);
-         }
-         if (!WebsocketJsonParsers.TryParse(request.parameters, "quantity", out double? quantity, out status) || quantity == null) {
-            return (false, status);
+         if (
+            !WebsocketJsonParsers.TryParse<uint>(request.parameters, "order_key", out uint? key, out string status) || key == null ||
+            !WebsocketJsonParsers.TryParse(request.parameters, "quantity", out double? quantity, out status) || quantity == null ||
+            !WebsocketJsonParsers.TryParse(request.parameters, "price", out double? price, out status) || price == null
+            ) {
+            return (false, null, status);
          }
 
-         if (!WebsocketJsonParsers.TryParse(request.parameters, "price", out double? price, out status) || price == null) {
-            return (false, status);
-         }
-         (var check, var customdef, status) = TryGetCustomDef(request);
+         (var check, var customdef, status) = TryGetCustomDef(request, Dynamic_Descriptors);
          if (check == false || customdef == null) {
-            return (false, status);
+            return (false, null, status);
          }
          Response res = await oc.TryUpdateOrderAsync(customdef, (double)price, (double)quantity, (uint)key);
          status = res.ErrorCode.ToString();
-         return (res.ErrorCode != OrderError.Ok, status);
+         return (res.ErrorCode != OrderError.Ok, null, status);
       }
       public static async Task<(bool success, uint? orderId, string status)> EnterOrderAsync(
-        WebsocketRequest request, OrdersClient ordersClient, IReadOnlyDictionary<uint, SecurityMD> securities) {
-         double? quantity = null;
-         double? price = null;
+        WebsocketRequest request, OrdersClient ordersClient, IReadOnlyDictionary<uint, SecurityMD> securities, IReadOnlyDictionary<string, DynamicOrderDescriptor> Dynamic_Descriptors) {
          OrderDefinition? orderDefinition = null;
          uint? orderId = null;
          string status = string.Empty;
 
-         if (!WebsocketJsonParsers.TryParse(request.parameters, "quantity", out quantity, out status) || quantity == null) {
+         (bool success, orderDefinition, status) = await TryGetOrderDefAsync(request, securities, Dynamic_Descriptors);
+         
+         if (
+            !success || orderDefinition == null ||
+            !WebsocketJsonParsers.TryParse(request.parameters, "quantity", out double? quantity, out status) || quantity == null ||
+            !WebsocketJsonParsers.TryParse(request.parameters, "price", out double? price, out status) || price == null ) {
             return (false, null, status);
          }
 
-         if (!WebsocketJsonParsers.TryParse(request.parameters, "price", out price, out status) || price == null) {
-            return (false, null, status);
-         }
-
-         (bool success, orderDefinition, status) = await TryGetOrderDefAsync(request, securities);
-         if (!success || orderDefinition == null) {
-            return (success, null, status);
-         }
-
          var result = await ordersClient.TryEnterOrderAsync(orderDefinition, (uint)price, (double)quantity);
 
          if (result.ErrorCode != OrderError.Ok) {

src/client/spark_websocket/Core/WebsocketSubscriptionHandler.cs

@@ -100,36 +100,28 @@ namespace bts.websocket_core {
       }
 
       static bool TryGetTradesFilter(WebsocketRequest req, [NotNullWhen(true)] out TradesSubscriptionFilter? trades_filter, out string message) {
-         SecurityCriteria? cri;
          trades_filter = null;
          JObject? filter = req.parameters?["filter"] as JObject;
          if (filter == null) {
             message = "filter not present";
             return false;
          }
-         if (!TryGetFilter(req, out cri, out message)) {
+         if (!TryGetFilter(req, out SecurityCriteria? cri, out message)) {
             return false;
          }
          trades_filter = new TradesSubscriptionFilter(cri);
          return true;
       }
       static bool TryGetFillsFilter(WebsocketRequest req, [NotNullWhen(true)] out FillsSubscriptionFilter? order_filter, out string message) {
-         SecurityCriteria? cri;
          order_filter = null;
-         JObject? filter = req.parameters?["filter"] as JObject;
-         if (filter == null) {
+         if (req.parameters?["filter"] is not JObject filter) {
             message = "filter not present";
             return false;
          }
-         if (!TryGetFilter(req, out cri, out message)) {
-            return false;
-         }
-         uint[]? accounts;
-         if (!WebsocketJsonParsers.TryParse<uint>(filter, "accounts", out accounts, out message)) {
-            return false;
-         }
-         OrderStatus[]? statuses;
-         if (!WebsocketJsonParsers.TryParse<OrderStatus>(filter, "statuses", out statuses, out message)) {
+         if (!TryGetFilter(req, out SecurityCriteria? cri, out message) ||
+            !WebsocketJsonParsers.TryParse<uint>(filter, "accounts", out uint[]? accounts, out message) ||
+            !WebsocketJsonParsers.TryParse<OrderStatus>(filter, "statuses", out OrderStatus[]? statuses, out message)
+            ) {
             return false;
          }
          order_filter = new FillsSubscriptionFilter(
@@ -138,22 +130,16 @@ namespace bts.websocket_core {
          return true;
       }
       static bool TryGetOrdersFilter(WebsocketRequest req, [NotNullWhen(true)] out OrdersSubscriptionFilter? order_filter, out string message) {
-         SecurityCriteria? cri;
          order_filter = null;
-         JObject? filter = req.parameters?["filter"] as JObject;
-         if (filter == null) {
+         if (req.parameters?["filter"] is not JObject filter) {
             message = "filter not present";
             return false;
          }
-         if (!TryGetFilter(req, out cri, out message)) {
-            return false;
-         }
-         uint[]? accounts;
-         if (!WebsocketJsonParsers.TryParse<uint>(filter, "accounts", out accounts, out message)) {
-            return false;
-         }
-         OrderStatus[]? statuses;
-         if (!WebsocketJsonParsers.TryParse<OrderStatus>(filter, "statuses", out statuses, out message)) {
+         if (
+            !TryGetFilter(req, out SecurityCriteria? cri, out message) ||
+            !WebsocketJsonParsers.TryParse<uint>(filter, "accounts", out uint[]? accounts, out message) ||
+            !WebsocketJsonParsers.TryParse<OrderStatus>(filter, "statuses", out OrderStatus[]? statuses, out message)
+            ) {
             return false;
          }
          order_filter = new OrdersSubscriptionFilter(
@@ -165,39 +151,19 @@ namespace bts.websocket_core {
 
       static bool TryGetFilter(WebsocketRequest req, [NotNullWhen(true)] out SecurityCriteria? cri, out string message) {
          cri = null;
-         JObject? filter = req.parameters?["filter"] as JObject;
-         if (filter == null) {
+         if (req.parameters?["filter"] is not JObject filter) {
             message = "filter not present";
             return false;
          }
-         Venue[]? exchanges;
-         if (!WebsocketJsonParsers.TryParse<Venue>(filter, "venue", out exchanges, out message)) {
-            return false;
-         }
-         SecurityType[]? types;
-         if (!WebsocketJsonParsers.TryParse<SecurityType>(filter, "kind", out types, out message)) {
-            return false;
-         }
-         PutOrCall[]? callputs;
-         if (!WebsocketJsonParsers.TryParse<PutOrCall>(filter, "call_or_put", out callputs, out message)) {
-            return false;
-         }
-         DateTime[]? expiration;
-         if (!WebsocketJsonParsers.TryParse<DateTime>(filter, "expiration", out expiration, out message)) {
-            return false;
-         }
-         // string[]? symbols = (string[]?)(filter.GetValueOrDefault("symbol")?.ToString()?.Split(",").Select(s => s.ToUpper()));
-         WebsocketJsonParsers.ListWrapper? symbols;
-         if (!WebsocketJsonParsers.TryParse<WebsocketJsonParsers.ListWrapper>(filter, "symbol", out symbols, out message)) {
-            return false;
-         }
-         WebsocketJsonParsers.ListWrapper? names;
-         if (!WebsocketJsonParsers.TryParse<WebsocketJsonParsers.ListWrapper>(filter, "name", out names, out message)) {
-            return false;
-         }
-         // filter.GetValueOrDefault("symbol")?.ToString()?.Split(",");
-         double[]? strikes;
-         if (!WebsocketJsonParsers.TryParse<double>(filter, "strikes", out strikes, out message)) {
+         if (
+            !WebsocketJsonParsers.TryParse<Venue>(filter, "venue", out Venue[]? exchanges, out message) ||
+            !WebsocketJsonParsers.TryParse<SecurityType>(filter, "kind", out SecurityType[]? types, out message) ||
+            !WebsocketJsonParsers.TryParse<PutOrCall>(filter, "call_or_put", out PutOrCall[]? callputs, out message) ||
+            !WebsocketJsonParsers.TryParse<DateTime>(filter, "expiration", out DateTime[]? expiration, out message) || 
+            !WebsocketJsonParsers.TryParse<WebsocketJsonParsers.StringListWrapper>(filter, "symbol", out WebsocketJsonParsers.StringListWrapper? symbols, out message) ||
+            !WebsocketJsonParsers.TryParse<WebsocketJsonParsers.StringListWrapper>(filter, "name", out WebsocketJsonParsers.StringListWrapper? names, out message) ||
+            !WebsocketJsonParsers.TryParse<double>(filter, "strikes", out double[]? strikes, out message)
+         ) {
             return false;
          }
          var clause = new SecurityCriteriaClause(
@@ -223,13 +189,12 @@ namespace bts.websocket_core {
          out string status) {
          hdlr = null;
 
-         SUBSCRIPTION_TYPE? type;
-         if (!WebsocketJsonParsers.TryParse<SUBSCRIPTION_TYPE>(req.parameters, "type", out type, out status) || status != "") {
-            return false;
-         }
-         if (!WebsocketJsonParsers.TryParse<WebsocketJsonParsers.StringWrapper>(req.parameters, "subscription_id", out WebsocketJsonParsers.StringWrapper? id, out status) || id == null) {
+         if (
+            !WebsocketJsonParsers.TryParse<SUBSCRIPTION_TYPE>(req.parameters, "type", out SUBSCRIPTION_TYPE? type, out status) || status != "" ||
+            !WebsocketJsonParsers.TryParse<WebsocketJsonParsers.StringWrapper>(req.parameters, "subscription_id", out WebsocketJsonParsers.StringWrapper? id, out status) || id == null) {
             return false;
          }
+        
          Action<WebsocketSubscriptionHandler> sub;
          switch (type) {
             case SUBSCRIPTION_TYPE.BOOKS: {

src/data_files/documents/order_flow.txt

@@ -79,6 +79,24 @@ Misc:
 To search for a publisher for an event search for make_stream<Event_Name>, for 
 example make_stream<OrderStateUpdateEvent>()
 
+is_gateway_connected can be false in the NativeOrder class (GlbxNativeOrder) ->
+OrderServices->is_gateway_connected->
+SessionServices->is_gateway_connected
+SessionServicesData->gw_connected
+OrderManager sets SessionServicesData->gw_connected on void OrderManager::on_gateway_state(const mercury::Update<GatewayStatusChangeEvent> &m)
+
+
+Exchange gateway is down (debugging):
+-------------------------------------
+This happens at times and if often a config issue.
+GlbCNativeOrder::_on_activate ->
+OrderServices::is_gateway_connected ->
+SessionServices::is_gateway_connected ->
+SessionServicesData::
+
+IExecutionGateway::on_gw_status_change is what sends out the status
+
+
 --------------------------------------------------------------------------
 market data
 --------------------------------------------------------------------------

src/lib/api/md/GlbxMarketDataConfig.proto

@@ -6,17 +6,19 @@ package bts.api.md;
 
 option csharp_namespace = "bts.api.md";
 
+message GlbxActivePlatform {
+  string host = 1;
+  string user = 2;
+  string password = 3;
+}
+message GlbxOnixPlatform {
+  string temp =1; // will add everything related to onix here.
+}
 
 message GlbxMarketDataConfig {
-  uint32 unique_check_value = 1;
-  //uint32 num_depth_levels = 1;
-  //uint32 depth_throttle_ms = 2;
-  bts.api.domain.SecurityCriteria security_criteria = 3;
-  // Market data isn't subscribed for securities listed here (even if they are
-  // included by 'security_criteria'). There are some securities that don't
-  // fit into our system just yet, such as 'BTCBIDR', which trades in prices
-  // larger than our cpp 'domain::Price' class supports.
-  //repeated string excluded_securities = 4;
-  //string url_rest_base = 5;
-  //string url_ws_base = 6;
+  bts.api.domain.SecurityCriteria security_criteria = 2;
+  oneof Platform {
+    GlbxActivePlatform glbx_active_platform = 10;
+    GlbxOnixPlatform glbx_onix_platform = 11;
+  }
 }

src/lib/cmake/GetLinuxClientSources.cmake

@@ -93,10 +93,6 @@ function(get_cpp_sources dest)
     md/MDConsumerInterop.cpp
     md/MDProducer.cpp
     md/MDProducerHandle.cpp
-    md/MarketDataError.cpp
-    md/MarketDataGateway.cpp
-    md/MarketDataRegistrationFactory.cpp
-    md/MarketDataSubscription.cpp
     md/MemorySegmentUtils.cpp
     md/PriceSize.cpp
     md/Product.cpp
@@ -113,8 +109,6 @@ function(get_cpp_sources dest)
     md/cme/MDP3Utils.cpp
     md/common.cpp
     md/ice/ICEUtils.cpp
-    md/recording/MDRecorder.cpp
-    md/recording/MDReplayer.cpp
     md/relay/QueryServer.cpp
     md/relay/RelayClient.cpp
     md/relay/RelayRecorder.cpp

src/lib/cmake/GetLinuxServerLinkLibraries.cmake

@@ -25,8 +25,23 @@ function(get_link_libraries dest)
     Packet++
     Common++
     ssh2
+    ACE-5.7.5
+    ActivBase_x86-64_ubuntu22-04_gcc123_mdid
+    ActivOneApi_x86-64_ubuntu22-04_gcc123_mdid
+    ActivAgentApplication_x86-64_ubuntu22-04_gcc123_mdid
+    ActivDatabase_x86-64_ubuntu22-04_gcc123_mdid
+    ActivFramework_x86-64_ubuntu22-04_gcc123_mdid
+    ActivServices_x86-64_ubuntu22-04_gcc123_mdid
+    BoostAtomic_x86-64_ubuntu22-04_gcc123_mdid
+    BoostChrono_x86-64_ubuntu22-04_gcc123_mdid
+    BoostProgramOptions_x86-64_ubuntu22-04_gcc123_mdid
+    BoostFilesystem_x86-64_ubuntu22-04_gcc123_mdid
+    -lm
+    -lgcc_s
+    -lc
     -lrt
     -ldl
+    -lstdc++
     -lpcap
     -static-libgcc
     -static-libstdc++

src/lib/cmake/GetWindowsClientSources.cmake

@@ -87,10 +87,6 @@ function(get_cpp_sources dest)
     md/MDConsumerInterop.cpp
     md/MDProducer.cpp
     md/MDProducerHandle.cpp
-    md/MarketDataError.cpp
-    md/MarketDataGateway.cpp
-    md/MarketDataRegistrationFactory.cpp
-    md/MarketDataSubscription.cpp
     md/MemorySegmentUtils.cpp
     md/PriceSize.cpp
     md/Product.cpp

src/lib/config/profiles/default/jetfix.conf

@@ -3,7 +3,7 @@ JetFIXSimConnection {
    value {
       sim {
         name: "sim"
-        venue: VENUE_SIM
+        venue: VENUE_GLBX
         socket_connect_host_ip: "127.0.0.1"
         socket_connect_port: @Ports.glbx_sim_fix_port
         sender_compid: "SPARK" # this is the FIX session name
@@ -17,7 +17,9 @@ JetFIXSimConnection {
         raw_data: "1JE9X"
         use_cme_secure_logon: false
         seconds_to_pause : 1
-	max_connection_attempts: 9999999
+        max_order_msgs_per_window: 300
+        window_duration_msecs: 1000
+	     max_connection_attempts: 9999999
       }
       sim_ice {
         name: "sim_ice"

src/lib/core/EnumMacros.h

@@ -1,5 +1,7 @@
 #pragma once
 
+#include <ostream>
+
 #define ENUM_BASIC_MEMBERS(type)                                               \
    Value value;                                                                \
                                                                                \

src/lib/core/TimeStamp.cpp

@@ -23,7 +23,18 @@ TimeStamp::TimeStamp(uint32_t year, uint32_t month, uint32_t day) {
    value_ =
        std::chrono::duration_cast<std::chrono::nanoseconds>(duration).count();
 }
+bool TimeStamp::from_ymd(
+    std::string yearMonthDay, TimeStamp *result) {
+   uint32_t year, month, day;
+   std::istringstream ss(yearMonthDay);
+    char dash; // to consume the dash character in the input
 
+    // Use stringstream to extract year, month, and day from the input string
+    ss >> year >> dash >> month >> dash >> day;
+    if(ss.fail())
+      return false;
+    return TimeStamp::from_ymd(year,month,day,result);
+}
 bool TimeStamp::from_ymd(
     std::uint32_t year, std::uint32_t month, std::uint32_t day, TimeStamp *result) {
    if (!is_valid_ymd(year, month, day)) {

src/lib/core/TimeStamp.h

@@ -41,6 +41,9 @@ struct TimeStamp {
        std::uint32_t month,
        std::uint32_t day,
        TimeStamp *result);
+   static bool from_ymd(
+      std::string yearMonthDay,
+       TimeStamp *result);
 
    constexpr std::uint64_t nanos() const;
    constexpr std::uint64_t micros() const;

src/lib/domain_model/SecurityKey.h

@@ -1,5 +1,14 @@
 #pragma once
+
+#include "lib/core/FMT_IMPL.h"
 #include "lib/utils/ValueWrapper.h"
+
+#include <cstdint>
+
 namespace bts {
-using SecurityKey = utils::ValueWrapper<struct SecurityKeyTag, uint32_t>;
-}
+
+using SecurityKey = utils::ValueWrapper<struct SecurityKeyTag, std::uint32_t>;
+
+} // namespace bts
+
+FMT_IMPL(bts::SecurityKey, k, "{}", k.value());

src/lib/io/websocket/JsonRpcRequest.h

@@ -69,8 +69,6 @@ inline void from_json(const nlohmann::json &j, JsonRpcRequestT<ParamsT> &req) {
    j.at("method").get_to(req.method);
    j.at("params").get_to(req.params);
    std::vector<std::string> vec;
-   *vec.begin();
-   vec.begin()->size();
    auto id_j = j.find("id");
    if (id_j == j.end()) {
       req.id = std::nullopt;

src/lib/io/websocket/WebsocketMessage.h

@@ -1,6 +1,7 @@
 #pragma once
+
+#include <cstdint>
 #include <ostream>
-#include <string>
 #include <vector>
 
 namespace bts::io::websocket {

src/lib/logging/ccapi_logger.h

@@ -0,0 +1,87 @@
+#include "logging.h"
+
+#include "LoggerHub.h"
+#include "lib/core/LockGuard.h"
+#include "lib/utils/bts_assert.h"
+
+#include <boost/circular_buffer.hpp>
+#include <boost/container/flat_set.hpp>
+#include <boost/filesystem.hpp>
+#include <boost/filesystem/operations.hpp>
+#include <boost/filesystem/path.hpp>
+#include <ccapi_cpp/ccapi_logger.h>
+#include <ccapi_cpp/service/ccapi_market_data_service_deribit.h>
+#include <fmt/format.h>
+#include <google/protobuf/io/gzip_stream.h>
+#include <google/protobuf/io/zero_copy_stream_impl.h>
+#include <google/protobuf/text_format.h>
+#include <mutex>
+#include <string>
+
+namespace ccapi {
+
+class CcapiLogger final : public Logger {
+ public:
+   void logMessage(
+       const std::string &severity,
+       const std::string & /*thread_id*/,
+       const std::string & /*time_iso*/,
+       const std::string &file_name,
+       const std::string &line_num,
+       const std::string &msg) override {
+      auto level = to_bts_level_(severity);
+      if (!bts::log::_thread_logger.logger->is_loggable(bts::log::md, level)) {
+         return;
+      }
+
+      // Add file name to cache to preserve it in memory until the log is
+      // [asynchronously] committed
+      const auto &cached_file_name = store_file_name_(file_name);
+      bts::log::_thread_logger.logger->enqueue(
+          bts::log::log_ts(), bts::log::md, level, cached_file_name.c_str(),
+          stoi(line_num), "{}", msg);
+   }
+
+   static bts::log::Level to_bts_level_(const std::string &ccapi_severity) {
+      bts_assert(!ccapi_severity.empty());
+      // TRACE and DEBUG translate to Diag; all other levels have identical
+      // names in ccapi and spark
+      if (ccapi_severity[0] == 'T' || ccapi_severity[0] == 'D') {
+         return bts::Diag;
+      }
+      if (ccapi_severity[0] == 'I') {
+         return bts::Info;
+      }
+      if (ccapi_severity[0] == 'W') {
+         return bts::Warn;
+      }
+      if (ccapi_severity[0] == 'E') {
+         return bts::Error;
+      }
+      if (ccapi_severity[0] == 'F') {
+         return bts::Fatal;
+      }
+      bts_assert(false);
+      return bts::Fatal;
+   }
+
+ private:
+   std::mutex instance_mx_;
+   std::unordered_set<std::string> file_name_store_;
+   // This buffer is intended to keep file name strings in memory until the log
+   // is [asynchronously] committed. There is no mechanism for determining when
+   // the log is actually committed; we just hope that the log system won't be
+   // flooded to the point that file names are pushed out of the buffer before
+   // their corresponding logs are committed.
+   //
+   // Note: Another implementation, which would guarentee file names are
+   // preserved, is to maintain a set of unique file names. Naturally, it's
+   // size would be limited, but not without cost in additional overhead for
+   // lookups.
+   const std::string &store_file_name_(const std::string &file_name) {
+      LOCK_GUARD(instance_mx_);
+      return *file_name_store_.insert(file_name).first;
+   }
+};
+
+} // namespace ccapi

src/lib/logging/logging.cpp

@@ -1,17 +1,16 @@
 #include "logging.h"
-
 #include "LoggerHub.h"
 #include "lib/config/config.h"
 #include "lib/config/protos/deployment_type.pb.h"
 #include "lib/config/protos/logging.pb.h"
 #include "lib/config/protos/system_config.pb.h"
-#include "lib/core/LockGuard.h"
 #include "lib/os/os.h"
 #include "lib/utils/Dispatcher.h"
 #include "lib/utils/bts_assert.h"
-#include "lib/utils/bts_chrono.h"
-#include "lib/utils/file_utils.h"
 #include "lib/utils/thread_manager.h"
+#ifdef SPARK_ENABLE_CCAPI
+#include "lib/logging/ccapi_logger.h"
+#endif
 
 #include <boost/circular_buffer.hpp>
 #include <boost/container/flat_set.hpp>
@@ -19,16 +18,12 @@
 #include <boost/filesystem/operations.hpp>
 #include <boost/filesystem/path.hpp>
 #include <fmt/format.h>
-#include <string>
-#ifdef SPARK_ENABLE_CCAPI
-#include <ccapi_cpp/ccapi_logger.h>
-#include <ccapi_cpp/service/ccapi_market_data_service_deribit.h>
-#endif
 #include <fstream>
 #include <google/protobuf/io/gzip_stream.h>
 #include <google/protobuf/io/zero_copy_stream_impl.h>
 #include <google/protobuf/text_format.h>
 #include <mutex>
+#include <string>
 
 namespace bts {
 namespace log {
@@ -223,72 +218,9 @@ extern "C" BTS_EXPORT void Logger_log(
 
 #ifdef SPARK_ENABLE_CCAPI
 namespace ccapi {
-class SparkLoggerAdaptor final : public Logger {
- public:
-   void logMessage(
-       const std::string &severity,
-       const std::string & /*thread_id*/,
-       const std::string & /*time_iso*/,
-       const std::string &file_name,
-       const std::string &line_num,
-       const std::string &msg) override {
-      auto level = to_bts_level_(severity);
-      if (!bts::log::_thread_logger.logger->is_loggable(bts::log::md, level)) {
-         return;
-      }
 
-      // Add file name to cache to preserve it in memory until the log is
-      // [asynchronously] committed
-      const auto &cached_file_name = store_file_name_(file_name);
-      bts::log::_thread_logger.logger->enqueue(
-          bts::log::log_ts(), bts::log::md, level, cached_file_name.c_str(),
-          stoi(line_num), "{}", msg);
-   }
-
-   static bts::log::Level to_bts_level_(const std::string &ccapi_severity) {
-      bts_assert(!ccapi_severity.empty());
-      // TRACE and DEBUG translate to Diag; all other levels have identical
-      // names in ccapi and spark
-      if (ccapi_severity[0] == 'T' || ccapi_severity[0] == 'D') {
-         return bts::Diag;
-      }
-      if (ccapi_severity[0] == 'I') {
-         return bts::Info;
-      }
-      if (ccapi_severity[0] == 'W') {
-         return bts::Warn;
-      }
-      if (ccapi_severity[0] == 'E') {
-         return bts::Error;
-      }
-      if (ccapi_severity[0] == 'F') {
-         return bts::Fatal;
-      }
-      bts_assert(false);
-      return bts::Fatal;
-   }
-
- private:
-   std::mutex instance_mx_;
-   std::unordered_set<std::string> file_name_store_;
-   // This buffer is intended to keep file name strings in memory until the log
-   // is [asynchronously] committed. There is no mechanism for determining when
-   // the log is actually committed; we just hope that the log system won't be
-   // flooded to the point that file names are pushed out of the buffer before
-   // their corresponding logs are committed.
-   //
-   // Note: Another implementation, which would guarentee file names are
-   // preserved, is to maintain a set of unique file names. Naturally, it's
-   // size would be limited, but not without cost in additional overhead for
-   // lookups.
-   const std::string &store_file_name_(const std::string &file_name) {
-      LOCK_GUARD(instance_mx_);
-      return *file_name_store_.insert(file_name).first;
-   }
-};
-
-SparkLoggerAdaptor spark_logger_adaptor;
-Logger *Logger::logger = &spark_logger_adaptor;
+CcapiLogger ccapi_logger;
+Logger *Logger::logger = &ccapi_logger;
 
 } // namespace ccapi
 #endif

src/lib/md/ExchangeOrderKey.h

@@ -1,9 +1,15 @@
 #pragma once
+
 #include "lib/utils/ValueWrapper.h"
-namespace bts {
-namespace md {
+
+#include <cstdint>
+
+namespace bts::md {
+
 using ExchangeOrderKey =
-    utils::ValueWrapper<struct ExchangeOrderKeyTag, uint64_t>;
-using SparkOrderKey = utils::ValueWrapper<struct SparkOrderKeyTag, uint64_t>;
-} // namespace md
-} // namespace bts
+    utils::ValueWrapper<struct ExchangeOrderKeyTag, std::uint64_t>;
+
+using SparkOrderKey =
+    utils::ValueWrapper<struct SparkOrderKeyTag, std::uint64_t>;
+
+} // namespace bts::md

src/lib/md/ExchangeSecurityKey.h

@@ -1,6 +1,12 @@
 #pragma once
+
 #include "lib/utils/ValueWrapper.h"
+
+#include <cstdint>
+
 namespace bts {
+
 using ExchangeSecurityKey =
-    utils::ValueWrapper<struct ExchangeSecurityKeyTag, uint64_t>;
-}
+    utils::ValueWrapper<struct ExchangeSecurityKeyTag, std::uint64_t>;
+
+} // namespace bts

src/lib/md/GeneralMarketDataListener.cpp

@@ -1,4 +1,4 @@
-#ifdef SPARK_ENABLE_CCAPI
+//#ifdef SPARK_ENABLE_CCAPI
 #include "GeneralMarketDataListener.h"
 #include "lib/api/domain/Currency.pb.h"
 #include "lib/api/domain/SecurityType.pb.h"
@@ -464,4 +464,4 @@ void GeneralMarketDataListener::_possibly_match_with_sim_orders(
 }
 
 } // namespace bts::md
-#endif
+//#endif

src/lib/md/MDConsumer.cpp

@@ -537,7 +537,7 @@ class SyntheticConsumerInst {
    IncompleteStrategy to_strategy(const shared_ptr<api::Security> &security) {
       IncompleteStrategy res;
       res.security = security;
-      auto meta = _ss->get_metadata(SecurityKey(security->id()));
+      auto meta = _ss->get_metadata(bts::SecurityKey(security->id()));
       // If metadata is set copy it otherwise we use the default proto
       if (meta) {
          res.strategy.CopyFrom(meta->strategy());

src/lib/md/MDConsumerInterop.cpp

@@ -128,6 +128,32 @@ CAPI bool md_consumer_is_heart_beating(const ColdHeader *header) {
    return is_heart_beat_ok(*header);
 }
 
+typedef void (*raw_receive_pb)(const char *, size_t);
+CAPI int md_consumer_read_trades_linux(
+    MDConsumer *consumer,
+    const char* symbol,
+    int venue_i,
+    int type_i,
+    int64_t &cursor,
+    raw_receive_pb cb) {
+   auto type = static_cast<SecurityType>(type_i);
+   auto venue = static_cast<Venue>(venue_i);
+   bts_assert(consumer);
+   google::protobuf::Arena arena;
+   co2::TradeList *ptrades =
+       google::protobuf::Arena::CreateMessage<co2::TradeList>(&arena);
+   auto channel =
+       consumer->get_channel(symbol, type, venue);
+   if (channel.is_initialized()) {
+      cursor = channel.read_trades(*ptrades->mutable_items(), cursor);
+      std::string buffer;
+      ptrades->SerializeToString(&buffer);
+      cb(buffer.c_str(), buffer.size());
+      return ptrades->items_size();
+   }
+   return 0;
+}
+
 typedef void (*raw_receive_pb)(const char *, size_t);
 CAPI int md_consumer_read_trades(
     MDConsumer *consumer,

src/lib/md/Shm/FixedSecurity.h

@@ -2,6 +2,7 @@
 
 #include "../common.h"
 #include "lib/core/FMT_IMPL.h"
+#include "lib/core/TimeStamp.h"
 #include "lib/io/JsonDumper.h"
 #include "lib/io/JsonReader.h"
 #include "lib/utils/ValueWrapper.h"
@@ -9,8 +10,8 @@
 #include "lib/utils/bts_assert.h"
 #include "lib/utils/bts_export.h"
 #include "lib/utils/bts_no_copy.h"
-#include "lib/core/TimeStamp.h"
 #include "lib/utils/warn_ctrl.h"
+#include "lib/domain_model/SecurityKey.h"
 
 #include <nlohmann/json.hpp>
 #include <sstream>
@@ -19,10 +20,13 @@
 namespace bts {
 namespace md {
 
-using SecurityKey = utils::ValueWrapper<struct SecurityKeyTag, uint32_t>;
+using SecurityKey = bts::SecurityKey;
 using ExchangeKey = utils::ValueWrapper<struct ExchangeKeyTag, uint64_t>;
 
 GCC_DIAG_OFF("-Weffc++") // No Virtual Destructors by design
+#ifndef __clang__
+GCC_DIAG_OFF("-Wstringop-truncation")
+#endif
 
 #define STRING_PROPERTY(NAME)                                                  \
    const char *NAME() const { return _##NAME; }                                \
@@ -129,7 +133,7 @@ struct BTS_EXPORT FixedSpreadLeg {
    double leg_delta;
 };
 
-template <int LEGCOUNT = 16>
+template <int LEGCOUNT = 64>
 class FixedSpreadSecurity : public FixedSecurity {
    char _summary[128];
 
@@ -216,7 +220,6 @@ SecurityType securitytype_of(const Security &);
 size_t num_legs(const Security &);
 uint64_t exchange_id(const Security &security);
 
-
 void to_json(nlohmann::json &j, const FixedSpreadLeg &f);
 
 void to_json(nlohmann::json &j, const FixedSecurity &f);
@@ -386,5 +389,3 @@ void with_security_like(const api::Security &security, C cb) {
 
 } // namespace md
 } // namespace bts
-
-FMT_IMPL(bts::md::SecurityKey, k, "{}", *k);

src/lib/md/ccApi/CCSecurityIDFactory.h

@@ -1,16 +0,0 @@
-#pragma once
-
-#include "lib/api/domain/Venue.pb.h"
-#include "lib/md/common.h"
-#include <bits/stdint-uintn.h>
-namespace bts::md {
-
-// Conveniant class
-class CCSecurityIDFactory {
- public:
-   std::optional<uint64_t> get_id(
-       bts::md::Venue venue, const std::string &exchange_id);
-   std::optional<uint64_t> get_id(bts::md::Venue venue, uint64_t &exchange_id);
-};
-
-} // namespace bts::md

src/lib/md/deribit/DeribitSubscriptionFactory.h

@@ -1,82 +0,0 @@
-#pragma once
-
-#include <ccapi_cpp/ccapi_subscription.h>
-#include <string>
-#include <vector>
-
-namespace bts::md::deribit {
-class DeribitSubscriptionFactory {
- public:
-   /**
- * @brief Creates a Subscription for the specified instrument \p name
- *
- * @param name The name of the instrument for which to subscribe
- */
-   static ccapi::Subscription instrument_sub(
-       const std::string &name, JsonRpcId id) {
-      const auto corr_id = CCApiCorrelationID::from_parts(
-          VENUE_NAME, DERIBIT_MSG_TYPE_NAME(DeribitMessageType::INSTRUMENT),
-          name);
-      JsonRpcRequest rpc_req;
-      rpc_req.method = JSON_RPC_GET_INSTRUMENT_METHOD;
-      rpc_req.params["instrument_name"] = name;
-      rpc_req.id = id;
-      return {
-          CCAPI_VENUE_NAME, "", CCAPI_GENERIC_PUBLIC_SUBSCRIPTION,
-          json{rpc_req}.dump(2), corr_id.to_string()};
-   }
-
-   /**
- * @brief Creates a Subscription for instruments of the specified \p currency
- *
- * @param currency The currency for which to get instruments
- */
-   static ccapi::Subscription instruments_sub(
-       const string &currency, DeribitInstrumentKind kind, JsonRpcId id) {
-      const auto corr_id = CCApiCorrelationID::from_parts(
-          VENUE_NAME, DeribitMessageType::INSTRUMENTS, currency, kind);
-      JsonRpcRequest rpc_req;
-      rpc_req.method = JSON_RPC_GET_INSTRUMENTS_METHOD;
-      rpc_req.params["currency"] = currency;
-      if (kind != DeribitInstrumentKind::NONE) {
-         rpc_req.params["kind"] = kind;
-      }
-      rpc_req.id = id;
-      return {
-          CCAPI_VENUE_NAME, "", CCAPI_GENERIC_PUBLIC_SUBSCRIPTION,
-          json{rpc_req}.dump(2), corr_id.to_string()};
-   }
-
-   /**
- * @brief Creates a Subscription for combos of the specified \p currency
- *
- * @param currency The currency for which to get instruments
- */
-   static ccapi::Subscription combos_sub(const string &currency, JsonRpcId id) {
-      const auto corr_id = CCApiCorrelationID::from_parts(
-          VENUE_NAME, DeribitMessageType::COMBOS, currency);
-      JsonRpcRequest rpc_req;
-      rpc_req.method = JSON_RPC_GET_COMBOS_METHOD;
-      rpc_req.params["currency"] = currency;
-      rpc_req.id = id;
-      return {
-          CCAPI_VENUE_NAME, "", CCAPI_GENERIC_PUBLIC_SUBSCRIPTION,
-          json{rpc_req}.dump(2), corr_id.to_string()};
-   }
-
-   static ccapi::Subscription combo_details_sub(
-       const string &combo_name, JsonRpcId id) {
-      auto corr_id = CCApiCorrelationID::from_parts(
-          VENUE_NAME, DERIBIT_MSG_TYPE_NAME(DeribitMessageType::COMBO_DETAILS),
-          combo_name);
-      JsonRpcRequest rpc_req;
-      rpc_req.method = JSON_RPC_GET_COMBO_DETAILS_METHOD;
-      rpc_req.params["combo_id"] = combo_name;
-      rpc_req.id = id;
-      return {
-          CCAPI_VENUE_NAME, "", CCAPI_GENERIC_PUBLIC_SUBSCRIPTION,
-          json{rpc_req}.dump(2), corr_id.to_string()};
-   }
-};
-
-} // namespace bts::md::deribit

src/lib/orderman/ChildOrder.cpp

@@ -0,0 +1,102 @@
+#include "ChildOrder.h"
+
+namespace bts::orderman {
+
+ChildOrder::ChildOrder()
+    : _child_order(nullptr), _parent_order_services(nullptr),
+      _child_order_services(nullptr), _active(false) {}
+
+ChildOrder::ChildOrder(Order &child, OrderServices &parent_services)
+    : _child_order(&child), _parent_order_services(&parent_services),
+      _child_order_services(), _active(false) {}
+
+// This is used in attach, where the child order is already activated
+// and ready to go.
+ChildOrder::ChildOrder(
+    OrderServices &child_services, OrderServices &parent_services)
+    : _child_order(child_services._owner),
+      _parent_order_services(&parent_services),
+      _child_order_services(&child_services), _active(true) {}
+
+ChildOrder::ChildOrder(ChildOrder &&src) { operator=(std::move(src)); }
+ChildOrder::ChildOrder(ChildOrder const &src) { operator=(src); }
+
+ChildOrder::ChildOrder(
+    api::orderman::OrderDefinition const &def, OrderServices &parent_services)
+    : _child_order(&parent_services.make_child(def)),
+      _parent_order_services(&parent_services), _child_order_services(),
+      _active(false) {}
+
+ChildOrder &ChildOrder::operator=(ChildOrder &&src) {
+   _child_order = src._child_order;
+   _parent_order_services = src._parent_order_services;
+   _child_order_services = src._child_order_services;
+   _active = src._active;
+
+   src._child_order = nullptr;
+   src._parent_order_services = nullptr;
+   src._child_order_services = nullptr;
+   src._active = false;
+   return *this;
+}
+
+ChildOrder &ChildOrder::operator=(ChildOrder const &src) {
+   _child_order = src._child_order;
+   _parent_order_services = src._parent_order_services;
+   _child_order_services = src._child_order_services;
+   _active = src._active;
+   return *this;
+}
+
+bool ChildOrder::empty() const { return _child_order == nullptr; }
+OrderKey ChildOrder::key() const {
+   bts_assert_d(_child_order);
+   return _child_order->key();
+}
+
+bool ChildOrder::is_active() const { return _active; }
+void ChildOrder::warm_up_activate() {
+   bts_assert(!_active);
+   _parent_order_services->warm_up_activate_child(*_child_order);
+}
+
+void ChildOrder::activate(Price p, Quantity q) {
+   bts_assert(_child_order);
+   bts_assert(!_active);
+
+   _parent_order_services->activate_child(*_child_order, p, q);
+   _child_order_services = _child_order->_services.get();
+   _active = true;
+}
+
+void ChildOrder::mark_detached_from_parent() {
+   bts_assert(_child_order);
+   bts_assert(_active);
+   _child_order_services->mark_detached_from_parent();
+}
+
+void ChildOrder::update(Price p, Quantity q) {
+   bts_assert(_child_order);
+   bts_assert(_active);
+   _child_order_services->update(p, q);
+}
+
+void ChildOrder::update(api::orderman::Update const &update) {
+   bts_assert(_child_order);
+   bts_assert(_active);
+   _child_order_services->update(update);
+}
+
+void ChildOrder::detach() {
+   bts_assert(_child_order);
+   bts_assert(_active);
+   _child_order_services->detach();
+}
+
+void ChildOrder::cancel() {
+   bts_assert(_child_order);
+   bts_assert(_active);
+   _child_order_services->cancel();
+}
+
+} // namespace bts::orderman

src/lib/orderman/ChildOrder.h

@@ -3,173 +3,57 @@
 #include "lib/orderman/OrderServices.h"
 #include "lib/orderman/orderman_utils/orderman_common.h"
 #include "lib/utils/bts_assert.h"
+
 #include <memory>
 
-namespace bts {
-namespace orderman {
+namespace bts::orderman {
 
 class ChildOrder {
  public:
-   ChildOrder()
-       : _child_order(nullptr), _parent_order_services(nullptr),
-         _child_order_services(nullptr), _active(false) {}
+   ChildOrder();
 
-   ChildOrder(Order &child, OrderServices &parent_services)
-       : _child_order(&child), _parent_order_services(&parent_services),
-         _child_order_services(), _active(false) {}
+   ChildOrder(Order &child, OrderServices &parent_services);
 
    // This is used in attach, where the child order is already activated
    // and ready to go.
-   ChildOrder(OrderServices &child_services, OrderServices &parent_services)
-       : _child_order(child_services._owner),
-         _parent_order_services(&parent_services),
-         _child_order_services(&child_services), _active(true) {}
+   ChildOrder(OrderServices &child_services, OrderServices &parent_services);
 
-   ChildOrder(ChildOrder &&src) { operator=(std::move(src)); }
-   ChildOrder(ChildOrder const &src) { operator=(src); }
+   ChildOrder(ChildOrder &&src);
+   ChildOrder(ChildOrder const &src);
 
    ChildOrder(
        api::orderman::OrderDefinition const &def,
-       OrderServices &parent_services)
-       : _child_order(&parent_services.make_child(def)),
-         _parent_order_services(&parent_services), _child_order_services(),
-         _active(false) {}
+       OrderServices &parent_services);
 
-   ChildOrder &operator=(ChildOrder &&src) {
-      _child_order = src._child_order;
-      _parent_order_services = src._parent_order_services;
-      _child_order_services = src._child_order_services;
-      _active = src._active;
+   ChildOrder &operator=(ChildOrder &&src);
 
-      src._child_order = nullptr;
-      src._parent_order_services = nullptr;
-      src._child_order_services = nullptr;
-      src._active = false;
-      return *this;
-   }
+   ChildOrder &operator=(ChildOrder const &src);
 
-   ChildOrder &operator=(ChildOrder const &src) {
-      _child_order = src._child_order;
-      _parent_order_services = src._parent_order_services;
-      _child_order_services = src._child_order_services;
-      _active = src._active;
-      return *this;
-   }
-
-   bool empty() const { return _child_order == nullptr; }
-   OrderKey key() const {
-      bts_assert_d(_child_order);
-      return _child_order->key();
-   }
-
-   bool is_active() const { return _active; }
-   void warm_up_activate() {
-      bts_assert(!_active);
-      _parent_order_services->warm_up_activate_child(*_child_order);
-   }
-
-   // Activate
-   void activate(Price p, Quantity q) {
-      bts_assert(_child_order);
-      bts_assert(!_active);
-
-      _parent_order_services->activate_child(*_child_order, p, q);
-      _child_order_services = _child_order->_services.get();
-      _active = true;
-   }
+   bool empty() const;
+   OrderKey key() const;
 
+   bool is_active() const;
+   void warm_up_activate();
+   void activate(Price p, Quantity q);
    template <class CB>
-   void activate(Price p, Quantity q, CB const &on_activate_result) {
-      bts_assert(_child_order);
-      bts_assert(!_active);
+   void activate(Price p, Quantity q, CB const &on_activate_result);
 
-      auto on_activate_result_wrapper = [this, on_activate_result](
-                                            OrderOpResult const &r) {
-         _parent_order_services->call_with_reentrancy_guard(
-             [r, on_activate_result](auto const &) { on_activate_result(r); });
-      };
+   void mark_detached_from_parent();
 
-      _parent_order_services->activate_child(
-          *_child_order, p, q, on_activate_result_wrapper);
-      _child_order_services = _child_order->_services.get();
-      _active = true;
-   }
-   // End Activate
-
-   // Mark detached
-   void mark_detached_from_parent() {
-      bts_assert(_child_order);
-      bts_assert(_active);
-      _child_order_services->mark_detached_from_parent();
-   }
-
-   // Update
-   void update(Price p, Quantity q) {
-      bts_assert(_child_order);
-      bts_assert(_active);
-      _child_order_services->update(p, q);
-   }
-
-   template <class CB> void update(Price p, Quantity q, CB const &on_result) {
-      bts_assert(_child_order);
-      bts_assert(_active);
-
-      auto on_result_w = [this, on_result](OrderOpResult const &r) {
-         _parent_order_services->call_with_reentrancy_guard(
-             [r, on_result](auto const &) { on_result(r); });
-      };
-      _child_order_services->update(p, q, on_result_w);
-   }
-   void update(api::orderman::Update const &update) {
-      bts_assert(_child_order);
-      bts_assert(_active);
-      _child_order_services->update(update);
-   }
+   void update(Price p, Quantity q);
+   void update(api::orderman::Update const &update);
    template <class CB>
-   void update(api::orderman::Update const &update, CB const &on_result) {
-      bts_assert(_child_order);
-      bts_assert(_active);
-      auto on_result_w = [this, on_result](OrderOpResult const &r) {
-         _parent_order_services->call_with_reentrancy_guard(
-             [r, on_result](auto const &) { on_result(r); });
-      };
-      _child_order_services->update(update, on_result_w);
-   }
-   // End Update
+   void update(Price p, Quantity q, CB const &on_result);
+   template <class CB>
+   void update(api::orderman::Update const &update, CB const &on_result);
 
-   // Detach
-   void detach() {
-      bts_assert(_child_order);
-      bts_assert(_active);
-      _child_order_services->detach();
-   }
-   template <class CB> void detach(CB const &on_result) {
-      bts_assert(_child_order);
-      bts_assert(_active);
-      auto on_result_w = [this, on_result](OrderOpResult const &r) {
-         _parent_order_services->call_with_reentrancy_guard(
-             [r, on_result](auto const &) { on_result(r); });
-      };
-      _child_order_services->detach(on_result);
-   }
-   // End Detach
+   void detach();
+   template <class CB>
+   void detach(CB const &on_result);
 
-   // Cancel
-   void cancel() {
-      bts_assert(_child_order);
-      bts_assert(_active);
-      _child_order_services->cancel();
-   }
-   template <class CB> void cancel(CB const &on_result) {
-      bts_assert(_child_order);
-      bts_assert(_active);
-      auto on_result_w = [this, on_result](OrderOpResult const &r) {
-         _parent_order_services->call_with_reentrancy_guard(
-             [r, on_result](auto const &) { on_result(r); });
-      };
-      _child_order_services->cancel(on_result_w);
-   }
-   // End Cancel
+   void cancel();
+   template <class CB>
+   void cancel(CB const &on_result);
 
  private:
    friend class OrderBuilder;
@@ -179,5 +63,73 @@ class ChildOrder {
    bool _active;
 };
 
-} // namespace orderman
-} // namespace bts
+//////////////////////////////////////////////////
+// Definition
+//////////////////////////////////////////////////
+
+template <class CB>
+void ChildOrder::activate(Price p, Quantity q, CB const &on_activate_result) {
+   bts_assert(_child_order);
+   bts_assert(!_active);
+
+   auto on_activate_result_wrapper =
+       [this, on_activate_result](OrderOpResult const &r) {
+          _parent_order_services->call_with_reentrancy_guard(
+              [r, on_activate_result](auto const &) { on_activate_result(r); });
+       };
+
+   _parent_order_services->activate_child(
+       *_child_order, p, q, on_activate_result_wrapper);
+   _child_order_services = _child_order->_services.get();
+   _active = true;
+}
+
+template <class CB>
+void ChildOrder::update(Price p, Quantity q, CB const &on_result) {
+   bts_assert(_child_order);
+   bts_assert(_active);
+
+   auto on_result_w = [this, on_result](OrderOpResult const &r) {
+      _parent_order_services->call_with_reentrancy_guard(
+          [r, on_result](auto const &) { on_result(r); });
+   };
+   _child_order_services->update(p, q, on_result_w);
+}
+
+template <class CB>
+void ChildOrder::update(
+    api::orderman::Update const &update, CB const &on_result) {
+   bts_assert(_child_order);
+   bts_assert(_active);
+   auto on_result_w = [this, on_result](OrderOpResult const &r) {
+      _parent_order_services->call_with_reentrancy_guard(
+          [r, on_result](auto const &) { on_result(r); });
+   };
+   _child_order_services->update(update, on_result_w);
+}
+
+// Detach
+template <class CB>
+void ChildOrder::detach(CB const &on_result) {
+   bts_assert(_child_order);
+   bts_assert(_active);
+   auto on_result_w = [this, on_result](OrderOpResult const &r) {
+      _parent_order_services->call_with_reentrancy_guard(
+          [r, on_result](auto const &) { on_result(r); });
+   };
+   _child_order_services->detach(on_result);
+}
+// End Detach
+
+template <class CB>
+void ChildOrder::cancel(CB const &on_result) {
+   bts_assert(_child_order);
+   bts_assert(_active);
+   auto on_result_w = [this, on_result](OrderOpResult const &r) {
+      _parent_order_services->call_with_reentrancy_guard(
+          [r, on_result](auto const &) { on_result(r); });
+   };
+   _child_order_services->cancel(on_result_w);
+}
+
+} // namespace bts::orderman

src/lib/orderman/OrderBuilder.cpp

@@ -1,8 +1,136 @@
 #include "OrderBuilder.h"
 
-namespace bts {
-namespace orderman {
+namespace bts::orderman {
+
+using bts::api::orderman::OrderDefinition;
+using std::size_t;
 
 std::atomic<size_t> OrderBuilder::_current_builder_id(0);
+
+struct OrderBuilder::PImpl {
+   PImpl()
+       : parent_key(null_order_key()), definition(), reserve_sz(0),
+         order_stash(), builder_id(), order_services(nullptr) {}
+
+   PImpl(
+       OrderKey parent_key,
+       const api::orderman::OrderDefinition &def,
+       IOrderFactory &factory,
+       std::size_t reserve_sz = 1)
+       : parent_key(parent_key), definition(def), reserve_sz(reserve_sz),
+         order_stash(), builder_id(++_current_builder_id),
+         order_services(nullptr) {
+      bts_assert(parent_key != null_order_key());
+      while (order_stash.size() != reserve_sz)
+         order_stash.push_back(&factory.create(parent_key, definition));
+   }
+
+   PImpl(
+       OrderKey parent_key,
+       const api::orderman::OrderDefinition &def,
+       std::size_t reserve_sz = 1)
+       : parent_key(parent_key), definition(def), reserve_sz(reserve_sz),
+         order_stash(), builder_id(++_current_builder_id),
+         order_services(nullptr) {
+      bts_assert(parent_key != null_order_key());
+   }
+
+   PImpl &operator=(PImpl &&src) {
+      move_here(src);
+      return *this;
+   }
+
+   PImpl(PImpl &&src) { operator=(std::move(src)); }
+
+   OrderKey parent_key;
+   bts::api::orderman::OrderDefinition definition;
+   std::size_t reserve_sz;
+   std::vector<Order *> order_stash;
+   std::size_t builder_id;
+   OrderServices *order_services;
+
+   ChildOrder build() {
+      bts_assert_msg(
+          order_services != nullptr,
+          "OrderBuilder::activate() must be called before orders can be built");
+
+      ChildOrder result;
+      if (order_stash.size()) {
+         result = ChildOrder(*order_stash.back(), *order_services);
+         order_stash.pop_back();
+      } else {
+         LOG(OrderMan, Warn,
+             "Ran out of pre-allocated child orders for order {}.  Will "
+             "allocate directly. "
+             "This will have an impact on performance",
+             parent_key);
+         result = ChildOrder(
+             order_services->order_factory().create(parent_key, definition),
+             *order_services);
+      }
+      if (reserve_sz > order_stash.size()) {
+         auto need = reserve_sz - order_stash.size();
+         order_services->send_order_create_req(
+             builder_id, parent_key, definition, need);
+      }
+      return result;
+   }
+
+   void move_here(PImpl &src) {
+      order_services = src.order_services;
+      parent_key = src.parent_key;
+      definition = src.definition;
+      reserve_sz = src.reserve_sz;
+      order_stash = src.order_stash;
+      builder_id = src.builder_id;
+      src.order_stash.clear();
+      src.builder_id = 0;
+      src.order_services = nullptr;
+      bts_assert(parent_key != null_order_key());
+   }
+};
+
+OrderBuilder::OrderBuilder() : me_(std::make_unique<PImpl>()) {}
+
+OrderBuilder::OrderBuilder(
+    OrderKey parent_key,
+    OrderDefinition const &def,
+    IOrderFactory &factory,
+    size_t reserve_sz)
+    : me_(std::make_unique<PImpl>(parent_key, def, factory, reserve_sz)) {}
+
+OrderBuilder::OrderBuilder(
+    OrderKey parent_key,
+    bts::api::orderman::OrderDefinition const &def,
+    std::size_t reserve_sz)
+    : me_(std::make_unique<PImpl>(parent_key, def, reserve_sz)) {}
+
+OrderBuilder::OrderBuilder(OrderBuilder &&src) = default;
+
+OrderBuilder &OrderBuilder::operator=(OrderBuilder &&rhs) = default;
+
+OrderBuilder::~OrderBuilder() = default;
+
+bool OrderBuilder::is_activated() const {
+   return me_->order_services != nullptr;
 }
-} // namespace bts
+
+void OrderBuilder::activate(OrderServices &order_services) {
+   bts_assert(!me_->order_services);
+   me_->order_services = &order_services;
+   me_->order_services->subscribe_to_order_creation_reply(
+       me_->builder_id, [this](auto const &order_ev) {
+          me_->order_stash.insert(
+              me_->order_stash.end(), order_ev.orders.begin(),
+              order_ev.orders.end());
+       });
+}
+
+ChildOrder OrderBuilder::build() { return me_->build(); }
+
+OrderServices &OrderBuilder::services() {
+   bts_assert(me_->order_services != nullptr);
+   return *me_->order_services;
+}
+
+} // namespace bts::orderman

src/lib/orderman/OrderBuilder.h

@@ -1,123 +1,46 @@
 #pragma once
+
 #include "lib/api/services/OrderService.pb.h"
-#include "lib/logging/logging.h"
 #include "lib/orderman/ChildOrder.h"
 #include "lib/orderman/OrderServices.h"
 #include "lib/orderman/orderman_utils/orderman_common.h"
-#include <atomic>
-#include <memory>
 
-namespace bts {
-namespace orderman {
+#include <atomic>
+
+namespace bts::orderman {
 
 class OrderBuilder {
  public:
-   OrderBuilder()
-       : _parent_key(null_order_key()), _definition(), _reserve_sz(0),
-         _order_stash(), _builder_id(), _order_services(nullptr) {}
-
-   OrderBuilder(OrderBuilder &&src) { operator=(std::move(src)); }
+   OrderBuilder();
+   OrderBuilder(OrderBuilder &&src);
 
    OrderBuilder(
        OrderKey parent_key,
        api::orderman::OrderDefinition const &def,
        IOrderFactory &factory,
-       size_t reserve_sz = 1)
-       : _parent_key(parent_key), _definition(def), _reserve_sz(reserve_sz),
-         _order_stash(), _builder_id(++_current_builder_id),
-         _order_services(nullptr)
-
-   {
-      bts_assert(parent_key != null_order_key());
-      while (_order_stash.size() != reserve_sz)
-         _order_stash.push_back(&factory.create(parent_key, _definition));
-   }
+       size_t reserve_sz = 1);
 
    OrderBuilder(
        OrderKey parent_key,
        api::orderman::OrderDefinition const &def,
-       size_t reserve_sz = 1)
-       : _parent_key(parent_key), _definition(def), _reserve_sz(reserve_sz),
-         _order_stash(), _builder_id(++_current_builder_id),
-         _order_services(nullptr)
+       size_t reserve_sz = 1);
 
-   {
-      bts_assert(parent_key != null_order_key());
-   }
+   ~OrderBuilder();
 
-   bool is_activated() { return _order_services != nullptr; }
+   OrderBuilder &operator=(OrderBuilder &&src);
 
-   void activate(OrderServices &order_services) {
-      bts_assert(!_order_services);
-      _order_services = &order_services;
-      _order_services->subscribe_to_order_creation_reply(
-          _builder_id, [this](auto const &order_ev) {
-             _order_stash.insert(
-                 _order_stash.end(), order_ev.orders.begin(),
-                 order_ev.orders.end());
-          });
-   }
-
-   ChildOrder build() {
-      bts_assert_msg(
-          _order_services,
-          "OrderBuilder::activate() must be called before orders can be built");
-
-      ChildOrder result;
-      if (_order_stash.size()) {
-         result = ChildOrder(*_order_stash.back(), *_order_services);
-         _order_stash.pop_back();
-      } else {
-         LOG(OrderMan, Warn,
-             "Ran out of pre-allocated child orders for order {}.  Will "
-             "allocate directly. "
-             "This will have an impact on performance",
-             _parent_key);
-         result = ChildOrder(
-             _order_services->order_factory().create(_parent_key, _definition),
-             *_order_services);
-      }
-      if (_reserve_sz > _order_stash.size()) {
-         auto need = _reserve_sz - _order_stash.size();
-         _order_services->send_order_create_req(
-             _builder_id, _parent_key, _definition, need);
-      }
-      return result;
-   }
-
-   OrderBuilder &operator=(OrderBuilder &&src) {
-      move_here(src);
-      return *this;
-   }
-
-   OrderServices &services() {
-      bts_assert(_order_services != nullptr);
-      return *_order_services;
-   }
+   bool is_activated() const;
+   void activate(OrderServices &order_services);
+   ChildOrder build();
+   OrderServices &services();
 
  private:
-   void move_here(OrderBuilder &src) {
-      _order_services = src._order_services;
-      _parent_key = src._parent_key;
-      _definition = src._definition;
-      _reserve_sz = src._reserve_sz;
-      _order_stash = src._order_stash;
-      _builder_id = src._builder_id;
-      src._order_stash.clear();
-      src._builder_id = 0;
-      src._order_services = nullptr;
-      bts_assert(_parent_key != null_order_key());
-   }
+   static std::atomic<std::size_t> _current_builder_id;
 
-   OrderKey _parent_key;
-   api::orderman::OrderDefinition _definition;
-   size_t _reserve_sz;
-   std::vector<Order *> _order_stash;
-   size_t _builder_id;
-   OrderServices *_order_services;
+   struct PImpl;
+   std::unique_ptr<PImpl> me_;
 
-   static std::atomic<size_t> _current_builder_id;
+   void move_here(OrderBuilder &src);
 };
 
-} // namespace orderman
-} // namespace bts
+} // namespace bts::orderman

src/lib/orderman/Trade.h

@@ -0,0 +1,17 @@
+#pragma once
+
+#include "lib/domain_model/common.h"
+#include "lib/md/protos/co2.pb.h"
+#include "lib/utils/bts_chrono.h"
+
+namespace bts::orderman {
+
+struct Trade {
+   TimeStamp ts;
+   Price price;
+   Quantity quantity;
+   Quantity total_volume;
+   co2::AggressorSide aggressor_side;
+};
+
+} // namespace bts::orderman

src/lib/orderman/TradeVector.h

@@ -0,0 +1,11 @@
+#pragma once
+
+#include "lib/orderman/Trade.h"
+
+#include <boost/container/small_vector.hpp>
+
+namespace bts::orderman {
+
+typedef boost::container::small_vector<Trade, 5> TradeVector;
+
+} // namespace bts::orderman

src/lib/orderman/actors/MDSource.h

@@ -5,7 +5,7 @@
 #include "lib/md/IntPrice.h"
 #include "lib/md/MDConsumer.h"
 #include "lib/md/MDConsumerHandle.h"
-#include "lib/md/cme/MDP3Utils.h"
+#include "server/sparkd/md/cme/MDP3Utils.h"
 #include "lib/mercury/actors/Actor.h"
 #include "lib/mercury/actors/IPollableActor.h"
 #include "lib/mercury/events/TimerEvent.h"
@@ -51,6 +51,7 @@ class MDSource : public mercury::Actor, public mercury::IPollableActor {
 
    void init() override {
       LOG(OrderMan, Warn, "Starting MDSource actor");
+      // TODO: why is this hitting?
       bts_assert_msg(
           _md_consumer.all_heartbeats_ok(),
           "Marketdata heartbeats are bad. MDConsumer will not recover from "
@@ -159,7 +160,7 @@ class MDSource : public mercury::Actor, public mercury::IPollableActor {
             for (auto &per_sec_ev : c.securities) {
                if (trade.key() == per_sec_ev.key().value()) {
                   trade_count++;
-                  per_sec_ev.trades.emplace_back(TickerUpdateEvent::Trade{
+                  per_sec_ev.trades.emplace_back(Trade{
                       datetime::utc_ll_to_timepoint(trade.time()),
                       Price(md::IntPrice(
                                 trade.mantissa(), (int8_t)trade.exponent())

src/lib/orderman/actors/order_manager/OrderManager.cpp

@@ -381,6 +381,9 @@ void OrderManager::on_gateway_state(
     const mercury::Update<GatewayStatusChangeEvent> &m) {
    if (m.events)
       for (auto const &ev : *m.events) {
+         LOG(OrderMan, Info,
+             " OrderManager::on_gateway_state gw: {} connected: {}", ev.gw_name,
+             ev.is_connected);
          auto &sr = me->services_data.get_session(ev.gw_name.value());
          sr.gw_connected = ev.is_connected;
       }

src/lib/orderman/ccapi/CCAPIDeribitHelpers.cpp

@@ -1,24 +1,9 @@
 #include "CCAPIDeribitHelpers.h"
-#include "CCAPIDeribitOrderMux.h"
-#include "CCAPIOrderState.h"
-#include "CCAPISingleOrderManager.h"
-#include "ccapi_cpp/ccapi_session.h"
 #include "lib/api/services/OrderService.pb.h"
-#include "lib/core/TimeStamp.h"
 #include "lib/domain_model/Security.pb.h"
 #include "lib/io/JsonReader.h"
-#include "lib/logging/logging.h"
-#include "lib/md/SecurityOracle.h"
-#include "lib/md/binance/BinanceSpot.h"
-#include "lib/orderman/CancelOrderMsg.h"
-#include "lib/orderman/CancelReplaceOrderMsg.h"
-#include "lib/orderman/CreateOrderMsg.h"
-#include "lib/orderman/RFQMsg.h"
-#include "lib/orderman/ccapi/CCAPIOrderCallbackMsgs.h"
 #include "lib/orderman/ccapi/ICCAPIOrderMux.h"
-#include "lib/orderman/orderman_utils/orderman_common.h"
-#include "lib/utils/PBUtils.h"
-#include "lib/utils/bts_assert.h"
+
 #include <bits/stdint-uintn.h>
 #include <boost/lockfree/spsc_queue.hpp>
 #include <ccapi_cpp/ccapi_event.h>
@@ -27,16 +12,11 @@
 #include <ccapi_cpp/ccapi_request.h>
 #include <ccapi_cpp/ccapi_subscription.h>
 #include <ccapi_cpp/ccapi_util_private.h>
-#include <chrono>
 #include <google/protobuf/timestamp.pb.h>
 #include <google/protobuf/util/time_util.h>
-#include <memory>
 #include <nlohmann/json_fwd.hpp>
 #include <optional>
-#include <ostream>
-#include <random>
 #include <string>
-#include <unordered_map>
 #include <vector>
 
 namespace bts::orderman {
@@ -155,4 +135,4 @@ bool CCAPIDeribitHelpers::parse_working_position(
    return true;
 }
 
-} // namespace bts::orderman
+} // namespace bts::orderman

src/lib/orderman/ccapi/CCAPIDeribitOrderMux.cpp

@@ -1,3 +1,5 @@
+#ifdef SPARK_ENABLE_CCAPI
+
 #include "CCAPIDeribitOrderMux.h"
 #include "CCAPIOrderState.h"
 #include "CCAPISingleOrderManager.h"
@@ -953,4 +955,6 @@ void CCAPIDeribitOrderMux::_send_buffered_orders(uint32_t sec_id) {
        "n_buffered_orders: {}",
        me->buffered_creates.size());
 }
-} // namespace bts::orderman
+} // namespace bts::orderman
+
+#endif

src/lib/orderman/ccapi/CCAPIDeribitOrderMux.h

@@ -1,4 +1,6 @@
 #pragma once
+#ifdef SPARK_ENABLE_CCAPI
+
 #include "ICCAPIOrderMux.h"
 #include "lib/md/SecurityOracle.h"
 #include "lib/orderman/CreateOrderMsg.h"
@@ -72,4 +74,6 @@ class CCAPIDeribitOrderMux : public ICCAPIOrderMux {
    void _handle_generic_pvt_request(const nlohmann::json &json_obj);
 };
 
-} // namespace bts::orderman
+} // namespace bts::orderman
+
+#endif

src/lib/orderman/events/GatewayStatusChangeEvent.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include "lib/logging/logging.h"
 #include "lib/mercury/synopsis/CacheLast.h"
 #include "lib/orderman/orderman_utils/orderman_common.h"
 #include <memory>
@@ -14,7 +15,11 @@ class GatewayStatusChangeEvent
    using SynopsisPolicy = mercury::CacheLast;
 
    GatewayStatusChangeEvent(GatewayName const &n, bool c)
-       : gw_name(n), is_connected(c) {}
+       : gw_name(n), is_connected(c) {
+      LOG(OrderMan, Info,
+          "GatewayStatusChangeEvent::cons gw_name: {} is_connected: {}",
+          gw_name, is_connected);
+   }
 
    GatewayName gw_name;
    bool is_connected;

src/lib/orderman/events/TickerUpdateEvent.h

@@ -1,35 +1,30 @@
 #pragma once
 
-#include <memory>
-
-#include "boost/container/small_vector.hpp"
-#include "lib/md/protos/co2.pb.h"
 #include "lib/mercury/events/Event.h"
-#include "lib/orderman/orderman_utils/orderman_common.h"
-#include "lib/utils/bts_chrono.h"
+#include "lib/orderman/TradeVector.h"
 
-namespace bts {
-namespace orderman {
+namespace bts::orderman {
 
 class TickerUpdateEvent : public mercury::Event<TickerUpdateEvent> {
-   SecurityKey _key;
+   SecurityKey key_;
 
  public:
    using SynopsisPolicy = mercury::NoSynopsis;
 
-   SecurityKey key() const { return _key; };
+   TradeVector trades;
 
-   struct Trade {
-      TimeStamp ts;
-      Price price;
-      Quantity quantity;
-      Quantity total_volume;
-      co2::AggressorSide aggressor_side;
-   };
+   TickerUpdateEvent(SecurityKey key);
 
-   boost::container::small_vector<TickerUpdateEvent::Trade, 5> trades;
-   TickerUpdateEvent(SecurityKey key) : _key(key), trades() {}
+   SecurityKey key() const;
 };
-typedef boost::container::small_vector<TickerUpdateEvent::Trade, 5> TradeVector;
-} // namespace orderman
-} // namespace bts
+
+//////////////////////////////////////////////////
+// Definition
+//////////////////////////////////////////////////
+
+inline SecurityKey TickerUpdateEvent::key() const { return key_; };
+
+inline TickerUpdateEvent::TickerUpdateEvent(SecurityKey key)
+    : key_(key), trades() {}
+
+} // namespace bts::orderman

src/lib/orderman/gateways/CCAPIExecutionGateway.cpp

@@ -1,3 +1,5 @@
+#ifdef SPARK_ENABLE_CCAPI
+
 #include "CCAPIExecutionGateway.h"
 #include "lib/api/service_helpers/AdminServiceUtils.h"
 #include "lib/api/services/AdminService.pb.h"
@@ -492,4 +494,6 @@ SingleOrderMux *CCAPIExecutionGateway::_find_mux_from_orderkey(OrderKey key) {
    return _find_mux(it->second.routing_record);
 }
 
-} // namespace bts::orderman
+} // namespace bts::orderman
+
+#endif

src/lib/orderman/gateways/CCAPIExecutionGateway.h

@@ -1,6 +1,7 @@
 #pragma once
+#ifdef SPARK_ENABLE_CCAPI
 
-#include "IExecutionGateway.h"
+#include "lib/orderman/gateways/IExecutionGateway.h"
 #include "lib/api/services/AdminService.pb.h"
 #include "lib/api/services/OrderService.pb.h"
 #include "lib/domain_model/Security.pb.h"
@@ -10,7 +11,7 @@
 #include "lib/orderman/RFQMsg.h"
 #include "lib/orderman/ccapi/CCAPIOrderState.h"
 #include "lib/orderman/orderman_utils/orderman_common.h"
-#include "server/sparkd/md/platforms/emumba/IEMOrderMuxer.h"
+
 #include <bits/stdint-uintn.h>
 #include <cstdint>
 
@@ -73,3 +74,5 @@ class CCAPIExecutionGateway : public IExecutionGateway {
 
 } // namespace orderman
 } // namespace bts
+
+#endif

src/lib/orderman/gateways/ExecutionGatewayFactory.cpp

@@ -2,7 +2,7 @@
 #include "lib/domain_model/Security.pb.h"
 #include "lib/md/SecurityOracle.h"
 #include "lib/orderman/gateways/CCAPIExecutionGateway.h"
-#include "lib/orderman/gateways/EMExecutionGateway.h"
+// #include "lib/orderman/gateways/EMExecutionGateway.h"
 #include "lib/orderman/gateways/FakeExecutionGateway.h"
 #include "lib/orderman/gateways/GLBXExecutionGateway.h"
 #include "lib/orderman/gateways/GLBXiLink3ExecutionGateway.h"
@@ -34,9 +34,11 @@ std::unique_ptr<IExecutionGateway> ExecutionGatewayFactory::create_gateway(
          return std::make_unique<ilink3::GLBXiLink3ExecutionGateway>(
              conn, pl, o);
       return std::make_unique<GLBXExecutionGateway>(conn, pl, o);
-   } else if (conn.venue() == bts::api::VENUE_DERIBIT)
-      //   return std::make_unique<EMExecutionGateway>(o);
+   } else if (conn.venue() == bts::api::VENUE_DERIBIT) {
+#ifdef SPARK_ENABLE_CCAPI
       return std::make_unique<CCAPIExecutionGateway>(conn.venue(), o);
+#endif
+   }
    return std::make_unique<FakeExecutionGateway>(o);
 }
 

src/lib/orderman/gateways/GLBXiLink3ExecutionGatewayState.h

@@ -1,5 +1,4 @@
 #pragma once
-#ifdef SPARK_ENABLE_CCAPI
 
 #include "lib/core/EnumMacros.h"
 #include "lib/core/FMT_IMPL.h"
@@ -26,7 +25,6 @@ ENUM_CLASS_DEF(
 #undef FOR_EACH_GLBX_ILINK3_EXECUTION_GATEWAY_STATE
 
 } // namespace bts::orderman::ilink3
-#endif
 
 FMT_IMPL(
     bts::orderman::ilink3::GLBXiLink3ExecutionGatewayState, s, "{}", s.name());

src/lib/orderman/gateways/sim/SimExecutionGateway.cpp

@@ -30,6 +30,7 @@
 #include "lib/utils/bts_assert.h"
 #include "lib/utils/bts_chrono.h"
 #include "lib/utils/helper_macros.h"
+#include "lib/utils/thread_manager.h"
 #include "server/sparkd/md/platforms/emumba/IEMOrderMuxer.h"
 
 #include <bits/stdint-uintn.h>
@@ -227,13 +228,19 @@ struct SEGSecurityData {
          const CancelReplaceOrderMsg &cxlr_msg =
              std::get<CancelReplaceOrderMsg>(msg);
          SEGOrderState *seg_os = find_order(cxlr_msg.create_fields.key);
-         bts_assert(seg_os != nullptr);
+         if(seg_os == nullptr){
+            LOG(OrderMan, Info, "SimExecutionGateway::apply_order_action tried updating unkown order");
+            return {ActionResult::NoOp, nullptr};
+         }
          auto action_result = seg_os->apply_change(cxlr_msg);
          return {action_result, seg_os};
       } else if (std::holds_alternative<CancelOrderMsg>(msg)) {
          const CancelOrderMsg &cxl_msg = std::get<CancelOrderMsg>(msg);
          SEGOrderState *seg_os = find_order(cxl_msg.key);
-         bts_assert(seg_os != nullptr);
+         if(seg_os == nullptr){
+            LOG(OrderMan, Info, "SimExecutionGateway::apply_order_action tried canceling unkown order");
+            return {ActionResult::NoOp, nullptr};
+         }
          auto action_result = seg_os->apply_cxl(cxl_msg);
          return {action_result, seg_os};
       }
@@ -246,11 +253,12 @@ struct SEGSecurityData {
       }
    }
    void remove_inactive_orders() {
-      orders.erase(
-          std::remove_if(
-              orders.begin(), orders.end(),
-              [](const auto &ord) { return !ord.active(); }),
-          orders.end());
+      return;
+      // orders.erase(
+      //     std::remove_if(
+      //         orders.begin(), orders.end(),
+      //         [](const auto &ord) { return !ord.active(); }),
+      //     orders.end());
    }
    void record_trade(const SEGTradeEntry &seg_te) { trades.push_back(seg_te); }
 };
@@ -347,12 +355,26 @@ void SimExecutionGateway::on_admin_data(
        bts::os::get_thread_name());
 }
 
-void SimExecutionGateway::send_fix(const jetfix::OutFIXPtr &, trace::CorID) {
-   bts_assert(false);
+void SimExecutionGateway::send_fix(const jetfix::OutFIXPtr &p, trace::CorID) {
+   LOG(OrderMan, Warn, "SimExecutionGateway::send_fix p sz: {}", p->fix.size());
+   // TODO
+   //bts_assert(false);
 }
 
-// This is where the messages are sent to the exchange. We simply write the
-// operation to a concurrent queue and dequeue in the market data thread
+int msg_type(const OrderMsg &msg) {
+   // if (std::holds_alternative<CreateOrderMsg>(msg)) {
+   //    return 1;
+   // } else if (std::holds_alternative<CancelReplaceOrderMsg>(msg)) {
+   //    return 2;
+   // } else if (std::holds_alternative<CancelOrderMsg>(msg)) {
+   //    return 3;
+   // }
+   return msg.index();
+}
+
+// This is where the messages are received from the client. For sim,
+// we simply write the operation to a concurrent queue (buffered_order_ops)
+// and dequeue in the market data thread (when the market data source needs the matched_books)
 // (the resulting execution rerports are written to another conrrent queue)
 // which is then dequeued on dispatch_incoming_fix_messages)
 void SimExecutionGateway::send_msgs(
@@ -364,15 +386,20 @@ void SimExecutionGateway::send_msgs(
        "SimExecutionGateway::send_msgs tid: {} tname: {} count: {}",
        bts::os::get_thread_id(), bts::os::get_thread_name(), msg_vec.size());
    for (auto &m : msg_vec) {
-      LOG(OrderMan, Diag,
-          "SimExecutionGateway::send_msgs tid: {} tname: {} m: {}",
-          bts::os::get_thread_id(), bts::os::get_thread_name(), m);
+      auto mtype = msg_type(m);
+      LOG(OrderMan, Info,
+          "SimExecutionGateway::send_msgs tid: {} tname: {} m_type: {}",
+          bts::os::get_thread_id(), bts::os::get_thread_name(), mtype);
       me->buffered_order_ops.push(m);
    }
 }
 
 OnExeReport &SimExecutionGateway::on_exe_report() { return me->on_exe_report; }
 OnGwStatusChange &SimExecutionGateway::on_gw_status_change() {
+   std::string tid_name = "SimExecutionGateway::_send_gw_status_once";
+   bts::threading::make_thread(tid_name, [&]() {
+      _send_gw_status_once();
+   }).detach();
    return me->on_gw_status_change;
 }
 OnCancelReject &SimExecutionGateway::on_cancel_reject() {
@@ -394,15 +421,16 @@ void SimExecutionGateway::set_producers(
 
 void SimExecutionGateway::do_work() {}
 
+// this gets called by GeneralMarketDataListener, so this can alter the based on order activity
 bool SimExecutionGateway::get_matched_book(
     bts::md::SecurityKey k,
-    std::array<md::PriceSize, md::COCONUT_MAX_DEPTH> &bids,
-    std::array<md::PriceSize, md::COCONUT_MAX_DEPTH> &asks) {
+    std::array<md::PriceSize, md::COCONUT_MAX_DEPTH> &out_bids,
+    std::array<md::PriceSize, md::COCONUT_MAX_DEPTH> &out_asks) {
    bts::SecurityKey key = k;
    // security entry is added here
    SEGSecurityData &seg_sec = *_find_add_security_entry(key, true, true);
    // this is where the book is updated
-   to_book(bids, asks, seg_sec.real_book);
+   to_book(out_bids, out_asks, seg_sec.real_book);
    if (seg_sec.orders.empty() && seg_sec.trades.empty() &&
        me->buffered_order_ops.empty())
       return false;
@@ -417,6 +445,10 @@ bool SimExecutionGateway::get_matched_book(
    OrderMsg ord_msg;
    // all ord_msgs are popped and enqueued in the security's SEGSecurityData
    while (me->buffered_order_ops.try_pop(ord_msg)) {
+      // LOG(OrderMan, Info,
+      //     "SimExecutionGateway::get_matched_book ord_msg dequeued from "
+      //     "get_matched_book, ord_msg: {}",
+      //     ord_msg);
       std::optional<SecurityKey> op_sec_key = _extract_sec_key(ord_msg);
       if (!op_sec_key.has_value()) {
          LOG(OrderMan, Error,
@@ -436,7 +468,7 @@ bool SimExecutionGateway::get_matched_book(
    }
    // this takes care of everything !
    _process_transaction();
-   from_book(seg_sec.matched_book, bids, asks);
+   from_book(seg_sec.matched_book, out_bids, out_asks);
    // cleanup
    for (SecurityKey key : me->touched_secs) {
       SEGSecurityData &t_seg_sec = *_find_add_security_entry(key, false, true);
@@ -675,6 +707,13 @@ SimExecutionGateway::_find_order(OrderKey key) const {
        nullptr, nullptr);
 }
 
+void SimExecutionGateway::_send_gw_status_once() {
+   auto sleep_period = std::chrono::milliseconds(2500);
+   std::this_thread::sleep_for(sleep_period);
+   // fire the status and exit the thread
+   me->on_gw_status_change(true);
+}
+
 // this returns the security key from an OrdMsg. The reason it has more code
 // than needed is because CancelOrderMsg might not have the security set.
 // therefore we need to find the security from the order key

src/lib/orderman/gateways/sim/SimExecutionGateway.h

@@ -76,6 +76,7 @@ class SimExecutionGateway : public IExecutionGateway {
    std::tuple<SEGSecurityData const *, SEGOrderState const *> _find_order(
        OrderKey key) const;
    std::optional<SecurityKey> _extract_sec_key(const OrderMsg &msg);
+   void _send_gw_status_once();
 
  private:
    struct PImpl;

src/lib/orderman/gateways/sim/SimExecutionGatewayFactory.cpp

@@ -61,6 +61,8 @@ SimExecutionGatewayFactory *SimExecutionGatewayFactory::get() {
 }
 
 bool SimExecutionGatewayFactory::is_sim_execution_on(api::Venue venue) {
+   // TODO: Arham, fix this !
+   return true;
    const auto &sim_exec = config::get<bts::config::SimExecution>();
    for (const auto &e : sim_exec.entries()) {
       if (e.venue() == venue) {

src/lib/orderman/gateways/sim/SimExecutionGatewayUniqueWrapper.cpp

@@ -4,6 +4,8 @@
 #include "lib/domain_model/Security.pb.h"
 #include "lib/logging/logging.h"
 #include <memory>
+#include <sstream>
+#include <streambuf>
 
 namespace bts::orderman {
 
@@ -52,10 +54,10 @@ void SimExecutionGatewayUniqueWrapper::send_fix(
 }
 void SimExecutionGatewayUniqueWrapper::send_msgs(
     std::vector<OrderMsg> const &msgs, trace::CorID cid) {
-   for (const auto &m : msgs) {
-      LOG(OrderMan, Info, "SimExecutionGatewayUniqueWrapper::send_msgs m: {}",
-          m);
-   }
+   LOG(OrderMan, Info,
+       "SimExecutionGatewayUniqueWrapper::send_msgs tid: {} tname: {} count: "
+       "{}",
+       bts::os::get_thread_id(), bts::os::get_thread_name(), msgs.size());
    me->gateway->send_msgs(msgs, cid);
 }
 

src/lib/orderman/ilink3/IL3State.h

@@ -1,5 +1,4 @@
 #pragma once
-#ifdef SPARK_ENABLE_CCAPI
 
 #include "lib/core/EnumMacros.h"
 #include "lib/core/FMT_IMPL.h"
@@ -27,6 +26,5 @@ ENUM_CLASS_DEF(IL3State, FOR_EACH_IL3_STATE)
 #undef FOR_EACH_IL3_STATE
 
 } // namespace bts::orderman::ilink3
-#endif
 
 FMT_IMPL(bts::orderman::ilink3::IL3State, s, "{}", s.name());

src/lib/orderman/order_factory/IOrderFactory.h

@@ -1,35 +1,35 @@
 #pragma once
 
 #include "lib/orderman/orderman_utils/orderman_common.h"
-#include <memory>
 
-// fwd
-namespace bts {
-namespace orderman {
+namespace bts::orderman {
 class Order;
-}
-namespace api {
-namespace orderman {
+} // namespace bts::orderman
+
+namespace bts::api::orderman {
 class OrderDefinition;
 class OrderRecord;
-} // namespace orderman
-} // namespace api
-} // namespace bts
-// end fwd
+} // namespace bts::api::orderman
 
-namespace bts {
-namespace orderman {
+namespace bts::orderman {
 
-inline OrderKey null_order_key() { return OrderKey(0); }
+constexpr OrderKey null_order_key();
 
 class IOrderFactory {
  public:
-   Order &create(api::orderman::OrderDefinition const &def) {
-      return create(null_order_key(), def);
-   }
-   virtual Order &
-   create(OrderKey parent, api::orderman::OrderDefinition const &) = 0;
+   Order &create(api::orderman::OrderDefinition const &def);
+   virtual Order &create(
+       OrderKey parent, api::orderman::OrderDefinition const &) = 0;
 };
 
-} // namespace orderman
-} // namespace bts
+//////////////////////////////////////////////////
+// Definition
+//////////////////////////////////////////////////
+
+constexpr OrderKey null_order_key() { return OrderKey{0}; }
+
+inline Order &IOrderFactory::create(api::orderman::OrderDefinition const &def) {
+   return create(null_order_key(), def);
+}
+
+} // namespace bts::orderman

src/lib/orderman/orderman_utils/Book.h

@@ -1,50 +1,35 @@
 #pragma once
 
-#include "lib/config/protos/market_data.pb.h"
-#include "lib/domain_model/common.h"
+#include "lib/core/FMT_IMPL.h"
 #include "lib/md/PriceSize.h"
-#include <memory>
 
-namespace bts {
-namespace orderman {
+#include <array>
+
+namespace bts::orderman {
 
 using BookSide = std::array<md::PriceSize, 5>;
 
 class Book {
  public:
-   Book() : _security_key(), _book() {}
+   Book();
+   Book(SecurityKey security_key);
+   Book(SecurityKey security_key, const BookSide& bid, const BookSide& ask);
+   Book(const Book& rhs);
 
-   Book(SecurityKey key) : _security_key(key), _book() {}
+   Book& operator=(Book const& rhs);
 
-   Book(SecurityKey key, BookSide const &bid, BookSide const &ask)
-       : _security_key(key), _book{bid, ask} {}
+   constexpr SecurityKey security_key() const;
 
-   Book(Book const &rhs) : _security_key(rhs._security_key), _book(rhs._book) {}
+   const BookSide& bid() const;
+   BookSide& bid();
 
-   Book &operator=(Book const &rhs) {
-      _security_key = rhs._security_key;
-      _book = rhs._book;
-      return *this;
-   }
+   const BookSide& ask() const;
+   BookSide& ask();
 
-   SecurityKey security_key() const { return _security_key; };
+   size_t n_levels() const;
 
-   BookSide const &bid() const { return _book[0]; }
-   BookSide const &ask() const { return _book[1]; }
-   BookSide &bid() { return _book[0]; }
-   BookSide &ask() { return _book[1]; }
-
-   size_t n_levels() const { return std::tuple_size<BookSide>(); }
-
-   BookSide const &side(api::Side bs) const {
-      bts_assert_d(bs != api::SIDE_UNDEFINED);
-      return _book[bs - api::SIDE_BUY];
-   }
-
-   BookSide &side(api::Side bs) {
-      bts_assert_d(bs != api::SIDE_UNDEFINED);
-      return _book[bs - api::SIDE_BUY];
-   }
+   const BookSide& side(api::Side bs) const;
+   BookSide& side(api::Side bs);
 
  private:
    friend class MDSource;
@@ -52,5 +37,57 @@ class Book {
    std::array<BookSide, 2> _book;
 };
 
-} // namespace orderman
-} // namespace bts
+//////////////////////////////////////////////////
+// Definition
+//////////////////////////////////////////////////
+
+inline Book::Book() : _security_key(), _book() {}
+
+inline Book::Book(SecurityKey security_key)
+    : _security_key(security_key), _book() {}
+
+inline Book::Book(
+    SecurityKey security_key, const BookSide& bid, const BookSide& ask)
+    : _security_key(security_key), _book{bid, ask} {}
+
+inline Book::Book(const Book& rhs)
+    : _security_key(rhs._security_key), _book(rhs._book) {}
+
+inline Book& Book::operator=(const Book& rhs) {
+   _security_key = rhs._security_key;
+   _book = rhs._book;
+   return *this;
+}
+
+constexpr SecurityKey Book::security_key() const { return _security_key; };
+
+inline const BookSide& Book::bid() const { return _book[0]; }
+inline const BookSide& Book::ask() const { return _book[1]; }
+inline BookSide& Book::bid() { return _book[0]; }
+inline BookSide& Book::ask() { return _book[1]; }
+
+inline std::size_t Book::n_levels() const {
+   return std::tuple_size<BookSide>();
+}
+
+inline const BookSide& Book::side(api::Side bs) const {
+   bts_assert_d(bs != api::SIDE_UNDEFINED);
+   return _book[bs - api::SIDE_BUY];
+}
+
+inline BookSide& Book::side(api::Side bs) {
+   bts_assert_d(bs != api::SIDE_UNDEFINED);
+   return _book[bs - api::SIDE_BUY];
+}
+
+} // namespace bts::orderman
+
+// Formats bbo
+FMT_IMPL(
+    bts::orderman::Book,
+    b,
+    "({}) {} <=> {} ({})",
+    b.bid()[0].size(),
+    b.bid()[0].price(),
+    b.ask()[0].price(),
+    b.ask()[0].size());

src/lib/orderman/orders/GLBXNativeOrder.cpp

@@ -4,11 +4,14 @@
 #include "lib/config/protos/orderman_cfg.pb.h"
 #include "lib/config/protos/toggles.pb.h"
 #include "lib/domain_model/Security.h"
+#include "lib/domain_model/Security.pb.h"
+#include "lib/domain_model/common.h"
 #include "lib/jetfix/codec/FIXFragment.h"
 #include "lib/jetfix/codec/FieldDefinitions.h"
 #include "lib/logging/logging.h"
 #include "lib/orderman/OrderRouter.h"
 #include "lib/orderman/OrderServices.h"
+#include "lib/orderman/gateways/sim/SimExecutionGatewayFactory.h"
 #include "lib/orderman/order_factory/IOrderFactory.h"
 #include "lib/orderman/orderman_utils/ExecutionReport.h"
 #include "lib/orderman/orderman_utils/fix_order_codec.h"
@@ -65,6 +68,8 @@ struct GLBXNativeOrder::PImpl {
    std::optional<api::orderman::Update> buffered_update;
    RoutingRecord routing_record;
    bool stop_order_still_has_stop_px = true;
+   const bool in_sim_mode =
+       SimExecutionGatewayFactory::is_sim_execution_on(api::VENUE_GLBX);
 };
 
 GLBXNativeOrder::GLBXNativeOrder() : me(new PImpl()) {}
@@ -471,8 +476,13 @@ OrderOpResult GLBXNativeOrder::on_activate(
              "delta:{}",
              string{corid.channel}, corid.sn, corid.app_arrival.to_uint64(),
              now.to_uint64(), (now - corid.app_arrival).nanoseconds());
-
+         // in sim mode, we dont need this,
          me->encoder->send_create(*me->services, *me->session_services);
+         if (me->in_sim_mode) {
+            CreateOrderMsg c_msg = {};
+            _create_create_msg(c_msg);
+            me->services->queue_order_msg(*me->session_services, c_msg);
+         }
          me->services->governor(*me->session_services).record_sent();
          me->state.set_status(ORDER_STATUS_PENDING_WORKING);
 
@@ -568,6 +578,19 @@ OrderOpResult GLBXNativeOrder::on_update(Price p, Quantity q) {
 OrderOpResult GLBXNativeOrder::on_update(api::orderman::Update const &update) {
    auto p = Price(update.price());
    auto q = Quantity(update.quantity());
+   auto send_update_in_sim_mode = [&]() {
+      if (me->in_sim_mode) {
+         CancelReplaceOrderMsg cxl_repl_msg = {};
+         _create_create_msg(cxl_repl_msg.create_fields);
+         Price new_p = Price(update.price());
+         Quantity new_q = bts::orderman::get_update_quantity(update);
+         cxl_repl_msg.create_fields.price = new_p;
+         cxl_repl_msg.create_fields.quantity = new_q;
+         me->services->queue_order_msg(*me->session_services, cxl_repl_msg);
+      }
+   };
+   LOG(OrderMan, Info, "GLBXNativeOrder::on_update update: {}",
+       utils::pb::to_string(update));
 
    auto scoped_log =
        me->services->log_scope("NativeOrder::on_update", key().value());
@@ -630,6 +653,7 @@ OrderOpResult GLBXNativeOrder::on_update(api::orderman::Update const &update) {
                me->services->report_definition(me->definition);
                me->encoder->update_cancel_replace_stop_px(
                    stop_px, me->price_encoding_precision);
+               send_update_in_sim_mode();
             }
          } else if (
              update.custom().custom_case() ==
@@ -673,6 +697,7 @@ OrderOpResult GLBXNativeOrder::on_update(api::orderman::Update const &update) {
       }
       me->encoder->update_cancel_replace_encoding(q, *me->services);
       me->encoder->send_cancel_replace(*me->services, *me->session_services);
+      send_update_in_sim_mode();
 
       me->services->governor(*me->session_services).record_sent();
       update_working_position();
@@ -773,6 +798,13 @@ OrderOpResult GLBXNativeOrder::on_cancel() {
    case ORDER_STATUS_WORKING: {
       me->encoder->update_cancel_encoding(*me->services);
       me->encoder->send_cancel(*me->services, *me->session_services);
+      // TODO: check for sim mode here
+      if (me->in_sim_mode) {
+         CancelOrderMsg cxl_msg = {};
+         cxl_msg.key = key();
+         LOG(OrderMan, Info, "GLBXNativeOrder queue_order_msg for cxl");
+         me->services->queue_order_msg(*me->session_services, cxl_msg);
+      }
       break;
    }
 
@@ -1326,5 +1358,37 @@ void GLBXNativeOrder::fixup_routing_record(api::orderman::RoutingRecord *rr) {
                                           .fallback_cust_order_handling_inst());
 }
 
+// TODO, make it static
+NativeOrderType get_glbx_native_ord_type(const OrderDefinition &def) {
+   switch (def.custom_def().custom_case()) {
+   case CustomOrderDefinition::kMarket:
+      return NativeOrderType::NATIVE_ORDER_TYPE_MARKET;
+      break;
+   case CustomOrderDefinition::kLimit:
+      return NativeOrderType::NATIVE_ORDER_TYPE_LIMIT;
+      break;
+   default:
+      break;
+   }
+   bts_assert(false);
+   return api::orderman::NATIVE_ORDER_TYPE_LIMIT;
+}
+
+void GLBXNativeOrder::_create_create_msg(CreateOrderMsg &c) {
+   c.key = key();
+   c.price = Price(me->state.price());
+   c.quantity =
+       bts::orderman::get_quantity(me->state, StateQuantityType::OrderQuantity);
+   c.security = me->sec;
+   c.side = me->definition.side();
+   c.order_type = get_glbx_native_ord_type(me->definition);
+   c.time_in_force = me->definition.time_in_force();
+   c.order_type = api::orderman::NATIVE_ORDER_TYPE_LIMIT;
+   c.manual_order_indicator = me->definition.manual_order_indicator();
+   c.routing_record = &me->routing_record;
+   LOG(OrderMan, Info, "GLBXNativeOrder::_create_create_msg c.qty: {}",
+       c.quantity);
+}
+
 } // namespace orderman
 } // namespace bts

src/lib/orderman/orders/GLBXNativeOrder.h

@@ -11,6 +11,7 @@ class NewExecutionReportEvent;
 class CancelRejectEvent;
 class GatewayStatusChangeEvent;
 class ExecutionReport;
+struct CreateOrderMsg;
 
 class GLBXNativeOrder : public NativeOrderBase {
  public:
@@ -42,7 +43,7 @@ class GLBXNativeOrder : public NativeOrderBase {
    struct PImpl;
    std::unique_ptr<PImpl> me;
    void init_encodings();
-
+   void _create_create_msg(CreateOrderMsg &);
    void fixup_routing_record(api::orderman::RoutingRecord *rr);
    void on_exe_report(const NewExecutionReportEvent &);
    void on_cancel_reject(const CancelRejectEvent &);
@@ -60,8 +61,8 @@ class GLBXNativeOrder : public NativeOrderBase {
        log::Level,
        api::orderman::OrderStatus new_state,
        std::string const &msg);
-   void
-   transition(api::orderman::OrderStatus new_state, std::string const &msg);
+   void transition(
+       api::orderman::OrderStatus new_state, std::string const &msg);
    void transition(api::orderman::OrderStatus new_state);
    void transition_to_error(
        api::orderman::NativeOrderState::SubError sub_error,
@@ -74,8 +75,8 @@ class GLBXNativeOrder : public NativeOrderBase {
        api::orderman::NativeOrderState &,
        const NewExecutionReportEvent &) const;
    bool terminal_state(api::orderman::OrderStatus status) const;
-   api::orderman::NativeOrderState &
-   get_native_state(api::orderman::OrderState &);
+   api::orderman::NativeOrderState &get_native_state(
+       api::orderman::OrderState &);
    void ignore_or_cancel(
        std::string const &, api::orderman::OrderError, OrderOpResult *out);
 

src/lib/orderman/orders/algo/IAlgoOrder.cpp

@@ -1,18 +0,0 @@
-#include "lib/orderman/orders/algo/IAlgoOrder.h"
-#include "lib/orderman/orders/algo/IAlgoServices.h"
-
-namespace bts {
-namespace orderman {
-#include "lib/orderman/orders/algo/IAlgoOrder.h"
-IAlgoOrder::~IAlgoOrder() {}
-
-OrderOpResult IAlgoOrder::do_before_launch(IBeforeLaunchAlgoServices &) {
-   return {};
-}
-
-void IAlgoOrder::do_launch_attached(IAlgoServices &serv, OrderKey, ChildState) {
-   serv.terminate_error("Algo does not implement attachment functionality");
-}
-
-} // namespace orderman
-} // namespace bts