fix build: Add a few includes, suppress some werrors

Not sure how this slipped through the cracks, but I'm guessing the build
broke when we upgraded the cpp compilers (to clang 17 and gcc 13).

* Include <cstdint> in a few places
* Suppress -Wstringop-overflow in sparkd-gcc-release build
* Change const char * param to explicit std::string for proper type
  deduction in call to 'std::make_tuple'. For some reason, the gcc build
  was failing with an array-bounds werror (changing the arg fixed it).

src/DataStreamSuite/BooleanExpressions/BooleanExpressionTokens.lex

@@ -42,7 +42,6 @@ OpenParenthesis          \(
 CloseParenthesis         \)
 Comma                    ,
 Negate                   '!'
-MapsTo                   ':'
 IsOfType                 '/'
 
 %%
@@ -98,10 +97,6 @@ IsOfType                 '/'
    return (int)Token.NEGATE;
 }
 
-{MapsTo} {
-   return (int)Token.MAPS_TO;
-}
-
 {Equal} {
    return (int)Token.EQUAL;
 }

src/DataStreamSuite/DataStream/Primitives/MongoDB/IncrementalMongoDbStream.cs

@@ -1,35 +0,0 @@
-namespace Bts.DataStream;
-
-using BCG;
-
-using System;
-using System.Collections.Generic;
-
-public sealed class IncrementalMongoDbStream : ViewStream {
-   readonly string collectionName;
-   readonly string databaseName;
-   readonly string url;
-   readonly string index;
-   readonly Dictionary<string, int> idMap = [];
-   readonly Dictionary<string, string> columnMap = [];
-   readonly MongoDbChangeStreamObservable source;
-
-   public IncrementalMongoDbStream()
-      : this(
-         Environment.GetEnvironmentVariable("MONGO_DB_COLL") ?? string.Empty,
-         Environment.GetEnvironmentVariable("MONGO_DB_NAME") ?? string.Empty,
-         Environment.GetEnvironmentVariable("MONGO_DB_URL") ?? string.Empty) {
-   }
-
-   public IncrementalMongoDbStream(string collectionName, string databaseName, string url)
-      : this(collectionName, databaseName, url, "_id") { }
-
-   public IncrementalMongoDbStream(string collectionName, string databaseName, string url, string index) {
-      this.collectionName = collectionName;
-      this.databaseName = databaseName;
-      this.url = url;
-      this.index = index;
-      source = new(collectionName, databaseName, url);
-      mySetRealSource(source.AsViewStream());
-   }
-}

src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoCollectionStream.cs

@@ -0,0 +1,203 @@
+namespace Bts.DataStream;
+
+using BCG;
+
+using bts;
+
+using Bts.DataStreamCore;
+
+using mbts;
+
+using MongoDB.Bson;
+using MongoDB.Driver;
+
+using Spark.Core.Disposables;
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Reactive.Subjects;
+using System.Threading;
+using System.Threading.Tasks;
+
+public class MongoCollectionStream : ViewStream, IObservable<DataPacket> {
+   readonly string collectionName;
+   readonly string databaseName;
+   readonly string url;
+   readonly object syncRoot = new();
+   readonly CancellationTokenSource cts = new();
+   readonly Subject<DataPacket> subject = new();
+
+   bool hasStarted = false;
+   IBlueRowFactory<BsonDocument>? rowFactory;
+   MongoClient? clientBacking;
+   IMongoDatabase? databaseBacking;
+   IMongoCollection<BsonDocument>? collectionBacking = null;
+   BlueTableImmutable? table = null;
+
+   public MongoCollectionStream()
+      : this(
+         Environment.GetEnvironmentVariable("MONGO_DB_COLL") ?? string.Empty,
+         Environment.GetEnvironmentVariable("MONGO_DB_NAME") ?? string.Empty,
+         Environment.GetEnvironmentVariable("MONGO_DB_URL") ?? string.Empty) {
+   }
+
+   public MongoCollectionStream(string collectionName)
+      : this(
+         collectionName,
+         Environment.GetEnvironmentVariable("MONGO_DB_NAME") ?? string.Empty,
+         Environment.GetEnvironmentVariable("MONGO_DB_URL") ?? string.Empty) {
+   }
+
+   public MongoCollectionStream(string collectionName, string databaseName)
+      : this(
+         collectionName,
+         databaseName,
+         Environment.GetEnvironmentVariable("MONGO_DB_URL") ?? string.Empty) {
+   }
+
+   public MongoCollectionStream(string collectionName, string databaseName, string url)
+      : this(collectionName, databaseName, url, null) { }
+
+   internal MongoCollectionStream(
+      string collectionName, string databaseName, string url, IBlueRowFactory<BsonDocument>? rowFactory) {
+      this.collectionName = collectionName;
+      this.databaseName = databaseName;
+      this.url = url;
+      this.rowFactory = rowFactory;
+      mySetRealSource(new ObservableViewStream(this));
+   }
+
+   MongoClient client => clientBacking ??= new(url);
+   IMongoDatabase database => databaseBacking ??= client.GetDatabase(databaseName);
+   IMongoCollection<BsonDocument>? collection {
+      get => Volatile.Read(ref collectionBacking);
+      set => Volatile.Write(ref collectionBacking, value);
+   }
+
+   public IDisposable Subscribe(IObserver<DataPacket> observer) {
+      lock (syncRoot) {
+         if (hasStarted) {
+            if (table is not null) {
+               // The first update has already been fired. Seed observer w/ current state.
+               observer.OnNext(new DataPacket(table, table, new BlueTable(table.schema)));
+            }
+            return subject.Subscribe(observer);
+         }
+         hasStarted = true;
+      }
+      var subscription = subject.Subscribe(observer);
+      // Retrieve collection snapshot to bootstrap sequence
+      initTable().ignore();
+      // Watch changes
+      Task.Run(watchChanges);
+      return Disposable.Create(Dispose);
+
+      void Dispose() {
+         Log.Gen.info("Disposing MongoDbChangeObservable subscription");
+         subscription.Dispose();
+         cts.Cancel();
+         cts.Dispose();
+      }
+   }
+
+   protected virtual IBlueRowFactory<BsonDocument> getRowFactory(IEnumerable<BsonDocument> docs) =>
+      MongoSchemaAdaptor.InferSchema(docs);
+
+   async Task watchChanges() {
+      Log.Gen.info(
+         $"MongoCollectionStreamSource. Watching changes. thread: {Environment.CurrentManagedThreadId}");
+      while (collection is null) {
+         await Task.Delay(100);
+      }
+      Log.notnull(collection);
+      using var cursor = collection.Watch(new ChangeStreamOptions {
+         FullDocument = ChangeStreamFullDocumentOption.UpdateLookup
+      });
+      try {
+         while (await cursor.MoveNextAsync(cts.Token)) {
+            // Ignore while waiting for init
+            lock (syncRoot) {
+               if (table is null) {
+                  continue;
+               }
+            }
+            // schemaAdaptor is initialized after table; redundantly assert both
+            // are non-null to be safe, and clarify expectations.
+            Log.notnull(rowFactory);
+            Log.notnull(table);
+
+            var bd = table.ToDataPacketBuilder();
+            foreach (var ch in cursor.Current) {
+               var opDescJson = ch.OperationDescription?.ToJson() ?? string.Empty;
+               var updDescJson = ch.UpdateDescription?.ToJson() ?? string.Empty;
+               Log.Gen.info(
+                  $"MongoCollectionStream.watchChanges. Doc changed. type: {ch.OperationType} " +
+                  $"changes:\n\t\t{updDescJson}");
+               switch (ch.OperationType) {
+                  case ChangeStreamOperationType.Insert: {
+                        bd.AddOrUpdate(Row());
+                        break;
+                     }
+                  case ChangeStreamOperationType.Update: {
+                        bd.Remove(OldRow().Id());
+                        bd.AddOrUpdate(Row());
+                        break;
+                     }
+                  case ChangeStreamOperationType.Replace: {
+                        bd.Remove(OldRow().Id());
+                        bd.AddOrUpdate(Row());
+                        break;
+                     }
+                  case ChangeStreamOperationType.Delete: {
+                        bd.Remove(OldRow().Id());
+                        break;
+                     }
+                  default:
+                     Log.assert(false, $"Unsupported op type '{ch.OperationType}'");
+                     break;
+               }
+
+               IBlueRow Row() =>
+                  rowFactory.GetBlueRow(ch.FullDocumentBeforeChange);
+
+               IBlueRow OldRow() =>
+                  rowFactory.GetBlueRow(ch.FullDocumentBeforeChange);
+            }
+            var dp = bd.ToDataPacket();
+            if (!dp.HasChanges()) {
+               Log.Gen.warn(
+                  $"MongoDbChangeObservable Ignoring update because it didn't change any rows.");
+               continue;
+            }
+            lock (syncRoot) {
+               table = (BlueTableImmutable)dp.newRows;
+               subject.OnNext(dp);
+            }
+         }
+      } catch (TaskCanceledException ex) {
+         Log.Gen.warn(
+            $"MongoDbChangeObservable 'Watch' task cancelled (this is " +
+            $"expected if subscription was disposed). {ex}");
+      }
+   }
+
+
+   [MemberNotNull(nameof(collection))]
+   async Task initTable() {
+      collection = database.GetCollection<BsonDocument>(collectionName);
+      var cursor = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
+      var docList = await cursor.ToListAsync();
+      rowFactory ??= MongoSchemaAdaptor.InferSchema(docList);
+      var firstTable = new BlueTableImmutable(rowFactory.Schema).ToDataPacketBuilder();
+      foreach (var doc in docList) {
+         var row = rowFactory.GetBlueRow(doc);
+         firstTable.AddOrUpdate(row);
+      }
+      var dp = firstTable.ToDataPacket();
+      lock (syncRoot) {
+         table = (BlueTableImmutable)dp.newRows;
+         subject.OnNext(dp);
+      }
+   }
+}

src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDbChangeObservable.cs

@@ -1,208 +0,0 @@
-namespace Bts.DataStream;
-
-using BCG;
-
-using bts;
-using bts.collections;
-
-using Bts.DataStreamCore;
-
-using mbts;
-
-using MongoDB.Bson;
-using MongoDB.Driver;
-
-using Spark.Core;
-using Spark.Core.Disposables;
-
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Reactive.Subjects;
-using System.Threading;
-using System.Threading.Tasks;
-
-public class MongoDbChangeStreamObservable : IObservable<DataPacket> {
-   readonly string collectionName;
-   readonly string databaseName;
-   readonly string url;
-   readonly object locker = new();
-   readonly CancellationTokenSource cts = new();
-   readonly BlueTableSchemaAdaptor<BsonDocument> schemaBd = new("MongoDbChangeStream");
-
-   bool hasStarted = false;
-   MongoClient? clientBacking;
-   IMongoDatabase? databaseBacking;
-   IMongoCollection<BsonDocument>? collectionBacking = null;
-   BlueTableImmutable? table = null;
-   Subject<DataPacket> subject = new();
-
-   public MongoDbChangeStreamObservable(string collectionName, string databaseName, string url) {
-      this.collectionName = collectionName;
-      this.databaseName = databaseName;
-      this.url = url;
-   }
-
-   MongoClient client => clientBacking ??= new(url);
-   IMongoDatabase database => databaseBacking ??= client.GetDatabase(databaseName);
-   IMongoCollection<BsonDocument>? collection {
-      get => Volatile.Read(ref collectionBacking);
-      set => Volatile.Write(ref collectionBacking, value);
-   }
-
-   public IDisposable Subscribe(IObserver<DataPacket> observer) {
-      lock (locker) {
-         if (hasStarted) {
-            if (table is not null) {
-               // The first update has already been fired. Seed observer w/ current state.
-               observer.OnNext(new DataPacket(table, table, new BlueTable(table.schema)));
-            }
-            return subject.Subscribe(observer);
-         }
-         hasStarted = true;
-      }
-      var subscription = subject.Subscribe(observer);
-      // Retrieve collection snapshot in background
-      initTable().ignore();
-      // Listen for changes in background
-      Listen().ignore();
-      return Disposable.Create(Dispose);
-
-      void Dispose() {
-         Log.Gen.info("Disposing MongoDbChangeObservable subscription");
-         subscription.Dispose();
-         cts.Cancel();
-         cts.Dispose();
-      }
-
-      async Task Listen() {
-         while (collection is null) {
-            Thread.Yield();
-         }
-         using var cursor = collection.Watch(new ChangeStreamOptions {
-            FullDocument = ChangeStreamFullDocumentOption.UpdateLookup
-         });
-         try {
-            while (await cursor.MoveNextAsync(cts.Token)) {
-               // Ignore changes while waiting for collection snapshot
-               lock (locker) {
-                  if (table is null) {
-                     continue;
-                  }
-               }
-               var bd = table.ToDataPacketBuilder();
-               foreach (var ch in cursor.Current) {
-                  var opDescJson = ch.OperationDescription?.ToJson() ?? string.Empty;
-                  var updDescJson = ch.UpdateDescription?.ToJson() ?? string.Empty;
-                  var doc = ch.FullDocument;
-                  var docJson = doc.ToJson();
-                  Log.Gen.info(
-                     $"IncrementalMongoDbStream doc changed. type: {ch.OperationType} " +
-                     $"\n\tchanges:\n\t\t{updDescJson}\n\tnewValue:\n\t\t{docJson}");
-                  switch (ch.OperationType) {
-                     case ChangeStreamOperationType.Insert: {
-                           Log.Gen.info(
-                              $"IncrementalMongoDbStream doc changed. type: {ch.OperationType} " +
-                              $"\n\tchanges:\n\t\t{updDescJson}\n\tnewValue:\n\t\t{docJson}");
-                           var row = schemaBd.GetBlueRow(doc);
-                           bd.AddOrUpdate(row);
-                           break;
-                        }
-                     case ChangeStreamOperationType.Update: {
-                           var oldRow = schemaBd.GetBlueRow(ch.FullDocumentBeforeChange);
-                           bd.Remove(oldRow.Id());
-                           var newRow = schemaBd.GetBlueRow(ch.FullDocument);
-                           bd.AddOrUpdate(newRow);
-                           break;
-                        }
-                     case ChangeStreamOperationType.Replace: {
-                           var oldRow = schemaBd.GetBlueRow(ch.FullDocumentBeforeChange);
-                           bd.Remove(oldRow.Id());
-                           var newRow = schemaBd.GetBlueRow(ch.FullDocument);
-                           bd.AddOrUpdate(newRow);
-                           break;
-                        }
-                     default:
-                        Log.assert(false, $"Unsupported op type '{ch.OperationType}'");
-                        break;
-                  }
-               }
-               var dp = bd.ToDataPacket();
-               if (!dp.HasChanges()) {
-                  Log.Gen.warn(
-                     $"MongoDbChangeObservable Ignoring update because it didn't change any rows.");
-                  continue;
-               }
-               lock (locker) {
-                  table = (BlueTableImmutable)dp.newRows;
-                  subject.OnNext(dp);
-               }
-            }
-         } catch (TaskCanceledException ex) {
-            Log.Gen.warn(
-               $"MongoDbChangeObservable 'Watch' task cancelled (this is " +
-               $"expected if subscription was disposed). {ex}");
-         }
-      }
-   }
-
-   async Task initTable() {
-      collection = database.GetCollection<BsonDocument>(collectionName);
-      var cursor = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
-      var docList = await cursor.ToListAsync();
-      initSchema(docList);
-      var firstTable = new BlueTableImmutable(schemaBd.Schema).ToDataPacketBuilder();
-      foreach (var doc in docList) {
-         var row = schemaBd.GetBlueRow(doc);
-         firstTable.AddOrUpdate(row);
-      }
-      var dp = firstTable.ToDataPacket();
-      lock (locker) {
-         table = (BlueTableImmutable)dp.newRows;
-         subject.OnNext(dp);
-      }
-   }
-
-   void initSchema(IEnumerable<BsonDocument> docs) {
-      foreach (var doc in docs) {
-         foreach (var el in doc) {
-            if (schemaBd.Contains(el.Name)) {
-               continue;
-            }
-            switch (el.Value.BsonType) {
-               case BsonType.Array:
-                  schemaBd.AddColumn(el.Name, d => d.GetArrayOrDefault(el.Name)?.ToString() ?? string.Empty);
-                  break;
-               case BsonType.String:
-                  schemaBd.AddColumn(el.Name, d => d.GetStringOrDefault(el.Name) ?? string.Empty);
-                  break;
-               case BsonType.Int32:
-                  schemaBd.AddColumn(el.Name, d => d.GetInt32OrDefault(el.Name));
-                  break;
-               case BsonType.Int64:
-                  schemaBd.AddColumn(el.Name, d => d.GetInt64OrDefault(el.Name));
-                  break;
-               case BsonType.Double:
-                  schemaBd.AddColumn(el.Name, d => d.GetDoubleOrDefault(el.Name));
-                  break;
-               case BsonType.DateTime:
-                  schemaBd.AddColumn(el.Name, d => d.GetDateTimeOrDefault(el.Name));
-                  break;
-               case BsonType.Timestamp:
-                  schemaBd.AddColumn(el.Name, d => d.GetTimestampOrDefault(el.Name));
-                  break;
-               case BsonType.ObjectId:
-                  schemaBd.AddColumn(el.Name, d => d.GetObjectIdOrDefault(el.Name)?.ToString());
-                  break;
-               case BsonType.Null:
-                  Log.Gen.warn(
-                     $"Ignoring column '{el.Name}' because at least one element is null");
-                  break;
-               default:
-                  throw new InvalidOperationException(
-                     $"Unsupported BsonType '{el.Value.BsonType}'");
-            }
-         }
-      }
-   } 
-}

src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDbExtensions.cs

@@ -1,39 +1,63 @@
-namespace Bts.DataStream;
-
-using MongoDB.Bson;
-
-using System;
-
-public static class MongoDbExtensions {
-   public static BsonArray? GetArrayOrDefault(this BsonDocument doc, string elementName) =>
-      doc.TryGetValue(elementName, out var val) && val.IsBsonArray ?
-         val.AsBsonArray : null;
-
-   public static string? GetStringOrDefault(this BsonDocument doc, string elementName) =>
-      doc.TryGetValue(elementName, out var val) && val.IsString ?
-         val.AsString : null;
-
-   public static int? GetInt32OrDefault(this BsonDocument doc, string elementName) =>
-      doc.TryGetValue(elementName, out var val) && val.IsInt32 ?
-         val.AsInt32 : null;
-
-   public static long? GetInt64OrDefault(this BsonDocument doc, string elementName) =>
-      doc.TryGetValue(elementName, out var val) && val.IsInt64 ?
-         val.AsInt64 : null;
-
-   public static double? GetDoubleOrDefault(this BsonDocument doc, string elementName) =>
-      doc.TryGetValue(elementName, out var val) && val.IsDouble ?
-         val.AsDouble : null;
-
-   public static DateTime? GetDateTimeOrDefault(this BsonDocument doc, string elementName) =>
-      doc.TryGetValue(elementName, out var val) && val.IsValidDateTime ?
-         val.ToUniversalTime() : null;
-
-   public static DateTime? GetTimestampOrDefault(this BsonDocument doc, string elementName) =>
-      doc.TryGetValue(elementName, out var val) && val.IsValidDateTime ?
-         val.ToUniversalTime() : null;
-
-   public static ObjectId? GetObjectIdOrDefault(this BsonDocument doc, string elementName) =>
-      doc.TryGetValue(elementName, out var val) && val.IsObjectId ?
-         val.AsObjectId : null;
-}
+namespace Bts.DataStream;
+
+using MongoDB.Bson;
+
+using System;
+
+public static class MongoDbExtensions {
+   public static BsonArray GetArray(this BsonDocument doc, string elementName) =>
+      doc[elementName].AsBsonArray;
+
+   public static BsonArray? GetArrayOrDefault(this BsonDocument doc, string elementName) =>
+      doc.TryGetValue(elementName, out var val) && val.IsBsonArray ?
+         val.AsBsonArray : null;
+
+   public static string GetString(this BsonDocument doc, string elementName) =>
+      doc[elementName].AsString;
+
+   public static string? GetStringOrDefault(this BsonDocument doc, string elementName) =>
+      doc.TryGetValue(elementName, out var val) && val.IsString ?
+         val.AsString : null;
+
+   public static int GetInt32(this BsonDocument doc, string elementName) =>
+      doc[elementName].AsInt32;
+
+   public static int? GetInt32OrDefault(this BsonDocument doc, string elementName) =>
+      doc.TryGetValue(elementName, out var val) && val.IsInt32 ?
+         val.AsInt32 : null;
+
+   public static long GetInt64(this BsonDocument doc, string elementName) =>
+      doc[elementName].AsInt64;
+
+   public static long? GetInt64OrDefault(this BsonDocument doc, string elementName) =>
+      doc.TryGetValue(elementName, out var val) && val.IsInt64 ?
+         val.AsInt64 : null;
+
+   public static double GetDouble(this BsonDocument doc, string elementName) =>
+      doc[elementName].AsDouble;
+
+   public static double? GetDoubleOrDefault(this BsonDocument doc, string elementName) =>
+      doc.TryGetValue(elementName, out var val) && val.IsDouble ?
+         val.AsDouble : null;
+
+   public static DateTime GetDateTime(this BsonDocument doc, string elementName) =>
+      doc[elementName].ToUniversalTime();
+
+   public static DateTime? GetDateTimeOrDefault(this BsonDocument doc, string elementName) =>
+      doc.TryGetValue(elementName, out var val) && val.IsValidDateTime ?
+         val.ToUniversalTime() : null;
+
+   public static DateTime GetTimestamp(this BsonDocument doc, string elementName) =>
+      doc[elementName].ToUniversalTime();
+
+   public static DateTime? GetTimestampOrDefault(this BsonDocument doc, string elementName) =>
+      doc.TryGetValue(elementName, out var val) && val.IsValidDateTime ?
+         val.ToUniversalTime() : null;
+
+   public static ObjectId GetObjectId(this BsonDocument doc, string elementName) =>
+      doc[elementName].AsObjectId;
+
+   public static ObjectId? GetObjectIdOrDefault(this BsonDocument doc, string elementName) =>
+      doc.TryGetValue(elementName, out var val) && val.IsObjectId ?
+         val.AsObjectId : null;
+}

src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoSchemaAdaptor.cs

@@ -0,0 +1,57 @@
+namespace Bts.DataStream;
+
+using BCG;
+
+using bts;
+
+using MongoDB.Bson;
+
+using System;
+using System.Collections.Generic;
+
+public static class MongoSchemaAdaptor {
+   public static BlueTableSchemaAdaptor<BsonDocument> InferSchema(IEnumerable<BsonDocument> docs) {
+      var schemaAdaptor = new BlueTableSchemaAdaptor<BsonDocument>("MongoCollectionStream");
+      foreach (var doc in docs) {
+         foreach (var el in doc) {
+            if (schemaAdaptor.Contains(el.Name)) {
+               continue;
+            }
+            switch (el.Value.BsonType) {
+               case BsonType.Array:
+                  schemaAdaptor.AddColumn(el.Name, d => d.GetArrayOrDefault(el.Name)?.ToString() ?? string.Empty);
+                  break;
+               case BsonType.String:
+                  schemaAdaptor.AddColumn(el.Name, d => d.GetStringOrDefault(el.Name) ?? string.Empty);
+                  break;
+               case BsonType.Int32:
+                  schemaAdaptor.AddColumn(el.Name, d => d.GetInt32OrDefault(el.Name));
+                  break;
+               case BsonType.Int64:
+                  schemaAdaptor.AddColumn(el.Name, d => d.GetInt64OrDefault(el.Name));
+                  break;
+               case BsonType.Double:
+                  schemaAdaptor.AddColumn(el.Name, d => d.GetDoubleOrDefault(el.Name));
+                  break;
+               case BsonType.DateTime:
+                  schemaAdaptor.AddColumn(el.Name, d => d.GetDateTimeOrDefault(el.Name));
+                  break;
+               case BsonType.Timestamp:
+                  schemaAdaptor.AddColumn(el.Name, d => d.GetTimestampOrDefault(el.Name));
+                  break;
+               case BsonType.ObjectId:
+                  schemaAdaptor.AddColumn(el.Name, d => d.GetObjectIdOrDefault(el.Name)?.ToString());
+                  break;
+               case BsonType.Null:
+                  Log.Gen.warn(
+                     $"Ignoring column '{el.Name}' because at least one element is null");
+                  break;
+               default:
+                  throw new InvalidOperationException(
+                     $"Unsupported BsonType '{el.Value.BsonType}'");
+            }
+         }
+      }
+      return schemaAdaptor;
+   }
+}

src/client/Spark.Console/Spark.Console.csproj

@@ -4,7 +4,7 @@
     <OutputType>Exe</OutputType>
     <TargetFrameworks>net8.0</TargetFrameworks>
     <Nullable>enable</Nullable>
-    <RuntimeIdentifiers>win10-x64;ubuntu.16.04-x64</RuntimeIdentifiers>
+    <RuntimeIdentifiers>win-x64;linux-x64</RuntimeIdentifiers>
     <Authors>Blue Trading Systems</Authors>
     <Company>Blue Trading Systems</Company>
     <Product>Spark</Product>

src/client/spark/Modules/Trading/AlgoStudio/Model/AlgoDataVM.cs

@@ -1,39 +1,42 @@
-using bts.api.orderman;
-using bts.api.orderman.algo;
-using bts.utils;
-
-using mbts.spark.utilities;
-
-using System;
-using System.Windows.Media;
-
-namespace mbts.spark {
-   struct AlgoDataVM : IEquatable<AlgoDataVM> {
-      public static readonly Func<AlgoDataVM, AlgoID> id_fn = vm => AlgoID.from_algo_key(vm.data.key);
-
-      public readonly AlgoData data;
-      public readonly AlgoDefinitionMetadata? meta;
-
-      public AlgoDataVM(AlgoData data, AlgoDefinitionMetadata? meta = null) {
-         this.data = data;
-         this.meta = meta;
-      }
-
-      public string name => data.name;
-      public Brush badge_fill => OrderUtils.badge_fill(meta, OrderDataHelper.get_metadata(data.value.CustomCase));
-      public AlgoDisplayGroup group => data.key.display_group();
-
-      public override string ToString() =>
-         data.name;
-
-      public static AlgoDataVM from_data(AlgoData data, AlgoDefinitionMetadata? meta) =>
-         new AlgoDataVM(data, meta);
-
-      public static AlgoDataVM from_data((AlgoData algo, AlgoDefinitionMetadata? meta) tup) =>
-         new AlgoDataVM(tup.algo, tup.meta);
-
-      public bool Equals(AlgoDataVM other) => data == other.data && meta == other.meta;
-      public override bool Equals(object obj) => obj is AlgoDataVM other && Equals(other);
-      public override int GetHashCode() => EqualityComparer.aggregate_hash(data.GetHashCode(), meta.GetHashCode());
-   }
-}
+using bts.api.orderman;
+using bts.api.orderman.algo;
+using bts.utils;
+
+using mbts.spark.utilities;
+
+using System;
+using System.Windows.Media;
+
+namespace mbts.spark {
+   struct AlgoDataVM : IEquatable<AlgoDataVM> {
+      public static readonly Func<AlgoDataVM, AlgoID> id_fn = vm => AlgoID.from_algo_key(vm.data.key);
+
+      public readonly AlgoData data;
+      public readonly AlgoDefinitionMetadata? meta;
+
+      public AlgoDataVM(AlgoData data, AlgoDefinitionMetadata? meta = null) {
+         this.data = data;
+         this.meta = meta;
+      }
+
+      public string name => data.name;
+
+      public Brush badge_fill => OrderUtils.badge_fill(
+         meta, OrderMetadataStore.get_metadata(data.value.CustomCase));
+
+      public AlgoDisplayGroup group => data.key.display_group();
+
+      public override string ToString() =>
+         data.name;
+
+      public static AlgoDataVM from_data(AlgoData data, AlgoDefinitionMetadata? meta) =>
+         new AlgoDataVM(data, meta);
+
+      public static AlgoDataVM from_data((AlgoData algo, AlgoDefinitionMetadata? meta) tup) =>
+         new AlgoDataVM(tup.algo, tup.meta);
+
+      public bool Equals(AlgoDataVM other) => data == other.data && meta == other.meta;
+      public override bool Equals(object obj) => obj is AlgoDataVM other && Equals(other);
+      public override int GetHashCode() => EqualityComparer.aggregate_hash(data.GetHashCode(), meta.GetHashCode());
+   }
+}

src/client/spark/Modules/Trading/OrderBook/OrderBookWindow.cs

@@ -298,7 +298,7 @@ namespace mbts.spark {
          }
 
          bool has_algo_params(OrderSecurityAlgo? order) =>
-            order.HasValue && !OrderDataHelper.get_metadata(order.Value.algo.value.CustomCase).NoProps;
+            order.HasValue && !OrderMetadataStore.get_metadata(order.Value.algo.value.CustomCase).NoProps;
 
          Window mk_algo_params_dialog(OrderSecurityAlgo? order) {
             if (!has_algo_params(order))

src/client/spark/Orders/OrderFactory.cs

@@ -1,323 +1,323 @@
-#nullable enable
-namespace mbts.spark.utilities {
-   using bts;
-   using bts.api;
-   using bts.api.core;
-   using bts.api.orderman;
-   using bts.collections;
-   using bts.spark;
-
-   using mbts.ui;
-   using mbts.ui.utilities;
-   using mbts.ui.utility;
-   using mbts.utils;
-
-   using Spark.Core;
-   using Spark.Core.Disposables;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Composition;
-   using System.Linq;
-   using System.Reactive.Linq;
-
-   using OrderType = bts.api.orderman.CustomOrderDefinition.CustomOneofCase;
-   using DynamicOrderParamType = bts.api.orderman.DynamicOrderParam.ValueOneofCase;
-
-   [Export(typeof(OrderFactory))]
-   class OrderFactory : IDisposable {
-      static readonly OrderMetadata _default_order_metadata = new() {
-         KeyBinding = "",
-         Color = BTSColors.Orders.default_order
-      };
-      readonly AdminModel _admin_model;
-      readonly IReadOnlyDictionary<OneOf<OrderType, string>, string> _algo_abbreviation_index;
-      readonly IReadOnlyDictionary<string, DynamicOrderDescriptor> _dynamic_descriptors;
-
-      IDisposable? _dispose;
-
-      [ImportingConstructor]
-      public OrderFactory(OrderService service, AdminModel admin_model) {
-         _admin_model = admin_model;
-         _dynamic_descriptors = service.get_registered_dynamic_order_types(new VoidMessage())
-            .DynamicOrderDesc
-            .ToDictionary(d => d.DynamicOrderType);
-         var algo_name_index = new HashSet<OneOf<OrderType, string>>(
-            OneOf<OrderType, string>.ValueEqualityComparer);
-         // add standard types
-         algo_name_index.AddRange(
-            EnumHelper.get_values_except(OrderType.None)
-            .Select(t => new OneOf<OrderType, string>(t)));
-         // add dynamic types
-         algo_name_index.AddRange(_dynamic_descriptors.Keys
-            .Select(dt => new OneOf<OrderType, string>(dt)));
-         // build abbreviation index
-         _algo_abbreviation_index = DisplayNameHelper.get_word_tree_abbreviations(
-            algo_name_index,
-            oneof => oneof.value.ToString(),
-            OneOf<OrderType, string>.ValueEqualityComparer);
-      }
-
-      public static string? fmt_dynamic_param(DynamicOrderParam? dp) {
-         if (dp is null) {
-            return null;
-         }
-         return dp.ValueCase switch {
-            DynamicOrderParamType.AsArray => ", ".join(dp.AsArray.Element.Select(fmt_dynamic_param)),
-            DynamicOrderParamType.AsBool => dp.AsBool.ToString(),
-            DynamicOrderParamType.AsDouble => dp.AsDouble.ToString(),
-            DynamicOrderParamType.AsEnumerated => dp.AsEnumerated.EnumValue,
-            DynamicOrderParamType.AsInt32 => dp.AsInt32.ToString(),
-            DynamicOrderParamType.AsQuantity => dp.AsQuantity.ToString(),
-            DynamicOrderParamType.AsString => dp.AsString,
-            _ => $"({dp.ValueCase})",
-         };
-      }
-
-      public static DynamicOrderParam init_dynamic_param(DynamicOrderParamDescriptor descriptor) {
-         var rv = new DynamicOrderParam();
-         switch (descriptor.ParamType.DescriptorCase) {
-            case DynamicOrderTypeDescriptor.DescriptorOneofCase.Array:
-               rv.AsArray = new DynamicOrderArray();
-               break;
-            case DynamicOrderTypeDescriptor.DescriptorOneofCase.Enumerated:
-               rv.AsEnumerated = new() {
-                  EnumValue = descriptor.ParamType.Enumerated.Element.FirstOrDefault()
-               };
-               break;
-            case DynamicOrderTypeDescriptor.DescriptorOneofCase.Primitive:
-               switch (descriptor.ParamType.Primitive) {
-                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveBool: rv.AsBool = false; break;
-                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveDouble: rv.AsDouble = 0.0; break;
-                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveInt: rv.AsInt32 = 0; break;
-                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveOrderDef: rv.AsOrderDef = new(); break;
-                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveString: rv.AsString = string.Empty; break;
-                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveSecurityKey:
-                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveUint: rv.AsUint32 = 0u; break;
-                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveQuantity: rv.AsQuantity = new(); break;
-               }
-               break;
-            case DynamicOrderTypeDescriptor.DescriptorOneofCase.Structure:
-               rv.AsStruct = new();
-               break;
-         }
-         return rv;
-      }
-
-      public static SpreaderOrderDefinition spreader_def() => new() {
-         ExecutionStrategy = new ExecutionStrategy()
-      };
-
-      public DynamicOrderDescriptor? dynamic_descriptor(string type) =>
-         _dynamic_descriptors.GetValueOrDefault(type);
-
-      public DynamicOrderDefinition? dynamic_def(string type) {
-         if (!_dynamic_descriptors.TryGetValue(type, out var desc)) {
-            return null;
-         }
-         var def = new DynamicOrderDefinition { DynamicOrderType = type };
-         foreach (var p in desc.ParamDesc) {
-            def.OrderParams[p.ParamName] = init_dynamic_param(p);
-         }
-         return def;
-      }
-
-      public OrderType default_order_type(SecurityImmutable security) =>
-         security.ContractCase switch {
-            Security.ContractOneofCase.Synthetic => OrderType.Spreader,
-            _ => OrderType.Limit,
-         };
-
-      public TimeInForce default_time_in_force() =>
-         TimeInForce.Day;
-
-      public TimeInForce default_time_in_force(SecurityImmutable security) =>
-         default_time_in_force();
-
-      public IEnumerable<OrderType> default_order_types() =>
-         OrderDataHelper.exchange_order_types;
-
-      public IEnumerable<TimeInForce> default_time_in_forces(
-         SecurityImmutable security,
-         OrderType order_type,
-         bool is_child = false
-      ) {
-         var tifs = EnumHelper.get_values_except(TimeInForce.Undefined);
-         if (order_type != OrderType.Limit) {
-            tifs = tifs.Except(new[] { TimeInForce.Ioc });
-         }
-         return tifs;
-      }
-
-      public IEnumerable<string> dynamic_types() =>
-         _dynamic_descriptors.Keys;
-
-      public CustomOrderDefinitionImmutable? custom_defi(string type_name) {
-         if (Enum.TryParse(type_name, out OrderType sys_type)) {
-            return _custom_def(sys_type);
-         }
-         var dyn_def = dynamic_def(type_name);
-         if (dyn_def is null) {
-            return null;
-         }
-         return new(new() { Dynamic = dyn_def });
-      }
-
-      public OrderDefinition order(SecurityImmutable security, AccountTuple? account = null) {
-         var rv = new OrderDefinition {
-            SecurityKey = security.Id,
-            Side = Side.Buy,
-            TimeInForce = TimeInForce.Day,
-            CustomDef = OrderDataHelper.Defaults.CustomOrderDefinition(),
-            CtiCode = CTICode.CtiCode1,
-            CustomerOrFirm = CustomerOrFirm.Firm,
-            // this will be the tag 50 on CME orders
-            // current user might be okay, but we might also want to get it
-            // from the store that serves the risk admin gui
-            ManualOrderIndicator = ManualOrderIndicator.Yes,
-            User = _admin_model.user.name,
-            UserId = _admin_model.user.id
-         };
-         if (account.HasValue) {
-            rv.Account = account.Value.name;
-            rv.AccountId = account.Value.id;
-         }
-         return rv;
-      }
-
-      public string abbreviate_algo(OneOf<OrderType, string> id) {
-         if (id is null) {
-            throw new ArgumentNullException(nameof(id));
-         }
-         if (id.value is string) {
-            return _algo_abbreviation_index.GetValueOrDefault(id) ??
-               id.value.ToString() ??
-               string.Empty;
-         }
-         if (id.value is OrderType t) {
-            return OrderDataHelper.abbreviate_algo_type(t);
-         }
-         return id.ToString() ?? string.Empty;
-      }
-
-      public OrderMetadata get_metadata(OrderRecordImmutable order) =>
-         get_metadata(order.Def.CustomDef.CustomCase);
-
-      public OrderMetadata get_metadata(OrderType order_type) =>
-         order_type switch {
-            OrderType.Market => new() { KeyBinding = "M", Color = BTSColors.Orders.market_order, NoProps = true },
-            OrderType.Limit => new() { KeyBinding = "L", Color = BTSColors.Orders.limit_order, NoProps = true },
-            OrderType.Stop => new() { KeyBinding = "S", Color = BTSColors.Orders.stop_order, NoProps = true },
-            OrderType.StopLimit => new() { KeyBinding = "SL", Color = BTSColors.Orders.stop_limit_order },
-            OrderType.GlbxIceberg => new() { KeyBinding = "GB", Color = BTSColors.Orders.glbx_iceberg_order },
-            OrderType.Iceberg => new() { KeyBinding = "I", Color = BTSColors.Orders.iceberg_order },
-            OrderType.TimeBox => new() { KeyBinding = "TB", Color = BTSColors.Orders.time_box_order },
-            OrderType.Tick => new() { KeyBinding = "T", Color = BTSColors.Orders.tick_order },
-            OrderType.Lean => new() { KeyBinding = "L", Color = BTSColors.Orders.lean_order },
-            OrderType.Join => new() { KeyBinding = "J", Color = BTSColors.Orders.join_order },
-            OrderType.Top => new() { KeyBinding = "D", Color = BTSColors.Orders.top_order },
-            OrderType.Cover => new() { KeyBinding = "C", Color = BTSColors.Orders.cover_order },
-            OrderType.Oco => new() { KeyBinding = "C", Color = BTSColors.Orders.oco_order },
-            OrderType.Pricer => new() { KeyBinding = "C", Color = BTSColors.Orders.pricer_order },
-            OrderType.Offset => new() { KeyBinding = "O", Color = BTSColors.Orders.off_order },
-            OrderType.Trigger => new() { KeyBinding = "T", Color = BTSColors.Orders.trigger_order },
-            OrderType.Spreader => new() { KeyBinding = "", Color = BTSColors.Orders.spreader_order },
-            OrderType.Dynamic => new() { KeyBinding = "", Color = BTSColors.Orders.top_order },
-            _ => _default_order_metadata
-         };
-
-      public SpreaderOrderDefinition spreader_order(
-         SecurityImmutable security, ExecutionStrategyImmutable strategy) =>
-         new() {
-            Security = security.CloneToMutable(),
-            ExecutionStrategy = strategy.CloneToMutable()
-         };
-
-      public SpreaderOrderDefinitionImmutable spreader_orderi(
-         SecurityImmutable security, ExecutionStrategyImmutable strategy) =>
-         new(spreader_order(security, strategy));
-
-      public void Dispose() =>
-         Disposable.dispose(ref _dispose);
-
-      static CustomOrderDefinition _custom_def() =>
-         new() { Limit = _limit_def() };
-
-      static CustomOrderDefinition _custom_def(
-         OrderType type) {
-         var rv = new CustomOrderDefinition();
-         switch (type) {
-            case OrderType.Market: rv.Market = _market_def(); break;
-            case OrderType.Limit: rv.Limit = _limit_def(); break;
-            case OrderType.Stop: rv.Stop = _stop_def(); break;
-            case OrderType.StopLimit: rv.StopLimit = _stop_limit_def(); break;
-            case OrderType.Iceberg: rv.Iceberg = _iceberg_def(); break;
-            case OrderType.GlbxIceberg: rv.GlbxIceberg = _glbx_iceberg_def(); break;
-            case OrderType.TimeBox: rv.TimeBox = _time_box_def(); break;
-            case OrderType.Tick: rv.Tick = _tick_def(); break;
-            case OrderType.Lean: rv.Lean = _lean_def(); break;
-            case OrderType.Join: rv.Join = _join_def(); break;
-            case OrderType.Top: rv.Top = _top_def(); break;
-            case OrderType.Cover: rv.Cover = _cover_def(); break;
-            case OrderType.Oco: rv.Oco = _oco_def(); break;
-            case OrderType.Pricer: rv.Pricer = _pricer_def(); break;
-            case OrderType.Offset: rv.Offset = _offset_def(); break;
-            case OrderType.Trigger: rv.Trigger = _trigger_def(); break;
-            case OrderType.Spreader: rv.Spreader = spreader_def(); break;
-            case OrderType.Dynamic: rv.Dynamic = _dynamic_def(); break;
-            default:
-               OrderTypeVMFactory.GenericOrderFactory.setup_generic_custom_order(rv, type);
-               break;
-         }
-         return rv;
-      }
-
-      static MarketOrderDefinition _market_def() => new();
-      static LimitOrderDefinition _limit_def() => new();
-      static StopOrderDefinition _stop_def() => new();
-      static StopLimitOrderDefinition _stop_limit_def() => new();
-      static GLBXIcebergOrderDefinition _glbx_iceberg_def() => new() { MaxShowQuantity = 1 };
-      static IcebergOrderDefinition _iceberg_def() =>
-         new() { Child = _custom_def(), ShowQuantity = 1 };
-      static TimeBoxOrderDefinition _time_box_def() => new() {
-         Child = _custom_def(),
-         Start = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTime(DateTime.UtcNow),
-         Stop = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTime(DateTime.UtcNow)
-      };
-      static TickOrderDefinition _tick_def() => new() {
-         Child = _custom_def(),
-         CrossThreshold = new() { Absolute = 1 }
-      };
-      static LeanOrderDefinition _lean_def() => new() {
-         Child = _custom_def(),
-         CrossThreshold = new() { Absolute = 1 },
-         MaxChaseTicks = 2
-      };
-      static JoinOrderDefinition _join_def() => new() {
-         Child = _custom_def(),
-         JoinThreshold = new() { Absolute = 1 }
-      };
-      static TopOrderDefinition _top_def() => new() {
-         Child = _custom_def(),
-         TopThreshold = new() { Absolute = 1 }
-      };
-      static CoverOrderDefinition _cover_def() => new() {
-         Child = _custom_def(),
-         Hedge = _custom_def(),
-         HedgeTicks = 1
-      };
-      static OCOOrderDefinition _oco_def() => new() {
-         Child = _custom_def(),
-         Other = _custom_def(),
-         OtherSide = RelativeSide.Same
-      };
-      static PricerOrderDefinition _pricer_def() => new() { Child = _custom_def() };
-      static OffsetOrderDefinition _offset_def() => new() { Child = _custom_def(), };
-      static TriggerOrderDefinition _trigger_def() => new() {
-         Child = _custom_def(),
-         TriggerCondition = new TriggerCondition(),
-         TriggerAction = new TriggerAction(),
-      };
-      static DynamicOrderDefinition _dynamic_def() => new();
-   }
-}
+#nullable enable
+namespace mbts.spark.utilities {
+   using bts;
+   using bts.api;
+   using bts.api.core;
+   using bts.api.orderman;
+   using bts.collections;
+   using bts.spark;
+
+   using mbts.ui;
+   using mbts.ui.utilities;
+   using mbts.ui.utility;
+   using mbts.utils;
+
+   using Spark.Core;
+   using Spark.Core.Disposables;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Composition;
+   using System.Linq;
+   using System.Reactive.Linq;
+
+   using OrderType = bts.api.orderman.CustomOrderDefinition.CustomOneofCase;
+   using DynamicOrderParamType = bts.api.orderman.DynamicOrderParam.ValueOneofCase;
+
+   [Export(typeof(OrderFactory))]
+   class OrderFactory : IDisposable {
+      static readonly OrderMetadata _default_order_metadata = new() {
+         KeyBinding = "",
+         Color = BTSColors.Orders.default_order
+      };
+      readonly AdminModel _admin_model;
+      readonly IReadOnlyDictionary<OneOf<OrderType, string>, string> _algo_abbreviation_index;
+      readonly IReadOnlyDictionary<string, DynamicOrderDescriptor> _dynamic_descriptors;
+
+      IDisposable? _dispose;
+
+      [ImportingConstructor]
+      public OrderFactory(OrderService service, AdminModel admin_model) {
+         _admin_model = admin_model;
+         _dynamic_descriptors = service.get_registered_dynamic_order_types(new VoidMessage())
+            .DynamicOrderDesc
+            .ToDictionary(d => d.DynamicOrderType);
+         var algo_name_index = new HashSet<OneOf<OrderType, string>>(
+            OneOf<OrderType, string>.ValueEqualityComparer);
+         // add standard types
+         algo_name_index.AddRange(
+            EnumHelper.get_values_except(OrderType.None)
+            .Select(t => new OneOf<OrderType, string>(t)));
+         // add dynamic types
+         algo_name_index.AddRange(_dynamic_descriptors.Keys
+            .Select(dt => new OneOf<OrderType, string>(dt)));
+         // build abbreviation index
+         _algo_abbreviation_index = DisplayNameHelper.get_word_tree_abbreviations(
+            algo_name_index,
+            oneof => oneof.value.ToString(),
+            OneOf<OrderType, string>.ValueEqualityComparer);
+      }
+
+      public static string? fmt_dynamic_param(DynamicOrderParam? dp) {
+         if (dp is null) {
+            return null;
+         }
+         return dp.ValueCase switch {
+            DynamicOrderParamType.AsArray => ", ".join(dp.AsArray.Element.Select(fmt_dynamic_param)),
+            DynamicOrderParamType.AsBool => dp.AsBool.ToString(),
+            DynamicOrderParamType.AsDouble => dp.AsDouble.ToString(),
+            DynamicOrderParamType.AsEnumerated => dp.AsEnumerated.EnumValue,
+            DynamicOrderParamType.AsInt32 => dp.AsInt32.ToString(),
+            DynamicOrderParamType.AsQuantity => dp.AsQuantity.ToString(),
+            DynamicOrderParamType.AsString => dp.AsString,
+            _ => $"({dp.ValueCase})",
+         };
+      }
+
+      public static DynamicOrderParam init_dynamic_param(DynamicOrderParamDescriptor descriptor) {
+         var rv = new DynamicOrderParam();
+         switch (descriptor.ParamType.DescriptorCase) {
+            case DynamicOrderTypeDescriptor.DescriptorOneofCase.Array:
+               rv.AsArray = new DynamicOrderArray();
+               break;
+            case DynamicOrderTypeDescriptor.DescriptorOneofCase.Enumerated:
+               rv.AsEnumerated = new() {
+                  EnumValue = descriptor.ParamType.Enumerated.Element.FirstOrDefault()
+               };
+               break;
+            case DynamicOrderTypeDescriptor.DescriptorOneofCase.Primitive:
+               switch (descriptor.ParamType.Primitive) {
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveBool: rv.AsBool = false; break;
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveDouble: rv.AsDouble = 0.0; break;
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveInt: rv.AsInt32 = 0; break;
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveOrderDef: rv.AsOrderDef = new(); break;
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveString: rv.AsString = string.Empty; break;
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveSecurityKey:
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveUint: rv.AsUint32 = 0u; break;
+                  case DynamicOrderPrimitiveDescriptor.DynamicOrderPrimitiveQuantity: rv.AsQuantity = new(); break;
+               }
+               break;
+            case DynamicOrderTypeDescriptor.DescriptorOneofCase.Structure:
+               rv.AsStruct = new();
+               break;
+         }
+         return rv;
+      }
+
+      public static SpreaderOrderDefinition spreader_def() => new() {
+         ExecutionStrategy = new ExecutionStrategy()
+      };
+
+      public DynamicOrderDescriptor? dynamic_descriptor(string type) =>
+         _dynamic_descriptors.GetValueOrDefault(type);
+
+      public DynamicOrderDefinition? dynamic_def(string type) {
+         if (!_dynamic_descriptors.TryGetValue(type, out var desc)) {
+            return null;
+         }
+         var def = new DynamicOrderDefinition { DynamicOrderType = type };
+         foreach (var p in desc.ParamDesc) {
+            def.OrderParams[p.ParamName] = init_dynamic_param(p);
+         }
+         return def;
+      }
+
+      public OrderType default_order_type(SecurityImmutable security) =>
+         security.ContractCase switch {
+            Security.ContractOneofCase.Synthetic => OrderType.Spreader,
+            _ => OrderType.Limit,
+         };
+
+      public TimeInForce default_time_in_force() =>
+         TimeInForce.Day;
+
+      public TimeInForce default_time_in_force(SecurityImmutable security) =>
+         default_time_in_force();
+
+      public IEnumerable<OrderType> default_order_types() =>
+         OrderDataHelper.exchange_order_types;
+
+      public IEnumerable<TimeInForce> default_time_in_forces(
+         SecurityImmutable security,
+         OrderType order_type,
+         bool is_child = false
+      ) {
+         var tifs = EnumHelper.get_values_except(TimeInForce.Undefined);
+         if (order_type != OrderType.Limit) {
+            tifs = tifs.Except(new[] { TimeInForce.Ioc });
+         }
+         return tifs;
+      }
+
+      public IEnumerable<string> dynamic_types() =>
+         _dynamic_descriptors.Keys;
+
+      public CustomOrderDefinitionImmutable? custom_defi(string type_name) {
+         if (Enum.TryParse(type_name, out OrderType sys_type)) {
+            return _custom_def(sys_type);
+         }
+         var dyn_def = dynamic_def(type_name);
+         if (dyn_def is null) {
+            return null;
+         }
+         return new(new() { Dynamic = dyn_def });
+      }
+
+      public OrderDefinition order(SecurityImmutable security, AccountTuple? account = null) {
+         var rv = new OrderDefinition {
+            SecurityKey = security.Id,
+            Side = Side.Buy,
+            TimeInForce = TimeInForce.Day,
+            CustomDef = OrderDataHelper.Defaults.CustomOrderDefinition(),
+            CtiCode = CTICode.CtiCode1,
+            CustomerOrFirm = CustomerOrFirm.Firm,
+            // this will be the tag 50 on CME orders
+            // current user might be okay, but we might also want to get it
+            // from the store that serves the risk admin gui
+            ManualOrderIndicator = ManualOrderIndicator.Yes,
+            User = _admin_model.user.name,
+            UserId = _admin_model.user.id
+         };
+         if (account.HasValue) {
+            rv.Account = account.Value.name;
+            rv.AccountId = account.Value.id;
+         }
+         return rv;
+      }
+
+      public string abbreviate_algo(OneOf<OrderType, string> id) {
+         if (id is null) {
+            throw new ArgumentNullException(nameof(id));
+         }
+         if (id.value is string) {
+            return _algo_abbreviation_index.GetValueOrDefault(id) ??
+               id.value.ToString() ??
+               string.Empty;
+         }
+         if (id.value is OrderType t) {
+            return OrderDataHelper.abbreviate_algo_type(t);
+         }
+         return id.ToString() ?? string.Empty;
+      }
+
+      public OrderMetadata get_metadata(OrderRecordImmutable order) =>
+         get_metadata(order.Def.CustomDef.CustomCase);
+
+      public OrderMetadata get_metadata(OrderType order_type) =>
+         order_type switch {
+            OrderType.Market => new() { KeyBinding = "M", Color = BTSColors.Orders.market_order, NoProps = true },
+            OrderType.Limit => new() { KeyBinding = "L", Color = BTSColors.Orders.limit_order, NoProps = true },
+            OrderType.Stop => new() { KeyBinding = "S", Color = BTSColors.Orders.stop_order, NoProps = true },
+            OrderType.StopLimit => new() { KeyBinding = "SL", Color = BTSColors.Orders.stop_limit_order },
+            OrderType.GlbxIceberg => new() { KeyBinding = "GB", Color = BTSColors.Orders.glbx_iceberg_order },
+            OrderType.Iceberg => new() { KeyBinding = "I", Color = BTSColors.Orders.iceberg_order },
+            OrderType.TimeBox => new() { KeyBinding = "TB", Color = BTSColors.Orders.time_box_order },
+            OrderType.Tick => new() { KeyBinding = "T", Color = BTSColors.Orders.tick_order },
+            OrderType.Lean => new() { KeyBinding = "L", Color = BTSColors.Orders.lean_order },
+            OrderType.Join => new() { KeyBinding = "J", Color = BTSColors.Orders.join_order },
+            OrderType.Top => new() { KeyBinding = "D", Color = BTSColors.Orders.top_order },
+            OrderType.Cover => new() { KeyBinding = "C", Color = BTSColors.Orders.cover_order },
+            OrderType.Oco => new() { KeyBinding = "C", Color = BTSColors.Orders.oco_order },
+            OrderType.Pricer => new() { KeyBinding = "C", Color = BTSColors.Orders.pricer_order },
+            OrderType.Offset => new() { KeyBinding = "O", Color = BTSColors.Orders.off_order },
+            OrderType.Trigger => new() { KeyBinding = "T", Color = BTSColors.Orders.trigger_order },
+            OrderType.Spreader => new() { KeyBinding = "", Color = BTSColors.Orders.spreader_order },
+            OrderType.Dynamic => new() { KeyBinding = "", Color = BTSColors.Orders.top_order },
+            _ => _default_order_metadata
+         };
+
+      public SpreaderOrderDefinition spreader_order(
+         SecurityImmutable security, ExecutionStrategyImmutable strategy) =>
+         new() {
+            Security = security.CloneToMutable(),
+            ExecutionStrategy = strategy.CloneToMutable()
+         };
+
+      public SpreaderOrderDefinitionImmutable spreader_orderi(
+         SecurityImmutable security, ExecutionStrategyImmutable strategy) =>
+         new(spreader_order(security, strategy));
+
+      public void Dispose() =>
+         Disposable.dispose(ref _dispose);
+
+      static CustomOrderDefinition _custom_def() =>
+         new() { Limit = _limit_def() };
+
+      static CustomOrderDefinition _custom_def(
+         OrderType type) {
+         var rv = new CustomOrderDefinition();
+         switch (type) {
+            case OrderType.Market: rv.Market = _market_def(); break;
+            case OrderType.Limit: rv.Limit = _limit_def(); break;
+            case OrderType.Stop: rv.Stop = _stop_def(); break;
+            case OrderType.StopLimit: rv.StopLimit = _stop_limit_def(); break;
+            case OrderType.Iceberg: rv.Iceberg = _iceberg_def(); break;
+            case OrderType.GlbxIceberg: rv.GlbxIceberg = _glbx_iceberg_def(); break;
+            case OrderType.TimeBox: rv.TimeBox = _time_box_def(); break;
+            case OrderType.Tick: rv.Tick = _tick_def(); break;
+            case OrderType.Lean: rv.Lean = _lean_def(); break;
+            case OrderType.Join: rv.Join = _join_def(); break;
+            case OrderType.Top: rv.Top = _top_def(); break;
+            case OrderType.Cover: rv.Cover = _cover_def(); break;
+            case OrderType.Oco: rv.Oco = _oco_def(); break;
+            case OrderType.Pricer: rv.Pricer = _pricer_def(); break;
+            case OrderType.Offset: rv.Offset = _offset_def(); break;
+            case OrderType.Trigger: rv.Trigger = _trigger_def(); break;
+            case OrderType.Spreader: rv.Spreader = spreader_def(); break;
+            case OrderType.Dynamic: rv.Dynamic = _dynamic_def(); break;
+            default:
+               OrderDataHelper.setup_generic_custom_order(rv, type);
+               break;
+         }
+         return rv;
+      }
+
+      static MarketOrderDefinition _market_def() => new();
+      static LimitOrderDefinition _limit_def() => new();
+      static StopOrderDefinition _stop_def() => new();
+      static StopLimitOrderDefinition _stop_limit_def() => new();
+      static GLBXIcebergOrderDefinition _glbx_iceberg_def() => new() { MaxShowQuantity = 1 };
+      static IcebergOrderDefinition _iceberg_def() =>
+         new() { Child = _custom_def(), ShowQuantity = 1 };
+      static TimeBoxOrderDefinition _time_box_def() => new() {
+         Child = _custom_def(),
+         Start = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTime(DateTime.UtcNow),
+         Stop = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTime(DateTime.UtcNow)
+      };
+      static TickOrderDefinition _tick_def() => new() {
+         Child = _custom_def(),
+         CrossThreshold = new() { Absolute = 1 }
+      };
+      static LeanOrderDefinition _lean_def() => new() {
+         Child = _custom_def(),
+         CrossThreshold = new() { Absolute = 1 },
+         MaxChaseTicks = 2
+      };
+      static JoinOrderDefinition _join_def() => new() {
+         Child = _custom_def(),
+         JoinThreshold = new() { Absolute = 1 }
+      };
+      static TopOrderDefinition _top_def() => new() {
+         Child = _custom_def(),
+         TopThreshold = new() { Absolute = 1 }
+      };
+      static CoverOrderDefinition _cover_def() => new() {
+         Child = _custom_def(),
+         Hedge = _custom_def(),
+         HedgeTicks = 1
+      };
+      static OCOOrderDefinition _oco_def() => new() {
+         Child = _custom_def(),
+         Other = _custom_def(),
+         OtherSide = RelativeSide.Same
+      };
+      static PricerOrderDefinition _pricer_def() => new() { Child = _custom_def() };
+      static OffsetOrderDefinition _offset_def() => new() { Child = _custom_def(), };
+      static TriggerOrderDefinition _trigger_def() => new() {
+         Child = _custom_def(),
+         TriggerCondition = new TriggerCondition(),
+         TriggerAction = new TriggerAction(),
+      };
+      static DynamicOrderDefinition _dynamic_def() => new();
+   }
+}

src/client/spark/Orders/OrderTypeVMs/DefaultAlgoVM.cs

@@ -35,7 +35,7 @@ namespace mbts.spark {
       public IAlgoVM impl { get; private set; }
       public AlgoKey id { get; } = AlgoKey.from_known_type(AlgoType.None);
       public AlgoType order_type => _definition.value.CustomCase;
-      public OrderMetadata meta => OrderDataHelper.get_metadata(order_type);
+      public OrderMetadata meta => OrderMetadataStore.get_metadata(order_type);
       public bool can_expand => impl?.can_expand ?? false;
       public bool is_expanded {
          get => false;

src/client/spark/Orders/OrderTypeVMs/GenericOrderVM.cs

@@ -41,17 +41,6 @@ namespace mbts.spark {
             object vm = Activator.CreateInstance(makevm, key, algo_type, display_name, editing_permissions);
             return (IAlgoVM)vm;
          }
-         static public void setup_generic_custom_order(CustomOrderDefinition custom, CustomOrderDefinition.CustomOneofCase type) {
-            var prop = typeof(CustomOrderDefinition).GetProperty(type.ToString(),
-               BindingFlags.Instance | BindingFlags.Public);
-            var rt = prop.PropertyType;
-            var rtt = Activator.CreateInstance(rt);
-
-            var ci = rt.GetProperties().Where(x => x.PropertyType == typeof(CustomOrderDefinition));
-            foreach (var cid in ci)
-               cid.SetValue(rtt, new CustomOrderDefinition { Limit = new LimitOrderDefinition() });
-            prop.SetValue(custom, rtt);
-         }
       }
 
       class GenericOrderVM<TMutable, TImmutable> : OrderTypeVMBase<TMutable, TImmutable>

src/client/spark/Orders/OrderTypeVMs/OrderMetadataStore.cs

@@ -0,0 +1,32 @@
+namespace mbts.spark;
+
+using bts.api.orderman;
+using bts.spark;
+
+using mbts.ui;
+
+static class OrderMetadataStore {
+   public static OrderMetadata get_metadata(CustomOrderDefinition.CustomOneofCase order_type) =>
+      order_type switch {
+         CustomOrderDefinition.CustomOneofCase.Market => new() { KeyBinding = "M", Color = BTSColors.Orders.market_order, NoProps = true },
+         CustomOrderDefinition.CustomOneofCase.Limit => new() { KeyBinding = "L", Color = BTSColors.Orders.limit_order, NoProps = true },
+         CustomOrderDefinition.CustomOneofCase.Stop => new() { KeyBinding = "S", Color = BTSColors.Orders.stop_order, NoProps = true },
+         CustomOrderDefinition.CustomOneofCase.StopLimit => new() { KeyBinding = "SL", Color = BTSColors.Orders.stop_limit_order },
+         CustomOrderDefinition.CustomOneofCase.GlbxIceberg => new() { KeyBinding = "GB", Color = BTSColors.Orders.glbx_iceberg_order },
+         CustomOrderDefinition.CustomOneofCase.Iceberg => new() { KeyBinding = "I", Color = BTSColors.Orders.iceberg_order },
+         CustomOrderDefinition.CustomOneofCase.TimeBox => new() { KeyBinding = "TB", Color = BTSColors.Orders.time_box_order },
+         CustomOrderDefinition.CustomOneofCase.Tick => new() { KeyBinding = "T", Color = BTSColors.Orders.tick_order },
+         CustomOrderDefinition.CustomOneofCase.Lean => new() { KeyBinding = "L", Color = BTSColors.Orders.lean_order },
+         CustomOrderDefinition.CustomOneofCase.Join => new() { KeyBinding = "J", Color = BTSColors.Orders.join_order },
+         CustomOrderDefinition.CustomOneofCase.Top => new() { KeyBinding = "D", Color = BTSColors.Orders.top_order },
+         CustomOrderDefinition.CustomOneofCase.Cover => new() { KeyBinding = "C", Color = BTSColors.Orders.cover_order },
+         CustomOrderDefinition.CustomOneofCase.Oco => new() { KeyBinding = "C", Color = BTSColors.Orders.oco_order },
+         CustomOrderDefinition.CustomOneofCase.Pricer => new() { KeyBinding = "C", Color = BTSColors.Orders.pricer_order },
+         CustomOrderDefinition.CustomOneofCase.Offset => new() { KeyBinding = "O", Color = BTSColors.Orders.off_order },
+         CustomOrderDefinition.CustomOneofCase.Trigger => new() { KeyBinding = "T", Color = BTSColors.Orders.trigger_order },
+         CustomOrderDefinition.CustomOneofCase.Spreader => new() { KeyBinding = "", Color = BTSColors.Orders.spreader_order },
+         CustomOrderDefinition.CustomOneofCase.Dynamic => new() { KeyBinding = "", Color = BTSColors.Orders.dynamic_order },
+         _ => new OrderMetadata { KeyBinding = "", Color = BTSColors.Orders.default_order },
+      };
+
+}

src/client/spark/Orders/OrderTypeVMs/OrderTypeVMFactory.cs

@@ -201,7 +201,7 @@ namespace mbts.spark {
          public virtual string abbreviation => model == null ? display_name : model.factory.abbreviate_algo(new OneOf<AlgoType, string>(order_type));
          public virtual AlgoDisplayGroup group => order_type.is_bts_algo_type() ? AlgoDisplayGroup.BTS : AlgoDisplayGroup.Exchange;
          public OrdersModel orders_model => model;
-         public virtual OrderMetadata meta => OrderDataHelper.get_metadata(order_type);
+         public virtual OrderMetadata meta => OrderMetadataStore.get_metadata(order_type);
          // child algos
          [EditorBrowsable(EditorBrowsableState.Never)]
          public dynamic children_vm {

src/client/spark/Spark.csproj

@@ -11,7 +11,7 @@
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <Configurations>Debug;Release</Configurations>
     <Platforms>x64</Platforms>
-    <RuntimeIdentifiers>win10-x64</RuntimeIdentifiers>
+    <RuntimeIdentifiers>win-x64</RuntimeIdentifiers>
     <PlatformTarget>x64</PlatformTarget>
     <LanguageTargets Condition="'$(VisualStudioVersion)'=='17.0'">
       C:/Program Files/Microsoft Visual Studio/2022/Community/Msbuild/Current/Bin/amd64/Microsoft.CSharp.targets

src/client/spark_api/Orders/OrderDataHelper.cs

@@ -1,5 +1,4 @@
-using bts;
-using bts.api;
+using bts.api;
 using bts.api.core;
 using bts.api.orderman;
 using bts.collections;
@@ -10,23 +9,19 @@ using Google.Protobuf;
 using Google.Protobuf.Reflection;
 using Google.Protobuf.WellKnownTypes;
 
-using mbts.ui;
-using mbts.ui.utilities;
 using mbts.ui.utility;
 using mbts.utils;
 
-using Spark.Core.Disposables;
-
 using System;
 using System.Collections.Generic;
-using System.Composition;
 using System.Linq;
 using System.Reactive.Linq;
+using System.Reflection;
 
 using OrderState = bts.api.orderman.OrderState;
 
 namespace mbts.spark.utilities {
-   struct ChildAlgoData {
+   public readonly struct ChildAlgoData {
       public readonly string name;
       public readonly Func<IMessage, CustomOrderDefinition> get_child_from_impl;
       public readonly Action<IMessage, CustomOrderDefinition> assign_child_to_impl;
@@ -48,27 +43,29 @@ namespace mbts.spark.utilities {
       }
    }
 
-   static class OrderDataHelper {
-      class AlgoTypeCaseDescriptorData {
-         public readonly IReadOnlyList<ChildAlgoData> child_algo_data;
-         public readonly IReadOnlyList<FieldDescriptor> value_descriptors;
-         public readonly FieldDescriptor type_case_descriptor;
+   public class AlgoTypeCaseDescriptorData {
+      public readonly IReadOnlyList<ChildAlgoData> child_algo_data;
+      public readonly IReadOnlyList<FieldDescriptor> value_descriptors;
+      public readonly FieldDescriptor type_case_descriptor;
 
-         public AlgoTypeCaseDescriptorData(FieldDescriptor type_case_descriptor, IReadOnlyList<ChildAlgoData> child_algo_data, IReadOnlyList<FieldDescriptor> value_descriptors) {
-            this.type_case_descriptor = type_case_descriptor;
-            this.child_algo_data = child_algo_data;
-            this.value_descriptors = value_descriptors;
-         }
+      public AlgoTypeCaseDescriptorData(
+         FieldDescriptor type_case_descriptor,
+         IReadOnlyList<ChildAlgoData> child_algo_data,
+         IReadOnlyList<FieldDescriptor> value_descriptors
+      ) {
+         this.type_case_descriptor = type_case_descriptor;
+         this.child_algo_data = child_algo_data;
+         this.value_descriptors = value_descriptors;
       }
+   }
 
+   public static class OrderDataHelper {
       const string _primary_child_field = "child";
       // algo-type-case maps
       // Note: obv this will break if we change the name of the 'CustomOneofCase' field
       static readonly OneofDescriptor _oneof_desc = CustomOrderDefinition.Descriptor.Oneofs.FirstOrDefault(fd => fd.Name == "custom");
       static readonly IReadOnlyDictionary<System.Type, CustomOrderDefinition.CustomOneofCase> _algo_type_case_index;
       static readonly IReadOnlyDictionary<CustomOrderDefinition.CustomOneofCase, AlgoTypeCaseDescriptorData> _algo_type_case_descriptor_data;
-      // 
-      static readonly OrderMetadata _default_order_metadata = new OrderMetadata { KeyBinding = "", Color = BTSColors.Orders.default_order };
 
       static OrderDataHelper() {
          var algo_type_case_index = new Dictionary<System.Type, CustomOrderDefinition.CustomOneofCase>();
@@ -106,6 +103,12 @@ namespace mbts.spark.utilities {
       }
 
       #region Algo Structure/Hierarchy
+      public static AlgoTypeCaseDescriptorData get_algo_type_descriptor(
+         CustomOrderDefinition.CustomOneofCase algo_type) =>
+         _algo_type_case_descriptor_data.TryGetValue(algo_type, out var d) ? d :
+            throw new ArgumentException(
+               $"Failed to find descriptor data for algo type '{algo_type}'", nameof(algo_type));
+
       public static IEnumerable<(CustomOrderDefinition.CustomOneofCase type, string dyn, uint studio, uint spreader_sec)> enumerate_algo_structure(CustomOrderDefinitionImmutable custom_defi) {
          string dyn = null;
          uint studio = 0, spreader_sec = 0;
@@ -205,29 +208,6 @@ namespace mbts.spark.utilities {
          _algo_type_case_descriptor_data.TryGetValue(algo_type, out AlgoTypeCaseDescriptorData d) ? d.child_algo_data : Array.Empty<ChildAlgoData>();
       #endregion
 
-      public static OrderMetadata get_metadata(CustomOrderDefinition.CustomOneofCase order_type) =>
-         order_type switch {
-            CustomOrderDefinition.CustomOneofCase.Market => new() { KeyBinding = "M", Color = BTSColors.Orders.market_order, NoProps = true },
-            CustomOrderDefinition.CustomOneofCase.Limit => new() { KeyBinding = "L", Color = BTSColors.Orders.limit_order, NoProps = true },
-            CustomOrderDefinition.CustomOneofCase.Stop => new() { KeyBinding = "S", Color = BTSColors.Orders.stop_order, NoProps = true },
-            CustomOrderDefinition.CustomOneofCase.StopLimit => new() { KeyBinding = "SL", Color = BTSColors.Orders.stop_limit_order },
-            CustomOrderDefinition.CustomOneofCase.GlbxIceberg => new() { KeyBinding = "GB", Color = BTSColors.Orders.glbx_iceberg_order },
-            CustomOrderDefinition.CustomOneofCase.Iceberg => new() { KeyBinding = "I", Color = BTSColors.Orders.iceberg_order },
-            CustomOrderDefinition.CustomOneofCase.TimeBox => new() { KeyBinding = "TB", Color = BTSColors.Orders.time_box_order },
-            CustomOrderDefinition.CustomOneofCase.Tick => new() { KeyBinding = "T", Color = BTSColors.Orders.tick_order },
-            CustomOrderDefinition.CustomOneofCase.Lean => new() { KeyBinding = "L", Color = BTSColors.Orders.lean_order },
-            CustomOrderDefinition.CustomOneofCase.Join => new() { KeyBinding = "J", Color = BTSColors.Orders.join_order },
-            CustomOrderDefinition.CustomOneofCase.Top => new() { KeyBinding = "D", Color = BTSColors.Orders.top_order },
-            CustomOrderDefinition.CustomOneofCase.Cover => new() { KeyBinding = "C", Color = BTSColors.Orders.cover_order },
-            CustomOrderDefinition.CustomOneofCase.Oco => new() { KeyBinding = "C", Color = BTSColors.Orders.oco_order },
-            CustomOrderDefinition.CustomOneofCase.Pricer => new() { KeyBinding = "C", Color = BTSColors.Orders.pricer_order },
-            CustomOrderDefinition.CustomOneofCase.Offset => new() { KeyBinding = "O", Color = BTSColors.Orders.off_order },
-            CustomOrderDefinition.CustomOneofCase.Trigger => new() { KeyBinding = "T", Color = BTSColors.Orders.trigger_order },
-            CustomOrderDefinition.CustomOneofCase.Spreader => new() { KeyBinding = "", Color = BTSColors.Orders.spreader_order },
-            CustomOrderDefinition.CustomOneofCase.Dynamic => new() { KeyBinding = "", Color = BTSColors.Orders.dynamic_order },
-            _ => _default_order_metadata,
-         };
-
       public static string abbreviate_algo_type(CustomOrderDefinition.CustomOneofCase algo_type) {
          switch (algo_type) {
             case CustomOrderDefinition.CustomOneofCase.Limit: return "LM";
@@ -331,6 +311,19 @@ namespace mbts.spark.utilities {
          order_type_def != null && _algo_type_case_index.TryGetValue(order_type_def.Descriptor.ClrType, out CustomOrderDefinition.CustomOneofCase algo_case) ?
             algo_case : CustomOrderDefinition.CustomOneofCase.None;
 
+      public static void setup_generic_custom_order(
+         CustomOrderDefinition custom, CustomOrderDefinition.CustomOneofCase type) {
+         var prop = typeof(CustomOrderDefinition).GetProperty(type.ToString(),
+            BindingFlags.Instance | BindingFlags.Public);
+         var rt = prop.PropertyType;
+         var rtt = Activator.CreateInstance(rt);
+
+         var ci = rt.GetProperties().Where(x => x.PropertyType == typeof(CustomOrderDefinition));
+         foreach (var cid in ci)
+            cid.SetValue(rtt, new CustomOrderDefinition { Limit = new LimitOrderDefinition() });
+         prop.SetValue(custom, rtt);
+      }
+
       #region Private Members
       // get child
       static CustomOrderDefinitionImmutable? _get_iceberg_child(IcebergOrderDefinitionImmutable iceberg) => iceberg.HasChild ? iceberg.Child : (CustomOrderDefinitionImmutable?)null;
@@ -655,7 +648,7 @@ namespace mbts.spark.utilities {
                case bts.api.orderman.CustomOrderDefinition.CustomOneofCase.Spreader: rv.Spreader = SpreaderOrderDefinition(); break;
                case bts.api.orderman.CustomOrderDefinition.CustomOneofCase.Dynamic: rv.Dynamic = DynamicOrderDefinition(); break;
                default:
-                  OrderTypeVMFactory.GenericOrderFactory.setup_generic_custom_order(rv, order_type);
+                  setup_generic_custom_order(rv, order_type);
                   break;
             }
             return rv;

src/data_files/documents/order_flow.txt

@@ -79,6 +79,24 @@ Misc:
 To search for a publisher for an event search for make_stream<Event_Name>, for 
 example make_stream<OrderStateUpdateEvent>()
 
+is_gateway_connected can be false in the NativeOrder class (GlbxNativeOrder) ->
+OrderServices->is_gateway_connected->
+SessionServices->is_gateway_connected
+SessionServicesData->gw_connected
+OrderManager sets SessionServicesData->gw_connected on void OrderManager::on_gateway_state(const mercury::Update<GatewayStatusChangeEvent> &m)
+
+
+Exchange gateway is down (debugging):
+-------------------------------------
+This happens at times and if often a config issue.
+GlbCNativeOrder::_on_activate ->
+OrderServices::is_gateway_connected ->
+SessionServices::is_gateway_connected ->
+SessionServicesData::
+
+IExecutionGateway::on_gw_status_change is what sends out the status
+
+
 --------------------------------------------------------------------------
 market data
 --------------------------------------------------------------------------

src/lib/config/profiles/default/jetfix.conf

@@ -3,7 +3,7 @@ JetFIXSimConnection {
    value {
       sim {
         name: "sim"
-        venue: VENUE_SIM
+        venue: VENUE_GLBX
         socket_connect_host_ip: "127.0.0.1"
         socket_connect_port: @Ports.glbx_sim_fix_port
         sender_compid: "SPARK" # this is the FIX session name

src/lib/md/MarketDataGateway.cpp

@@ -1,4 +1,4 @@
-#if !defined(_WIN32) && defined(SPARK_ENABLE_CCAPI)
+#if !defined(_WIN32) //&& defined(SPARK_ENABLE_CCAPI)
 #include "MarketDataGateway.h"
 #include "lib/api/md/EmumbaDeribitMarketDataConfig.pb.h"
 #include "lib/api/md/MarketDataConfig.pb.h"

src/lib/md/recording/MDRecorder.cpp

@@ -1,4 +1,4 @@
-#if !defined(_WIN32) && defined(SPARK_ENABLE_CCAPI)
+#if !defined(_WIN32) //&& defined(SPARK_ENABLE_CCAPI)
 #include "MDRecorder.h"
 #include "lib/api/md/MarketDataConfig.pb.h"
 #include "lib/api/md/MarketDataConfigUtils.h"
@@ -184,7 +184,7 @@ void MDRecorder::_poll_md() {
       uint64_t now_ns = core::TimeStamp::now_ns();
       for (auto &kv : me->handles) {
          MDRecordingSecEntry &md_entry = kv.second;
-         if(me->settings.n_book_levels > 0)
+         if (me->settings.n_book_levels > 0)
             _poll_books(md_entry);
          _poll_trades(md_entry);
          _poll_misc(md_entry);

src/lib/md/recording/MDReplayer.cpp

@@ -1,4 +1,4 @@
-#if !defined(_WIN32) && defined(SPARK_ENABLE_CCAPI)
+#if !defined(_WIN32) //&& defined(SPARK_ENABLE_CCAPI)
 #include "MDReplayer.h"
 #include "lib/config/config.h"
 #include "lib/config/protos/system_config.pb.h"

src/lib/orderman/actors/order_manager/OrderManager.cpp

@@ -381,6 +381,9 @@ void OrderManager::on_gateway_state(
     const mercury::Update<GatewayStatusChangeEvent> &m) {
    if (m.events)
       for (auto const &ev : *m.events) {
+         LOG(OrderMan, Info,
+             " OrderManager::on_gateway_state gw: {} connected: {}", ev.gw_name,
+             ev.is_connected);
          auto &sr = me->services_data.get_session(ev.gw_name.value());
          sr.gw_connected = ev.is_connected;
       }

src/lib/orderman/events/GatewayStatusChangeEvent.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include "lib/logging/logging.h"
 #include "lib/mercury/synopsis/CacheLast.h"
 #include "lib/orderman/orderman_utils/orderman_common.h"
 #include <memory>
@@ -14,7 +15,11 @@ class GatewayStatusChangeEvent
    using SynopsisPolicy = mercury::CacheLast;
 
    GatewayStatusChangeEvent(GatewayName const &n, bool c)
-       : gw_name(n), is_connected(c) {}
+       : gw_name(n), is_connected(c) {
+      LOG(OrderMan, Info,
+          "GatewayStatusChangeEvent::cons gw_name: {} is_connected: {}",
+          gw_name, is_connected);
+   }
 
    GatewayName gw_name;
    bool is_connected;

src/lib/orderman/gateways/sim/SimExecutionGateway.cpp

@@ -30,6 +30,7 @@
 #include "lib/utils/bts_assert.h"
 #include "lib/utils/bts_chrono.h"
 #include "lib/utils/helper_macros.h"
+#include "lib/utils/thread_manager.h"
 #include "server/sparkd/md/platforms/emumba/IEMOrderMuxer.h"
 
 #include <bits/stdint-uintn.h>
@@ -347,8 +348,10 @@ void SimExecutionGateway::on_admin_data(
        bts::os::get_thread_name());
 }
 
-void SimExecutionGateway::send_fix(const jetfix::OutFIXPtr &, trace::CorID) {
-   bts_assert(false);
+void SimExecutionGateway::send_fix(const jetfix::OutFIXPtr &p, trace::CorID) {
+   LOG(OrderMan, Warn, "SimExecutionGateway::send_fix p sz: {}", p->fix.size());
+   // TODO
+   //bts_assert(false);
 }
 
 // This is where the messages are sent to the exchange. We simply write the
@@ -364,7 +367,7 @@ void SimExecutionGateway::send_msgs(
        "SimExecutionGateway::send_msgs tid: {} tname: {} count: {}",
        bts::os::get_thread_id(), bts::os::get_thread_name(), msg_vec.size());
    for (auto &m : msg_vec) {
-      LOG(OrderMan, Diag,
+      LOG(OrderMan, Info,
           "SimExecutionGateway::send_msgs tid: {} tname: {} m: {}",
           bts::os::get_thread_id(), bts::os::get_thread_name(), m);
       me->buffered_order_ops.push(m);
@@ -373,6 +376,10 @@ void SimExecutionGateway::send_msgs(
 
 OnExeReport &SimExecutionGateway::on_exe_report() { return me->on_exe_report; }
 OnGwStatusChange &SimExecutionGateway::on_gw_status_change() {
+   std::string tid_name = "SimExecutionGateway::_send_gw_status_once";
+   bts::threading::make_thread(tid_name, [&]() {
+      _send_gw_status_once();
+   }).detach();
    return me->on_gw_status_change;
 }
 OnCancelReject &SimExecutionGateway::on_cancel_reject() {
@@ -675,6 +682,13 @@ SimExecutionGateway::_find_order(OrderKey key) const {
        nullptr, nullptr);
 }
 
+void SimExecutionGateway::_send_gw_status_once() {
+   auto sleep_period = std::chrono::milliseconds(2500);
+   std::this_thread::sleep_for(sleep_period);
+   // fire the status and exit the thread
+   me->on_gw_status_change(true);
+}
+
 // this returns the security key from an OrdMsg. The reason it has more code
 // than needed is because CancelOrderMsg might not have the security set.
 // therefore we need to find the security from the order key

src/lib/orderman/gateways/sim/SimExecutionGateway.h

@@ -76,6 +76,7 @@ class SimExecutionGateway : public IExecutionGateway {
    std::tuple<SEGSecurityData const *, SEGOrderState const *> _find_order(
        OrderKey key) const;
    std::optional<SecurityKey> _extract_sec_key(const OrderMsg &msg);
+   void _send_gw_status_once();
 
  private:
    struct PImpl;

src/lib/orderman/gateways/sim/SimExecutionGatewayFactory.cpp

@@ -61,6 +61,8 @@ SimExecutionGatewayFactory *SimExecutionGatewayFactory::get() {
 }
 
 bool SimExecutionGatewayFactory::is_sim_execution_on(api::Venue venue) {
+   // TODO: Arham, fix this !
+   return true;
    const auto &sim_exec = config::get<bts::config::SimExecution>();
    for (const auto &e : sim_exec.entries()) {
       if (e.venue() == venue) {

src/lib/orderman/orders/GLBXNativeOrder.cpp

@@ -4,11 +4,14 @@
 #include "lib/config/protos/orderman_cfg.pb.h"
 #include "lib/config/protos/toggles.pb.h"
 #include "lib/domain_model/Security.h"
+#include "lib/domain_model/Security.pb.h"
+#include "lib/domain_model/common.h"
 #include "lib/jetfix/codec/FIXFragment.h"
 #include "lib/jetfix/codec/FieldDefinitions.h"
 #include "lib/logging/logging.h"
 #include "lib/orderman/OrderRouter.h"
 #include "lib/orderman/OrderServices.h"
+#include "lib/orderman/gateways/sim/SimExecutionGatewayFactory.h"
 #include "lib/orderman/order_factory/IOrderFactory.h"
 #include "lib/orderman/orderman_utils/ExecutionReport.h"
 #include "lib/orderman/orderman_utils/fix_order_codec.h"
@@ -65,6 +68,8 @@ struct GLBXNativeOrder::PImpl {
    std::optional<api::orderman::Update> buffered_update;
    RoutingRecord routing_record;
    bool stop_order_still_has_stop_px = true;
+   const bool in_sim_mode =
+       SimExecutionGatewayFactory::is_sim_execution_on(api::VENUE_GLBX);
 };
 
 GLBXNativeOrder::GLBXNativeOrder() : me(new PImpl()) {}
@@ -471,8 +476,13 @@ OrderOpResult GLBXNativeOrder::on_activate(
              "delta:{}",
              string{corid.channel}, corid.sn, corid.app_arrival.to_uint64(),
              now.to_uint64(), (now - corid.app_arrival).nanoseconds());
-
+         // in sim mode, we dont need this,
          me->encoder->send_create(*me->services, *me->session_services);
+         if (me->in_sim_mode) {
+            CreateOrderMsg c_msg = {};
+            _create_create_msg(c_msg);
+            me->services->queue_order_msg(*me->session_services, c_msg);
+         }
          me->services->governor(*me->session_services).record_sent();
          me->state.set_status(ORDER_STATUS_PENDING_WORKING);
 
@@ -630,6 +640,16 @@ OrderOpResult GLBXNativeOrder::on_update(api::orderman::Update const &update) {
                me->services->report_definition(me->definition);
                me->encoder->update_cancel_replace_stop_px(
                    stop_px, me->price_encoding_precision);
+               if (me->in_sim_mode) {
+                  CancelReplaceOrderMsg cxl_repl_msg = {};
+                  _create_create_msg(cxl_repl_msg.create_fields);
+                  Price new_p = Price(update.price());
+                  Quantity new_q = bts::orderman::get_update_quantity(update);
+                  cxl_repl_msg.create_fields.price = new_p;
+                  cxl_repl_msg.create_fields.quantity = new_q;
+                  me->services->queue_order_msg(
+                      *me->session_services, cxl_repl_msg);
+               }
             }
          } else if (
              update.custom().custom_case() ==
@@ -773,6 +793,13 @@ OrderOpResult GLBXNativeOrder::on_cancel() {
    case ORDER_STATUS_WORKING: {
       me->encoder->update_cancel_encoding(*me->services);
       me->encoder->send_cancel(*me->services, *me->session_services);
+      // TODO: check for sim mode here
+      if (me->in_sim_mode) {
+         CancelOrderMsg cxl_msg = {};
+         cxl_msg.key = key();
+         LOG(OrderMan, Info, "GLBXNativeOrder queue_order_msg for cxl");
+         me->services->queue_order_msg(*me->session_services, cxl_msg);
+      }
       break;
    }
 
@@ -1326,5 +1353,37 @@ void GLBXNativeOrder::fixup_routing_record(api::orderman::RoutingRecord *rr) {
                                           .fallback_cust_order_handling_inst());
 }
 
+// TODO, make it static
+NativeOrderType get_glbx_native_ord_type(const OrderDefinition &def) {
+   switch (def.custom_def().custom_case()) {
+   case CustomOrderDefinition::kMarket:
+      return NativeOrderType::NATIVE_ORDER_TYPE_MARKET;
+      break;
+   case CustomOrderDefinition::kLimit:
+      return NativeOrderType::NATIVE_ORDER_TYPE_LIMIT;
+      break;
+   default:
+      break;
+   }
+   bts_assert(false);
+   return api::orderman::NATIVE_ORDER_TYPE_LIMIT;
+}
+
+void GLBXNativeOrder::_create_create_msg(CreateOrderMsg &c) {
+   c.key = key();
+   c.price = Price(me->state.price());
+   c.quantity =
+       bts::orderman::get_quantity(me->state, StateQuantityType::OrderQuantity);
+   c.security = me->sec;
+   c.side = me->definition.side();
+   c.order_type = get_glbx_native_ord_type(me->definition);
+   c.time_in_force = me->definition.time_in_force();
+   c.order_type = api::orderman::NATIVE_ORDER_TYPE_LIMIT;
+   c.manual_order_indicator = me->definition.manual_order_indicator();
+   c.routing_record = &me->routing_record;
+   LOG(OrderMan, Info, "GLBXNativeOrder::_create_create_msg c.qty: {}",
+       c.quantity);
+}
+
 } // namespace orderman
 } // namespace bts

src/lib/orderman/orders/GLBXNativeOrder.h

@@ -11,6 +11,7 @@ class NewExecutionReportEvent;
 class CancelRejectEvent;
 class GatewayStatusChangeEvent;
 class ExecutionReport;
+struct CreateOrderMsg;
 
 class GLBXNativeOrder : public NativeOrderBase {
  public:
@@ -42,7 +43,7 @@ class GLBXNativeOrder : public NativeOrderBase {
    struct PImpl;
    std::unique_ptr<PImpl> me;
    void init_encodings();
-
+   void _create_create_msg(CreateOrderMsg &);
    void fixup_routing_record(api::orderman::RoutingRecord *rr);
    void on_exe_report(const NewExecutionReportEvent &);
    void on_cancel_reject(const CancelRejectEvent &);
@@ -60,8 +61,8 @@ class GLBXNativeOrder : public NativeOrderBase {
        log::Level,
        api::orderman::OrderStatus new_state,
        std::string const &msg);
-   void
-   transition(api::orderman::OrderStatus new_state, std::string const &msg);
+   void transition(
+       api::orderman::OrderStatus new_state, std::string const &msg);
    void transition(api::orderman::OrderStatus new_state);
    void transition_to_error(
        api::orderman::NativeOrderState::SubError sub_error,
@@ -74,8 +75,8 @@ class GLBXNativeOrder : public NativeOrderBase {
        api::orderman::NativeOrderState &,
        const NewExecutionReportEvent &) const;
    bool terminal_state(api::orderman::OrderStatus status) const;
-   api::orderman::NativeOrderState &
-   get_native_state(api::orderman::OrderState &);
+   api::orderman::NativeOrderState &get_native_state(
+       api::orderman::OrderState &);
    void ignore_or_cancel(
        std::string const &, api::orderman::OrderError, OrderOpResult *out);