all exectuion work

The previous regex was overly ambitious (and needlessly complex as a
result). The new regex is simpler, and doesn't impose bounds on month
and day ints, aside from requiring 1-2 digits.

Bugs (fixed):
  * The DataStreamExpressionInterpretter fails to parse certain date
    constants, such as "8/30/2024"

.devcontainer/Dockerfile

@@ -87,6 +87,9 @@ RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
 # Pcap
 RUN apt-get update && apt-get install -y libpcap0.8-dev
 
+# Databento (deps)
+RUN apt-get update && apt-get -y install libzstd-dev
+
 # Set up default executables (update-alternatives)
 RUN \
    update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \

.devcontainer/devcontainer.json

@@ -27,7 +27,7 @@
     "-p",
     "8003:8003",
     "-p",
-    "12345:12345",
+    "12345:12345"
   ],
   "mounts": [
     "source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind"

.gitattributes

@@ -1,3 +1,3 @@
-*.cmd eol=crlf
-*.cs eol=crlf
-*.sh eol=lf
+*.cmd text=auto eol=crlf
+*.cs text=auto eol=crlf
+*.sh text=auto eol=lf

.vscode/launch.json

@@ -26,11 +26,23 @@
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
+    {
+      "name": "(gdb) Launch",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
+      "args": ["run"],
+      "stopAtEntry": false,
+      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
+      "environment": [],
+      "externalConsole": false,
+      "MIMode": "gdb",
+    },
     {
       "name": "DataStreamQueryAnalyzer",
       "type": "coreclr",
       "request": "launch",
-      "preLaunchTask": "Build DataStream (Debug)",
+      //"preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
       "program": "",
       "windows": {
         "program": "DataStreamQueryAnalyzer.exe",
@@ -143,4 +155,4 @@
       "default": "http://localhost:8000/config",
     },
   ]
-}
+}

.vscode/tasks.json

@@ -526,6 +526,28 @@
         "color": "terminal.ansiGreen",
       },
     },
+    {
+      "label": "Run PubSubServer (Debug | Skip Build)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "PubSubServer",
+      },
+      "windows": {
+        "command": "PubSubServer.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
     {
       "label": "Run PubSubServer (Release)",
       "dependsOn": "Build DataStream (Release)",
@@ -549,6 +571,28 @@
         "color": "terminal.ansiGreen",
       },
     },
+    {
+      "label": "Run PubSubServer (Release | Skip Build)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "PubSubServer",
+      },
+      "windows": {
+        "command": "PubSubServer.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
     {
       "label": "Run Spark",
       "dependsOn": "Build Spark",
@@ -684,6 +728,36 @@
         "id": "wrench"
       },
     },
+    {
+      "label": "View PubSubServer Logs (Last Run)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/logging/show_last_run.sh",
+        "args": [
+          "${env:HOME}/.local/share/DataStream/data/logging.PubSubServer",
+          "Starting PubSubServer"
+        ]
+      },
+      "windows": {
+        "command": "tools/logging/show_last_run.cmd",
+        "args": [
+          "DataStream",
+          "PubSubServer.exe",
+          "Starting PubSubServer"
+        ]
+      },
+      "presentation": {
+        "reveal": "always",
+        "focus": true,
+        "panel": "shared",
+        "showReuseMessage": false,
+        "close": true,
+      },
+      "icon": {
+        "id": "wrench"
+      },
+    },
   ],
   "inputs": [
     {

CMakeLists.txt

@@ -132,6 +132,8 @@ else()
       COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
       COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
       COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/${CMAKE_BUILD_TYPE}/libonixs-fix-engine.so.4 ${CMAKE_BINARY_DIR}/server
+      
       COMMAND
         ln ARGS -sf
         ${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0

src/DataStreamSuite/DataStream/Primitives/Distinct/DistinctRowStream.cs

@@ -1,23 +1,24 @@
+using bts;
+
 using System;
 using System.Collections;
 
-namespace BCG
-{
+namespace BCG {
 	/// <summary>
 	/// Provides a stream that with distinct rows with respect to the columns
 	/// the user specifies in the argument
 	/// </summary>
-	[DataStreamUserRespresentation ("distinct")]
-	public class DistinctRowStream : ViewStream
-	{
-		public DistinctRowStream (ViewStream stream, params string[] cols)
-		{
-         ArrayList groupByCols = new ArrayList ();
-         foreach (string col in cols)
-            foreach (string c in col.Split (' '))
-               groupByCols.Add (c);
-         mySetRealSource (new GroupByStream (stream, 
-            new DistinctRowEvaluator (groupByCols.ToArray (typeof(string)) as string[]), false));
+	[DataStreamUserRespresentation("distinct")]
+	public class DistinctRowStream : ViewStream {
+		public DistinctRowStream(ViewStream stream, params string[] cols) {
+			var g_by_eval = new DistinctRowEvaluator(cols);
+			ViewStream r = new GroupByStream(stream, g_by_eval);
+			var proj_exp = String.Empty;
+			foreach (var c in cols)
+				proj_exp += $"{c} ";
+			proj_exp = proj_exp.Trim();
+			r = new NewProjectStream(r, proj_exp);
+			mySetRealSource(r);
 		}
 	}
 }

src/DataStreamSuite/DataStream/Primitives/FileStreams/DumpLoaderStream.cs

@@ -1,49 +0,0 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Text;
-// using System.IO;
-
-// using System.Runtime.Serialization.Formatters.Binary;
-
-// namespace BCG
-// {
-//    public class DumpLoaderStream : ViewStream, ISideEffectStream
-//    {
-//       //wtf
-//       //public class Exception : System.Exception
-//       //{public Exception(string msg) : base(msg) { }}
-
-//       public DumpLoaderStream(string fileName, int secondsBetweenPolls)
-//       {
-//          mySetRealSource(
-//             new PollStream(() => new LoaderStream(fileName).rows, secondsBetweenPolls, 
-//                ViewStreamDatabase.instance));
-//       }
-
-//       public DumpLoaderStream(string fileName)
-//       {
-//         if (fileName == null)
-//            throw new LoaderStream.Exception("File name cannot be null");
-
-//         for (int i = 0; i < 5; i++)
-//            try
-//            {
-//               using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
-//               {
-//                  IClientDeserializer deser = new ClientUniversalDeserializer();
-//                  byte[] the_ops;
-//                  Array the_rows;
-//                  deser.deserialize(new Stream[] { fs }, out the_ops, out the_rows);
-
-//                  mySetRealSource(new BlueRowCollectionStaticStream((IBlueRowCollection)the_rows.GetValue(0)));
-//               }
-//               break;
-//            }
-//            catch (IOException)
-//            {
-//               System.Threading.Thread.Sleep(50);
-//            }
-
-//       }
-//    }
-// }

src/DataStreamSuite/DataStream/Primitives/FileStreams/LoaderStream.cs

@@ -1,6 +1,7 @@
 using BCG;
 
 namespace bts {
+   // This is the other side of the SaverStream, its "loads" the stream and fires once
    [DataStreamUserRespresentation("loader")]
    public class LoaderStream : ViewStream {
 

src/DataStreamSuite/DataStream/Primitives/FileStreams/PersistorStream.cs

@@ -5,10 +5,11 @@ using System.Text;
 using System.IO;
 using Google.Protobuf;
 using System.Globalization;
-using bts;
+using BCG;
 
-namespace BCG {
-   // Writes all the udpates to a stream in a file.
+namespace bts {
+   // Writes all the udpates to a stream in a file or multiples files dependong on the rollover size (as opposed to Saver
+   // which just writes one update, as a create)
    // File naming:
    // token_epoch_yyyyMMdd_HHmmss
    // File location: directory(passed)\token\
@@ -94,10 +95,10 @@ namespace BCG {
       void _on_change(object new_value) {
          DataPacket dp = new_value as DataPacket;
          byte[] serialized_update = null;
-         bool _write_create = _serializer == null || _n_updates_in_current_file == 0;
+         bool first_write = _serializer == null || _n_updates_in_current_file == 0;
          _serializer = _serializer ?? new UniversalStreamIOSerializer();
-         if (_write_create) {
-            // new files always starts with create
+         if (first_write) {
+            // first file always starts with create
             serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.CreateMessage(dp.newRows)));
          } else
             serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows)));
@@ -105,8 +106,8 @@ namespace BCG {
          long utc_epoch = epoch_now();
 
          //time, length of data, data
-         bw.Write(utc_epoch);
-         bw.Write(serialized_update.Length);
+         //bw.Write(utc_epoch);
+         //bw.Write(serialized_update.Length);
          bw.Write(serialized_update);
          _out_fs.Flush();
          myFireEvent(dp);

src/DataStreamSuite/DataStream/Primitives/FileStreams/PlaybackStream.cs

@@ -1,190 +0,0 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Linq;
-// using System.Text;
-// using System.IO;
-// using System.Runtime.Serialization.Formatters.Binary;
-// using BCG.Utilities.General;
-// using bts.collections;
-// using System.Reflection.Metadata;
-
-
-// namespace BCG {
-
-//    public class PlaybackStream : ViewStream, IWorker {
-//       private readonly PlaybackType _type;
-//       private BlueTable _current_table;
-//       private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
-//       // these are the packets already downloaded from the current file
-//       private List<PlaybackPacket> _packets = new List<PlaybackPacket>();
-//       //private IEnumerable<FileStream> _file_streams = new List<FileStream>();
-//       private IEnumerable<string> _file_names;
-//       private Tuple<int, FileStream> _current_fs = new Tuple<int, FileStream>(-1, null);
-//       private DateTime _playback_t0, _playback_t1;
-//       private bool _all_files_read = false;
-//       SnapshotData _snapshot_data = null; // only initialized when in snapshot mode
-
-//       public PlaybackStream(string datetime_start, string datetime_end, string directory_name, string token) {
-//          _type = PlaybackType.Streaming;
-//          _playback_t0 = PersistorStream.parse_datetime(datetime_start);
-//          _playback_t1 = PersistorStream.parse_datetime(datetime_end);
-//          string token_directory_name = $"{directory_name}\\{token}";
-//          bts.Log.Gen.info($"PlaybackStream t0: {_playback_t0} t1: {_playback_t1} dir: {token_directory_name}");
-//          List<string> files_in_range = new List<string>();
-//          if (Directory.Exists(token_directory_name)) {
-//             _file_names =
-//          Directory.GetFiles(token_directory_name).Where(f => {
-//             DateTime file_time = PersistorStream.parse_datetime_from_filename(f);
-//             return file_time > _playback_t0 && file_time < _playback_t1;
-//          });
-//          }
-//       }
-
-//       protected override void myOnFirstSubscriber() {
-//          FileStream fs = new FileStream(_snapshot_data.snapshot_file, FileMode.Open, FileAccess.Read);
-//          _snapshot_data.packets = StreamPersistorUtils.load_packets_from_file(fs);
-//          PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_snapshot_data.packets, _snapshot_data.snapshot_time);
-//          if (snapshot_packet != null) {
-//             foreach (PlaybackPacket p in _snapshot_data.packets) {
-//                byte[] ops;
-//                Array row_array;
-//                _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
-//                for (int i = 0; i < ops.Length; i++) {
-//                   IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-//                   byte op = ops[i];
-//                   if (op == StreamOpCode.create) {
-//                      IBlueRowCollection full_table = rows_args[0];
-//                      _current_table = new BlueTable(full_table);
-//                   } else if (op == StreamOpCode.multipleRemoveAdd) {
-//                      //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-//                      IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-//                      IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-//                      BlueDataUtility.absorbChanges(_current_table, dels, adds);
-//                   } else {
-//                      bts.Log.assert(false, $"Oprator: {op} is not expected");
-//                   }
-//                   if (p.idx == snapshot_packet.idx) {
-//                      myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
-//                      break;
-//                   }
-//                }
-//             }
-//          }
-//       }
-
-
-//       private void _load_packets_from_current_file() {
-//          int curr_idx = _current_fs.Item1;
-//          FileStream curr_fs = _current_fs.Item2;
-//          _packets = new List<PlaybackPacket>();
-//          _packets.AddRange(StreamPersistorUtils.load_packets_from_file(curr_fs));
-//       }
-
-//       private bool current_file_all_replayed {
-//          get {
-//             bool rv = true;
-//             _packets.ForEach(p => rv = rv && p.replayed);
-//             return rv;
-//          }
-//       }
-
-//       protected override void myOnLastSubscriber() {
-//          var fs = _current_fs.Item2;
-//          if (fs != null) {
-//             fs.Close();
-//          }
-//          ViewStreamDatabase.instance.removeWorker(this);
-//       }
-
-//       // public event PlaybackStreamProgress onProgress;
-//       // public event Action onFileExhausted;
-//       private PlaybackPacket _locate_packet_to_replay() {
-//          foreach (var p in _packets) {
-//             // returns the first non replayed packet
-//             if (!p.replayed)
-//                return p;
-//          }
-//          //else
-//          {
-//             double secondsElapsed = new BinaryReader(myFileReader).ReadDouble();
-//             myNextPacketTime = myPlaybackStart + new TimeSpan(0, 0, 0, 0, (int)(1000 * secondsElapsed / myPlaybackSpeed));
-//             if (onProgress != null)
-//                onProgress(myFileReader.Position, myFileReader.Length);
-//          }
-//       }
-
-//       #region IWorker Members
-
-//       public void doWork() {
-//          if (_all_files_read)
-//             return;
-//          if (current_file_all_replayed) {
-//             _load_next_file();
-//          }
-//          PlaybackPacket curr_packet = _locate_packet_to_replay();
-//          if (curr_packet.idx == 0) { // first packet in the file, fire it rightaway 
-//             byte[] ops;
-//             Array row_array;
-//             // what a stupid interface!
-//             _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
-//             _fire_packet(ops, row_array);
-//             curr_packet.replay_timestamp = PersistorStream.epoch_now();
-//          } else {
-//             PlaybackPacket prev_packet = _get_previous_packet(curr_packet);
-//             // this is the time we need to fire (replay) this packet
-//             long time_to_fire = prev_packet.replay_timestamp + curr_packet.offset_from_previous;
-//             long now_epoch = PersistorStream.epoch_now();
-//             // fire when we are past the 
-//             if (now_epoch >= time_to_fire) {
-//                byte[] ops;
-//                Array row_array;
-//                _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
-//                _fire_packet(ops, row_array);
-//                curr_packet.replay_timestamp = PersistorStream.epoch_now();
-//             }
-//          }
-//       }
-
-//       private void _fire_packet(byte[] ops, Array row_array) {
-//          for (int i = 0; i < ops.Length; i++) {
-//             IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-//             byte op = ops[i];
-//             if (op == StreamOpCode.create) {
-//                IBlueRowCollection full_table = rows_args[0];
-//                _current_table = new BlueTable(full_table);
-//                //_current_table.addRows(full_table);
-//                myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
-//                                           (myLastValue as DataPacket).newRows));
-//             } else if (op == StreamOpCode.multipleRemoveAdd) {
-//                //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-//                IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-//                IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-//                BlueDataUtility.absorbChanges(_current_table, dels, adds);
-//                myFireEvent(new DataPacket(_current_table, adds, dels));
-//             } else {
-//                bts.Log.assert(false, $"Oprator: {op} is not expected");
-//             }
-
-//          }
-//       }
-
-//       private void _load_next_file() {
-//          var new_idx = _current_fs.Item1 + 1;
-//          if (new_idx >= _file_names.Count()) {
-//             _all_files_read = true;
-//             bts.Log.Gen.info("All files read");
-//             return;
-//          }
-//          var prev_fs = _current_fs.Item2;
-//          if (prev_fs != null)
-//             prev_fs.Close();
-//          var new_filename = _file_names.ElementAt(new_idx);
-//          bts.Log.Gen.info("PlaybackStream_load_next_file new_filename=: {new_filename}");
-//          var new_fs = new FileStream(new_filename, FileMode.Open, FileAccess.Read);
-//          _current_fs = new Tuple<int, FileStream>(new_idx, new_fs);
-//          _load_packets_from_current_file();
-//       }
-
-//       #endregion
-//    }
-// }

src/DataStreamSuite/DataStream/Primitives/FileStreams/RecorderStream.cs

@@ -1,68 +0,0 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Linq;
-// using System.Text;
-// using System.IO;
-
-// namespace BCG
-// {
-//    public class RecorderStream : ViewStream
-//    {
-//       private ViewStream mySource;
-//       private DateTime myStartTime;
-//       private FileStream myOutFile;
-//       private string myFileName;
-//       private IBlueDataSerializer mySerializer;
-
-//       public RecorderStream(ViewStream src, string recordFile)
-//       {
-//          mySource = src;
-//          myFileName = recordFile;
-
-//          myFileName = myFileName.Replace("%DATE%", DateTime.Now.ToString("yyyyMMdd_HHmmss")); 
-//       }
-
-//       protected override void myOnFirstSubscriber()
-//       {
-//          myStartTime = DateTime.Now;         
-//          mySource.onChange += new ChangeDelegate(mySource_onChange);
-//          myOutFile = new FileStream(myFileName, FileMode.OpenOrCreate);
-//       }
-
-//       protected override void myOnLastSubscriber()
-//       {
-//          mySource.onChange -= mySource_onChange;
-//          myOutFile.Close();
-//       }
-
-//       void mySource_onChange(object newValue)
-//       {
-//          DataPacket dp = (DataPacket)newValue;         
-
-//          BlueTable toWrite = new BlueTable(dp.schema);
-
-//          if (mySerializer == null)
-//          {
-//             mySerializer = new UniversalSerializer(dp.schema);
-//          }
-//          foreach (IBlueRow r in dp.newRows)
-//             toWrite.addRow(r);
-
-//          byte[] result;
-
-//          if (myLastValue == null)
-//             mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out result);
-//          else
-//             mySerializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out result);
-
-//          BinaryWriter bw = new BinaryWriter(myOutFile);
-//          //write down what time this happened
-//          bw.Write((DateTime.Now - myStartTime).TotalSeconds);
-//          bw.Write(result.Length);
-//          bw.Write(result);         
-//          myOutFile.Flush();
-
-//          myFireEvent(dp);
-//       }
-//    }
-// }

src/DataStreamSuite/DataStream/Primitives/FileStreams/SaverStream.cs

@@ -5,7 +5,8 @@ using System.IO;
 using BCG;
 
 namespace bts {
-   // this saves the total state in the filr provided in the cons
+   // this saves the updates as a Create on the same file, each update calls -save, which
+   // reqrites the whole data to the packet. The other side of this stream is the Loader
    [DataStreamUserRespresentation("saver")]
    public class SaverStream : ViewStream {
       private ViewStream _stream;
@@ -31,7 +32,6 @@ namespace bts {
          StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
          byte[] str_bytes = _serializer.serialize(new StreamIOMessage(cr_msg));
          StreamSerializationUtils.write_to_file(str_bytes, _filename);
-
       }
    }
 }

src/DataStreamSuite/DataStream/Primitives/FileStreams/StreamPersistorUtils.cs

@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Globalization;
 using System.IO;
+using bts;
 
 
 namespace BCG {

src/DataStreamSuite/DataStream/Primitives/Misc/PubSubTestStream.cs

@@ -8,26 +8,33 @@ using System;
 using System.IO;
 using System.Linq;
 
-namespace bts {
-    public class PubSubTestingStream : ViewStream, IWorker {
-        public class PayloadMessage {
+namespace bts
+{
+    public class PubSubTestingStream : ViewStream, IWorker
+    {
+        public class PayloadMessage
+        {
             public int msg_id = 0;
             public string guid;
             public Int64 travel_epoch_ms = 0;
             public byte[] payload = null;
-            public PayloadMessage(int msg_id_args, Int64 send_epoch_args, int payload_size, byte payload_byte) {
+            public PayloadMessage(int msg_id_args, Int64 send_epoch_args, int payload_size, byte payload_byte)
+            {
                 msg_id = msg_id_args;
                 travel_epoch_ms = send_epoch_args;
                 payload = new byte[payload_size];
-                for (int i = 0; i < payload_size; i++) {
+                for (int i = 0; i < payload_size; i++)
+                {
                     payload[i] = payload_byte;
                 }
             }
             private PayloadMessage() { }
             public PayloadMessage(int msg_id, int payload_size, byte payload_byte) :
-            this(msg_id, PubSubTestingStream.epoch_now(), payload_size, payload_byte) { }
+            this(msg_id, PubSubTestingStream.epoch_now(), payload_size, payload_byte)
+            { }
 
-            public byte[] serialize() {
+            public byte[] serialize()
+            {
                 var str = new MemoryStream();
                 BinaryWriter bw = new BinaryWriter(str);
                 bw.Write(msg_id);
@@ -40,7 +47,8 @@ namespace bts {
                 return bytes;
             }
 
-            public static PayloadMessage deserialize(byte[] ser_bytes) {
+            public static PayloadMessage deserialize(byte[] ser_bytes)
+            {
                 var str = new MemoryStream(ser_bytes);
                 BinaryReader br = new BinaryReader(str);
                 int msg_id = br.ReadInt32();
@@ -48,7 +56,8 @@ namespace bts {
                 Int64 epoch = br.ReadInt64();
                 int len = br.ReadInt32();
                 var payload = br.ReadBytes(len);
-                return new PayloadMessage() {
+                return new PayloadMessage()
+                {
                     msg_id = msg_id,
                     guid = guid,
                     travel_epoch_ms = epoch,
@@ -73,7 +82,8 @@ namespace bts {
         private readonly int _payload_sz;
         private const byte _payload_byte = (byte)'A';
         private readonly string _tag_name;
-        public PubSubTestingStream(int publish_period_msecs, int max_pubs, int payload_size, string tag_name, ViewStreamDatabase vsd) {
+        public PubSubTestingStream(int publish_period_msecs, int max_pubs, int payload_size, string tag_name, ViewStreamDatabase vsd)
+        {
             _vsd = vsd;
             _payload_sz = payload_size;
             var schema = BlueDataUtility.form_schema("PubSubTest",
@@ -96,43 +106,53 @@ namespace bts {
         public PubSubTestingStream(int publish_period_seconds, int max_pubs, int msg_size, string tag_name)
         : this(publish_period_seconds, max_pubs, msg_size, tag_name, ViewStreamDatabase.instance) { }
 
-        protected override void myOnFirstSubscriber() {
+        protected override void myOnFirstSubscriber()
+        {
             _vsd.addWorker(this);
             System.Threading.Thread th = new System.Threading.Thread(_subscribe_thread_func);
             th.IsBackground = true;
             th.Start();
         }
 
-        protected override void myOnLastSubscriber() {
+        protected override void myOnLastSubscriber()
+        {
             _vsd.removeWorker(this);
             _continue_sub_thread = false;
         }
 
-        void _log_updates(IEnumerable<PubSubServerMessage> updates) {
+        void _log_updates(IEnumerable<PubSubServerMessage> updates)
+        {
             var idx = 0;
-            foreach (var msg in updates) {
+            foreach (var msg in updates)
+            {
                 bts.Log.Gen.info($"PubSubTesStream._subscribe_thread_func idx: {idx} msg: {msg}");
                 idx++;
             }
         }
 
-        private void _subscribe_thread_func() {
+        private void _subscribe_thread_func()
+        {
             // subscription is done in another thread
             SubscriberCore sub_core = new SubscriberCore();
             sub_core.subscribe(_tag_name);
-            while (_continue_sub_thread) {
+            while (_continue_sub_thread)
+            {
                 // TODO recv_id
                 var updates = sub_core.get_updates(TimeSpan.FromMilliseconds(250));
                 if (updates == null || updates.Count() == 0)
                     continue;
                 _log_updates(updates);
                 _n_updates += updates.Count();
-                if (updates.Count() > 0) {
+                if (updates.Count() > 0)
+                {
                     var server_msg = updates.Last();
-                    if (server_msg.data != null && server_msg.data.Length > 0) {
+                    if (server_msg.data != null && server_msg.data.Length > 0)
+                    {
                         _last_recvd_msg = PayloadMessage.deserialize(server_msg.data);
                         _last_recvd_msg.travel_epoch_ms = epoch_now();
-                    } else {
+                    }
+                    else
+                    {
                         // why are we getting empty or null data ? (server is sending data regardless?)
                     }
                     _fire_dirty = true;
@@ -140,29 +160,36 @@ namespace bts {
             }
         }
 
-        private void _publish_next_packet() {
-            if (_last_sent_msg == null) {
-                _last_sent_msg = new PayloadMessage(0, _payload_sz, _payload_byte) {
+        private void _publish_next_packet()
+        {
+            if (_last_sent_msg == null)
+            {
+                _last_sent_msg = new PayloadMessage(0, _payload_sz, _payload_byte)
+                {
                     guid = Guid.NewGuid().ToString()
                 };
-            } else {
+            }
+            else
+            {
                 _last_sent_msg.msg_id++;
                 _last_sent_msg.guid = Guid.NewGuid().ToString();
                 _last_sent_msg.travel_epoch_ms = epoch_now();
             }
             var ser_bytes = _last_sent_msg.serialize();
             _n_pubs++;
-            _pub_core.publish(String.Empty, _tag_name, ser_bytes);
+            _pub_core.publish("Blob", _tag_name, ser_bytes);
             _fire_dirty = true;
         }
 
-        private void _fire_dp() {
+        private void _fire_dp()
+        {
             object last_sent_id = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.msg_id;
             object last_recv_id = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.msg_id;
             object echo_time = DBNull.Value;
             object last_sent_guid = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.guid;
             object last_recv_guid = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.guid;
-            if (_last_sent_msg != null && _last_recvd_msg != null) {
+            if (_last_sent_msg != null && _last_recvd_msg != null)
+            {
                 echo_time = _last_recvd_msg.travel_epoch_ms - _last_sent_msg.travel_epoch_ms;
             }
             var match = last_sent_id.Equals(last_recv_id) && last_sent_guid.Equals(last_recv_guid);
@@ -174,21 +201,25 @@ namespace bts {
             myFireEvent(dp);
         }
 
-        public void doWork() {
+        public void doWork()
+        {
             if (DateTime.Now - _last_publish_time > _publish_period)
-                if (_n_pubs <= _max_pubs) {
+                if (_n_pubs <= _max_pubs)
+                {
                     _publish_next_packet();
                     _last_publish_time = DateTime.Now;
                 }
             // publish 
             // TODO: send id
-            if (_fire_dirty) {
+            if (_fire_dirty)
+            {
                 _fire_dirty = false;
                 _fire_dp();
             }
         }
 
-        public static long epoch_now() {
+        public static long epoch_now()
+        {
             return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
         }
 

src/DataStreamSuite/DataStream/Primitives/Misc/PubsubEchoTestStream.cs

@@ -9,17 +9,22 @@ using System.Linq;
 using System.Runtime.CompilerServices;
 using bts.api;
 
-namespace bts {
+namespace bts
+{
    [DataStreamUserRespresentation("pss_echo")]
-   public class PubsubEchoTestStream : ViewStream, IWorker {
-      public class PayloadMessage {
+   public class PubsubEchoTestStream : ViewStream, IWorker
+   {
+      public class PayloadMessage
+      {
          public readonly string msg_id;
          public readonly Int64 timestamp = 0;
          public readonly byte[] payload = null;
 
-         public static byte[] create_data(int payload_size, byte payload_byte) {
+         public static byte[] create_data(int payload_size, byte payload_byte)
+         {
             var rv = new byte[payload_size];
-            for (int i = 0; i < payload_size; i++) {
+            for (int i = 0; i < payload_size; i++)
+            {
                rv[i] = payload_byte;
             }
             return rv;
@@ -35,14 +40,16 @@ namespace bts {
          //       payload[i] = payload_byte;
          //    }
          // }
-         public PayloadMessage(string guid_, Int64 timestamp_, byte[] payload_) {
+         public PayloadMessage(string guid_, Int64 timestamp_, byte[] payload_)
+         {
             msg_id = guid_;
             timestamp = timestamp_;
             payload = payload_;
          }
 
          private PayloadMessage() { }
-         public byte[] serialize() {
+         public byte[] serialize()
+         {
             var str = new MemoryStream();
             BinaryWriter bw = new BinaryWriter(str);
             bw.Write(msg_id);
@@ -54,7 +61,8 @@ namespace bts {
             return bytes;
          }
 
-         public static PayloadMessage deserialize(byte[] ser_bytes) {
+         public static PayloadMessage deserialize(byte[] ser_bytes)
+         {
             var str = new MemoryStream(ser_bytes);
             BinaryReader br = new BinaryReader(str);
             string guid = br.ReadString();
@@ -80,13 +88,15 @@ namespace bts {
       private readonly string _tag_name;
       private readonly int _payload_sz;
       private readonly IPubSubRouter _router;
-      private enum IOState {
+      private enum IOState
+      {
          SEND = 0,
          RECV = 1
       };
       IOState _state = IOState.SEND;
 
-      public PubsubEchoTestStream(string server_url, TimeSpan publish_period, int payload_size, string tag_name, ViewStreamDatabase vsd) {
+      public PubsubEchoTestStream(string server_url, TimeSpan publish_period, int payload_size, string tag_name, ViewStreamDatabase vsd)
+      {
          var route = new PubSubRoute(server_url, ".*");
          _router = new PubSubRouter(route);
          _vsd = vsd;
@@ -107,7 +117,8 @@ namespace bts {
       public PubsubEchoTestStream(string server_url, int publish_period_msecs, int payload_size, string tag_name)
       : this(server_url, TimeSpan.FromMilliseconds(publish_period_msecs), payload_size, tag_name, ViewStreamDatabase.instance) { }
 
-      protected override void myOnFirstSubscriber() {
+      protected override void myOnFirstSubscriber()
+      {
          _pub_core = new PublisherCore(_router);
          _sub_core = new SubscriberCore(_router);
          _sub_core.subscribe(_tag_name);
@@ -115,12 +126,14 @@ namespace bts {
 
       }
 
-      protected override void myOnLastSubscriber() {
+      protected override void myOnLastSubscriber()
+      {
          _vsd.removeWorker(this);
          _sub_core.unsubscribe(_tag_name);
       }
 
-      private void _fire_dp_after_send() {
+      private void _fire_dp_after_send()
+      {
          // as soon as the data is published, its reflected in the stream state
          Log.assert(_last_sent_msg != null);
          BlueRow r = new BlueRow(_table.schema, _table.Count, _last_sent_msg.msg_id, _last_sent_msg.timestamp, DBNull.Value, DBNull.Value,
@@ -130,7 +143,8 @@ namespace bts {
          myFireEvent(dp_out);
       }
 
-      private void _fire_dp_after_recv() {
+      private void _fire_dp_after_recv()
+      {
          Log.assert(_last_recvd_msg != null);
          // update the last publish row
          int id = _table.Count - 1;
@@ -144,11 +158,13 @@ namespace bts {
          myFireEvent(dp_out);
       }
 
-      private string _get_next_id() {
+      private string _get_next_id()
+      {
          return _table.Count.ToString();
       }
 
-      private void _publish_next_packet() {
+      private void _publish_next_packet()
+      {
          var bytes = PayloadMessage.create_data(_payload_sz, _payload_byte);
          _last_sent_msg = new PayloadMessage(_get_next_id(), epoch_now(), bytes);
          var ser_bytes = _last_sent_msg.serialize();
@@ -157,19 +173,25 @@ namespace bts {
          _last_publish_time = DateTime.Now;
       }
 
-      public void doWork() {
-         Func<IEnumerable<PubSubServerMessage>, PayloadMessage> find_packet = (updates) => {
-            foreach (var u in updates) {
+      public void doWork()
+      {
+         Func<IEnumerable<PubSubServerMessage>, PayloadMessage> find_packet = (updates) =>
+         {
+            foreach (var u in updates)
+            {
                var recv_msg = PayloadMessage.deserialize(u.data);
-               if (recv_msg.msg_id == _last_sent_msg.msg_id) {
+               if (recv_msg.msg_id == _last_sent_msg.msg_id)
+               {
                   return recv_msg;
                }
             }
             return null;
          };
-         switch (_state) {
+         switch (_state)
+         {
             case IOState.SEND:
-               if (DateTime.Now - _last_publish_time >= _publish_period) {
+               if (DateTime.Now - _last_publish_time >= _publish_period)
+               {
                   _publish_next_packet();
                   _last_publish_time = DateTime.Now;
                   _fire_dp_after_send();
@@ -181,7 +203,8 @@ namespace bts {
                var updates = _sub_core.get_updates(TimeSpan.Zero);
                updates = updates.Where(u => u.tag == _tag_name);
                var recv_msg = find_packet(updates);
-               if (recv_msg != null) {
+               if (recv_msg != null)
+               {
                   _last_recvd_msg = recv_msg;
                   _fire_dp_after_recv();
                   _state = IOState.SEND;
@@ -191,7 +214,8 @@ namespace bts {
          }
       }
 
-      public static long epoch_now() {
+      public static long epoch_now()
+      {
          return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
       }
 

src/DataStreamSuite/DataStream/Primitives/MongoDB/MongoDBStream.cs

@@ -7,122 +7,127 @@ using MongoDB.Driver;
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.Threading.Tasks;
+using System.Threading;
 
 namespace bts {
-    [DataStreamUserRespresentation("mongodb_sql")]
-    public class MongoDBSQLStream : ViewStream, IWorker {
-        private ViewStreamDatabase _vsd;
-        private readonly string _connection_string, _db_name, _collection_name;
-        private MongoClient _client;
-        // make this a sorted table
-        private BlueTable _table;
+    // [DataStreamUserRespresentation("mongodb_sql")]
+    // public class MongoDBSQLStream : ViewStream, IWorker {
+    //     private ViewStreamDatabase _vsd;
+    //     private readonly string _connection_string, _db_name, _collection_name;
+    //     private MongoClient _client;
+    //     // make this a sorted table
+    //     private BlueTable _table;
 
 
-        // Access the database
-        private IMongoDatabase _database;
-        private IDictionary<string, int> _id_map = new Dictionary<string, int>();
-        // mongodb://192.168.1.80:27017
+    //     // Access the database
+    //     private IMongoDatabase _database;
+    //     private IDictionary<string, int> _id_map = new Dictionary<string, int>();
+    //     // mongodb://192.168.1.80:27017
 
-        public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
-        : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
+    //     public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
+    //     : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
 
-        }
-        public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
-            bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
-                                    $"db_name: {db_name}");
-            _connection_string = connection_string;
-            _db_name = db_name;
-            _collection_name = collection_name;
-            _vsd = vsd;
-            _client = new MongoClient(_connection_string);
-            _database = _client.GetDatabase(_db_name);
-        }
+    //     }
+    //     public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
+    //         bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
+    //                                 $"db_name: {db_name}");
+    //         _connection_string = connection_string;
+    //         _db_name = db_name;
+    //         _collection_name = collection_name;
+    //         _vsd = vsd;
+    //         _client = new MongoClient(_connection_string);
+    //         _database = _client.GetDatabase(_db_name);
+    //     }
 
-        protected override void myOnFirstSubscriber() {
-            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
-        }
+    //     protected override void myOnFirstSubscriber() {
+    //         MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
+    //     }
 
-        protected override void myOnLastSubscriber() {
-            MasterWorker.getInstance(false).removeWorker(this);
-        }
+    //     protected override void myOnLastSubscriber() {
+    //         MasterWorker.getInstance(false).removeWorker(this);
+    //     }
 
-        public void doWork() {
-            _read_mogodb();
-        }
+    //     public void doWork() {
+    //         _read_mogodb();
+    //     }
 
-        private void _read_mogodb() {
-            var timer = Stopwatch.StartNew();
-            // Access the collection
-            IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
+    //     private void _read_mogodb() {
+    //         var timer = Stopwatch.StartNew();
+    //         // Access the collection
+    //         IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
 
-            // Fetch all documents from the collection
-            var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
-            _table = _table == null ? null : new BlueTable(_table.schema);
-            // Iterate through the documents and print values
-            foreach (var document in documents.ToEnumerable()) {
-                bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
-                if (_table == null) {
-                    _init_table(document);
-                }
-                _add_row(document);
-            }
-            timer.Stop();
-            Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.ElapsedMilliseconds}ms");
-            timer.Restart();
-            var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
-            timer.Stop();
-            Log.Gen.info($"MongoDBStream time to compare tables :{timer.ElapsedMilliseconds}ms");
-            if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
-                return;
-            myFireEvent(dp);
-        }
+    //         // Fetch all documents from the collection
+    //         var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
+    //         _table = _table == null ? null : new BlueTable(_table.schema);
+    //         // Iterate through the documents and print values
+    //         foreach (var document in documents.ToEnumerable()) {
+    //             bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
+    //             if (_table == null) {
+    //                 _init_table(document);
+    //             }
+    //             _add_row(document);
+    //         }
+    //         timer.Stop();
+    //         Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.ElapsedMilliseconds}ms");
+    //         timer.Restart();
+    //         var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
+    //         timer.Stop();
+    //         Log.Gen.info($"MongoDBStream time to compare tables :{timer.ElapsedMilliseconds}ms");
+    //         if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
+    //             return;
+    //         myFireEvent(dp);
+    //     }
 
-        private void _init_table(BsonDocument doc_row) {
-            bts.Log.assert(_table == null);
-            List<ColumnInfo> cis = new List<ColumnInfo>();
-            cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
-            foreach (var n in doc_row.Names) {
-                cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
-            }
-            IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
-            _table = new BlueTable(schema);
-        }
+    //     private void _init_table(BsonDocument doc_row) {
+    //         bts.Log.assert(_table == null);
+    //         List<ColumnInfo> cis = new List<ColumnInfo>();
+    //         cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
+    //         foreach (var n in doc_row.Names) {
+    //             cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
+    //         }
+    //         IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
+    //         _table = new BlueTable(schema);
+    //     }
 
-        private void _add_row(BsonDocument doc) {
-            var key_val = doc["_id"];
-            var key = key_val.ToString();
-            int id;
-            if (!_id_map.TryGetValue(key, out id)) {
-                id = _id_map.Count;
-                _id_map.Add(key, id);
-            }
-            MutableBlueRow nr = new MutableBlueRow(_table.schema);
-            nr.setValue("id", id);
-            foreach (var n in doc.Names) {
-                var val = doc[n].ToString();
-                nr.setValue(n, val);
-            }
-            _table.addRow(nr);
-        }
-    }
+    //     private void _add_row(BsonDocument doc) {
+    //         var key_val = doc["_id"];
+    //         var key = key_val.ToString();
+    //         int id;
+    //         if (!_id_map.TryGetValue(key, out id)) {
+    //             id = _id_map.Count;
+    //             _id_map.Add(key, id);
+    //         }
+    //         MutableBlueRow nr = new MutableBlueRow(_table.schema);
+    //         nr.setValue("id", id);
+    //         foreach (var n in doc.Names) {
+    //             var val = doc[n].ToString();
+    //             nr.setValue(n, val);
+    //         }
+    //         _table.addRow(nr);
+    //     }
+    // }
 
     [DataStreamUserRespresentation("mongodb")]
     public class MongoDBStream : ViewStream, IWorker {
         private ViewStreamDatabase _vsd;
         private readonly string _connection_string, _db_name, _collection_name;
+        private readonly TimeSpan _poll_period;
         private MongoClient _client;
         private BlueTable _table;
+        private SyncQueue<DataPacket> _table_queue = new SyncQueue<DataPacket>();
+        private BlockingQueue _work_queue = new BlockingQueue();
 
         // Access the database
         private IMongoDatabase _database;
         private IDictionary<string, int> _id_map = new Dictionary<string, int>();
         // mongodb://192.168.1.80:27017
 
-        public MongoDBStream(string connection_string, string db_name, string collection_name)
-        : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
+        public MongoDBStream(string connection_string, string db_name, string collection_name, string poll_period)
+        : this(connection_string, db_name, collection_name, TimeBasedFilter.parse_duration(poll_period), ViewStreamDatabase.instance) {
 
         }
-        public MongoDBStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
+        public MongoDBStream(string connection_string, string db_name, string collection_name, TimeSpan poll_period, ViewStreamDatabase vsd) {
             bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
                                     $"db_name: {db_name}");
             _connection_string = connection_string;
@@ -130,19 +135,30 @@ namespace bts {
             _collection_name = collection_name;
             _vsd = vsd;
             _client = new MongoClient(_connection_string);
+            _poll_period = poll_period;
             _database = _client.GetDatabase(_db_name);
         }
 
         protected override void myOnFirstSubscriber() {
-            MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
+            _vsd.addWorker(this);
+            //MasterWorker.getInstance(false).addWorker(this, _poll_period);
         }
 
         protected override void myOnLastSubscriber() {
-            MasterWorker.getInstance(false).removeWorker(this);
+            _vsd.removeWorker(this);
+            //MasterWorker.getInstance(false).removeWorker(this);
         }
 
+        private DateTime _last_poll_time = DateTime.MinValue;
         public void doWork() {
-            _read_mogodb();
+            var now = DateTime.Now;
+            if (now - _last_poll_time > _poll_period) {
+                ThreadPool.QueueUserWorkItem(_ => _read_mogodb());
+                _last_poll_time = DateTime.Now;
+            }
+            var dp = _table_queue.dequeue(0);
+            if (dp != null)
+                myFireEvent(dp);
         }
 
         private void _read_mogodb() {
@@ -166,10 +182,11 @@ namespace bts {
             timer.Restart();
             var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
             timer.Stop();
-            Log.Gen.info($"MongoDBStream time to compare tables: {timer.ElapsedMilliseconds}ms");;
+            Log.Gen.info($"MongoDBStream time to compare tables: {timer.ElapsedMilliseconds}ms"); ;
             if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
                 return;
-            myFireEvent(dp);
+            _table_queue.enqueue(dp);
+            //myFireEvent(dp);
         }
 
         private void _init_table(BsonDocument doc_row) {
@@ -208,46 +225,46 @@ namespace bts {
         }
     }
 
-    [DataStreamUserRespresentation("mongodb_flat")]
-    public class MongoDBFlatStream : ViewStream {
-        public MongoDBFlatStream(string connection_string, string db_name, string collection_name, string timestamp_col, string ttl_duration)
-        :
-        this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, timestamp_col, TimeBasedFilter.parse_duration(ttl_duration)) { }
+    // [DataStreamUserRespresentation("mongodb_flat")]
+    // public class MongoDBFlatStream : ViewStream {
+    //     public MongoDBFlatStream(string connection_string, string db_name, string collection_name, string timestamp_col, string ttl_duration)
+    //     :
+    //     this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, timestamp_col, TimeBasedFilter.parse_duration(ttl_duration)) { }
 
-        public MongoDBFlatStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name, string timestamp_col, TimeSpan ttl_duration) {
-            ViewStream rs = new MongoDBStream(connection_string, db_name, collection_name, vsd);
-            rs = new KeyValueToRelationalStream(rs, "doc_id", "key", "val");
-            rs = new DateStandardizationStream(rs, timestamp_col, DateConversion.ISO8601);
-            //rs = new TimeBasedFilterStream(vsd, rs, timestamp_col, ttl_duration);
-            mySetRealSource(rs);
-        }
-    }
+    //     public MongoDBFlatStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name, string timestamp_col, TimeSpan ttl_duration) {
+    //         ViewStream rs = new MongoDBStream(connection_string, db_name, collection_name, vsd);
+    //         rs = new KeyValueToRelationalStream(rs, "doc_id", "key", "val");
+    //         rs = new DateStandardizationStream(rs, timestamp_col, DateConversion.ISO8601);
+    //         //rs = new TimeBasedFilterStream(vsd, rs, timestamp_col, ttl_duration);
+    //         mySetRealSource(rs);
+    //     }
+    // }
 
 
 
-    [DataStreamUserRespresentation("newsreplay")]
-    public class NewsReplayStream : ViewStream {
-        public NewsReplayStream(string connection_string, string db_name, string collection_name,
-                                int buffer_sz, string fire_period) :
-        this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, buffer_sz, TimeBasedFilter.parse_duration(fire_period)) { }
+    // [DataStreamUserRespresentation("newsreplay")]
+    // public class NewsReplayStream : ViewStream {
+    //     public NewsReplayStream(string connection_string, string db_name, string collection_name,
+    //                             int buffer_sz, string fire_period) :
+    //     this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, buffer_sz, TimeBasedFilter.parse_duration(fire_period)) { }
 
-        public NewsReplayStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name,
-                                int buffer_sz, TimeSpan fire_period) {
-            ViewStream r;
+    //     public NewsReplayStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name,
+    //                             int buffer_sz, TimeSpan fire_period) {
+    //         ViewStream r;
 
-            r = new MongoDBStream(connection_string, db_name, collection_name, vsd);
-            r = new KeyValueToRelationalStream(r,
-                                                "doc_id",
-                                                "key",
-                                                "value");
-            r = new DateStandardizationStream(r,
-                                               "created_at",
-                                               "ISO8601");
-            r = new SnapshotReplayStream(vsd, r, "created_at", buffer_sz, fire_period);
+    //         r = new MongoDBStream(connection_string, db_name, collection_name, vsd);
+    //         r = new KeyValueToRelationalStream(r,
+    //                                             "doc_id",
+    //                                             "key",
+    //                                             "value");
+    //         r = new DateStandardizationStream(r,
+    //                                            "created_at",
+    //                                            "ISO8601");
+    //         r = new SnapshotReplayStream(vsd, r, "created_at", buffer_sz, fire_period);
 
-            mySetRealSource(r);
-        }
-    }
+    //         mySetRealSource(r);
+    //     }
+    // }
 
 
 }

src/DataStreamSuite/DataStream/Primitives/Project/New/NewProjectStream.cs

@@ -6,6 +6,7 @@ namespace bts {
    using System;
    using System.Collections;
    using System.Collections.Generic;
+   using System.Linq;
 
    public class ProjectData {
       // schema from the base stream
@@ -147,12 +148,23 @@ namespace bts {
                mapped_to_orig_map.Add(mapped_col, orig_col);
                orig_to_mapped.Add(orig_col, mapped_col);
             } else {
-               Log.Gen.error($"NewProjectStream unexpecetd token: {p_tok}");
+               if (input_schema.columnNames.Contains(p_tok)) {
+                  string orig_col = p_tok;
+                  string mapped_col = orig_col;
+                  orig_to_mapped.Add(orig_col, mapped_col);
+               }
             }
          }
          // make the new schema
          List<ColumnInfo> new_cols = new List<ColumnInfo>();
          foreach (ColumnInfo ci in input_schema.columnInfo) {
+            if (ci.colName == "id") {
+               // if there's a hidden col, other columns will show by default, but if cols are being
+               // projected epxlicitely, than others wil be hidden
+               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
+               new_cols.Add(nc);
+               continue;
+            }
             if (removed_cols.Contains(ci.colName)) {
                continue;
             } else if (orig_to_mapped.ContainsKey(ci.colName)) {
@@ -160,8 +172,12 @@ namespace bts {
                ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
                new_cols.Add(nc);
             } else {
-               ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
-               new_cols.Add(nc);
+               if (ci.colName == "id" || removed_cols.Count > 0) {
+                  // if there's a hidden col, other columns will show by default, but if cols are being
+                  // projected epxlicitely, than others wil be hidden
+                  ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
+                  new_cols.Add(nc);
+               }
             }
          }
          IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());

src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs

@@ -1783,6 +1783,18 @@ namespace BCG {
          #endregion
       }
 
+      public static IBlueRow find_max(IEnumerable<IBlueRow> rows, params string[] cols_to_compare) {
+         if (rows == null || rows.Count() == 0)
+            return null;
+         IBlueRow max_row = rows.ElementAt(0);
+         foreach (IBlueRow r in rows) {
+            int c = compare_rows(r, max_row, cols_to_compare);
+            if (c > 0)
+               max_row = r;
+         }
+         return max_row;
+      }
+
       public static int compare_rows(IBlueRow r1, IBlueRow r2, Dictionary<string, bool> sort_map) {
          foreach (KeyValuePair<string, bool> p in sort_map) {
             int comp = compare_send_dontcare_to_last(r1, p.Key, r2, p.Key, p.Value);

src/DataStreamSuite/DataStreamExpressions/DataStreamExpressionTokens.lex

@@ -9,7 +9,15 @@
 Letter                   [a-zA-Z]
 BooleanConstant          'true'|'false'
 CSStringConstant         \"([^\"\\]*|\\.)*\"
-DateTimeConstant         ([1-9]|'10'|'11'|'12')\/([1-9]|([1-2][0-9])|'30'|'31')\/[0-9][0-9][0-9]*
+YearInt                  [0-9]{4}
+MonthInt                 [0-9]{1,2}
+DayInt                   [0-9]{1,2}
+/**
+ * DateTimeConstant supports American (mm/dd/yyyy) and European (yyyy/mm/dd)
+ * formats. Month and day can be 1 or 2 digits. It does not validate that the
+ * numbers represent an actual date.
+ */
+DateTimeConstant         ({MonthInt}\/{DayInt}\/{YearInt}|{YearInt}\/{MonthInt}\/{DayInt})
 IntegerConstant          -?[0-9]+
 NullConstant             'null'
 RealConstant             -?[0-9]+\.[0-9]+

src/DataStreamSuite/DataStreamPublisher/Configurations/MarketDataPublisher.json

@@ -1,6 +1,6 @@
 {
   "user": "bts",
-  "sparkd": "http://127.0.0.1:8000/config",
+  "sparkd": "http://192.168.1.80:8000/config",
   "pubsub": {
     "milliseconds_between_sub_info_updates": 1000,
     "milliseconds_between_time_slice_updates": 5000,
@@ -105,5 +105,25 @@
         }
       ]
     }
+    // {
+    //   "name": "news_data",
+    //   "streams": [
+    //     {
+    //       "name": "benzinga_news",
+    //       "expression": "benzingaNews(\"mongodb://localhost:27017\",\"local\",\"news_headlines\", \"30s\", \"5d\")",
+    //       "is_on_websocket": true
+    //     },
+    //     {
+    //       "name": "benzinga_news_symbols",
+    //       "expression": "benzingaNewsSymbols(benzinga_news)",
+    //       "is_on_websocket": true
+    //     },
+    //     {
+    //       "name": "benzinga_news_symbol_state",
+    //       "expression": "benzingaNewsSymbolState(benzinga_news)",
+    //       "is_on_websocket": true
+    //     }
+    //   ]
+    // }
   ]
 }

src/DataStreamSuite/DataStreamPublisher/Configurations/TestDataPublisher.json

@@ -9,6 +9,21 @@
           "expression": "milliseconds(1000,10)",
           "is_on_websocket": true
         },
+        {
+          "name": "benzinga_news",
+          "expression": "benzingaNews(\"mongodb://localhost:27017\",\"local\",\"news_headlines\", \"30s\", \"5d\")",
+          "is_on_websocket": true
+        },
+        {
+          "name": "benzinga_news_symbols",
+          "expression": "benzingaNewsSymbols(benzinga_news)",
+          "is_on_websocket": true
+        },
+        {
+          "name": "benzinga_news_symbol_state",
+          "expression": "benzingaNewsSymbolState(benzinga_news)",
+          "is_on_websocket": true
+        },
         {
           "name": "test_literal",
           "expression": "literal(\"x\", \"420.0\")",

src/DataStreamSuite/DataStreamPublisher/DataStreamPublisherConfig.cs

@@ -10,7 +10,8 @@ using System.IO;
 using System.Text.Json;
 using System.Text.Json.Serialization;
 
-public class DataStreamPublisherConfig {
+public class DataStreamPublisherConfig
+{
    /// <summary>
    /// The login user for sparkd
    /// </summary>
@@ -38,6 +39,18 @@ public class DataStreamPublisherConfig {
    [JsonPropertyName("publishers")]
    public List<PublisherNode>? Publishers { get; set; }
 
+   [JsonPropertyName("mongodb_url")]
+   public string MONGO_DB_URL { get; set; }
+
+   [JsonPropertyName("mongodb_db_name")]
+   public string MONGO_DB_NAME { get; set; }
+
+   [JsonPropertyName("mongodb_db_collection")]
+   public string MONGO_DB_COLLECTION { get; set; }
+
+   [JsonPropertyName("benzinga_news_ttl")]
+   public string BENZINGA_NEWS_TTL { get; set; }
+
    [JsonPropertyName("pubsub")]
    public PubSubConfig? PubSub { get; set; }
 
@@ -49,11 +62,13 @@ public class DataStreamPublisherConfig {
    public string? ConfigFile { get; set; }
 
    public static DataStreamPublisherConfig Default() =>
-      new() {
+      new()
+      {
          PubSub = PubSubConfig.Default(),
       };
 
-   public void MergeWith(DataStreamPublisherConfig other) {
+   public void MergeWith(DataStreamPublisherConfig other)
+   {
       User ??= other.User;
       Password ??= other.Password;
       Sparkd ??= other.Sparkd;
@@ -69,8 +84,10 @@ public class DataStreamPublisherConfig {
    /// <exception cref="InvalidOperationException">
    /// JsonSerializer.Deserialize returns null
    /// </exception>
-   public void MergeWithConfigFile() {
-      if (ConfigFile is null) {
+   public void MergeWithConfigFile()
+   {
+      if (ConfigFile is null)
+      {
          return;
       }
       var cfgString = File.ReadAllText(ConfigFile);

src/DataStreamSuite/DataStreamPublisher/Program.cs

@@ -23,8 +23,10 @@ using System.Threading.Tasks;
 await Parser.Default.ParseArguments<DataStreamPublisherConfig>(args)
    .MapResult(RunAsync, errs => Task.FromResult(1));
 
-async Task<int> RunAsync(DataStreamPublisherConfig cfg) {
-   try {
+async Task<int> RunAsync(DataStreamPublisherConfig cfg)
+{
+   try
+   {
       cfg.MergeWithConfigFile();
       cfg.MergeWith(DataStreamPublisherConfig.Default());
       PubSubConfig.SetGlobalConfig(Log.notnull(cfg.PubSub));
@@ -33,7 +35,9 @@ async Task<int> RunAsync(DataStreamPublisherConfig cfg) {
       var service_ctx = ServiceContext.create(cfg.Sparkd);
       await service_ctx.login(cfg.User ?? "bts", cfg.Password ?? string.Empty);
       _publish_streams(cfg.Publishers ?? new(0));
-   } catch (Exception ex) {
+   }
+   catch (Exception ex)
+   {
       // Write the error to console first, in case the logging system is compromised.
       var error_msg = $"DataStreamPublisher threw exception:\n{ex}";
       Console.Error.WriteLine(error_msg);
@@ -44,29 +48,39 @@ async Task<int> RunAsync(DataStreamPublisherConfig cfg) {
    return 0;
 }
 
-static ViewStream? build_stream(string? stream_expression) {
-   if (stream_expression is null) {
+static ViewStream? build_stream(string? stream_expression)
+{
+   // Console.WriteLine("Hit key to continue");
+   // Console.ReadLine();
+   if (stream_expression is null)
+   {
       return null;
    }
-   try {
+   try
+   {
       bts.Log.Gen.info($"build_stream stream_expression: {stream_expression}");
       var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
       var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
       return rv;
 
-   } catch (Exception ex) {
+   }
+   catch (Exception ex)
+   {
       bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
       return null;
    }
 }
 
-void _publish_streams(IEnumerable<PublisherNode> publishers) {
+void _publish_streams(IEnumerable<PublisherNode> publishers)
+{
    PubTableBroker pb_broker = new PubTableBroker();
    RemoteQueryManager csm = new RemoteQueryManager();
    var csvPublishers = new List<CSVPublisher>();
 
-   foreach (var publisher in publishers) {
-      if (Directory.Exists(publisher.CsvDir)) {
+   foreach (var publisher in publishers)
+   {
+      if (Directory.Exists(publisher.CsvDir))
+      {
          csvPublishers.Add(new(publisher.CsvDir, pb_broker));
       }
    }
@@ -75,7 +89,8 @@ void _publish_streams(IEnumerable<PublisherNode> publishers) {
       .SelectMany(p => p.Streams.empty_if_null().Select(createViewPublisher))
       .ToList();
 
-   while (true) {
+   while (true)
+   {
       System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1));
       ViewStreamDatabase.instance.doWork();
       csm.doWork();
@@ -83,16 +98,19 @@ void _publish_streams(IEnumerable<PublisherNode> publishers) {
       ManagedSubscriber.instance.doWork();
    }
 
-   ViewPublisher? createViewPublisher(PublishedStream ps) {
+   ViewPublisher? createViewPublisher(PublishedStream ps)
+   {
       Log.Gen.info(
          $"Instantiating ViewPublisher for: {JsonSerializer.Serialize(ps)}");
-      if (string.IsNullOrEmpty(ps.Name)) {
+      if (string.IsNullOrEmpty(ps.Name))
+      {
          Log.Gen.warn(
             $"Failed to instantiate ViewPublisher. Null/empty 'name': {JsonSerializer.Serialize(ps)}");
          return null;
       }
       var vs = build_stream(ps.Expression);
-      if (vs is null) {
+      if (vs is null)
+      {
          Log.Gen.warn(
             $"Failed to instantiate ViewPublisher. Invalid 'expression': {JsonSerializer.Serialize(ps)}");
          return null;
@@ -103,31 +121,39 @@ void _publish_streams(IEnumerable<PublisherNode> publishers) {
    }
 }
 
-void on_change(object obj) {
+void on_change(object obj)
+{
    bts.Log.Gen.info($"DSPublisher on_change dp: {obj}");
 }
 
-public class CSVPublisher : IWorker {
+public class CSVPublisher : IWorker
+{
    private IDictionary<string, ViewPublisher> _pubs = new Dictionary<string, ViewPublisher>();
    private readonly string _csv_directory;
    private PubTableBroker _pub_table_broker;
 
-   public CSVPublisher(string csv_directory, PubTableBroker pub_table_broker) {
+   public CSVPublisher(string csv_directory, PubTableBroker pub_table_broker)
+   {
       _csv_directory = csv_directory;
       _pub_table_broker = pub_table_broker;
       MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
    }
-   public void doWork() {
+   public void doWork()
+   {
       var file_names = Directory.GetFiles(_csv_directory);
-      foreach (var fn in file_names) {
-         if (!_pubs.ContainsKey(fn)) {
+      foreach (var fn in file_names)
+      {
+         if (!_pubs.ContainsKey(fn))
+         {
             _handle_new_file(fn);
          }
       }
    }
 
-   private void _handle_new_file(string file_name) {
-      if (file_name.EndsWith(".csv")) {
+   private void _handle_new_file(string file_name)
+   {
+      if (file_name.EndsWith(".csv"))
+      {
          ViewStream vs = new CSVReaderStream(file_name);
          vs = new SchemaGuesserStream(vs);
          var toks = file_name.Split('\\');

src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json

@@ -1,61 +1,60 @@
-{
-  "user": "bts",
-  "pubsub": {
-    "milliseconds_between_sub_info_updates": 1000,
-    "milliseconds_between_time_slice_updates": 5000,
-    "milliseconds_between_no_buffer_space_retries": 100,
-    "retries_on_no_buffer_space": 1,
-    "do_it_all_seconds_between_logs": 0,
-    "heartbeat_interval": 15000,
-    "missing_heartbeats_before_gc": 50,
-    "use_managed_sockets": true,
-    "state_update_period": 5000,
-    "statistics_update_period": 5000,
-    "disposed_connection_check_period": 1000,
-    "message_processing_warning_threshold": 1000,
-    "server_port": 12345,
-    "publisher_routes": [
-      {
-        "name": "default",
-        "server_address": "bluto://192.168.1.80:12345",
-        "tag": ".*"
-      }
-    ],
-    "subscriber_routes": [
-      {
-        "name": "default",
-        "server_address": "bluto://192.168.1.80:12345",
-        "tag": ".*"
-      }
-    ],
-    "max_packet_size": 500000000,
-    "blue_comm_must_die": true,
-    "crushable_tags": [],
-    "tag_type_to_assembly_map": [
-      {
-        "handler_class": "BCG.PubSubServerTagHandler",
-        "assembly": "DataStreamCore.dll"
-      }
-    ],
-    "server_queue": "PubSubQueue",
-    "port_range": {
-      "first": 30400,
-      "last": 31700
-    },
-    "bind_to_random_port": false,
-    "debug_tags": "^NOTAGS$",
-    "latency_report_threshold_ms": 1500,
-    "use_substations": false,
-    "use_dynamic_clients": false,
-    "is_substation": false,
-    "internally_publish_sub_info": true,
-    "use_managed_clients_in_native_code": false,
-    "compressed_tags": "^NOTAGS$",
-    "tags_to_report_compression_ratio": "^NOTAGS$",
-    "disable_msmq": false,
-    "use_prepublished_server_state": false,
-    "ip_translation": null,
-    "gc_activation_threshold": 0,
-    "wait_period_after_gc": 1000
-  }
+"user": "bts",
+"pubsub": {
+  "milliseconds_between_sub_info_updates": 1000,
+  "milliseconds_between_time_slice_updates": 5000,
+  "milliseconds_between_no_buffer_space_retries": 100,
+  "retries_on_no_buffer_space": 1,
+  "do_it_all_seconds_between_logs": 0,
+  "heartbeat_interval": 15000,
+  "missing_heartbeats_before_gc": 50,
+  "use_managed_sockets": true,
+  "state_update_period": 5000,
+  "statistics_update_period": 5000,
+  "disposed_connection_check_period": 1000,
+  "message_processing_warning_threshold": 1000,
+  "server_port": 12345,
+  "publisher_routes": [
+    {
+      "name": "default",
+      "server_address": "bts-cloud://192.168.1.80:12345",
+      "tag": ".*"
+    }
+  ],
+  "subscriber_routes": [
+    {
+      "name": "default",
+      "server_address": "bts-cloud://192.168.1.80:12345",
+      "tag": ".*"
+    }
+  ],
+  "max_packet_size": 500000000,
+  "blue_comm_must_die": true,
+  "crushable_tags": [],
+  "tag_type_to_assembly_map": [
+    {
+      "handler_class": "BCG.PubSubServerTagHandler",
+      "assembly": "DataStreamCore.dll"
+    }
+  ],
+  "server_queue": "PubSubQueue",
+  "port_range": {
+    "first": 30400,
+    "last": 31700
+  },
+  "bind_to_random_port": false,
+  "debug_tags": "^NOTAGS$",
+  "latency_report_threshold_ms": 1500,
+  "use_substations": false,
+  "use_dynamic_clients": false,
+  "is_substation": false,
+  "internally_publish_sub_info": true,
+  "use_managed_clients_in_native_code": false,
+  "compressed_tags": "^NOTAGS$",
+  "tags_to_report_compression_ratio": "^NOTAGS$",
+  "disable_msmq": false,
+  "use_prepublished_server_state": false,
+  "ip_translation": null,
+  "gc_activation_threshold": 0,
+  "wait_period_after_gc": 1000
+}
 }

src/DataStreamSuite/PubSub/Config/PubSubConfig.cs

@@ -117,10 +117,10 @@ public class PubSubConfig {
          WaitPeriodAfterGc = 1000,
          SimpleEchoServer = false,
          PublisherRoutes = new() {
-            new("bts-cloud://127.0.0.1:12345", ".*")
+            new("bluto://127.0.0.1:12345", ".*")
          },
          SubscriberRoutes = new() {
-            new("bts-cloud://127.0.0.1:12345", ".*")
+            new("bluto://127.0.0.1:12345", ".*")
          }
       };
       return rv;

src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_APM.cs

@@ -36,14 +36,17 @@ using System.Linq;
 using Bts.PubSub;
 using Spark.Core;
 
-namespace bts {
-   public class AsyncTCPConnection_APM : ITCPConnection, IWorker, IDisposable {
+namespace bts
+{
+   public class AsyncTCPConnection_APM : ITCPConnection, IWorker, IDisposable
+   {
 
       // this class is used as the argument of the completion function for BeginSend/Receive   
       // serves as the base class for both Send and ReeceiveMessage classes.
       // since both TCP sends and receives happen in multiple transfers, this
       // provides a wrapper around copying the partial transfers into the complete buffer
-      public class AsyncMessageBase {
+      public class AsyncMessageBase
+      {
          readonly string msg_id = Guid.NewGuid().ToString();
          // every BeginSend or BeginReceive has a unique key associated with it
          public int key;
@@ -55,37 +58,45 @@ namespace bts {
          public int transfer_ptr;
          protected int _n_transfers = 0;
          public bool completed_sync = false;
-         public AsyncMessageBase(byte[] the_buffer, int the_offset) {
+         public AsyncMessageBase(byte[] the_buffer, int the_offset)
+         {
             buffer = the_buffer;
             transfer_ptr = the_offset;
          }
-         public int remaining_size {
-            get {
+         public int remaining_size
+         {
+            get
+            {
                return buffer.Length - transfer_ptr;
             }
          }
 
          public int original_size => buffer.Length;
 
-         public void advance_transfer_pointer(int advance_by) {
+         public void advance_transfer_pointer(int advance_by)
+         {
             transfer_ptr += advance_by;
             _n_transfers++;
-            if (transfer_ptr > original_size) {
+            if (transfer_ptr > original_size)
+            {
                bts.Log.Gen.error($"transfer went beyond initial size {ToString()}");
             }
             // TODO: log if it goes past the buffer
          }
 
-         public bool fully_transfered() {
+         public bool fully_transfered()
+         {
             return transfer_ptr >= original_size;
          }
 
-         public override string ToString() {
+         public override string ToString()
+         {
             return $"{GetType().ToString()} msg_id: {msg_id} key: {key} original_length: {original_size} transfer_ptr: {transfer_ptr} n_transfers: {_n_transfers} " +
                    $"fully_transfered: {fully_transfered()} completed_sync: {completed_sync}";
          }
       }
-      public class AsyncSendMessage : AsyncMessageBase {
+      public class AsyncSendMessage : AsyncMessageBase
+      {
          public string message_id;
          public SendConfirmationDelegate on_send_del;
          //private int _n_transfers = 0;
@@ -94,35 +105,43 @@ namespace bts {
             : this(the_buffer, the_offset, String.Empty, null) { }
 
          public AsyncSendMessage(byte[] the_buffer, int the_transfer_ptr, string the_message_id, SendConfirmationDelegate the_on_send_del)
-         : base(the_buffer, the_transfer_ptr) {
+         : base(the_buffer, the_transfer_ptr)
+         {
             message_id = the_message_id;
             on_send_del = the_on_send_del;
          }
-         public override string ToString() {
+         public override string ToString()
+         {
             return base.ToString();
          }
 
       }
 
-      public class AsyncReceiveMessage : AsyncMessageBase {
-         public enum ReadState {
+      public class AsyncReceiveMessage : AsyncMessageBase
+      {
+         public enum ReadState
+         {
             reading_size,
             reading_data
          };
          readonly ReadState read_state;
 
          public AsyncReceiveMessage(int size, ReadState the_read_state) :
-         this(new byte[size], 0, the_read_state) { }
+         this(new byte[size], 0, the_read_state)
+         { }
          public AsyncReceiveMessage(byte[] the_buffer, int the_transfer_ptr, ReadState the_read_state)
-         : base(the_buffer, the_transfer_ptr) {
+         : base(the_buffer, the_transfer_ptr)
+         {
             read_state = the_read_state;
          }
 
-         public int read_int32(int start_idx = 0) {
+         public int read_int32(int start_idx = 0)
+         {
             return BitConverter.ToInt32(buffer, start_idx);
          }
 
-         public byte[] read_bytes(int start_idx) {
+         public byte[] read_bytes(int start_idx)
+         {
             byte[] rv = new byte[buffer.Length - start_idx];
             Array.Copy(buffer, start_idx, rv, 0, rv.Length);
             return rv;
@@ -131,16 +150,19 @@ namespace bts {
          public bool is_reading_size() => read_state == ReadState.reading_size;
       }
 
-      public class ErrorMessage {
+      public class ErrorMessage
+      {
          public readonly SocketError? socket_error = null;
          public readonly Exception exception = null;
          public readonly bool is_exception = false;
          public readonly string context = String.Empty;
-         public ErrorMessage(SocketError socket_error_, string context_) {
+         public ErrorMessage(SocketError socket_error_, string context_)
+         {
             socket_error = socket_error_;
             context = context_;
          }
-         public ErrorMessage(Exception exception_, string context_) {
+         public ErrorMessage(Exception exception_, string context_)
+         {
             is_exception = true;
             exception = exception_;
             context = context_;
@@ -148,7 +170,8 @@ namespace bts {
 
          public bool oom => exception != null && exception is OutOfMemoryException;
 
-         public override string ToString() {
+         public override string ToString()
+         {
             if (is_exception)
                return $"ErrorMessage exception: {exception} context: {context}";
             return $"socket_error: {socket_error} context: {context}";
@@ -174,13 +197,15 @@ namespace bts {
       private bool _hb_received_yet = false, _data_received_yet = false, _hb_sent_once = false;
       private DateTime _last_hb_received_time = new DateTime(1900, 1, 1);
 
-      public AsyncTCPConnection_APM(Socket sock, TimeSpan hb_period, ITCPConnection.ConnectionSide connection_side) {
+      public AsyncTCPConnection_APM(Socket sock, TimeSpan hb_period, ITCPConnection.ConnectionSide connection_side)
+      {
          _sock = sock;
          _hb_period = hb_period;
          _connection_side = connection_side;
          _sock.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.DontRoute, true);
          #region testing options
-         switch (connection_side) {
+         switch (connection_side)
+         {
             case ITCPConnection.ConnectionSide.Server:
                TCPSocketUtils.set_server_socket_options(ref _sock);
                break;
@@ -195,10 +220,13 @@ namespace bts {
          _detect_connection_status_change();
       }
 
-      private void _detect_connection_status_change() {
-         Action testing_func = () => {
+      private void _detect_connection_status_change()
+      {
+         Action testing_func = () =>
+         {
             bool connected = _sock.Connected;
-            while (true) {
+            while (true)
+            {
                if (connected != _sock.Connected)
                   Log.Gen.error($"Connection status changed this: {ToString()} {connected} -> {_sock.Connected}");
                connected = _sock.Connected;
@@ -208,26 +236,30 @@ namespace bts {
          ThreadPool.QueueUserWorkItem(_ => testing_func());
       }
 
-      public override string ToString() {
+      public override string ToString()
+      {
          return $"AsyncTCPConnection_APM side: {_connection_side} peer: {_peer_conn_info}, self: {_self_conn_info}";
       }
 
       #region ITCPConnection Members
 
-      public void sendWithConfirmation(byte[] data, ref string conf_id, SendConfirmationDelegate del) {
+      public void sendWithConfirmation(byte[] data, ref string conf_id, SendConfirmationDelegate del)
+      {
          conf_id = Guid.NewGuid().ToString();
          byte[] msg = _build_data_message(data);
          _begin_or_resume_sending(new AsyncSendMessage(msg, 0, conf_id, del));
       }
 
-      public void send(byte[] data) {
-         Log.Gen.info($"AsyncTCPConnection_APM.send data: {AsyncTCPConnection_AsyncWrapper.to_hex(data, 0, data.Length)}");
+      public void send(byte[] data)
+      {
+         //Log.Gen.info($"AsyncTCPConnection_APM.send data: {AsyncTCPConnection_AsyncWrapper.to_hex(data, 0, data.Length)}");
          byte[] msg = _build_data_message(data);
-         Log.Gen.info($"AsyncTCPConnection_APM.send msg: {AsyncTCPConnection_AsyncWrapper.to_hex(msg, 0, msg.Length)}");
+         //Log.Gen.info($"AsyncTCPConnection_APM.send msg: {AsyncTCPConnection_AsyncWrapper.to_hex(msg, 0, msg.Length)}");
          _begin_or_resume_sending(new AsyncSendMessage(msg, 0));
       }
 
-      public void register(ITCPMessageHandler messageHandler) {
+      public void register(ITCPMessageHandler messageHandler)
+      {
          bts.Log.assert_d(_message_handler == null, "Message Handler is already assigned");
          //bts.Log.Gen.info($"AsyncTCPConnection_APM for peer = {_peer_conn_info} being registered");
          _message_handler = messageHandler;
@@ -236,15 +268,18 @@ namespace bts {
          _read_next_msg();
       }
 
-      public RemoteConnectionInfo get_peer_address() {
+      public RemoteConnectionInfo get_peer_address()
+      {
          return TCPSocketUtils.get_peer_address(_sock);
       }
 
-      public RemoteConnectionInfo get_self_address() {
+      public RemoteConnectionInfo get_self_address()
+      {
          return TCPSocketUtils.get_peer_address(_sock);
       }
 
-      public bool isDisposed {
+      public bool isDisposed
+      {
          get { return !_is_alive; }
       }
 
@@ -253,12 +288,14 @@ namespace bts {
       #region IWorker Members
 
       bool _last_connected = false;
-      public void doWork() {
+      public void doWork()
+      {
          if (!_is_alive)
             return;
          return;
          // disable hbs for now
-         if (!_hb_sent_once) {
+         if (!_hb_sent_once)
+         {
             bts.Log.Gen.info("Sending heartbeat to peer {_peer_conn_info} for the first time");
             _hb_sent_once = true;
          }
@@ -266,12 +303,14 @@ namespace bts {
          _begin_or_resume_sending(new AsyncSendMessage(msg, 0));
       }
 
-      private static int _tid() {
+      private static int _tid()
+      {
          var tid = System.Threading.Thread.CurrentThread.ManagedThreadId;
          return tid;
       }
 
-      private static string _cb_txt(IAsyncResult res) {
+      private static string _cb_txt(IAsyncResult res)
+      {
          return $"completed?: {res.IsCompleted} sync?: {res.CompletedSynchronously}";
       }
 
@@ -279,10 +318,13 @@ namespace bts {
 
       #region sending data
 
-      private void _begin_or_resume_sending(AsyncSendMessage message) {
+      private void _begin_or_resume_sending(AsyncSendMessage message)
+      {
          //bts.Log.Gen.info($"AsyncTCPConnection_APM._begin_or_resume_sending this: {ToString()} tid: {_tid()} message: {message} _is_alive: {_is_alive}");
-         if (_is_alive) {
-            try {
+         if (_is_alive)
+         {
+            try
+            {
                SocketError begin_send_err;
                // since we get message on on_send we dont need to keep the state around
                message.key = _send_key++;
@@ -290,35 +332,45 @@ namespace bts {
                                  out begin_send_err, new AsyncCallback(BeginSend_callback_function), message);
                if (begin_send_err != SocketError.Success)
                   _handle_error(new ErrorMessage(begin_send_err, "_begin_or_resume_sending"));
-            } catch (Exception ex) {
+            }
+            catch (Exception ex)
+            {
                _handle_error(new ErrorMessage(ex, "_begin_or_resume_sending"));
             }
          }
       }
 
-      private void BeginSend_callback_function(IAsyncResult res) {
-         if (res.CompletedSynchronously) {
+      private void BeginSend_callback_function(IAsyncResult res)
+      {
+         if (res.CompletedSynchronously)
+         {
             ThreadPool.QueueUserWorkItem(_ => _on_send_async(res));
-         } else
+         }
+         else
             _on_send_async(res);
       }
 
-      private void _on_send_async(IAsyncResult res) {
+      private void _on_send_async(IAsyncResult res)
+      {
          AsyncSendMessage message = res.AsyncState as AsyncSendMessage;
-         if (_send_key_map.Contains(message.key)) {
+         if (_send_key_map.Contains(message.key))
+         {
             bts.Log.Gen.warn($"AsyncTCPConnection_APM._on_send key: {message.key} already handled");
             return;
          }
          _send_key_map.Add(message.key);
          // bts.Log.Gen.info($"AsyncTCPConnection_APM._on_send: pre endSend this: {ToString()} tid: {_tid()} " +
          //                         $"res: {_cb_txt(res)} message: {res.AsyncState} _is_alive: {_is_alive}");
-         if (_is_alive) {
-            try {
+         if (_is_alive)
+         {
+            try
+            {
                SocketError send_err;
                int bytes_sent = _sock.EndSend(res, out send_err);
                //message.offset += bytesSent;
                message.advance_transfer_pointer(bytes_sent);
-               if (bytes_sent == 0) {
+               if (bytes_sent == 0)
+               {
                   bts.Log.Gen.error($"AsyncTCPConnection_APM._on_send this: {this} sent 0 bytes, throwing exception");
                   _handle_error(new ErrorMessage(new ApplicationException("Sent 0 bytes, peer socket is likely shutdown"),
                         "_on_send"));
@@ -326,22 +378,31 @@ namespace bts {
                }
                bts.Log.Gen.info($"AsyncTCPConnection_APM._on_send: post endSend this: {ToString()} tid: {_tid()} res: {_cb_txt(res)} " +
                $"message: {res.AsyncState} bytes_sent: {bytes_sent} _is_alive: {_is_alive} send_err: {send_err}");
-               if (send_err == SocketError.Success) {
+               if (send_err == SocketError.Success)
+               {
                   // successful send of some bytes
                   //if (message.offset == message.buffer.Length) {
-                  if (message.fully_transfered()) {
+                  if (message.fully_transfered())
+                  {
                      // fully sent, if confirmation needs to be sent out                     
-                     if (!String.IsNullOrEmpty(message.message_id)) {
+                     if (!String.IsNullOrEmpty(message.message_id))
+                     {
                         message.on_send_del(message.message_id, true);
                      }
-                  } else {
+                  }
+                  else
+                  {
                      // there is more to send
                      _begin_or_resume_sending(message);
                   }
-               } else {
+               }
+               else
+               {
                   _handle_error(new ErrorMessage(send_err, "_on_send"));
                }
-            } catch (Exception ex) {
+            }
+            catch (Exception ex)
+            {
                _handle_error(new ErrorMessage(ex, "_on_send"));
             }
          }
@@ -352,8 +413,10 @@ namespace bts {
       /// future references
       /// </summary>
       /// <returns></returns>
-      private byte[] _build_hb_message() {
-         if (_cached_hb_message == null) {
+      private byte[] _build_hb_message()
+      {
+         if (_cached_hb_message == null)
+         {
             BinaryWriter bw = new BinaryWriter(new MemoryStream());
             // the total msg =[msg_sz, msg_type] 
             int msg_sz = 4;
@@ -365,7 +428,8 @@ namespace bts {
       }
 
       // writes size prefixed msg
-      private byte[] _build_data_message(byte[] data) {
+      private byte[] _build_data_message(byte[] data)
+      {
          // TODO, avoid unecessary memory creation (write directly into byte[] using BitConverter)
          MemoryStream str = new MemoryStream();
          BinaryWriter bw = new BinaryWriter(str);
@@ -384,28 +448,34 @@ namespace bts {
 
       #region receiving data
 
-      private void _read_next_msg() {
+      private void _read_next_msg()
+      {
          _read_size();
       }
 
-      private void _read_size() {
+      private void _read_size()
+      {
          _begin_or_resume_reading(new AsyncReceiveMessage(4, AsyncReceiveMessage.ReadState.reading_size));
       }
 
-      private void _read_data(int data_size) {
+      private void _read_data(int data_size)
+      {
          // TODO, this could be a problem is the size is too big - maybe read it in chunks (later)
          _begin_or_resume_reading(new AsyncReceiveMessage(data_size, AsyncReceiveMessage.ReadState.reading_data));
       }
 
-      private static string IAsyncResult_txt(IAsyncResult res) {
+      private static string IAsyncResult_txt(IAsyncResult res)
+      {
          return $"IsCompleted: {res.IsCompleted} CompletedSynchronously: {res.CompletedSynchronously}";
       }
 
-      private void _begin_or_resume_reading(AsyncReceiveMessage rcv_msg) {
+      private void _begin_or_resume_reading(AsyncReceiveMessage rcv_msg)
+      {
          //bts.Log.Gen.info($"AsyncTCPConnection_APM._begin_or_resume_reading: this: {ToString()} tid: {_tid()} message: {rcv_msg} _is_alive: {_is_alive}");
          if (!_is_alive)
             return;
-         try {
+         try
+         {
             SocketError read_sock_err;
             rcv_msg.key = _receive_key++;
             var res = _sock.BeginReceive(rcv_msg.buffer, rcv_msg.transfer_ptr,
@@ -413,50 +483,61 @@ namespace bts {
                                  SocketFlags.None, out read_sock_err,
                                  new AsyncCallback(BeginReceive_callback_function),
                                  rcv_msg);
-            if (read_sock_err != SocketError.Success) {
+            if (read_sock_err != SocketError.Success)
+            {
                bts.Log.Gen.error($"_begin_or_resume_reading this: {this} error: {read_sock_err} rcv_msg: {rcv_msg}");
                _handle_error(new ErrorMessage(read_sock_err, "_beging_or_resume_reading"));
             }
-         } catch (Exception ex) {
+         }
+         catch (Exception ex)
+         {
             _handle_error(new ErrorMessage(ex, "_beging_or_resume_reading"));
          }
       }
 
-      private void _log_incoming_msg(IAsyncResult res) {
-         AsyncReceiveMessage rcv_message = res.AsyncState as AsyncReceiveMessage;
-         var buff_txt = AsyncTCPConnection_AsyncWrapper.to_hex(rcv_message.buffer, 0, rcv_message.buffer.Length);
-         Log.Gen.info($"BeginReceive_callback_function _log_incoming_msg buff: {buff_txt}");
+      private void _log_incoming_msg(IAsyncResult res)
+      {
+         // AsyncReceiveMessage rcv_message = res.AsyncState as AsyncReceiveMessage;
+         // var buff_txt = AsyncTCPConnection_AsyncWrapper.to_hex(rcv_message.buffer, 0, rcv_message.buffer.Length);
+         // Log.Gen.info($"BeginReceive_callback_function _log_incoming_msg buff: {buff_txt}");
 
       }
 
       // this is the function passed to BeginReceive, we 
-      private void BeginReceive_callback_function(IAsyncResult res) {
+      private void BeginReceive_callback_function(IAsyncResult res)
+      {
          _log_incoming_msg(res);
-         if (res.CompletedSynchronously) {
+         if (res.CompletedSynchronously)
+         {
             ThreadPool.QueueUserWorkItem(_ => _on_read_async(res));
-         } else
+         }
+         else
             _on_read_async(res);
       }
 
       // this will always be called on the non main-thread. the reason this is on its own thread is that this function ends up
       // calling _begin_or_resume_reading, which in tuns calls _on_read_async. So to not cause a stack overflow, this is broken
       // always called on a different thread than _begin_or_resume_reading
-      private void _on_read_async(IAsyncResult res) {
+      private void _on_read_async(IAsyncResult res)
+      {
          AsyncReceiveMessage rcv_message = res.AsyncState as AsyncReceiveMessage;
-         if (_read_key_map.Contains(rcv_message.key)) {
+         if (_read_key_map.Contains(rcv_message.key))
+         {
             bts.Log.Gen.warn($"AsyncTCPConnection_APM._on_read key: {rcv_message.key} already handled");
             return;
          }
          _read_key_map.Add(rcv_message.key);
          if (!_is_alive)
             return;
-         try {
+         try
+         {
             SocketError read_err;
             Log.Gen.info($"AsyncTCPConnection_APM._on_read pre-EndReceive rcv_message: {rcv_message} res: {IAsyncResult_txt(res)}");
             int bytes_read = _sock.EndReceive(res, out read_err);
             rcv_message.advance_transfer_pointer(bytes_read);
             Log.Gen.info($"AsyncTCPConnection_APM._on_read post-EndReceive rcv_message: {rcv_message} res: {IAsyncResult_txt(res)} bytes_read: {bytes_read}");
-            if (bytes_read == 0) {
+            if (bytes_read == 0)
+            {
                bts.Log.Gen.error($"AsyncTCPConnection_APM._on_read this: {this} read 0 bytes, throwing exception");
                //bts.Log.Gen.error($"AsyncTCPConnection_APM._on_read this: {this} read 0 bytes, sleeping and ignoring");
                //System.Threading.Thread.Sleep(50);
@@ -464,23 +545,32 @@ namespace bts {
                return;
             }
             //return;
-            if (read_err == SocketError.Success) {
-               if (rcv_message.fully_transfered()) {
-                  if (rcv_message.is_reading_size()) {
+            if (read_err == SocketError.Success)
+            {
+               if (rcv_message.fully_transfered())
+               {
+                  if (rcv_message.is_reading_size())
+                  {
                      var msg_size = _on_read_size_completely(rcv_message);
-                     if (msg_size != null && msg_size.HasValue) {
+                     if (msg_size != null && msg_size.HasValue)
+                     {
                         _read_data(msg_size.Value);
                      }
                      return;
-                  } else {
+                  }
+                  else
+                  {
                      // data is read
                      int msg_type = rcv_message.read_int32();
                      bts.Log.assert(msg_type == HB_MSG_SIGNAL || msg_type == DATA_MSG_SIGNAL);
-                     if (msg_type == DATA_MSG_SIGNAL) {
+                     if (msg_type == DATA_MSG_SIGNAL)
+                     {
                         _on_read_data_completely(rcv_message);
                         _read_next_msg();
                         return;
-                     } else if (msg_type == HB_MSG_SIGNAL) {
+                     }
+                     else if (msg_type == HB_MSG_SIGNAL)
+                     {
                         _on_read_hb_completely(rcv_message);
                         _read_next_msg();
                         return;
@@ -488,45 +578,57 @@ namespace bts {
                      // go back to reading size for a new message
                      //_read_size();
                   }
-               } else {
+               }
+               else
+               {
                   // msg has not been fully rd, we have already advanced the transfer ptr
                   _begin_or_resume_reading(rcv_msg: rcv_message);
                }
-            } else {
+            }
+            else
+            {
                _handle_error(new ErrorMessage(read_err, "_on_read_async"));
             }
-         } catch (Exception ex) {
+         }
+         catch (Exception ex)
+         {
             _handle_error(new ErrorMessage(ex, "_on_read_async"));
          }
       }
 
       #endregion
 
-      private int? _on_read_size_completely(AsyncReceiveMessage msg) {
+      private int? _on_read_size_completely(AsyncReceiveMessage msg)
+      {
          bts.Log.assert(msg.is_reading_size());
          bts.Log.assert(msg.fully_transfered());
          bts.Log.assert(msg.original_size == 4);
          var msg_sz = msg.read_int32();
-         var buff_txt = AsyncTCPConnection_AsyncWrapper.to_hex(msg.buffer, 0, msg.buffer.Length);
-         bts.Log.Gen.info($"AsyncTCPConnection_APM._on_read_size_completely buff: {buff_txt}");
+         //var buff_txt = AsyncTCPConnection_AsyncWrapper.to_hex(msg.buffer, 0, msg.buffer.Length);
+         //bts.Log.Gen.info($"AsyncTCPConnection_APM._on_read_size_completely buff: {buff_txt}");
          bts.Log.Gen.info($"AsyncTCPConnection_APM._on_read_size_completely msg: {msg} msg_sz: {msg_sz}");
-         if (msg_sz < 0 || msg_sz > _max_msg_size) {
+         if (msg_sz < 0 || msg_sz > _max_msg_size)
+         {
             string err_msg = $"Encountered malformed message, prefixed_size: {msg_sz}";
             _handle_error(new ErrorMessage(new ApplicationException(err_msg), "_on_read_size_completely"));
             return null;
-         } else {
+         }
+         else
+         {
             // sz is determined, now we read that many bytes
             //_read_data(msg_sz);
             return msg_sz;
          }
       }
 
-      private void _on_read_hb_completely(AsyncReceiveMessage msg) {
+      private void _on_read_hb_completely(AsyncReceiveMessage msg)
+      {
          bts.Log.assert(!msg.is_reading_size());
          bts.Log.assert(msg.fully_transfered());
          bts.Log.assert(msg.original_size == 4);
          bts.Log.assert(msg.read_int32(0) == HB_MSG_SIGNAL);
-         if (!_hb_received_yet) {
+         if (!_hb_received_yet)
+         {
             _hb_received_yet = true;
             //bts.Log.Gen.info($"AsyncTCPConnection_APM HB received for the first time from peer {_peer_conn_info}");
          }
@@ -534,7 +636,8 @@ namespace bts {
          _message_handler.onData(new RemoteMessage(null, RemoteMessageType.Heartbeat, _peer_conn_info));
       }
 
-      private void _on_read_data_completely(AsyncReceiveMessage msg) {
+      private void _on_read_data_completely(AsyncReceiveMessage msg)
+      {
          bts.Log.assert(!msg.is_reading_size());
          bts.Log.assert(msg.fully_transfered());
          var type = msg.read_int32(0);
@@ -543,7 +646,8 @@ namespace bts {
          byte[] data = msg.read_bytes(4);
          ///callbacks can be made through thread pool to avoid this
          ///thread from being blocked, by this callback
-         if (!_data_received_yet) {
+         if (!_data_received_yet)
+         {
             _data_received_yet = true;
             //bts.Log.Gen.info($"AsyncTCPConnection_APM data received for the first time from peer {_peer_conn_info}");
          }
@@ -553,18 +657,24 @@ namespace bts {
 
       #region IDisposable Members
 
-      public void Dispose() {
+      public void Dispose()
+      {
          // socket is closed and disposed here
-         lock (this) {
+         lock (this)
+         {
             bts.Log.Gen.info($"AsyncTCPConnection_APM.Dispose this: {this} Dispose called _dipose_impetus: {_dipose_impetus}");
-            if (_is_alive) {
+            if (_is_alive)
+            {
                _master_worker.removeWorker(this);
                _is_alive = false;
                bts.Log.Gen.info($"AsyncTCPConnection_APM.Dispose this: {this} being disposed, last HB was received at {_last_hb_received_time}");
-               try {
+               try
+               {
                   bts.Log.Gen.info($"AsyncTCPConnection_APM.Dispose this: {this} shutting down socket now");
                   _sock.Shutdown(SocketShutdown.Both);
-               } catch (Exception ex) {
+               }
+               catch (Exception ex)
+               {
                   String.Format("AsyncTCPConnection_APM.Dispose exception: {ex} thrown when closing socket");
                   if (ex is OutOfMemoryException)
                      throw ex;
@@ -577,8 +687,10 @@ namespace bts {
       #endregion
 
       private string _dipose_impetus = String.Empty;
-      private void _handle_error(ErrorMessage err_msg) {
-         lock (this) {
+      private void _handle_error(ErrorMessage err_msg)
+      {
+         lock (this)
+         {
             Log.Gen.error($"AsyncTCPConnection_APM._handle_error this: {this} error_msg: {err_msg}");
             if (err_msg.oom)
                throw err_msg.exception;

src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnection_AsyncWrapper.cs

@@ -1,4 +1,5 @@
-namespace bts {
+namespace bts
+{
    using BCG;
    using BCG.Utilities.General;
    using System;
@@ -11,15 +12,19 @@ namespace bts {
 
    // this ITCPConnection uses the TCPClient class instead of a raw socket (in the hope, that it would configure the socket correctly)
 
-   public class AsyncTCPConnection_AsyncWrapper : ITCPConnection, IWorker, IDisposable {
-      public class ConnectionID {
+   public class AsyncTCPConnection_AsyncWrapper : ITCPConnection, IWorker, IDisposable
+   {
+      public class ConnectionID
+      {
          public readonly RemoteConnectionInfo self;
          public readonly RemoteConnectionInfo peer;
          public readonly Type conn_type;
-         public override string ToString() {
+         public override string ToString()
+         {
             return $"{conn_type} self: {self} peer: {peer}";
          }
-         public ConnectionID(RemoteConnectionInfo self_, RemoteConnectionInfo peer_, Type conn_type_) {
+         public ConnectionID(RemoteConnectionInfo self_, RemoteConnectionInfo peer_, Type conn_type_)
+         {
             self = self_;
             peer = peer_;
             conn_type = conn_type_;
@@ -33,30 +38,34 @@ namespace bts {
       private bool _is_alive = true;
       ConnectionID _connection_id;
 
-      public static string to_hex(byte[] byte_arr, int start_pos, int end_pos) {
-         var len = end_pos - start_pos;
-         StringBuilder hex_builder = new StringBuilder(len * 2);
-         int i = 0;
+      // public static string to_hex(byte[] byte_arr, int start_pos, int end_pos) {
+      //    var len = end_pos - start_pos;
+      //    StringBuilder hex_builder = new StringBuilder(len * 2);
+      //    int i = 0;
 
-         foreach (byte b in byte_arr) {
-            if (i >= start_pos && i < end_pos)
-               hex_builder.Append($"{b:X2} ");
-            i++;
-         }
-         return hex_builder.ToString().Trim();
-      }
+      //    foreach (byte b in byte_arr) {
+      //       if (i >= start_pos && i < end_pos)
+      //          hex_builder.Append($"{b:X2} ");
+      //       i++;
+      //    }
+      //    return hex_builder.ToString().Trim();
+      // }
 
-      public AsyncTCPConnection_AsyncWrapper(TcpClient tcp_client) {
+      public AsyncTCPConnection_AsyncWrapper(TcpClient tcp_client)
+      {
          _tcp_client = tcp_client;
          _nw_stream = _tcp_client.GetStream();
          _connection_id = new ConnectionID(TCPSocketUtils.get_self_address(_tcp_client.Client), TCPSocketUtils.get_peer_address(_tcp_client.Client), GetType());
          Log.Gen.info("AsyncTCPConnection_AsyncWrapper.cons connection: {_connection_id}");
       }
 
-      private void _detect_connection_status_change() {
-         Action testing_func = () => {
+      private void _detect_connection_status_change()
+      {
+         Action testing_func = () =>
+         {
             bool connected = _tcp_client.Connected;
-            while (true) {
+            while (true)
+            {
                if (connected != _tcp_client.Connected)
                   Log.Gen.error($"Connection status changed this: {ToString()} {connected} -> {_tcp_client.Connected}");
                connected = _tcp_client.Connected;
@@ -69,14 +78,17 @@ namespace bts {
       }
 
       #region Sending 
-      public void _send_fixed_size_msg(byte[] buffer) {
-         //Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._send_fixed_size_msg buffer: {to_hex(buffer, 0, buffer.Length)}");
+      public void _send_fixed_size_msg(byte[] buffer)
+      {
+         // string buff_txt = to_hex(buffer, 0, buffer.Length);
+         // Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._send_fixed_size_msg buffer: {buff_txt}");
 
          int total_bytes_sent = 0;
          int n_bytes = buffer.Length;
 
          int n_transfers = 0;
-         while (_is_alive && total_bytes_sent < n_bytes) {
+         while (_is_alive && total_bytes_sent < n_bytes)
+         {
             int remaining_bytes = n_bytes - total_bytes_sent;
             _nw_stream.Write(buffer, total_bytes_sent, remaining_bytes);
             total_bytes_sent += remaining_bytes;
@@ -85,8 +97,10 @@ namespace bts {
          Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._send_fixed_size_msg sent {buffer.Length} bytes in {n_transfers} transfers");
       }
 
-      private void _send_one_msg(byte[] msg) {
-         try {
+      private void _send_one_msg(byte[] msg)
+      {
+         try
+         {
             // the total msg = [msg_sz, msg_type, data] 
             int sz = msg.Length + 4;
             MemoryStream str = new MemoryStream();
@@ -97,15 +111,20 @@ namespace bts {
             str.Position = 0;
             byte[] buff = str.ToArray();
             _send_fixed_size_msg(buff);
-         } catch (Exception ex) {
+         }
+         catch (Exception ex)
+         {
             _handle_error(new ErrorMessage(ex, $"_send_one_msg call_stacK: {ex.StackTrace}"));
          }
       }
 
-      private void _send_forever() {
-         while (_is_alive) {
+      private void _send_forever()
+      {
+         while (_is_alive)
+         {
             byte[] data = _send_queue.dequeue();
-            if (data != null) {
+            if (data != null)
+            {
                _send_one_msg(data);
             }
          }
@@ -118,39 +137,47 @@ namespace bts {
       }
       #endregion
 
-      public void register(ITCPMessageHandler message_handler) {
+      public void register(ITCPMessageHandler message_handler)
+      {
          _msg_handler = message_handler;
          // starting thread pool threads to keep reading and sending
          ThreadPool.QueueUserWorkItem(_ => _read_msgs_forever());
          ThreadPool.QueueUserWorkItem(_ => _send_forever());
       }
 
-      public RemoteConnectionInfo get_peer_address() {
+      public RemoteConnectionInfo get_peer_address()
+      {
          return _connection_id.peer;
       }
 
-      public RemoteConnectionInfo get_self_address() {
+      public RemoteConnectionInfo get_self_address()
+      {
          return _connection_id.self;
       }
 
       public bool isDisposed { get { return !_is_alive; } }
 
-      public void doWork() {
+      public void doWork()
+      {
          // send hb
       }
 
       #region Reading
       // This makes the blocking IO call, the callstack is _read_msgs_forever -> _read_one_message -> _read_fixed_size_msg
-      private byte[] _read_fixed_size_msg(int n_bytes) {
-         if (n_bytes <= 0) {
+      private byte[] _read_fixed_size_msg(int n_bytes)
+      {
+         if (n_bytes <= 0)
+         {
             throw new ArgumentException("Number of bytes to read must be greater than zero.", nameof(n_bytes));
          }
          byte[] buffer = new byte[n_bytes];
          int total_bytes_read = 0;
          int n_transfers = 0;
-         while (_is_alive && total_bytes_read < n_bytes) {
+         while (_is_alive && total_bytes_read < n_bytes)
+         {
             int bytes_read = _nw_stream.Read(buffer, total_bytes_read, n_bytes - total_bytes_read);
-            if (bytes_read <= 0) {
+            if (bytes_read <= 0)
+            {
                throw new IOException("Read 0 bytes, the connection was closed before all data could be read.");
             }
             n_transfers++;
@@ -161,37 +188,44 @@ namespace bts {
          return buffer;
       }
 
-      private class OneRecvMessage {
+      private class OneRecvMessage
+      {
          public int msg_type;
          public byte[] data;
-         public OneRecvMessage(int msg_type_, byte[] data_) {
+         public OneRecvMessage(int msg_type_, byte[] data_)
+         {
             msg_type = msg_type_;
             data = data_;
          }
-         public OneRecvMessage(byte[] msg) {
+         public OneRecvMessage(byte[] msg)
+         {
             MemoryStream str = new MemoryStream(msg);
             BinaryReader br = new BinaryReader(str);
             int msg_type = br.ReadInt32();
-            if (msg_type == DATA_MSG_SIGNAL) {
+            if (msg_type == DATA_MSG_SIGNAL)
+            {
                // TODO, polish it, too much memory bneing allocated
                data = br.ReadBytes(msg.Length - 4);
             }
          }
 
-         public override string ToString() {
+         public override string ToString()
+         {
             string rv = $"OneMessage msg_type: {msg_type} data_len: {data.Length}";
             return rv;
          }
       }
 
-      private OneRecvMessage _read_one_message() {
+      private OneRecvMessage _read_one_message()
+      {
          // read size first
          byte[] sz_arr = _read_fixed_size_msg(4);
          BinaryReader br = new BinaryReader(new MemoryStream(sz_arr));
          br.BaseStream.Position = 0;
          int msg_sz = br.ReadInt32();
          Log.Gen.info($"AsyncTCPConnection_AsyncWrapper._read_one_message read msg_sz: {msg_sz}");
-         if (msg_sz < 0) {
+         if (msg_sz < 0)
+         {
             throw new IOException("Read 0 bytes, the connection was closed before all data could be read.");
          }
          byte[] msg = _read_fixed_size_msg(msg_sz);
@@ -201,19 +235,29 @@ namespace bts {
          return rv;
       }
 
-      private void _read_msgs_forever() {
-         while (_is_alive) {
-            try {
+      private void _read_msgs_forever()
+      {
+         while (_is_alive)
+         {
+            try
+            {
                OneRecvMessage one_msg = _read_one_message();
-               if (one_msg.msg_type == DATA_MSG_SIGNAL) {
+               if (one_msg.msg_type == DATA_MSG_SIGNAL)
+               {
                   _msg_handler.onData(new RemoteMessage(one_msg.data, RemoteMessageType.Data, _connection_id.peer));
-               } else if (one_msg.msg_type == HB_MSG_SIGNAL) {
+               }
+               else if (one_msg.msg_type == HB_MSG_SIGNAL)
+               {
                   // keep a last hb time an dudpate it here atomically
                   _msg_handler.onData(new RemoteMessage(null, RemoteMessageType.Heartbeat, _connection_id.peer));
-               } else {
+               }
+               else
+               {
                   throw new ApplicationException($"Unknown message type: {one_msg.msg_type}");
                }
-            } catch (Exception ex) {
+            }
+            catch (Exception ex)
+            {
                _handle_error(new ErrorMessage(ex, $"_read_msgs_forever, call_stack: {ex.StackTrace}"));
             }
          }
@@ -223,16 +267,19 @@ namespace bts {
 
       public bool is_disposed => !_is_alive;
 
-      public class ErrorMessage {
+      public class ErrorMessage
+      {
          public readonly SocketError? socket_error = null;
          public readonly Exception exception = null;
          public readonly bool is_exception = false;
          public readonly string context = String.Empty;
-         public ErrorMessage(SocketError socket_error_, string context_) {
+         public ErrorMessage(SocketError socket_error_, string context_)
+         {
             socket_error = socket_error_;
             context = context_;
          }
-         public ErrorMessage(Exception exception_, string context_) {
+         public ErrorMessage(Exception exception_, string context_)
+         {
             is_exception = true;
             exception = exception_;
             context = context_;
@@ -240,23 +287,30 @@ namespace bts {
 
          public bool oom => exception != null && exception is OutOfMemoryException;
 
-         public override string ToString() {
+         public override string ToString()
+         {
             if (is_exception)
                return $"ErrorMessage exception: {exception} context: {context}";
             return $"socket_error: {socket_error} context: {context}";
          }
       }
 
-      public void Dispose() {
+      public void Dispose()
+      {
          // socket is closed and disposed here
-         lock (this) {
-            if (_is_alive) {
+         lock (this)
+         {
+            if (_is_alive)
+            {
                //_master_worker.removeWorker(this);
                _is_alive = false;
-               try {
+               try
+               {
                   bts.Log.Gen.info($"AsyncTCPConnection_APM.Dispose this: {this} closing tcp_client nin an orderly fashion now");
                   _tcp_client.Close();
-               } catch (Exception ex) {
+               }
+               catch (Exception ex)
+               {
                   String.Format("AsyncTCPConnection_APM.Dispose exception: {ex} thrown when closing socket");
                   if (ex is OutOfMemoryException)
                      throw ex;
@@ -266,8 +320,10 @@ namespace bts {
          }
       }
 
-      private void _handle_error(ErrorMessage err_msg) {
-         lock (this) {
+      private void _handle_error(ErrorMessage err_msg)
+      {
+         lock (this)
+         {
             Log.Gen.error($"AsyncTCPConnection_APM._handle_error this: {this} error_msg: {err_msg}");
             if (err_msg.oom)
                throw err_msg.exception;
@@ -275,7 +331,8 @@ namespace bts {
          }
       }
 
-      public override string ToString() {
+      public override string ToString()
+      {
          return _connection_id.ToString();
       }
    }

src/DataStreamSuite/PubSub/Server/Communication/ManagedTCP/ManagedTCPMessageHub.cs

@@ -285,4 +285,4 @@ namespace BCG {
 
 
 
-}
+}

src/DataStreamSuite/PubSub/Server/PubSubSingleTagState.cs

@@ -1,126 +1,126 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
+// using System;
+// using System.Collections.Generic;
+// using System.Linq;
+// using System.Text;
 
-/*
- *  This class contains the recent channges of state of a single pubsub tag, with respect to who has subscribed, who has published, etc. 
- *  It has a list of new subscription events. 
- * 
- */
-namespace BCG {
-   public class PubSubSingleTagState {
-      private string myTag;
-      public string tag {
-         get { return myTag; }
-         set { myTag = value; }
+// /*
+//  *  This class contains the recent channges of state of a single pubsub tag, with respect to who has subscribed, who has published, etc. 
+//  *  It has a list of new subscription events. 
+//  * 
+//  */
+// namespace BCG {
+//    public class PubSubSingleTagState {
+//       private string myTag;
+//       public string tag {
+//          get { return myTag; }
+//          set { myTag = value; }
 
-      }
-      //we have this here as SortedList so that we can compare for equality
-      private SortedList<string, string> myPublishers;
-      private SortedList<string, string> mySubscribers;
-      private Dictionary<string, PubSubSingleTagStateEvent> myNewSubscribers;
-      private Dictionary<string, PubSubSingleTagStateEvent> myRemovedSubscribers;
-      private Dictionary<string, PubSubSingleTagStateEvent> myNewPublishers;
-      private Dictionary<string, PubSubSingleTagStateEvent> myRemovedPublishers;
+//       }
+//       //we have this here as SortedList so that we can compare for equality
+//       private SortedList<string, string> myPublishers;
+//       private SortedList<string, string> mySubscribers;
+//       private Dictionary<string, PubSubSingleTagStateEvent> myNewSubscribers;
+//       private Dictionary<string, PubSubSingleTagStateEvent> myRemovedSubscribers;
+//       private Dictionary<string, PubSubSingleTagStateEvent> myNewPublishers;
+//       private Dictionary<string, PubSubSingleTagStateEvent> myRemovedPublishers;
 
 
 
-      public PubSubSingleTagState() {
-         //the total state of the tag
-         myPublishers = new SortedList<string, string>();
-         mySubscribers = new SortedList<string, string>();
+//       public PubSubSingleTagState() {
+//          //the total state of the tag
+//          myPublishers = new SortedList<string, string>();
+//          mySubscribers = new SortedList<string, string>();
 
-         //how has the state changed recentlty
-         myNewSubscribers = new Dictionary<string, PubSubSingleTagStateEvent>();
-         myNewPublishers = new Dictionary<string, PubSubSingleTagStateEvent>();
-         myRemovedPublishers = new Dictionary<string, PubSubSingleTagStateEvent>();
-         myRemovedSubscribers = new Dictionary<string, PubSubSingleTagStateEvent>();
-      }
+//          //how has the state changed recentlty
+//          myNewSubscribers = new Dictionary<string, PubSubSingleTagStateEvent>();
+//          myNewPublishers = new Dictionary<string, PubSubSingleTagStateEvent>();
+//          myRemovedPublishers = new Dictionary<string, PubSubSingleTagStateEvent>();
+//          myRemovedSubscribers = new Dictionary<string, PubSubSingleTagStateEvent>();
+//       }
 
-      //returns a list of all the events that need to be published.
-      public PubSubSingleTagStateEvent[] getNewEventsForPublishing() {
-         PubSubSingleTagStateEvent[] newEvents = new PubSubSingleTagStateEvent[myNewSubscribers.Count + myNewPublishers.Count + myRemovedPublishers.Count + myRemovedSubscribers.Count];
+//       //returns a list of all the events that need to be published.
+//       public PubSubSingleTagStateEvent[] getNewEventsForPublishing() {
+//          PubSubSingleTagStateEvent[] newEvents = new PubSubSingleTagStateEvent[myNewSubscribers.Count + myNewPublishers.Count + myRemovedPublishers.Count + myRemovedSubscribers.Count];
 
-         myNewSubscribers.Values.CopyTo(newEvents, 0);
-         int index = myNewSubscribers.Count;
-         myNewPublishers.Values.CopyTo(newEvents, index);
-         index += myNewPublishers.Count;
-         myRemovedSubscribers.Values.CopyTo(newEvents, index);
-         index += myRemovedSubscribers.Count;
-         myRemovedPublishers.Values.CopyTo(newEvents, index);
+//          myNewSubscribers.Values.CopyTo(newEvents, 0);
+//          int index = myNewSubscribers.Count;
+//          myNewPublishers.Values.CopyTo(newEvents, index);
+//          index += myNewPublishers.Count;
+//          myRemovedSubscribers.Values.CopyTo(newEvents, index);
+//          index += myRemovedSubscribers.Count;
+//          myRemovedPublishers.Values.CopyTo(newEvents, index);
 
-         return newEvents;
+//          return newEvents;
 
-      }
+//       }
 
-      //call this after you have published the new events
-      //this is made public for code readability reasons 
-      //i.e. so that someone reading the SubInfoStateWatcher can understand what's going on more easily
-      public void resetNewEvents() {
-         myNewSubscribers = new Dictionary<string, PubSubSingleTagStateEvent>();
-         myNewPublishers = new Dictionary<string, PubSubSingleTagStateEvent>();
-         myRemovedSubscribers = new Dictionary<string, PubSubSingleTagStateEvent>();
-         myRemovedPublishers = new Dictionary<string, PubSubSingleTagStateEvent>();
-      }
+//       //call this after you have published the new events
+//       //this is made public for code readability reasons 
+//       //i.e. so that someone reading the SubInfoStateWatcher can understand what's going on more easily
+//       public void resetNewEvents() {
+//          myNewSubscribers = new Dictionary<string, PubSubSingleTagStateEvent>();
+//          myNewPublishers = new Dictionary<string, PubSubSingleTagStateEvent>();
+//          myRemovedSubscribers = new Dictionary<string, PubSubSingleTagStateEvent>();
+//          myRemovedPublishers = new Dictionary<string, PubSubSingleTagStateEvent>();
+//       }
 
-      public void newPublish(PubSubSingleTagStateEvent newPublisher) {
-         //don't do anything here if we already knew about the publishers because we were already dealing with this publishers
-         if (!myPublishers.ContainsKey(newPublisher.app)) {
-            myPublishers.Add(newPublisher.app, newPublisher.app);
+//       public void newPublish(PubSubSingleTagStateEvent newPublisher) {
+//          //don't do anything here if we already knew about the publishers because we were already dealing with this publishers
+//          if (!myPublishers.ContainsKey(newPublisher.app)) {
+//             myPublishers.Add(newPublisher.app, newPublisher.app);
 
-            //if we were going to delete it, that means it was already being published by this app
-            //that means we should no logner delete it, but we don't have to announce that it was published again
-            if (myRemovedPublishers.ContainsKey(newPublisher.app))
-               myRemovedPublishers.Remove(newPublisher.app);
-            else
-               myNewPublishers[newPublisher.app] = newPublisher;
+//             //if we were going to delete it, that means it was already being published by this app
+//             //that means we should no logner delete it, but we don't have to announce that it was published again
+//             if (myRemovedPublishers.ContainsKey(newPublisher.app))
+//                myRemovedPublishers.Remove(newPublisher.app);
+//             else
+//                myNewPublishers[newPublisher.app] = newPublisher;
 
 
-         }
-      }
+//          }
+//       }
 
-      public void removePublisher(PubSubSingleTagStateEvent removedPublisher) {
-         if (myPublishers.ContainsKey(removedPublisher.app)) {
-            myPublishers.Remove(removedPublisher.app);
+//       public void removePublisher(PubSubSingleTagStateEvent removedPublisher) {
+//          if (myPublishers.ContainsKey(removedPublisher.app)) {
+//             myPublishers.Remove(removedPublisher.app);
 
-            //same kind of logic here. If we were about to publish this new publisher, we no longer need to announce it
-            if (myNewPublishers.ContainsKey(removedPublisher.app))
-               myNewPublishers.Remove(removedPublisher.app);
-            else
-               myRemovedPublishers[removedPublisher.app] = removedPublisher;
-         }
+//             //same kind of logic here. If we were about to publish this new publisher, we no longer need to announce it
+//             if (myNewPublishers.ContainsKey(removedPublisher.app))
+//                myNewPublishers.Remove(removedPublisher.app);
+//             else
+//                myRemovedPublishers[removedPublisher.app] = removedPublisher;
+//          }
 
-      }
+//       }
 
-      public void newSubscription(PubSubSingleTagStateEvent newSubscriber) {
-         if (!mySubscribers.ContainsKey(newSubscriber.app)) {
-            mySubscribers.Add(newSubscriber.app, newSubscriber.app);
+//       public void newSubscription(PubSubSingleTagStateEvent newSubscriber) {
+//          if (!mySubscribers.ContainsKey(newSubscriber.app)) {
+//             mySubscribers.Add(newSubscriber.app, newSubscriber.app);
 
-            if (myRemovedSubscribers.ContainsKey(newSubscriber.app))
-               myRemovedSubscribers.Remove(newSubscriber.app);
-            else
-               myNewSubscribers[newSubscriber.app] = newSubscriber;
+//             if (myRemovedSubscribers.ContainsKey(newSubscriber.app))
+//                myRemovedSubscribers.Remove(newSubscriber.app);
+//             else
+//                myNewSubscribers[newSubscriber.app] = newSubscriber;
 
 
-         }
-      }
+//          }
+//       }
 
-      public void removeSubscriber(PubSubSingleTagStateEvent removedSubscriber) {
-         if (mySubscribers.ContainsKey(removedSubscriber.app)) {
-            mySubscribers.Remove(removedSubscriber.app);
-            //if we were going to add this subscriber, we no logner want to
-            //this means no event needs to be published
-            if (myNewSubscribers.ContainsKey(removedSubscriber.app))
-               myNewSubscribers.Remove(removedSubscriber.app);
-            else
-               //we have to remove this guy in this updated
-               myRemovedSubscribers[removedSubscriber.app] = removedSubscriber;
-         }
+//       public void removeSubscriber(PubSubSingleTagStateEvent removedSubscriber) {
+//          if (mySubscribers.ContainsKey(removedSubscriber.app)) {
+//             mySubscribers.Remove(removedSubscriber.app);
+//             //if we were going to add this subscriber, we no logner want to
+//             //this means no event needs to be published
+//             if (myNewSubscribers.ContainsKey(removedSubscriber.app))
+//                myNewSubscribers.Remove(removedSubscriber.app);
+//             else
+//                //we have to remove this guy in this updated
+//                myRemovedSubscribers[removedSubscriber.app] = removedSubscriber;
+//          }
 
 
-      }
+//       }
 
 
-   }
-}
+//    }
+// }

src/DataStreamSuite/PubSubServer/Configurations/PubsubServer_Linux.json

@@ -0,0 +1,61 @@
+{
+   "user": "bts",
+   "pubsub": {
+      "milliseconds_between_sub_info_updates": 1000,
+      "milliseconds_between_time_slice_updates": 5000,
+      "milliseconds_between_no_buffer_space_retries": 100,
+      "retries_on_no_buffer_space": 1,
+      "do_it_all_seconds_between_logs": 0,
+      "heartbeat_interval": 15000,
+      "missing_heartbeats_before_gc": 50,
+      "use_managed_sockets": true,
+      "state_update_period": 5000,
+      "statistics_update_period": 5000,
+      "disposed_connection_check_period": 1000,
+      "message_processing_warning_threshold": 1000,
+      "server_port": 12345,
+      "publisher_routes": [
+         {
+            "name": "default",
+            "server_address": "bts-cloud://192.168.1.80:12345",
+            "tag": ".*"
+         }
+      ],
+      "subscriber_routes": [
+         {
+            "name": "default",
+            "server_address": "bts-cloud://192.168.1.80:12345",
+            "tag": ".*"
+         }
+      ],
+      "max_packet_size": 500000000,
+      "blue_comm_must_die": true,
+      "crushable_tags": [],
+      "tag_type_to_assembly_map": [
+         {
+            "handler_class": "BCG.PubSubServerTagHandler",
+            "assembly": "DataStreamCore.dll"
+         }
+      ],
+      "server_queue": "PubSubQueue",
+      "port_range": {
+         "first": 30400,
+         "last": 31700
+      },
+      "bind_to_random_port": false,
+      "debug_tags": "^NOTAGS$",
+      "latency_report_threshold_ms": 1500,
+      "use_substations": false,
+      "use_dynamic_clients": false,
+      "is_substation": false,
+      "internally_publish_sub_info": true,
+      "use_managed_clients_in_native_code": false,
+      "compressed_tags": "^NOTAGS$",
+      "tags_to_report_compression_ratio": "^NOTAGS$",
+      "disable_msmq": false,
+      "use_prepublished_server_state": false,
+      "ip_translation": null,
+      "gc_activation_threshold": 0,
+      "wait_period_after_gc": 1000
+   }
+}

src/DataStreamSuite/SharedStreams/Business/BenzingaNews/BenzingaNewsStream.cs

@@ -7,26 +7,45 @@ namespace bts {
    using System.Text.Json;
    using System.Text.Json.Nodes;
    [DataStreamUserRespresentation("benzingaNews")]
-   public class BenzingaNewStream : ViewStream {
+   public class BenzingaNewsStream : ViewStream {
       private IBlueTableSchema _schema = null;
       private int _id = 0;
-      // constructor takes in the raw mongo db stream, and the time to live (ttl), "7d" for example
-      public BenzingaNewStream(ViewStream mongodb_raw_stream, string ttl) {
+
+
+      private static ViewStream _pre_process_raw_kvstream(ViewStream mongo_db_str) {
+         ViewStream r = new KeyValueToRelationalStream(mongo_db_str, "doc_id", "key", "value");
+         r = new DateStandardizationStream(r, "created_at", "ISO8601");
+         return r;
+      }
+
+      public BenzingaNewsStream(string connection_string, string db_name, string collection_name, string poll_period, string ttl)
+      : this(new MongoDBStream(connection_string, db_name, collection_name, poll_period), ttl) {
+      }
+
+      // public BenzingaNewsStream()
+      // : this(new MongoDBStream(DataStreamPub connection_string, db_name, collection_name, poll_period), ttl) {
+      // }
+
+      // assumes it gets a raw mongodb_raw_kv_stream with key value pairs
+      public BenzingaNewsStream(ViewStream mongodb_raw_kv_stream, string ttl) {
+         ViewStream output_stream = _pre_process_raw_kvstream(mongodb_raw_kv_stream);
          // Hiding body (with project, columns can be hidden, or renamed)
-         ViewStream output_stream = new ProjectStream(mongodb_raw_stream, "!body");
+         output_stream = new ProjectStream(output_stream, "!body");
          // add column news_source = 'Benzinga', so we can append other news sources
          output_stream = new AddColumnsStream(output_stream, @"news_source : Benzinga");
          // now SuperMapStream is used to map one row into multiple, one per symbol,
          // (SuperMapStream takes 2 lambdas, one to get the enw schema and another to 
          // return an array of rows, per one row)
-         output_stream = new SuperMapStream(mongodb_raw_stream,
+         output_stream = new SuperMapStream(output_stream,
 
         (IBlueTableSchema s) => {
            // this lambda returns the new schema (ie with extra cols symbol, exchange, is_primary_exchange)
            if (_schema == null) {
               var out_schema = s;
               out_schema = BlueDataUtility.remove_columns(out_schema, "securities");
-              out_schema = BlueDataUtility.add_column(s, "doc_id_internal", typeof(string));
+              out_schema = BlueDataUtility.remove_columns(out_schema, "body");
+              out_schema = BlueDataUtility.remove_columns(out_schema, "tags");
+              out_schema = BlueDataUtility.add_column(out_schema, "doc_id_internal", typeof(string));
               out_schema = BlueDataUtility.add_column(out_schema, "symbol", typeof(string));
               out_schema = BlueDataUtility.add_column(out_schema, "exchange", typeof(string));
               out_schema = BlueDataUtility.add_column(out_schema, "is_primary_exchange", typeof(bool));
@@ -44,11 +63,13 @@ namespace bts {
            }
            foreach (JsonNode jnode in json_objs) {
               if (jnode is JsonObject json_object) {
+                 string symbol = jnode["symbol"] == null ? String.Empty : jnode["symbol"].ToString();
+                 string exchange = jnode["exchange"] == null ? String.Empty : jnode["exchange"].ToString();
                  MutableBlueRow nr = new MutableBlueRow(_schema, r);
                  nr.setValue("id", _id++);
                  nr.setValue("doc_id_internal", $"{r["doc_id"]}_{jnode["symbol"]}");
-                 nr.setValue("symbol", jnode["symbol"]);
-                 nr.setValue("exchange", jnode["exchange"]);
+                 nr.setValue("symbol", symbol);
+                 nr.setValue("exchange", exchange);
                  nr.setValue("is_primary_exchange", Boolean.Parse(jnode["primary"].ToString()));
                  expanded_rows.Add(nr);
               }

src/DataStreamSuite/SharedStreams/Business/BenzingaNews/BenzingaNewsSymbolStateStream.cs

@@ -0,0 +1,33 @@
+namespace bts {
+   using BCG;
+
+   using System.Collections.Generic;
+   using System.Linq;
+   using System.Security.Cryptography;
+
+   //[DataStreamUserRespresentation("benzingaNewsSymbols")]
+   public class BenzingaNewsSymbolStateStream : ViewStream, INonIncGroupByEvaluator {
+      public BenzingaNewsSymbolStateStream(ViewStream news_str) {
+         ViewStream r = new NonIncGroupByStream(news_str, this);
+         mySetRealSource(r);
+      }
+      public BenzingaNewsSymbolStateStream(string connection_string, string db_name, string collection_name, string poll_period, string ttl) :
+      this(new BenzingaNewsStream(connection_string, db_name, collection_name, poll_period, ttl)) {
+      }
+
+      public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
+         BlueTable rv = null;
+         foreach (var kv in grouped_rows) {
+            var symbol_rows = kv.Value;
+            IBlueRow state_row = BlueDataUtility.find_max(symbol_rows, "created_at");
+            rv = rv == null ? new BlueTable(state_row.schema) : rv;
+            rv.addRow(state_row);
+         }
+         return rv;
+      }
+
+      public string[] get_group_by_cols() {
+         return new string[] { "symbol" };
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/Business/BenzingaNews/BenzingaNewsSymbolsStream.cs

@@ -0,0 +1,14 @@
+namespace bts {
+   using BCG;
+   [DataStreamUserRespresentation("benzingaNewsSymbols")]
+
+   public class BenzingaNewsSymbolsStream : ViewStream {
+      public BenzingaNewsSymbolsStream(ViewStream news_str) {
+         ViewStream rs = new DistinctRowStream(news_str, "symbol");
+         mySetRealSource(rs);
+      }
+      public BenzingaNewsSymbolsStream(string connection_string, string db_name, string collection_name, string poll_period, string ttl) :
+      this(new BenzingaNewsStream(connection_string, db_name, collection_name, poll_period, ttl)) {
+      }
+   }
+}

src/DataStreamSuite/SharedStreams/Config/SharedStreamConfig.cs

@@ -0,0 +1,48 @@
+namespace Bts.DataStreamPublisher;
+
+using Bts.PubSub;
+
+//using CommandLine;
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+public class SharedStreamConfig {
+
+   [JsonPropertyName("mongodb_url")]
+   public string MONGO_DB_URL { get; set; }
+
+   [JsonPropertyName("mongodb_db_name")]
+   public string MONGO_DB_NAME { get; set; }
+
+   [JsonPropertyName("mongodb_db_collection")]
+   public string MONGO_DB_COLLECTION { get; set; }
+
+   [JsonPropertyName("benzinga_news_ttl")]
+   public string BENZINGA_NEWS_TTL { get; set; }
+
+   [JsonPropertyName("pubsub")]
+   public PubSubConfig? PubSub { get; set; }
+
+
+   /// <summary>
+   /// Merges options with those defined in `ConfigFile` (does not overwrite
+   /// non-null existing values).
+   /// </summary>
+   /// <exception cref="InvalidOperationException">
+   /// JsonSerializer.Deserialize returns null
+   /// </exception>
+   public void MergeWithConfigFile() {
+      // if (ConfigFile is null) {
+      //    return;
+      // }
+      // var cfgString = File.ReadAllText(ConfigFile);
+      // var parsedCfg = JsonSerializer.Deserialize<DataStreamPublisherConfig>(cfgString) ??
+      //    throw new InvalidOperationException($"Failed to parse json from {ConfigFile}");
+      // MergeWith(parsedCfg);
+      // ConfigFile = null;
+   }
+}

src/Spark.Admin.WebApp/src/Accounts/Types.fs

@@ -17,7 +17,7 @@ type Tab =
   | PositionManagementTab
   | MarginTab
 
-let venues = [| Venue.BINANCE; Venue.BinanceUs; Venue.BinanceUsdsFutures; Venue.COINBASE; Venue.DERIBIT; Venue.GLBX; Venue.ICE; Venue.SIM; |]
+let venues = [| Venue.BINANCE; Venue.BinanceUs; Venue.BinanceUsdsFutures; Venue.CBOE; Venue.COINBASE; Venue.DERIBIT; Venue.GLBX; Venue.ICE; Venue.TAIFEX; Venue.SIM; |]
 
 let securityTypes = [| SecurityType.SPOT; SecurityType.FUTURE; SecurityType.OPTION; SecurityType.SPREAD; |]
 
@@ -69,6 +69,11 @@ type ICEExchangeField =
   | AccountCode of string
   | MiscTagMapping of JSMap<string>
 
+type TaifexExchangeField =
+  | CTI of CTICode
+  | Origin of CustomerOrFirm
+  | ClearingAccount of string
+
 type SIMExchangeField =
   | CTI of CTICode
   | Origin of CustomerOrFirm

src/Spark.Admin.WebApp/src/Accounts/View.fs

@@ -797,9 +797,9 @@ let venueToString (venue:bts.api.Venue) =
     | bts.api.Venue.BINANCE -> "BINANCE"
     | bts.api.Venue.BinanceUs -> "BinanceUs"
     | bts.api.Venue.BinanceUsdsFutures -> "BinanceUsdsFutures"
-    | bts.api.Venue.COINBASE -> "COINBASE"
     | bts.api.Venue.FTX -> "FTX"
     | bts.api.Venue.OKX -> "OKX"
+    | bts.api.Venue.TAIFEX -> "TAIFEX"
 
 let exchange_display expirationNameList =
   let s = sprintf "%O" expirationNameList.product.exchange

src/client/spark/AW2/AnalyticsParameters/AnalyticsExpirationParametersSettings.cs

@@ -1,4 +1,6 @@
 namespace mbts.spark {
+   using bts.api;
+
    using ProtoBuf;
 
    [ProtoContract]
@@ -10,6 +12,12 @@ namespace mbts.spark {
       [ProtoMember(10)]
       public string[] selected_symbols {get;set;}
 
+      [ProtoMember(11)]
+      public Venue[] selected_venues {get;set;}
+
+      [ProtoMember(12)]
+      public SecurityType[] selected_types {get;set;}
+
       [ProtoMember(20)]
       public bool auto_copy_rates_is_checked {get;set;}
 

src/client/spark/AW2/AnalyticsParameters/AnalyticsExpirationParametersVM.cs

@@ -44,6 +44,8 @@ namespace mbts.spark {
       readonly DeferrableAction _raise_selected_rows_changed;
       readonly IRaiseCanExecuteChangedCommand _set_driver_to_nearest_exp_cmd;
       IList<string>? _pending_selected_symbols;
+      IList<Venue>? _pending_selected_venues;
+      IList<SecurityType>? _pending_selected_types;
 
       AnalyticsExpirationParametersMap? _parameter_map;
       (ICommand cmd, IDisposable can_exe_sub)? _smart_copy_roll_target_to_selection_cmd;
@@ -56,9 +58,13 @@ namespace mbts.spark {
          _auto_copy_rolls_is_checked = backing(nameof(auto_copy_rolls_is_checked), false);
          _raise_selected_rows_changed = new(() => selected_rows_changed?.Invoke(_selected_rows));
          var selected_symbol_ref_count = new RefCountMap<string>();
+         var selected_venue_ref_count = new RefCountMap<Venue>();
+         var selected_type_ref_count = new RefCountMap<SecurityType>();
          _sync_visible_rows = new(() =>
             visible_rows.sync_with(
-               rows.Where(x => selected_symbol_ref_count.ContainsKey(x.security.symbol))));
+               rows.Where(x => selected_symbol_ref_count.ContainsKey(x.security.symbol)
+                  && selected_venue_ref_count.ContainsKey(x.security.exchange)
+                  && selected_type_ref_count.ContainsKey(x.security.security_type))));
          non_options_with_empty = new ObservableAggregateCollection<SecurityExpiration>(
             new() { SecurityExpiration.empty }, non_options);
 
@@ -67,6 +73,8 @@ namespace mbts.spark {
 
          if (startup_data is not null) {
             _pending_selected_symbols = startup_data.selected_symbols;
+            _pending_selected_venues = startup_data.selected_venues;
+            _pending_selected_types = startup_data.selected_types;
             auto_copy_rolls_is_checked = startup_data.auto_copy_rolls_is_checked;
             auto_copy_rates_is_checked = startup_data.auto_copy_rates_is_checked;
          }
@@ -103,6 +111,26 @@ namespace mbts.spark {
                   }
                });
             }),
+            ObservableAutoWrapper.connect_item_hooks(selected_venues, sym => {
+               if (selected_venue_ref_count.increment_adds_key(sym)) {
+                  _sync_visible_rows.Invoke();
+               }
+               return Disposable.Create(() => {
+                  if (selected_venue_ref_count.decrement_removes_key(sym)) {
+                     _sync_visible_rows.Invoke();
+                  }
+               });
+            }),
+            ObservableAutoWrapper.connect_item_hooks(selected_types, sym => {
+               if (selected_type_ref_count.increment_adds_key(sym)) {
+                  _sync_visible_rows.Invoke();
+               }
+               return Disposable.Create(() => {
+                  if (selected_type_ref_count.decrement_removes_key(sym)) {
+                     _sync_visible_rows.Invoke();
+                  }
+               });
+            }),
             ObservableAutoWrapper.connect_item_hooks(
                rows, r => r.observe_is_dirty().Subscribe(_ => _is_dirty.refresh())),
             _is_dirty.observe_current(true).OnDispatcher().Subscribe(can_revert_observer),
@@ -128,6 +156,10 @@ namespace mbts.spark {
 
       public ObservableCollection<string> symbols { get; } = new();
       public ObservableCollection<string> selected_symbols { get; } = new();
+      public ObservableCollection<Venue> venues { get; } = new();
+      public ObservableCollection<Venue> selected_venues { get; } = new();
+      public ObservableCollection<SecurityType> types { get; } = new() { SecurityType.Future, SecurityType.Option };
+      public ObservableCollection<SecurityType> selected_types { get; } = new();
 
       public bool is_dirty => _is_dirty.Value;
 
@@ -191,6 +223,8 @@ namespace mbts.spark {
       public AnalyticsExpirationParametersSettings get_state() =>
          new() {
             selected_symbols = selected_symbols.Cast<string>().ToArray(),
+            selected_venues = selected_venues.ToArray(),
+            selected_types = selected_types.ToArray(),
             auto_copy_rates_is_checked = auto_copy_rates_is_checked,
             auto_copy_rolls_is_checked = auto_copy_rolls_is_checked
          };
@@ -206,7 +240,16 @@ namespace mbts.spark {
                .Where(d => d.HasSecurity && d.Security.Symbol == r.security.symbol)
                .Where(d => is_unexpired(d))
                .TryGetMinElement(
-                  d => (r.security.expiration - d.Security.Expiration.date_time()).Ticks.abs(),
+                  d => {
+                     var expDiff = (r.security.expiration - d.Security.Expiration.date_time()).Ticks.abs();
+                     if (r.security.exchange != d.Security.Exchange.to_venue()) {
+                        // This is a hack to favor the security from the same venue as the row's
+                        // security in the event of a tie (for the nearest exp). As long as exp
+                        // dates are separated by more than 1 tick (100ns), the hack will work.
+                        expDiff += 1;
+                     }
+                     return expDiff;
+                  },
                   out var nearest_exp)) {
                r.driver = nearest_exp;
             }
@@ -294,10 +337,16 @@ namespace mbts.spark {
          }
 
          symbols.sync_with(_parameters.Select(p => p.security.symbol).Distinct().Ordered());
+         venues.sync_with(_parameters.Select(p => p.security.exchange).Distinct().OrderBy(v => v.ToString()));
+         types.sync_with(_parameters.Select(p => p.security.security_type).Distinct().OrderBy(v => v.ToString()));
          using var _ = _sync_visible_rows.defer();
          if (is_first_update) {
             selected_symbols.sync_with(_pending_selected_symbols?.Intersect(symbols) ?? symbols);
             _pending_selected_symbols = null;
+            selected_venues.sync_with(_pending_selected_venues?.Intersect(venues) ?? venues);
+            _pending_selected_venues = null;
+            selected_types.sync_with(_pending_selected_types?.Intersect(types) ?? types);
+            _pending_selected_types = null;
          }
       }
 

src/client/spark/AW2/AnalyticsParameters/AnalyticsExpirationParametersView.xaml

@@ -162,6 +162,34 @@
         </c:ComboCheckBox.HeaderContentTemplate>
       </c:ComboCheckBox>
 
+      <c:ComboCheckBox
+        ItemsSource="{Binding venues}"
+        SelectedItems="{Binding selected_venues}"
+        HorizontalAlignment="Left"
+        Margin="5,2,2,2"
+        HeaderContent="Venues"
+        ShowSelectAll="True">
+        <c:ComboCheckBox.HeaderContentTemplate>
+          <DataTemplate>
+            <TextBlock Text="{Binding}" Foreground="{u:BTSColor UI.default_text}" />
+          </DataTemplate>
+        </c:ComboCheckBox.HeaderContentTemplate>
+      </c:ComboCheckBox>
+
+      <c:ComboCheckBox
+        ItemsSource="{Binding types}"
+        SelectedItems="{Binding selected_types}"
+        HorizontalAlignment="Left"
+        Margin="5,2,2,2"
+        HeaderContent="Types"
+        ShowSelectAll="True">
+        <c:ComboCheckBox.HeaderContentTemplate>
+          <DataTemplate>
+            <TextBlock Text="{Binding}" Foreground="{u:BTSColor UI.default_text}" />
+          </DataTemplate>
+        </c:ComboCheckBox.HeaderContentTemplate>
+      </c:ComboCheckBox>
+
       <WrapPanel>
         <TextBlock
           Margin="5,5,0,0" Foreground="{u:BTSBrush UI.default_text}" Text="Auto copy rolls:" />

src/client/spark/AW2/AnalyticsParameters/AnalyticsExpirationParametersView.xaml.cs

@@ -39,7 +39,8 @@ namespace mbts.ui.shell {
          new() {
             ShowSymbol = true,
             ShowType = true,
-            MapExp = true
+            MapExp = true,
+            ShowVenue = true,
          };
 
       public static IValueConverter driver_display_text_converter =>

src/client/spark/AW2/Bus/SingleOptionDriverPricePublisher.cs

@@ -4,6 +4,8 @@ using System.Linq;
 using mbts.analytics;
 using bts.api;
 using bts.api.analytics;
+using Spark.Core.Disposables;
+using System.Reactive.Linq;
 
 public class DriverMD {
    public readonly DriverCalculationType calc_type = DriverCalculationType.Direct;
@@ -46,57 +48,63 @@ namespace mbts.spark {
    // Publishes the driver prices for expirations of a single option (this is not AW2 specific)
    // f(AnalyticsExpirationParametersMap, SystemContractData) -> OptionDriverPriceMap
    // This is constructed by the ResourceAllocator
-   public class SingleOptionDriverPricePublisher {
+   public sealed class SingleOptionDriverPricePublisher : IDisposable {
+      readonly object syncRoot = new();
       private readonly SecurityTuple _sec_tup;
-      private ThreadedBlackboard _bb;
+      readonly IBus _bus;
       // tags
-      private ITag<AnalyticsExpirationParametersMap> _in_bb_exp_params_tag =
-         SparkTags.ExpirationParametersInputTag;
-      private ITag<SystemContractData> _in_bb_contracts_tag =
-         SparkTags.SystemContractsTag;
       private OptionDriverPriceMap _last_published;
       private readonly PerfStatsLogger _logger;
       private readonly TimeSpan _fire_period = TimeSpan.FromMilliseconds(300);
       private DateTime _last_fire_time = DateTime.MinValue;
+      SystemContractData sc_data;
+      AnalyticsExpirationParametersMap param_map;
+      IDisposable dispose;
 
-      public SingleOptionDriverPricePublisher(SecurityTuple sec_tup,
-                                               ThreadedBlackboard bb) {
+      public SingleOptionDriverPricePublisher(SecurityTuple sec_tup, IBus bus) {
          bts.Log.Gen.info($"SingleOptionDriverPricePublisher initialized st:{sec_tup}");
          _sec_tup = sec_tup;
-         _bb = bb;
+         _bus = bus;
          _logger = new PerfStatsLogger(50, 50,
             $"SingleOptionDriverPricePublisher_{sec_tup.security_name}_{GetHashCode()}");
          _last_published = new OptionDriverPriceMap(sec_tup.security_name,
             new Dictionary<SecurityExpiration, DriverMD>());
-         _bb.subscribe(_in_bb_exp_params_tag);
-         _bb.subscribe(_in_bb_contracts_tag);
-         _bb.data_callback += _on_bb_data;
-         _bb.timer_callback += _on_bb_timer;
+         dispose = Disposable.Create(
+            Observable.Interval(_fire_period).Subscribe(onInterval),
+            bus.subscribe(SparkTags.ExpirationParametersInputTag).Subscribe(onExpirationParameters),
+            bus.subscribe(SparkTags.SystemContractsTag).Subscribe(onSystemContractData));
          _compute_and_fire();
-      }
 
-      private void _on_bb_data(object tag, object data) {
-         if (_in_bb_exp_params_tag == tag || _in_bb_contracts_tag == tag) {
-            _on_bb_timer();
+         void onInterval(long _) {
+            _compute_and_fire();
          }
-      }
 
-      private void _on_bb_timer() {
-         if (DateTime.Now - _last_fire_time >= _fire_period) {
-            _last_fire_time = DateTime.Now;
+         void onExpirationParameters(AnalyticsExpirationParametersMap expParams) {
+            param_map = expParams;
+            _compute_and_fire();
+         }
+
+         void onSystemContractData(SystemContractData data) {
+            sc_data = data;
             _compute_and_fire();
          }
       }
 
-      private void _compute_and_fire() {
+      public void Dispose() =>
+         Disposable.dispose(ref dispose);
+
+      void _compute_and_fire() {
+         lock (syncRoot) {
+            _compute_and_fire_impl();
+         }
+      }
+
+      private void _compute_and_fire_impl() {
          _logger.start_default_timer();
-         AnalyticsExpirationParametersMap param_map = _bb.read_state(_in_bb_exp_params_tag);
-         SystemContractData sc_data = _bb.read_state(_in_bb_contracts_tag);
          if (param_map == null || sc_data == null) {
             return;
          }
-         IDictionary<mbts.SecurityExpiration, mbts.SecurityExpiration> driver_map =
-            param_map.get_option_to_driver_map(_sec_tup.security_name);
+         var driver_map = param_map.get_option_to_driver_map(_sec_tup.security_name, _sec_tup.security_exchange);
          // look up the market data
          IDictionary<SecurityExpiration, DriverMD> res =
               new Dictionary<SecurityExpiration, DriverMD>();
@@ -128,7 +136,7 @@ namespace mbts.spark {
             // bts.Log.Gen.info("SingleOptionDriverPricePublisher._compute_and_fire \n" +
             //                   $"map:{to_fire.log_string()}");
             _logger.stop_default_timer();
-            _bb.post(SparkTags.OptionDriverBBOTag(_sec_tup.security_name),
+            _bus.post(SparkTags.OptionDriverBBOTag((_sec_tup.security_name, _sec_tup.security_exchange)),
                      new OptionDriverPriceMap(_sec_tup.security_name, res));
          }
       }
@@ -168,15 +176,13 @@ namespace mbts.spark {
       public static Tuple<SecurityExpiration, SecurityExpiration>
             find_neighbors(SystemContractData sc_data,
                            SecurityTuple sec_tup, DateTime exp) {
-         var exps = sc_data.exp_map.Where(
-            kv => sec_tup.Equals(kv.Key.security_tuple)).Select(kv => kv.Key);
+         var exps = sc_data.GetExpirations(sec_tup.security_exchange, sec_tup.security_type, sec_tup.security_name);
          if (exps == null) {
             return null;
          }
-         SecurityExpiration low, high;
          VolSurfaceUtilityFunctions.find_neighbors(exps, exp,
-            (t, p) => t.expiration.Subtract(p).TotalMilliseconds, out low, out high);
-         return Tuple.Create(low, high);
+            (t, p) => t.Subtract(p).TotalMilliseconds, out var low, out var high);
+         return new(new(sec_tup, low), new(sec_tup, high));
       }
 
       public static double interp_driver(SystemContractData sc_data,

src/client/spark/AW2/Bus/SingleVolSurfaceCommittedOutputRetriever.cs

@@ -1,11 +1,11 @@
-using System;
+#nullable enable
+using System;
 using System.Collections.Generic;
 using System.Diagnostics;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 using mbts.analytics;
 using bts.api;
+using Spark.Core.Disposables;
+using System.Reactive.Linq;
 
 namespace mbts.spark {
    // Publishes the vol outputs for a single vol surface.
@@ -19,41 +19,26 @@ namespace mbts.spark {
    // 1. VS Out
    public class SingleVolSurfaceCommittedOutputRetriever : IDisposable {
       private readonly int _vs_id;
-      private readonly ITag<VolSurfaceInput> _in_bb_vs_input_tag;
-      private readonly ITag<VolSurfaceNonVolData> _in_bb_nonvol_data_tag;
-      private readonly ITag<VolSurfaceStandardTime> _in_vs_standard_time_tag;
-      private readonly ITag<VolSurfaceOutput> _out_bb_computed_vol_tag;
-      private readonly ThreadedBlackboard _bb;
       private readonly PerfStatsLogger _logger = new PerfStatsLogger(50, 100);
       private readonly string _log_id;
+      readonly IBus _bus;
+      IDisposable? _dispose;
 
-      public SingleVolSurfaceCommittedOutputRetriever(int vs_id,
-         ThreadedBlackboard threaded_bb, IEnumerable<string> symbols) {
+      public SingleVolSurfaceCommittedOutputRetriever(int vs_id, IBus bus) {
          bts.Log.Gen.info($"SingleVolSurfaceCommittedOutputRetriever cons vs_id:{vs_id} tid:{_tid()}");
          _vs_id = vs_id;
-         _bb = threaded_bb;
-         _in_bb_vs_input_tag = SparkTags.VolSurfaceCommittedInputTag(vs_id);
-         _in_bb_nonvol_data_tag = SparkTags.VolSurfaceNonVolDataTag(vs_id);
-         _in_vs_standard_time_tag = SparkTags.VolSurfaceStandardTimeTag(vs_id);
-         _out_bb_computed_vol_tag = SparkTags.VolSurfaceCommittedOutputTag(vs_id);
+         _bus = bus;
          _log_id = String.Format("SingleVolSurfaceCommittedOutputRetriever_{0}", vs_id);
-         _bb.subscribe(_in_bb_vs_input_tag);
-         _bb.subscribe(_in_bb_nonvol_data_tag);
-         _bb.subscribe(_in_vs_standard_time_tag);
-         _bb.data_callback += _on_bb_data;
-         _recompute_and_publish();
+         _dispose = Observable.CombineLatest(
+            _bus.subscribe(SparkTags.VolSurfaceCommittedInputTag(vs_id)),
+            _bus.subscribe(SparkTags.VolSurfaceNonVolDataTag(vs_id)),
+            _bus.subscribe(SparkTags.VolSurfaceStandardTimeTag(vs_id)),
+            (vs_input, vs_nonvol_data, vs_standard_time) => (vs_input, vs_nonvol_data, vs_standard_time))
+            .Subscribe(_recompute_and_publish);
       }
       private static int _tid() { return VolSurfaceTroubleshootingFunctions.tid(); }
-      private void _on_bb_data(object tag, object data) {
-         if (_in_bb_vs_input_tag.Equals(tag) || _in_bb_nonvol_data_tag.Equals(tag) ||
-             _in_vs_standard_time_tag.Equals(tag))
-            _recompute_and_publish(tag);
-      }
-
-      private void _recompute_and_publish(object tag = null) {
-         VolSurfaceInput vs_input = _bb.read_state(_in_bb_vs_input_tag);
-         VolSurfaceNonVolData vs_nonvol_data = _bb.read_state(_in_bb_nonvol_data_tag);
-         VolSurfaceStandardTime vs_standard_time = _bb.read_state(_in_vs_standard_time_tag);
+      private void _recompute_and_publish((VolSurfaceInput, VolSurfaceNonVolData, VolSurfaceStandardTime) data_tup) {
+         var (vs_input, vs_nonvol_data, vs_standard_time) = data_tup;
          if (vs_input != null && vs_nonvol_data != null && vs_standard_time != null) {
             var timer = Stopwatch.StartNew();
             IDictionary<VolSurfaceExpiration, VolSurfaceExpirationAuxiliaryData> aux_data_map =
@@ -62,17 +47,12 @@ namespace mbts.spark {
             VolSurfaceOutput vs_output = GeneralVolSurfaceCalculator.compute(
                vs_input, vs_nonvol_data.contracts, aux_data_map);
             timer.Stop();
-            var tag_str = tag == null ? "Null" : tag.ToString();
-            // bts.Log.Gen.info($"SingleVolSurfaceCommittedOutputRetriever._recompute_and_publish " +
-            //                  $"vs_id: {_vs_id} time:{_timer.lastIntervalTime} ms tag:{tag_str} " +
-            //                  $"tid:{_tid()}"); //output: {vs_output.log_string()}*/");
             _logger.log(_log_id, _log_id, timer.Elapsed.TotalMilliseconds);
-            _bb.post(_out_bb_computed_vol_tag, vs_output);
+            _bus.post(SparkTags.VolSurfaceCommittedOutputTag(_vs_id), vs_output);
          }
       }
 
-      #region IDisposable
-      public void Dispose() { _bb.data_callback -= _on_bb_data; }
-      #endregion
+      public void Dispose() =>
+         Disposable.dispose(ref _dispose);
    }
 }

src/client/spark/AW2/Bus/SingleVolSurfaceDataSource.cs

@@ -24,15 +24,15 @@ namespace mbts.spark {
       private readonly SymbolListType _list_type;
       private readonly int _vs_id;
       private readonly ThreadedBlackboard _bb;
-      private readonly Func<IEnumerable<string>, IDisposable> _create_func;
+      private readonly Func<IEnumerable<(string, Venue)>, IDisposable> _create_func;
       private readonly ITag<VolSurfaceInput> _in_bb_committed_data_tag;
       private readonly ITag<VolSurfaceCalculationRequest> _in_bb_calc_req_tag;
 
-      private HashSet<string> _option_symbols = new HashSet<string>();
+      private HashSet<(string, Venue)> _option_symbols = new();
       private IDisposable _source = null;
 
       public SingleVolSurfaceDataSource(SymbolListType list_type, int vs_id, ThreadedBlackboard threaded_bb,
-                                        Func<IEnumerable<string>, IDisposable> create_func) {
+                                        Func<IEnumerable<(string, Venue)>, IDisposable> create_func) {
          Log.Gen.info(String.Format("SingleVolSurfaceDataSource constructed vs_id {0}", vs_id));
          _list_type = list_type;
          _vs_id = vs_id;
@@ -56,33 +56,33 @@ namespace mbts.spark {
          VolSurfaceInput vs_comm_inp = _bb.read_state(_in_bb_committed_data_tag);
          if (_list_type == SymbolListType.Candidate) {
             if (vs_calc_req != null && vs_calc_req.volsurface_input != null) {
-               HashSet<string> new_sym_map = new HashSet<string>(VolSurfaceDataFunctions.get_symbols(vs_calc_req.volsurface_input));
+               var new_sym_map = VolSurfaceDataFunctions.get_symbols(vs_calc_req.volsurface_input).ToHashSet();
                _create_if_needed(new_sym_map);
             }
          } else if (_list_type == SymbolListType.Committed) {
             if (vs_comm_inp != null) {
-               HashSet<string> new_sym_map = new HashSet<string>(VolSurfaceDataFunctions.get_symbols(vs_comm_inp));
+               var new_sym_map = VolSurfaceDataFunctions.get_symbols(vs_comm_inp).ToHashSet();
                _create_if_needed(new_sym_map);
             }
          } else if (_list_type == SymbolListType.Combined) {
-            List<string> new_sym_list = new List<string>();
+            var new_syms = new HashSet<(string, Venue)>();
             if (vs_calc_req != null && vs_calc_req.volsurface_input != null)
-               new_sym_list.AddRange(VolSurfaceDataFunctions.get_symbols(vs_calc_req.volsurface_input));
+               new_syms.AddRange(VolSurfaceDataFunctions.get_symbols(vs_calc_req.volsurface_input));
             if (vs_comm_inp != null)
-               new_sym_list.AddRange(VolSurfaceDataFunctions.get_symbols(vs_comm_inp));
-            _create_if_needed(new HashSet<string>(new_sym_list));
+               new_syms.AddRange(VolSurfaceDataFunctions.get_symbols(vs_comm_inp));
+            _create_if_needed(new_syms);
          }
       }
 
-      private void _create_if_needed(HashSet<string> new_sym_map) {
+      private void _create_if_needed(HashSet<(string, Venue)> new_sym_map) {
          if (!CollectionHelper.hashset_equals(_option_symbols, new_sym_map)) {
-            _option_symbols = new HashSet<string>(new_sym_map);
+            _option_symbols = new(new_sym_map);
             if (_source != null)
                _source.Dispose();
             _source = _create_func(_option_symbols);
             _bb.post(SparkTags.AquireTag(Guid.NewGuid().ToString()),
                                  new SymbolResourceAllocationUnit(
-                                    _option_symbols.Select(i => new mbts.SecurityTuple(SecurityType.Option, i, Venue.Unset)), this, true));
+                                    _option_symbols.Select(i => new mbts.SecurityTuple(SecurityType.Option, i.Item1, i.Item2)), this, true));
          }
       }
 
@@ -95,4 +95,4 @@ namespace mbts.spark {
       #endregion
 
    }
-}
+}

src/client/spark/AW2/Bus/SingleVolSurfaceInterpGreeksPublisher.cs

@@ -25,12 +25,12 @@ namespace mbts.spark {
    public class SingleVolSurfaceInterpGreeksPublisher : IDisposable {
       private readonly int _vs_id;
       private readonly ThreadedBlackboard _bb;
-      private HashSet<string> _all_symbols;
+      private HashSet<(string, Venue)> _all_symbols;
       private VolSurfaceInterpGreeksProducer _theo_producer;
       private readonly ITag<VolSurfaceInput> _in_vs_input_tag;
       private readonly ITag<VolSurfaceNonVolInput> _in_vs_nonvol_input_tag;
       private readonly ITag<VolSurfaceStrikeMap> _in_vs_contracts_tag;
-      private ITag<IReadOnlyDictionary<string, OptionDriverPriceMap>>
+      private ITag<IReadOnlyDictionary<(string, Venue), OptionDriverPriceMap>>
          _in_all_syms_driver_price_tag = SparkTags.AllOptionDriverBBOTag;
       private readonly ITag<VolSurfaceStandardTime> _in_vs_standard_time_tag;
       private readonly ITag<bool> _in_ui_up_tag;
@@ -38,11 +38,11 @@ namespace mbts.spark {
       private readonly PerfStatsLogger _logger;
 
       public SingleVolSurfaceInterpGreeksPublisher(int vs_id,
-               ThreadedBlackboard threaded_bb, IEnumerable<string> symbols) {
+               ThreadedBlackboard threaded_bb, IEnumerable<(string, Venue)> symbols) {
          bts.Log.Gen.info($"SingleVolSurfaceInterpGreeksPublisher cons {vs_id}");
          _vs_id = vs_id;
          _bb = threaded_bb;
-         _all_symbols = new HashSet<string>(symbols);
+         _all_symbols = symbols.ToHashSet();
          _logger = new PerfStatsLogger(10, 100,
             $"SingleVolSurfaceInterpGreeksPublisher_{vs_id}");
          _theo_producer = new VolSurfaceInterpGreeksProducer(vs_id);
@@ -191,4 +191,4 @@ namespace mbts.spark {
       }
       #endregion
    }
-}
+}

src/client/spark/AW2/Bus/SingleVolSurfaceManager.cs

@@ -9,16 +9,17 @@ namespace mbts.spark {
       private readonly int _vs_id;
       private IEnumerable<IDisposable> _resources;
       private readonly ThreadedBlackboard _bb;
+      readonly IBus _bus;
       private readonly AnalyticsService _analytics_service;
       private ITag<bool> _vs_activation_tag;
       private bool _activated = false;
 
-      public SingleVolSurfaceManager(int vs_id, ThreadedBlackboard t_bb,
-                                     AnalyticsService analytics_service) {
+      public SingleVolSurfaceManager(int vs_id, ThreadedBlackboard t_bb, AnalyticsService analytics_service, IBus bus) {
          var tid = System.Threading.Thread.CurrentThread.ManagedThreadId;
          bts.Log.Ui.info($"SingleVolSurfaceManager.cons vs_id:{vs_id} tid:{tid}");
          _vs_id = vs_id;
          _bb = t_bb;
+         _bus = bus;
          _analytics_service = analytics_service;
          _vs_activation_tag = SparkTags.VolSurfaceActiveUITag(vs_id);
          _bb.subscribe(_vs_activation_tag);
@@ -34,7 +35,7 @@ namespace mbts.spark {
             new SingleVolSurfaceDataSource(SingleVolSurfaceDataSource.SymbolListType.Combined,
                _vs_id, _bb, syms => new SingleVolSurfaceNonVolDataSource(_vs_id, syms, _bb)),
             new SingleVolSurfaceDataSource(SingleVolSurfaceDataSource.SymbolListType.Committed, _vs_id, _bb,
-               a_syms => new SingleVolSurfaceCommittedOutputRetriever(_vs_id, _bb, a_syms)),
+               a_syms => new SingleVolSurfaceCommittedOutputRetriever(_vs_id, _bus)),
             new SingleVolSurfaceDataSource(SingleVolSurfaceDataSource.SymbolListType.Combined, _vs_id, _bb,
                a_syms => new SingleVolSurfaceRequestCalculator(
                   SingleVolSurfaceRequestCalculator.RequestType.UI, _vs_id, a_syms, _bb)),
@@ -76,4 +77,4 @@ namespace mbts.spark {
       }
       #endregion
    }
-}
+}

src/client/spark/AW2/Bus/SingleVolSurfaceMarketImpliedVolSource.cs

@@ -1,4 +1,5 @@
-namespace mbts.spark {
+namespace mbts.spark
+{
    using bts;
    using bts.api;
 
@@ -13,29 +14,37 @@
 
    /// This class publishes the market implied vols for the
    /// selected expiration.
-   public class SingleVolSurfaceMarketImpliedVolSource : IDisposable {
+   public class SingleVolSurfaceMarketImpliedVolSource : IDisposable
+   {
       // This keeps the last computed exp and its bbo
       #region MD caching
-      private class CachedExpMD {
+      private class CachedExpMD
+      {
          public SecurityExpiration exp = SecurityExpiration.empty;
          public IDictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>> k_md_map =
             new Dictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>>();
          private bool _has_changed(
                SecurityExpiration new_exp,
-               IDictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>> new_k_md_map) {
-            if (new_exp != exp) {
+               IDictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>> new_k_md_map)
+         {
+            if (new_exp != exp)
+            {
                return true;
             }
-            if (k_md_map.Count() != new_k_md_map.Count) {
+            if (k_md_map.Count() != new_k_md_map.Count)
+            {
                return true;
             }
-            foreach (var kv in k_md_map) {
+            foreach (var kv in k_md_map)
+            {
                Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit> k_md;
-               if (!new_k_md_map.TryGetValue(kv.Key, out k_md)) {
+               if (!new_k_md_map.TryGetValue(kv.Key, out k_md))
+               {
                   return true;
                }
                if (!kv.Value.Item1.Equals(k_md.Item1) ||
-                   !kv.Value.Item2.Equals(k_md.Item2)) {
+                   !kv.Value.Item2.Equals(k_md.Item2))
+               {
                   return true;
                }
             }
@@ -43,9 +52,11 @@
          }
          public bool set(
                SecurityExpiration new_exp,
-               IDictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>> new_k_md_map) {
+               IDictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>> new_k_md_map)
+         {
             bool has_changed = _has_changed(new_exp, new_k_md_map);
-            if (has_changed) {
+            if (has_changed)
+            {
                exp = new_exp;
                k_md_map = new_k_md_map;
             }
@@ -69,7 +80,8 @@
       private TimeSpan _compute_period = TimeSpan.FromMilliseconds(333);
       private bool _is_dirty = true;
 
-      public SingleVolSurfaceMarketImpliedVolSource(int vs_id, ThreadedBlackboard bb) {
+      public SingleVolSurfaceMarketImpliedVolSource(int vs_id, ThreadedBlackboard bb)
+      {
          bts.Log.Gen.info($"SingleVolSurfaceMarketImpliedVolSource cons vs_id:{vs_id}");
          _vs_id = vs_id;
          _bb = bb;
@@ -91,7 +103,8 @@
          _bb.timer_callback += _on_bb_timer;
       }
 
-      private void _on_bb_data(object tag, object data) {
+      private void _on_bb_data(object tag, object data)
+      {
          if (tag != null)
             if (_in_bb_committed_output_tag.Equals(tag) ||
                 _in_bb_calc_resp_tag.Equals(tag) ||
@@ -101,22 +114,26 @@
                _is_dirty = true;
       }
 
-      private void _check_bbo_change() {
+      private void _check_bbo_change()
+      {
          var sc_data = _bb.read_state(SparkTags.SystemContractsTag);
          Tuple<VolSurfaceDisplayGridExpiration, bool> sel_exp_tup =
             _bb.read_state(_in_bb_selected_exp_tag);
-         if (sc_data == null || sel_exp_tup == null) {
+         if (sc_data == null || sel_exp_tup == null)
+         {
             return;
          }
-         VolSurfaceExpiration new_sel_exp = sel_exp_tup.Item1.ToVSExp();
+         VolSurfaceExpiration new_sel_exp = sel_exp_tup.Item1.ToVSExp(SecurityType.Option);
          var new_md = sc_data.get_option_bbo_vsexp(new_sel_exp);
          bool bbo_dirty = _cached_exp_md.set(new_md.Item1, new_md.Item2);
          _is_dirty = _is_dirty || bbo_dirty;
       }
 
       private DateTime _last_publish_time = DateTime.MinValue;
-      private void _on_bb_timer() {
-         if (DateTime.Now - _last_publish_time >= _compute_period) {
+      private void _on_bb_timer()
+      {
+         if (DateTime.Now - _last_publish_time >= _compute_period)
+         {
             _check_bbo_change();
             if (_is_dirty)
                _recompute_and_publish();
@@ -125,18 +142,17 @@
          }
       }
 
-      private void _recompute_and_publish() {
+      private void _recompute_and_publish()
+      {
          Tuple<VolSurfaceDisplayGridExpiration, bool> sel_exp =
             _bb.read_state(_in_bb_selected_exp_tag);
          VolSurfaceOutput vs_output = _resolve_curr_cand();
          if (vs_output != null && vs_output.volsurface_input != null &&
             vs_output.aux_data_map != null &&
-            sel_exp != null && sel_exp.Item1 != null) {
+            sel_exp != null && sel_exp.Item1 != null)
+         {
             var impliedvol_side = _get_market_impliedvol_side(_bb, _vs_id);
-            VolSurfaceExpiration orig_exp = new VolSurfaceExpiration() {
-               symbol = sel_exp.Item1.symbol,
-               expiration = sel_exp.Item1.expiration
-            };
+            var orig_exp = sel_exp.Item1.ToVSExp(SecurityType.Option);
             var timer = Stopwatch.StartNew();
             VolSurfaceExpiration md_exp;
             VolCurveMarketImpliedVolData exp_impl_vol_map =
@@ -152,7 +168,8 @@
                "SingleVolSurfaceMarketImpliedVolSource",
                "SingleVolSurfaceMarketImpliedVolSource",
                timer.Elapsed.TotalMilliseconds);
-            if (exp_impl_vol_map is null) {
+            if (exp_impl_vol_map is null)
+            {
                return;
             }
             bts.Log.Gen.info(
@@ -167,17 +184,21 @@
       }
 
       // Returns the candidate or committed VolSurface depending
-      private VolSurfaceOutput _resolve_curr_cand() {
+      private VolSurfaceOutput _resolve_curr_cand()
+      {
          bool is_dirty = _bb.read_state(_in_bb_inp_dirty_tag, false);
-         if (is_dirty) {
+         if (is_dirty)
+         {
             VolSurfaceCalculationResponse vs_resp = _bb.read_state(_in_bb_calc_resp_tag);
             return vs_resp != null ? vs_resp.volsurface_calculation_output : null;
-         } else
+         }
+         else
             return _bb.read_state(_in_bb_committed_output_tag);
       }
 
       private IDictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>>
-      _get_bbo(VolSurfaceExpiration vs_exp) {
+      _get_bbo(VolSurfaceExpiration vs_exp)
+      {
          return get_option_bbo(_bb, vs_exp);
       }
 
@@ -189,10 +210,14 @@
       // option prices are in BTC
       internal static double get_option_price_mult(
                                                 ThreadedBlackboard bb,
-                                                VolSurfaceExpiration option_exp) {
+                                                VolSurfaceExpiration option_exp)
+      {
+         if (option_exp.venue != Venue.Deribit)
+            return 1.0;
          var driver_map = bb.read_state(SparkTags.AllOptionDriverBBOTag);
          if (driver_map != null &&
-            driver_map.TryGetValue(option_exp.symbol, out var sym_map)) {
+            driver_map.TryGetValue((option_exp.symbol, option_exp.venue), out var sym_map))
+         {
             var vs_md = sym_map.get_driver_md(option_exp.expiration);
             return vs_md.get_driver();
          }
@@ -200,7 +225,8 @@
       }
       internal static
       IDictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>>
-      get_option_bbo(ThreadedBlackboard bb, VolSurfaceExpiration vs_exp) {
+      get_option_bbo(ThreadedBlackboard bb, VolSurfaceExpiration vs_exp)
+      {
          var sc_data = bb.read_state(SparkTags.SystemContractsTag);
          IDictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>> rv =
             sc_data.get_option_bbo_vsexp(vs_exp).Item2;
@@ -213,13 +239,15 @@
       }
 
       internal static bool _ignore_market_implied_vol_source_settings(
-            ThreadedBlackboard bb, int vs_id) {
+            ThreadedBlackboard bb, int vs_id)
+      {
          return bb.read_state(
             SparkTags.VolSurfaceIgnoreImpliedVolSettingsTag(vs_id), false);
       }
 
       internal static VolSurfaceEnums.MarketSide
-      _get_market_impliedvol_side(ThreadedBlackboard bb, int vs_id) {
+      _get_market_impliedvol_side(ThreadedBlackboard bb, int vs_id)
+      {
          return bb.read_state(
             SparkTags.VolSurfaceMarketImpliedVolSide(vs_id),
             VolSurfaceEnums.MarketSide.Mid);
@@ -228,7 +256,8 @@
       #endregion
 
       #region IDisposable
-      public void Dispose() {
+      public void Dispose()
+      {
          _bb.data_callback -= _on_bb_data;
          _bb.timer_callback -= _on_bb_timer;
       }

src/client/spark/AW2/Bus/SingleVolSurfaceNonVolDataSource.cs

@@ -18,15 +18,15 @@ namespace mbts.spark {
    // 2. VolSurfaceStrikeMap
    public class SingleVolSurfaceNonVolDataSource : IDisposable {
       private readonly int _vs_id;
-      private readonly IEnumerable<string> _option_symbols;
+      private readonly IEnumerable<(string, Venue)> _option_symbols;
       private readonly ThreadedBlackboard _bb;
       private readonly ITag<SystemContractData> _in_bb_contract_data_tag;
       private readonly ITag<VolSurfaceNonVolInput> _in_bb_vs_nonvol_input_tag;
-      private readonly ITag<IReadOnlyDictionary<string, OptionDriverPriceMap>> _in_bb_all_driver_price_tag;
+      private readonly ITag<IReadOnlyDictionary<(string, Venue), OptionDriverPriceMap>> _in_bb_all_driver_price_tag;
       private readonly ITag<mbts.analytics.VolSurfaceNonVolData> _out_bb_vs_nonvol_data_tag;
       private readonly ITag<mbts.analytics.VolSurfaceStrikeMap> _out_bb_vs_contracts_tag;
 
-      public SingleVolSurfaceNonVolDataSource(int vs_id, IEnumerable<string> option_symbols,
+      public SingleVolSurfaceNonVolDataSource(int vs_id, IEnumerable<(string, Venue)> option_symbols,
             ThreadedBlackboard bb) {
          bts.Log.Gen.info($"SingleVolSurfaceNonVolDataSource vs_id:{vs_id}");
          _vs_id = vs_id;

src/client/spark/AW2/Bus/SingleVolSurfaceRequestCalculator.cs

@@ -25,7 +25,7 @@ namespace mbts.spark {
       }
       private readonly RequestType _request_type = RequestType.UI;
       private readonly int _vs_id;
-      private readonly HashSet<string> _symbols;
+      private readonly HashSet<(string, Venue)> _symbols;
       private readonly ThreadedBlackboard _bb;
       private readonly ITag<VolSurfaceCalculationRequest> _in_bb_request_tag;
       private readonly ITag<VolSurfaceCalculationRequest> _in_bb_auto_request_tag;
@@ -37,12 +37,12 @@ namespace mbts.spark {
       private VolSurfaceOutput _last_output;
 
       public SingleVolSurfaceRequestCalculator(RequestType request_type,
-                                               int vs_id, IEnumerable<string> symbols,
+                                               int vs_id, IEnumerable<(string, Venue)> symbols,
                                                ThreadedBlackboard threaded_bb) {
          bts.Log.Gen.info($"SingleVolSurfaceRequestCalculator cons vs_id: {vs_id} request_type: {request_type}");
          _request_type = request_type;
          _vs_id = vs_id;
-         _symbols = new HashSet<string>(symbols);
+         _symbols = symbols.ToHashSet();
          _bb = threaded_bb;
          _in_bb_request_tag = SparkTags.VolSurfaceCalculationRequestTag(vs_id);
          _in_bb_auto_request_tag = SparkTags.VolSurfaceAutoCalculationRequestTag(vs_id);

src/client/spark/AW2/Bus/SymbolAggregationPublisher.cs

@@ -4,92 +4,79 @@ using System;
 using System.Collections.Generic;
 using bts.api;
 using System.Linq;
+using System.Reactive.Linq;
+using Spark.Core.Disposables;
+using Spark.Core.Reactive;
 
 namespace mbts.analytics {
 
    // Thin class that assembles all symbol based data published on different
    // tags,  into IReadOnlyDictionary of symbol vs data publsihed under one tag
    public class SymbolAggregationPublisher<TData> : IDisposable {
-      private readonly ThreadedBlackboard _bb;
+      readonly IBus _bus;
       private readonly SecurityType _sec_type;
-      Func<string, ITag<TData>> _get_tag;
+      Func<(string, Venue), ITag<TData>> _get_tag;
       // the tag on which the output will be published
-      ITag<IReadOnlyDictionary<string, TData>> _out_tag;
-      private readonly ITag<SystemContractData> _sc_data_tag;
-      private Dictionary<string, ITag<TData>> _tag_map =
-         new Dictionary<string, ITag<TData>>();
-      private HashSet<object> _obj_tags = new HashSet<object>();
+      ITag<IReadOnlyDictionary<(string, Venue), TData>> _out_tag;
+      Dictionary<(string, Venue), TData> _tag_map = new();
       private bool _dirty = true;
+      readonly object _lock = new();
+      IDisposable _dispose;
+
       public SymbolAggregationPublisher(
-                              ThreadedBlackboard bb,
+                              IBus bus,
                               SecurityType sec_type,
-                              Func<string, ITag<TData>> get_tag,
-                              ITag<IReadOnlyDictionary<string, TData>> out_tag) {
-         _bb = bb;
+                              Func<(string, Venue), ITag<TData>> get_tag,
+                              ITag<IReadOnlyDictionary<(string, Venue), TData>> out_tag,
+                              TimeSpan? contracts_throttle = null) {
+         _bus = bus;
+         var contracts_source = bus.subscribe(SparkTags.SystemContractsTag);
+         if (contracts_throttle.HasValue) {
+            contracts_source = contracts_source.Sample(contracts_throttle.Value);            
+         }
+         var timer_source = Observable.Interval(TimeSpan.FromMilliseconds(133));
+         _dispose = Disposable.Create(
+            contracts_source.SubscribeEach(contracts => Disposable.Create(
+               contracts.sectup_map.Where(st => st.security_type == sec_type).Select(subscribe_sec_data).ToList())),
+            timer_source.Subscribe(_on_timer));
          _sec_type = sec_type;
          _get_tag = get_tag;
          _out_tag = out_tag;
-         _sc_data_tag = SparkTags.SystemContractsTag;
-         _bb.subscribe(SparkTags.SystemContractsTag);
-         _bb.data_callback += _on_bb_data;
-         _bb.timer_callback += _on_timer;
-         _handle_sc_data();
-      }
 
-      private void _on_bb_data(object tag, object data) {
-         if (_sc_data_tag.Equals(tag)) {
-            _handle_sc_data();
-         } else {
-            if (_obj_tags.Contains(tag)) {
-               _dirty = true;
-            }
-         }
-      }
-
-      private void _handle_sc_data() {
-         var sc_data = _bb.read_state(_sc_data_tag);
-         if (sc_data == null) {
-            return;
-         }
-         foreach (var st in sc_data.sectup_map) {
-            if (st.security_type != _sec_type) {
-               continue;
-            }
-            // this is not correct since it is assuming symbol/name is 
-            // unique across securty_tuples
-            string symbol = st.security_name;
-            if (!_tag_map.ContainsKey(symbol)) {
-               var tag = _get_tag(symbol);
-               if (tag != null) {
-                  _tag_map.Add(symbol, tag);
-                  _obj_tags.Add(tag);
-                  _bb.subscribe(tag);
-                  _dirty = true;
+         IDisposable subscribe_sec_data(SecurityTuple st) =>
+            _bus.subscribe(_get_tag((st.security_name, st.security_exchange))).Subscribe(d => {
+               lock (_lock) {
+                  _tag_map[(st.security_name, st.security_exchange)] = d;
                }
-            }
-         }
+               _dirty = true;
+            });
       }
 
-      private void _on_timer() {
+      private void _on_timer(long _) {
          if (_dirty) {
             _dirty = false;
-            Dictionary<string, TData> output = new Dictionary<string, TData>();
-            foreach (var kv in _tag_map) {
-               var symbol = kv.Key;
-               var symbol_tag = kv.Value;
-               TData symbol_data = _bb.read_state(symbol_tag);
-               output.Add(symbol, symbol_data);
+            var output = new Dictionary<(string, Venue), TData>();
+            lock (_lock) {
+               foreach (var kv in _tag_map) {
+                  var symbol = kv.Key;
+                  output.Add(symbol, kv.Value);
+               }
             }
-            IReadOnlyDictionary<string, TData> ro_output = output;
+            IReadOnlyDictionary<(string, Venue), TData> ro_output = output;
             bts.Log.Gen.diag($"SymbolAggregationPublisher._on_timer publishing " +
                              $"tag: {_out_tag} sz: {ro_output.Count}");
-            _bb.post(_out_tag, ro_output);
+            _bus.post(_out_tag, ro_output);
          }
       }
 
       public void Dispose() {
-         _bb.data_callback -= _on_bb_data;
-         _bb.timer_callback -= _on_timer;
+         lock (_lock) {
+            Disposable.dispose(ref _dispose);
+         }
+      }
+
+      class SymbolData {
+         public TData data { get; set; }
       }
    }
 }

src/client/spark/AW2/Bus/VolSurfaceBlackboardManager.cs

@@ -18,6 +18,8 @@ using mbts.spark;
 using System.DirectoryServices.ActiveDirectory;
 using Spark.Core.Protobuf;
 using mbts.ui;
+using bts.config.toggles;
+using System.Diagnostics.CodeAnalysis;
 // using mbts.bb;
 
 namespace mbts.spark
@@ -27,12 +29,27 @@ namespace mbts.spark
       private List<IDisposable> _disps = new List<IDisposable>();
       private Dictionary<SecurityTuple, SingleOptionDriverPricePublisher> _option_driver_map =
          new Dictionary<SecurityTuple, SingleOptionDriverPricePublisher>();
+      readonly IBus bus;
       ThreadedBlackboard _bb;
-      public ResourceAllocator(IBus bus, ThreadedBlackboard bb)
+      readonly object _lock = new object();
+
+      public ResourceAllocator(IBus bus, ThreadedBlackboard bb, TimeSpan? contracts_throttle = null)
       {
          bts.Log.Gen.info("ResourceAllocator started");
+         this.bus = bus;
          _bb = bb;
-         var s = bus.subscribe(SparkTags.SystemContractsTag).OnDispatcher().Subscribe(_on_data);
+         var contracts_source = bus.subscribe(SparkTags.SystemContractsTag);
+         if (contracts_throttle.HasValue)
+         {
+            contracts_source = contracts_source.Sample(contracts_throttle.Value);
+         }
+         var s = contracts_source.Subscribe(contracts =>
+         {
+            lock (_lock)
+            {
+               _on_data(contracts);
+            }
+         });
          _disps.Add(s);
       }
       private void _on_data(SystemContractData d)
@@ -43,13 +60,18 @@ namespace mbts.spark
             if (!_option_driver_map.ContainsKey(sec_tup))
             {
                bts.Log.Gen.info($"ResourceAllocator._on_data allocating SingleOptionDriverPricePublisher for '{sec_tup}'");
-               _option_driver_map.Add(sec_tup, new SingleOptionDriverPricePublisher(sec_tup, _bb));
+               _option_driver_map.Add(sec_tup, new(sec_tup, bus));
             }
          }
       }
       public void Dispose()
       {
-         _disps.ForEach(d => d.Dispose());
+         lock (_lock)
+         {
+            _disps.ForEach(d => d.Dispose());
+            _option_driver_map.Values.DisposeAll();
+            _option_driver_map.Clear();
+         }
       }
    }
    // This is the anologous to the QuoteSetBlackboardManager. There
@@ -100,6 +122,7 @@ namespace mbts.spark
          _bb.subscribe(_in_bb_req_vol);
          _bb.data_callback += _on_bb_update;
          _bb.timer_callback += _on_timer;
+         var contracts_throttle = TimeSpan.FromMilliseconds(200);
          // sub to service
          var vs_sub = _analytics_service.subscribei_volsurfaces()
             .ObserveOn(SynchronizationContext.Current)
@@ -110,18 +133,15 @@ namespace mbts.spark
          _subs.Add(vs_sub);
          _subs.Add(vs_usage_sub);
          _subs.Add(new AnalyticsParametersManager(bus, _analytics_service));
-         _subs.Add(new ResourceAllocator(bus, _bb));
+         _subs.Add(new ResourceAllocator(bus, _bb, contracts_throttle));
          _subs.Add(new SymbolAggregationPublisher<OptionDriverPriceMap>(
-            _bb, SecurityType.Option, sym => SparkTags.OptionDriverBBOTag(sym),
-            SparkTags.AllOptionDriverBBOTag));
+            bus, SecurityType.Option, SparkTags.OptionDriverBBOTag, SparkTags.AllOptionDriverBBOTag, contracts_throttle));
          _subs.Add(new SymbolAggregationPublisher<OptionTheoInterpolationData>(
-            _bb, SecurityType.Option, sym => SparkTags.SymbolInterpGreeksTag(sym),
-            SparkTags.AllInterpGreeks));
+            bus, SecurityType.Option, SparkTags.SymbolInterpGreeksTag, SparkTags.AllInterpGreeks, contracts_throttle));
          _subs.Add(new SymbolAggregationPublisher<ExpStrikeKeyedData<TheoPoint>>(
-            _bb, SecurityType.Option, sym => SparkTags.TC2Tags.LiveTheosTag(sym),
-            SparkTags.TC2Tags.AllLiveTheosTag));
+            bus, SecurityType.Option, SparkTags.TC2Tags.LiveTheosTag, SparkTags.TC2Tags.AllLiveTheosTag, contracts_throttle));
          _subs.Add(new LiveTheoPublisher(_bb));
-         _subs.Add(new InterpTheosServerPublisher(_bb, order_service));
+         _subs.Add(new InterpTheosServerPublisher(_bb, bus, order_service, contracts_throttle));
          _subs.Add(new VolSurfaceInterpGreeksMultiplexer(_bb));
          // usage         
          var list_sub = _bus.subscribe(SparkTags.VolSurfaceListTag).OnDispatcher().Subscribe(t =>
@@ -137,6 +157,15 @@ namespace mbts.spark
             Log.Ui.info($"VolSurfaceUsageList vs_usage acount: {vs_usage.Items.Count}");
             foreach (var u in vs_usage.Items)
             {
+               if (string.Compare(u.OptionName, "SPX", 0) == 0 || string.Compare(u.OptionName, "SPXW", 0) == 0 && u.OptionVenue == Venue.Unset)
+               {
+                  u.OptionVenue = Venue.Cboe;
+               }
+               if (!validate_vs_usage_unit(u, out var error))
+               {
+                  Log.Gen.error($"Invalid VolSurfaceUsageUnit config. error: {error}; value: {u}");
+                  continue;
+               }
                string option_name = u.OptionName;
                string vs_name = u.VsName;
                int vs_key;
@@ -145,10 +174,11 @@ namespace mbts.spark
                   ulist.UsageList.Add(new VolSurfaceUsageEntry()
                   {
                      VsId = vs_key,
+                     SymbolName = option_name,
                      SecTup = new bts.api.SecurityTuple
                      {
                         Type = SecurityType.Option,
-                        Venue = Venue.Deribit,
+                        Venue = u.OptionVenue,
                         Symbol = option_name
                      }
                   });
@@ -177,11 +207,32 @@ namespace mbts.spark
                }
             );
          _disps.Add(theo_sub);
+
+         static bool validate_vs_usage_unit(VolSurfaceUsageUnit vs_usage_unit, [NotNullWhen(false)] out string error)
+         {
+            if (vs_usage_unit.OptionVenue == Venue.Unset)
+            {
+               error = "OptionVenue is unspecified";
+               return false;
+            }
+            if (string.IsNullOrEmpty(vs_usage_unit.OptionName))
+            {
+               error = "OptionName is null/empty";
+               return false;
+            }
+            if (string.IsNullOrEmpty(vs_usage_unit.VsName))
+            {
+               error = "VsName is null/empty";
+               return false;
+            }
+            error = null;
+            return true;
+         }
       }
 
       static IReadOnlyDictionary<uint, VolSurfaceTheoPoint> _assemble_keyed_theos(
          SystemContractData sc_data,
-         IReadOnlyDictionary<string, ExpStrikeKeyedData<TheoPoint>> theos)
+         IReadOnlyDictionary<(string, Venue), ExpStrikeKeyedData<TheoPoint>> theos)
       {
          if (sc_data is null || theos is null)
          {
@@ -203,7 +254,7 @@ namespace mbts.spark
                TheoPoint tp = s_kv.third;
                // TODO: fix exchange later
                Tuple<uint?, uint?> keys =
-                  sc_data.get_option_sec_id(Venue.Deribit, symbol, exp, strike);
+                  sc_data.get_option_sec_id(symbol.Item2, symbol.Item1, exp, strike);
                if (keys != null)
                {
                   var c_key = keys.Item1;
@@ -267,8 +318,7 @@ namespace mbts.spark
          {
             if (!_vs_manager_map.ContainsKey(vs_id))
             {
-               _vs_manager_map.Add(vs_id, new SingleVolSurfaceManager(
-                  vs_id, _bb, _analytics_service));
+               _vs_manager_map.Add(vs_id, new SingleVolSurfaceManager(vs_id, _bb, _analytics_service, _bus));
             }
          }
       }
@@ -355,7 +405,7 @@ namespace mbts.spark
             return;
          }
          mbts.analytics.VolSurfaceSettings vs_settings = _waiting_creations.Dequeue();
-         IDictionary<string, IEnumerable<DateTime>> exp_map = _contract_data.get_option_expirations(VolSurfaceDataFunctions.get_symbols(vs_settings), false);
+         var exp_map = _contract_data.get_option_expirations(VolSurfaceDataFunctions.get_symbols(vs_settings), false);
          VolSurfaceInput vs_input = mbts.analytics.VolSurfaceDataFunctions.get_default_vs_input(vs_settings, exp_map);
          var reply = _analytics_service.get_new_volsurface_id(new VoidMessage());
          if (!reply.Ok)

src/client/spark/AW2/Grid/VolSurfaceDisplayGridExpiration.cs

@@ -1,18 +1,19 @@
 namespace mbts.spark {
    using bts.api;
-   using bts.utils;
 
    using System;
 
    public class VolSurfaceDisplayGridExpiration : IComparable<VolSurfaceDisplayGridExpiration>, IEquatable<VolSurfaceDisplayGridExpiration> {
-      public VolSurfaceDisplayGridExpiration(string symbol, DateTime expiration, analytics.VolSurfaceEnums.VolSurfaceExpirationCalculationMethod calc_mode) {
+      public VolSurfaceDisplayGridExpiration(string symbol, DateTime expiration, Venue venue, analytics.VolSurfaceEnums.VolSurfaceExpirationCalculationMethod calc_mode) {
          this.symbol = symbol;
          this.expiration = expiration;
+         this.venue = venue;
          this.calc_mode = calc_mode;
       }
 
       public string symbol { get; }
       public DateTime expiration { get; }
+      public Venue venue { get; }
       public analytics.VolSurfaceEnums.VolSurfaceExpirationCalculationMethod calc_mode { get; }
 
       /// Slave exps are drawn differently and selecting them does not activate
@@ -24,16 +25,22 @@
          other != null &&
          expiration == other.expiration &&
          symbol == other.symbol &&
+         venue == other.venue &&
          calc_mode == other.calc_mode;
 
       public override int GetHashCode() =>
-         EqualityComparer.auto_phase(symbol.GetHashCode(), expiration.GetHashCode());
+         HashCode.Combine(symbol.GetHashCode(), expiration.GetHashCode(), venue.GetHashCode());
 
       public override bool Equals(object obj) =>
          Equals(obj as VolSurfaceDisplayGridExpiration);
 
-      public override string ToString() =>
-         SecurityFormatter.format_future(symbol, expiration);
+      public override string ToString() {
+         var rv = SecurityFormatter.format_future(symbol, expiration);
+         if (venue != Venue.Unset) {
+            rv = $"{rv} ({venue})";
+         }
+         return rv;
+      }
 
       public static bool operator ==(VolSurfaceDisplayGridExpiration x, VolSurfaceDisplayGridExpiration y) =>
          x is null ? y is null : x.Equals(y);

src/client/spark/AW2/InterpTheosServerPublisher.cs

@@ -8,6 +8,8 @@ using System.Linq;
 using bts.api.orderman;
 using bts.api.analytics;
 using bts;
+using System.Reactive.Linq;
+using Spark.Core.Disposables;
 
 namespace mbts.spark {
    // Listens to the interp theos, decorates the data 
@@ -15,29 +17,52 @@ namespace mbts.spark {
    public class InterpTheosServerPublisher : IDisposable {
       private readonly ThreadedBlackboard _bb;
       private readonly bts.api.orderman.OrderService _order_service;
+      IReadOnlyDictionary<(string Symbol, Venue Venue), OptionTheoInterpolationData> _interp_greeks;
+      IReadOnlyDictionary<(string Symbol, Venue Venue), OptionDriverPriceMap> _driver_bbos;
+      VolSurfaceUsageListImmutable? _vs_usage;
+      UserTuple _user;
+      SystemContractData _system_contracts;
+      IDisposable _dispose;
 
-      public InterpTheosServerPublisher(ThreadedBlackboard bb,
-         bts.api.orderman.OrderService order_service) {
+      public InterpTheosServerPublisher(
+         ThreadedBlackboard bb, IBus bus, OrderService order_service, TimeSpan? contracts_throttle = null) {
          bts.Log.Ui.info(
             $"InterpTheosServerPublisher cons tid:{VolSurfaceTroubleshootingFunctions.tid()}");
          _bb = bb;
          _order_service = order_service;
-         _bb.subscribe(SparkTags.AllInterpGreeks);
-         _bb.subscribe(SparkTags.SystemContractsTag);
-         _bb.subscribe(SparkTags.CurrentUser);
-         _bb.subscribe(SparkTags.VolSurfaceUsageTag);
-         _bb.data_callback += _on_data;
-      }
+         var contracts_source = bus.subscribe(SparkTags.SystemContractsTag);
+         if (contracts_throttle.HasValue) {
+            contracts_source = contracts_source.Sample(contracts_throttle.Value);            
+         }
+         _dispose = Disposable.Create(
+            bus.subscribe(SparkTags.CurrentUser).Subscribe(on_current_user),
+            bus.subscribe(SparkTags.VolSurfaceUsageTag).Subscribe(on_vs_usage),
+            bus.subscribe(SparkTags.AllOptionDriverBBOTag).Subscribe(on_driver_bbos),
+            bus.subscribe(SparkTags.AllInterpGreeks).Subscribe(on_interp_greeks),
+            contracts_source.Subscribe(on_system_contracts));
 
-      private void _on_data(object tag, object data) {
-         if (tag.Equals(SparkTags.SystemContractsTag) ||
-            tag.Equals(SparkTags.AllInterpGreeks)) {
+         void on_current_user(UserTuple user) =>
+            _user = user;
+
+         void on_vs_usage(VolSurfaceUsageListImmutable? vs_usage) =>
+            _vs_usage = vs_usage;
+
+         void on_driver_bbos(IReadOnlyDictionary<(string Symbol, Venue Venue), OptionDriverPriceMap> driver_bbos) =>
+            _driver_bbos = driver_bbos;
+
+         void on_interp_greeks(IReadOnlyDictionary<(string Symbol, Venue Venue), OptionTheoInterpolationData> interp_greeks) {
+            _interp_greeks = interp_greeks;
+            _publish_data();
+         }
+
+         void on_system_contracts(SystemContractData system_contracts) {
+            _system_contracts = system_contracts;
             _publish_data();
          }
       }
 
       private static IDictionary<UInt32, DriverMD> _get_drivers(
-            IReadOnlyDictionary<string, OptionDriverPriceMap> driver_map,
+            IReadOnlyDictionary<(string, Venue), OptionDriverPriceMap> driver_map,
             SystemContractData sc_data) {
          var rv = new Dictionary<UInt32, DriverMD>();
          foreach (var kv in driver_map) {
@@ -48,7 +73,7 @@ namespace mbts.spark {
       }
 
       private static IEnumerable<Tuple<SecurityImmutable, TheoPoint>> _get_theos(
-         IReadOnlyDictionary<string, OptionTheoInterpolationData> theo_data,
+         IReadOnlyDictionary<(string, Venue), OptionTheoInterpolationData> theo_data,
          SystemContractData sc_data) {
          var rv = new List<Tuple<SecurityImmutable, TheoPoint>>();
          foreach (var kv in theo_data) {
@@ -58,20 +83,20 @@ namespace mbts.spark {
          return rv;
       }
 
-      private IReadOnlyDictionary<string, OptionTheoInterpolationData>
-         _filter_for_publish(IReadOnlyDictionary<string, OptionTheoInterpolationData> all_theos) {
+      private IReadOnlyDictionary<(string, Venue), OptionTheoInterpolationData>
+         _filter_for_publish(IReadOnlyDictionary<(string, Venue), OptionTheoInterpolationData> all_theos) {
          // find the symbol vs vs_id, get the committed vs, see if it is
          // owned by the current user, and only then publish it
-         HashSet<string> syms = new HashSet<string>();
-         var usage = _bb.read_state(SparkTags.VolSurfaceUsageTag);
-         string current_user = _bb.read_state(SparkTags.CurrentUser).name;
+         var syms = new HashSet<(string, Venue)>();
+         var usage = _vs_usage;
+         var current_user = _user.name;
          foreach (var u in (usage?.UsageList).empty_if_null()) {
             VolSurfaceInput vs_input =
                _bb.read_state(SparkTags.VolSurfaceCommittedInputTag(u.VsId));
             if (vs_input != null) {
                if (vs_input.surface_settings.core_settings.vs_manager ==
                   current_user) {
-                  syms.Add(u.SecTup.Symbol);
+                  syms.Add((u.SecTup.Symbol, u.SecTup.Venue));
                }
             }
          }
@@ -79,12 +104,12 @@ namespace mbts.spark {
             all_theos.Where(kv => syms.Contains(kv.Key)).ToDictionary(kv => kv.Key, kv => kv.Value);
       }
 
-      private void _publish_data() {
-         SystemContractData sc_data = _bb.read_state(SparkTags.SystemContractsTag, null);
-         IReadOnlyDictionary<string, OptionTheoInterpolationData> all_theos =
-            _bb.read_state(SparkTags.AllInterpGreeks, null);
-         IReadOnlyDictionary<string, OptionDriverPriceMap> all_drivers =
-            _bb.read_state(SparkTags.AllOptionDriverBBOTag, null);
+      void _publish_data() {
+         // copy data to local vars because updates are multi-threaded,
+         // and data must be consistent (during computation).
+         var sc_data = _system_contracts;
+         var all_theos = _interp_greeks;
+         var all_drivers = _driver_bbos;
          if (sc_data == null || all_theos == null || all_drivers == null)
             return;
          var timer = Stopwatch.StartNew();
@@ -132,7 +157,7 @@ namespace mbts.spark {
       }
 
       public void Dispose() {
-         _bb.data_callback -= _on_data;
+         Disposable.dispose(ref _dispose);
       }
    }
 }

src/client/spark/AW2/LiveTheoPublisher.cs

@@ -10,12 +10,13 @@ using System.Windows.Automation;
 using bts.collections;
 
 namespace mbts.spark {
-   // Publishes theos for all options (on the side, also publishes the theos for TC2)
+   // Publishes theos for all options (on the side, also publishes the theos for TC2).
+   // Listens to driver and interp theos, on a change, calls _interp_and_publish
    public class LiveTheoPublisher : IDisposable {
       private ThreadedBlackboard _bb;
-      private readonly ITag<IReadOnlyDictionary<string, OptionTheoInterpolationData>>
+      private readonly ITag<IReadOnlyDictionary<(string, Venue), OptionTheoInterpolationData>>
          _greeks_tag = SparkTags.AllInterpGreeks;
-      private readonly ITag<IReadOnlyDictionary<string, OptionDriverPriceMap>>
+      private readonly ITag<IReadOnlyDictionary<(string, Venue), OptionDriverPriceMap>>
          _drivers_tag = SparkTags.AllOptionDriverBBOTag;
       private readonly PerfStatsLogger _logger = new PerfStatsLogger(50, 100);
       private readonly string _log_id = "LiveTheoPublisher";
@@ -84,17 +85,17 @@ namespace mbts.spark {
       }
 
       private static DriverMD _get_option_driver(
-                  IReadOnlyDictionary<string, OptionDriverPriceMap> driver_map,
+                  IReadOnlyDictionary<(string, Venue), OptionDriverPriceMap> driver_map,
                   SecurityStrike k) {
-         if (driver_map.TryGetValue(k.expiration.symbol, out var exp_map) &&
+         if (driver_map.TryGetValue((k.expiration.symbol, k.expiration.exchange), out var exp_map) &&
             exp_map != null)
             return exp_map.get_driver_md(k.expiration);
          return DriverMD.empty;
       }
       private static TheoPoint _interp_theo(
-         IReadOnlyDictionary<string, OptionTheoInterpolationData> interp_greeks_map,
+         IReadOnlyDictionary<(string, Venue), OptionTheoInterpolationData> interp_greeks_map,
          SecurityStrike k, double driver_price) {
-         if (interp_greeks_map.TryGetValue(k.expiration.symbol, out var interp_greeks) &&
+         if (interp_greeks_map.TryGetValue((k.expiration.symbol, k.expiration.exchange), out var interp_greeks) &&
             interp_greeks != null)
             return interp_greeks.interpolate_theos(k.expiration.expiration, k.strike,
                                                    driver_price);
@@ -102,14 +103,14 @@ namespace mbts.spark {
       }
 
       private void _publish_tc2_theos(IDictionary<SecurityStrike, TheoPoint> theos) {
-         var rv = new Dictionary<string, ExpStrikeKeyedData<TheoPoint>>();
+         var rv = new Dictionary<(string, Venue), ExpStrikeKeyedData<TheoPoint>>();
          foreach (var kv in theos) {
             SecurityStrike k = kv.Key;
             TheoPoint tp = kv.Value;
             ExpStrikeKeyedData<TheoPoint> sym_theos = null;
-            if (!rv.TryGetValue(k.expiration.symbol, out sym_theos)) {
+            if (!rv.TryGetValue((k.expiration.symbol, k.expiration.exchange), out sym_theos)) {
                sym_theos = new ExpStrikeKeyedData<TheoPoint>();
-               rv.Add(k.expiration.symbol, sym_theos);
+               rv.Add((k.expiration.symbol, k.expiration.exchange), sym_theos);
             }
             sym_theos.set_data(k, tp);
          }
@@ -122,10 +123,8 @@ namespace mbts.spark {
       private void _interp_and_publish() {
          var timer = Stopwatch.StartNew();
          var res = new Dictionary<uint, GreeksTuple>();
-         IReadOnlyDictionary<string, OptionDriverPriceMap> driver_map =
-            _bb.read_state(_drivers_tag);
-         IReadOnlyDictionary<string, OptionTheoInterpolationData> interp_greeks =
-            _bb.read_state(SparkTags.AllInterpGreeks);
+         var driver_map = _bb.read_state(_drivers_tag);
+         var interp_greeks = _bb.read_state(SparkTags.AllInterpGreeks);
          SystemContractData sc_data = _bb.read_state(SparkTags.SystemContractsTag);
          if (driver_map == null || interp_greeks == null || sc_data == null) {
             return;
@@ -178,17 +177,10 @@ namespace mbts.spark {
          SystemContractData sc_data = _bb.read_state(SparkTags.SystemContractsTag);
          if (sc_data == null)
             return rv;
-         foreach (var kv in sc_data.exp_map) {
-            var id = kv.Value;
-            SecurityMD sec_md;
-            if (sc_data.sec_md_map.TryGetValue(id, out sec_md)) {
-               if (sec_md.security.HasFuture) {
-                  SecurityExpiration exp = kv.Key;
-                  double pr = sec_md.md.fixing_price;
-                  rv.Add(sec_md.security.Id,
-                        new GreeksTuple(0, pr, pr, 1, 0, 0, 0, 0, pr));
-               }
-            }
+         foreach (var sec_md in sc_data.Futures) {
+            double pr = sec_md.md.fixing_price;
+            rv.Add(sec_md.security.Id,
+                  new GreeksTuple(0, pr, pr, 1, 0, 0, 0, 0, pr));
          }
          return rv;
       }
@@ -202,16 +194,12 @@ namespace mbts.spark {
          SystemContractData sc_data = _bb.read_state(SparkTags.SystemContractsTag);
          if (sc_data == null)
             return;
-         foreach (var kv in sc_data.exp_map) {
-            var id = kv.Value;
-            SecurityMD sec_md;
-            if (sc_data.sec_md_map.TryGetValue(id, out sec_md)) {
-               if (sec_md.security.HasFuture) {
-                  SecurityExpiration exp = kv.Key;
-                  double pr = sec_md.md.fixing_price;
-                  rv.Add(exp, new FutureTheoStructure(exp.expiration, pr, pr, 1, 0));
-               }
-            }
+         foreach (var sec_md in sc_data.Futures) {
+            var sec = sec_md.security;
+            var exp = new SecurityExpiration(
+               sec.Exchange, sec.ContractCase, sec.Symbol, sec.Future.expiration_date());
+            double pr = sec_md.md.fixing_price;
+            rv.Add(exp, new FutureTheoStructure(exp.expiration, pr, pr, 1, 0));
          }
          _bb.post(SparkTags.TC2Tags.LiveFutureTheosTag, rv);
       }

src/client/spark/AW2/Misc/AnalyticsExpirationParametersMap.cs

@@ -6,6 +6,7 @@ using System.Linq;
 using bts.api;
 
 using mbts.analytics;
+using mbts.utils;
 
 
 namespace mbts.spark {
@@ -20,11 +21,16 @@ namespace mbts.spark {
          var empty = SecurityExpiration.empty;
          var temp = new Dictionary<SecurityExpiration,
                                    AnalyticsExpirationParameters>(exp_map_arg);
-         foreach (var kv in sc_data.exp_map) {
-            var exp = kv.Key;
-            if (!temp.ContainsKey(kv.Key)) {
-               temp.Add(kv.Key,
-                  new AnalyticsExpirationParameters(exp, null, empty, empty, 0, 0, 0, 0, 0));
+         foreach (var smd in sc_data.Futures) {
+            var sec = smd.security;
+            var exp = new SecurityExpiration(sec.Exchange, SecurityType.Future, sec.Symbol, sec.Future.expiration_date());
+            if (!temp.ContainsKey(exp)) {
+               temp.Add(exp, new AnalyticsExpirationParameters(exp, null, empty, empty, 0, 0, 0, 0, 0));
+            }
+         }
+         foreach (var sec_exp in sc_data.strike_map.Keys) {
+            if (!temp.ContainsKey(sec_exp)) {
+               temp.Add(sec_exp, new(sec_exp, null, empty, empty, 0, 0, 0, 0, 0));
             }
          }
          exp_map = temp;
@@ -32,14 +38,11 @@ namespace mbts.spark {
 
       public IEnumerable<mbts.SecurityExpiration> get_sorted_exps() {
          List<SecurityExpiration> sorted_exps = new List<SecurityExpiration>(exp_map.Keys);
-         sorted_exps.Sort((e1, e2) => {
-            int rv = e1.expiration.CompareTo(e2.expiration);
-            if (rv == 0)
-               rv = e1.symbol.CompareTo(e2.symbol);
-            if (rv == 0)
-               rv = e1.security_type.ToString().CompareTo(e2.security_type.ToString());
-            return rv;
-         });
+         sorted_exps.Sort(SecurityExpiration.get_comparer(
+            SecurityExpiration.Properties.Expiration,
+            SecurityExpiration.Properties.Symbol,
+            SecurityExpiration.Properties.Type,
+            SecurityExpiration.Properties.Venue));
          return sorted_exps;
       }
 
@@ -88,23 +91,21 @@ namespace mbts.spark {
       // }
 
       // Returns a driver to expiration it drives
-      public IDictionary<mbts.SecurityExpiration, IEnumerable<mbts.SecurityExpiration>>
-      get_option_driver_map() {
-         var rv = new Dictionary<mbts.SecurityExpiration, HashSet<mbts.SecurityExpiration>>();
-         foreach (var kv in exp_map.Where(kv_arg => kv_arg.Key.security_type == SecurityType.Option)) {
-            mbts.SecurityExpiration sec = kv.Key, driver_sec = kv.Value.driver_security;
-            HashSet<mbts.SecurityExpiration> driven_by_this_driver;
-            if (!rv.TryGetValue(driver_sec, out driven_by_this_driver))
-               rv[driver_sec] = driven_by_this_driver = new HashSet<mbts.SecurityExpiration>();
-            driven_by_this_driver.Add(sec);
-         }
-         return rv.ToDictionary(kv => kv.Key, kv => kv.Value as IEnumerable<mbts.SecurityExpiration>);
-      }
+      public IEnumerable<IGrouping<SecurityExpiration, SecurityExpiration>> get_option_driver_map() =>
+         exp_map
+            .Where(kv => kv.Value.driver_security.exchange != Venue.Unset && kv.Key.security_type == SecurityType.Option)
+            .GroupBy(kv => kv.Value.driver_security, kv => kv.Key);
+
+      public IEnumerable<IGrouping<(string symbol, Venue venue), SecurityExpiration>> get_option_driver_symbol_map() =>
+         exp_map
+            .Where(kv => kv.Value.driver_security.exchange != Venue.Unset && kv.Key.security_type == SecurityType.Option)
+            .GroupBy(kv => (kv.Value.driver_security.symbol, kv.Value.driver_security.exchange), kv => kv.Key);
 
       public IDictionary<mbts.SecurityExpiration, mbts.SecurityExpiration>
-      get_option_to_driver_map(string option_name) {
+      get_option_to_driver_map(string option_name, Venue venue) {
          var filt = exp_map.Where(kv => kv.Key.security_type == SecurityType.Option &&
-                                   kv.Key.symbol == option_name);
+                                   kv.Key.symbol == option_name &&
+                                   kv.Key.exchange == venue);
          return filt.ToDictionary(kv => kv.Key, kv => kv.Value.driver_security);
       }
 

src/client/spark/AW2/Misc/OptionDriverPriceMap.cs

@@ -33,8 +33,8 @@ namespace mbts.spark {
          exp_price_map.GetValueOrDefault(exp) ?? DriverMD.empty;
 
       public static IDictionary<VolSurfaceExpiration, DriverMD> get_vs_price_map(
-         IReadOnlyDictionary<string, OptionDriverPriceMap> all_sym_map,
-         Func<string, bool>? sym_filter = null
+         IReadOnlyDictionary<(string, Venue), OptionDriverPriceMap> all_sym_map,
+         Func<(string, Venue), bool>? sym_filter = null
       ) {
          sym_filter ??= _ => true;
          return all_sym_map.SafeWhere(kv => kv.Value is not null && sym_filter(kv.Key))

src/client/spark/AW2/Misc/OptionInterpolationData.cs

@@ -1,4 +1,6 @@
-using System;
+using bts.api;
+
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
@@ -10,11 +12,15 @@ namespace mbts.spark {
    // indexed by option_expiration->driver_price->option_strike->TheoPoint
    public class OptionTheoInterpolationData {
       public readonly string symbol;
+      public readonly Venue venue;
       private IDictionary<DateTime, Dictionary<double, Dictionary<double, TheoPoint>>> _map =
          new Dictionary<DateTime, Dictionary<double, Dictionary<double, TheoPoint>>>();
       private IDictionary<DateTime, double> _zero_move_driver_map = new Dictionary<DateTime, double>();
 
-      public OptionTheoInterpolationData(string symbol) { this.symbol = symbol; }
+      public OptionTheoInterpolationData(string symbol, Venue venue) {
+         this.symbol = symbol;
+         this.venue = venue;
+      }
 
       public void set_data(DateTime option_expiration, double driver_price,
           double move, double option_strike, TheoPoint theo_point) {
@@ -125,4 +131,4 @@ namespace mbts.spark {
 
       #endregion
    }
-}
+}

src/client/spark/AW2/Misc/SecurityNormalizer.cs

@@ -51,15 +51,15 @@ namespace mbts.spark {
                foreach (var k_kv in exp_kv.Value) {
                   double strike = k_kv.Key;
                   TheoPoint tp = k_kv.Value;
-                  IEnumerable<SecurityMD> calls = sc_data.get_options(
-                     null, symbol_theos.symbol, exp, strike, PutOrCall.Call);
-                  IEnumerable<SecurityMD> puts = sc_data.get_options(
-                     null, symbol_theos.symbol, exp, strike, PutOrCall.Put);
-                  foreach (SecurityMD sec_md in calls) {
-                     rv.Add(Tuple.Create(sec_md.security, tp));
+                  var call = sc_data.get_option(
+                     symbol_theos.venue, symbol_theos.symbol, exp, strike, PutOrCall.Call);
+                  var put = sc_data.get_option(
+                     symbol_theos.venue, symbol_theos.symbol, exp, strike, PutOrCall.Put);
+                  if (call.HasValue) {
+                     rv.Add(Tuple.Create(call.Value.security, tp));
                   }
-                  foreach (SecurityMD sec_md in puts) {
-                     rv.Add(Tuple.Create(sec_md.security, tp));
+                  if (put.HasValue) {
+                     rv.Add(Tuple.Create(put.Value.security, tp));
                   }
                }
             }

src/client/spark/AW2/Misc/SystemContractData.cs

@@ -3,7 +3,9 @@ using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Linq;
 
+using bts;
 using bts.api;
+using bts.collections;
 using bts.utils;
 
 using mbts.analytics;
@@ -15,98 +17,97 @@ namespace mbts.spark {
    public class SystemContractData {
       internal readonly IReadOnlyDictionary<UInt32, SecurityMD> sec_md_map;
       public readonly HashSet<SecurityTuple> sectup_map = new HashSet<SecurityTuple>();
-      // for expirations that are contracts themselves (ie non options)
-      public readonly IReadOnlyDictionary<SecurityExpiration, UInt32> exp_map =
-         new Dictionary<SecurityExpiration, UInt32>();
       public readonly IReadOnlyDictionary<SecurityExpiration, HashSet<SecurityStrike>>
          strike_map = new Dictionary<SecurityExpiration, HashSet<SecurityStrike>>();
 
-      private SystemContractData() { }
-      public static readonly SystemContractData empty = new SystemContractData();
+      readonly Lazy<Dictionary<(Venue Venue, string Symbol, DateTime Expiration), SecurityMD>> futuresIndex;
+      readonly Lazy<Dictionary<(Venue Venue, string Symbol, DateTime Expiration, double Strike, PutOrCall CallPut), SecurityMD>> optionsIndex;
+      readonly Lazy<Dictionary<(Venue Venue, SecurityType Type, string Symbol), HashSet<DateTime>>> expirationsIndex;
 
-      class CPKey {
-         public Nullable<UInt32> ckey = null, pkey = null;
-      }
-
-      static void _add_strike(
-            ref Dictionary<SecurityExpiration, Dictionary<double, CPKey>> exp_k_map,
-            SecurityImmutable sec) {
-         bts.Log.assert(sec.HasOption);
-         SecurityExpiration sec_exp = new SecurityExpiration(sec.Exchange,
-            SecurityType.Option, sec.Symbol, sec.Option.expiration_date());
-         Dictionary<double, CPKey> k_map = null;
-         if (!exp_k_map.TryGetValue(sec_exp, out k_map)) {
-            k_map = new Dictionary<double, CPKey>();
-            exp_k_map.Add(sec_exp, k_map);
-         }
-         CPKey cp_key = null;
-         if (!k_map.TryGetValue(sec.Option.Strike, out cp_key)) {
-            cp_key = new CPKey();
-            k_map.Add(sec.Option.Strike, cp_key);
-         }
-         if (sec.Option.CallPut == PutOrCall.Put) {
-            cp_key.pkey = sec.Id;
-         } else {
-            cp_key.ckey = sec.Id;
-         }
-      }
+      public static readonly SystemContractData empty = new(new Dictionary<uint, SecurityMD>(0));
 
       internal SystemContractData(IReadOnlyDictionary<uint, SecurityMD> sec_map_arg) {
          sec_md_map = sec_map_arg;
          HashSet<SecurityTuple> sectup_map_aux = new HashSet<SecurityTuple>();
-         Dictionary<SecurityExpiration, UInt32> exp_map_aux =
-            new Dictionary<SecurityExpiration, uint>();
-         Dictionary<SecurityExpiration, Dictionary<double, CPKey>> strike_map_temp =
-           new Dictionary<SecurityExpiration, Dictionary<double, CPKey>>();
-         foreach (var kv in sec_md_map) {
-            SecurityImmutable sec = kv.Value.security;
-            SecurityTuple sec_tup =
-               new SecurityTuple(sec.security_type(), sec.Symbol, sec.Exchange);
-            sectup_map_aux.Add(sec_tup);
-            SecurityExpiration sec_exp =
-               new SecurityExpiration(sec_tup, sec.security_expiration() ?? default, sec.Id);
-            exp_map_aux[sec_exp] = sec.Id;
-            if (sec.HasOption) {
-               _add_strike(ref strike_map_temp, sec);
+         sectup_map = sec_md_map.Values
+            .Select(smd => new SecurityTuple(smd.security.security_type(), smd.security.Symbol, smd.security.Exchange))
+            .ToHashSet();
+         // Index options
+         strike_map = sec_md_map.Values
+            .Select(smd => smd.security)
+            .Where(s => s.security_type() == SecurityType.Option)
+            .GroupBy(s => new SecurityExpiration(s.Exchange, SecurityType.Option, s.Symbol, s.Option.expiration_date()))
+            .ToDictionary(
+               expGroup => expGroup.Key,
+               expGroup => expGroup.GroupBy(s => s.Option.Strike).Select(strikeGroup => {
+                  // 'strikeGroup' should have 2 securities: the call and the put.
+                  uint? callId = null;
+                  uint? putId = null;
+                  foreach (var s in strikeGroup) {
+                     switch (s.Option.CallPut) {
+                        case PutOrCall.Call:
+                           Log.assert(!callId.HasValue);
+                           callId = s.Id;
+                           break;
+                        case PutOrCall.Put:
+                           Log.assert(!putId.HasValue);
+                           putId = s.Id;
+                           break;
+                        default:
+                           Log.assert(false, $"Option sec has invalid CallPut '{s.Option.CallPut}'");
+                           break;
+                     }
+                  }
+                  return new SecurityStrike(expGroup.Key, strikeGroup.Key, new(callId, putId));
+               })
+               .ToHashSet());
+         // Index exp dates (for futures and options)
+         expirationsIndex = new(() => {
+            var rv = new Dictionary<(Venue Venue, SecurityType Type, string Symbol), HashSet<DateTime>>();
+            foreach (var sec in sec_md_map.Values.Select(s => s.security)) {
+               switch (sec.security_type()) {
+                  case SecurityType.Future:
+                  case SecurityType.Option:
+                     rv.GetOrAdd((sec.Exchange, sec.security_type(), sec.Symbol), _ => new())
+                        .Add(sec.earliest_expiration());
+                     break;
+               }
             }
-         }
-         Dictionary<SecurityExpiration, HashSet<SecurityStrike>> strike_map_aux =
-            new Dictionary<SecurityExpiration, HashSet<SecurityStrike>>();
-         foreach (var kv in strike_map_temp) {
-            SecurityExpiration exp = kv.Key;
-            HashSet<SecurityStrike> k_map = new HashSet<SecurityStrike>();
-            foreach (var kv_i in kv.Value) {
-               double k = kv_i.Key;
-               CPKey cp_key = kv_i.Value;
-               SecurityStrike sec_k =
-                  new SecurityStrike(exp, k, new Tuple<uint?, uint?>(cp_key.ckey, cp_key.pkey));
-               k_map.Add(sec_k);
+            return rv;
+         }, true);
+         // Index futures
+         futuresIndex = new(() => {
+            var rv = new Dictionary<(Venue, string, DateTime), SecurityMD>();
+            foreach (var smd in sec_md_map.Values.Where(smd => smd.security.security_type() == SecurityType.Future)) {
+               var key = (smd.security.Exchange, smd.security.Symbol, smd.security.Future.expiration_date());
+               if (rv.TryGetValue(key, out var existing)) {
+                  continue;
+               }
+               rv.Add(key, smd);
             }
-            strike_map_aux.Add(exp, k_map);
-         }
-         sectup_map = sectup_map_aux;
-         exp_map = exp_map_aux;
-         strike_map = strike_map_aux;
-         _log_data();
+            return rv;
+         },
+         true);
+         // Index options
+         optionsIndex = new(() => {
+            var rv = new Dictionary<(Venue, string, DateTime, double, PutOrCall), SecurityMD>();
+            foreach (var smd in sec_md_map.Values.Where(smd => smd.security.security_type() == SecurityType.Option)) {
+               var sec = smd.security;
+               var key = (sec.Exchange, sec.Symbol, sec.Option.expiration_date(), sec.Option.Strike, sec.Option.CallPut);
+               if (rv.TryGetValue(key, out var existing)) {
+                  continue;
+               }
+               rv.Add(key, smd);
+            }
+            return rv;
+         });
       }
 
-      private void _log_data() {
-         bts.Log.Gen.info(
-            String.Format(
-               "SystemContractData constructed, sec_tups:{0} total_exps:{1} option_exps:{2}",
-               sectup_map.Count, exp_map.Count, strike_map.Count));
-         sectup_map.ForEach(st => bts.Log.Gen.info(
-            String.Format("SystemContractData loaded sectup:{0}", st.ToString())));
-         foreach (var kv in exp_map) {
-            var log_txt = String.Format("SystemContractData loaded exp:{0} id:{1}",
-                                          kv.Key.ToString(), kv.Value);
-            if (kv.Key.security_type == SecurityType.Option) {
-               strike_map.TryGetValue(kv.Key, out var k_map);
-               log_txt += " n_strikes:" + ((k_map == null) ? 0 : k_map.Count);
-            }
-            bts.Log.Gen.info(log_txt);
-         }
-      }
+      public IEnumerable<SecurityMD> Futures =>
+         futuresIndex.Value.Values;
+
+      public ISet<DateTime> GetExpirations(Venue venue, SecurityType type, string symbol) =>
+         expirationsIndex.Value.GetValueOrDefault((venue, type, symbol)) ?? new(0);
 
       public SecurityImmutable? get_option_contract(
          Venue venue, string symbol, DateTime exp, double strike, PutOrCall cp) {
@@ -167,20 +168,19 @@ namespace mbts.spark {
                              new VolSurfaceMDUnit(bid, ask, sec_md.md.fixing_price));
       }
 
-      public Tuple<SecurityImmutable, VolSurfaceMDUnit>
-      get_exp_md(SecurityExpiration sec_exp) {
-         uint sec_id = 0;
-         if (!exp_map.TryGetValue(sec_exp, out sec_id)) {
+      public Tuple<SecurityImmutable, VolSurfaceMDUnit> get_exp_md(SecurityExpiration sec_exp) {
+         Log.assert(sec_exp.security_type == SecurityType.Future);
+         if (!futuresIndex.Value.TryGetValue((sec_exp.exchange, sec_exp.symbol, sec_exp.expiration), out var smd)) {
             return null;
          }
-         return get_sec_md(sec_id);
+         ulong time = 0;
+         smd.md.get_bbo(ref time, out var bid, out var ask);
+         return Tuple.Create(smd.security, new VolSurfaceMDUnit(bid, ask, smd.md.fixing_price));
       }
 
-      internal Tuple<SecurityExpiration,
-                     IDictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>>>
-      get_option_bbo_vsexp(VolSurfaceExpiration exp) {
-         // this is a workaround for VSExp not having an exchange
-         foreach (Venue v in System.Enum.GetValues(typeof(Venue))) {
+      internal Tuple<SecurityExpiration, IDictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>>> get_option_bbo_vsexp(VolSurfaceExpiration exp) {
+         // Prefer bbo from the same venue as the exp, but fallback to another venue if necessary.
+         foreach (Venue v in Enum.GetValues<Venue>().Except(exp.venue).Prepend(exp.venue)) {
             var sec_exp =
                new SecurityExpiration(v, SecurityType.Option, exp.symbol, exp.expiration);
             var rv = get_option_bbo(sec_exp);
@@ -190,7 +190,7 @@ namespace mbts.spark {
          }
          IDictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>> rv_data =
             new Dictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>>();
-         return Tuple.Create(exp.to_security_expiration(type: SecurityType.Option), rv_data);
+         return Tuple.Create(exp.with(security_type: SecurityType.Option).to_security_expiration(), rv_data);
       }
 
       internal IDictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>> get_option_bbo(
@@ -211,33 +211,34 @@ namespace mbts.spark {
          return rv;
       }
 
-      public IDictionary<DateTime, IEnumerable<double>> get_option_strikes(string option_name) {
-         return strike_map.Where(kv => kv.Key.symbol == option_name)
+      public IDictionary<DateTime, IEnumerable<double>> get_option_strikes(string option_name, Venue venue) {
+         return strike_map.Where(kv => kv.Key.symbol == option_name && kv.Key.exchange == venue)
             .ToDictionary(kv => kv.Key.expiration, kv => kv.Value.Select(k => k.strike));
       }
 
       public IDictionary<SecurityExpiration, IEnumerable<double>>
-      get_option_strikes(HashSet<string> option_names) {
-         return strike_map.Where(kv => option_names.Contains(kv.Key.symbol))
+      get_option_strikes(HashSet<(string, Venue)> option_names) {
+         return strike_map.Where(kv => option_names.Contains((kv.Key.symbol, kv.Key.exchange)))
             .ToDictionary(kv => kv.Key, kv => kv.Value.Select(k => k.strike));
       }
 
-      public Nullable<UInt32> get_exp_sec_id(SecurityExpiration sec_exp) {
-         UInt32 rv;
-         if (exp_map.TryGetValue(sec_exp, out rv)) {
-            return rv;
+      public IEnumerable<DateTime> get_option_expirations(string option_name, Venue option_venue) {
+         foreach (var expGroup in expirationsIndex.Value) {
+            var (venue, type, symbol) = expGroup.Key;
+            if (type != SecurityType.Option || symbol != option_name || venue != option_venue) {
+               continue;
+            }
+            foreach (var exp in expGroup.Value) {
+               yield return exp;
+            }
          }
-         return null;
       }
 
-      public IEnumerable<DateTime> get_option_expirations(string option_name,
-         bool is_display_expiration = false) {
-         return exp_map.Keys.Where(e => e.security_type == SecurityType.Option &&
-                                    e.symbol == option_name).Select(e => e.expiration);
-      }
+      public IEnumerable<DateTime> get_option_expirations((string, Venue) option_symbol) =>
+         get_option_expirations(option_symbol.Item1, option_symbol.Item2);
 
-      public IDictionary<string, IEnumerable<DateTime>> get_option_expirations(
-         IEnumerable<string> option_names, bool is_display_expiration) {
+      public IDictionary<(string, Venue), IEnumerable<DateTime>> get_option_expirations(
+         IEnumerable<(string, Venue)> option_names, bool is_display_expiration) {
          return option_names.ToDictionaryDupes(kv => kv, kv => get_option_expirations(kv));
       }
 
@@ -270,17 +271,8 @@ namespace mbts.spark {
          return sec_md_map.Values.Where(sec_md => criteria(sec_md));
       }
 
-      public IEnumerable<SecurityMD> get_options(
-         Venue? exchange, string option_name, DateTime? exp, double? strike, PutOrCall? cp) {
-         return get_securities(sc => {
-            var s = sc.security;
-            return s.security_type() == SecurityType.Option &&
-                  (exchange == null || s.Exchange == exchange) &&
-                  (exp == null || s.earliest_expiration() == exp) &&
-                  (strike == null || s.strike() == strike) &&
-                  (cp == null || s.Option.CallPut == cp);
-         });
-      }
-
+      public SecurityMD? get_option(
+         Venue exchange, string symbol, DateTime exp, double strike, PutOrCall cp) =>
+         optionsIndex.Value.TryGetValue((exchange, symbol, exp, strike, cp), out var smd) ? smd : null;
    }
 }

src/client/spark/AW2/Misc/ThreadedBlackboard.cs

@@ -37,15 +37,6 @@ namespace mbts.spark {
          _bus = bus;
          _bb_name = name;
          subscribe(SparkTags.SparkTimer);
-         System.Threading.Thread timer_thread = new System.Threading.Thread(() => {
-            DateTime start_time = DateTime.Now;
-            while (true) {
-               System.Threading.Thread.Sleep(100);
-               _bus.post(SparkTags.SparkTimer, DateTime.Now - start_time);
-            }
-         });
-         timer_thread.IsBackground = true;
-         timer_thread.Start();
       }
 
       #region Public API
@@ -58,13 +49,17 @@ namespace mbts.spark {
          return (T)rv;
       }
 
-      public void subscribe<T>(ITag<T> tag) {
+      public void subscribe<T>(ITag<T> tag, bool onDispatcher = true) {
          if (_tag_map.Contains(tag)) {
             return;
          }
          _tag_map.Add(tag);
          bts.Log.Ui.info($"ThreadedBlackBoard.subscribe tag:{tag} tid:{_tid()}");
-         var observable = _bus.subscribe(tag).OnDispatcher().Subscribe(
+         var source = _bus.subscribe(tag);
+         if (onDispatcher) {
+            source = source.OnDispatcher();
+         }
+         var observable = source.Subscribe(
             data => _on_bus_data(tag, data));
          _subs.Add(observable);
       }

src/client/spark/AW2/UIAdapters/DriverUIAdapter.cs

@@ -1,5 +1,7 @@
 
 namespace mbts.ui.controls {
+   using bts.api;
+
    using mbts.analytics;
    using mbts.spark;
    using mbts.user.settings;
@@ -56,7 +58,11 @@ namespace mbts.ui.controls {
       {
          foreach (VolSurfaceDisplayGridExpiration disp_exp in exp_disp_grid.get_ordered_expirations())
          {
-            VolSurfaceExpiration exp = new VolSurfaceExpiration() { symbol = disp_exp.symbol, expiration = disp_exp.expiration };
+            var exp = VolSurfaceExpiration.Default.with(
+               symbol: disp_exp.symbol,
+               expiration: disp_exp.expiration,
+               venue: disp_exp.venue,
+               security_type: SecurityType.Option);
             VolCurveParameters curve_parameter = VolSurfaceDataFunctions.get_curve_parameters(vs_input, exp);
             curve_parameter.core_parameters.ref_forward = VolSurfaceDisplayUtilities.parse_double(exp_disp_grid.get_grid_data(disp_exp, _ref_fwd_col, Double.NaN));
             curve_parameter.core_parameters.driver_override = VolSurfaceDisplayUtilities.parse_double(exp_disp_grid.get_grid_data(disp_exp, _driver_override_col, Double.NaN));

src/client/spark/AW2/UIAdapters/DualQuadraticModelUIAdapter.cs

@@ -1,4 +1,6 @@
 namespace mbts.ui.controls {
+   using bts.api;
+
    using mbts.analytics;
    using mbts.spark;
    using mbts.user.settings;
@@ -68,7 +70,7 @@
       {
          foreach (VolSurfaceDisplayGridExpiration disp_exp in exp_disp_grid.get_ordered_expirations())
          {
-            VolSurfaceExpiration exp = new VolSurfaceExpiration() { symbol = disp_exp.symbol, expiration = disp_exp.expiration };
+            var exp = disp_exp.ToVSExp(SecurityType.Option);
             VolCurveParameters input_general_parameters = VolSurfaceDataFunctions.get_curve_parameters(vs_input, exp);
             DualQuadraticModelVolCurveParameters dq_parameters = input_general_parameters.dualquadratic_model_parameters;
             /// atm

src/client/spark/AW2/UIAdapters/ExplicitSlopeWingUIAdapter.cs

@@ -1,4 +1,6 @@
 namespace mbts.ui.controls {
+   using bts.api;
+
    using mbts.analytics;
    using mbts.spark;
 
@@ -41,7 +43,7 @@
       {
          foreach (VolSurfaceDisplayGridExpiration disp_exp in exp_disp_grid.get_ordered_expirations())
          {
-            VolSurfaceExpiration exp = new VolSurfaceExpiration() { symbol = disp_exp.symbol, expiration = disp_exp.expiration };
+            var exp = disp_exp.ToVSExp(SecurityType.Option);
             VolCurveParameters curve_parameter = VolSurfaceDataFunctions.get_curve_parameters(vs_input, exp);
             curve_parameter.core_parameters.put_wing_parameters.explicit_slope_wing_parameters.slope = VolSurfaceDisplayUtilities.parse_double(exp_disp_grid.get_grid_data(disp_exp, _p_wing_slope, Double.NaN));
             curve_parameter.core_parameters.call_wing_parameters.explicit_slope_wing_parameters.slope = VolSurfaceDisplayUtilities.parse_double(exp_disp_grid.get_grid_data(disp_exp, _c_wing_slope, Double.NaN));

src/client/spark/AW2/UIAdapters/ImplicitSlopeWingUIAdapter.cs

@@ -1,5 +1,7 @@
 namespace mbts.ui.controls {
-using mbts.analytics;
+   using bts.api;
+
+   using mbts.analytics;
 using mbts.spark;
 using mbts.user.settings;
 
@@ -40,7 +42,11 @@ using System.Collections.Generic;
          /// just for the input values, which in this case are all
          foreach (VolSurfaceDisplayGridExpiration disp_exp in exp_disp_grid.get_ordered_expirations())
          {
-            VolSurfaceExpiration exp = new VolSurfaceExpiration() { symbol = disp_exp.symbol, expiration = disp_exp.expiration };
+            var exp = VolSurfaceExpiration.Default.with(
+               symbol: disp_exp.symbol,
+               expiration: disp_exp.expiration,
+               venue: disp_exp.venue,
+               security_type: SecurityType.Option);
             VolCurveParameters curve_parameter = VolSurfaceDataFunctions.get_curve_parameters(vs_input, exp);
             curve_parameter.core_parameters.put_wing_parameters.simple_slope_wing_parameters.max_vol = VolSurfaceDisplayUtilities.parse_double(exp_disp_grid.get_grid_data(disp_exp, _p_wing_max, Double.NaN));
             curve_parameter.core_parameters.call_wing_parameters.simple_slope_wing_parameters.max_vol = VolSurfaceDisplayUtilities.parse_double(exp_disp_grid.get_grid_data(disp_exp, _c_wing_max, Double.NaN));

src/client/spark/AW2/UIAdapters/MiscColsUIAdapter.cs

@@ -1,5 +1,7 @@
 
 namespace mbts.ui.controls {
+   using bts.api;
+
    using mbts.analytics;
    using mbts.spark;
    using mbts.user.settings;
@@ -64,7 +66,7 @@ namespace mbts.ui.controls {
       {
          foreach (VolSurfaceDisplayGridExpiration disp_exp in exp_disp_grid.get_ordered_expirations())
          {
-            VolSurfaceExpiration exp = new VolSurfaceExpiration() { symbol = disp_exp.symbol, expiration = disp_exp.expiration };
+            var exp = disp_exp.ToVSExp(SecurityType.Option);
             VolCurveParameters curve_parameter = VolSurfaceDataFunctions.get_curve_parameters(vs_input, exp);
             curve_parameter.core_parameters.time_shift = VolSurfaceDisplayUtilities.parse_double(exp_disp_grid.get_grid_data(disp_exp, _time_shift_col, 0.0));
             //curve_parameter.core_parameters.unhinge_exp_from_master = VolSurfaceDisplayUtilities.parse_boolean(exp_disp_grid.get_grid_data(disp_exp, _slave_unhinged_col, false), false);

src/client/spark/AW2/UIAdapters/ParabolicVolPathUIAdapter.cs

@@ -1,4 +1,6 @@
 namespace mbts.ui.controls {
+   using bts.api;
+
    using mbts.analytics;
    using mbts.spark;
    using mbts.user.settings;
@@ -44,7 +46,7 @@
       {
          foreach (VolSurfaceDisplayGridExpiration disp_exp in exp_disp_grid.get_ordered_expirations())
          {
-            VolSurfaceExpiration exp = new VolSurfaceExpiration() { symbol = disp_exp.symbol, expiration = disp_exp.expiration };
+            var exp = disp_exp.ToVSExp(SecurityType.Option);
             ParabolicVolPathParameters exp_params = new ParabolicVolPathParameters()
             {
                down_move_point = Tuple.Create (VolSurfaceDisplayUtilities.parse_double(exp_disp_grid.get_grid_data(disp_exp, _down_market_move, Double.NaN)),

src/client/spark/AW2/UIAdapters/PolynomialModelUIAdapter.cs

@@ -1,4 +1,6 @@
 namespace mbts.ui.controls {
+   using bts.api;
+
    using mbts.analytics;
    using mbts.spark;
    using mbts.user.settings;
@@ -107,7 +109,7 @@
       {
          foreach (VolSurfaceDisplayGridExpiration disp_exp in exp_disp_grid.get_ordered_expirations())
          {
-            VolSurfaceExpiration exp = new VolSurfaceExpiration() { symbol = disp_exp.symbol, expiration = disp_exp.expiration };
+            var exp = disp_exp.ToVSExp(SecurityType.Option);
             PolynomialModelVolCurveParameters input_polynomial_parameters = VolSurfaceDataFunctions.get_polynomial_curve_parameters(vs_input, exp);
             VolCurveParameters input_general_parameters = VolSurfaceDataFunctions.get_curve_parameters(vs_input, exp);
             for (uint p_point_no = 0; p_point_no < _put_cols.Count() ; p_point_no++)

src/client/spark/AW2/UIAdapters/SplineModelUIAdapter.cs

@@ -1,4 +1,6 @@
 namespace mbts.ui.controls {
+   using bts.api;
+
    using mbts.analytics;
    using mbts.spark;
    using mbts.user.settings;
@@ -111,7 +113,11 @@
       {
          foreach (VolSurfaceDisplayGridExpiration disp_exp in exp_disp_grid.get_ordered_expirations())
          {
-            VolSurfaceExpiration exp = new VolSurfaceExpiration() { symbol = disp_exp.symbol, expiration = disp_exp.expiration };
+            var exp = VolSurfaceExpiration.Default.with(
+               symbol: disp_exp.symbol,
+               expiration: disp_exp.expiration,
+               venue: disp_exp.venue,
+               security_type: SecurityType.Option);
             SplineModelVolCurveParameters input_spline_parameters = VolSurfaceDataFunctions.get_spline_curve_parameters(vs_input, exp);
             VolCurveParameters input_general_parameters = VolSurfaceDataFunctions.get_curve_parameters(vs_input, exp);
             for (uint p_point_no = 0; p_point_no < _put_cols.Count(); p_point_no++)

src/client/spark/AW2/UIAdapters/StrikeConversionUIAdapter.cs

@@ -1,4 +1,6 @@
 namespace mbts.ui.controls {
+   using bts.api;
+
    using mbts.analytics;
    using mbts.spark;
    using mbts.user.settings;
@@ -43,7 +45,7 @@
       {
          foreach (VolSurfaceDisplayGridExpiration disp_exp in exp_disp_grid.get_ordered_expirations())
          {
-            VolSurfaceExpiration exp = new VolSurfaceExpiration() { symbol = disp_exp.symbol, expiration = disp_exp.expiration };
+            var exp = disp_exp.ToVSExp(SecurityType.Option);
             VolCurveParameters curve_parameter = VolSurfaceDataFunctions.get_curve_parameters(vs_input, exp);
             curve_parameter.core_parameters.conversion_fwd_slider = VolSurfaceDisplayUtilities.parse_double(exp_disp_grid.get_grid_data(disp_exp, _conv_fwd_slider_col, 0.0));
             curve_parameter.core_parameters.conversion_vol_slider = VolSurfaceDisplayUtilities.parse_double(exp_disp_grid.get_grid_data(disp_exp, _conv_vol_slider_col, 0.0));

src/client/spark/AW2/UIAdapters/TwoWaySlopeVolPathUIAdapter.cs

@@ -1,4 +1,6 @@
 namespace mbts.ui.controls {
+   using bts.api;
+
    using mbts.analytics;
    using mbts.spark;
    using mbts.user.settings;
@@ -49,7 +51,7 @@
       {
          foreach (VolSurfaceDisplayGridExpiration disp_exp in exp_disp_grid.get_ordered_expirations())
          {
-            VolSurfaceExpiration exp = new VolSurfaceExpiration() { symbol = disp_exp.symbol, expiration = disp_exp.expiration };
+            var exp = disp_exp.ToVSExp(SecurityType.Option);
             TwoWaySlopeVolPathParameters exp_params = new TwoWaySlopeVolPathParameters()
             {
                vol_move_for_unit_fwd_down = VolSurfaceDisplayUtilities.parse_double (exp_disp_grid.get_grid_data(disp_exp, _down_slope_col, Double.NaN)),

src/client/spark/AW2/UIAdapters/VolSurfaceUIAdapter.cs

@@ -8,6 +8,7 @@ using mbts.analytics;
 using mbts;
 using mbts.spark;
 using bts;
+using bts.api;
 
 namespace mbts.ui.controls {
    class VolSurfaceUIAdapter {
@@ -91,7 +92,7 @@ namespace mbts.ui.controls {
          IDictionary<VolSurfaceDisplayGridExpiration, IDictionary<VolSurfaceDisplayGridColumn, object>> exp_display_map = new Dictionary<VolSurfaceDisplayGridExpiration, IDictionary<VolSurfaceDisplayGridColumn, object>>();
          foreach (KeyValuePair<VolSurfaceExpiration, IDictionary<VolSurfaceDisplayGridColumn, object>> p in exp_map) {
             VolCurveOutput vc_out = vs_output.exp_output_map.TryGetValue(p.Key, out vc_out) ? vc_out : null;
-            VolSurfaceDisplayGridExpiration disp_exp = new VolSurfaceDisplayGridExpiration(p.Key.symbol, p.Key.expiration, vc_out != null ? vc_out.calc_method : VolSurfaceEnums.VolSurfaceExpirationCalculationMethod.Master);
+            VolSurfaceDisplayGridExpiration disp_exp = new VolSurfaceDisplayGridExpiration(p.Key.symbol, p.Key.expiration, p.Key.venue, vc_out != null ? vc_out.calc_method : VolSurfaceEnums.VolSurfaceExpirationCalculationMethod.Master);
             exp_display_map.Add(disp_exp, p.Value);
          }
          exp_grid = new VolSurfaceDisplayGrid(exp_display_map.Keys.OrderBy(t => t), ordered_cols, exp_display_map);
@@ -103,7 +104,8 @@ namespace mbts.ui.controls {
       public static VolSurfaceInput copy_ui_to_input(VolSurfaceDisplayGrid exp_disp_grid, VolSurfaceSettings settings, VolSurfaceMacroGridSettings grid_settings) {
          // HighPerformanceTimer timer = new HighPerformanceTimer();
          // timer.start();
-         VolSurfaceInput rv = VolSurfaceDataFunctions.get_default_vs_input(settings, exp_disp_grid.get_ordered_expirations().Select(exp => new VolSurfaceExpiration() { expiration = exp.expiration, symbol = exp.symbol }));
+         var rv = VolSurfaceDataFunctions.get_default_vs_input(settings, exp_disp_grid.get_ordered_expirations()
+            .Select(exp => new VolSurfaceExpiration(SecurityType.Option, exp.symbol, exp.expiration, exp.venue)));
          IEnumerable<IReadWriteGridAdapter> grid_adapters = _get_grid_adapters(settings, grid_settings);
          foreach (IReadWriteGridAdapter g in grid_adapters)
             g.copy_grid_to_data(exp_disp_grid, settings, ref rv);
@@ -117,7 +119,7 @@ namespace mbts.ui.controls {
                                              out IDictionary<VolSurfaceDisplayGridExpiration, VolCurveDisplayGraph> exp_graph_map) {
          exp_graph_map = new Dictionary<VolSurfaceDisplayGridExpiration, VolCurveDisplayGraph>();
          foreach (KeyValuePair<VolSurfaceExpiration, VolCurveOutput> p in vs_output.exp_output_map) {
-            VolSurfaceDisplayGridExpiration exp = new VolSurfaceDisplayGridExpiration(p.Key.symbol, p.Key.expiration, p.Value.calc_method);
+            var exp = new VolSurfaceDisplayGridExpiration(p.Key.symbol, p.Key.expiration, p.Key.venue, p.Value.calc_method);
             VolCurveOutput curve_output = p.Value;
             IEnumerable<Tuple<double, double>> exp_graph_series = curve_output.strike_vol_map.Select(kvp => Tuple.Create(kvp.Key, kvp.Value)).OrderBy(t => t.Item1);
             exp_graph_map[exp] = new VolCurveDisplayGraph(exp_graph_series.OrderBy(t => t.Item1),

src/client/spark/AW2/ViewModel/ExpSelectionGridVM.cs

@@ -15,6 +15,7 @@ using System.Collections.Generic;
 using System.Collections.ObjectModel;
 using System.Collections.Specialized;
 using System.Linq;
+using bts.api;
 
 namespace mbts.spark {
    public interface IExpirationBasedRow {
@@ -48,15 +49,15 @@ namespace mbts.spark {
             expirations.sync_with(_model.get_ordered_expirations());
 
             if (added.Any() && !removed.Any()) {
-               visible_rows.sync_with(rows.Where(r => !_hidden_exps.Contains(r.expiration.ToVSExp())).Cast<object>());
+               visible_rows.sync_with(rows.Where(r => !_hidden_exps.Contains(r.expiration.ToVSExp(SecurityType.Option))).Cast<object>());
             } else {
-               var vs_exps = expirations.Select(exp => exp.ToVSExp());
-               var vs_adds = added.Select(exp => exp.ToVSExp());
-               var vs_removes = removed.Select(exp => exp.ToVSExp());
+               var vs_exps = expirations.Select(exp => exp.ToVSExp(SecurityType.Option));
+               var vs_adds = added.Select(exp => exp.ToVSExp(SecurityType.Option));
+               var vs_removes = removed.Select(exp => exp.ToVSExp(SecurityType.Option));
 
-               var true_removed = removed.Where(rmv => !vs_adds.Contains(rmv.ToVSExp()));
-               var true_added = added.Except(vs_removes.Concat(_hidden_exps), v => v.ToVSExp());
-               var changed_calc_modes = added.Where(add => vs_removes.Contains(add.ToVSExp()));
+               var true_removed = removed.Where(rmv => !vs_adds.Contains(rmv.ToVSExp(SecurityType.Option)));
+               var true_added = added.Except(vs_removes.Concat(_hidden_exps), v => v.ToVSExp(SecurityType.Option));
+               var changed_calc_modes = added.Where(add => vs_removes.Contains(add.ToVSExp(SecurityType.Option)));
 
                foreach (var tr in true_removed) {
                   int index = visible_rows.Cast<IExpirationBasedRow>().Select(r => r.expiration).indexOf(tr);
@@ -73,8 +74,8 @@ namespace mbts.spark {
                foreach (var ccm in changed_calc_modes) {
                   int index = visible_rows
                      .Cast<IExpirationBasedRow>()
-                     .Select(r => r.expiration.ToVSExp())
-                     .indexOf(ccm.ToVSExp());
+                     .Select(r => r.expiration.ToVSExp(SecurityType.Option))
+                     .indexOf(ccm.ToVSExp(SecurityType.Option));
                   if (index < 0 || index >= visible_rows.Count) {
                      continue;
                   }
@@ -105,12 +106,12 @@ namespace mbts.spark {
       void _visible_rows_changed(object sender, NotifyCollectionChangedEventArgs e) {
          if (e.Action == NotifyCollectionChangedAction.Remove) {
             foreach (IExpirationBasedRow row in e.OldItems)
-               if (!_hidden_exps.Contains(row.expiration.ToVSExp()))
-                  _hidden_exps.Add(row.expiration.ToVSExp());
+               if (!_hidden_exps.Contains(row.expiration.ToVSExp(SecurityType.Option)))
+                  _hidden_exps.Add(row.expiration.ToVSExp(SecurityType.Option));
          } else if (e.Action == NotifyCollectionChangedAction.Add)
             foreach (IExpirationBasedRow row in e.NewItems)
-               if (_hidden_exps.Contains(row.expiration.ToVSExp()))
-                  _hidden_exps.Remove(row.expiration.ToVSExp());
+               if (_hidden_exps.Contains(row.expiration.ToVSExp(SecurityType.Option)))
+                  _hidden_exps.Remove(row.expiration.ToVSExp(SecurityType.Option));
       }
    }
 }

src/client/spark/AW2/ViewModel/Graph/VolSurfaceGraphVM.cs

@@ -176,7 +176,7 @@ namespace mbts.spark {
 
       public IEnumerable<VolSurfaceExpiration> get_visible_series() {
          if (_earliest_selected_expiration != null) {
-            yield return _earliest_selected_expiration.ToVSExp();
+            yield return _earliest_selected_expiration.ToVSExp(SecurityType.Option);
          }
       }
 
@@ -323,7 +323,7 @@ namespace mbts.spark {
          if (show_implied_vol && _md_exp != null && _md_exp.symbol != _expiration.symbol) {
             var suffix = _md_exp.expiration == _expiration.expiration ? _md_exp.symbol :
                new VolSurfaceDisplayGridExpiration(
-                  _md_exp.symbol, _md_exp.expiration, VolSurfaceEnums.VolSurfaceExpirationCalculationMethod.Master)
+                  _md_exp.symbol, _md_exp.expiration, _md_exp.venue, VolSurfaceEnums.VolSurfaceExpirationCalculationMethod.Master)
                   .ToString();
             return $"{_expiration} ({suffix})";
          }

src/client/spark/AW2/ViewModel/Grid/VolSurfaceGridVM.cs

@@ -26,6 +26,7 @@ namespace mbts.spark.VolSurface {
    using System.Windows.Data;
    using System.Windows.Input;
    using System.Windows.Media;
+   using bts.api;
 
    class VolSurfaceGridVM : DisposableViewModelBase {
       readonly VolSurfaceVM _parent;
@@ -62,7 +63,7 @@ namespace mbts.spark.VolSurface {
          this.model.on_committed_grid_change += _on_grid_updated;
 
          current_grid = this.model.get_active_grid();
-         committed_grid = is_dirty ? this.model.get_committed_grid() : current_grid;
+         committed_grid = this.model.get_committed_grid();
          _columns_model.models.SetItems(current_grid.get_ordered_columns());
 
          // Rows
@@ -263,7 +264,7 @@ namespace mbts.spark.VolSurface {
          if (old_row_count == 0) {
             // Initial data download
             if (_hidden_expirations != null) {
-               visible_rows.AddRange(rows.Where(r => !_hidden_expirations.Contains(r.expiration.ToVSExp())));
+               visible_rows.AddRange(rows.Where(r => !_hidden_expirations.Contains(r.expiration.ToVSExp(SecurityType.Option))));
                _hidden_expirations = null;
             } else {
                visible_rows.AddRange(rows);

src/client/spark/AW2/ViewModel/Grid/VolSurfaceRowContextMenuVM.cs

@@ -1,6 +1,7 @@
 #nullable enable
 namespace mbts.spark {
    using bts;
+   using bts.api;
    using bts.collections;
 
    using mbts.spark.VolSurface;
@@ -170,7 +171,7 @@ namespace mbts.spark {
 
       public ICommand smart_copy_rolls_rates_cmd =>
          _smart_copy_rolls_rates_cmd ??= new Command(
-            () => _parent.model.sync_slave_ref_forwards(expiration.ToVSExp()));
+            () => _parent.model.sync_slave_ref_forwards(expiration.ToVSExp(SecurityType.Option)));
 
       public ICommand import_csv_cmd =>
          _import_csv_cmd ??= FileDialogCommand.open_file(

src/client/spark/AW2/ViewModel/SettingsEditors/VolSurfaceDriverGridVM.cs

@@ -0,0 +1,30 @@
+namespace mbts.spark;
+
+using mbts.ui.utility;
+
+using Spark.Core.Disposables;
+
+using System;
+using System.Collections.ObjectModel;
+
+public class VolSurfaceDriverGridVM : IDisposable {
+   IDisposable _slaves_hook;
+
+   public VolSurfaceDriverGridVM(VolSurfaceDriverVM driver) {
+      this.driver = driver;
+      _slaves_hook = ObservableAutoWrapper.connect(
+         driver.symbols,
+         slaves,
+         sym => new VolSurfaceSlaveVM(driver, new(bts.api.SecurityType.Option, sym.symbol, sym.venue)),
+         sym_vm => sym_vm.Dispose());
+   }
+
+   public VolSurfaceDriverVM driver { get; }
+   public string header => $"{driver.name} ({driver.exchange})";
+   public ObservableCollection<VolSurfaceSlaveVM> slaves { get; } = new();
+
+   public void Dispose() {
+      Disposable.dispose(ref _slaves_hook);
+      slaves.DisposeAll(true);
+   }
+}

src/client/spark/AW2/ViewModel/SettingsEditors/VolSurfaceDriverVM.cs

@@ -0,0 +1,28 @@
+namespace mbts.spark;
+
+using bts.api;
+
+using mbts.analytics;
+
+using System;
+using System.Collections.ObjectModel;
+
+public class VolSurfaceDriverVM : IEquatable<VolSurfaceDriverVM> {
+   public VolSurfaceDriverVM(Venue exchange, string name) {
+      this.exchange = exchange;
+      this.name = name;
+   }
+
+   public string name { get; }
+   public Venue exchange { get; }
+   public ObservableCollection<VolSurfaceSymbolVenueVM> symbols { get; } = new();
+
+   public bool Equals(VolSurfaceDriverVM other) =>
+      other != null && other.name == name && other.exchange == exchange;
+
+   public override bool Equals(object obj) =>
+      Equals(obj as VolSurfaceDriverVM);
+
+   public override int GetHashCode() =>
+      name.GetHashCode();
+}

src/client/spark/AW2/ViewModel/SettingsEditors/VolSurfaceSettingsEditorVM.cs

@@ -3,28 +3,27 @@ namespace mbts.spark {
    using bts.api;
    using bts.collections;
 
-   using mbts;
    using mbts.analytics;
    using mbts.ui;
    using mbts.ui.shell;
    using mbts.ui.utility;
+   using mbts.utils;
 
    using Spark.Core.Disposables;
    using Spark.Core.Protobuf;
 
    using System;
    using System.Collections.Generic;
-   using System.Collections.Immutable;
    using System.Collections.ObjectModel;
-   using System.Collections.Specialized;
    using System.Linq;
    using System.Reactive.Linq;
    using System.Windows.Input;
 
-   public class VolSurfaceSettingsEditorVM : ViewModelBase {
+   public sealed class VolSurfaceSettingsEditorVM : DisposableViewModelBase {
       readonly uint _atm_shifts_id = 0;
       readonly VolSurfaceSettings _settings;
       readonly VolSurfaceCoreSettings _core_settings;
+      readonly List<IDisposable> _subscriptions = new(1);
 
       string _selected_section;
       bool _can_edit_name = true;
@@ -35,7 +34,7 @@ namespace mbts.spark {
          }
          _settings = settings.ensure_no_nulls();
          _core_settings = settings.core_settings;
-         symbol_structure_vm = new VSSymbolStructureVM(bus, settings);
+         symbol_structure_vm = new VolSurfaceSymbolStructureVM(settings);
          vol_model_polynomial_vm.load_from(settings.polynomial_model_settings);
          vol_model_cubic_spline_vm.load_from(settings.spline_model_settings);
          vol_model_dual_quadratic_vm.load_from(settings.dualquadratic_model_settings);
@@ -69,23 +68,26 @@ namespace mbts.spark {
 
          this.commit = new DelegateCommand(commit_impl);
 
+         _dispose = Disposable.Create(
+            bus.subscribe(SparkTags.ExpirationParametersInputTag).OnDispatcher().Subscribe(symbol_structure_vm.on_exp_params_input),
+            symbol_structure_vm);
+
          void commit_impl() {
             var symbols = symbol_structure_vm.driver_grids
-               .SelectMany(dg => dg.symbols.Where(sym => sym.is_selected))
-               .ToList();
-            _core_settings.master_slave_relationships = symbols
-               .Where(sym => sym.masters.Any())
-               .Select(sym => new VolSurfaceSymbolRelationship {
-                  symbol = sym.name,
-                  connected_symbols = sym.masters.Cast<string>().ToList()
-               })
+               .SelectMany(dg => dg.slaves.Where(sym => sym.is_selected))
                .ToList();
+            _core_settings.master_slave_relationships.SetItems(symbols
+               .Where(sym => sym.selected_master_symbols.Any())
+               .Select(sym => VolSurfaceSymbolRelationship.create(
+                  (sym.name, sym.venue),
+                  sym.selected_master_symbols.Select(s => new SymbolVenue() { symbol = s.security_name, venue = s.security_exchange })
+               )));
             _core_settings.implied_vol_marketdata_relationships = symbols
-               .Where(sym => sym.fits.Any())
-               .Select(sym => new VolSurfaceSymbolRelationship {
-                  symbol = sym.name,
-                  connected_symbols = sym.fits.Cast<string>().ToList()
-               })
+               .Where(sym => sym.selected_fits.Any())
+               .Select(sym => VolSurfaceSymbolRelationship.create(
+                  (sym.name, sym.venue),
+                  sym.selected_fits.Select(s => new SymbolVenue() { symbol = s.security_name, venue = s.security_exchange})
+               ))
                .ToList();
 
             vol_model_polynomial_vm.assign_to(_settings.polynomial_model_settings);
@@ -93,7 +95,9 @@ namespace mbts.spark {
             vol_model_dual_quadratic_vm.assign_to(_settings.dualquadratic_model_settings);
             vol_path_parabolic_vm.assign_to(_settings.core_settings.volpath_settings.parabolic_vol_path_settings);
 
-            _core_settings.symbol_pricing_model_map = symbols.Select(sym => Tuple.Create(sym.name, sym.pricing_model)).ToList();
+            foreach (var sym in symbols) {
+               _core_settings.symbol_pricing_model_map[(sym.name, sym.venue)] = sym.pricing_model;
+            }
             _core_settings.put_rotation.strike_vol_movement_pairs = put_rotations
                .Select(r => Tuple.Create(r.target_strike, r.move))
                .ToList();
@@ -139,7 +143,7 @@ namespace mbts.spark {
          set => Set(ref _selected_section, value);
       }
 
-      public VSSymbolStructureVM symbol_structure_vm { get; set; }
+      public VolSurfaceSymbolStructureVM symbol_structure_vm { get; }
       public VolModelPolynomialVM vol_model_polynomial_vm { get; } = new();
       public VolModelCubicSplineVM vol_model_cubic_spline_vm { get; } = new();
       public VolModelDualQuadraticVM vol_model_dual_quadratic_vm { get; } = new();
@@ -363,31 +367,31 @@ namespace mbts.spark {
       public VolSurfaceSettings get_settings() {
          var rv = ProtobufSerializationHelper.clone(_settings);
          var symbols = symbol_structure_vm.driver_grids
-            .SelectMany(dg => dg.symbols.Where(sym => sym.is_selected))
+            .SelectMany(dg => dg.slaves.Where(sym => sym.is_selected))
             .ToList();
          var settings = new VolSurfaceSettings();
          var core_settings = settings.core_settings;
-         core_settings.master_slave_relationships = symbols
-            .Where(sym => sym.masters.Any())
-            .Select(sym => new VolSurfaceSymbolRelationship {
-               symbol = sym.name,
-               connected_symbols = sym.masters.Cast<string>().ToList()
-            })
-            .ToList();
-         core_settings.implied_vol_marketdata_relationships = symbols
-            .Where(sym => sym.fits.Any())
-            .Select(sym => new VolSurfaceSymbolRelationship {
-               symbol = sym.name,
-               connected_symbols = sym.fits.Cast<string>().ToList()
-            })
-            .ToList();
+         core_settings.master_slave_relationships.SetItems(symbols
+            .Where(sym => sym.selected_master_symbols.Any())
+            .Select(sym => VolSurfaceSymbolRelationship.create(
+               (sym.name, sym.venue),
+               sym.selected_master_symbols.Select(s => new SymbolVenue() { symbol = s.security_name, venue = s.security_exchange })
+            )));
+         core_settings.implied_vol_marketdata_relationships.SetItems(symbols
+            .Where(sym => sym.selected_fits.Any())
+            .Select(sym => VolSurfaceSymbolRelationship.create(
+               (sym.name, sym.venue),
+               sym.selected_fits.Select(s => new SymbolVenue() { symbol = s.security_name, venue = s.security_exchange })
+            )));
 
          vol_model_polynomial_vm.assign_to(_settings.polynomial_model_settings);
          vol_model_cubic_spline_vm.assign_to(_settings.spline_model_settings);
          vol_model_dual_quadratic_vm.assign_to(_settings.dualquadratic_model_settings);
          vol_path_parabolic_vm.assign_to(_settings.core_settings.volpath_settings.parabolic_vol_path_settings);
 
-         core_settings.symbol_pricing_model_map = symbols.Select(sym => Tuple.Create(sym.name, sym.pricing_model)).ToList();
+         foreach (var sym in symbols) {
+            _core_settings.symbol_pricing_model_map[(sym.name, sym.venue)] = sym.pricing_model;
+         }
          core_settings.put_rotation.strike_vol_movement_pairs = put_rotations
             .Select(r => Tuple.Create(r.target_strike, r.move))
             .ToList();
@@ -455,154 +459,19 @@ namespace mbts.spark {
          _core_settings.atm_shift.target_terms = term;
       }
 
-      public class VSDriver : IEquatable<VSDriver> {
-         public VSDriver(Venue exchange, string name) {
-            this.exchange = exchange;
-            this.name = name;
+      public class SymbolVenueVM : IEquatable<SymbolVenueVM> {
+         public SymbolVenueVM(string symbol, Venue venue) {
+            this.symbol = symbol;
+            this.venue = venue;
          }
 
-         public string name { get; }
-         public Venue exchange;
-         public ObservableCollection<string> symbols { get; } = new();
+         public string symbol { get; }
+         public Venue venue { get; }
 
-         public bool Equals(VSDriver other) =>
-            other != null && other.name == name && other.exchange == exchange;
-
-         public override bool Equals(object obj) =>
-            Equals(obj as VSDriver);
-
-         public override int GetHashCode() =>
-            name.GetHashCode();
-      }
-
-      public class VSSymbol : DisposableViewModelBase, IEquatable<VSSymbol> {
-         ImmutableList<string> _fitsi = ImmutableList<string>.Empty;
-         ImmutableList<string> _mastersi = ImmutableList<string>.Empty;
-
-         public VSSymbol(VSDriver driver, string name) {
-            this.driver = driver;
-            this.name = name;
-            _dispose = Disposable.Create(
-               fits.observe_immutable().OnDispatcher().Subscribe(f => fitsi = f),
-               masters.observe_immutable().OnDispatcher().Subscribe(m => mastersi = m));
-         }
-
-         public VSDriver driver { get; }
-         public string name { get; }
-         public bool is_selected { get; set; }
-
-         public ObservableCollection<string> fits { get; } = new();
-         public ImmutableList<string> fitsi {
-            get => _fitsi;
-            set => Set(ref _fitsi, value);
-         }
-
-         public ObservableCollection<string> masters { get; } = new();
-         public ImmutableList<string> mastersi {
-            get => _mastersi;
-            set => Set(ref _mastersi, value);
-         }
-
-         public VolSurfaceEnums.VolSurfacePricingModel pricing_model { get; set; }
-
-         public bool Equals(VSSymbol other) =>
-            other != null && name == other.name && driver.Equals(other.driver);
-
-         public override bool Equals(object obj) =>
-            Equals(obj as VSSymbol);
-
-         public override int GetHashCode() =>
-            name.GetHashCode();
-      }
-
-      public class VSSymbolGridVM : IDisposable {
-         public VSSymbolGridVM(VSDriver driver) {
-            this.driver = driver;
-            symbols = new ObservableCollection<VSSymbol>();
-         }
-
-         public VSDriver driver { get; private set; }
-         public string header { get { return "{0} ({1})".fmt(driver.name, driver.exchange); } }
-         public ObservableCollection<VSSymbol> symbols { get; private set; }
-
-         public void Dispose() =>
-            symbols.DisposeAll(true);
-      }
-
-      public class VSSymbolStructureVM : DisposableViewModelBase {
-         readonly Property<VSSymbolGridVM> _selected_driver_grid;
-
-         public VSSymbolStructureVM(IBus bus, VolSurfaceSettings settings) {
-            driver_grids = new ObservableCollection<VSSymbolGridVM>();
-            _selected_driver_grid = backing<VSSymbolGridVM>(nameof(selected_driver_grid));
-            settings.core_settings.master_slave_relationships ??= new List<VolSurfaceSymbolRelationship>();
-            settings.core_settings.implied_vol_marketdata_relationships ??= new List<VolSurfaceSymbolRelationship>();
-            _dispose = Disposable.Create(
-               driver_grids.Subscribe((s, e) =>
-                  selected_driver_grid ??= driver_grids.FirstOrDefault()),
-               bus.subscribe(SparkTags.ExpirationParametersInputTag).OnDispatcher().Subscribe(on_exp_params_input),
-               Disposable.Create(() => driver_grids.DisposeAll(true)));
-
-            void on_exp_params_input(AnalyticsExpirationParametersMap data) {
-               var driver_map = data.get_option_driver_map();
-               var driver_symbols_aux = new Dictionary<VSDriver, HashSet<string>>();
-               bool init_with_settings = driver_grids.Count == 0;
-
-               foreach (var (driver_exp, slave_exps) in driver_map.Where(kv => kv.Key.exchange != Venue.Unset)) {
-                  var driver = new VSDriver(driver_exp.exchange, driver_exp.symbol);
-                  if (!driver_symbols_aux.TryGetValue(driver, out var driven_by_this_driver)) {
-                     driver_symbols_aux[driver] = driven_by_this_driver = new HashSet<string>();
-                  }
-                  driven_by_this_driver.AddRange(slave_exps.Select(s => s.symbol));
-               }
-
-               var driver_symbols = driver_symbols_aux.ToDictionary(kv => kv.Key, kv => new List<string>(kv.Value));
-               driver_grids.RemoveWhere(driver_sym_grid => !driver_symbols.ContainsKey(driver_sym_grid.driver));
-
-               foreach (var symbols in driver_symbols.Values) {
-                  symbols.Sort();
-               }
-
-               foreach (var (key, i) in driver_symbols.OrderBy(kv => kv.Key.name).Select((kv, i) => (key: kv.Key, i))) {
-                  if (!driver_grids.Select(dg => dg.driver).Contains(key)) {
-                     driver_grids.Insert(i, new VSSymbolGridVM(key));
-                  }
-               }
-
-               foreach (var sym_grid in driver_grids) {
-                  sym_grid.driver.symbols.sync_with(driver_symbols[sym_grid.driver]);
-                  sym_grid.symbols.sync_with(driver_symbols[sym_grid.driver].Select(sym => new VSSymbol(sym_grid.driver, sym)));
-               }
-
-               if (!init_with_settings) {
-                  return;
-               }
-
-               foreach (var sym in driver_grids.SelectMany(grid => grid.symbols)) {
-                  if (settings.core_settings.symbol_pricing_model_map is null) {
-                     continue;
-                  }
-                  var pricing_model = settings.core_settings.symbol_pricing_model_map.FirstOrDefault(pm => pm.Item1 == sym.name);
-                  if (pricing_model != null) {
-                     sym.is_selected = true;
-                     sym.pricing_model = pricing_model.Item2;
-                     var master_slaves = settings.core_settings.master_slave_relationships.FirstOrDefault(msr => msr.symbol == sym.name);
-                     if (master_slaves != null)
-                        sym.masters.sync_with(master_slaves.connected_symbols);
-                     var implied_vols = settings.core_settings.implied_vol_marketdata_relationships.FirstOrDefault(iv => iv.symbol == sym.name);
-                     if (implied_vols != null)
-                        sym.fits.sync_with(implied_vols.connected_symbols);
-                  }
-               }
-            }
-         }
-
-         public IReadOnlyList<VolSurfaceEnums.VolSurfacePricingModel> pricing_models => EnumHelper.get_values<VolSurfaceEnums.VolSurfacePricingModel>();
-         public ObservableCollection<VSSymbolGridVM> driver_grids { get; private set; }
-         public VSSymbolGridVM selected_driver_grid {
-            get => _selected_driver_grid.Value;
-            set => _selected_driver_grid.set(value);
-         }
+         public bool Equals(SymbolVenueVM other) =>
+            other is not null
+               && symbol == other.symbol
+               && venue == other.venue;
       }
 
       public class RotationVM : ViewModelBase, IEquatable<RotationVM> {

src/client/spark/AW2/ViewModel/SettingsEditors/VolSurfaceSlaveVM.cs

@@ -0,0 +1,98 @@
+namespace mbts.spark;
+
+using bts.api;
+using bts.collections;
+
+using mbts.analytics;
+using mbts.ui;
+using mbts.ui.shell;
+using mbts.ui.utility;
+
+using Spark.Core.Disposables;
+
+using System;
+using System.Collections.Immutable;
+using System.Collections.ObjectModel;
+
+using SecurityTuple = mbts.SecurityTuple;
+
+public class VolSurfaceSlaveVM : DisposableViewModelBase, IEquatable<VolSurfaceSlaveVM> {
+   ImmutableList<SecurityTuple> _selected_fits = ImmutableList<SecurityTuple>.Empty;
+   ImmutableList<SecurityTuple> _selected_master_symbols = ImmutableList<SecurityTuple>.Empty;
+   VolSurfaceEnums.VolSurfacePricingModel _pricing_model;
+   bool _is_selected;
+
+   public VolSurfaceSlaveVM(VolSurfaceDriverVM driver, SecurityTuple security_tuple) {
+      this.driver = driver;
+      this.security_tuple = security_tuple;
+      _dispose = Disposable.Create(
+         ObservableAutoWrapper.connect(driver.symbols, fit_items, s => new(new(SecurityType.Option, s.symbol, s.venue))),
+         ObservableAutoWrapper.connect_item_hooks(fit_items, init_fit),
+         ObservableAutoWrapper.connect(driver.symbols, master_symbol_items, s => new(new(SecurityType.Option, s.symbol, s.venue))),
+         ObservableAutoWrapper.connect_item_hooks(master_symbol_items, init_master_symbol));
+
+      IDisposable init_master_symbol(SelectableItemVM<SecurityTuple> vm) {
+         return vm.subscribe(on_is_selected_changed);
+
+         void on_is_selected_changed(bool is_selected) =>
+            selected_master_symbols = is_selected
+               ? selected_master_symbols.InsertSorted(vm.value)
+               : selected_master_symbols.RemoveSorted(vm.value);
+      }
+
+      IDisposable init_fit(SelectableItemVM<SecurityTuple> vm) {
+         return vm.subscribe(on_is_selected_changed);
+
+         void on_is_selected_changed(bool is_selected) =>
+            selected_fits = is_selected
+               ? selected_fits.InsertSorted(vm.value)
+               : selected_fits.RemoveSorted(vm.value);
+      }
+   }
+
+   public VolSurfaceDriverVM driver { get; }
+   public SecurityTuple security_tuple { get; }
+   public string name => security_tuple.security_name;
+   public Venue venue => security_tuple.security_exchange;
+
+   public bool is_selected {
+      get => _is_selected;
+      set => Set(ref _is_selected, value);
+   }
+
+   public ImmutableList<SecurityTuple> selected_fits {
+      get => _selected_fits;
+      private set {
+         if (_selected_fits != value) {
+            _selected_fits = value;
+            RaisePropertyChanged();
+         }
+      }
+   }
+   public ObservableCollection<SelectableItemVM<SecurityTuple>> fit_items { get; } = new();
+
+   public ImmutableList<SecurityTuple> selected_master_symbols {
+      get => _selected_master_symbols;
+      private set {
+         if (_selected_master_symbols != value) {
+            _selected_master_symbols = value;
+            RaisePropertyChanged();
+         }
+      }
+   }
+   public ObservableCollection<SelectableItemVM<SecurityTuple>> master_symbol_items { get; } = new();
+
+   public VolSurfaceEnums.VolSurfacePricingModel pricing_model {
+      get => _pricing_model;
+      set => Set(ref _pricing_model, value);
+   }
+
+   public bool Equals(VolSurfaceSlaveVM other) =>
+      other != null && name == other.name && driver.Equals(other.driver);
+
+   public override bool Equals(object obj) =>
+      Equals(obj as VolSurfaceSlaveVM);
+
+   public override int GetHashCode() =>
+      name.GetHashCode();
+}

src/client/spark/AW2/ViewModel/SettingsEditors/VolSurfaceSymbolStructureVM.cs

@@ -0,0 +1,87 @@
+namespace mbts.spark;
+
+using bts.collections;
+
+using mbts.analytics;
+using mbts.ui.shell;
+using mbts.ui.utility;
+
+using Spark.Core.Disposables;
+
+using System;
+using System.Collections.Generic;
+using System.Collections.ObjectModel;
+using System.Collections.Specialized;
+using System.Linq;
+using System.Reactive.Linq;
+
+public class VolSurfaceSymbolStructureVM : DisposableViewModelBase {
+   readonly Property<VolSurfaceDriverGridVM> _selected_driver_grid;
+   readonly VolSurfaceSettings _settings;
+
+   public VolSurfaceSymbolStructureVM(VolSurfaceSettings settings) {
+      _settings = settings;
+      _settings.core_settings.implied_vol_marketdata_relationships ??= new List<VolSurfaceSymbolRelationship>();
+      _selected_driver_grid = backing<VolSurfaceDriverGridVM>(nameof(selected_driver_grid));
+      _dispose = Disposable.Create(
+         driver_grids.Subscribe((s, e) =>
+            selected_driver_grid ??= driver_grids.FirstOrDefault()),
+         Disposable.Create(() => driver_grids.DisposeAll(true)));
+   }
+
+   public IReadOnlyList<VolSurfaceEnums.VolSurfacePricingModel> pricing_models => EnumHelper.get_values<VolSurfaceEnums.VolSurfacePricingModel>();
+   public ObservableCollection<VolSurfaceDriverGridVM> driver_grids { get; } = new();
+   public VolSurfaceDriverGridVM selected_driver_grid {
+      get => _selected_driver_grid.Value;
+      set => _selected_driver_grid.set(value);
+   }
+
+   public void on_exp_params_input(AnalyticsExpirationParametersMap data) {
+      var driver_groups = data.get_option_driver_symbol_map().ToDictionary(
+         g => g.Key,
+         g => g.Select(e => e.security_tuple).Distinct().Ordered().ToList());
+      bool init_with_settings = driver_grids.Count == 0;
+
+      driver_grids.RemoveWhere(dg => !driver_groups.ContainsKey((dg.driver.name, dg.driver.exchange)));
+      driver_grids.sync_with_models(
+         driver_groups.OrderBy(kv => kv.Key),
+         dg => {
+            var driver = new VolSurfaceDriverVM(dg.Key.venue, dg.Key.symbol);
+            driver.symbols.SetItems(dg.Value.Select(s => new VolSurfaceSymbolVenueVM(s.security_name, s.security_exchange)));
+            return new VolSurfaceDriverGridVM(driver);
+         },
+         (vm, dg) => {
+            vm.driver.symbols.sync_with(dg.Value.Select(s => new VolSurfaceSymbolVenueVM(s.security_name, s.security_exchange)));
+            return vm;
+         });
+
+      if (!init_with_settings) {
+         return;
+      }
+
+      var sec_tuple_comp = EqualityComparer<SecurityTuple>.Default;
+      var price_models = _settings.core_settings.symbol_pricing_model_map;
+      if (price_models is null) {
+         return;
+      }
+      foreach (var sym in driver_grids.SelectMany(grid => grid.slaves)) {
+         if (!price_models.TryGetValue((sym.name, sym.venue), out var pricing_model)) {
+            continue;
+         }
+         sym.is_selected = true;
+         sym.pricing_model = pricing_model;
+         var master_slaves = _settings.core_settings.master_slave_relationships.FirstOrDefault(msr => msr.symbol == (sym.name, sym.venue));
+         if (master_slaves != null) {
+            foreach (var m in sym.master_symbol_items) {
+               m.is_selected = master_slaves.connected_symbols.Any(s => s == (m.value.security_name, m.value.security_exchange));
+            }
+         }
+         var implied_vols = _settings.core_settings.implied_vol_marketdata_relationships.FirstOrDefault(iv => iv.symbol == (sym.name, sym.venue));
+         if (implied_vols != null) {
+            foreach (var f in sym.fit_items) {
+               f.is_selected = implied_vols.connected_symbols.Any(s => s == (f.value.security_name, f.value.security_exchange));
+            }
+         }
+      }
+   }
+}

src/client/spark/AW2/ViewModel/SettingsEditors/VolSurfaceSymbolVenueVM.cs

@@ -0,0 +1,34 @@
+#nullable enable
+namespace mbts.spark;
+
+using bts;
+using bts.api;
+
+using System;
+
+public class VolSurfaceSymbolVenueVM : IEquatable<VolSurfaceSymbolVenueVM>, ICloneable<VolSurfaceSymbolVenueVM> {
+   public VolSurfaceSymbolVenueVM(string? symbol, Venue venue) {
+      this.symbol = symbol;
+      this.venue = venue;
+   }
+
+   public string? symbol { get; }
+   public Venue venue { get; }
+
+   public VolSurfaceSymbolVenueVM Clone() =>
+      new(symbol, venue);
+
+   public bool Equals(VolSurfaceSymbolVenueVM? other) =>
+      other is not null
+         && other.symbol == symbol
+         && other.venue == venue;
+
+   public override bool Equals(object? obj) =>
+      Equals(obj as VolSurfaceSymbolVenueVM);
+
+   public override int GetHashCode() =>
+      HashCode.Combine(symbol, venue);
+
+   object ICloneable.Clone() =>
+      Clone();
+}

src/client/spark/AW2/ViewModel/VolTermsGraphVM.cs

@@ -165,16 +165,16 @@
 
       void _update_points() {
          var settings = _model.get_active_settings();
-         var master_symbols = settings == null ? new HashSet<string>() : VolSurfaceUtilityFunctions.get_master_symbols(settings);
+         var master_symbols = settings == null ? new HashSet<(string, Venue)>() : VolSurfaceUtilityFunctions.get_master_symbols(settings);
          var committed_map = _get_exp_date_map(_model.get_committed_term_structure_map(), master_symbols);
          var candidate_map = _get_exp_date_map(_model.get_candidate_term_structure_map(), master_symbols);
          foreach (var s in graph_vm.series.Cast<IVolTermsSeries>()) s.update_points(s.is_committed ? committed_map : candidate_map);
       }
 
       Dictionary<DateTime, double> _get_exp_date_map(
-         IReadOnlyDictionary<VolSurfaceExpiration, double> exp_map, HashSet<string> master_symbols) =>
+         IReadOnlyDictionary<VolSurfaceExpiration, double> exp_map, HashSet<(string, Venue)> master_symbols) =>
          exp_map.Where(kv => selected_symbols.Contains(kv.Key.symbol) &&
-            master_symbols.Contains(kv.Key.symbol))
+            master_symbols.Contains((kv.Key.symbol, kv.Key.venue)))
             .ToDictionaryCrushed(kv => kv.Key.expiration, kv => kv.Value);
 
       public class SymbolItem : IEquatable<SymbolItem> {

src/client/spark/AW2/VolSurfaceDisplayModel.cs

@@ -250,7 +250,11 @@
 
       public IEnumerable<Tuple<double, double>> get_committed_volpath(VolSurfaceDisplayGridExpiration exp) {
          if (_committed_data != null && _committed_data.vs_output != null) {
-            VolCurveOutput exp_out = _committed_data.vs_output[new VolSurfaceExpiration(exp.symbol, exp.expiration)];
+            VolCurveOutput exp_out = _committed_data.vs_output[VolSurfaceExpiration.Default.with(
+               security_type: SecurityType.Option,
+               symbol: exp.symbol,
+               expiration: exp.expiration,
+               venue: exp.venue)];
             return VolSurfaceUtilityFunctions.get_vol_path_points(exp_out);
          } else
             return Enumerable.Empty<Tuple<double, double>>();
@@ -258,7 +262,11 @@
 
       public IEnumerable<Tuple<double, double>> get_candidate_volpath(VolSurfaceDisplayGridExpiration exp) {
          if (is_dirty && _candidate_data != null && _candidate_data.vs_output != null) {
-            VolCurveOutput exp_out = _candidate_data.vs_output[new VolSurfaceExpiration(exp.symbol, exp.expiration)];
+            VolCurveOutput exp_out = _candidate_data.vs_output[VolSurfaceExpiration.Default.with(
+               security_type: SecurityType.Option,
+               symbol: exp.symbol,
+               expiration: exp.expiration,
+               venue: exp.venue)];
             return VolSurfaceUtilityFunctions.get_vol_path_points(exp_out);
          } else
             return Enumerable.Empty<Tuple<double, double>>();
@@ -277,15 +285,11 @@
          return !is_dirty || _candidate_data == null || _candidate_data.vs_output == null ? new Dictionary<VolSurfaceExpiration, double>() : _candidate_data.vs_output.exp_output_map.ToDictionary(kv => kv.Key, kv => kv.Value.atm_vol);
       }
 
-      public VolCurveMarketImpliedVolData get_market_market_implied_vols(VolSurfaceDisplayGridExpiration exp) {
-         Tuple<VolSurfaceExpiration, VolCurveMarketImpliedVolData> rv = null;
-         return _impl_vol_map.TryGetValue(new VolSurfaceExpiration() { symbol = exp.symbol, expiration = exp.expiration }, out rv) ? rv.Item2 : null;
-      }
+      public VolCurveMarketImpliedVolData get_market_market_implied_vols(VolSurfaceDisplayGridExpiration exp) =>
+         _impl_vol_map.GetValueOrDefault(exp.ToVSExp(SecurityType.Option))?.Item2;
 
-      public VolSurfaceExpiration get_market_implied_vol_source(VolSurfaceDisplayGridExpiration exp) {
-         Tuple<VolSurfaceExpiration, VolCurveMarketImpliedVolData> rv = null;
-         return _impl_vol_map.TryGetValue(new VolSurfaceExpiration() { symbol = exp.symbol, expiration = exp.expiration }, out rv) ? rv.Item1 : null;
-      }
+      public VolSurfaceExpiration get_market_implied_vol_source(VolSurfaceDisplayGridExpiration exp) =>
+         _impl_vol_map.GetValueOrDefault(exp.ToVSExp(SecurityType.Option))?.Item1;
 
       #endregion
 
@@ -374,7 +378,11 @@
                string[] data_items = line.Split(',').Select(cn => cn.Trim()).ToArray();
                if (data_items.Length != col_names.Length)
                   throw new ApplicationException("Data is not well formed");
-               VolSurfaceDisplayGridExpiration grid_exp = new VolSurfaceDisplayGridExpiration(data_items[0], DateTime.Parse(data_items[1]), VolSurfaceEnums.VolSurfaceExpirationCalculationMethod.Master);
+               VolSurfaceDisplayGridExpiration grid_exp = new(
+                  data_items[0],
+                  DateTime.Parse(data_items[1]),
+                  Enum.Parse<Venue>(data_items[2]),
+                  VolSurfaceEnums.VolSurfaceExpirationCalculationMethod.Master);
                for (int i = 2; i < col_names.Length; i++) {
                   string col_name = file_col_map[i];
                   string data_text = data_items[i];
@@ -508,7 +516,7 @@
       public void fit_to_market(IEnumerable<VolSurfaceDisplayGridExpiration> exp_list, bool is_all_curve, bool is_live_market,
                                 bool use_weighted_regression_fit, bool cascade_up, bool cascade_down) {
          DisplayModelData curr_data = _get_current_data();
-         IEnumerable<VolSurfaceExpiration> vs_exps = exp_list.Select(i => i.ToVSExp());
+         IEnumerable<VolSurfaceExpiration> vs_exps = exp_list.Select(i => i.ToVSExp(SecurityType.Option));
          _send_curve_action_request(vs_exps.ToDictionary(e => e,
                                                          e => {
                                                             VolCurveMarketFitAction.ActionType action_type = is_all_curve ? VolCurveMarketFitAction.ActionType.FitWholeCurve : VolCurveMarketFitAction.ActionType.FitATM;
@@ -518,7 +526,7 @@
       }
 
       public void rebuild(IEnumerable<VolSurfaceDisplayGridExpiration> exp_list, bool cascade_up, bool cascade_down) {
-         _send_curve_action_request(exp_list.ToDictionary(e => e.ToVSExp(), e => new VolCurveAction(VolCurveAction.ActionType.Rebuild)), cascade_up, cascade_down);
+         _send_curve_action_request(exp_list.ToDictionary(e => e.ToVSExp(SecurityType.Option), e => new VolCurveAction(VolCurveAction.ActionType.Rebuild)), cascade_up, cascade_down);
       }
 
       public void publish_new_grid_selection(string window_id, VolSurfaceDisplayGridExpiration selected_expiration, bool show_implied_vol_checked) {
@@ -564,7 +572,7 @@
       /// operation go call this
       private void _send_curve_operation_request(IEnumerable<VolSurfaceDisplayGridExpiration> exp_list, VolCurveMovementOperation curve_operation,
                                                  bool is_up, bool cascade_up, bool cascade_down) {
-         _send_curve_action_request(exp_list.ToDictionary(e => e.ToVSExp(), e => new VolCurveAction(VolSurfaceUtilityFunctions.add_side_to_curve_operation(curve_operation, is_up))),
+         _send_curve_action_request(exp_list.ToDictionary(e => e.ToVSExp(SecurityType.Option), e => new VolCurveAction(VolSurfaceUtilityFunctions.add_side_to_curve_operation(curve_operation, is_up))),
                                                           cascade_up, cascade_down);
       }
 
@@ -604,15 +612,10 @@
       }
 
       private void _copy_time_parameters(VolSurfaceInput source, ref VolSurfaceInput target) {
-         target.surface_parameters = target.surface_parameters ?? new VolSurfaceParameters();
-         target.surface_parameters.time_parameters = target.surface_parameters.time_parameters ?? new VolSurfaceTimeParameters() { input_time = DateTime.MinValue, auto_decay_points = new List<Tuple<DateTime, double>>() };
-         if (source != null && source.surface_parameters != null) {
-            target.surface_parameters.vol_time = source.surface_parameters.vol_time;
-            if (source.surface_parameters.time_parameters != null) {
-               target.surface_parameters.time_parameters.input_time = source.surface_parameters.time_parameters.input_time;
-               target.surface_parameters.time_parameters.auto_decay_points = source.surface_parameters.time_parameters.auto_decay_points == null ? target.surface_parameters.time_parameters.auto_decay_points : source.surface_parameters.time_parameters.auto_decay_points.ToList();
-            }
-         }
+         target.surface_parameters ??= new();
+         target.surface_parameters.vol_time = source?.surface_parameters?.vol_time ?? default;
+         target.surface_parameters.time_parameters = source?.surface_parameters?.time_parameters?.Clone()
+            ?? VolSurfaceTimeParameters.Default;
       }
 
       #region Stateless Helpers

src/client/spark/AW2/VolSurfaceExtensions.cs

@@ -18,8 +18,8 @@ namespace mbts.spark {
 
       // returns true if the option_name is driven by vs_id
       public static bool symbol_driven_by_vs(
-         this VolSurfaceUsageListImmutable list, string option_name, int vs_id) =>
-         list.UsageList.Any(u => u.SecTup.Symbol == option_name && u.VsId == vs_id);
+         this VolSurfaceUsageListImmutable list, (string, Venue) option_name, int vs_id) =>
+         list.UsageList.Any(u => (u.SecTup.Symbol, u.SecTup.Venue) == option_name && u.VsId == vs_id);
 
       // returns all sec_tups that are driven by this vs_id
       public static IEnumerable<bts.api.SecurityTuple> get_sectups(
@@ -37,8 +37,8 @@ namespace mbts.spark {
       public static bool is_editable(this VolSurfaceColumnType vs_col_type) =>
          vs_col_type is VolSurfaceColumnType.ReadWriteTick;
 
-      public static analytics.VolSurfaceExpiration ToVSExp(this VolSurfaceDisplayGridExpiration vsdge) =>
-         new() { expiration = vsdge.expiration, symbol = vsdge.symbol };
+      public static analytics.VolSurfaceExpiration ToVSExp(this VolSurfaceDisplayGridExpiration vsdge, SecurityType security_type) =>
+         new(security_type, vsdge.symbol, vsdge.expiration, vsdge.venue);
 
       public static SecurityExpiration ToSecurityExp(this VolSurfaceDisplayGridExpiration vsdge) =>
          new(Venue.Unset, SecurityType.Option, vsdge.symbol, vsdge.expiration);
@@ -84,7 +84,7 @@ namespace mbts.spark {
          };
 
       public static IEnumerable<VolSurfaceExpiration> select_exp(this IEnumerable<VolSurfaceRowVM> rows) =>
-         rows.Select(r => r.expiration.ToVSExp());
+         rows.Select(r => r.expiration.ToVSExp(SecurityType.Option));
 
       public static IEnumerable<VolSurfaceDisplayGridExpiration> select_display_exp(this IEnumerable<VolSurfaceRowVM> rows) =>
          rows.Select(r => r.expiration);
@@ -142,7 +142,12 @@ namespace mbts.spark {
          };
 
       public static VolSurfaceExpiration to_vs_exp(this SecurityExpiration sec_exp) =>
-         new() { security_type = sec_exp.security_type, symbol = sec_exp.symbol, expiration = sec_exp.expiration };
+         new() {
+            security_type = sec_exp.security_type,
+            symbol = sec_exp.symbol,
+            expiration = sec_exp.expiration,
+            venue = sec_exp.exchange,
+         };
 
       public static DriverCalculationType to_driver_calc_type(
          this ExpirationDriver.ModeOneofCase driver_mode) =>
@@ -170,7 +175,7 @@ namespace mbts.spark {
 
       public static SecurityExpiration to_sec_exp(this SecurityImmutable sec) {
          var exp_date = sec.security_expiration() ?? throw new InvalidOperationException();
-         return new(sec.Exchange, sec.ContractCase, sec.Symbol, exp_date, sec.Id);
+         return new(sec.Exchange, sec.ContractCase, sec.Symbol, exp_date);
       }
 
       public static SecurityTuple security_tuple(this Security sec) {
@@ -217,6 +222,9 @@ namespace mbts.spark {
          this SecurityExpiration sec_exp) =>
          new(to_api_sec_exp(sec_exp));
 
+   public static SecurityExpiration to_security_expiration(this VolSurfaceExpiration vs_exp) =>
+      new(vs_exp.venue, vs_exp.security_type, vs_exp.symbol, vs_exp.expiration);
+
       static IEnumerable<(double x, double y)> _select_real_points(IEnumerable<Tuple<double, double>> points) =>
          points.Where(pt => pt.Item1.is_real() && pt.Item2.is_real()).Select(t => (t.Item1, t.Item2));
    }

src/client/spark/AW2/VolSurfaceInterpGreeksMultiplexer.cs

@@ -77,7 +77,7 @@ namespace mbts.spark {
          if (theos != null) {
             var theo_map = _convert_theos(theos);
             foreach (var kv in theo_map) {
-               string option_name = kv.Key;
+               var option_name = kv.Key;
                if (_vs_usage_map.symbol_driven_by_vs(option_name, vs_id)) {
                   bts.Log.Gen.info(
                      "VolSurfaceInterpGreeksMultiplexer._read_theos_and_publish " +
@@ -89,15 +89,14 @@ namespace mbts.spark {
       }
 
       #region Stateless Helpers
-      private static IDictionary<string, OptionTheoInterpolationData> _convert_theos(
+      private static IDictionary<(string, Venue), OptionTheoInterpolationData> _convert_theos(
          IDictionary<VolSurfaceExpiration, IEnumerable<Theos>> vs_theos) {
-         IDictionary<string, OptionTheoInterpolationData> rv =
-            new Dictionary<string, OptionTheoInterpolationData>();
+         var rv = new Dictionary<(string, Venue), OptionTheoInterpolationData>();
          foreach (var kv in vs_theos) {
             OptionTheoInterpolationData sym_data = null;
-            if (!rv.TryGetValue(kv.Key.symbol, out sym_data)) {
-               sym_data = new OptionTheoInterpolationData(kv.Key.symbol);
-               rv.Add(kv.Key.symbol, sym_data);
+            if (!rv.TryGetValue((kv.Key.symbol, kv.Key.venue), out sym_data)) {
+               sym_data = new OptionTheoInterpolationData(kv.Key.symbol, kv.Key.venue);
+               rv.Add((kv.Key.symbol, kv.Key.venue), sym_data);
             }
             foreach (var t in kv.Value)
                sym_data.set_data(kv.Key.expiration, t.driver,

src/client/spark/Analytics/Core/Parameters/VolSurfaceTimeParameters.cs

@@ -1,17 +1,46 @@
-using System;
+#nullable enable
+using bts.collections;
+
+using ProtoBuf;
+
+using Spark.Core.Protobuf;
+
+using System;
 using System.Collections.Generic;
 using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 
-namespace mbts.analytics
-{
-   [ProtoBuf.ProtoContract(Name = "VolSurfaceTimeParameters")]
-   public class VolSurfaceTimeParameters
-   {
-      [ProtoBuf.ProtoMember(1)]
-      public DateTime input_time;
-      [ProtoBuf.ProtoMember(10)]
-      public List<Tuple<DateTime, double>> auto_decay_points;
-   }
+namespace mbts.analytics;
+
+[ProtoContract(Name = "VolSurfaceTimeParameters")]
+public class VolSurfaceTimeParameters : IProtobuf<VolSurfaceTimeParameters> {
+   public static VolSurfaceTimeParameters Default =>
+      new() {
+         input_time = DateTime.MinValue,
+         auto_decay_points = new(0)
+      };
+
+   [ProtoMember(1)]
+   public DateTime input_time { get; set; }
+
+   [ProtoMember(10)]
+   public List<Tuple<DateTime, double>>? auto_decay_points { get; set; }
+
+   public VolSurfaceTimeParameters Clone() =>
+      new() {
+         input_time = input_time,
+         auto_decay_points = auto_decay_points
+            ?.WrapSelector(dp => Tuple.Create(dp.Item1, dp.Item2), null)
+            ?.ToList()
+      };
+
+   public bool Equals(VolSurfaceTimeParameters? other) =>
+      other is not null
+      && input_time == other.input_time
+      && auto_decay_points.safe_sequence_equals(other.auto_decay_points);
+
+   public override bool Equals(object? obj) =>
+      Equals(obj as VolSurfaceTimeParameters);
+
+   public override int GetHashCode() =>
+      HashCode.Combine(input_time, auto_decay_points?.Count);
 }

src/client/spark/Analytics/Core/Settings/SecurityExpiration.cs

@@ -37,19 +37,17 @@ namespace mbts {
          Venue exchange,
          Security.ContractOneofCase type,
          string symbol,
-         DateTime exp,
-         uint? id = null)
-         : this(exchange, type.security_type(), symbol, exp, id) { }
+         DateTime exp)
+         : this(exchange, type.security_type(), symbol, exp) { }
 
       public SecurityExpiration(
-         Venue exchange, SecurityType type, string symbol, DateTime expiration, uint? id = null)
-         : this(new SecurityTuple(type, symbol, exchange), expiration, id) { }
+         Venue exchange, SecurityType type, string symbol, DateTime expiration)
+         : this(new SecurityTuple(type, symbol, exchange), expiration) { }
 
       public SecurityExpiration(
-         SecurityTuple security_tuple, DateTime expiration, uint? id = null) {
+         SecurityTuple security_tuple, DateTime expiration) {
          this.security_tuple = security_tuple;
          this.expiration = expiration;
-         this.id = id;
       }
 
       [ProtoMember(1)]
@@ -58,9 +56,6 @@ namespace mbts {
       [ProtoMember(2)]
       public DateTime expiration { get; }
 
-      [ProtoMember(3)]
-      public uint? id { get; }
-
       public SecurityType security_type => security_tuple?.security_type ?? SecurityType.Unset;
       public string symbol => security_tuple?.security_name ?? string.Empty;
       public Venue exchange => security_tuple?.security_exchange ?? Venue.Unset;
@@ -71,7 +66,6 @@ namespace mbts {
          Symbol = 1 << 1,
          Expiration = 1 << 2,
          Venue = 1 << 3,
-         Id = 1 << 4
       }
 
       public static IEqualityComparer<SecurityExpiration> get_eq_comparer() =>
@@ -88,7 +82,6 @@ namespace mbts {
                Properties.Symbol => wrap(s => s.symbol).to_safe(),
                Properties.Expiration => wrap(s => s.expiration),
                Properties.Venue => wrap(s => s.exchange),
-               Properties.Id => wrap(s => s.id).to_safe(),
                _ => throw new NotImplementedException()
             };
 
@@ -109,7 +102,6 @@ namespace mbts {
             Properties.Symbol => ComparerWrapper<SecurityExpiration>.wrap(s => s.symbol),
             Properties.Expiration => ComparerWrapper<SecurityExpiration>.wrap(s => s.expiration),
             Properties.Venue => ComparerWrapper<SecurityExpiration>.wrap(s => s.exchange),
-            Properties.Id => ComparerWrapper<SecurityExpiration>.wrap(s => s.id),
             _ => throw new NotImplementedException()
          };
       }
@@ -126,19 +118,18 @@ namespace mbts {
          new(Venue.Unset, security_type, symbol, expiration);
 
       public SecurityExpiration Clone() =>
-         new(security_tuple.Clone(), expiration, id);
+         new(security_tuple.Clone(), expiration);
 
       public bool Equals(SecurityExpiration? other) =>
          other is not null &&
          security_tuple.safe_equals(other.security_tuple) &&
-         expiration == other.expiration &&
-         id == other.id;
+         expiration == other.expiration;
 
       public override bool Equals(object? obj) =>
          Equals(obj as SecurityExpiration);
 
       public override int GetHashCode() =>
-         HashCode.Combine(security_tuple, expiration, id ?? 0);
+         HashCode.Combine(security_tuple, expiration);
 
       object ICloneable.Clone() =>
          Clone();
@@ -186,11 +177,10 @@ namespace mbts {
          Venue? exchange = null,
          SecurityType? security_type = null,
          string? symbol = null,
-         DateTime? expiration = null,
-         Optional<uint> id = default) => new(
-            exchange ?? sec_exp.exchange, security_type ?? sec_exp.security_type,
+         DateTime? expiration = null) => new(
+            exchange ?? sec_exp.exchange,
+            security_type ?? sec_exp.security_type,
             symbol ?? sec_exp.symbol,
-            expiration ?? sec_exp.expiration,
-            id.has_value ? id : sec_exp.id);
+            expiration ?? sec_exp.expiration);
    }
 }

src/client/spark/Analytics/Core/Settings/SymbolVenue.cs

@@ -0,0 +1,52 @@
+#nullable enable
+namespace mbts.analytics;
+
+using bts.api;
+
+using ProtoBuf;
+
+using Spark.Core.Protobuf;
+
+using System;
+
+[ProtoContract]
+public class SymbolVenue : IProtobuf<SymbolVenue> {
+   [ProtoMember(1)]
+   public string? symbol { get; set; }
+
+   [ProtoMember(2)]
+   public Venue venue { get; set; }
+
+   public SymbolVenue Clone() =>
+      new() {
+         symbol = symbol,
+         venue = venue
+      };
+
+   public bool Equals(SymbolVenue? other) =>
+      other is not null
+         && other.symbol == symbol
+         && other.venue == venue;
+
+   public override bool Equals(object? obj) =>
+      Equals(obj as SymbolVenue);
+
+
+   public override int GetHashCode() =>
+      HashCode.Combine(symbol, venue);
+
+   public static bool operator ==(SymbolVenue left, SymbolVenue right) =>
+      left is null ? right is null : left.Equals(right);
+
+   public static bool operator !=(SymbolVenue left, SymbolVenue right) =>
+      !(left == right);
+
+   public static implicit operator SymbolVenue((string symbol, Venue venue) tup) =>
+      new() {
+         symbol = tup.symbol,
+         venue = tup.venue
+      };
+
+   public static implicit operator (string symbol, Venue venue)(SymbolVenue symbol_venue) =>
+      (symbol_venue.symbol ?? string.Empty, symbol_venue.venue);
+}

src/client/spark/Analytics/Core/Settings/VolSurfaceCoreSettings.cs

@@ -1,3 +1,5 @@
+using Google.Protobuf.Collections;
+
 using System;
 using System.Collections.Generic;
 using System.Linq;
@@ -15,10 +17,10 @@ namespace mbts.analytics {
       public int vol_surface_id;
       [ProtoBuf.ProtoMember(10)]
       public VolSurfaceEnums.VolSurfaceModel vol_surface_model;
-      [ProtoBuf.ProtoMember(20)]
-      public List<string> symbols;
-      [ProtoBuf.ProtoMember(25)]
-      public List<VolSurfaceSymbolStructure> symbol_structure_list;
+      // [ProtoBuf.ProtoMember(20)]
+      // public List<string> symbols;
+      // [ProtoBuf.ProtoMember(25)]
+      // public List<VolSurfaceSymbolRelationship> symbol_structure_list;
       [ProtoBuf.ProtoMember(30)]
       public VolSurfaceWingSettings wing_settings;
       [ProtoBuf.ProtoMember(40)]
@@ -62,13 +64,13 @@ namespace mbts.analytics {
       /// Some symbols could use other symbols's market data ex
       /// spx using es's
       [ProtoBuf.ProtoMember(160)]
-      public List<VolSurfaceSymbolStructure> symbol_implied_vol_marketdata_source_list;
+      public List<VolSurfaceSymbolRelationship> symbol_implied_vol_marketdata_source_list;
       [ProtoBuf.ProtoMember(170)]
-      public List<Tuple<string, VolSurfaceEnums.VolSurfacePricingModel>> symbol_pricing_model_map;
+      public MapField<SymbolVenue, VolSurfaceEnums.VolSurfacePricingModel> symbol_pricing_model_map { get; } = new();
       [ProtoBuf.ProtoMember(180)]
       public List<VolSurfaceSymbolRelationship> implied_vol_marketdata_relationships;
       [ProtoBuf.ProtoMember(190)]
-      public List<VolSurfaceSymbolRelationship> master_slave_relationships;
+      public List<VolSurfaceSymbolRelationship> master_slave_relationships { get; } = new(1);
       /// This tells how the abstract strike axis (moneyness, delta etc)
       /// is converted to a real strike. That depends on vol and fwd.
       /// This tells what vol and fwd to use for that.

src/client/spark/Analytics/Core/Settings/VolSurfaceSettings.cs

@@ -14,8 +14,6 @@ namespace mbts.analytics {
             strangle = new() { strike_vol_movement_pairs = new() },
             full_rotation = new() { strike_vol_movement_pairs = new() },
             atm_shift = new() { strike_vol_movement_pairs = new() },
-            symbols = new(),
-            symbol_structure_list = new(),
             wing_settings = new() {
                explicit_slope_wing_settings = new(),
                implicit_slope_wing_settings = new(),

src/client/spark/Analytics/Core/VolSurfaceExpiration.cs

@@ -1,98 +1,95 @@
-using bts.api;
-using bts.settings;
-using bts.spark;
-using bts.utils;
+#nullable enable
+using bts.api;
 
-using mbts.spark;
-using mbts.ui.converters;
+using mbts.utils;
 
 using ProtoBuf;
 
 using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 
-namespace mbts.analytics {
-   /// <summary>
-   /// An expiration in the Vol Surface is expressed
-   /// with a symbol and an expiration. This is a 
-   /// conveniant way to express unique expirations,
-   /// but it could very well be a string, that maps
-   /// to a real expiration.
-   /// </summary>
-   [ProtoBuf.ProtoContract(Name = "VolSurfaceExpiration")]
-   public class VolSurfaceExpiration : IEquatable<VolSurfaceExpiration>, IComparable<VolSurfaceExpiration> {
-      public VolSurfaceExpiration() : this(string.Empty, default(DateTime)) { }
+namespace mbts.analytics;
 
-      public VolSurfaceExpiration(string symbol, DateTime expiration)
-      :
-      this(bts.api.SecurityType.Option, symbol, expiration) {
-      }
+/// <summary>
+/// An expiration in the Vol Surface is expressed
+/// with a symbol and an expiration. This is a 
+/// conveniant way to express unique expirations,
+/// but it could very well be a string, that maps
+/// to a real expiration.
+/// </summary>
+[ProtoContract(Name = "VolSurfaceExpiration")]
+public class VolSurfaceExpiration : IEquatable<VolSurfaceExpiration>, IComparable<VolSurfaceExpiration> {
+   static readonly ComparerBuilder<VolSurfaceExpiration> _comparer = new() {
+      x => x.expiration,
+      x => x.symbol,
+      x => x.security_type,
+      x => x.venue,
+   };
 
-      public VolSurfaceExpiration(bts.api.SecurityType security_type, string symbol, DateTime expiration) {
-         this.security_type = security_type;
-         this.symbol = symbol;
-         this.expiration = expiration;
-      }
+   public VolSurfaceExpiration()
+      : this(SecurityType.Unset, string.Empty, default, Venue.Unset) { }
 
-      public VolSurfaceExpiration(SecurityExpiration exp) {
-         symbol = exp.symbol;
-         expiration = exp.expiration;
-      }
-
-      public SecurityExpiration to_security_expiration(
-         Venue exchange = Venue.Unset, SecurityType type = SecurityType.Unset) =>
-         new(exchange, security_type, symbol, expiration);
-
-      [ProtoBuf.ProtoMember(10)]
-      public string symbol;
-      [ProtoBuf.ProtoMember(20)]
-      public DateTime expiration;
-      [ProtoBuf.ProtoMember(30)]
-      public bts.api.SecurityType security_type = bts.api.SecurityType.Option;
-
-
-      public bool Equals(VolSurfaceExpiration other) {
-         return other != null && symbol.Equals(other.symbol) && expiration.Equals(other.expiration);
-      }
-
-      public override int GetHashCode() { return security_type.GetHashCode() ^ symbol.GetHashCode() ^ expiration.GetHashCode(); }
-
-      public override bool Equals(object obj) {
-         return Equals(obj as VolSurfaceExpiration);
-      }
-
-      public override string ToString() {
-         return String.Format("{0} {1} {2}",
-            security_type.ToString().ToUpper().First(), symbol, expiration.ToShortDateString());
-      }
-
-      public int CompareTo(VolSurfaceExpiration other) {
-         other = other ?? new VolSurfaceExpiration() { expiration = DateTime.MinValue, symbol = String.Empty };
-         int rv = expiration.CompareTo(other.expiration);
-         return rv != 0 ? rv : symbol.CompareTo(other.symbol);
-      }
-
-      public double diff_days(VolSurfaceExpiration exp_other) {
-         TimeSpan ts = expiration - exp_other.expiration;
-         return ts.TotalDays;
-      }
+   public VolSurfaceExpiration(SecurityType security_type, string symbol, DateTime expiration, Venue venue) {
+      this.security_type = security_type;
+      this.symbol = symbol;
+      this.expiration = expiration;
+      this.venue = venue;
    }
 
-   public static class VolSurfaceExpirationExtensions {
-      public static SecurityExpiration ToSecurityExp(
-         this VolSurfaceExpiration vs_exp, SecurityType type) =>
-         new(Venue.Unset, type, vs_exp.symbol, vs_exp.expiration);
+   public VolSurfaceExpiration(SecurityExpiration exp) {
+      symbol = exp.symbol;
+      expiration = exp.expiration;
+      security_type = exp.security_type;
+      venue = exp.exchange;
    }
 
-   public class VolSurfaceStrikeMap {
-      public readonly IDictionary<VolSurfaceExpiration, IEnumerable<double>> strike_map = null;
-      public VolSurfaceStrikeMap(IEnumerable<string> option_names, SystemContractData sc_data) {
-         var on_map = new HashSet<string>(option_names);
-         var aux_map = sc_data.get_option_strikes(on_map);
-         strike_map = aux_map.ToDictionaryDupes(kv => new VolSurfaceExpiration(kv.Key), kv => kv.Value);
+   public static VolSurfaceExpiration Default =>
+      new(SecurityType.Unset, string.Empty, default, Venue.Unset);
+
+   [ProtoMember(10)]
+   public string symbol { get; set; }
+
+   [ProtoMember(20)]
+   public DateTime expiration { get; set; }
+
+   [ProtoMember(30)]
+   public SecurityType security_type { get; set; }
+
+   [ProtoMember(40)]
+   public Venue venue { get; set; }
+
+   public VolSurfaceExpiration with(
+      string? symbol = null,
+      DateTime? expiration = null,
+      SecurityType? security_type = null,
+      Venue? venue = null
+   ) =>
+      new(
+         security_type ?? this.security_type,
+         symbol ?? this.symbol,
+         expiration ?? this.expiration,
+         venue ?? this.venue);
+
+   public bool Equals(VolSurfaceExpiration? other) =>
+      other is not null
+         && symbol == other.symbol
+         && expiration == other.expiration
+         && security_type == other.security_type
+         && venue == other.venue;
+
+   public int CompareTo(VolSurfaceExpiration? other) =>
+      _comparer.Compare(this, other);
+
+   public override int GetHashCode() =>
+      HashCode.Combine(security_type, symbol, expiration, venue);
+
+   public override bool Equals(object? obj) =>
+      Equals(obj as VolSurfaceExpiration);
+
+   public override string ToString() {
+      var rv = $"{char.ToUpper(security_type.ToString()[0])} {symbol} {expiration.ToShortDateString()}";
+      if (venue != Venue.Unset) {
+         rv += " " + venue.ToString();
       }
+      return rv;
    }
 }

src/client/spark/Analytics/Core/VolSurfaceStrikeMap.cs

@@ -0,0 +1,18 @@
+#nullable enable
+namespace mbts.analytics;
+
+using bts.api;
+
+using mbts.spark;
+
+using System.Collections.Generic;
+
+public class VolSurfaceStrikeMap {
+   public readonly IDictionary<VolSurfaceExpiration, IEnumerable<double>>? strike_map = null;
+
+   public VolSurfaceStrikeMap(IEnumerable<(string, Venue)> option_names, SystemContractData sc_data) {
+      var on_map = new HashSet<(string, Venue)>(option_names);
+      var aux_map = sc_data.get_option_strikes(on_map);
+      strike_map = aux_map.ToDictionaryCrushed(kv => new VolSurfaceExpiration(kv.Key), kv => kv.Value);
+   }
+}

src/client/spark/Analytics/Core/VolSurfaceSymbolRelationship.cs

@@ -0,0 +1,24 @@
+using ProtoBuf;
+
+using System.Collections.Generic;
+using System.Linq;
+
+namespace mbts.analytics;
+
+[ProtoContract(Name = "VolSurfaceSymbolRelationship")]
+public class VolSurfaceSymbolRelationship {
+   [ProtoMember(10)]
+   public SymbolVenue symbol { get; set; }
+
+   [ProtoMember(20)]
+   public List<SymbolVenue> connected_symbols { get; set; } = new(1);
+
+   public static VolSurfaceSymbolRelationship create(SymbolVenue symbol, params SymbolVenue[] connected_symbols) =>
+      create(symbol, connected_symbols.AsEnumerable());
+
+   public static VolSurfaceSymbolRelationship create(SymbolVenue symbol, IEnumerable<SymbolVenue> connected_symbols) {
+      var rv = new VolSurfaceSymbolRelationship() { symbol = symbol };
+      rv.connected_symbols.AddRange(connected_symbols);
+      return rv;
+   }
+}

src/client/spark/Analytics/Core/VolSurfaceSymbolStructure.cs

@@ -1,26 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace mbts.analytics
-{
-   [ProtoBuf.ProtoContract(Name = "VolSurfaceSymbolStructure")]
-   public class VolSurfaceSymbolStructure
-   {
-      [ProtoBuf.ProtoMember(10)]
-      public string master_symbol;
-      [ProtoBuf.ProtoMember(20)]
-      public List<string> slave_symbol_list;
-   }
-
-   [ProtoBuf.ProtoContract(Name = "VolSurfaceSymbolRelationship")]
-   public class VolSurfaceSymbolRelationship
-   {
-      [ProtoBuf.ProtoMember(10)]
-      public string symbol;
-      [ProtoBuf.ProtoMember(20)]
-      public List<string> connected_symbols;
-   }
-}

src/client/spark/Analytics/Misc/VolSurfaceDataFunctions.cs

@@ -1,4 +1,8 @@
-using Spark.Core.Collections;
+using bts;
+using bts.api;
+using bts.collections;
+
+using Spark.Core.Collections;
 
 using System;
 using System.Collections.Generic;
@@ -26,6 +30,9 @@ namespace mbts.analytics
             out HashSet<VolSurfaceExpiration> deleted_exps,
             out HashSet<VolSurfaceExpiration> added_exps);
          if (deleted_exps.Any() || added_exps.Any()) {
+            Log.Gen.info(
+               $"VolSurfaceDataFunctions.sync_input_expirations_to_master_list exps changed. "
+               + $"deleted: {string.Join(", ", deleted_exps)}; added: {string.Join(", ", added_exps)}");
             VolSurfaceInput rv = VolSurfaceUtilityFunctions.clone_proto(vs_input);
             rv.surface_parameters.curve_parameters_map = rv.surface_parameters.curve_parameters_map ?? new List<Tuple<VolSurfaceExpiration, VolCurveParameters>>();
             rv.surface_parameters.curve_parameters_map.RemoveAll(t => deleted_exps.Contains(t.Item1));
@@ -53,12 +60,12 @@ namespace mbts.analytics
          };
       }
 
-      public static VolSurfaceInput get_default_vs_input(VolSurfaceSettings vs_settings, IDictionary<string, IEnumerable<DateTime>> exp_list)
+      public static VolSurfaceInput get_default_vs_input(VolSurfaceSettings vs_settings, IDictionary<(string, Venue), IEnumerable<DateTime>> exp_list)
       {
          List<VolSurfaceExpiration> vs_exp_list = new List<VolSurfaceExpiration>();
-         foreach (KeyValuePair<string, IEnumerable<DateTime>> kv in exp_list)
+         foreach (var kv in exp_list)
             foreach (DateTime exp in kv.Value)
-               vs_exp_list.Add(new VolSurfaceExpiration() { symbol = kv.Key, expiration = exp });
+               vs_exp_list.Add(new(SecurityType.Option, kv.Key.Item1, exp, kv.Key.Item2));
          return get_default_vs_input(vs_settings, vs_exp_list);
       }
 
@@ -220,9 +227,9 @@ namespace mbts.analytics
       public static VolSurfaceEnums.VolSurfacePricingModel get_pricing_model(VolSurfaceSettings settings, VolSurfaceExpiration exp)
       {
          if (settings.core_settings.symbol_pricing_model_map != null)
-            foreach (Tuple<string, VolSurfaceEnums.VolSurfacePricingModel> t in settings.core_settings.symbol_pricing_model_map)
-               if (t.Item1 == exp.symbol)
-                  return t.Item2;
+            foreach (var (sym, model) in settings.core_settings.symbol_pricing_model_map)
+               if (sym == (exp.symbol, exp.venue))
+                  return model;
          return VolSurfaceEnums.VolSurfacePricingModel.BlackScholes;
       }
 
@@ -231,13 +238,13 @@ namespace mbts.analytics
          return VolSurfaceUtilityFunctions.are_protos_equal(exp_params, VolSurfaceDataFunctions.get_default_curve_params(vs_settings));
       }
 
-      public static IEnumerable<string> get_symbols(VolSurfaceInput vs_input)
+      public static IEnumerable<(string, Venue)> get_symbols(VolSurfaceInput vs_input)
       {
-         return vs_input != null ? get_symbols(vs_input.surface_settings) : new string[] { };
+         return vs_input != null ? get_symbols(vs_input.surface_settings) : Enumerable.Empty<(string, Venue)>();
       }
 
-      public static IEnumerable<string> get_symbols(VolSurfaceSettings vs_settings) =>
-         vs_settings.core_settings.symbol_pricing_model_map.empty_if_null().Select(i => i.Item1);
+      public static IEnumerable<(string, Venue)> get_symbols(VolSurfaceSettings vs_settings) =>
+         vs_settings.core_settings.symbol_pricing_model_map.empty_if_null().Select(i => (i.Key.symbol, i.Key.venue));
 
       public static VolCurveParameters get_curve_parameters(VolSurfaceInput vs_input, VolSurfaceExpiration exp)
       {
@@ -477,7 +484,7 @@ namespace mbts.analytics
       private static void _force_distinct_connected_symbols(VolSurfaceSymbolRelationship vssr) {
          var distinct_syms = vssr.connected_symbols.Distinct().ToList();
          if (distinct_syms.Count != vssr.connected_symbols.Count)
-            vssr.connected_symbols = distinct_syms;
+            vssr.connected_symbols.SetItems(distinct_syms);
       }
 
       #endregion
@@ -702,7 +709,7 @@ namespace mbts.analytics
             var this_exp = kv.Item1;
             var this_exp_params = kv.Item2;
             if (exp_has_input(vs_input.surface_settings, this_exp_params)) {
-               double this_exp_diff_days = Math.Abs(vs_exp.diff_days(this_exp));
+               double this_exp_diff_days = Math.Abs((vs_exp.expiration - this_exp.expiration).TotalDays);
                if (this_exp_diff_days < rv_diff_days) {
                   rv = this_exp;
                   rv_diff_days = this_exp_diff_days;

src/client/spark/Analytics/Misc/VolSurfaceUtilityFunctions.cs

@@ -373,17 +373,17 @@ namespace mbts.analytics {
       /// <summary>
       /// Puts VolSurfaceSymbolRelationship's into a map of symbol vs its connected symbols
       /// </summary>
-      static IDictionary<string, HashSet<string>> _hash_relationship(
+      static IDictionary<(string, Venue), HashSet<(string, Venue)>> _hash_relationship(
          VolSurfaceSettings vs_settings, IEnumerable<VolSurfaceSymbolRelationship> rels) {
          var rv = rels.empty_if_null().ToDictionaryCrushed(
-            r => r.symbol, r => r.connected_symbols.ToHashSet());
+            r => (r.symbol.symbol, r.symbol.venue), r => r.connected_symbols.Select(s => (s.symbol, s.venue)).ToHashSet());
          foreach (var s in VolSurfaceDataFunctions.get_symbols(vs_settings)) {
-            rv.GetOrAdd(s, _ => new HashSet<string>()).Add(s);
+            rv.GetOrAdd(s, _ => new()).Add(s);
          }
          return rv;
       }
 
-      static HashSet<string> _get_slave_symbols(VolSurfaceSettings vs_settings) {
+      static HashSet<(string, Venue)> _get_slave_symbols(VolSurfaceSettings vs_settings) {
          /// just all - masters
          var slave_symbols = VolSurfaceDataFunctions.get_symbols(vs_settings).ToHashSet();
          var master_symbols = get_master_symbols(vs_settings);
@@ -391,12 +391,12 @@ namespace mbts.analytics {
          return slave_symbols;
       }
 
-      static HashSet<string> _get_masters_for_symbol(VolSurfaceSettings vs_settings, string symbol) =>
+      static HashSet<(string, Venue)> _get_masters_for_symbol(VolSurfaceSettings vs_settings, string symbol, Venue venue) =>
          _hash_relationship(
             vs_settings, vs_settings.core_settings.master_slave_relationships)
-            .GetValueOrDefault(symbol) ?? new();
+            .GetValueOrDefault((symbol, venue)) ?? new();
 
-      public static HashSet<string> get_master_symbols(VolSurfaceSettings vs_settings) {
+      public static HashSet<(string, Venue)> get_master_symbols(VolSurfaceSettings vs_settings) {
          var master_slaves = _hash_relationship(
             vs_settings, vs_settings.core_settings.master_slave_relationships ?? new());
          return master_slaves.Values.SelectMany().ToHashSet();
@@ -411,7 +411,7 @@ namespace mbts.analytics {
          var master_symbols = get_master_symbols(vs_inp.surface_settings);
          return vs_inp.surface_parameters.curve_parameters_map
             .Where(t =>
-               master_symbols.Contains(t.Item1.symbol) ||
+               master_symbols.Contains((t.Item1.symbol, t.Item1.venue)) ||
                (include_unlinked_slaves && t.Item2.core_parameters.unhinge_exp_from_master))
             .Select(t => t.Item1)
             .ToHashSet();
@@ -424,25 +424,25 @@ namespace mbts.analytics {
          var slave_symbols = _get_slave_symbols(vs_inp.surface_settings);
          return vs_inp.surface_parameters.curve_parameters_map
             .Select(t => t.Item1)
-            .Where(t => slave_symbols.Contains(t.symbol))
+            .Where(t => slave_symbols.Contains((t.symbol, t.venue)))
             .ToHashSet();
       }
 
       public static bool is_slave_expiration(
          VolSurfaceExpiration vs_exp, VolSurfaceSettings vs_settings) =>
-         _get_slave_symbols(vs_settings).Contains(vs_exp.symbol);
+         _get_slave_symbols(vs_settings).Contains((vs_exp.symbol, vs_exp.venue));
 
       public static bool is_master_expiration(
          VolSurfaceExpiration vs_exp, VolSurfaceSettings vs_settings) =>
-         get_master_symbols(vs_settings).Contains(vs_exp.symbol);
+         get_master_symbols(vs_settings).Contains((vs_exp.symbol, vs_exp.venue));
 
       public static VolSurfaceExpiration find_master_expiration(
          VolSurfaceExpiration slave_vs_exp,
          VolSurfaceSettings vs_settings,
          IEnumerable<VolSurfaceExpiration> vs_exps
       ) {
-         var master_symbols = _get_masters_for_symbol(vs_settings, slave_vs_exp.symbol);
-         vs_exps = vs_exps.Where(e => master_symbols.Contains(e.symbol));
+         var master_symbols = _get_masters_for_symbol(vs_settings, slave_vs_exp.symbol, slave_vs_exp.venue);
+         vs_exps = vs_exps.Where(e => master_symbols.Contains((e.symbol, e.venue)));
          VolSurfaceExpiration rv = find_nearest(vs_exps, slave_vs_exp, exp_date_diff) ??
             new() { expiration = DateTime.MinValue };
          /// TODO get this 2 from settings
@@ -457,10 +457,10 @@ namespace mbts.analytics {
          var md_rel_map = _hash_relationship(
             vs_input.surface_settings,
             vs_input.surface_settings.core_settings.implied_vol_marketdata_relationships);
-         var source_symbols = md_rel_map.GetValueOrDefault(vs_exp.symbol) ?? new();
+         var source_symbols = md_rel_map.GetValueOrDefault((vs_exp.symbol, vs_exp.venue)) ?? new();
          var source_exps = vs_input.surface_parameters.curve_parameters_map
             .Select(t => t.Item1)
-            .Where(e => source_symbols.Contains(e.symbol));
+            .Where(e => source_symbols.Contains((e.symbol, e.venue)));
          var rv = find_nearest(source_exps, vs_exp, exp_date_diff) ??
             new() { expiration = DateTime.MinValue };
          /// TODO get this from settings
@@ -546,20 +546,6 @@ namespace mbts.analytics {
          return polynomial_str;
       }
 
-      /// <summary>
-      /// Transforms <paramref name="implied_vol_md_source"/> to a map of symbols -> md source
-      /// </summary>
-      /// <returns>
-      /// An <see cref="IDictionary{string, string}"/> whose keys are slave symbols and whose
-      /// values are the corresponding master symbols
-      /// </returns>
-      public static IDictionary<string, string> flatten_implied_vol_md_source(
-         IEnumerable<VolSurfaceSymbolStructure> implied_vol_md_source) =>
-         implied_vol_md_source.empty_if_null()
-            .SelectMany(s => s.slave_symbol_list
-               .Select(i => (i, s.master_symbol)))
-            .ToDictionary();
-
       public static double restrict_bw_min_and_max(
          double v, double min, double max) => v.max(min).min(max);
 
@@ -807,7 +793,7 @@ namespace mbts.analytics {
          IDictionary<VolSurfaceExpiration, IEnumerable<double>> exp_strike_map) =>
          exp_strike_map
             .SelectMany(kv => {
-               var sec_exp = kv.Key.to_security_expiration(type: SecurityType.Option);
+               var sec_exp = kv.Key.with(security_type: SecurityType.Option).to_security_expiration();
                return kv.Value.Select(k => new SecurityStrike(sec_exp, k));
             })
             .ToHashSet();

src/client/spark/Analytics/Pricing/VolSurfaceInterpGreeksProducer.cs

@@ -154,11 +154,8 @@ namespace mbts.analytics {
       private IDictionary<SecurityExpiration, AnalyticsExpirationParameters> _sync_exp_params() {
          var master_exp_keys =
             _exp_strikes_map.Keys.Select(k =>
-               new SecurityExpiration(Venue.Unset, SecurityType.Option, k.symbol, k.expiration));
-         // TODO: add exchange to key (so we don't have to 'Crush' the duplicate keys)
-         var exp_params_no_exch = _exp_params_map.ToDictionaryCrushed(
-            kv => kv.Key.unset_exchange(), kv => kv.Value);
-         return exp_params_no_exch.sync(master_exp_keys, true,
+               new SecurityExpiration(k.venue, SecurityType.Option, k.symbol, k.expiration));
+         return _exp_params_map.sync(master_exp_keys, true,
             exp => AnalyticsExpirationParameters.empty);
       }
 

src/client/spark/Analytics/SupportingData/VolSurfaceNonVolInput.cs

@@ -4,6 +4,8 @@ using System.Linq;
 
 using bts.api;
 
+using mbts.spark;
+
 namespace mbts.analytics {
    /// This is the input data (other than vol) that
    /// is used for vol/price calculations. It is not
@@ -30,7 +32,7 @@ namespace mbts.analytics {
       public IDictionary<VolSurfaceExpiration, AnalyticsExpirationParameters> get_option_exp_input() {
          var list = exp_input_map.Where(kv => kv.Key.security_type == SecurityType.Option);
          return list.ToDictionary(
-            kv => new VolSurfaceExpiration() { symbol = kv.Key.symbol, expiration = kv.Key.expiration },
+            kv => kv.Key.to_vs_exp(),
             kv => kv.Value);
       }
 

src/client/spark/Analytics/Tests/MarketFitFunctionsTests.cs

@@ -5,6 +5,7 @@ using System.Text;
 using System.Threading.Tasks;
 using System.IO;
 using NUnit.Framework;
+using bts.api;
 
 namespace mbts.analytics {
    [TestFixture]
@@ -17,14 +18,15 @@ namespace mbts.analytics {
             minimum_top_book_size = 1,
             graph_update_period_ms = 1000
          };
-         IDictionary<VolSurfaceExpiration, IDictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>>> bbo = _parse_bbo("spx", @"P:\Dropbox\Data\VolManager\TestData\processed_bbo_spx.csv");
+         var venue = Venue.Cboe;
+         IDictionary<VolSurfaceExpiration, IDictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>>> bbo = _parse_bbo("spx", venue, @"P:\Dropbox\Data\VolManager\TestData\processed_bbo_spx.csv");
          DateTime exp = DateTime.Parse("12/19/2015");
          DateTime theo_time = new DateTime(2015, 10, 13, 9, 0, 0);
          double cal_t = VolSurfaceUtilityFunctions.to_years(exp.Subtract(theo_time)),
                 biz_t = cal_t * 0.95,
                 rate = 0.3,
                 fwd = 2009.25;
-         IDictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>> k_map = bbo[new VolSurfaceExpiration() { expiration = exp, symbol = "spx" }];
+         IDictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>> k_map = bbo[new() { expiration = exp, symbol = "spx", venue = venue }];
          IDictionary<double, double> put_mid_prices, call_mid_prices;
          VolSurfaceCoreFunctions.get_filtered_bbo_mid(fit_settings, fwd, k_map, VolSurfaceEnums.MarketSide.Mid, out put_mid_prices, out call_mid_prices);
          int n_iters = 10;
@@ -51,7 +53,7 @@ namespace mbts.analytics {
          bts.Log.Testing.info(String.Format("MarketFitFunctionsTests.test_implied_vols puts = {0}, calls = {1}, runs = {2}, total time = {3}", put_mid_prices.Count, call_mid_prices.Count, n_iters, stop_watch.ElapsedMilliseconds));
       }
 
-      private static IDictionary<VolSurfaceExpiration, IDictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>>> _parse_bbo(string sym, string file_name) {
+      private static IDictionary<VolSurfaceExpiration, IDictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>>> _parse_bbo(string sym, Venue venue, string file_name) {
          IDictionary<VolSurfaceExpiration, IDictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>>> t_rv = new Dictionary<VolSurfaceExpiration, IDictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>>>();
          StreamReader sr = new StreamReader(file_name);
          while (!sr.EndOfStream) {
@@ -65,7 +67,7 @@ namespace mbts.analytics {
             bool is_call = toks[c++].ToLower().Contains("call");
             DateTime exp = DateTime.Parse(toks[c++]);
             double strike = Double.Parse(toks[c++]);
-            VolSurfaceExpiration vs_exp = new VolSurfaceExpiration() { symbol = sym, expiration = exp };
+            var vs_exp = VolSurfaceExpiration.Default.with(sym, exp, SecurityType.Option, venue);
             VolSurfaceMDUnit vs_bbo_unit = new VolSurfaceMDUnit(bid_price, bid_size, ask_price, ask_size, Double.NaN);
             IDictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>> k_map;
             if (!t_rv.TryGetValue(vs_exp, out k_map))