all exectuion work

The previous regex was overly ambitious (and needlessly complex as a
result). The new regex is simpler, and doesn't impose bounds on month
and day ints, aside from requiring 1-2 digits.

Bugs (fixed):
  * The DataStreamExpressionInterpretter fails to parse certain date
    constants, such as "8/30/2024"

.devcontainer/Dockerfile

@@ -87,6 +87,9 @@ RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
 # Pcap
 RUN apt-get update && apt-get install -y libpcap0.8-dev
 
+# Databento (deps)
+RUN apt-get update && apt-get -y install libzstd-dev
+
 # Set up default executables (update-alternatives)
 RUN \
    update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \

.gitattributes

@@ -1,3 +1,3 @@
-*.cmd eol=crlf
-*.cs eol=crlf
-*.sh eol=lf
+*.cmd text=auto eol=crlf
+*.cs text=auto eol=crlf
+*.sh text=auto eol=lf

.vscode/launch.json

@@ -26,6 +26,18 @@
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
+    {
+      "name": "(gdb) Launch",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
+      "args": ["run"],
+      "stopAtEntry": false,
+      "cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
+      "environment": [],
+      "externalConsole": false,
+      "MIMode": "gdb",
+    },
     {
       "name": "DataStreamQueryAnalyzer",
       "type": "coreclr",

.vscode/tasks.json

@@ -526,6 +526,28 @@
         "color": "terminal.ansiGreen",
       },
     },
+    {
+      "label": "Run PubSubServer (Debug | Skip Build)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "PubSubServer",
+      },
+      "windows": {
+        "command": "PubSubServer.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
     {
       "label": "Run PubSubServer (Release)",
       "dependsOn": "Build DataStream (Release)",
@@ -549,6 +571,28 @@
         "color": "terminal.ansiGreen",
       },
     },
+    {
+      "label": "Run PubSubServer (Release | Skip Build)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "PubSubServer",
+      },
+      "windows": {
+        "command": "PubSubServer.exe",
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
+      ],
+      "options": {
+        "cwd": "${workspaceFolder}/build/Release/bin/net7.0"
+      },
+      "icon": {
+        "id": "run",
+        "color": "terminal.ansiGreen",
+      },
+    },
     {
       "label": "Run Spark",
       "dependsOn": "Build Spark",
@@ -684,6 +728,36 @@
         "id": "wrench"
       },
     },
+    {
+      "label": "View PubSubServer Logs (Last Run)",
+      "type": "process",
+      "command": "",
+      "linux": {
+        "command": "tools/logging/show_last_run.sh",
+        "args": [
+          "${env:HOME}/.local/share/DataStream/data/logging.PubSubServer",
+          "Starting PubSubServer"
+        ]
+      },
+      "windows": {
+        "command": "tools/logging/show_last_run.cmd",
+        "args": [
+          "DataStream",
+          "PubSubServer.exe",
+          "Starting PubSubServer"
+        ]
+      },
+      "presentation": {
+        "reveal": "always",
+        "focus": true,
+        "panel": "shared",
+        "showReuseMessage": false,
+        "close": true,
+      },
+      "icon": {
+        "id": "wrench"
+      },
+    },
   ],
   "inputs": [
     {

CMakeLists.txt

@@ -132,6 +132,8 @@ else()
       COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
       COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
       COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
+      COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/${CMAKE_BUILD_TYPE}/libonixs-fix-engine.so.4 ${CMAKE_BINARY_DIR}/server
+      
       COMMAND
         ln ARGS -sf
         ${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0

src/DataStreamSuite/DataStream/Primitives/FileStreams/DumpLoaderStream.cs

@@ -1,49 +0,0 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Text;
-// using System.IO;
-
-// using System.Runtime.Serialization.Formatters.Binary;
-
-// namespace BCG
-// {
-//    public class DumpLoaderStream : ViewStream, ISideEffectStream
-//    {
-//       //wtf
-//       //public class Exception : System.Exception
-//       //{public Exception(string msg) : base(msg) { }}
-
-//       public DumpLoaderStream(string fileName, int secondsBetweenPolls)
-//       {
-//          mySetRealSource(
-//             new PollStream(() => new LoaderStream(fileName).rows, secondsBetweenPolls, 
-//                ViewStreamDatabase.instance));
-//       }
-
-//       public DumpLoaderStream(string fileName)
-//       {
-//         if (fileName == null)
-//            throw new LoaderStream.Exception("File name cannot be null");
-
-//         for (int i = 0; i < 5; i++)
-//            try
-//            {
-//               using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
-//               {
-//                  IClientDeserializer deser = new ClientUniversalDeserializer();
-//                  byte[] the_ops;
-//                  Array the_rows;
-//                  deser.deserialize(new Stream[] { fs }, out the_ops, out the_rows);
-
-//                  mySetRealSource(new BlueRowCollectionStaticStream((IBlueRowCollection)the_rows.GetValue(0)));
-//               }
-//               break;
-//            }
-//            catch (IOException)
-//            {
-//               System.Threading.Thread.Sleep(50);
-//            }
-
-//       }
-//    }
-// }

src/DataStreamSuite/DataStream/Primitives/FileStreams/LoaderStream.cs

@@ -1,33 +1,7 @@
 using BCG;
 
 namespace bts {
-   //[DataStreamUserRespresentation("saver")]
-   // public class SaverStream : ViewStream {
-   //    private readonly ViewStream _vs;
-   //    private readonly string _file_name;
-   //    public SaverStream(ViewStream vs, string file_name) {
-   //       _vs = vs;
-   //       _file_name = file_name;
-   //    }
-
-   //    protected override void myOnFirstSubscriber() {
-   //       _vs.onChange += _handle_dp;
-   //    }
-
-   //    protected override void myOnLastSubscriber() {
-
-   //    }
-
-   //    private void _handle_dp(object dp_obj) {
-   //       DataPacket dp = dp_obj as DataPacket;
-   //       byte[] serialized_update = null;
-   //       var _serializer = new UniversalStreamIOSerializer();
-   //       serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.CreateMessage(dp.newRows)));
-   //       StreamSerializationUtils.write_to_file(serialized_update, _file_name);
-   //       myFireEvent(dp);
-   //    }
-   // }
-
+   // This is the other side of the SaverStream, its "loads" the stream and fires once
    [DataStreamUserRespresentation("loader")]
    public class LoaderStream : ViewStream {
 

src/DataStreamSuite/DataStream/Primitives/FileStreams/PersistorStream.cs

@@ -5,10 +5,11 @@ using System.Text;
 using System.IO;
 using Google.Protobuf;
 using System.Globalization;
-using bts;
+using BCG;
 
-namespace BCG {
-   // Writes all the udpates to a stream in a file.
+namespace bts {
+   // Writes all the udpates to a stream in a file or multiples files dependong on the rollover size (as opposed to Saver
+   // which just writes one update, as a create)
    // File naming:
    // token_epoch_yyyyMMdd_HHmmss
    // File location: directory(passed)\token\
@@ -94,10 +95,10 @@ namespace BCG {
       void _on_change(object new_value) {
          DataPacket dp = new_value as DataPacket;
          byte[] serialized_update = null;
-         bool _write_create = _serializer == null || _n_updates_in_current_file == 0;
+         bool first_write = _serializer == null || _n_updates_in_current_file == 0;
          _serializer = _serializer ?? new UniversalStreamIOSerializer();
-         if (_write_create) {
-            // new files always starts with create
+         if (first_write) {
+            // first file always starts with create
             serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.CreateMessage(dp.newRows)));
          } else
             serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows)));
@@ -105,8 +106,8 @@ namespace BCG {
          long utc_epoch = epoch_now();
 
          //time, length of data, data
-         bw.Write(utc_epoch);
-         bw.Write(serialized_update.Length);
+         //bw.Write(utc_epoch);
+         //bw.Write(serialized_update.Length);
          bw.Write(serialized_update);
          _out_fs.Flush();
          myFireEvent(dp);

src/DataStreamSuite/DataStream/Primitives/FileStreams/PlaybackStream.cs

@@ -1,190 +0,0 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Linq;
-// using System.Text;
-// using System.IO;
-// using System.Runtime.Serialization.Formatters.Binary;
-// using BCG.Utilities.General;
-// using bts.collections;
-// using System.Reflection.Metadata;
-
-
-// namespace BCG {
-
-//    public class PlaybackStream : ViewStream, IWorker {
-//       private readonly PlaybackType _type;
-//       private BlueTable _current_table;
-//       private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
-//       // these are the packets already downloaded from the current file
-//       private List<PlaybackPacket> _packets = new List<PlaybackPacket>();
-//       //private IEnumerable<FileStream> _file_streams = new List<FileStream>();
-//       private IEnumerable<string> _file_names;
-//       private Tuple<int, FileStream> _current_fs = new Tuple<int, FileStream>(-1, null);
-//       private DateTime _playback_t0, _playback_t1;
-//       private bool _all_files_read = false;
-//       SnapshotData _snapshot_data = null; // only initialized when in snapshot mode
-
-//       public PlaybackStream(string datetime_start, string datetime_end, string directory_name, string token) {
-//          _type = PlaybackType.Streaming;
-//          _playback_t0 = PersistorStream.parse_datetime(datetime_start);
-//          _playback_t1 = PersistorStream.parse_datetime(datetime_end);
-//          string token_directory_name = $"{directory_name}\\{token}";
-//          bts.Log.Gen.info($"PlaybackStream t0: {_playback_t0} t1: {_playback_t1} dir: {token_directory_name}");
-//          List<string> files_in_range = new List<string>();
-//          if (Directory.Exists(token_directory_name)) {
-//             _file_names =
-//          Directory.GetFiles(token_directory_name).Where(f => {
-//             DateTime file_time = PersistorStream.parse_datetime_from_filename(f);
-//             return file_time > _playback_t0 && file_time < _playback_t1;
-//          });
-//          }
-//       }
-
-//       protected override void myOnFirstSubscriber() {
-//          FileStream fs = new FileStream(_snapshot_data.snapshot_file, FileMode.Open, FileAccess.Read);
-//          _snapshot_data.packets = StreamPersistorUtils.load_packets_from_file(fs);
-//          PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_snapshot_data.packets, _snapshot_data.snapshot_time);
-//          if (snapshot_packet != null) {
-//             foreach (PlaybackPacket p in _snapshot_data.packets) {
-//                byte[] ops;
-//                Array row_array;
-//                _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
-//                for (int i = 0; i < ops.Length; i++) {
-//                   IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-//                   byte op = ops[i];
-//                   if (op == StreamOpCode.create) {
-//                      IBlueRowCollection full_table = rows_args[0];
-//                      _current_table = new BlueTable(full_table);
-//                   } else if (op == StreamOpCode.multipleRemoveAdd) {
-//                      //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-//                      IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-//                      IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-//                      BlueDataUtility.absorbChanges(_current_table, dels, adds);
-//                   } else {
-//                      bts.Log.assert(false, $"Oprator: {op} is not expected");
-//                   }
-//                   if (p.idx == snapshot_packet.idx) {
-//                      myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
-//                      break;
-//                   }
-//                }
-//             }
-//          }
-//       }
-
-
-//       private void _load_packets_from_current_file() {
-//          int curr_idx = _current_fs.Item1;
-//          FileStream curr_fs = _current_fs.Item2;
-//          _packets = new List<PlaybackPacket>();
-//          _packets.AddRange(StreamPersistorUtils.load_packets_from_file(curr_fs));
-//       }
-
-//       private bool current_file_all_replayed {
-//          get {
-//             bool rv = true;
-//             _packets.ForEach(p => rv = rv && p.replayed);
-//             return rv;
-//          }
-//       }
-
-//       protected override void myOnLastSubscriber() {
-//          var fs = _current_fs.Item2;
-//          if (fs != null) {
-//             fs.Close();
-//          }
-//          ViewStreamDatabase.instance.removeWorker(this);
-//       }
-
-//       // public event PlaybackStreamProgress onProgress;
-//       // public event Action onFileExhausted;
-//       private PlaybackPacket _locate_packet_to_replay() {
-//          foreach (var p in _packets) {
-//             // returns the first non replayed packet
-//             if (!p.replayed)
-//                return p;
-//          }
-//          //else
-//          {
-//             double secondsElapsed = new BinaryReader(myFileReader).ReadDouble();
-//             myNextPacketTime = myPlaybackStart + new TimeSpan(0, 0, 0, 0, (int)(1000 * secondsElapsed / myPlaybackSpeed));
-//             if (onProgress != null)
-//                onProgress(myFileReader.Position, myFileReader.Length);
-//          }
-//       }
-
-//       #region IWorker Members
-
-//       public void doWork() {
-//          if (_all_files_read)
-//             return;
-//          if (current_file_all_replayed) {
-//             _load_next_file();
-//          }
-//          PlaybackPacket curr_packet = _locate_packet_to_replay();
-//          if (curr_packet.idx == 0) { // first packet in the file, fire it rightaway 
-//             byte[] ops;
-//             Array row_array;
-//             // what a stupid interface!
-//             _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
-//             _fire_packet(ops, row_array);
-//             curr_packet.replay_timestamp = PersistorStream.epoch_now();
-//          } else {
-//             PlaybackPacket prev_packet = _get_previous_packet(curr_packet);
-//             // this is the time we need to fire (replay) this packet
-//             long time_to_fire = prev_packet.replay_timestamp + curr_packet.offset_from_previous;
-//             long now_epoch = PersistorStream.epoch_now();
-//             // fire when we are past the 
-//             if (now_epoch >= time_to_fire) {
-//                byte[] ops;
-//                Array row_array;
-//                _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
-//                _fire_packet(ops, row_array);
-//                curr_packet.replay_timestamp = PersistorStream.epoch_now();
-//             }
-//          }
-//       }
-
-//       private void _fire_packet(byte[] ops, Array row_array) {
-//          for (int i = 0; i < ops.Length; i++) {
-//             IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
-//             byte op = ops[i];
-//             if (op == StreamOpCode.create) {
-//                IBlueRowCollection full_table = rows_args[0];
-//                _current_table = new BlueTable(full_table);
-//                //_current_table.addRows(full_table);
-//                myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
-//                                           (myLastValue as DataPacket).newRows));
-//             } else if (op == StreamOpCode.multipleRemoveAdd) {
-//                //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
-//                IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
-//                IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
-//                BlueDataUtility.absorbChanges(_current_table, dels, adds);
-//                myFireEvent(new DataPacket(_current_table, adds, dels));
-//             } else {
-//                bts.Log.assert(false, $"Oprator: {op} is not expected");
-//             }
-
-//          }
-//       }
-
-//       private void _load_next_file() {
-//          var new_idx = _current_fs.Item1 + 1;
-//          if (new_idx >= _file_names.Count()) {
-//             _all_files_read = true;
-//             bts.Log.Gen.info("All files read");
-//             return;
-//          }
-//          var prev_fs = _current_fs.Item2;
-//          if (prev_fs != null)
-//             prev_fs.Close();
-//          var new_filename = _file_names.ElementAt(new_idx);
-//          bts.Log.Gen.info("PlaybackStream_load_next_file new_filename=: {new_filename}");
-//          var new_fs = new FileStream(new_filename, FileMode.Open, FileAccess.Read);
-//          _current_fs = new Tuple<int, FileStream>(new_idx, new_fs);
-//          _load_packets_from_current_file();
-//       }
-
-//       #endregion
-//    }
-// }

src/DataStreamSuite/DataStream/Primitives/FileStreams/RecorderStream.cs

@@ -1,68 +0,0 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Linq;
-// using System.Text;
-// using System.IO;
-
-// namespace BCG
-// {
-//    public class RecorderStream : ViewStream
-//    {
-//       private ViewStream mySource;
-//       private DateTime myStartTime;
-//       private FileStream myOutFile;
-//       private string myFileName;
-//       private IBlueDataSerializer mySerializer;
-
-//       public RecorderStream(ViewStream src, string recordFile)
-//       {
-//          mySource = src;
-//          myFileName = recordFile;
-
-//          myFileName = myFileName.Replace("%DATE%", DateTime.Now.ToString("yyyyMMdd_HHmmss")); 
-//       }
-
-//       protected override void myOnFirstSubscriber()
-//       {
-//          myStartTime = DateTime.Now;         
-//          mySource.onChange += new ChangeDelegate(mySource_onChange);
-//          myOutFile = new FileStream(myFileName, FileMode.OpenOrCreate);
-//       }
-
-//       protected override void myOnLastSubscriber()
-//       {
-//          mySource.onChange -= mySource_onChange;
-//          myOutFile.Close();
-//       }
-
-//       void mySource_onChange(object newValue)
-//       {
-//          DataPacket dp = (DataPacket)newValue;         
-
-//          BlueTable toWrite = new BlueTable(dp.schema);
-
-//          if (mySerializer == null)
-//          {
-//             mySerializer = new UniversalSerializer(dp.schema);
-//          }
-//          foreach (IBlueRow r in dp.newRows)
-//             toWrite.addRow(r);
-
-//          byte[] result;
-
-//          if (myLastValue == null)
-//             mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out result);
-//          else
-//             mySerializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out result);
-
-//          BinaryWriter bw = new BinaryWriter(myOutFile);
-//          //write down what time this happened
-//          bw.Write((DateTime.Now - myStartTime).TotalSeconds);
-//          bw.Write(result.Length);
-//          bw.Write(result);         
-//          myOutFile.Flush();
-
-//          myFireEvent(dp);
-//       }
-//    }
-// }

src/DataStreamSuite/DataStream/Primitives/FileStreams/SaverStream.cs

@@ -5,7 +5,8 @@ using System.IO;
 using BCG;
 
 namespace bts {
-   // this saves the total state in the filr provided in the cons
+   // this saves the updates as a Create on the same file, each update calls -save, which
+   // reqrites the whole data to the packet. The other side of this stream is the Loader
    [DataStreamUserRespresentation("saver")]
    public class SaverStream : ViewStream {
       private ViewStream _stream;

src/DataStreamSuite/DataStream/Primitives/FileStreams/StreamPersistorUtils.cs

@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Globalization;
 using System.IO;
+using bts;
 
 
 namespace BCG {

src/DataStreamSuite/DataStreamExpressions/DataStreamExpressionTokens.lex

@@ -9,7 +9,15 @@
 Letter                   [a-zA-Z]
 BooleanConstant          'true'|'false'
 CSStringConstant         \"([^\"\\]*|\\.)*\"
-DateTimeConstant         ([1-9]|'10'|'11'|'12')\/([1-9]|([1-2][0-9])|'30'|'31')\/[0-9][0-9][0-9]*
+YearInt                  [0-9]{4}
+MonthInt                 [0-9]{1,2}
+DayInt                   [0-9]{1,2}
+/**
+ * DateTimeConstant supports American (mm/dd/yyyy) and European (yyyy/mm/dd)
+ * formats. Month and day can be 1 or 2 digits. It does not validate that the
+ * numbers represent an actual date.
+ */
+DateTimeConstant         ({MonthInt}\/{DayInt}\/{YearInt}|{YearInt}\/{MonthInt}\/{DayInt})
 IntegerConstant          -?[0-9]+
 NullConstant             'null'
 RealConstant             -?[0-9]+\.[0-9]+

src/DataStreamSuite/DataStreamPublisher/Configurations/MarketDataPublisher.json

@@ -1,6 +1,6 @@
 {
   "user": "bts",
-  "sparkd": "http://127.0.0.1:8000/config",
+  "sparkd": "http://192.168.1.80:8000/config",
   "pubsub": {
     "milliseconds_between_sub_info_updates": 1000,
     "milliseconds_between_time_slice_updates": 5000,
@@ -105,5 +105,25 @@
         }
       ]
     }
+    // {
+    //   "name": "news_data",
+    //   "streams": [
+    //     {
+    //       "name": "benzinga_news",
+    //       "expression": "benzingaNews(\"mongodb://localhost:27017\",\"local\",\"news_headlines\", \"30s\", \"5d\")",
+    //       "is_on_websocket": true
+    //     },
+    //     {
+    //       "name": "benzinga_news_symbols",
+    //       "expression": "benzingaNewsSymbols(benzinga_news)",
+    //       "is_on_websocket": true
+    //     },
+    //     {
+    //       "name": "benzinga_news_symbol_state",
+    //       "expression": "benzingaNewsSymbolState(benzinga_news)",
+    //       "is_on_websocket": true
+    //     }
+    //   ]
+    // }
   ]
 }

src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json

@@ -16,14 +16,14 @@
   "publisher_routes": [
     {
       "name": "default",
-      "server_address": "bluto://192.168.1.80:12345",
+      "server_address": "bts-cloud://192.168.1.80:12345",
       "tag": ".*"
     }
   ],
   "subscriber_routes": [
     {
       "name": "default",
-      "server_address": "bluto://192.168.1.80:12345",
+      "server_address": "bts-cloud://192.168.1.80:12345",
       "tag": ".*"
     }
   ],

src/DataStreamSuite/PubSub/Config/PubSubConfig.cs

@@ -117,10 +117,10 @@ public class PubSubConfig {
          WaitPeriodAfterGc = 1000,
          SimpleEchoServer = false,
          PublisherRoutes = new() {
-            new("bts-cloud://127.0.0.1:12345", ".*")
+            new("bluto://127.0.0.1:12345", ".*")
          },
          SubscriberRoutes = new() {
-            new("bts-cloud://127.0.0.1:12345", ".*")
+            new("bluto://127.0.0.1:12345", ".*")
          }
       };
       return rv;

src/DataStreamSuite/PubSub/Server/Communication/ManagedTCP/ManagedTCPMessageHub.cs

@@ -285,4 +285,4 @@ namespace BCG {
 
 
 
-}
+}

src/Spark.Admin.WebApp/src/Accounts/Types.fs

@@ -17,7 +17,7 @@ type Tab =
   | PositionManagementTab
   | MarginTab
 
-let venues = [| Venue.BINANCE; Venue.BinanceUs; Venue.BinanceUsdsFutures; Venue.COINBASE; Venue.DERIBIT; Venue.GLBX; Venue.ICE; Venue.SIM; |]
+let venues = [| Venue.BINANCE; Venue.BinanceUs; Venue.BinanceUsdsFutures; Venue.CBOE; Venue.COINBASE; Venue.DERIBIT; Venue.GLBX; Venue.ICE; Venue.TAIFEX; Venue.SIM; |]
 
 let securityTypes = [| SecurityType.SPOT; SecurityType.FUTURE; SecurityType.OPTION; SecurityType.SPREAD; |]
 
@@ -69,6 +69,11 @@ type ICEExchangeField =
   | AccountCode of string
   | MiscTagMapping of JSMap<string>
 
+type TaifexExchangeField =
+  | CTI of CTICode
+  | Origin of CustomerOrFirm
+  | ClearingAccount of string
+
 type SIMExchangeField =
   | CTI of CTICode
   | Origin of CustomerOrFirm

src/Spark.Admin.WebApp/src/Accounts/View.fs

@@ -797,9 +797,9 @@ let venueToString (venue:bts.api.Venue) =
     | bts.api.Venue.BINANCE -> "BINANCE"
     | bts.api.Venue.BinanceUs -> "BinanceUs"
     | bts.api.Venue.BinanceUsdsFutures -> "BinanceUsdsFutures"
-    | bts.api.Venue.COINBASE -> "COINBASE"
     | bts.api.Venue.FTX -> "FTX"
     | bts.api.Venue.OKX -> "OKX"
+    | bts.api.Venue.TAIFEX -> "TAIFEX"
 
 let exchange_display expirationNameList =
   let s = sprintf "%O" expirationNameList.product.exchange

src/client/spark/AW2/AnalyticsParameters/AnalyticsExpirationParametersSettings.cs

@@ -1,4 +1,6 @@
 namespace mbts.spark {
+   using bts.api;
+
    using ProtoBuf;
 
    [ProtoContract]
@@ -10,6 +12,12 @@ namespace mbts.spark {
       [ProtoMember(10)]
       public string[] selected_symbols {get;set;}
 
+      [ProtoMember(11)]
+      public Venue[] selected_venues {get;set;}
+
+      [ProtoMember(12)]
+      public SecurityType[] selected_types {get;set;}
+
       [ProtoMember(20)]
       public bool auto_copy_rates_is_checked {get;set;}
 

src/client/spark/AW2/AnalyticsParameters/AnalyticsExpirationParametersVM.cs

@@ -44,6 +44,8 @@ namespace mbts.spark {
       readonly DeferrableAction _raise_selected_rows_changed;
       readonly IRaiseCanExecuteChangedCommand _set_driver_to_nearest_exp_cmd;
       IList<string>? _pending_selected_symbols;
+      IList<Venue>? _pending_selected_venues;
+      IList<SecurityType>? _pending_selected_types;
 
       AnalyticsExpirationParametersMap? _parameter_map;
       (ICommand cmd, IDisposable can_exe_sub)? _smart_copy_roll_target_to_selection_cmd;
@@ -56,9 +58,13 @@ namespace mbts.spark {
          _auto_copy_rolls_is_checked = backing(nameof(auto_copy_rolls_is_checked), false);
          _raise_selected_rows_changed = new(() => selected_rows_changed?.Invoke(_selected_rows));
          var selected_symbol_ref_count = new RefCountMap<string>();
+         var selected_venue_ref_count = new RefCountMap<Venue>();
+         var selected_type_ref_count = new RefCountMap<SecurityType>();
          _sync_visible_rows = new(() =>
             visible_rows.sync_with(
-               rows.Where(x => selected_symbol_ref_count.ContainsKey(x.security.symbol))));
+               rows.Where(x => selected_symbol_ref_count.ContainsKey(x.security.symbol)
+                  && selected_venue_ref_count.ContainsKey(x.security.exchange)
+                  && selected_type_ref_count.ContainsKey(x.security.security_type))));
          non_options_with_empty = new ObservableAggregateCollection<SecurityExpiration>(
             new() { SecurityExpiration.empty }, non_options);
 
@@ -67,6 +73,8 @@ namespace mbts.spark {
 
          if (startup_data is not null) {
             _pending_selected_symbols = startup_data.selected_symbols;
+            _pending_selected_venues = startup_data.selected_venues;
+            _pending_selected_types = startup_data.selected_types;
             auto_copy_rolls_is_checked = startup_data.auto_copy_rolls_is_checked;
             auto_copy_rates_is_checked = startup_data.auto_copy_rates_is_checked;
          }
@@ -103,6 +111,26 @@ namespace mbts.spark {
                   }
                });
             }),
+            ObservableAutoWrapper.connect_item_hooks(selected_venues, sym => {
+               if (selected_venue_ref_count.increment_adds_key(sym)) {
+                  _sync_visible_rows.Invoke();
+               }
+               return Disposable.Create(() => {
+                  if (selected_venue_ref_count.decrement_removes_key(sym)) {
+                     _sync_visible_rows.Invoke();
+                  }
+               });
+            }),
+            ObservableAutoWrapper.connect_item_hooks(selected_types, sym => {
+               if (selected_type_ref_count.increment_adds_key(sym)) {
+                  _sync_visible_rows.Invoke();
+               }
+               return Disposable.Create(() => {
+                  if (selected_type_ref_count.decrement_removes_key(sym)) {
+                     _sync_visible_rows.Invoke();
+                  }
+               });
+            }),
             ObservableAutoWrapper.connect_item_hooks(
                rows, r => r.observe_is_dirty().Subscribe(_ => _is_dirty.refresh())),
             _is_dirty.observe_current(true).OnDispatcher().Subscribe(can_revert_observer),
@@ -128,6 +156,10 @@ namespace mbts.spark {
 
       public ObservableCollection<string> symbols { get; } = new();
       public ObservableCollection<string> selected_symbols { get; } = new();
+      public ObservableCollection<Venue> venues { get; } = new();
+      public ObservableCollection<Venue> selected_venues { get; } = new();
+      public ObservableCollection<SecurityType> types { get; } = new() { SecurityType.Future, SecurityType.Option };
+      public ObservableCollection<SecurityType> selected_types { get; } = new();
 
       public bool is_dirty => _is_dirty.Value;
 
@@ -191,6 +223,8 @@ namespace mbts.spark {
       public AnalyticsExpirationParametersSettings get_state() =>
          new() {
             selected_symbols = selected_symbols.Cast<string>().ToArray(),
+            selected_venues = selected_venues.ToArray(),
+            selected_types = selected_types.ToArray(),
             auto_copy_rates_is_checked = auto_copy_rates_is_checked,
             auto_copy_rolls_is_checked = auto_copy_rolls_is_checked
          };
@@ -206,7 +240,16 @@ namespace mbts.spark {
                .Where(d => d.HasSecurity && d.Security.Symbol == r.security.symbol)
                .Where(d => is_unexpired(d))
                .TryGetMinElement(
-                  d => (r.security.expiration - d.Security.Expiration.date_time()).Ticks.abs(),
+                  d => {
+                     var expDiff = (r.security.expiration - d.Security.Expiration.date_time()).Ticks.abs();
+                     if (r.security.exchange != d.Security.Exchange.to_venue()) {
+                        // This is a hack to favor the security from the same venue as the row's
+                        // security in the event of a tie (for the nearest exp). As long as exp
+                        // dates are separated by more than 1 tick (100ns), the hack will work.
+                        expDiff += 1;
+                     }
+                     return expDiff;
+                  },
                   out var nearest_exp)) {
                r.driver = nearest_exp;
             }
@@ -294,10 +337,16 @@ namespace mbts.spark {
          }
 
          symbols.sync_with(_parameters.Select(p => p.security.symbol).Distinct().Ordered());
+         venues.sync_with(_parameters.Select(p => p.security.exchange).Distinct().OrderBy(v => v.ToString()));
+         types.sync_with(_parameters.Select(p => p.security.security_type).Distinct().OrderBy(v => v.ToString()));
          using var _ = _sync_visible_rows.defer();
          if (is_first_update) {
             selected_symbols.sync_with(_pending_selected_symbols?.Intersect(symbols) ?? symbols);
             _pending_selected_symbols = null;
+            selected_venues.sync_with(_pending_selected_venues?.Intersect(venues) ?? venues);
+            _pending_selected_venues = null;
+            selected_types.sync_with(_pending_selected_types?.Intersect(types) ?? types);
+            _pending_selected_types = null;
          }
       }
 

src/client/spark/AW2/AnalyticsParameters/AnalyticsExpirationParametersView.xaml

@@ -162,6 +162,34 @@
         </c:ComboCheckBox.HeaderContentTemplate>
       </c:ComboCheckBox>
 
+      <c:ComboCheckBox
+        ItemsSource="{Binding venues}"
+        SelectedItems="{Binding selected_venues}"
+        HorizontalAlignment="Left"
+        Margin="5,2,2,2"
+        HeaderContent="Venues"
+        ShowSelectAll="True">
+        <c:ComboCheckBox.HeaderContentTemplate>
+          <DataTemplate>
+            <TextBlock Text="{Binding}" Foreground="{u:BTSColor UI.default_text}" />
+          </DataTemplate>
+        </c:ComboCheckBox.HeaderContentTemplate>
+      </c:ComboCheckBox>
+
+      <c:ComboCheckBox
+        ItemsSource="{Binding types}"
+        SelectedItems="{Binding selected_types}"
+        HorizontalAlignment="Left"
+        Margin="5,2,2,2"
+        HeaderContent="Types"
+        ShowSelectAll="True">
+        <c:ComboCheckBox.HeaderContentTemplate>
+          <DataTemplate>
+            <TextBlock Text="{Binding}" Foreground="{u:BTSColor UI.default_text}" />
+          </DataTemplate>
+        </c:ComboCheckBox.HeaderContentTemplate>
+      </c:ComboCheckBox>
+
       <WrapPanel>
         <TextBlock
           Margin="5,5,0,0" Foreground="{u:BTSBrush UI.default_text}" Text="Auto copy rolls:" />

src/client/spark/AW2/AnalyticsParameters/AnalyticsExpirationParametersView.xaml.cs

@@ -39,7 +39,8 @@ namespace mbts.ui.shell {
          new() {
             ShowSymbol = true,
             ShowType = true,
-            MapExp = true
+            MapExp = true,
+            ShowVenue = true,
          };
 
       public static IValueConverter driver_display_text_converter =>

src/client/spark/AW2/Bus/SingleOptionDriverPricePublisher.cs

@@ -4,6 +4,8 @@ using System.Linq;
 using mbts.analytics;
 using bts.api;
 using bts.api.analytics;
+using Spark.Core.Disposables;
+using System.Reactive.Linq;
 
 public class DriverMD {
    public readonly DriverCalculationType calc_type = DriverCalculationType.Direct;
@@ -46,57 +48,63 @@ namespace mbts.spark {
    // Publishes the driver prices for expirations of a single option (this is not AW2 specific)
    // f(AnalyticsExpirationParametersMap, SystemContractData) -> OptionDriverPriceMap
    // This is constructed by the ResourceAllocator
-   public class SingleOptionDriverPricePublisher {
+   public sealed class SingleOptionDriverPricePublisher : IDisposable {
+      readonly object syncRoot = new();
       private readonly SecurityTuple _sec_tup;
-      private ThreadedBlackboard _bb;
+      readonly IBus _bus;
       // tags
-      private ITag<AnalyticsExpirationParametersMap> _in_bb_exp_params_tag =
-         SparkTags.ExpirationParametersInputTag;
-      private ITag<SystemContractData> _in_bb_contracts_tag =
-         SparkTags.SystemContractsTag;
       private OptionDriverPriceMap _last_published;
       private readonly PerfStatsLogger _logger;
       private readonly TimeSpan _fire_period = TimeSpan.FromMilliseconds(300);
       private DateTime _last_fire_time = DateTime.MinValue;
+      SystemContractData sc_data;
+      AnalyticsExpirationParametersMap param_map;
+      IDisposable dispose;
 
-      public SingleOptionDriverPricePublisher(SecurityTuple sec_tup,
-                                               ThreadedBlackboard bb) {
+      public SingleOptionDriverPricePublisher(SecurityTuple sec_tup, IBus bus) {
          bts.Log.Gen.info($"SingleOptionDriverPricePublisher initialized st:{sec_tup}");
          _sec_tup = sec_tup;
-         _bb = bb;
+         _bus = bus;
          _logger = new PerfStatsLogger(50, 50,
             $"SingleOptionDriverPricePublisher_{sec_tup.security_name}_{GetHashCode()}");
          _last_published = new OptionDriverPriceMap(sec_tup.security_name,
             new Dictionary<SecurityExpiration, DriverMD>());
-         _bb.subscribe(_in_bb_exp_params_tag);
-         _bb.subscribe(_in_bb_contracts_tag);
-         _bb.data_callback += _on_bb_data;
-         _bb.timer_callback += _on_bb_timer;
+         dispose = Disposable.Create(
+            Observable.Interval(_fire_period).Subscribe(onInterval),
+            bus.subscribe(SparkTags.ExpirationParametersInputTag).Subscribe(onExpirationParameters),
+            bus.subscribe(SparkTags.SystemContractsTag).Subscribe(onSystemContractData));
          _compute_and_fire();
-      }
 
-      private void _on_bb_data(object tag, object data) {
-         if (_in_bb_exp_params_tag == tag || _in_bb_contracts_tag == tag) {
-            _on_bb_timer();
+         void onInterval(long _) {
+            _compute_and_fire();
          }
-      }
 
-      private void _on_bb_timer() {
-         if (DateTime.Now - _last_fire_time >= _fire_period) {
-            _last_fire_time = DateTime.Now;
+         void onExpirationParameters(AnalyticsExpirationParametersMap expParams) {
+            param_map = expParams;
+            _compute_and_fire();
+         }
+
+         void onSystemContractData(SystemContractData data) {
+            sc_data = data;
             _compute_and_fire();
          }
       }
 
-      private void _compute_and_fire() {
+      public void Dispose() =>
+         Disposable.dispose(ref dispose);
+
+      void _compute_and_fire() {
+         lock (syncRoot) {
+            _compute_and_fire_impl();
+         }
+      }
+
+      private void _compute_and_fire_impl() {
          _logger.start_default_timer();
-         AnalyticsExpirationParametersMap param_map = _bb.read_state(_in_bb_exp_params_tag);
-         SystemContractData sc_data = _bb.read_state(_in_bb_contracts_tag);
          if (param_map == null || sc_data == null) {
             return;
          }
-         IDictionary<mbts.SecurityExpiration, mbts.SecurityExpiration> driver_map =
-            param_map.get_option_to_driver_map(_sec_tup.security_name);
+         var driver_map = param_map.get_option_to_driver_map(_sec_tup.security_name, _sec_tup.security_exchange);
          // look up the market data
          IDictionary<SecurityExpiration, DriverMD> res =
               new Dictionary<SecurityExpiration, DriverMD>();
@@ -128,7 +136,7 @@ namespace mbts.spark {
             // bts.Log.Gen.info("SingleOptionDriverPricePublisher._compute_and_fire \n" +
             //                   $"map:{to_fire.log_string()}");
             _logger.stop_default_timer();
-            _bb.post(SparkTags.OptionDriverBBOTag(_sec_tup.security_name),
+            _bus.post(SparkTags.OptionDriverBBOTag((_sec_tup.security_name, _sec_tup.security_exchange)),
                      new OptionDriverPriceMap(_sec_tup.security_name, res));
          }
       }
@@ -168,15 +176,13 @@ namespace mbts.spark {
       public static Tuple<SecurityExpiration, SecurityExpiration>
             find_neighbors(SystemContractData sc_data,
                            SecurityTuple sec_tup, DateTime exp) {
-         var exps = sc_data.exp_map.Where(
-            kv => sec_tup.Equals(kv.Key.security_tuple)).Select(kv => kv.Key);
+         var exps = sc_data.GetExpirations(sec_tup.security_exchange, sec_tup.security_type, sec_tup.security_name);
          if (exps == null) {
             return null;
          }
-         SecurityExpiration low, high;
          VolSurfaceUtilityFunctions.find_neighbors(exps, exp,
-            (t, p) => t.expiration.Subtract(p).TotalMilliseconds, out low, out high);
-         return Tuple.Create(low, high);
+            (t, p) => t.Subtract(p).TotalMilliseconds, out var low, out var high);
+         return new(new(sec_tup, low), new(sec_tup, high));
       }
 
       public static double interp_driver(SystemContractData sc_data,

src/client/spark/AW2/Bus/SingleVolSurfaceCommittedOutputRetriever.cs

@@ -1,11 +1,11 @@
-using System;
+#nullable enable
+using System;
 using System.Collections.Generic;
 using System.Diagnostics;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 using mbts.analytics;
 using bts.api;
+using Spark.Core.Disposables;
+using System.Reactive.Linq;
 
 namespace mbts.spark {
    // Publishes the vol outputs for a single vol surface.
@@ -19,41 +19,26 @@ namespace mbts.spark {
    // 1. VS Out
    public class SingleVolSurfaceCommittedOutputRetriever : IDisposable {
       private readonly int _vs_id;
-      private readonly ITag<VolSurfaceInput> _in_bb_vs_input_tag;
-      private readonly ITag<VolSurfaceNonVolData> _in_bb_nonvol_data_tag;
-      private readonly ITag<VolSurfaceStandardTime> _in_vs_standard_time_tag;
-      private readonly ITag<VolSurfaceOutput> _out_bb_computed_vol_tag;
-      private readonly ThreadedBlackboard _bb;
       private readonly PerfStatsLogger _logger = new PerfStatsLogger(50, 100);
       private readonly string _log_id;
+      readonly IBus _bus;
+      IDisposable? _dispose;
 
-      public SingleVolSurfaceCommittedOutputRetriever(int vs_id,
-         ThreadedBlackboard threaded_bb, IEnumerable<string> symbols) {
+      public SingleVolSurfaceCommittedOutputRetriever(int vs_id, IBus bus) {
          bts.Log.Gen.info($"SingleVolSurfaceCommittedOutputRetriever cons vs_id:{vs_id} tid:{_tid()}");
          _vs_id = vs_id;
-         _bb = threaded_bb;
-         _in_bb_vs_input_tag = SparkTags.VolSurfaceCommittedInputTag(vs_id);
-         _in_bb_nonvol_data_tag = SparkTags.VolSurfaceNonVolDataTag(vs_id);
-         _in_vs_standard_time_tag = SparkTags.VolSurfaceStandardTimeTag(vs_id);
-         _out_bb_computed_vol_tag = SparkTags.VolSurfaceCommittedOutputTag(vs_id);
+         _bus = bus;
          _log_id = String.Format("SingleVolSurfaceCommittedOutputRetriever_{0}", vs_id);
-         _bb.subscribe(_in_bb_vs_input_tag);
-         _bb.subscribe(_in_bb_nonvol_data_tag);
-         _bb.subscribe(_in_vs_standard_time_tag);
-         _bb.data_callback += _on_bb_data;
-         _recompute_and_publish();
+         _dispose = Observable.CombineLatest(
+            _bus.subscribe(SparkTags.VolSurfaceCommittedInputTag(vs_id)),
+            _bus.subscribe(SparkTags.VolSurfaceNonVolDataTag(vs_id)),
+            _bus.subscribe(SparkTags.VolSurfaceStandardTimeTag(vs_id)),
+            (vs_input, vs_nonvol_data, vs_standard_time) => (vs_input, vs_nonvol_data, vs_standard_time))
+            .Subscribe(_recompute_and_publish);
       }
       private static int _tid() { return VolSurfaceTroubleshootingFunctions.tid(); }
-      private void _on_bb_data(object tag, object data) {
-         if (_in_bb_vs_input_tag.Equals(tag) || _in_bb_nonvol_data_tag.Equals(tag) ||
-             _in_vs_standard_time_tag.Equals(tag))
-            _recompute_and_publish(tag);
-      }
-
-      private void _recompute_and_publish(object tag = null) {
-         VolSurfaceInput vs_input = _bb.read_state(_in_bb_vs_input_tag);
-         VolSurfaceNonVolData vs_nonvol_data = _bb.read_state(_in_bb_nonvol_data_tag);
-         VolSurfaceStandardTime vs_standard_time = _bb.read_state(_in_vs_standard_time_tag);
+      private void _recompute_and_publish((VolSurfaceInput, VolSurfaceNonVolData, VolSurfaceStandardTime) data_tup) {
+         var (vs_input, vs_nonvol_data, vs_standard_time) = data_tup;
          if (vs_input != null && vs_nonvol_data != null && vs_standard_time != null) {
             var timer = Stopwatch.StartNew();
             IDictionary<VolSurfaceExpiration, VolSurfaceExpirationAuxiliaryData> aux_data_map =
@@ -62,17 +47,12 @@ namespace mbts.spark {
             VolSurfaceOutput vs_output = GeneralVolSurfaceCalculator.compute(
                vs_input, vs_nonvol_data.contracts, aux_data_map);
             timer.Stop();
-            var tag_str = tag == null ? "Null" : tag.ToString();
-            // bts.Log.Gen.info($"SingleVolSurfaceCommittedOutputRetriever._recompute_and_publish " +
-            //                  $"vs_id: {_vs_id} time:{_timer.lastIntervalTime} ms tag:{tag_str} " +
-            //                  $"tid:{_tid()}"); //output: {vs_output.log_string()}*/");
             _logger.log(_log_id, _log_id, timer.Elapsed.TotalMilliseconds);
-            _bb.post(_out_bb_computed_vol_tag, vs_output);
+            _bus.post(SparkTags.VolSurfaceCommittedOutputTag(_vs_id), vs_output);
          }
       }
 
-      #region IDisposable
-      public void Dispose() { _bb.data_callback -= _on_bb_data; }
-      #endregion
+      public void Dispose() =>
+         Disposable.dispose(ref _dispose);
    }
 }

src/client/spark/AW2/Bus/SingleVolSurfaceDataSource.cs

@@ -24,15 +24,15 @@ namespace mbts.spark {
       private readonly SymbolListType _list_type;
       private readonly int _vs_id;
       private readonly ThreadedBlackboard _bb;
-      private readonly Func<IEnumerable<string>, IDisposable> _create_func;
+      private readonly Func<IEnumerable<(string, Venue)>, IDisposable> _create_func;
       private readonly ITag<VolSurfaceInput> _in_bb_committed_data_tag;
       private readonly ITag<VolSurfaceCalculationRequest> _in_bb_calc_req_tag;
 
-      private HashSet<string> _option_symbols = new HashSet<string>();
+      private HashSet<(string, Venue)> _option_symbols = new();
       private IDisposable _source = null;
 
       public SingleVolSurfaceDataSource(SymbolListType list_type, int vs_id, ThreadedBlackboard threaded_bb,
-                                        Func<IEnumerable<string>, IDisposable> create_func) {
+                                        Func<IEnumerable<(string, Venue)>, IDisposable> create_func) {
          Log.Gen.info(String.Format("SingleVolSurfaceDataSource constructed vs_id {0}", vs_id));
          _list_type = list_type;
          _vs_id = vs_id;
@@ -56,33 +56,33 @@ namespace mbts.spark {
          VolSurfaceInput vs_comm_inp = _bb.read_state(_in_bb_committed_data_tag);
          if (_list_type == SymbolListType.Candidate) {
             if (vs_calc_req != null && vs_calc_req.volsurface_input != null) {
-               HashSet<string> new_sym_map = new HashSet<string>(VolSurfaceDataFunctions.get_symbols(vs_calc_req.volsurface_input));
+               var new_sym_map = VolSurfaceDataFunctions.get_symbols(vs_calc_req.volsurface_input).ToHashSet();
                _create_if_needed(new_sym_map);
             }
          } else if (_list_type == SymbolListType.Committed) {
             if (vs_comm_inp != null) {
-               HashSet<string> new_sym_map = new HashSet<string>(VolSurfaceDataFunctions.get_symbols(vs_comm_inp));
+               var new_sym_map = VolSurfaceDataFunctions.get_symbols(vs_comm_inp).ToHashSet();
                _create_if_needed(new_sym_map);
             }
          } else if (_list_type == SymbolListType.Combined) {
-            List<string> new_sym_list = new List<string>();
+            var new_syms = new HashSet<(string, Venue)>();
             if (vs_calc_req != null && vs_calc_req.volsurface_input != null)
-               new_sym_list.AddRange(VolSurfaceDataFunctions.get_symbols(vs_calc_req.volsurface_input));
+               new_syms.AddRange(VolSurfaceDataFunctions.get_symbols(vs_calc_req.volsurface_input));
             if (vs_comm_inp != null)
-               new_sym_list.AddRange(VolSurfaceDataFunctions.get_symbols(vs_comm_inp));
-            _create_if_needed(new HashSet<string>(new_sym_list));
+               new_syms.AddRange(VolSurfaceDataFunctions.get_symbols(vs_comm_inp));
+            _create_if_needed(new_syms);
          }
       }
 
-      private void _create_if_needed(HashSet<string> new_sym_map) {
+      private void _create_if_needed(HashSet<(string, Venue)> new_sym_map) {
          if (!CollectionHelper.hashset_equals(_option_symbols, new_sym_map)) {
-            _option_symbols = new HashSet<string>(new_sym_map);
+            _option_symbols = new(new_sym_map);
             if (_source != null)
                _source.Dispose();
             _source = _create_func(_option_symbols);
             _bb.post(SparkTags.AquireTag(Guid.NewGuid().ToString()),
                                  new SymbolResourceAllocationUnit(
-                                    _option_symbols.Select(i => new mbts.SecurityTuple(SecurityType.Option, i, Venue.Unset)), this, true));
+                                    _option_symbols.Select(i => new mbts.SecurityTuple(SecurityType.Option, i.Item1, i.Item2)), this, true));
          }
       }
 
@@ -95,4 +95,4 @@ namespace mbts.spark {
       #endregion
 
    }
-}
+}

src/client/spark/AW2/Bus/SingleVolSurfaceInterpGreeksPublisher.cs

@@ -25,12 +25,12 @@ namespace mbts.spark {
    public class SingleVolSurfaceInterpGreeksPublisher : IDisposable {
       private readonly int _vs_id;
       private readonly ThreadedBlackboard _bb;
-      private HashSet<string> _all_symbols;
+      private HashSet<(string, Venue)> _all_symbols;
       private VolSurfaceInterpGreeksProducer _theo_producer;
       private readonly ITag<VolSurfaceInput> _in_vs_input_tag;
       private readonly ITag<VolSurfaceNonVolInput> _in_vs_nonvol_input_tag;
       private readonly ITag<VolSurfaceStrikeMap> _in_vs_contracts_tag;
-      private ITag<IReadOnlyDictionary<string, OptionDriverPriceMap>>
+      private ITag<IReadOnlyDictionary<(string, Venue), OptionDriverPriceMap>>
          _in_all_syms_driver_price_tag = SparkTags.AllOptionDriverBBOTag;
       private readonly ITag<VolSurfaceStandardTime> _in_vs_standard_time_tag;
       private readonly ITag<bool> _in_ui_up_tag;
@@ -38,11 +38,11 @@ namespace mbts.spark {
       private readonly PerfStatsLogger _logger;
 
       public SingleVolSurfaceInterpGreeksPublisher(int vs_id,
-               ThreadedBlackboard threaded_bb, IEnumerable<string> symbols) {
+               ThreadedBlackboard threaded_bb, IEnumerable<(string, Venue)> symbols) {
          bts.Log.Gen.info($"SingleVolSurfaceInterpGreeksPublisher cons {vs_id}");
          _vs_id = vs_id;
          _bb = threaded_bb;
-         _all_symbols = new HashSet<string>(symbols);
+         _all_symbols = symbols.ToHashSet();
          _logger = new PerfStatsLogger(10, 100,
             $"SingleVolSurfaceInterpGreeksPublisher_{vs_id}");
          _theo_producer = new VolSurfaceInterpGreeksProducer(vs_id);
@@ -191,4 +191,4 @@ namespace mbts.spark {
       }
       #endregion
    }
-}
+}

src/client/spark/AW2/Bus/SingleVolSurfaceManager.cs

@@ -9,16 +9,17 @@ namespace mbts.spark {
       private readonly int _vs_id;
       private IEnumerable<IDisposable> _resources;
       private readonly ThreadedBlackboard _bb;
+      readonly IBus _bus;
       private readonly AnalyticsService _analytics_service;
       private ITag<bool> _vs_activation_tag;
       private bool _activated = false;
 
-      public SingleVolSurfaceManager(int vs_id, ThreadedBlackboard t_bb,
-                                     AnalyticsService analytics_service) {
+      public SingleVolSurfaceManager(int vs_id, ThreadedBlackboard t_bb, AnalyticsService analytics_service, IBus bus) {
          var tid = System.Threading.Thread.CurrentThread.ManagedThreadId;
          bts.Log.Ui.info($"SingleVolSurfaceManager.cons vs_id:{vs_id} tid:{tid}");
          _vs_id = vs_id;
          _bb = t_bb;
+         _bus = bus;
          _analytics_service = analytics_service;
          _vs_activation_tag = SparkTags.VolSurfaceActiveUITag(vs_id);
          _bb.subscribe(_vs_activation_tag);
@@ -34,7 +35,7 @@ namespace mbts.spark {
             new SingleVolSurfaceDataSource(SingleVolSurfaceDataSource.SymbolListType.Combined,
                _vs_id, _bb, syms => new SingleVolSurfaceNonVolDataSource(_vs_id, syms, _bb)),
             new SingleVolSurfaceDataSource(SingleVolSurfaceDataSource.SymbolListType.Committed, _vs_id, _bb,
-               a_syms => new SingleVolSurfaceCommittedOutputRetriever(_vs_id, _bb, a_syms)),
+               a_syms => new SingleVolSurfaceCommittedOutputRetriever(_vs_id, _bus)),
             new SingleVolSurfaceDataSource(SingleVolSurfaceDataSource.SymbolListType.Combined, _vs_id, _bb,
                a_syms => new SingleVolSurfaceRequestCalculator(
                   SingleVolSurfaceRequestCalculator.RequestType.UI, _vs_id, a_syms, _bb)),
@@ -76,4 +77,4 @@ namespace mbts.spark {
       }
       #endregion
    }
-}
+}

src/client/spark/AW2/Bus/SingleVolSurfaceMarketImpliedVolSource.cs

@@ -1,4 +1,5 @@
-namespace mbts.spark {
+namespace mbts.spark
+{
    using bts;
    using bts.api;
 
@@ -13,29 +14,37 @@
 
    /// This class publishes the market implied vols for the
    /// selected expiration.
-   public class SingleVolSurfaceMarketImpliedVolSource : IDisposable {
+   public class SingleVolSurfaceMarketImpliedVolSource : IDisposable
+   {
       // This keeps the last computed exp and its bbo
       #region MD caching
-      private class CachedExpMD {
+      private class CachedExpMD
+      {
          public SecurityExpiration exp = SecurityExpiration.empty;
          public IDictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>> k_md_map =
             new Dictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>>();
          private bool _has_changed(
                SecurityExpiration new_exp,
-               IDictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>> new_k_md_map) {
-            if (new_exp != exp) {
+               IDictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>> new_k_md_map)
+         {
+            if (new_exp != exp)
+            {
                return true;
             }
-            if (k_md_map.Count() != new_k_md_map.Count) {
+            if (k_md_map.Count() != new_k_md_map.Count)
+            {
                return true;
             }
-            foreach (var kv in k_md_map) {
+            foreach (var kv in k_md_map)
+            {
                Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit> k_md;
-               if (!new_k_md_map.TryGetValue(kv.Key, out k_md)) {
+               if (!new_k_md_map.TryGetValue(kv.Key, out k_md))
+               {
                   return true;
                }
                if (!kv.Value.Item1.Equals(k_md.Item1) ||
-                   !kv.Value.Item2.Equals(k_md.Item2)) {
+                   !kv.Value.Item2.Equals(k_md.Item2))
+               {
                   return true;
                }
             }
@@ -43,9 +52,11 @@
          }
          public bool set(
                SecurityExpiration new_exp,
-               IDictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>> new_k_md_map) {
+               IDictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>> new_k_md_map)
+         {
             bool has_changed = _has_changed(new_exp, new_k_md_map);
-            if (has_changed) {
+            if (has_changed)
+            {
                exp = new_exp;
                k_md_map = new_k_md_map;
             }
@@ -69,7 +80,8 @@
       private TimeSpan _compute_period = TimeSpan.FromMilliseconds(333);
       private bool _is_dirty = true;
 
-      public SingleVolSurfaceMarketImpliedVolSource(int vs_id, ThreadedBlackboard bb) {
+      public SingleVolSurfaceMarketImpliedVolSource(int vs_id, ThreadedBlackboard bb)
+      {
          bts.Log.Gen.info($"SingleVolSurfaceMarketImpliedVolSource cons vs_id:{vs_id}");
          _vs_id = vs_id;
          _bb = bb;
@@ -91,7 +103,8 @@
          _bb.timer_callback += _on_bb_timer;
       }
 
-      private void _on_bb_data(object tag, object data) {
+      private void _on_bb_data(object tag, object data)
+      {
          if (tag != null)
             if (_in_bb_committed_output_tag.Equals(tag) ||
                 _in_bb_calc_resp_tag.Equals(tag) ||
@@ -101,22 +114,26 @@
                _is_dirty = true;
       }
 
-      private void _check_bbo_change() {
+      private void _check_bbo_change()
+      {
          var sc_data = _bb.read_state(SparkTags.SystemContractsTag);
          Tuple<VolSurfaceDisplayGridExpiration, bool> sel_exp_tup =
             _bb.read_state(_in_bb_selected_exp_tag);
-         if (sc_data == null || sel_exp_tup == null) {
+         if (sc_data == null || sel_exp_tup == null)
+         {
             return;
          }
-         VolSurfaceExpiration new_sel_exp = sel_exp_tup.Item1.ToVSExp();
+         VolSurfaceExpiration new_sel_exp = sel_exp_tup.Item1.ToVSExp(SecurityType.Option);
          var new_md = sc_data.get_option_bbo_vsexp(new_sel_exp);
          bool bbo_dirty = _cached_exp_md.set(new_md.Item1, new_md.Item2);
          _is_dirty = _is_dirty || bbo_dirty;
       }
 
       private DateTime _last_publish_time = DateTime.MinValue;
-      private void _on_bb_timer() {
-         if (DateTime.Now - _last_publish_time >= _compute_period) {
+      private void _on_bb_timer()
+      {
+         if (DateTime.Now - _last_publish_time >= _compute_period)
+         {
             _check_bbo_change();
             if (_is_dirty)
                _recompute_and_publish();
@@ -125,18 +142,17 @@
          }
       }
 
-      private void _recompute_and_publish() {
+      private void _recompute_and_publish()
+      {
          Tuple<VolSurfaceDisplayGridExpiration, bool> sel_exp =
             _bb.read_state(_in_bb_selected_exp_tag);
          VolSurfaceOutput vs_output = _resolve_curr_cand();
          if (vs_output != null && vs_output.volsurface_input != null &&
             vs_output.aux_data_map != null &&
-            sel_exp != null && sel_exp.Item1 != null) {
+            sel_exp != null && sel_exp.Item1 != null)
+         {
             var impliedvol_side = _get_market_impliedvol_side(_bb, _vs_id);
-            VolSurfaceExpiration orig_exp = new VolSurfaceExpiration() {
-               symbol = sel_exp.Item1.symbol,
-               expiration = sel_exp.Item1.expiration
-            };
+            var orig_exp = sel_exp.Item1.ToVSExp(SecurityType.Option);
             var timer = Stopwatch.StartNew();
             VolSurfaceExpiration md_exp;
             VolCurveMarketImpliedVolData exp_impl_vol_map =
@@ -152,7 +168,8 @@
                "SingleVolSurfaceMarketImpliedVolSource",
                "SingleVolSurfaceMarketImpliedVolSource",
                timer.Elapsed.TotalMilliseconds);
-            if (exp_impl_vol_map is null) {
+            if (exp_impl_vol_map is null)
+            {
                return;
             }
             bts.Log.Gen.info(
@@ -167,17 +184,21 @@
       }
 
       // Returns the candidate or committed VolSurface depending
-      private VolSurfaceOutput _resolve_curr_cand() {
+      private VolSurfaceOutput _resolve_curr_cand()
+      {
          bool is_dirty = _bb.read_state(_in_bb_inp_dirty_tag, false);
-         if (is_dirty) {
+         if (is_dirty)
+         {
             VolSurfaceCalculationResponse vs_resp = _bb.read_state(_in_bb_calc_resp_tag);
             return vs_resp != null ? vs_resp.volsurface_calculation_output : null;
-         } else
+         }
+         else
             return _bb.read_state(_in_bb_committed_output_tag);
       }
 
       private IDictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>>
-      _get_bbo(VolSurfaceExpiration vs_exp) {
+      _get_bbo(VolSurfaceExpiration vs_exp)
+      {
          return get_option_bbo(_bb, vs_exp);
       }
 
@@ -189,10 +210,14 @@
       // option prices are in BTC
       internal static double get_option_price_mult(
                                                 ThreadedBlackboard bb,
-                                                VolSurfaceExpiration option_exp) {
+                                                VolSurfaceExpiration option_exp)
+      {
+         if (option_exp.venue != Venue.Deribit)
+            return 1.0;
          var driver_map = bb.read_state(SparkTags.AllOptionDriverBBOTag);
          if (driver_map != null &&
-            driver_map.TryGetValue(option_exp.symbol, out var sym_map)) {
+            driver_map.TryGetValue((option_exp.symbol, option_exp.venue), out var sym_map))
+         {
             var vs_md = sym_map.get_driver_md(option_exp.expiration);
             return vs_md.get_driver();
          }
@@ -200,7 +225,8 @@
       }
       internal static
       IDictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>>
-      get_option_bbo(ThreadedBlackboard bb, VolSurfaceExpiration vs_exp) {
+      get_option_bbo(ThreadedBlackboard bb, VolSurfaceExpiration vs_exp)
+      {
          var sc_data = bb.read_state(SparkTags.SystemContractsTag);
          IDictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>> rv =
             sc_data.get_option_bbo_vsexp(vs_exp).Item2;
@@ -213,13 +239,15 @@
       }
 
       internal static bool _ignore_market_implied_vol_source_settings(
-            ThreadedBlackboard bb, int vs_id) {
+            ThreadedBlackboard bb, int vs_id)
+      {
          return bb.read_state(
             SparkTags.VolSurfaceIgnoreImpliedVolSettingsTag(vs_id), false);
       }
 
       internal static VolSurfaceEnums.MarketSide
-      _get_market_impliedvol_side(ThreadedBlackboard bb, int vs_id) {
+      _get_market_impliedvol_side(ThreadedBlackboard bb, int vs_id)
+      {
          return bb.read_state(
             SparkTags.VolSurfaceMarketImpliedVolSide(vs_id),
             VolSurfaceEnums.MarketSide.Mid);
@@ -228,7 +256,8 @@
       #endregion
 
       #region IDisposable
-      public void Dispose() {
+      public void Dispose()
+      {
          _bb.data_callback -= _on_bb_data;
          _bb.timer_callback -= _on_bb_timer;
       }

src/client/spark/AW2/Bus/SingleVolSurfaceNonVolDataSource.cs

@@ -18,15 +18,15 @@ namespace mbts.spark {
    // 2. VolSurfaceStrikeMap
    public class SingleVolSurfaceNonVolDataSource : IDisposable {
       private readonly int _vs_id;
-      private readonly IEnumerable<string> _option_symbols;
+      private readonly IEnumerable<(string, Venue)> _option_symbols;
       private readonly ThreadedBlackboard _bb;
       private readonly ITag<SystemContractData> _in_bb_contract_data_tag;
       private readonly ITag<VolSurfaceNonVolInput> _in_bb_vs_nonvol_input_tag;
-      private readonly ITag<IReadOnlyDictionary<string, OptionDriverPriceMap>> _in_bb_all_driver_price_tag;
+      private readonly ITag<IReadOnlyDictionary<(string, Venue), OptionDriverPriceMap>> _in_bb_all_driver_price_tag;
       private readonly ITag<mbts.analytics.VolSurfaceNonVolData> _out_bb_vs_nonvol_data_tag;
       private readonly ITag<mbts.analytics.VolSurfaceStrikeMap> _out_bb_vs_contracts_tag;
 
-      public SingleVolSurfaceNonVolDataSource(int vs_id, IEnumerable<string> option_symbols,
+      public SingleVolSurfaceNonVolDataSource(int vs_id, IEnumerable<(string, Venue)> option_symbols,
             ThreadedBlackboard bb) {
          bts.Log.Gen.info($"SingleVolSurfaceNonVolDataSource vs_id:{vs_id}");
          _vs_id = vs_id;

src/client/spark/AW2/Bus/SingleVolSurfaceRequestCalculator.cs

@@ -25,7 +25,7 @@ namespace mbts.spark {
       }
       private readonly RequestType _request_type = RequestType.UI;
       private readonly int _vs_id;
-      private readonly HashSet<string> _symbols;
+      private readonly HashSet<(string, Venue)> _symbols;
       private readonly ThreadedBlackboard _bb;
       private readonly ITag<VolSurfaceCalculationRequest> _in_bb_request_tag;
       private readonly ITag<VolSurfaceCalculationRequest> _in_bb_auto_request_tag;
@@ -37,12 +37,12 @@ namespace mbts.spark {
       private VolSurfaceOutput _last_output;
 
       public SingleVolSurfaceRequestCalculator(RequestType request_type,
-                                               int vs_id, IEnumerable<string> symbols,
+                                               int vs_id, IEnumerable<(string, Venue)> symbols,
                                                ThreadedBlackboard threaded_bb) {
          bts.Log.Gen.info($"SingleVolSurfaceRequestCalculator cons vs_id: {vs_id} request_type: {request_type}");
          _request_type = request_type;
          _vs_id = vs_id;
-         _symbols = new HashSet<string>(symbols);
+         _symbols = symbols.ToHashSet();
          _bb = threaded_bb;
          _in_bb_request_tag = SparkTags.VolSurfaceCalculationRequestTag(vs_id);
          _in_bb_auto_request_tag = SparkTags.VolSurfaceAutoCalculationRequestTag(vs_id);

src/client/spark/AW2/Bus/SymbolAggregationPublisher.cs

@@ -4,92 +4,79 @@ using System;
 using System.Collections.Generic;
 using bts.api;
 using System.Linq;
+using System.Reactive.Linq;
+using Spark.Core.Disposables;
+using Spark.Core.Reactive;
 
 namespace mbts.analytics {
 
    // Thin class that assembles all symbol based data published on different
    // tags,  into IReadOnlyDictionary of symbol vs data publsihed under one tag
    public class SymbolAggregationPublisher<TData> : IDisposable {
-      private readonly ThreadedBlackboard _bb;
+      readonly IBus _bus;
       private readonly SecurityType _sec_type;
-      Func<string, ITag<TData>> _get_tag;
+      Func<(string, Venue), ITag<TData>> _get_tag;
       // the tag on which the output will be published
-      ITag<IReadOnlyDictionary<string, TData>> _out_tag;
-      private readonly ITag<SystemContractData> _sc_data_tag;
-      private Dictionary<string, ITag<TData>> _tag_map =
-         new Dictionary<string, ITag<TData>>();
-      private HashSet<object> _obj_tags = new HashSet<object>();
+      ITag<IReadOnlyDictionary<(string, Venue), TData>> _out_tag;
+      Dictionary<(string, Venue), TData> _tag_map = new();
       private bool _dirty = true;
+      readonly object _lock = new();
+      IDisposable _dispose;
+
       public SymbolAggregationPublisher(
-                              ThreadedBlackboard bb,
+                              IBus bus,
                               SecurityType sec_type,
-                              Func<string, ITag<TData>> get_tag,
-                              ITag<IReadOnlyDictionary<string, TData>> out_tag) {
-         _bb = bb;
+                              Func<(string, Venue), ITag<TData>> get_tag,
+                              ITag<IReadOnlyDictionary<(string, Venue), TData>> out_tag,
+                              TimeSpan? contracts_throttle = null) {
+         _bus = bus;
+         var contracts_source = bus.subscribe(SparkTags.SystemContractsTag);
+         if (contracts_throttle.HasValue) {
+            contracts_source = contracts_source.Sample(contracts_throttle.Value);            
+         }
+         var timer_source = Observable.Interval(TimeSpan.FromMilliseconds(133));
+         _dispose = Disposable.Create(
+            contracts_source.SubscribeEach(contracts => Disposable.Create(
+               contracts.sectup_map.Where(st => st.security_type == sec_type).Select(subscribe_sec_data).ToList())),
+            timer_source.Subscribe(_on_timer));
          _sec_type = sec_type;
          _get_tag = get_tag;
          _out_tag = out_tag;
-         _sc_data_tag = SparkTags.SystemContractsTag;
-         _bb.subscribe(SparkTags.SystemContractsTag);
-         _bb.data_callback += _on_bb_data;
-         _bb.timer_callback += _on_timer;
-         _handle_sc_data();
-      }
 
-      private void _on_bb_data(object tag, object data) {
-         if (_sc_data_tag.Equals(tag)) {
-            _handle_sc_data();
-         } else {
-            if (_obj_tags.Contains(tag)) {
-               _dirty = true;
-            }
-         }
-      }
-
-      private void _handle_sc_data() {
-         var sc_data = _bb.read_state(_sc_data_tag);
-         if (sc_data == null) {
-            return;
-         }
-         foreach (var st in sc_data.sectup_map) {
-            if (st.security_type != _sec_type) {
-               continue;
-            }
-            // this is not correct since it is assuming symbol/name is 
-            // unique across securty_tuples
-            string symbol = st.security_name;
-            if (!_tag_map.ContainsKey(symbol)) {
-               var tag = _get_tag(symbol);
-               if (tag != null) {
-                  _tag_map.Add(symbol, tag);
-                  _obj_tags.Add(tag);
-                  _bb.subscribe(tag);
-                  _dirty = true;
+         IDisposable subscribe_sec_data(SecurityTuple st) =>
+            _bus.subscribe(_get_tag((st.security_name, st.security_exchange))).Subscribe(d => {
+               lock (_lock) {
+                  _tag_map[(st.security_name, st.security_exchange)] = d;
                }
-            }
-         }
+               _dirty = true;
+            });
       }
 
-      private void _on_timer() {
+      private void _on_timer(long _) {
          if (_dirty) {
             _dirty = false;
-            Dictionary<string, TData> output = new Dictionary<string, TData>();
-            foreach (var kv in _tag_map) {
-               var symbol = kv.Key;
-               var symbol_tag = kv.Value;
-               TData symbol_data = _bb.read_state(symbol_tag);
-               output.Add(symbol, symbol_data);
+            var output = new Dictionary<(string, Venue), TData>();
+            lock (_lock) {
+               foreach (var kv in _tag_map) {
+                  var symbol = kv.Key;
+                  output.Add(symbol, kv.Value);
+               }
             }
-            IReadOnlyDictionary<string, TData> ro_output = output;
+            IReadOnlyDictionary<(string, Venue), TData> ro_output = output;
             bts.Log.Gen.diag($"SymbolAggregationPublisher._on_timer publishing " +
                              $"tag: {_out_tag} sz: {ro_output.Count}");
-            _bb.post(_out_tag, ro_output);
+            _bus.post(_out_tag, ro_output);
          }
       }
 
       public void Dispose() {
-         _bb.data_callback -= _on_bb_data;
-         _bb.timer_callback -= _on_timer;
+         lock (_lock) {
+            Disposable.dispose(ref _dispose);
+         }
+      }
+
+      class SymbolData {
+         public TData data { get; set; }
       }
    }
 }

src/client/spark/AW2/Bus/VolSurfaceBlackboardManager.cs

@@ -18,6 +18,8 @@ using mbts.spark;
 using System.DirectoryServices.ActiveDirectory;
 using Spark.Core.Protobuf;
 using mbts.ui;
+using bts.config.toggles;
+using System.Diagnostics.CodeAnalysis;
 // using mbts.bb;
 
 namespace mbts.spark
@@ -27,12 +29,27 @@ namespace mbts.spark
       private List<IDisposable> _disps = new List<IDisposable>();
       private Dictionary<SecurityTuple, SingleOptionDriverPricePublisher> _option_driver_map =
          new Dictionary<SecurityTuple, SingleOptionDriverPricePublisher>();
+      readonly IBus bus;
       ThreadedBlackboard _bb;
-      public ResourceAllocator(IBus bus, ThreadedBlackboard bb)
+      readonly object _lock = new object();
+
+      public ResourceAllocator(IBus bus, ThreadedBlackboard bb, TimeSpan? contracts_throttle = null)
       {
          bts.Log.Gen.info("ResourceAllocator started");
+         this.bus = bus;
          _bb = bb;
-         var s = bus.subscribe(SparkTags.SystemContractsTag).OnDispatcher().Subscribe(_on_data);
+         var contracts_source = bus.subscribe(SparkTags.SystemContractsTag);
+         if (contracts_throttle.HasValue)
+         {
+            contracts_source = contracts_source.Sample(contracts_throttle.Value);
+         }
+         var s = contracts_source.Subscribe(contracts =>
+         {
+            lock (_lock)
+            {
+               _on_data(contracts);
+            }
+         });
          _disps.Add(s);
       }
       private void _on_data(SystemContractData d)
@@ -43,13 +60,18 @@ namespace mbts.spark
             if (!_option_driver_map.ContainsKey(sec_tup))
             {
                bts.Log.Gen.info($"ResourceAllocator._on_data allocating SingleOptionDriverPricePublisher for '{sec_tup}'");
-               _option_driver_map.Add(sec_tup, new SingleOptionDriverPricePublisher(sec_tup, _bb));
+               _option_driver_map.Add(sec_tup, new(sec_tup, bus));
             }
          }
       }
       public void Dispose()
       {
-         _disps.ForEach(d => d.Dispose());
+         lock (_lock)
+         {
+            _disps.ForEach(d => d.Dispose());
+            _option_driver_map.Values.DisposeAll();
+            _option_driver_map.Clear();
+         }
       }
    }
    // This is the anologous to the QuoteSetBlackboardManager. There
@@ -100,6 +122,7 @@ namespace mbts.spark
          _bb.subscribe(_in_bb_req_vol);
          _bb.data_callback += _on_bb_update;
          _bb.timer_callback += _on_timer;
+         var contracts_throttle = TimeSpan.FromMilliseconds(200);
          // sub to service
          var vs_sub = _analytics_service.subscribei_volsurfaces()
             .ObserveOn(SynchronizationContext.Current)
@@ -110,18 +133,15 @@ namespace mbts.spark
          _subs.Add(vs_sub);
          _subs.Add(vs_usage_sub);
          _subs.Add(new AnalyticsParametersManager(bus, _analytics_service));
-         _subs.Add(new ResourceAllocator(bus, _bb));
+         _subs.Add(new ResourceAllocator(bus, _bb, contracts_throttle));
          _subs.Add(new SymbolAggregationPublisher<OptionDriverPriceMap>(
-            _bb, SecurityType.Option, sym => SparkTags.OptionDriverBBOTag(sym),
-            SparkTags.AllOptionDriverBBOTag));
+            bus, SecurityType.Option, SparkTags.OptionDriverBBOTag, SparkTags.AllOptionDriverBBOTag, contracts_throttle));
          _subs.Add(new SymbolAggregationPublisher<OptionTheoInterpolationData>(
-            _bb, SecurityType.Option, sym => SparkTags.SymbolInterpGreeksTag(sym),
-            SparkTags.AllInterpGreeks));
+            bus, SecurityType.Option, SparkTags.SymbolInterpGreeksTag, SparkTags.AllInterpGreeks, contracts_throttle));
          _subs.Add(new SymbolAggregationPublisher<ExpStrikeKeyedData<TheoPoint>>(
-            _bb, SecurityType.Option, sym => SparkTags.TC2Tags.LiveTheosTag(sym),
-            SparkTags.TC2Tags.AllLiveTheosTag));
+            bus, SecurityType.Option, SparkTags.TC2Tags.LiveTheosTag, SparkTags.TC2Tags.AllLiveTheosTag, contracts_throttle));
          _subs.Add(new LiveTheoPublisher(_bb));
-         _subs.Add(new InterpTheosServerPublisher(_bb, order_service));
+         _subs.Add(new InterpTheosServerPublisher(_bb, bus, order_service, contracts_throttle));
          _subs.Add(new VolSurfaceInterpGreeksMultiplexer(_bb));
          // usage         
          var list_sub = _bus.subscribe(SparkTags.VolSurfaceListTag).OnDispatcher().Subscribe(t =>
@@ -137,6 +157,15 @@ namespace mbts.spark
             Log.Ui.info($"VolSurfaceUsageList vs_usage acount: {vs_usage.Items.Count}");
             foreach (var u in vs_usage.Items)
             {
+               if (string.Compare(u.OptionName, "SPX", 0) == 0 || string.Compare(u.OptionName, "SPXW", 0) == 0 && u.OptionVenue == Venue.Unset)
+               {
+                  u.OptionVenue = Venue.Cboe;
+               }
+               if (!validate_vs_usage_unit(u, out var error))
+               {
+                  Log.Gen.error($"Invalid VolSurfaceUsageUnit config. error: {error}; value: {u}");
+                  continue;
+               }
                string option_name = u.OptionName;
                string vs_name = u.VsName;
                int vs_key;
@@ -145,10 +174,11 @@ namespace mbts.spark
                   ulist.UsageList.Add(new VolSurfaceUsageEntry()
                   {
                      VsId = vs_key,
+                     SymbolName = option_name,
                      SecTup = new bts.api.SecurityTuple
                      {
                         Type = SecurityType.Option,
-                        Venue = Venue.Deribit,
+                        Venue = u.OptionVenue,
                         Symbol = option_name
                      }
                   });
@@ -177,11 +207,32 @@ namespace mbts.spark
                }
             );
          _disps.Add(theo_sub);
+
+         static bool validate_vs_usage_unit(VolSurfaceUsageUnit vs_usage_unit, [NotNullWhen(false)] out string error)
+         {
+            if (vs_usage_unit.OptionVenue == Venue.Unset)
+            {
+               error = "OptionVenue is unspecified";
+               return false;
+            }
+            if (string.IsNullOrEmpty(vs_usage_unit.OptionName))
+            {
+               error = "OptionName is null/empty";
+               return false;
+            }
+            if (string.IsNullOrEmpty(vs_usage_unit.VsName))
+            {
+               error = "VsName is null/empty";
+               return false;
+            }
+            error = null;
+            return true;
+         }
       }
 
       static IReadOnlyDictionary<uint, VolSurfaceTheoPoint> _assemble_keyed_theos(
          SystemContractData sc_data,
-         IReadOnlyDictionary<string, ExpStrikeKeyedData<TheoPoint>> theos)
+         IReadOnlyDictionary<(string, Venue), ExpStrikeKeyedData<TheoPoint>> theos)
       {
          if (sc_data is null || theos is null)
          {
@@ -203,7 +254,7 @@ namespace mbts.spark
                TheoPoint tp = s_kv.third;
                // TODO: fix exchange later
                Tuple<uint?, uint?> keys =
-                  sc_data.get_option_sec_id(Venue.Deribit, symbol, exp, strike);
+                  sc_data.get_option_sec_id(symbol.Item2, symbol.Item1, exp, strike);
                if (keys != null)
                {
                   var c_key = keys.Item1;
@@ -267,8 +318,7 @@ namespace mbts.spark
          {
             if (!_vs_manager_map.ContainsKey(vs_id))
             {
-               _vs_manager_map.Add(vs_id, new SingleVolSurfaceManager(
-                  vs_id, _bb, _analytics_service));
+               _vs_manager_map.Add(vs_id, new SingleVolSurfaceManager(vs_id, _bb, _analytics_service, _bus));
             }
          }
       }
@@ -355,7 +405,7 @@ namespace mbts.spark
             return;
          }
          mbts.analytics.VolSurfaceSettings vs_settings = _waiting_creations.Dequeue();
-         IDictionary<string, IEnumerable<DateTime>> exp_map = _contract_data.get_option_expirations(VolSurfaceDataFunctions.get_symbols(vs_settings), false);
+         var exp_map = _contract_data.get_option_expirations(VolSurfaceDataFunctions.get_symbols(vs_settings), false);
          VolSurfaceInput vs_input = mbts.analytics.VolSurfaceDataFunctions.get_default_vs_input(vs_settings, exp_map);
          var reply = _analytics_service.get_new_volsurface_id(new VoidMessage());
          if (!reply.Ok)

src/client/spark/AW2/Grid/VolSurfaceDisplayGridExpiration.cs

@@ -1,18 +1,19 @@
 namespace mbts.spark {
    using bts.api;
-   using bts.utils;
 
    using System;
 
    public class VolSurfaceDisplayGridExpiration : IComparable<VolSurfaceDisplayGridExpiration>, IEquatable<VolSurfaceDisplayGridExpiration> {
-      public VolSurfaceDisplayGridExpiration(string symbol, DateTime expiration, analytics.VolSurfaceEnums.VolSurfaceExpirationCalculationMethod calc_mode) {
+      public VolSurfaceDisplayGridExpiration(string symbol, DateTime expiration, Venue venue, analytics.VolSurfaceEnums.VolSurfaceExpirationCalculationMethod calc_mode) {
          this.symbol = symbol;
          this.expiration = expiration;
+         this.venue = venue;
          this.calc_mode = calc_mode;
       }
 
       public string symbol { get; }
       public DateTime expiration { get; }
+      public Venue venue { get; }
       public analytics.VolSurfaceEnums.VolSurfaceExpirationCalculationMethod calc_mode { get; }
 
       /// Slave exps are drawn differently and selecting them does not activate
@@ -24,16 +25,22 @@
          other != null &&
          expiration == other.expiration &&
          symbol == other.symbol &&
+         venue == other.venue &&
          calc_mode == other.calc_mode;
 
       public override int GetHashCode() =>
-         EqualityComparer.auto_phase(symbol.GetHashCode(), expiration.GetHashCode());
+         HashCode.Combine(symbol.GetHashCode(), expiration.GetHashCode(), venue.GetHashCode());
 
       public override bool Equals(object obj) =>
          Equals(obj as VolSurfaceDisplayGridExpiration);
 
-      public override string ToString() =>
-         SecurityFormatter.format_future(symbol, expiration);
+      public override string ToString() {
+         var rv = SecurityFormatter.format_future(symbol, expiration);
+         if (venue != Venue.Unset) {
+            rv = $"{rv} ({venue})";
+         }
+         return rv;
+      }
 
       public static bool operator ==(VolSurfaceDisplayGridExpiration x, VolSurfaceDisplayGridExpiration y) =>
          x is null ? y is null : x.Equals(y);

src/client/spark/AW2/InterpTheosServerPublisher.cs

@@ -8,6 +8,8 @@ using System.Linq;
 using bts.api.orderman;
 using bts.api.analytics;
 using bts;
+using System.Reactive.Linq;
+using Spark.Core.Disposables;
 
 namespace mbts.spark {
    // Listens to the interp theos, decorates the data 
@@ -15,29 +17,52 @@ namespace mbts.spark {
    public class InterpTheosServerPublisher : IDisposable {
       private readonly ThreadedBlackboard _bb;
       private readonly bts.api.orderman.OrderService _order_service;
+      IReadOnlyDictionary<(string Symbol, Venue Venue), OptionTheoInterpolationData> _interp_greeks;
+      IReadOnlyDictionary<(string Symbol, Venue Venue), OptionDriverPriceMap> _driver_bbos;
+      VolSurfaceUsageListImmutable? _vs_usage;
+      UserTuple _user;
+      SystemContractData _system_contracts;
+      IDisposable _dispose;
 
-      public InterpTheosServerPublisher(ThreadedBlackboard bb,
-         bts.api.orderman.OrderService order_service) {
+      public InterpTheosServerPublisher(
+         ThreadedBlackboard bb, IBus bus, OrderService order_service, TimeSpan? contracts_throttle = null) {
          bts.Log.Ui.info(
             $"InterpTheosServerPublisher cons tid:{VolSurfaceTroubleshootingFunctions.tid()}");
          _bb = bb;
          _order_service = order_service;
-         _bb.subscribe(SparkTags.AllInterpGreeks);
-         _bb.subscribe(SparkTags.SystemContractsTag);
-         _bb.subscribe(SparkTags.CurrentUser);
-         _bb.subscribe(SparkTags.VolSurfaceUsageTag);
-         _bb.data_callback += _on_data;
-      }
+         var contracts_source = bus.subscribe(SparkTags.SystemContractsTag);
+         if (contracts_throttle.HasValue) {
+            contracts_source = contracts_source.Sample(contracts_throttle.Value);            
+         }
+         _dispose = Disposable.Create(
+            bus.subscribe(SparkTags.CurrentUser).Subscribe(on_current_user),
+            bus.subscribe(SparkTags.VolSurfaceUsageTag).Subscribe(on_vs_usage),
+            bus.subscribe(SparkTags.AllOptionDriverBBOTag).Subscribe(on_driver_bbos),
+            bus.subscribe(SparkTags.AllInterpGreeks).Subscribe(on_interp_greeks),
+            contracts_source.Subscribe(on_system_contracts));
 
-      private void _on_data(object tag, object data) {
-         if (tag.Equals(SparkTags.SystemContractsTag) ||
-            tag.Equals(SparkTags.AllInterpGreeks)) {
+         void on_current_user(UserTuple user) =>
+            _user = user;
+
+         void on_vs_usage(VolSurfaceUsageListImmutable? vs_usage) =>
+            _vs_usage = vs_usage;
+
+         void on_driver_bbos(IReadOnlyDictionary<(string Symbol, Venue Venue), OptionDriverPriceMap> driver_bbos) =>
+            _driver_bbos = driver_bbos;
+
+         void on_interp_greeks(IReadOnlyDictionary<(string Symbol, Venue Venue), OptionTheoInterpolationData> interp_greeks) {
+            _interp_greeks = interp_greeks;
+            _publish_data();
+         }
+
+         void on_system_contracts(SystemContractData system_contracts) {
+            _system_contracts = system_contracts;
             _publish_data();
          }
       }
 
       private static IDictionary<UInt32, DriverMD> _get_drivers(
-            IReadOnlyDictionary<string, OptionDriverPriceMap> driver_map,
+            IReadOnlyDictionary<(string, Venue), OptionDriverPriceMap> driver_map,
             SystemContractData sc_data) {
          var rv = new Dictionary<UInt32, DriverMD>();
          foreach (var kv in driver_map) {
@@ -48,7 +73,7 @@ namespace mbts.spark {
       }
 
       private static IEnumerable<Tuple<SecurityImmutable, TheoPoint>> _get_theos(
-         IReadOnlyDictionary<string, OptionTheoInterpolationData> theo_data,
+         IReadOnlyDictionary<(string, Venue), OptionTheoInterpolationData> theo_data,
          SystemContractData sc_data) {
          var rv = new List<Tuple<SecurityImmutable, TheoPoint>>();
          foreach (var kv in theo_data) {
@@ -58,20 +83,20 @@ namespace mbts.spark {
          return rv;
       }
 
-      private IReadOnlyDictionary<string, OptionTheoInterpolationData>
-         _filter_for_publish(IReadOnlyDictionary<string, OptionTheoInterpolationData> all_theos) {
+      private IReadOnlyDictionary<(string, Venue), OptionTheoInterpolationData>
+         _filter_for_publish(IReadOnlyDictionary<(string, Venue), OptionTheoInterpolationData> all_theos) {
          // find the symbol vs vs_id, get the committed vs, see if it is
          // owned by the current user, and only then publish it
-         HashSet<string> syms = new HashSet<string>();
-         var usage = _bb.read_state(SparkTags.VolSurfaceUsageTag);
-         string current_user = _bb.read_state(SparkTags.CurrentUser).name;
+         var syms = new HashSet<(string, Venue)>();
+         var usage = _vs_usage;
+         var current_user = _user.name;
          foreach (var u in (usage?.UsageList).empty_if_null()) {
             VolSurfaceInput vs_input =
                _bb.read_state(SparkTags.VolSurfaceCommittedInputTag(u.VsId));
             if (vs_input != null) {
                if (vs_input.surface_settings.core_settings.vs_manager ==
                   current_user) {
-                  syms.Add(u.SecTup.Symbol);
+                  syms.Add((u.SecTup.Symbol, u.SecTup.Venue));
                }
             }
          }
@@ -79,12 +104,12 @@ namespace mbts.spark {
             all_theos.Where(kv => syms.Contains(kv.Key)).ToDictionary(kv => kv.Key, kv => kv.Value);
       }
 
-      private void _publish_data() {
-         SystemContractData sc_data = _bb.read_state(SparkTags.SystemContractsTag, null);
-         IReadOnlyDictionary<string, OptionTheoInterpolationData> all_theos =
-            _bb.read_state(SparkTags.AllInterpGreeks, null);
-         IReadOnlyDictionary<string, OptionDriverPriceMap> all_drivers =
-            _bb.read_state(SparkTags.AllOptionDriverBBOTag, null);
+      void _publish_data() {
+         // copy data to local vars because updates are multi-threaded,
+         // and data must be consistent (during computation).
+         var sc_data = _system_contracts;
+         var all_theos = _interp_greeks;
+         var all_drivers = _driver_bbos;
          if (sc_data == null || all_theos == null || all_drivers == null)
             return;
          var timer = Stopwatch.StartNew();
@@ -132,7 +157,7 @@ namespace mbts.spark {
       }
 
       public void Dispose() {
-         _bb.data_callback -= _on_data;
+         Disposable.dispose(ref _dispose);
       }
    }
 }

src/client/spark/AW2/LiveTheoPublisher.cs

@@ -10,12 +10,13 @@ using System.Windows.Automation;
 using bts.collections;
 
 namespace mbts.spark {
-   // Publishes theos for all options (on the side, also publishes the theos for TC2)
+   // Publishes theos for all options (on the side, also publishes the theos for TC2).
+   // Listens to driver and interp theos, on a change, calls _interp_and_publish
    public class LiveTheoPublisher : IDisposable {
       private ThreadedBlackboard _bb;
-      private readonly ITag<IReadOnlyDictionary<string, OptionTheoInterpolationData>>
+      private readonly ITag<IReadOnlyDictionary<(string, Venue), OptionTheoInterpolationData>>
          _greeks_tag = SparkTags.AllInterpGreeks;
-      private readonly ITag<IReadOnlyDictionary<string, OptionDriverPriceMap>>
+      private readonly ITag<IReadOnlyDictionary<(string, Venue), OptionDriverPriceMap>>
          _drivers_tag = SparkTags.AllOptionDriverBBOTag;
       private readonly PerfStatsLogger _logger = new PerfStatsLogger(50, 100);
       private readonly string _log_id = "LiveTheoPublisher";
@@ -84,17 +85,17 @@ namespace mbts.spark {
       }
 
       private static DriverMD _get_option_driver(
-                  IReadOnlyDictionary<string, OptionDriverPriceMap> driver_map,
+                  IReadOnlyDictionary<(string, Venue), OptionDriverPriceMap> driver_map,
                   SecurityStrike k) {
-         if (driver_map.TryGetValue(k.expiration.symbol, out var exp_map) &&
+         if (driver_map.TryGetValue((k.expiration.symbol, k.expiration.exchange), out var exp_map) &&
             exp_map != null)
             return exp_map.get_driver_md(k.expiration);
          return DriverMD.empty;
       }
       private static TheoPoint _interp_theo(
-         IReadOnlyDictionary<string, OptionTheoInterpolationData> interp_greeks_map,
+         IReadOnlyDictionary<(string, Venue), OptionTheoInterpolationData> interp_greeks_map,
          SecurityStrike k, double driver_price) {
-         if (interp_greeks_map.TryGetValue(k.expiration.symbol, out var interp_greeks) &&
+         if (interp_greeks_map.TryGetValue((k.expiration.symbol, k.expiration.exchange), out var interp_greeks) &&
             interp_greeks != null)
             return interp_greeks.interpolate_theos(k.expiration.expiration, k.strike,
                                                    driver_price);
@@ -102,14 +103,14 @@ namespace mbts.spark {
       }
 
       private void _publish_tc2_theos(IDictionary<SecurityStrike, TheoPoint> theos) {
-         var rv = new Dictionary<string, ExpStrikeKeyedData<TheoPoint>>();
+         var rv = new Dictionary<(string, Venue), ExpStrikeKeyedData<TheoPoint>>();
          foreach (var kv in theos) {
             SecurityStrike k = kv.Key;
             TheoPoint tp = kv.Value;
             ExpStrikeKeyedData<TheoPoint> sym_theos = null;
-            if (!rv.TryGetValue(k.expiration.symbol, out sym_theos)) {
+            if (!rv.TryGetValue((k.expiration.symbol, k.expiration.exchange), out sym_theos)) {
                sym_theos = new ExpStrikeKeyedData<TheoPoint>();
-               rv.Add(k.expiration.symbol, sym_theos);
+               rv.Add((k.expiration.symbol, k.expiration.exchange), sym_theos);
             }
             sym_theos.set_data(k, tp);
          }
@@ -122,10 +123,8 @@ namespace mbts.spark {
       private void _interp_and_publish() {
          var timer = Stopwatch.StartNew();
          var res = new Dictionary<uint, GreeksTuple>();
-         IReadOnlyDictionary<string, OptionDriverPriceMap> driver_map =
-            _bb.read_state(_drivers_tag);
-         IReadOnlyDictionary<string, OptionTheoInterpolationData> interp_greeks =
-            _bb.read_state(SparkTags.AllInterpGreeks);
+         var driver_map = _bb.read_state(_drivers_tag);
+         var interp_greeks = _bb.read_state(SparkTags.AllInterpGreeks);
          SystemContractData sc_data = _bb.read_state(SparkTags.SystemContractsTag);
          if (driver_map == null || interp_greeks == null || sc_data == null) {
             return;
@@ -178,17 +177,10 @@ namespace mbts.spark {
          SystemContractData sc_data = _bb.read_state(SparkTags.SystemContractsTag);
          if (sc_data == null)
             return rv;
-         foreach (var kv in sc_data.exp_map) {
-            var id = kv.Value;
-            SecurityMD sec_md;
-            if (sc_data.sec_md_map.TryGetValue(id, out sec_md)) {
-               if (sec_md.security.HasFuture) {
-                  SecurityExpiration exp = kv.Key;
-                  double pr = sec_md.md.fixing_price;
-                  rv.Add(sec_md.security.Id,
-                        new GreeksTuple(0, pr, pr, 1, 0, 0, 0, 0, pr));
-               }
-            }
+         foreach (var sec_md in sc_data.Futures) {
+            double pr = sec_md.md.fixing_price;
+            rv.Add(sec_md.security.Id,
+                  new GreeksTuple(0, pr, pr, 1, 0, 0, 0, 0, pr));
          }
          return rv;
       }
@@ -202,16 +194,12 @@ namespace mbts.spark {
          SystemContractData sc_data = _bb.read_state(SparkTags.SystemContractsTag);
          if (sc_data == null)
             return;
-         foreach (var kv in sc_data.exp_map) {
-            var id = kv.Value;
-            SecurityMD sec_md;
-            if (sc_data.sec_md_map.TryGetValue(id, out sec_md)) {
-               if (sec_md.security.HasFuture) {
-                  SecurityExpiration exp = kv.Key;
-                  double pr = sec_md.md.fixing_price;
-                  rv.Add(exp, new FutureTheoStructure(exp.expiration, pr, pr, 1, 0));
-               }
-            }
+         foreach (var sec_md in sc_data.Futures) {
+            var sec = sec_md.security;
+            var exp = new SecurityExpiration(
+               sec.Exchange, sec.ContractCase, sec.Symbol, sec.Future.expiration_date());
+            double pr = sec_md.md.fixing_price;
+            rv.Add(exp, new FutureTheoStructure(exp.expiration, pr, pr, 1, 0));
          }
          _bb.post(SparkTags.TC2Tags.LiveFutureTheosTag, rv);
       }

src/client/spark/AW2/Misc/AnalyticsExpirationParametersMap.cs

@@ -6,6 +6,7 @@ using System.Linq;
 using bts.api;
 
 using mbts.analytics;
+using mbts.utils;
 
 
 namespace mbts.spark {
@@ -20,11 +21,16 @@ namespace mbts.spark {
          var empty = SecurityExpiration.empty;
          var temp = new Dictionary<SecurityExpiration,
                                    AnalyticsExpirationParameters>(exp_map_arg);
-         foreach (var kv in sc_data.exp_map) {
-            var exp = kv.Key;
-            if (!temp.ContainsKey(kv.Key)) {
-               temp.Add(kv.Key,
-                  new AnalyticsExpirationParameters(exp, null, empty, empty, 0, 0, 0, 0, 0));
+         foreach (var smd in sc_data.Futures) {
+            var sec = smd.security;
+            var exp = new SecurityExpiration(sec.Exchange, SecurityType.Future, sec.Symbol, sec.Future.expiration_date());
+            if (!temp.ContainsKey(exp)) {
+               temp.Add(exp, new AnalyticsExpirationParameters(exp, null, empty, empty, 0, 0, 0, 0, 0));
+            }
+         }
+         foreach (var sec_exp in sc_data.strike_map.Keys) {
+            if (!temp.ContainsKey(sec_exp)) {
+               temp.Add(sec_exp, new(sec_exp, null, empty, empty, 0, 0, 0, 0, 0));
             }
          }
          exp_map = temp;
@@ -32,14 +38,11 @@ namespace mbts.spark {
 
       public IEnumerable<mbts.SecurityExpiration> get_sorted_exps() {
          List<SecurityExpiration> sorted_exps = new List<SecurityExpiration>(exp_map.Keys);
-         sorted_exps.Sort((e1, e2) => {
-            int rv = e1.expiration.CompareTo(e2.expiration);
-            if (rv == 0)
-               rv = e1.symbol.CompareTo(e2.symbol);
-            if (rv == 0)
-               rv = e1.security_type.ToString().CompareTo(e2.security_type.ToString());
-            return rv;
-         });
+         sorted_exps.Sort(SecurityExpiration.get_comparer(
+            SecurityExpiration.Properties.Expiration,
+            SecurityExpiration.Properties.Symbol,
+            SecurityExpiration.Properties.Type,
+            SecurityExpiration.Properties.Venue));
          return sorted_exps;
       }
 
@@ -88,23 +91,21 @@ namespace mbts.spark {
       // }
 
       // Returns a driver to expiration it drives
-      public IDictionary<mbts.SecurityExpiration, IEnumerable<mbts.SecurityExpiration>>
-      get_option_driver_map() {
-         var rv = new Dictionary<mbts.SecurityExpiration, HashSet<mbts.SecurityExpiration>>();
-         foreach (var kv in exp_map.Where(kv_arg => kv_arg.Key.security_type == SecurityType.Option)) {
-            mbts.SecurityExpiration sec = kv.Key, driver_sec = kv.Value.driver_security;
-            HashSet<mbts.SecurityExpiration> driven_by_this_driver;
-            if (!rv.TryGetValue(driver_sec, out driven_by_this_driver))
-               rv[driver_sec] = driven_by_this_driver = new HashSet<mbts.SecurityExpiration>();
-            driven_by_this_driver.Add(sec);
-         }
-         return rv.ToDictionary(kv => kv.Key, kv => kv.Value as IEnumerable<mbts.SecurityExpiration>);
-      }
+      public IEnumerable<IGrouping<SecurityExpiration, SecurityExpiration>> get_option_driver_map() =>
+         exp_map
+            .Where(kv => kv.Value.driver_security.exchange != Venue.Unset && kv.Key.security_type == SecurityType.Option)
+            .GroupBy(kv => kv.Value.driver_security, kv => kv.Key);
+
+      public IEnumerable<IGrouping<(string symbol, Venue venue), SecurityExpiration>> get_option_driver_symbol_map() =>
+         exp_map
+            .Where(kv => kv.Value.driver_security.exchange != Venue.Unset && kv.Key.security_type == SecurityType.Option)
+            .GroupBy(kv => (kv.Value.driver_security.symbol, kv.Value.driver_security.exchange), kv => kv.Key);
 
       public IDictionary<mbts.SecurityExpiration, mbts.SecurityExpiration>
-      get_option_to_driver_map(string option_name) {
+      get_option_to_driver_map(string option_name, Venue venue) {
          var filt = exp_map.Where(kv => kv.Key.security_type == SecurityType.Option &&
-                                   kv.Key.symbol == option_name);
+                                   kv.Key.symbol == option_name &&
+                                   kv.Key.exchange == venue);
          return filt.ToDictionary(kv => kv.Key, kv => kv.Value.driver_security);
       }
 

src/client/spark/AW2/Misc/OptionDriverPriceMap.cs

@@ -33,8 +33,8 @@ namespace mbts.spark {
          exp_price_map.GetValueOrDefault(exp) ?? DriverMD.empty;
 
       public static IDictionary<VolSurfaceExpiration, DriverMD> get_vs_price_map(
-         IReadOnlyDictionary<string, OptionDriverPriceMap> all_sym_map,
-         Func<string, bool>? sym_filter = null
+         IReadOnlyDictionary<(string, Venue), OptionDriverPriceMap> all_sym_map,
+         Func<(string, Venue), bool>? sym_filter = null
       ) {
          sym_filter ??= _ => true;
          return all_sym_map.SafeWhere(kv => kv.Value is not null && sym_filter(kv.Key))

src/client/spark/AW2/Misc/OptionInterpolationData.cs

@@ -1,4 +1,6 @@
-using System;
+using bts.api;
+
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
@@ -10,11 +12,15 @@ namespace mbts.spark {
    // indexed by option_expiration->driver_price->option_strike->TheoPoint
    public class OptionTheoInterpolationData {
       public readonly string symbol;
+      public readonly Venue venue;
       private IDictionary<DateTime, Dictionary<double, Dictionary<double, TheoPoint>>> _map =
          new Dictionary<DateTime, Dictionary<double, Dictionary<double, TheoPoint>>>();
       private IDictionary<DateTime, double> _zero_move_driver_map = new Dictionary<DateTime, double>();
 
-      public OptionTheoInterpolationData(string symbol) { this.symbol = symbol; }
+      public OptionTheoInterpolationData(string symbol, Venue venue) {
+         this.symbol = symbol;
+         this.venue = venue;
+      }
 
       public void set_data(DateTime option_expiration, double driver_price,
           double move, double option_strike, TheoPoint theo_point) {
@@ -125,4 +131,4 @@ namespace mbts.spark {
 
       #endregion
    }
-}
+}

src/client/spark/AW2/Misc/SecurityNormalizer.cs

@@ -51,15 +51,15 @@ namespace mbts.spark {
                foreach (var k_kv in exp_kv.Value) {
                   double strike = k_kv.Key;
                   TheoPoint tp = k_kv.Value;
-                  IEnumerable<SecurityMD> calls = sc_data.get_options(
-                     null, symbol_theos.symbol, exp, strike, PutOrCall.Call);
-                  IEnumerable<SecurityMD> puts = sc_data.get_options(
-                     null, symbol_theos.symbol, exp, strike, PutOrCall.Put);
-                  foreach (SecurityMD sec_md in calls) {
-                     rv.Add(Tuple.Create(sec_md.security, tp));
+                  var call = sc_data.get_option(
+                     symbol_theos.venue, symbol_theos.symbol, exp, strike, PutOrCall.Call);
+                  var put = sc_data.get_option(
+                     symbol_theos.venue, symbol_theos.symbol, exp, strike, PutOrCall.Put);
+                  if (call.HasValue) {
+                     rv.Add(Tuple.Create(call.Value.security, tp));
                   }
-                  foreach (SecurityMD sec_md in puts) {
-                     rv.Add(Tuple.Create(sec_md.security, tp));
+                  if (put.HasValue) {
+                     rv.Add(Tuple.Create(put.Value.security, tp));
                   }
                }
             }

src/client/spark/AW2/Misc/SystemContractData.cs

@@ -3,7 +3,9 @@ using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Linq;
 
+using bts;
 using bts.api;
+using bts.collections;
 using bts.utils;
 
 using mbts.analytics;
@@ -15,98 +17,97 @@ namespace mbts.spark {
    public class SystemContractData {
       internal readonly IReadOnlyDictionary<UInt32, SecurityMD> sec_md_map;
       public readonly HashSet<SecurityTuple> sectup_map = new HashSet<SecurityTuple>();
-      // for expirations that are contracts themselves (ie non options)
-      public readonly IReadOnlyDictionary<SecurityExpiration, UInt32> exp_map =
-         new Dictionary<SecurityExpiration, UInt32>();
       public readonly IReadOnlyDictionary<SecurityExpiration, HashSet<SecurityStrike>>
          strike_map = new Dictionary<SecurityExpiration, HashSet<SecurityStrike>>();
 
-      private SystemContractData() { }
-      public static readonly SystemContractData empty = new SystemContractData();
+      readonly Lazy<Dictionary<(Venue Venue, string Symbol, DateTime Expiration), SecurityMD>> futuresIndex;
+      readonly Lazy<Dictionary<(Venue Venue, string Symbol, DateTime Expiration, double Strike, PutOrCall CallPut), SecurityMD>> optionsIndex;
+      readonly Lazy<Dictionary<(Venue Venue, SecurityType Type, string Symbol), HashSet<DateTime>>> expirationsIndex;
 
-      class CPKey {
-         public Nullable<UInt32> ckey = null, pkey = null;
-      }
-
-      static void _add_strike(
-            ref Dictionary<SecurityExpiration, Dictionary<double, CPKey>> exp_k_map,
-            SecurityImmutable sec) {
-         bts.Log.assert(sec.HasOption);
-         SecurityExpiration sec_exp = new SecurityExpiration(sec.Exchange,
-            SecurityType.Option, sec.Symbol, sec.Option.expiration_date());
-         Dictionary<double, CPKey> k_map = null;
-         if (!exp_k_map.TryGetValue(sec_exp, out k_map)) {
-            k_map = new Dictionary<double, CPKey>();
-            exp_k_map.Add(sec_exp, k_map);
-         }
-         CPKey cp_key = null;
-         if (!k_map.TryGetValue(sec.Option.Strike, out cp_key)) {
-            cp_key = new CPKey();
-            k_map.Add(sec.Option.Strike, cp_key);
-         }
-         if (sec.Option.CallPut == PutOrCall.Put) {
-            cp_key.pkey = sec.Id;
-         } else {
-            cp_key.ckey = sec.Id;
-         }
-      }
+      public static readonly SystemContractData empty = new(new Dictionary<uint, SecurityMD>(0));
 
       internal SystemContractData(IReadOnlyDictionary<uint, SecurityMD> sec_map_arg) {
          sec_md_map = sec_map_arg;
          HashSet<SecurityTuple> sectup_map_aux = new HashSet<SecurityTuple>();
-         Dictionary<SecurityExpiration, UInt32> exp_map_aux =
-            new Dictionary<SecurityExpiration, uint>();
-         Dictionary<SecurityExpiration, Dictionary<double, CPKey>> strike_map_temp =
-           new Dictionary<SecurityExpiration, Dictionary<double, CPKey>>();
-         foreach (var kv in sec_md_map) {
-            SecurityImmutable sec = kv.Value.security;
-            SecurityTuple sec_tup =
-               new SecurityTuple(sec.security_type(), sec.Symbol, sec.Exchange);
-            sectup_map_aux.Add(sec_tup);
-            SecurityExpiration sec_exp =
-               new SecurityExpiration(sec_tup, sec.security_expiration() ?? default, sec.Id);
-            exp_map_aux[sec_exp] = sec.Id;
-            if (sec.HasOption) {
-               _add_strike(ref strike_map_temp, sec);
+         sectup_map = sec_md_map.Values
+            .Select(smd => new SecurityTuple(smd.security.security_type(), smd.security.Symbol, smd.security.Exchange))
+            .ToHashSet();
+         // Index options
+         strike_map = sec_md_map.Values
+            .Select(smd => smd.security)
+            .Where(s => s.security_type() == SecurityType.Option)
+            .GroupBy(s => new SecurityExpiration(s.Exchange, SecurityType.Option, s.Symbol, s.Option.expiration_date()))
+            .ToDictionary(
+               expGroup => expGroup.Key,
+               expGroup => expGroup.GroupBy(s => s.Option.Strike).Select(strikeGroup => {
+                  // 'strikeGroup' should have 2 securities: the call and the put.
+                  uint? callId = null;
+                  uint? putId = null;
+                  foreach (var s in strikeGroup) {
+                     switch (s.Option.CallPut) {
+                        case PutOrCall.Call:
+                           Log.assert(!callId.HasValue);
+                           callId = s.Id;
+                           break;
+                        case PutOrCall.Put:
+                           Log.assert(!putId.HasValue);
+                           putId = s.Id;
+                           break;
+                        default:
+                           Log.assert(false, $"Option sec has invalid CallPut '{s.Option.CallPut}'");
+                           break;
+                     }
+                  }
+                  return new SecurityStrike(expGroup.Key, strikeGroup.Key, new(callId, putId));
+               })
+               .ToHashSet());
+         // Index exp dates (for futures and options)
+         expirationsIndex = new(() => {
+            var rv = new Dictionary<(Venue Venue, SecurityType Type, string Symbol), HashSet<DateTime>>();
+            foreach (var sec in sec_md_map.Values.Select(s => s.security)) {
+               switch (sec.security_type()) {
+                  case SecurityType.Future:
+                  case SecurityType.Option:
+                     rv.GetOrAdd((sec.Exchange, sec.security_type(), sec.Symbol), _ => new())
+                        .Add(sec.earliest_expiration());
+                     break;
+               }
             }
-         }
-         Dictionary<SecurityExpiration, HashSet<SecurityStrike>> strike_map_aux =
-            new Dictionary<SecurityExpiration, HashSet<SecurityStrike>>();
-         foreach (var kv in strike_map_temp) {
-            SecurityExpiration exp = kv.Key;
-            HashSet<SecurityStrike> k_map = new HashSet<SecurityStrike>();
-            foreach (var kv_i in kv.Value) {
-               double k = kv_i.Key;
-               CPKey cp_key = kv_i.Value;
-               SecurityStrike sec_k =
-                  new SecurityStrike(exp, k, new Tuple<uint?, uint?>(cp_key.ckey, cp_key.pkey));
-               k_map.Add(sec_k);
+            return rv;
+         }, true);
+         // Index futures
+         futuresIndex = new(() => {
+            var rv = new Dictionary<(Venue, string, DateTime), SecurityMD>();
+            foreach (var smd in sec_md_map.Values.Where(smd => smd.security.security_type() == SecurityType.Future)) {
+               var key = (smd.security.Exchange, smd.security.Symbol, smd.security.Future.expiration_date());
+               if (rv.TryGetValue(key, out var existing)) {
+                  continue;
+               }
+               rv.Add(key, smd);
             }
-            strike_map_aux.Add(exp, k_map);
-         }
-         sectup_map = sectup_map_aux;
-         exp_map = exp_map_aux;
-         strike_map = strike_map_aux;
-         _log_data();
+            return rv;
+         },
+         true);
+         // Index options
+         optionsIndex = new(() => {
+            var rv = new Dictionary<(Venue, string, DateTime, double, PutOrCall), SecurityMD>();
+            foreach (var smd in sec_md_map.Values.Where(smd => smd.security.security_type() == SecurityType.Option)) {
+               var sec = smd.security;
+               var key = (sec.Exchange, sec.Symbol, sec.Option.expiration_date(), sec.Option.Strike, sec.Option.CallPut);
+               if (rv.TryGetValue(key, out var existing)) {
+                  continue;
+               }
+               rv.Add(key, smd);
+            }
+            return rv;
+         });
       }
 
-      private void _log_data() {
-         bts.Log.Gen.info(
-            String.Format(
-               "SystemContractData constructed, sec_tups:{0} total_exps:{1} option_exps:{2}",
-               sectup_map.Count, exp_map.Count, strike_map.Count));
-         sectup_map.ForEach(st => bts.Log.Gen.info(
-            String.Format("SystemContractData loaded sectup:{0}", st.ToString())));
-         foreach (var kv in exp_map) {
-            var log_txt = String.Format("SystemContractData loaded exp:{0} id:{1}",
-                                          kv.Key.ToString(), kv.Value);
-            if (kv.Key.security_type == SecurityType.Option) {
-               strike_map.TryGetValue(kv.Key, out var k_map);
-               log_txt += " n_strikes:" + ((k_map == null) ? 0 : k_map.Count);
-            }
-            bts.Log.Gen.info(log_txt);
-         }
-      }
+      public IEnumerable<SecurityMD> Futures =>
+         futuresIndex.Value.Values;
+
+      public ISet<DateTime> GetExpirations(Venue venue, SecurityType type, string symbol) =>
+         expirationsIndex.Value.GetValueOrDefault((venue, type, symbol)) ?? new(0);
 
       public SecurityImmutable? get_option_contract(
          Venue venue, string symbol, DateTime exp, double strike, PutOrCall cp) {
@@ -167,20 +168,19 @@ namespace mbts.spark {
                              new VolSurfaceMDUnit(bid, ask, sec_md.md.fixing_price));
       }
 
-      public Tuple<SecurityImmutable, VolSurfaceMDUnit>
-      get_exp_md(SecurityExpiration sec_exp) {
-         uint sec_id = 0;
-         if (!exp_map.TryGetValue(sec_exp, out sec_id)) {
+      public Tuple<SecurityImmutable, VolSurfaceMDUnit> get_exp_md(SecurityExpiration sec_exp) {
+         Log.assert(sec_exp.security_type == SecurityType.Future);
+         if (!futuresIndex.Value.TryGetValue((sec_exp.exchange, sec_exp.symbol, sec_exp.expiration), out var smd)) {
             return null;
          }
-         return get_sec_md(sec_id);
+         ulong time = 0;
+         smd.md.get_bbo(ref time, out var bid, out var ask);
+         return Tuple.Create(smd.security, new VolSurfaceMDUnit(bid, ask, smd.md.fixing_price));
       }
 
-      internal Tuple<SecurityExpiration,
-                     IDictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>>>
-      get_option_bbo_vsexp(VolSurfaceExpiration exp) {
-         // this is a workaround for VSExp not having an exchange
-         foreach (Venue v in System.Enum.GetValues(typeof(Venue))) {
+      internal Tuple<SecurityExpiration, IDictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>>> get_option_bbo_vsexp(VolSurfaceExpiration exp) {
+         // Prefer bbo from the same venue as the exp, but fallback to another venue if necessary.
+         foreach (Venue v in Enum.GetValues<Venue>().Except(exp.venue).Prepend(exp.venue)) {
             var sec_exp =
                new SecurityExpiration(v, SecurityType.Option, exp.symbol, exp.expiration);
             var rv = get_option_bbo(sec_exp);
@@ -190,7 +190,7 @@ namespace mbts.spark {
          }
          IDictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>> rv_data =
             new Dictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>>();
-         return Tuple.Create(exp.to_security_expiration(type: SecurityType.Option), rv_data);
+         return Tuple.Create(exp.with(security_type: SecurityType.Option).to_security_expiration(), rv_data);
       }
 
       internal IDictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>> get_option_bbo(
@@ -211,33 +211,34 @@ namespace mbts.spark {
          return rv;
       }
 
-      public IDictionary<DateTime, IEnumerable<double>> get_option_strikes(string option_name) {
-         return strike_map.Where(kv => kv.Key.symbol == option_name)
+      public IDictionary<DateTime, IEnumerable<double>> get_option_strikes(string option_name, Venue venue) {
+         return strike_map.Where(kv => kv.Key.symbol == option_name && kv.Key.exchange == venue)
             .ToDictionary(kv => kv.Key.expiration, kv => kv.Value.Select(k => k.strike));
       }
 
       public IDictionary<SecurityExpiration, IEnumerable<double>>
-      get_option_strikes(HashSet<string> option_names) {
-         return strike_map.Where(kv => option_names.Contains(kv.Key.symbol))
+      get_option_strikes(HashSet<(string, Venue)> option_names) {
+         return strike_map.Where(kv => option_names.Contains((kv.Key.symbol, kv.Key.exchange)))
             .ToDictionary(kv => kv.Key, kv => kv.Value.Select(k => k.strike));
       }
 
-      public Nullable<UInt32> get_exp_sec_id(SecurityExpiration sec_exp) {
-         UInt32 rv;
-         if (exp_map.TryGetValue(sec_exp, out rv)) {
-            return rv;
+      public IEnumerable<DateTime> get_option_expirations(string option_name, Venue option_venue) {
+         foreach (var expGroup in expirationsIndex.Value) {
+            var (venue, type, symbol) = expGroup.Key;
+            if (type != SecurityType.Option || symbol != option_name || venue != option_venue) {
+               continue;
+            }
+            foreach (var exp in expGroup.Value) {
+               yield return exp;
+            }
          }
-         return null;
       }
 
-      public IEnumerable<DateTime> get_option_expirations(string option_name,
-         bool is_display_expiration = false) {
-         return exp_map.Keys.Where(e => e.security_type == SecurityType.Option &&
-                                    e.symbol == option_name).Select(e => e.expiration);
-      }
+      public IEnumerable<DateTime> get_option_expirations((string, Venue) option_symbol) =>
+         get_option_expirations(option_symbol.Item1, option_symbol.Item2);
 
-      public IDictionary<string, IEnumerable<DateTime>> get_option_expirations(
-         IEnumerable<string> option_names, bool is_display_expiration) {
+      public IDictionary<(string, Venue), IEnumerable<DateTime>> get_option_expirations(
+         IEnumerable<(string, Venue)> option_names, bool is_display_expiration) {
          return option_names.ToDictionaryDupes(kv => kv, kv => get_option_expirations(kv));
       }
 
@@ -270,17 +271,8 @@ namespace mbts.spark {
          return sec_md_map.Values.Where(sec_md => criteria(sec_md));
       }
 
-      public IEnumerable<SecurityMD> get_options(
-         Venue? exchange, string option_name, DateTime? exp, double? strike, PutOrCall? cp) {
-         return get_securities(sc => {
-            var s = sc.security;
-            return s.security_type() == SecurityType.Option &&
-                  (exchange == null || s.Exchange == exchange) &&
-                  (exp == null || s.earliest_expiration() == exp) &&
-                  (strike == null || s.strike() == strike) &&
-                  (cp == null || s.Option.CallPut == cp);
-         });
-      }
-
+      public SecurityMD? get_option(
+         Venue exchange, string symbol, DateTime exp, double strike, PutOrCall cp) =>
+         optionsIndex.Value.TryGetValue((exchange, symbol, exp, strike, cp), out var smd) ? smd : null;
    }
 }

src/client/spark/AW2/Misc/ThreadedBlackboard.cs

@@ -37,15 +37,6 @@ namespace mbts.spark {
          _bus = bus;
          _bb_name = name;
          subscribe(SparkTags.SparkTimer);
-         System.Threading.Thread timer_thread = new System.Threading.Thread(() => {
-            DateTime start_time = DateTime.Now;
-            while (true) {
-               System.Threading.Thread.Sleep(100);
-               _bus.post(SparkTags.SparkTimer, DateTime.Now - start_time);
-            }
-         });
-         timer_thread.IsBackground = true;
-         timer_thread.Start();
       }
 
       #region Public API
@@ -58,13 +49,17 @@ namespace mbts.spark {
          return (T)rv;
       }
 
-      public void subscribe<T>(ITag<T> tag) {
+      public void subscribe<T>(ITag<T> tag, bool onDispatcher = true) {
          if (_tag_map.Contains(tag)) {
             return;
          }
          _tag_map.Add(tag);
          bts.Log.Ui.info($"ThreadedBlackBoard.subscribe tag:{tag} tid:{_tid()}");
-         var observable = _bus.subscribe(tag).OnDispatcher().Subscribe(
+         var source = _bus.subscribe(tag);
+         if (onDispatcher) {
+            source = source.OnDispatcher();
+         }
+         var observable = source.Subscribe(
             data => _on_bus_data(tag, data));
          _subs.Add(observable);
       }

src/client/spark/AW2/UIAdapters/DriverUIAdapter.cs

@@ -1,5 +1,7 @@
 
 namespace mbts.ui.controls {
+   using bts.api;
+
    using mbts.analytics;
    using mbts.spark;
    using mbts.user.settings;
@@ -56,7 +58,11 @@ namespace mbts.ui.controls {
       {
          foreach (VolSurfaceDisplayGridExpiration disp_exp in exp_disp_grid.get_ordered_expirations())
          {
-            VolSurfaceExpiration exp = new VolSurfaceExpiration() { symbol = disp_exp.symbol, expiration = disp_exp.expiration };
+            var exp = VolSurfaceExpiration.Default.with(
+               symbol: disp_exp.symbol,
+               expiration: disp_exp.expiration,
+               venue: disp_exp.venue,
+               security_type: SecurityType.Option);
             VolCurveParameters curve_parameter = VolSurfaceDataFunctions.get_curve_parameters(vs_input, exp);
             curve_parameter.core_parameters.ref_forward = VolSurfaceDisplayUtilities.parse_double(exp_disp_grid.get_grid_data(disp_exp, _ref_fwd_col, Double.NaN));
             curve_parameter.core_parameters.driver_override = VolSurfaceDisplayUtilities.parse_double(exp_disp_grid.get_grid_data(disp_exp, _driver_override_col, Double.NaN));

src/client/spark/AW2/UIAdapters/DualQuadraticModelUIAdapter.cs

@@ -1,4 +1,6 @@
 namespace mbts.ui.controls {
+   using bts.api;
+
    using mbts.analytics;
    using mbts.spark;
    using mbts.user.settings;
@@ -68,7 +70,7 @@
       {
          foreach (VolSurfaceDisplayGridExpiration disp_exp in exp_disp_grid.get_ordered_expirations())
          {
-            VolSurfaceExpiration exp = new VolSurfaceExpiration() { symbol = disp_exp.symbol, expiration = disp_exp.expiration };
+            var exp = disp_exp.ToVSExp(SecurityType.Option);
             VolCurveParameters input_general_parameters = VolSurfaceDataFunctions.get_curve_parameters(vs_input, exp);
             DualQuadraticModelVolCurveParameters dq_parameters = input_general_parameters.dualquadratic_model_parameters;
             /// atm

src/client/spark/AW2/UIAdapters/ExplicitSlopeWingUIAdapter.cs

@@ -1,4 +1,6 @@
 namespace mbts.ui.controls {
+   using bts.api;
+
    using mbts.analytics;
    using mbts.spark;
 
@@ -41,7 +43,7 @@
       {
          foreach (VolSurfaceDisplayGridExpiration disp_exp in exp_disp_grid.get_ordered_expirations())
          {
-            VolSurfaceExpiration exp = new VolSurfaceExpiration() { symbol = disp_exp.symbol, expiration = disp_exp.expiration };
+            var exp = disp_exp.ToVSExp(SecurityType.Option);
             VolCurveParameters curve_parameter = VolSurfaceDataFunctions.get_curve_parameters(vs_input, exp);
             curve_parameter.core_parameters.put_wing_parameters.explicit_slope_wing_parameters.slope = VolSurfaceDisplayUtilities.parse_double(exp_disp_grid.get_grid_data(disp_exp, _p_wing_slope, Double.NaN));
             curve_parameter.core_parameters.call_wing_parameters.explicit_slope_wing_parameters.slope = VolSurfaceDisplayUtilities.parse_double(exp_disp_grid.get_grid_data(disp_exp, _c_wing_slope, Double.NaN));

src/client/spark/AW2/UIAdapters/ImplicitSlopeWingUIAdapter.cs

@@ -1,5 +1,7 @@
 namespace mbts.ui.controls {
-using mbts.analytics;
+   using bts.api;
+
+   using mbts.analytics;
 using mbts.spark;
 using mbts.user.settings;
 
@@ -40,7 +42,11 @@ using System.Collections.Generic;
          /// just for the input values, which in this case are all
          foreach (VolSurfaceDisplayGridExpiration disp_exp in exp_disp_grid.get_ordered_expirations())
          {
-            VolSurfaceExpiration exp = new VolSurfaceExpiration() { symbol = disp_exp.symbol, expiration = disp_exp.expiration };
+            var exp = VolSurfaceExpiration.Default.with(
+               symbol: disp_exp.symbol,
+               expiration: disp_exp.expiration,
+               venue: disp_exp.venue,
+               security_type: SecurityType.Option);
             VolCurveParameters curve_parameter = VolSurfaceDataFunctions.get_curve_parameters(vs_input, exp);
             curve_parameter.core_parameters.put_wing_parameters.simple_slope_wing_parameters.max_vol = VolSurfaceDisplayUtilities.parse_double(exp_disp_grid.get_grid_data(disp_exp, _p_wing_max, Double.NaN));
             curve_parameter.core_parameters.call_wing_parameters.simple_slope_wing_parameters.max_vol = VolSurfaceDisplayUtilities.parse_double(exp_disp_grid.get_grid_data(disp_exp, _c_wing_max, Double.NaN));

src/client/spark/AW2/UIAdapters/MiscColsUIAdapter.cs

@@ -1,5 +1,7 @@
 
 namespace mbts.ui.controls {
+   using bts.api;
+
    using mbts.analytics;
    using mbts.spark;
    using mbts.user.settings;
@@ -64,7 +66,7 @@ namespace mbts.ui.controls {
       {
          foreach (VolSurfaceDisplayGridExpiration disp_exp in exp_disp_grid.get_ordered_expirations())
          {
-            VolSurfaceExpiration exp = new VolSurfaceExpiration() { symbol = disp_exp.symbol, expiration = disp_exp.expiration };
+            var exp = disp_exp.ToVSExp(SecurityType.Option);
             VolCurveParameters curve_parameter = VolSurfaceDataFunctions.get_curve_parameters(vs_input, exp);
             curve_parameter.core_parameters.time_shift = VolSurfaceDisplayUtilities.parse_double(exp_disp_grid.get_grid_data(disp_exp, _time_shift_col, 0.0));
             //curve_parameter.core_parameters.unhinge_exp_from_master = VolSurfaceDisplayUtilities.parse_boolean(exp_disp_grid.get_grid_data(disp_exp, _slave_unhinged_col, false), false);

src/client/spark/AW2/UIAdapters/ParabolicVolPathUIAdapter.cs

@@ -1,4 +1,6 @@
 namespace mbts.ui.controls {
+   using bts.api;
+
    using mbts.analytics;
    using mbts.spark;
    using mbts.user.settings;
@@ -44,7 +46,7 @@
       {
          foreach (VolSurfaceDisplayGridExpiration disp_exp in exp_disp_grid.get_ordered_expirations())
          {
-            VolSurfaceExpiration exp = new VolSurfaceExpiration() { symbol = disp_exp.symbol, expiration = disp_exp.expiration };
+            var exp = disp_exp.ToVSExp(SecurityType.Option);
             ParabolicVolPathParameters exp_params = new ParabolicVolPathParameters()
             {
                down_move_point = Tuple.Create (VolSurfaceDisplayUtilities.parse_double(exp_disp_grid.get_grid_data(disp_exp, _down_market_move, Double.NaN)),

src/client/spark/AW2/UIAdapters/PolynomialModelUIAdapter.cs

@@ -1,4 +1,6 @@
 namespace mbts.ui.controls {
+   using bts.api;
+
    using mbts.analytics;
    using mbts.spark;
    using mbts.user.settings;
@@ -107,7 +109,7 @@
       {
          foreach (VolSurfaceDisplayGridExpiration disp_exp in exp_disp_grid.get_ordered_expirations())
          {
-            VolSurfaceExpiration exp = new VolSurfaceExpiration() { symbol = disp_exp.symbol, expiration = disp_exp.expiration };
+            var exp = disp_exp.ToVSExp(SecurityType.Option);
             PolynomialModelVolCurveParameters input_polynomial_parameters = VolSurfaceDataFunctions.get_polynomial_curve_parameters(vs_input, exp);
             VolCurveParameters input_general_parameters = VolSurfaceDataFunctions.get_curve_parameters(vs_input, exp);
             for (uint p_point_no = 0; p_point_no < _put_cols.Count() ; p_point_no++)

src/client/spark/AW2/UIAdapters/SplineModelUIAdapter.cs

@@ -1,4 +1,6 @@
 namespace mbts.ui.controls {
+   using bts.api;
+
    using mbts.analytics;
    using mbts.spark;
    using mbts.user.settings;
@@ -111,7 +113,11 @@
       {
          foreach (VolSurfaceDisplayGridExpiration disp_exp in exp_disp_grid.get_ordered_expirations())
          {
-            VolSurfaceExpiration exp = new VolSurfaceExpiration() { symbol = disp_exp.symbol, expiration = disp_exp.expiration };
+            var exp = VolSurfaceExpiration.Default.with(
+               symbol: disp_exp.symbol,
+               expiration: disp_exp.expiration,
+               venue: disp_exp.venue,
+               security_type: SecurityType.Option);
             SplineModelVolCurveParameters input_spline_parameters = VolSurfaceDataFunctions.get_spline_curve_parameters(vs_input, exp);
             VolCurveParameters input_general_parameters = VolSurfaceDataFunctions.get_curve_parameters(vs_input, exp);
             for (uint p_point_no = 0; p_point_no < _put_cols.Count(); p_point_no++)

src/client/spark/AW2/UIAdapters/StrikeConversionUIAdapter.cs

@@ -1,4 +1,6 @@
 namespace mbts.ui.controls {
+   using bts.api;
+
    using mbts.analytics;
    using mbts.spark;
    using mbts.user.settings;
@@ -43,7 +45,7 @@
       {
          foreach (VolSurfaceDisplayGridExpiration disp_exp in exp_disp_grid.get_ordered_expirations())
          {
-            VolSurfaceExpiration exp = new VolSurfaceExpiration() { symbol = disp_exp.symbol, expiration = disp_exp.expiration };
+            var exp = disp_exp.ToVSExp(SecurityType.Option);
             VolCurveParameters curve_parameter = VolSurfaceDataFunctions.get_curve_parameters(vs_input, exp);
             curve_parameter.core_parameters.conversion_fwd_slider = VolSurfaceDisplayUtilities.parse_double(exp_disp_grid.get_grid_data(disp_exp, _conv_fwd_slider_col, 0.0));
             curve_parameter.core_parameters.conversion_vol_slider = VolSurfaceDisplayUtilities.parse_double(exp_disp_grid.get_grid_data(disp_exp, _conv_vol_slider_col, 0.0));

src/client/spark/AW2/UIAdapters/TwoWaySlopeVolPathUIAdapter.cs

@@ -1,4 +1,6 @@
 namespace mbts.ui.controls {
+   using bts.api;
+
    using mbts.analytics;
    using mbts.spark;
    using mbts.user.settings;
@@ -49,7 +51,7 @@
       {
          foreach (VolSurfaceDisplayGridExpiration disp_exp in exp_disp_grid.get_ordered_expirations())
          {
-            VolSurfaceExpiration exp = new VolSurfaceExpiration() { symbol = disp_exp.symbol, expiration = disp_exp.expiration };
+            var exp = disp_exp.ToVSExp(SecurityType.Option);
             TwoWaySlopeVolPathParameters exp_params = new TwoWaySlopeVolPathParameters()
             {
                vol_move_for_unit_fwd_down = VolSurfaceDisplayUtilities.parse_double (exp_disp_grid.get_grid_data(disp_exp, _down_slope_col, Double.NaN)),

src/client/spark/AW2/UIAdapters/VolSurfaceUIAdapter.cs

@@ -8,6 +8,7 @@ using mbts.analytics;
 using mbts;
 using mbts.spark;
 using bts;
+using bts.api;
 
 namespace mbts.ui.controls {
    class VolSurfaceUIAdapter {
@@ -91,7 +92,7 @@ namespace mbts.ui.controls {
          IDictionary<VolSurfaceDisplayGridExpiration, IDictionary<VolSurfaceDisplayGridColumn, object>> exp_display_map = new Dictionary<VolSurfaceDisplayGridExpiration, IDictionary<VolSurfaceDisplayGridColumn, object>>();
          foreach (KeyValuePair<VolSurfaceExpiration, IDictionary<VolSurfaceDisplayGridColumn, object>> p in exp_map) {
             VolCurveOutput vc_out = vs_output.exp_output_map.TryGetValue(p.Key, out vc_out) ? vc_out : null;
-            VolSurfaceDisplayGridExpiration disp_exp = new VolSurfaceDisplayGridExpiration(p.Key.symbol, p.Key.expiration, vc_out != null ? vc_out.calc_method : VolSurfaceEnums.VolSurfaceExpirationCalculationMethod.Master);
+            VolSurfaceDisplayGridExpiration disp_exp = new VolSurfaceDisplayGridExpiration(p.Key.symbol, p.Key.expiration, p.Key.venue, vc_out != null ? vc_out.calc_method : VolSurfaceEnums.VolSurfaceExpirationCalculationMethod.Master);
             exp_display_map.Add(disp_exp, p.Value);
          }
          exp_grid = new VolSurfaceDisplayGrid(exp_display_map.Keys.OrderBy(t => t), ordered_cols, exp_display_map);
@@ -103,7 +104,8 @@ namespace mbts.ui.controls {
       public static VolSurfaceInput copy_ui_to_input(VolSurfaceDisplayGrid exp_disp_grid, VolSurfaceSettings settings, VolSurfaceMacroGridSettings grid_settings) {
          // HighPerformanceTimer timer = new HighPerformanceTimer();
          // timer.start();
-         VolSurfaceInput rv = VolSurfaceDataFunctions.get_default_vs_input(settings, exp_disp_grid.get_ordered_expirations().Select(exp => new VolSurfaceExpiration() { expiration = exp.expiration, symbol = exp.symbol }));
+         var rv = VolSurfaceDataFunctions.get_default_vs_input(settings, exp_disp_grid.get_ordered_expirations()
+            .Select(exp => new VolSurfaceExpiration(SecurityType.Option, exp.symbol, exp.expiration, exp.venue)));
          IEnumerable<IReadWriteGridAdapter> grid_adapters = _get_grid_adapters(settings, grid_settings);
          foreach (IReadWriteGridAdapter g in grid_adapters)
             g.copy_grid_to_data(exp_disp_grid, settings, ref rv);
@@ -117,7 +119,7 @@ namespace mbts.ui.controls {
                                              out IDictionary<VolSurfaceDisplayGridExpiration, VolCurveDisplayGraph> exp_graph_map) {
          exp_graph_map = new Dictionary<VolSurfaceDisplayGridExpiration, VolCurveDisplayGraph>();
          foreach (KeyValuePair<VolSurfaceExpiration, VolCurveOutput> p in vs_output.exp_output_map) {
-            VolSurfaceDisplayGridExpiration exp = new VolSurfaceDisplayGridExpiration(p.Key.symbol, p.Key.expiration, p.Value.calc_method);
+            var exp = new VolSurfaceDisplayGridExpiration(p.Key.symbol, p.Key.expiration, p.Key.venue, p.Value.calc_method);
             VolCurveOutput curve_output = p.Value;
             IEnumerable<Tuple<double, double>> exp_graph_series = curve_output.strike_vol_map.Select(kvp => Tuple.Create(kvp.Key, kvp.Value)).OrderBy(t => t.Item1);
             exp_graph_map[exp] = new VolCurveDisplayGraph(exp_graph_series.OrderBy(t => t.Item1),

src/client/spark/AW2/ViewModel/ExpSelectionGridVM.cs

@@ -15,6 +15,7 @@ using System.Collections.Generic;
 using System.Collections.ObjectModel;
 using System.Collections.Specialized;
 using System.Linq;
+using bts.api;
 
 namespace mbts.spark {
    public interface IExpirationBasedRow {
@@ -48,15 +49,15 @@ namespace mbts.spark {
             expirations.sync_with(_model.get_ordered_expirations());
 
             if (added.Any() && !removed.Any()) {
-               visible_rows.sync_with(rows.Where(r => !_hidden_exps.Contains(r.expiration.ToVSExp())).Cast<object>());
+               visible_rows.sync_with(rows.Where(r => !_hidden_exps.Contains(r.expiration.ToVSExp(SecurityType.Option))).Cast<object>());
             } else {
-               var vs_exps = expirations.Select(exp => exp.ToVSExp());
-               var vs_adds = added.Select(exp => exp.ToVSExp());
-               var vs_removes = removed.Select(exp => exp.ToVSExp());
+               var vs_exps = expirations.Select(exp => exp.ToVSExp(SecurityType.Option));
+               var vs_adds = added.Select(exp => exp.ToVSExp(SecurityType.Option));
+               var vs_removes = removed.Select(exp => exp.ToVSExp(SecurityType.Option));
 
-               var true_removed = removed.Where(rmv => !vs_adds.Contains(rmv.ToVSExp()));
-               var true_added = added.Except(vs_removes.Concat(_hidden_exps), v => v.ToVSExp());
-               var changed_calc_modes = added.Where(add => vs_removes.Contains(add.ToVSExp()));
+               var true_removed = removed.Where(rmv => !vs_adds.Contains(rmv.ToVSExp(SecurityType.Option)));
+               var true_added = added.Except(vs_removes.Concat(_hidden_exps), v => v.ToVSExp(SecurityType.Option));
+               var changed_calc_modes = added.Where(add => vs_removes.Contains(add.ToVSExp(SecurityType.Option)));
 
                foreach (var tr in true_removed) {
                   int index = visible_rows.Cast<IExpirationBasedRow>().Select(r => r.expiration).indexOf(tr);
@@ -73,8 +74,8 @@ namespace mbts.spark {
                foreach (var ccm in changed_calc_modes) {
                   int index = visible_rows
                      .Cast<IExpirationBasedRow>()
-                     .Select(r => r.expiration.ToVSExp())
-                     .indexOf(ccm.ToVSExp());
+                     .Select(r => r.expiration.ToVSExp(SecurityType.Option))
+                     .indexOf(ccm.ToVSExp(SecurityType.Option));
                   if (index < 0 || index >= visible_rows.Count) {
                      continue;
                   }
@@ -105,12 +106,12 @@ namespace mbts.spark {
       void _visible_rows_changed(object sender, NotifyCollectionChangedEventArgs e) {
          if (e.Action == NotifyCollectionChangedAction.Remove) {
             foreach (IExpirationBasedRow row in e.OldItems)
-               if (!_hidden_exps.Contains(row.expiration.ToVSExp()))
-                  _hidden_exps.Add(row.expiration.ToVSExp());
+               if (!_hidden_exps.Contains(row.expiration.ToVSExp(SecurityType.Option)))
+                  _hidden_exps.Add(row.expiration.ToVSExp(SecurityType.Option));
          } else if (e.Action == NotifyCollectionChangedAction.Add)
             foreach (IExpirationBasedRow row in e.NewItems)
-               if (_hidden_exps.Contains(row.expiration.ToVSExp()))
-                  _hidden_exps.Remove(row.expiration.ToVSExp());
+               if (_hidden_exps.Contains(row.expiration.ToVSExp(SecurityType.Option)))
+                  _hidden_exps.Remove(row.expiration.ToVSExp(SecurityType.Option));
       }
    }
 }

src/client/spark/AW2/ViewModel/Graph/VolSurfaceGraphVM.cs

@@ -176,7 +176,7 @@ namespace mbts.spark {
 
       public IEnumerable<VolSurfaceExpiration> get_visible_series() {
          if (_earliest_selected_expiration != null) {
-            yield return _earliest_selected_expiration.ToVSExp();
+            yield return _earliest_selected_expiration.ToVSExp(SecurityType.Option);
          }
       }
 
@@ -323,7 +323,7 @@ namespace mbts.spark {
          if (show_implied_vol && _md_exp != null && _md_exp.symbol != _expiration.symbol) {
             var suffix = _md_exp.expiration == _expiration.expiration ? _md_exp.symbol :
                new VolSurfaceDisplayGridExpiration(
-                  _md_exp.symbol, _md_exp.expiration, VolSurfaceEnums.VolSurfaceExpirationCalculationMethod.Master)
+                  _md_exp.symbol, _md_exp.expiration, _md_exp.venue, VolSurfaceEnums.VolSurfaceExpirationCalculationMethod.Master)
                   .ToString();
             return $"{_expiration} ({suffix})";
          }

src/client/spark/AW2/ViewModel/Grid/VolSurfaceGridVM.cs

@@ -26,6 +26,7 @@ namespace mbts.spark.VolSurface {
    using System.Windows.Data;
    using System.Windows.Input;
    using System.Windows.Media;
+   using bts.api;
 
    class VolSurfaceGridVM : DisposableViewModelBase {
       readonly VolSurfaceVM _parent;
@@ -62,7 +63,7 @@ namespace mbts.spark.VolSurface {
          this.model.on_committed_grid_change += _on_grid_updated;
 
          current_grid = this.model.get_active_grid();
-         committed_grid = is_dirty ? this.model.get_committed_grid() : current_grid;
+         committed_grid = this.model.get_committed_grid();
          _columns_model.models.SetItems(current_grid.get_ordered_columns());
 
          // Rows
@@ -263,7 +264,7 @@ namespace mbts.spark.VolSurface {
          if (old_row_count == 0) {
             // Initial data download
             if (_hidden_expirations != null) {
-               visible_rows.AddRange(rows.Where(r => !_hidden_expirations.Contains(r.expiration.ToVSExp())));
+               visible_rows.AddRange(rows.Where(r => !_hidden_expirations.Contains(r.expiration.ToVSExp(SecurityType.Option))));
                _hidden_expirations = null;
             } else {
                visible_rows.AddRange(rows);

src/client/spark/AW2/ViewModel/Grid/VolSurfaceRowContextMenuVM.cs

@@ -1,6 +1,7 @@
 #nullable enable
 namespace mbts.spark {
    using bts;
+   using bts.api;
    using bts.collections;
 
    using mbts.spark.VolSurface;
@@ -170,7 +171,7 @@ namespace mbts.spark {
 
       public ICommand smart_copy_rolls_rates_cmd =>
          _smart_copy_rolls_rates_cmd ??= new Command(
-            () => _parent.model.sync_slave_ref_forwards(expiration.ToVSExp()));
+            () => _parent.model.sync_slave_ref_forwards(expiration.ToVSExp(SecurityType.Option)));
 
       public ICommand import_csv_cmd =>
          _import_csv_cmd ??= FileDialogCommand.open_file(

src/client/spark/AW2/ViewModel/SettingsEditors/VolSurfaceDriverGridVM.cs

@@ -0,0 +1,30 @@
+namespace mbts.spark;
+
+using mbts.ui.utility;
+
+using Spark.Core.Disposables;
+
+using System;
+using System.Collections.ObjectModel;
+
+public class VolSurfaceDriverGridVM : IDisposable {
+   IDisposable _slaves_hook;
+
+   public VolSurfaceDriverGridVM(VolSurfaceDriverVM driver) {
+      this.driver = driver;
+      _slaves_hook = ObservableAutoWrapper.connect(
+         driver.symbols,
+         slaves,
+         sym => new VolSurfaceSlaveVM(driver, new(bts.api.SecurityType.Option, sym.symbol, sym.venue)),
+         sym_vm => sym_vm.Dispose());
+   }
+
+   public VolSurfaceDriverVM driver { get; }
+   public string header => $"{driver.name} ({driver.exchange})";
+   public ObservableCollection<VolSurfaceSlaveVM> slaves { get; } = new();
+
+   public void Dispose() {
+      Disposable.dispose(ref _slaves_hook);
+      slaves.DisposeAll(true);
+   }
+}

src/client/spark/AW2/ViewModel/SettingsEditors/VolSurfaceDriverVM.cs

@@ -0,0 +1,28 @@
+namespace mbts.spark;
+
+using bts.api;
+
+using mbts.analytics;
+
+using System;
+using System.Collections.ObjectModel;
+
+public class VolSurfaceDriverVM : IEquatable<VolSurfaceDriverVM> {
+   public VolSurfaceDriverVM(Venue exchange, string name) {
+      this.exchange = exchange;
+      this.name = name;
+   }
+
+   public string name { get; }
+   public Venue exchange { get; }
+   public ObservableCollection<VolSurfaceSymbolVenueVM> symbols { get; } = new();
+
+   public bool Equals(VolSurfaceDriverVM other) =>
+      other != null && other.name == name && other.exchange == exchange;
+
+   public override bool Equals(object obj) =>
+      Equals(obj as VolSurfaceDriverVM);
+
+   public override int GetHashCode() =>
+      name.GetHashCode();
+}

src/client/spark/AW2/ViewModel/SettingsEditors/VolSurfaceSettingsEditorVM.cs

@@ -3,28 +3,27 @@ namespace mbts.spark {
    using bts.api;
    using bts.collections;
 
-   using mbts;
    using mbts.analytics;
    using mbts.ui;
    using mbts.ui.shell;
    using mbts.ui.utility;
+   using mbts.utils;
 
    using Spark.Core.Disposables;
    using Spark.Core.Protobuf;
 
    using System;
    using System.Collections.Generic;
-   using System.Collections.Immutable;
    using System.Collections.ObjectModel;
-   using System.Collections.Specialized;
    using System.Linq;
    using System.Reactive.Linq;
    using System.Windows.Input;
 
-   public class VolSurfaceSettingsEditorVM : ViewModelBase {
+   public sealed class VolSurfaceSettingsEditorVM : DisposableViewModelBase {
       readonly uint _atm_shifts_id = 0;
       readonly VolSurfaceSettings _settings;
       readonly VolSurfaceCoreSettings _core_settings;
+      readonly List<IDisposable> _subscriptions = new(1);
 
       string _selected_section;
       bool _can_edit_name = true;
@@ -35,7 +34,7 @@ namespace mbts.spark {
          }
          _settings = settings.ensure_no_nulls();
          _core_settings = settings.core_settings;
-         symbol_structure_vm = new VSSymbolStructureVM(bus, settings);
+         symbol_structure_vm = new VolSurfaceSymbolStructureVM(settings);
          vol_model_polynomial_vm.load_from(settings.polynomial_model_settings);
          vol_model_cubic_spline_vm.load_from(settings.spline_model_settings);
          vol_model_dual_quadratic_vm.load_from(settings.dualquadratic_model_settings);
@@ -69,23 +68,26 @@ namespace mbts.spark {
 
          this.commit = new DelegateCommand(commit_impl);
 
+         _dispose = Disposable.Create(
+            bus.subscribe(SparkTags.ExpirationParametersInputTag).OnDispatcher().Subscribe(symbol_structure_vm.on_exp_params_input),
+            symbol_structure_vm);
+
          void commit_impl() {
             var symbols = symbol_structure_vm.driver_grids
-               .SelectMany(dg => dg.symbols.Where(sym => sym.is_selected))
-               .ToList();
-            _core_settings.master_slave_relationships = symbols
-               .Where(sym => sym.masters.Any())
-               .Select(sym => new VolSurfaceSymbolRelationship {
-                  symbol = sym.name,
-                  connected_symbols = sym.masters.Cast<string>().ToList()
-               })
+               .SelectMany(dg => dg.slaves.Where(sym => sym.is_selected))
                .ToList();
+            _core_settings.master_slave_relationships.SetItems(symbols
+               .Where(sym => sym.selected_master_symbols.Any())
+               .Select(sym => VolSurfaceSymbolRelationship.create(
+                  (sym.name, sym.venue),
+                  sym.selected_master_symbols.Select(s => new SymbolVenue() { symbol = s.security_name, venue = s.security_exchange })
+               )));
             _core_settings.implied_vol_marketdata_relationships = symbols
-               .Where(sym => sym.fits.Any())
-               .Select(sym => new VolSurfaceSymbolRelationship {
-                  symbol = sym.name,
-                  connected_symbols = sym.fits.Cast<string>().ToList()
-               })
+               .Where(sym => sym.selected_fits.Any())
+               .Select(sym => VolSurfaceSymbolRelationship.create(
+                  (sym.name, sym.venue),
+                  sym.selected_fits.Select(s => new SymbolVenue() { symbol = s.security_name, venue = s.security_exchange})
+               ))
                .ToList();
 
             vol_model_polynomial_vm.assign_to(_settings.polynomial_model_settings);
@@ -93,7 +95,9 @@ namespace mbts.spark {
             vol_model_dual_quadratic_vm.assign_to(_settings.dualquadratic_model_settings);
             vol_path_parabolic_vm.assign_to(_settings.core_settings.volpath_settings.parabolic_vol_path_settings);
 
-            _core_settings.symbol_pricing_model_map = symbols.Select(sym => Tuple.Create(sym.name, sym.pricing_model)).ToList();
+            foreach (var sym in symbols) {
+               _core_settings.symbol_pricing_model_map[(sym.name, sym.venue)] = sym.pricing_model;
+            }
             _core_settings.put_rotation.strike_vol_movement_pairs = put_rotations
                .Select(r => Tuple.Create(r.target_strike, r.move))
                .ToList();
@@ -139,7 +143,7 @@ namespace mbts.spark {
          set => Set(ref _selected_section, value);
       }
 
-      public VSSymbolStructureVM symbol_structure_vm { get; set; }
+      public VolSurfaceSymbolStructureVM symbol_structure_vm { get; }
       public VolModelPolynomialVM vol_model_polynomial_vm { get; } = new();
       public VolModelCubicSplineVM vol_model_cubic_spline_vm { get; } = new();
       public VolModelDualQuadraticVM vol_model_dual_quadratic_vm { get; } = new();
@@ -363,31 +367,31 @@ namespace mbts.spark {
       public VolSurfaceSettings get_settings() {
          var rv = ProtobufSerializationHelper.clone(_settings);
          var symbols = symbol_structure_vm.driver_grids
-            .SelectMany(dg => dg.symbols.Where(sym => sym.is_selected))
+            .SelectMany(dg => dg.slaves.Where(sym => sym.is_selected))
             .ToList();
          var settings = new VolSurfaceSettings();
          var core_settings = settings.core_settings;
-         core_settings.master_slave_relationships = symbols
-            .Where(sym => sym.masters.Any())
-            .Select(sym => new VolSurfaceSymbolRelationship {
-               symbol = sym.name,
-               connected_symbols = sym.masters.Cast<string>().ToList()
-            })
-            .ToList();
-         core_settings.implied_vol_marketdata_relationships = symbols
-            .Where(sym => sym.fits.Any())
-            .Select(sym => new VolSurfaceSymbolRelationship {
-               symbol = sym.name,
-               connected_symbols = sym.fits.Cast<string>().ToList()
-            })
-            .ToList();
+         core_settings.master_slave_relationships.SetItems(symbols
+            .Where(sym => sym.selected_master_symbols.Any())
+            .Select(sym => VolSurfaceSymbolRelationship.create(
+               (sym.name, sym.venue),
+               sym.selected_master_symbols.Select(s => new SymbolVenue() { symbol = s.security_name, venue = s.security_exchange })
+            )));
+         core_settings.implied_vol_marketdata_relationships.SetItems(symbols
+            .Where(sym => sym.selected_fits.Any())
+            .Select(sym => VolSurfaceSymbolRelationship.create(
+               (sym.name, sym.venue),
+               sym.selected_fits.Select(s => new SymbolVenue() { symbol = s.security_name, venue = s.security_exchange })
+            )));
 
          vol_model_polynomial_vm.assign_to(_settings.polynomial_model_settings);
          vol_model_cubic_spline_vm.assign_to(_settings.spline_model_settings);
          vol_model_dual_quadratic_vm.assign_to(_settings.dualquadratic_model_settings);
          vol_path_parabolic_vm.assign_to(_settings.core_settings.volpath_settings.parabolic_vol_path_settings);
 
-         core_settings.symbol_pricing_model_map = symbols.Select(sym => Tuple.Create(sym.name, sym.pricing_model)).ToList();
+         foreach (var sym in symbols) {
+            _core_settings.symbol_pricing_model_map[(sym.name, sym.venue)] = sym.pricing_model;
+         }
          core_settings.put_rotation.strike_vol_movement_pairs = put_rotations
             .Select(r => Tuple.Create(r.target_strike, r.move))
             .ToList();
@@ -455,154 +459,19 @@ namespace mbts.spark {
          _core_settings.atm_shift.target_terms = term;
       }
 
-      public class VSDriver : IEquatable<VSDriver> {
-         public VSDriver(Venue exchange, string name) {
-            this.exchange = exchange;
-            this.name = name;
+      public class SymbolVenueVM : IEquatable<SymbolVenueVM> {
+         public SymbolVenueVM(string symbol, Venue venue) {
+            this.symbol = symbol;
+            this.venue = venue;
          }
 
-         public string name { get; }
-         public Venue exchange;
-         public ObservableCollection<string> symbols { get; } = new();
+         public string symbol { get; }
+         public Venue venue { get; }
 
-         public bool Equals(VSDriver other) =>
-            other != null && other.name == name && other.exchange == exchange;
-
-         public override bool Equals(object obj) =>
-            Equals(obj as VSDriver);
-
-         public override int GetHashCode() =>
-            name.GetHashCode();
-      }
-
-      public class VSSymbol : DisposableViewModelBase, IEquatable<VSSymbol> {
-         ImmutableList<string> _fitsi = ImmutableList<string>.Empty;
-         ImmutableList<string> _mastersi = ImmutableList<string>.Empty;
-
-         public VSSymbol(VSDriver driver, string name) {
-            this.driver = driver;
-            this.name = name;
-            _dispose = Disposable.Create(
-               fits.observe_immutable().OnDispatcher().Subscribe(f => fitsi = f),
-               masters.observe_immutable().OnDispatcher().Subscribe(m => mastersi = m));
-         }
-
-         public VSDriver driver { get; }
-         public string name { get; }
-         public bool is_selected { get; set; }
-
-         public ObservableCollection<string> fits { get; } = new();
-         public ImmutableList<string> fitsi {
-            get => _fitsi;
-            set => Set(ref _fitsi, value);
-         }
-
-         public ObservableCollection<string> masters { get; } = new();
-         public ImmutableList<string> mastersi {
-            get => _mastersi;
-            set => Set(ref _mastersi, value);
-         }
-
-         public VolSurfaceEnums.VolSurfacePricingModel pricing_model { get; set; }
-
-         public bool Equals(VSSymbol other) =>
-            other != null && name == other.name && driver.Equals(other.driver);
-
-         public override bool Equals(object obj) =>
-            Equals(obj as VSSymbol);
-
-         public override int GetHashCode() =>
-            name.GetHashCode();
-      }
-
-      public class VSSymbolGridVM : IDisposable {
-         public VSSymbolGridVM(VSDriver driver) {
-            this.driver = driver;
-            symbols = new ObservableCollection<VSSymbol>();
-         }
-
-         public VSDriver driver { get; private set; }
-         public string header { get { return "{0} ({1})".fmt(driver.name, driver.exchange); } }
-         public ObservableCollection<VSSymbol> symbols { get; private set; }
-
-         public void Dispose() =>
-            symbols.DisposeAll(true);
-      }
-
-      public class VSSymbolStructureVM : DisposableViewModelBase {
-         readonly Property<VSSymbolGridVM> _selected_driver_grid;
-
-         public VSSymbolStructureVM(IBus bus, VolSurfaceSettings settings) {
-            driver_grids = new ObservableCollection<VSSymbolGridVM>();
-            _selected_driver_grid = backing<VSSymbolGridVM>(nameof(selected_driver_grid));
-            settings.core_settings.master_slave_relationships ??= new List<VolSurfaceSymbolRelationship>();
-            settings.core_settings.implied_vol_marketdata_relationships ??= new List<VolSurfaceSymbolRelationship>();
-            _dispose = Disposable.Create(
-               driver_grids.Subscribe((s, e) =>
-                  selected_driver_grid ??= driver_grids.FirstOrDefault()),
-               bus.subscribe(SparkTags.ExpirationParametersInputTag).OnDispatcher().Subscribe(on_exp_params_input),
-               Disposable.Create(() => driver_grids.DisposeAll(true)));
-
-            void on_exp_params_input(AnalyticsExpirationParametersMap data) {
-               var driver_map = data.get_option_driver_map();
-               var driver_symbols_aux = new Dictionary<VSDriver, HashSet<string>>();
-               bool init_with_settings = driver_grids.Count == 0;
-
-               foreach (var (driver_exp, slave_exps) in driver_map.Where(kv => kv.Key.exchange != Venue.Unset)) {
-                  var driver = new VSDriver(driver_exp.exchange, driver_exp.symbol);
-                  if (!driver_symbols_aux.TryGetValue(driver, out var driven_by_this_driver)) {
-                     driver_symbols_aux[driver] = driven_by_this_driver = new HashSet<string>();
-                  }
-                  driven_by_this_driver.AddRange(slave_exps.Select(s => s.symbol));
-               }
-
-               var driver_symbols = driver_symbols_aux.ToDictionary(kv => kv.Key, kv => new List<string>(kv.Value));
-               driver_grids.RemoveWhere(driver_sym_grid => !driver_symbols.ContainsKey(driver_sym_grid.driver));
-
-               foreach (var symbols in driver_symbols.Values) {
-                  symbols.Sort();
-               }
-
-               foreach (var (key, i) in driver_symbols.OrderBy(kv => kv.Key.name).Select((kv, i) => (key: kv.Key, i))) {
-                  if (!driver_grids.Select(dg => dg.driver).Contains(key)) {
-                     driver_grids.Insert(i, new VSSymbolGridVM(key));
-                  }
-               }
-
-               foreach (var sym_grid in driver_grids) {
-                  sym_grid.driver.symbols.sync_with(driver_symbols[sym_grid.driver]);
-                  sym_grid.symbols.sync_with(driver_symbols[sym_grid.driver].Select(sym => new VSSymbol(sym_grid.driver, sym)));
-               }
-
-               if (!init_with_settings) {
-                  return;
-               }
-
-               foreach (var sym in driver_grids.SelectMany(grid => grid.symbols)) {
-                  if (settings.core_settings.symbol_pricing_model_map is null) {
-                     continue;
-                  }
-                  var pricing_model = settings.core_settings.symbol_pricing_model_map.FirstOrDefault(pm => pm.Item1 == sym.name);
-                  if (pricing_model != null) {
-                     sym.is_selected = true;
-                     sym.pricing_model = pricing_model.Item2;
-                     var master_slaves = settings.core_settings.master_slave_relationships.FirstOrDefault(msr => msr.symbol == sym.name);
-                     if (master_slaves != null)
-                        sym.masters.sync_with(master_slaves.connected_symbols);
-                     var implied_vols = settings.core_settings.implied_vol_marketdata_relationships.FirstOrDefault(iv => iv.symbol == sym.name);
-                     if (implied_vols != null)
-                        sym.fits.sync_with(implied_vols.connected_symbols);
-                  }
-               }
-            }
-         }
-
-         public IReadOnlyList<VolSurfaceEnums.VolSurfacePricingModel> pricing_models => EnumHelper.get_values<VolSurfaceEnums.VolSurfacePricingModel>();
-         public ObservableCollection<VSSymbolGridVM> driver_grids { get; private set; }
-         public VSSymbolGridVM selected_driver_grid {
-            get => _selected_driver_grid.Value;
-            set => _selected_driver_grid.set(value);
-         }
+         public bool Equals(SymbolVenueVM other) =>
+            other is not null
+               && symbol == other.symbol
+               && venue == other.venue;
       }
 
       public class RotationVM : ViewModelBase, IEquatable<RotationVM> {

src/client/spark/AW2/ViewModel/SettingsEditors/VolSurfaceSlaveVM.cs

@@ -0,0 +1,98 @@
+namespace mbts.spark;
+
+using bts.api;
+using bts.collections;
+
+using mbts.analytics;
+using mbts.ui;
+using mbts.ui.shell;
+using mbts.ui.utility;
+
+using Spark.Core.Disposables;
+
+using System;
+using System.Collections.Immutable;
+using System.Collections.ObjectModel;
+
+using SecurityTuple = mbts.SecurityTuple;
+
+public class VolSurfaceSlaveVM : DisposableViewModelBase, IEquatable<VolSurfaceSlaveVM> {
+   ImmutableList<SecurityTuple> _selected_fits = ImmutableList<SecurityTuple>.Empty;
+   ImmutableList<SecurityTuple> _selected_master_symbols = ImmutableList<SecurityTuple>.Empty;
+   VolSurfaceEnums.VolSurfacePricingModel _pricing_model;
+   bool _is_selected;
+
+   public VolSurfaceSlaveVM(VolSurfaceDriverVM driver, SecurityTuple security_tuple) {
+      this.driver = driver;
+      this.security_tuple = security_tuple;
+      _dispose = Disposable.Create(
+         ObservableAutoWrapper.connect(driver.symbols, fit_items, s => new(new(SecurityType.Option, s.symbol, s.venue))),
+         ObservableAutoWrapper.connect_item_hooks(fit_items, init_fit),
+         ObservableAutoWrapper.connect(driver.symbols, master_symbol_items, s => new(new(SecurityType.Option, s.symbol, s.venue))),
+         ObservableAutoWrapper.connect_item_hooks(master_symbol_items, init_master_symbol));
+
+      IDisposable init_master_symbol(SelectableItemVM<SecurityTuple> vm) {
+         return vm.subscribe(on_is_selected_changed);
+
+         void on_is_selected_changed(bool is_selected) =>
+            selected_master_symbols = is_selected
+               ? selected_master_symbols.InsertSorted(vm.value)
+               : selected_master_symbols.RemoveSorted(vm.value);
+      }
+
+      IDisposable init_fit(SelectableItemVM<SecurityTuple> vm) {
+         return vm.subscribe(on_is_selected_changed);
+
+         void on_is_selected_changed(bool is_selected) =>
+            selected_fits = is_selected
+               ? selected_fits.InsertSorted(vm.value)
+               : selected_fits.RemoveSorted(vm.value);
+      }
+   }
+
+   public VolSurfaceDriverVM driver { get; }
+   public SecurityTuple security_tuple { get; }
+   public string name => security_tuple.security_name;
+   public Venue venue => security_tuple.security_exchange;
+
+   public bool is_selected {
+      get => _is_selected;
+      set => Set(ref _is_selected, value);
+   }
+
+   public ImmutableList<SecurityTuple> selected_fits {
+      get => _selected_fits;
+      private set {
+         if (_selected_fits != value) {
+            _selected_fits = value;
+            RaisePropertyChanged();
+         }
+      }
+   }
+   public ObservableCollection<SelectableItemVM<SecurityTuple>> fit_items { get; } = new();
+
+   public ImmutableList<SecurityTuple> selected_master_symbols {
+      get => _selected_master_symbols;
+      private set {
+         if (_selected_master_symbols != value) {
+            _selected_master_symbols = value;
+            RaisePropertyChanged();
+         }
+      }
+   }
+   public ObservableCollection<SelectableItemVM<SecurityTuple>> master_symbol_items { get; } = new();
+
+   public VolSurfaceEnums.VolSurfacePricingModel pricing_model {
+      get => _pricing_model;
+      set => Set(ref _pricing_model, value);
+   }
+
+   public bool Equals(VolSurfaceSlaveVM other) =>
+      other != null && name == other.name && driver.Equals(other.driver);
+
+   public override bool Equals(object obj) =>
+      Equals(obj as VolSurfaceSlaveVM);
+
+   public override int GetHashCode() =>
+      name.GetHashCode();
+}

src/client/spark/AW2/ViewModel/SettingsEditors/VolSurfaceSymbolStructureVM.cs

@@ -0,0 +1,87 @@
+namespace mbts.spark;
+
+using bts.collections;
+
+using mbts.analytics;
+using mbts.ui.shell;
+using mbts.ui.utility;
+
+using Spark.Core.Disposables;
+
+using System;
+using System.Collections.Generic;
+using System.Collections.ObjectModel;
+using System.Collections.Specialized;
+using System.Linq;
+using System.Reactive.Linq;
+
+public class VolSurfaceSymbolStructureVM : DisposableViewModelBase {
+   readonly Property<VolSurfaceDriverGridVM> _selected_driver_grid;
+   readonly VolSurfaceSettings _settings;
+
+   public VolSurfaceSymbolStructureVM(VolSurfaceSettings settings) {
+      _settings = settings;
+      _settings.core_settings.implied_vol_marketdata_relationships ??= new List<VolSurfaceSymbolRelationship>();
+      _selected_driver_grid = backing<VolSurfaceDriverGridVM>(nameof(selected_driver_grid));
+      _dispose = Disposable.Create(
+         driver_grids.Subscribe((s, e) =>
+            selected_driver_grid ??= driver_grids.FirstOrDefault()),
+         Disposable.Create(() => driver_grids.DisposeAll(true)));
+   }
+
+   public IReadOnlyList<VolSurfaceEnums.VolSurfacePricingModel> pricing_models => EnumHelper.get_values<VolSurfaceEnums.VolSurfacePricingModel>();
+   public ObservableCollection<VolSurfaceDriverGridVM> driver_grids { get; } = new();
+   public VolSurfaceDriverGridVM selected_driver_grid {
+      get => _selected_driver_grid.Value;
+      set => _selected_driver_grid.set(value);
+   }
+
+   public void on_exp_params_input(AnalyticsExpirationParametersMap data) {
+      var driver_groups = data.get_option_driver_symbol_map().ToDictionary(
+         g => g.Key,
+         g => g.Select(e => e.security_tuple).Distinct().Ordered().ToList());
+      bool init_with_settings = driver_grids.Count == 0;
+
+      driver_grids.RemoveWhere(dg => !driver_groups.ContainsKey((dg.driver.name, dg.driver.exchange)));
+      driver_grids.sync_with_models(
+         driver_groups.OrderBy(kv => kv.Key),
+         dg => {
+            var driver = new VolSurfaceDriverVM(dg.Key.venue, dg.Key.symbol);
+            driver.symbols.SetItems(dg.Value.Select(s => new VolSurfaceSymbolVenueVM(s.security_name, s.security_exchange)));
+            return new VolSurfaceDriverGridVM(driver);
+         },
+         (vm, dg) => {
+            vm.driver.symbols.sync_with(dg.Value.Select(s => new VolSurfaceSymbolVenueVM(s.security_name, s.security_exchange)));
+            return vm;
+         });
+
+      if (!init_with_settings) {
+         return;
+      }
+
+      var sec_tuple_comp = EqualityComparer<SecurityTuple>.Default;
+      var price_models = _settings.core_settings.symbol_pricing_model_map;
+      if (price_models is null) {
+         return;
+      }
+      foreach (var sym in driver_grids.SelectMany(grid => grid.slaves)) {
+         if (!price_models.TryGetValue((sym.name, sym.venue), out var pricing_model)) {
+            continue;
+         }
+         sym.is_selected = true;
+         sym.pricing_model = pricing_model;
+         var master_slaves = _settings.core_settings.master_slave_relationships.FirstOrDefault(msr => msr.symbol == (sym.name, sym.venue));
+         if (master_slaves != null) {
+            foreach (var m in sym.master_symbol_items) {
+               m.is_selected = master_slaves.connected_symbols.Any(s => s == (m.value.security_name, m.value.security_exchange));
+            }
+         }
+         var implied_vols = _settings.core_settings.implied_vol_marketdata_relationships.FirstOrDefault(iv => iv.symbol == (sym.name, sym.venue));
+         if (implied_vols != null) {
+            foreach (var f in sym.fit_items) {
+               f.is_selected = implied_vols.connected_symbols.Any(s => s == (f.value.security_name, f.value.security_exchange));
+            }
+         }
+      }
+   }
+}

src/client/spark/AW2/ViewModel/SettingsEditors/VolSurfaceSymbolVenueVM.cs

@@ -0,0 +1,34 @@
+#nullable enable
+namespace mbts.spark;
+
+using bts;
+using bts.api;
+
+using System;
+
+public class VolSurfaceSymbolVenueVM : IEquatable<VolSurfaceSymbolVenueVM>, ICloneable<VolSurfaceSymbolVenueVM> {
+   public VolSurfaceSymbolVenueVM(string? symbol, Venue venue) {
+      this.symbol = symbol;
+      this.venue = venue;
+   }
+
+   public string? symbol { get; }
+   public Venue venue { get; }
+
+   public VolSurfaceSymbolVenueVM Clone() =>
+      new(symbol, venue);
+
+   public bool Equals(VolSurfaceSymbolVenueVM? other) =>
+      other is not null
+         && other.symbol == symbol
+         && other.venue == venue;
+
+   public override bool Equals(object? obj) =>
+      Equals(obj as VolSurfaceSymbolVenueVM);
+
+   public override int GetHashCode() =>
+      HashCode.Combine(symbol, venue);
+
+   object ICloneable.Clone() =>
+      Clone();
+}

src/client/spark/AW2/ViewModel/VolTermsGraphVM.cs

@@ -165,16 +165,16 @@
 
       void _update_points() {
          var settings = _model.get_active_settings();
-         var master_symbols = settings == null ? new HashSet<string>() : VolSurfaceUtilityFunctions.get_master_symbols(settings);
+         var master_symbols = settings == null ? new HashSet<(string, Venue)>() : VolSurfaceUtilityFunctions.get_master_symbols(settings);
          var committed_map = _get_exp_date_map(_model.get_committed_term_structure_map(), master_symbols);
          var candidate_map = _get_exp_date_map(_model.get_candidate_term_structure_map(), master_symbols);
          foreach (var s in graph_vm.series.Cast<IVolTermsSeries>()) s.update_points(s.is_committed ? committed_map : candidate_map);
       }
 
       Dictionary<DateTime, double> _get_exp_date_map(
-         IReadOnlyDictionary<VolSurfaceExpiration, double> exp_map, HashSet<string> master_symbols) =>
+         IReadOnlyDictionary<VolSurfaceExpiration, double> exp_map, HashSet<(string, Venue)> master_symbols) =>
          exp_map.Where(kv => selected_symbols.Contains(kv.Key.symbol) &&
-            master_symbols.Contains(kv.Key.symbol))
+            master_symbols.Contains((kv.Key.symbol, kv.Key.venue)))
             .ToDictionaryCrushed(kv => kv.Key.expiration, kv => kv.Value);
 
       public class SymbolItem : IEquatable<SymbolItem> {

src/client/spark/AW2/VolSurfaceDisplayModel.cs

@@ -250,7 +250,11 @@
 
       public IEnumerable<Tuple<double, double>> get_committed_volpath(VolSurfaceDisplayGridExpiration exp) {
          if (_committed_data != null && _committed_data.vs_output != null) {
-            VolCurveOutput exp_out = _committed_data.vs_output[new VolSurfaceExpiration(exp.symbol, exp.expiration)];
+            VolCurveOutput exp_out = _committed_data.vs_output[VolSurfaceExpiration.Default.with(
+               security_type: SecurityType.Option,
+               symbol: exp.symbol,
+               expiration: exp.expiration,
+               venue: exp.venue)];
             return VolSurfaceUtilityFunctions.get_vol_path_points(exp_out);
          } else
             return Enumerable.Empty<Tuple<double, double>>();
@@ -258,7 +262,11 @@
 
       public IEnumerable<Tuple<double, double>> get_candidate_volpath(VolSurfaceDisplayGridExpiration exp) {
          if (is_dirty && _candidate_data != null && _candidate_data.vs_output != null) {
-            VolCurveOutput exp_out = _candidate_data.vs_output[new VolSurfaceExpiration(exp.symbol, exp.expiration)];
+            VolCurveOutput exp_out = _candidate_data.vs_output[VolSurfaceExpiration.Default.with(
+               security_type: SecurityType.Option,
+               symbol: exp.symbol,
+               expiration: exp.expiration,
+               venue: exp.venue)];
             return VolSurfaceUtilityFunctions.get_vol_path_points(exp_out);
          } else
             return Enumerable.Empty<Tuple<double, double>>();
@@ -277,15 +285,11 @@
          return !is_dirty || _candidate_data == null || _candidate_data.vs_output == null ? new Dictionary<VolSurfaceExpiration, double>() : _candidate_data.vs_output.exp_output_map.ToDictionary(kv => kv.Key, kv => kv.Value.atm_vol);
       }
 
-      public VolCurveMarketImpliedVolData get_market_market_implied_vols(VolSurfaceDisplayGridExpiration exp) {
-         Tuple<VolSurfaceExpiration, VolCurveMarketImpliedVolData> rv = null;
-         return _impl_vol_map.TryGetValue(new VolSurfaceExpiration() { symbol = exp.symbol, expiration = exp.expiration }, out rv) ? rv.Item2 : null;
-      }
+      public VolCurveMarketImpliedVolData get_market_market_implied_vols(VolSurfaceDisplayGridExpiration exp) =>
+         _impl_vol_map.GetValueOrDefault(exp.ToVSExp(SecurityType.Option))?.Item2;
 
-      public VolSurfaceExpiration get_market_implied_vol_source(VolSurfaceDisplayGridExpiration exp) {
-         Tuple<VolSurfaceExpiration, VolCurveMarketImpliedVolData> rv = null;
-         return _impl_vol_map.TryGetValue(new VolSurfaceExpiration() { symbol = exp.symbol, expiration = exp.expiration }, out rv) ? rv.Item1 : null;
-      }
+      public VolSurfaceExpiration get_market_implied_vol_source(VolSurfaceDisplayGridExpiration exp) =>
+         _impl_vol_map.GetValueOrDefault(exp.ToVSExp(SecurityType.Option))?.Item1;
 
       #endregion
 
@@ -374,7 +378,11 @@
                string[] data_items = line.Split(',').Select(cn => cn.Trim()).ToArray();
                if (data_items.Length != col_names.Length)
                   throw new ApplicationException("Data is not well formed");
-               VolSurfaceDisplayGridExpiration grid_exp = new VolSurfaceDisplayGridExpiration(data_items[0], DateTime.Parse(data_items[1]), VolSurfaceEnums.VolSurfaceExpirationCalculationMethod.Master);
+               VolSurfaceDisplayGridExpiration grid_exp = new(
+                  data_items[0],
+                  DateTime.Parse(data_items[1]),
+                  Enum.Parse<Venue>(data_items[2]),
+                  VolSurfaceEnums.VolSurfaceExpirationCalculationMethod.Master);
                for (int i = 2; i < col_names.Length; i++) {
                   string col_name = file_col_map[i];
                   string data_text = data_items[i];
@@ -508,7 +516,7 @@
       public void fit_to_market(IEnumerable<VolSurfaceDisplayGridExpiration> exp_list, bool is_all_curve, bool is_live_market,
                                 bool use_weighted_regression_fit, bool cascade_up, bool cascade_down) {
          DisplayModelData curr_data = _get_current_data();
-         IEnumerable<VolSurfaceExpiration> vs_exps = exp_list.Select(i => i.ToVSExp());
+         IEnumerable<VolSurfaceExpiration> vs_exps = exp_list.Select(i => i.ToVSExp(SecurityType.Option));
          _send_curve_action_request(vs_exps.ToDictionary(e => e,
                                                          e => {
                                                             VolCurveMarketFitAction.ActionType action_type = is_all_curve ? VolCurveMarketFitAction.ActionType.FitWholeCurve : VolCurveMarketFitAction.ActionType.FitATM;
@@ -518,7 +526,7 @@
       }
 
       public void rebuild(IEnumerable<VolSurfaceDisplayGridExpiration> exp_list, bool cascade_up, bool cascade_down) {
-         _send_curve_action_request(exp_list.ToDictionary(e => e.ToVSExp(), e => new VolCurveAction(VolCurveAction.ActionType.Rebuild)), cascade_up, cascade_down);
+         _send_curve_action_request(exp_list.ToDictionary(e => e.ToVSExp(SecurityType.Option), e => new VolCurveAction(VolCurveAction.ActionType.Rebuild)), cascade_up, cascade_down);
       }
 
       public void publish_new_grid_selection(string window_id, VolSurfaceDisplayGridExpiration selected_expiration, bool show_implied_vol_checked) {
@@ -564,7 +572,7 @@
       /// operation go call this
       private void _send_curve_operation_request(IEnumerable<VolSurfaceDisplayGridExpiration> exp_list, VolCurveMovementOperation curve_operation,
                                                  bool is_up, bool cascade_up, bool cascade_down) {
-         _send_curve_action_request(exp_list.ToDictionary(e => e.ToVSExp(), e => new VolCurveAction(VolSurfaceUtilityFunctions.add_side_to_curve_operation(curve_operation, is_up))),
+         _send_curve_action_request(exp_list.ToDictionary(e => e.ToVSExp(SecurityType.Option), e => new VolCurveAction(VolSurfaceUtilityFunctions.add_side_to_curve_operation(curve_operation, is_up))),
                                                           cascade_up, cascade_down);
       }
 
@@ -604,15 +612,10 @@
       }
 
       private void _copy_time_parameters(VolSurfaceInput source, ref VolSurfaceInput target) {
-         target.surface_parameters = target.surface_parameters ?? new VolSurfaceParameters();
-         target.surface_parameters.time_parameters = target.surface_parameters.time_parameters ?? new VolSurfaceTimeParameters() { input_time = DateTime.MinValue, auto_decay_points = new List<Tuple<DateTime, double>>() };
-         if (source != null && source.surface_parameters != null) {
-            target.surface_parameters.vol_time = source.surface_parameters.vol_time;
-            if (source.surface_parameters.time_parameters != null) {
-               target.surface_parameters.time_parameters.input_time = source.surface_parameters.time_parameters.input_time;
-               target.surface_parameters.time_parameters.auto_decay_points = source.surface_parameters.time_parameters.auto_decay_points == null ? target.surface_parameters.time_parameters.auto_decay_points : source.surface_parameters.time_parameters.auto_decay_points.ToList();
-            }
-         }
+         target.surface_parameters ??= new();
+         target.surface_parameters.vol_time = source?.surface_parameters?.vol_time ?? default;
+         target.surface_parameters.time_parameters = source?.surface_parameters?.time_parameters?.Clone()
+            ?? VolSurfaceTimeParameters.Default;
       }
 
       #region Stateless Helpers

src/client/spark/AW2/VolSurfaceExtensions.cs

@@ -18,8 +18,8 @@ namespace mbts.spark {
 
       // returns true if the option_name is driven by vs_id
       public static bool symbol_driven_by_vs(
-         this VolSurfaceUsageListImmutable list, string option_name, int vs_id) =>
-         list.UsageList.Any(u => u.SecTup.Symbol == option_name && u.VsId == vs_id);
+         this VolSurfaceUsageListImmutable list, (string, Venue) option_name, int vs_id) =>
+         list.UsageList.Any(u => (u.SecTup.Symbol, u.SecTup.Venue) == option_name && u.VsId == vs_id);
 
       // returns all sec_tups that are driven by this vs_id
       public static IEnumerable<bts.api.SecurityTuple> get_sectups(
@@ -37,8 +37,8 @@ namespace mbts.spark {
       public static bool is_editable(this VolSurfaceColumnType vs_col_type) =>
          vs_col_type is VolSurfaceColumnType.ReadWriteTick;
 
-      public static analytics.VolSurfaceExpiration ToVSExp(this VolSurfaceDisplayGridExpiration vsdge) =>
-         new() { expiration = vsdge.expiration, symbol = vsdge.symbol };
+      public static analytics.VolSurfaceExpiration ToVSExp(this VolSurfaceDisplayGridExpiration vsdge, SecurityType security_type) =>
+         new(security_type, vsdge.symbol, vsdge.expiration, vsdge.venue);
 
       public static SecurityExpiration ToSecurityExp(this VolSurfaceDisplayGridExpiration vsdge) =>
          new(Venue.Unset, SecurityType.Option, vsdge.symbol, vsdge.expiration);
@@ -84,7 +84,7 @@ namespace mbts.spark {
          };
 
       public static IEnumerable<VolSurfaceExpiration> select_exp(this IEnumerable<VolSurfaceRowVM> rows) =>
-         rows.Select(r => r.expiration.ToVSExp());
+         rows.Select(r => r.expiration.ToVSExp(SecurityType.Option));
 
       public static IEnumerable<VolSurfaceDisplayGridExpiration> select_display_exp(this IEnumerable<VolSurfaceRowVM> rows) =>
          rows.Select(r => r.expiration);
@@ -142,7 +142,12 @@ namespace mbts.spark {
          };
 
       public static VolSurfaceExpiration to_vs_exp(this SecurityExpiration sec_exp) =>
-         new() { security_type = sec_exp.security_type, symbol = sec_exp.symbol, expiration = sec_exp.expiration };
+         new() {
+            security_type = sec_exp.security_type,
+            symbol = sec_exp.symbol,
+            expiration = sec_exp.expiration,
+            venue = sec_exp.exchange,
+         };
 
       public static DriverCalculationType to_driver_calc_type(
          this ExpirationDriver.ModeOneofCase driver_mode) =>
@@ -170,7 +175,7 @@ namespace mbts.spark {
 
       public static SecurityExpiration to_sec_exp(this SecurityImmutable sec) {
          var exp_date = sec.security_expiration() ?? throw new InvalidOperationException();
-         return new(sec.Exchange, sec.ContractCase, sec.Symbol, exp_date, sec.Id);
+         return new(sec.Exchange, sec.ContractCase, sec.Symbol, exp_date);
       }
 
       public static SecurityTuple security_tuple(this Security sec) {
@@ -217,6 +222,9 @@ namespace mbts.spark {
          this SecurityExpiration sec_exp) =>
          new(to_api_sec_exp(sec_exp));
 
+   public static SecurityExpiration to_security_expiration(this VolSurfaceExpiration vs_exp) =>
+      new(vs_exp.venue, vs_exp.security_type, vs_exp.symbol, vs_exp.expiration);
+
       static IEnumerable<(double x, double y)> _select_real_points(IEnumerable<Tuple<double, double>> points) =>
          points.Where(pt => pt.Item1.is_real() && pt.Item2.is_real()).Select(t => (t.Item1, t.Item2));
    }

src/client/spark/AW2/VolSurfaceInterpGreeksMultiplexer.cs

@@ -77,7 +77,7 @@ namespace mbts.spark {
          if (theos != null) {
             var theo_map = _convert_theos(theos);
             foreach (var kv in theo_map) {
-               string option_name = kv.Key;
+               var option_name = kv.Key;
                if (_vs_usage_map.symbol_driven_by_vs(option_name, vs_id)) {
                   bts.Log.Gen.info(
                      "VolSurfaceInterpGreeksMultiplexer._read_theos_and_publish " +
@@ -89,15 +89,14 @@ namespace mbts.spark {
       }
 
       #region Stateless Helpers
-      private static IDictionary<string, OptionTheoInterpolationData> _convert_theos(
+      private static IDictionary<(string, Venue), OptionTheoInterpolationData> _convert_theos(
          IDictionary<VolSurfaceExpiration, IEnumerable<Theos>> vs_theos) {
-         IDictionary<string, OptionTheoInterpolationData> rv =
-            new Dictionary<string, OptionTheoInterpolationData>();
+         var rv = new Dictionary<(string, Venue), OptionTheoInterpolationData>();
          foreach (var kv in vs_theos) {
             OptionTheoInterpolationData sym_data = null;
-            if (!rv.TryGetValue(kv.Key.symbol, out sym_data)) {
-               sym_data = new OptionTheoInterpolationData(kv.Key.symbol);
-               rv.Add(kv.Key.symbol, sym_data);
+            if (!rv.TryGetValue((kv.Key.symbol, kv.Key.venue), out sym_data)) {
+               sym_data = new OptionTheoInterpolationData(kv.Key.symbol, kv.Key.venue);
+               rv.Add((kv.Key.symbol, kv.Key.venue), sym_data);
             }
             foreach (var t in kv.Value)
                sym_data.set_data(kv.Key.expiration, t.driver,

src/client/spark/Analytics/Core/Parameters/VolSurfaceTimeParameters.cs

@@ -1,17 +1,46 @@
-using System;
+#nullable enable
+using bts.collections;
+
+using ProtoBuf;
+
+using Spark.Core.Protobuf;
+
+using System;
 using System.Collections.Generic;
 using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 
-namespace mbts.analytics
-{
-   [ProtoBuf.ProtoContract(Name = "VolSurfaceTimeParameters")]
-   public class VolSurfaceTimeParameters
-   {
-      [ProtoBuf.ProtoMember(1)]
-      public DateTime input_time;
-      [ProtoBuf.ProtoMember(10)]
-      public List<Tuple<DateTime, double>> auto_decay_points;
-   }
+namespace mbts.analytics;
+
+[ProtoContract(Name = "VolSurfaceTimeParameters")]
+public class VolSurfaceTimeParameters : IProtobuf<VolSurfaceTimeParameters> {
+   public static VolSurfaceTimeParameters Default =>
+      new() {
+         input_time = DateTime.MinValue,
+         auto_decay_points = new(0)
+      };
+
+   [ProtoMember(1)]
+   public DateTime input_time { get; set; }
+
+   [ProtoMember(10)]
+   public List<Tuple<DateTime, double>>? auto_decay_points { get; set; }
+
+   public VolSurfaceTimeParameters Clone() =>
+      new() {
+         input_time = input_time,
+         auto_decay_points = auto_decay_points
+            ?.WrapSelector(dp => Tuple.Create(dp.Item1, dp.Item2), null)
+            ?.ToList()
+      };
+
+   public bool Equals(VolSurfaceTimeParameters? other) =>
+      other is not null
+      && input_time == other.input_time
+      && auto_decay_points.safe_sequence_equals(other.auto_decay_points);
+
+   public override bool Equals(object? obj) =>
+      Equals(obj as VolSurfaceTimeParameters);
+
+   public override int GetHashCode() =>
+      HashCode.Combine(input_time, auto_decay_points?.Count);
 }

src/client/spark/Analytics/Core/Settings/SecurityExpiration.cs

@@ -37,19 +37,17 @@ namespace mbts {
          Venue exchange,
          Security.ContractOneofCase type,
          string symbol,
-         DateTime exp,
-         uint? id = null)
-         : this(exchange, type.security_type(), symbol, exp, id) { }
+         DateTime exp)
+         : this(exchange, type.security_type(), symbol, exp) { }
 
       public SecurityExpiration(
-         Venue exchange, SecurityType type, string symbol, DateTime expiration, uint? id = null)
-         : this(new SecurityTuple(type, symbol, exchange), expiration, id) { }
+         Venue exchange, SecurityType type, string symbol, DateTime expiration)
+         : this(new SecurityTuple(type, symbol, exchange), expiration) { }
 
       public SecurityExpiration(
-         SecurityTuple security_tuple, DateTime expiration, uint? id = null) {
+         SecurityTuple security_tuple, DateTime expiration) {
          this.security_tuple = security_tuple;
          this.expiration = expiration;
-         this.id = id;
       }
 
       [ProtoMember(1)]
@@ -58,9 +56,6 @@ namespace mbts {
       [ProtoMember(2)]
       public DateTime expiration { get; }
 
-      [ProtoMember(3)]
-      public uint? id { get; }
-
       public SecurityType security_type => security_tuple?.security_type ?? SecurityType.Unset;
       public string symbol => security_tuple?.security_name ?? string.Empty;
       public Venue exchange => security_tuple?.security_exchange ?? Venue.Unset;
@@ -71,7 +66,6 @@ namespace mbts {
          Symbol = 1 << 1,
          Expiration = 1 << 2,
          Venue = 1 << 3,
-         Id = 1 << 4
       }
 
       public static IEqualityComparer<SecurityExpiration> get_eq_comparer() =>
@@ -88,7 +82,6 @@ namespace mbts {
                Properties.Symbol => wrap(s => s.symbol).to_safe(),
                Properties.Expiration => wrap(s => s.expiration),
                Properties.Venue => wrap(s => s.exchange),
-               Properties.Id => wrap(s => s.id).to_safe(),
                _ => throw new NotImplementedException()
             };
 
@@ -109,7 +102,6 @@ namespace mbts {
             Properties.Symbol => ComparerWrapper<SecurityExpiration>.wrap(s => s.symbol),
             Properties.Expiration => ComparerWrapper<SecurityExpiration>.wrap(s => s.expiration),
             Properties.Venue => ComparerWrapper<SecurityExpiration>.wrap(s => s.exchange),
-            Properties.Id => ComparerWrapper<SecurityExpiration>.wrap(s => s.id),
             _ => throw new NotImplementedException()
          };
       }
@@ -126,19 +118,18 @@ namespace mbts {
          new(Venue.Unset, security_type, symbol, expiration);
 
       public SecurityExpiration Clone() =>
-         new(security_tuple.Clone(), expiration, id);
+         new(security_tuple.Clone(), expiration);
 
       public bool Equals(SecurityExpiration? other) =>
          other is not null &&
          security_tuple.safe_equals(other.security_tuple) &&
-         expiration == other.expiration &&
-         id == other.id;
+         expiration == other.expiration;
 
       public override bool Equals(object? obj) =>
          Equals(obj as SecurityExpiration);
 
       public override int GetHashCode() =>
-         HashCode.Combine(security_tuple, expiration, id ?? 0);
+         HashCode.Combine(security_tuple, expiration);
 
       object ICloneable.Clone() =>
          Clone();
@@ -186,11 +177,10 @@ namespace mbts {
          Venue? exchange = null,
          SecurityType? security_type = null,
          string? symbol = null,
-         DateTime? expiration = null,
-         Optional<uint> id = default) => new(
-            exchange ?? sec_exp.exchange, security_type ?? sec_exp.security_type,
+         DateTime? expiration = null) => new(
+            exchange ?? sec_exp.exchange,
+            security_type ?? sec_exp.security_type,
             symbol ?? sec_exp.symbol,
-            expiration ?? sec_exp.expiration,
-            id.has_value ? id : sec_exp.id);
+            expiration ?? sec_exp.expiration);
    }
 }

src/client/spark/Analytics/Core/Settings/SymbolVenue.cs

@@ -0,0 +1,52 @@
+#nullable enable
+namespace mbts.analytics;
+
+using bts.api;
+
+using ProtoBuf;
+
+using Spark.Core.Protobuf;
+
+using System;
+
+[ProtoContract]
+public class SymbolVenue : IProtobuf<SymbolVenue> {
+   [ProtoMember(1)]
+   public string? symbol { get; set; }
+
+   [ProtoMember(2)]
+   public Venue venue { get; set; }
+
+   public SymbolVenue Clone() =>
+      new() {
+         symbol = symbol,
+         venue = venue
+      };
+
+   public bool Equals(SymbolVenue? other) =>
+      other is not null
+         && other.symbol == symbol
+         && other.venue == venue;
+
+   public override bool Equals(object? obj) =>
+      Equals(obj as SymbolVenue);
+
+
+   public override int GetHashCode() =>
+      HashCode.Combine(symbol, venue);
+
+   public static bool operator ==(SymbolVenue left, SymbolVenue right) =>
+      left is null ? right is null : left.Equals(right);
+
+   public static bool operator !=(SymbolVenue left, SymbolVenue right) =>
+      !(left == right);
+
+   public static implicit operator SymbolVenue((string symbol, Venue venue) tup) =>
+      new() {
+         symbol = tup.symbol,
+         venue = tup.venue
+      };
+
+   public static implicit operator (string symbol, Venue venue)(SymbolVenue symbol_venue) =>
+      (symbol_venue.symbol ?? string.Empty, symbol_venue.venue);
+}

src/client/spark/Analytics/Core/Settings/VolSurfaceCoreSettings.cs

@@ -1,3 +1,5 @@
+using Google.Protobuf.Collections;
+
 using System;
 using System.Collections.Generic;
 using System.Linq;
@@ -15,10 +17,10 @@ namespace mbts.analytics {
       public int vol_surface_id;
       [ProtoBuf.ProtoMember(10)]
       public VolSurfaceEnums.VolSurfaceModel vol_surface_model;
-      [ProtoBuf.ProtoMember(20)]
-      public List<string> symbols;
-      [ProtoBuf.ProtoMember(25)]
-      public List<VolSurfaceSymbolStructure> symbol_structure_list;
+      // [ProtoBuf.ProtoMember(20)]
+      // public List<string> symbols;
+      // [ProtoBuf.ProtoMember(25)]
+      // public List<VolSurfaceSymbolRelationship> symbol_structure_list;
       [ProtoBuf.ProtoMember(30)]
       public VolSurfaceWingSettings wing_settings;
       [ProtoBuf.ProtoMember(40)]
@@ -62,13 +64,13 @@ namespace mbts.analytics {
       /// Some symbols could use other symbols's market data ex
       /// spx using es's
       [ProtoBuf.ProtoMember(160)]
-      public List<VolSurfaceSymbolStructure> symbol_implied_vol_marketdata_source_list;
+      public List<VolSurfaceSymbolRelationship> symbol_implied_vol_marketdata_source_list;
       [ProtoBuf.ProtoMember(170)]
-      public List<Tuple<string, VolSurfaceEnums.VolSurfacePricingModel>> symbol_pricing_model_map;
+      public MapField<SymbolVenue, VolSurfaceEnums.VolSurfacePricingModel> symbol_pricing_model_map { get; } = new();
       [ProtoBuf.ProtoMember(180)]
       public List<VolSurfaceSymbolRelationship> implied_vol_marketdata_relationships;
       [ProtoBuf.ProtoMember(190)]
-      public List<VolSurfaceSymbolRelationship> master_slave_relationships;
+      public List<VolSurfaceSymbolRelationship> master_slave_relationships { get; } = new(1);
       /// This tells how the abstract strike axis (moneyness, delta etc)
       /// is converted to a real strike. That depends on vol and fwd.
       /// This tells what vol and fwd to use for that.

src/client/spark/Analytics/Core/Settings/VolSurfaceSettings.cs

@@ -14,8 +14,6 @@ namespace mbts.analytics {
             strangle = new() { strike_vol_movement_pairs = new() },
             full_rotation = new() { strike_vol_movement_pairs = new() },
             atm_shift = new() { strike_vol_movement_pairs = new() },
-            symbols = new(),
-            symbol_structure_list = new(),
             wing_settings = new() {
                explicit_slope_wing_settings = new(),
                implicit_slope_wing_settings = new(),

src/client/spark/Analytics/Core/VolSurfaceExpiration.cs

@@ -1,98 +1,95 @@
-using bts.api;
-using bts.settings;
-using bts.spark;
-using bts.utils;
+#nullable enable
+using bts.api;
 
-using mbts.spark;
-using mbts.ui.converters;
+using mbts.utils;
 
 using ProtoBuf;
 
 using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 
-namespace mbts.analytics {
-   /// <summary>
-   /// An expiration in the Vol Surface is expressed
-   /// with a symbol and an expiration. This is a 
-   /// conveniant way to express unique expirations,
-   /// but it could very well be a string, that maps
-   /// to a real expiration.
-   /// </summary>
-   [ProtoBuf.ProtoContract(Name = "VolSurfaceExpiration")]
-   public class VolSurfaceExpiration : IEquatable<VolSurfaceExpiration>, IComparable<VolSurfaceExpiration> {
-      public VolSurfaceExpiration() : this(string.Empty, default(DateTime)) { }
+namespace mbts.analytics;
 
-      public VolSurfaceExpiration(string symbol, DateTime expiration)
-      :
-      this(bts.api.SecurityType.Option, symbol, expiration) {
-      }
+/// <summary>
+/// An expiration in the Vol Surface is expressed
+/// with a symbol and an expiration. This is a 
+/// conveniant way to express unique expirations,
+/// but it could very well be a string, that maps
+/// to a real expiration.
+/// </summary>
+[ProtoContract(Name = "VolSurfaceExpiration")]
+public class VolSurfaceExpiration : IEquatable<VolSurfaceExpiration>, IComparable<VolSurfaceExpiration> {
+   static readonly ComparerBuilder<VolSurfaceExpiration> _comparer = new() {
+      x => x.expiration,
+      x => x.symbol,
+      x => x.security_type,
+      x => x.venue,
+   };
 
-      public VolSurfaceExpiration(bts.api.SecurityType security_type, string symbol, DateTime expiration) {
-         this.security_type = security_type;
-         this.symbol = symbol;
-         this.expiration = expiration;
-      }
+   public VolSurfaceExpiration()
+      : this(SecurityType.Unset, string.Empty, default, Venue.Unset) { }
 
-      public VolSurfaceExpiration(SecurityExpiration exp) {
-         symbol = exp.symbol;
-         expiration = exp.expiration;
-      }
-
-      public SecurityExpiration to_security_expiration(
-         Venue exchange = Venue.Unset, SecurityType type = SecurityType.Unset) =>
-         new(exchange, security_type, symbol, expiration);
-
-      [ProtoBuf.ProtoMember(10)]
-      public string symbol;
-      [ProtoBuf.ProtoMember(20)]
-      public DateTime expiration;
-      [ProtoBuf.ProtoMember(30)]
-      public bts.api.SecurityType security_type = bts.api.SecurityType.Option;
-
-
-      public bool Equals(VolSurfaceExpiration other) {
-         return other != null && symbol.Equals(other.symbol) && expiration.Equals(other.expiration);
-      }
-
-      public override int GetHashCode() { return security_type.GetHashCode() ^ symbol.GetHashCode() ^ expiration.GetHashCode(); }
-
-      public override bool Equals(object obj) {
-         return Equals(obj as VolSurfaceExpiration);
-      }
-
-      public override string ToString() {
-         return String.Format("{0} {1} {2}",
-            security_type.ToString().ToUpper().First(), symbol, expiration.ToShortDateString());
-      }
-
-      public int CompareTo(VolSurfaceExpiration other) {
-         other = other ?? new VolSurfaceExpiration() { expiration = DateTime.MinValue, symbol = String.Empty };
-         int rv = expiration.CompareTo(other.expiration);
-         return rv != 0 ? rv : symbol.CompareTo(other.symbol);
-      }
-
-      public double diff_days(VolSurfaceExpiration exp_other) {
-         TimeSpan ts = expiration - exp_other.expiration;
-         return ts.TotalDays;
-      }
+   public VolSurfaceExpiration(SecurityType security_type, string symbol, DateTime expiration, Venue venue) {
+      this.security_type = security_type;
+      this.symbol = symbol;
+      this.expiration = expiration;
+      this.venue = venue;
    }
 
-   public static class VolSurfaceExpirationExtensions {
-      public static SecurityExpiration ToSecurityExp(
-         this VolSurfaceExpiration vs_exp, SecurityType type) =>
-         new(Venue.Unset, type, vs_exp.symbol, vs_exp.expiration);
+   public VolSurfaceExpiration(SecurityExpiration exp) {
+      symbol = exp.symbol;
+      expiration = exp.expiration;
+      security_type = exp.security_type;
+      venue = exp.exchange;
    }
 
-   public class VolSurfaceStrikeMap {
-      public readonly IDictionary<VolSurfaceExpiration, IEnumerable<double>> strike_map = null;
-      public VolSurfaceStrikeMap(IEnumerable<string> option_names, SystemContractData sc_data) {
-         var on_map = new HashSet<string>(option_names);
-         var aux_map = sc_data.get_option_strikes(on_map);
-         strike_map = aux_map.ToDictionaryDupes(kv => new VolSurfaceExpiration(kv.Key), kv => kv.Value);
+   public static VolSurfaceExpiration Default =>
+      new(SecurityType.Unset, string.Empty, default, Venue.Unset);
+
+   [ProtoMember(10)]
+   public string symbol { get; set; }
+
+   [ProtoMember(20)]
+   public DateTime expiration { get; set; }
+
+   [ProtoMember(30)]
+   public SecurityType security_type { get; set; }
+
+   [ProtoMember(40)]
+   public Venue venue { get; set; }
+
+   public VolSurfaceExpiration with(
+      string? symbol = null,
+      DateTime? expiration = null,
+      SecurityType? security_type = null,
+      Venue? venue = null
+   ) =>
+      new(
+         security_type ?? this.security_type,
+         symbol ?? this.symbol,
+         expiration ?? this.expiration,
+         venue ?? this.venue);
+
+   public bool Equals(VolSurfaceExpiration? other) =>
+      other is not null
+         && symbol == other.symbol
+         && expiration == other.expiration
+         && security_type == other.security_type
+         && venue == other.venue;
+
+   public int CompareTo(VolSurfaceExpiration? other) =>
+      _comparer.Compare(this, other);
+
+   public override int GetHashCode() =>
+      HashCode.Combine(security_type, symbol, expiration, venue);
+
+   public override bool Equals(object? obj) =>
+      Equals(obj as VolSurfaceExpiration);
+
+   public override string ToString() {
+      var rv = $"{char.ToUpper(security_type.ToString()[0])} {symbol} {expiration.ToShortDateString()}";
+      if (venue != Venue.Unset) {
+         rv += " " + venue.ToString();
       }
+      return rv;
    }
 }

src/client/spark/Analytics/Core/VolSurfaceStrikeMap.cs

@@ -0,0 +1,18 @@
+#nullable enable
+namespace mbts.analytics;
+
+using bts.api;
+
+using mbts.spark;
+
+using System.Collections.Generic;
+
+public class VolSurfaceStrikeMap {
+   public readonly IDictionary<VolSurfaceExpiration, IEnumerable<double>>? strike_map = null;
+
+   public VolSurfaceStrikeMap(IEnumerable<(string, Venue)> option_names, SystemContractData sc_data) {
+      var on_map = new HashSet<(string, Venue)>(option_names);
+      var aux_map = sc_data.get_option_strikes(on_map);
+      strike_map = aux_map.ToDictionaryCrushed(kv => new VolSurfaceExpiration(kv.Key), kv => kv.Value);
+   }
+}

src/client/spark/Analytics/Core/VolSurfaceSymbolRelationship.cs

@@ -0,0 +1,24 @@
+using ProtoBuf;
+
+using System.Collections.Generic;
+using System.Linq;
+
+namespace mbts.analytics;
+
+[ProtoContract(Name = "VolSurfaceSymbolRelationship")]
+public class VolSurfaceSymbolRelationship {
+   [ProtoMember(10)]
+   public SymbolVenue symbol { get; set; }
+
+   [ProtoMember(20)]
+   public List<SymbolVenue> connected_symbols { get; set; } = new(1);
+
+   public static VolSurfaceSymbolRelationship create(SymbolVenue symbol, params SymbolVenue[] connected_symbols) =>
+      create(symbol, connected_symbols.AsEnumerable());
+
+   public static VolSurfaceSymbolRelationship create(SymbolVenue symbol, IEnumerable<SymbolVenue> connected_symbols) {
+      var rv = new VolSurfaceSymbolRelationship() { symbol = symbol };
+      rv.connected_symbols.AddRange(connected_symbols);
+      return rv;
+   }
+}

src/client/spark/Analytics/Core/VolSurfaceSymbolStructure.cs

@@ -1,26 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace mbts.analytics
-{
-   [ProtoBuf.ProtoContract(Name = "VolSurfaceSymbolStructure")]
-   public class VolSurfaceSymbolStructure
-   {
-      [ProtoBuf.ProtoMember(10)]
-      public string master_symbol;
-      [ProtoBuf.ProtoMember(20)]
-      public List<string> slave_symbol_list;
-   }
-
-   [ProtoBuf.ProtoContract(Name = "VolSurfaceSymbolRelationship")]
-   public class VolSurfaceSymbolRelationship
-   {
-      [ProtoBuf.ProtoMember(10)]
-      public string symbol;
-      [ProtoBuf.ProtoMember(20)]
-      public List<string> connected_symbols;
-   }
-}

src/client/spark/Analytics/Misc/VolSurfaceDataFunctions.cs

@@ -1,4 +1,8 @@
-using Spark.Core.Collections;
+using bts;
+using bts.api;
+using bts.collections;
+
+using Spark.Core.Collections;
 
 using System;
 using System.Collections.Generic;
@@ -26,6 +30,9 @@ namespace mbts.analytics
             out HashSet<VolSurfaceExpiration> deleted_exps,
             out HashSet<VolSurfaceExpiration> added_exps);
          if (deleted_exps.Any() || added_exps.Any()) {
+            Log.Gen.info(
+               $"VolSurfaceDataFunctions.sync_input_expirations_to_master_list exps changed. "
+               + $"deleted: {string.Join(", ", deleted_exps)}; added: {string.Join(", ", added_exps)}");
             VolSurfaceInput rv = VolSurfaceUtilityFunctions.clone_proto(vs_input);
             rv.surface_parameters.curve_parameters_map = rv.surface_parameters.curve_parameters_map ?? new List<Tuple<VolSurfaceExpiration, VolCurveParameters>>();
             rv.surface_parameters.curve_parameters_map.RemoveAll(t => deleted_exps.Contains(t.Item1));
@@ -53,12 +60,12 @@ namespace mbts.analytics
          };
       }
 
-      public static VolSurfaceInput get_default_vs_input(VolSurfaceSettings vs_settings, IDictionary<string, IEnumerable<DateTime>> exp_list)
+      public static VolSurfaceInput get_default_vs_input(VolSurfaceSettings vs_settings, IDictionary<(string, Venue), IEnumerable<DateTime>> exp_list)
       {
          List<VolSurfaceExpiration> vs_exp_list = new List<VolSurfaceExpiration>();
-         foreach (KeyValuePair<string, IEnumerable<DateTime>> kv in exp_list)
+         foreach (var kv in exp_list)
             foreach (DateTime exp in kv.Value)
-               vs_exp_list.Add(new VolSurfaceExpiration() { symbol = kv.Key, expiration = exp });
+               vs_exp_list.Add(new(SecurityType.Option, kv.Key.Item1, exp, kv.Key.Item2));
          return get_default_vs_input(vs_settings, vs_exp_list);
       }
 
@@ -220,9 +227,9 @@ namespace mbts.analytics
       public static VolSurfaceEnums.VolSurfacePricingModel get_pricing_model(VolSurfaceSettings settings, VolSurfaceExpiration exp)
       {
          if (settings.core_settings.symbol_pricing_model_map != null)
-            foreach (Tuple<string, VolSurfaceEnums.VolSurfacePricingModel> t in settings.core_settings.symbol_pricing_model_map)
-               if (t.Item1 == exp.symbol)
-                  return t.Item2;
+            foreach (var (sym, model) in settings.core_settings.symbol_pricing_model_map)
+               if (sym == (exp.symbol, exp.venue))
+                  return model;
          return VolSurfaceEnums.VolSurfacePricingModel.BlackScholes;
       }
 
@@ -231,13 +238,13 @@ namespace mbts.analytics
          return VolSurfaceUtilityFunctions.are_protos_equal(exp_params, VolSurfaceDataFunctions.get_default_curve_params(vs_settings));
       }
 
-      public static IEnumerable<string> get_symbols(VolSurfaceInput vs_input)
+      public static IEnumerable<(string, Venue)> get_symbols(VolSurfaceInput vs_input)
       {
-         return vs_input != null ? get_symbols(vs_input.surface_settings) : new string[] { };
+         return vs_input != null ? get_symbols(vs_input.surface_settings) : Enumerable.Empty<(string, Venue)>();
       }
 
-      public static IEnumerable<string> get_symbols(VolSurfaceSettings vs_settings) =>
-         vs_settings.core_settings.symbol_pricing_model_map.empty_if_null().Select(i => i.Item1);
+      public static IEnumerable<(string, Venue)> get_symbols(VolSurfaceSettings vs_settings) =>
+         vs_settings.core_settings.symbol_pricing_model_map.empty_if_null().Select(i => (i.Key.symbol, i.Key.venue));
 
       public static VolCurveParameters get_curve_parameters(VolSurfaceInput vs_input, VolSurfaceExpiration exp)
       {
@@ -477,7 +484,7 @@ namespace mbts.analytics
       private static void _force_distinct_connected_symbols(VolSurfaceSymbolRelationship vssr) {
          var distinct_syms = vssr.connected_symbols.Distinct().ToList();
          if (distinct_syms.Count != vssr.connected_symbols.Count)
-            vssr.connected_symbols = distinct_syms;
+            vssr.connected_symbols.SetItems(distinct_syms);
       }
 
       #endregion
@@ -702,7 +709,7 @@ namespace mbts.analytics
             var this_exp = kv.Item1;
             var this_exp_params = kv.Item2;
             if (exp_has_input(vs_input.surface_settings, this_exp_params)) {
-               double this_exp_diff_days = Math.Abs(vs_exp.diff_days(this_exp));
+               double this_exp_diff_days = Math.Abs((vs_exp.expiration - this_exp.expiration).TotalDays);
                if (this_exp_diff_days < rv_diff_days) {
                   rv = this_exp;
                   rv_diff_days = this_exp_diff_days;

src/client/spark/Analytics/Misc/VolSurfaceUtilityFunctions.cs

@@ -373,17 +373,17 @@ namespace mbts.analytics {
       /// <summary>
       /// Puts VolSurfaceSymbolRelationship's into a map of symbol vs its connected symbols
       /// </summary>
-      static IDictionary<string, HashSet<string>> _hash_relationship(
+      static IDictionary<(string, Venue), HashSet<(string, Venue)>> _hash_relationship(
          VolSurfaceSettings vs_settings, IEnumerable<VolSurfaceSymbolRelationship> rels) {
          var rv = rels.empty_if_null().ToDictionaryCrushed(
-            r => r.symbol, r => r.connected_symbols.ToHashSet());
+            r => (r.symbol.symbol, r.symbol.venue), r => r.connected_symbols.Select(s => (s.symbol, s.venue)).ToHashSet());
          foreach (var s in VolSurfaceDataFunctions.get_symbols(vs_settings)) {
-            rv.GetOrAdd(s, _ => new HashSet<string>()).Add(s);
+            rv.GetOrAdd(s, _ => new()).Add(s);
          }
          return rv;
       }
 
-      static HashSet<string> _get_slave_symbols(VolSurfaceSettings vs_settings) {
+      static HashSet<(string, Venue)> _get_slave_symbols(VolSurfaceSettings vs_settings) {
          /// just all - masters
          var slave_symbols = VolSurfaceDataFunctions.get_symbols(vs_settings).ToHashSet();
          var master_symbols = get_master_symbols(vs_settings);
@@ -391,12 +391,12 @@ namespace mbts.analytics {
          return slave_symbols;
       }
 
-      static HashSet<string> _get_masters_for_symbol(VolSurfaceSettings vs_settings, string symbol) =>
+      static HashSet<(string, Venue)> _get_masters_for_symbol(VolSurfaceSettings vs_settings, string symbol, Venue venue) =>
          _hash_relationship(
             vs_settings, vs_settings.core_settings.master_slave_relationships)
-            .GetValueOrDefault(symbol) ?? new();
+            .GetValueOrDefault((symbol, venue)) ?? new();
 
-      public static HashSet<string> get_master_symbols(VolSurfaceSettings vs_settings) {
+      public static HashSet<(string, Venue)> get_master_symbols(VolSurfaceSettings vs_settings) {
          var master_slaves = _hash_relationship(
             vs_settings, vs_settings.core_settings.master_slave_relationships ?? new());
          return master_slaves.Values.SelectMany().ToHashSet();
@@ -411,7 +411,7 @@ namespace mbts.analytics {
          var master_symbols = get_master_symbols(vs_inp.surface_settings);
          return vs_inp.surface_parameters.curve_parameters_map
             .Where(t =>
-               master_symbols.Contains(t.Item1.symbol) ||
+               master_symbols.Contains((t.Item1.symbol, t.Item1.venue)) ||
                (include_unlinked_slaves && t.Item2.core_parameters.unhinge_exp_from_master))
             .Select(t => t.Item1)
             .ToHashSet();
@@ -424,25 +424,25 @@ namespace mbts.analytics {
          var slave_symbols = _get_slave_symbols(vs_inp.surface_settings);
          return vs_inp.surface_parameters.curve_parameters_map
             .Select(t => t.Item1)
-            .Where(t => slave_symbols.Contains(t.symbol))
+            .Where(t => slave_symbols.Contains((t.symbol, t.venue)))
             .ToHashSet();
       }
 
       public static bool is_slave_expiration(
          VolSurfaceExpiration vs_exp, VolSurfaceSettings vs_settings) =>
-         _get_slave_symbols(vs_settings).Contains(vs_exp.symbol);
+         _get_slave_symbols(vs_settings).Contains((vs_exp.symbol, vs_exp.venue));
 
       public static bool is_master_expiration(
          VolSurfaceExpiration vs_exp, VolSurfaceSettings vs_settings) =>
-         get_master_symbols(vs_settings).Contains(vs_exp.symbol);
+         get_master_symbols(vs_settings).Contains((vs_exp.symbol, vs_exp.venue));
 
       public static VolSurfaceExpiration find_master_expiration(
          VolSurfaceExpiration slave_vs_exp,
          VolSurfaceSettings vs_settings,
          IEnumerable<VolSurfaceExpiration> vs_exps
       ) {
-         var master_symbols = _get_masters_for_symbol(vs_settings, slave_vs_exp.symbol);
-         vs_exps = vs_exps.Where(e => master_symbols.Contains(e.symbol));
+         var master_symbols = _get_masters_for_symbol(vs_settings, slave_vs_exp.symbol, slave_vs_exp.venue);
+         vs_exps = vs_exps.Where(e => master_symbols.Contains((e.symbol, e.venue)));
          VolSurfaceExpiration rv = find_nearest(vs_exps, slave_vs_exp, exp_date_diff) ??
             new() { expiration = DateTime.MinValue };
          /// TODO get this 2 from settings
@@ -457,10 +457,10 @@ namespace mbts.analytics {
          var md_rel_map = _hash_relationship(
             vs_input.surface_settings,
             vs_input.surface_settings.core_settings.implied_vol_marketdata_relationships);
-         var source_symbols = md_rel_map.GetValueOrDefault(vs_exp.symbol) ?? new();
+         var source_symbols = md_rel_map.GetValueOrDefault((vs_exp.symbol, vs_exp.venue)) ?? new();
          var source_exps = vs_input.surface_parameters.curve_parameters_map
             .Select(t => t.Item1)
-            .Where(e => source_symbols.Contains(e.symbol));
+            .Where(e => source_symbols.Contains((e.symbol, e.venue)));
          var rv = find_nearest(source_exps, vs_exp, exp_date_diff) ??
             new() { expiration = DateTime.MinValue };
          /// TODO get this from settings
@@ -546,20 +546,6 @@ namespace mbts.analytics {
          return polynomial_str;
       }
 
-      /// <summary>
-      /// Transforms <paramref name="implied_vol_md_source"/> to a map of symbols -> md source
-      /// </summary>
-      /// <returns>
-      /// An <see cref="IDictionary{string, string}"/> whose keys are slave symbols and whose
-      /// values are the corresponding master symbols
-      /// </returns>
-      public static IDictionary<string, string> flatten_implied_vol_md_source(
-         IEnumerable<VolSurfaceSymbolStructure> implied_vol_md_source) =>
-         implied_vol_md_source.empty_if_null()
-            .SelectMany(s => s.slave_symbol_list
-               .Select(i => (i, s.master_symbol)))
-            .ToDictionary();
-
       public static double restrict_bw_min_and_max(
          double v, double min, double max) => v.max(min).min(max);
 
@@ -807,7 +793,7 @@ namespace mbts.analytics {
          IDictionary<VolSurfaceExpiration, IEnumerable<double>> exp_strike_map) =>
          exp_strike_map
             .SelectMany(kv => {
-               var sec_exp = kv.Key.to_security_expiration(type: SecurityType.Option);
+               var sec_exp = kv.Key.with(security_type: SecurityType.Option).to_security_expiration();
                return kv.Value.Select(k => new SecurityStrike(sec_exp, k));
             })
             .ToHashSet();

src/client/spark/Analytics/Pricing/VolSurfaceInterpGreeksProducer.cs

@@ -154,11 +154,8 @@ namespace mbts.analytics {
       private IDictionary<SecurityExpiration, AnalyticsExpirationParameters> _sync_exp_params() {
          var master_exp_keys =
             _exp_strikes_map.Keys.Select(k =>
-               new SecurityExpiration(Venue.Unset, SecurityType.Option, k.symbol, k.expiration));
-         // TODO: add exchange to key (so we don't have to 'Crush' the duplicate keys)
-         var exp_params_no_exch = _exp_params_map.ToDictionaryCrushed(
-            kv => kv.Key.unset_exchange(), kv => kv.Value);
-         return exp_params_no_exch.sync(master_exp_keys, true,
+               new SecurityExpiration(k.venue, SecurityType.Option, k.symbol, k.expiration));
+         return _exp_params_map.sync(master_exp_keys, true,
             exp => AnalyticsExpirationParameters.empty);
       }
 

src/client/spark/Analytics/SupportingData/VolSurfaceNonVolInput.cs

@@ -4,6 +4,8 @@ using System.Linq;
 
 using bts.api;
 
+using mbts.spark;
+
 namespace mbts.analytics {
    /// This is the input data (other than vol) that
    /// is used for vol/price calculations. It is not
@@ -30,7 +32,7 @@ namespace mbts.analytics {
       public IDictionary<VolSurfaceExpiration, AnalyticsExpirationParameters> get_option_exp_input() {
          var list = exp_input_map.Where(kv => kv.Key.security_type == SecurityType.Option);
          return list.ToDictionary(
-            kv => new VolSurfaceExpiration() { symbol = kv.Key.symbol, expiration = kv.Key.expiration },
+            kv => kv.Key.to_vs_exp(),
             kv => kv.Value);
       }
 

src/client/spark/Analytics/Tests/MarketFitFunctionsTests.cs

@@ -5,6 +5,7 @@ using System.Text;
 using System.Threading.Tasks;
 using System.IO;
 using NUnit.Framework;
+using bts.api;
 
 namespace mbts.analytics {
    [TestFixture]
@@ -17,14 +18,15 @@ namespace mbts.analytics {
             minimum_top_book_size = 1,
             graph_update_period_ms = 1000
          };
-         IDictionary<VolSurfaceExpiration, IDictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>>> bbo = _parse_bbo("spx", @"P:\Dropbox\Data\VolManager\TestData\processed_bbo_spx.csv");
+         var venue = Venue.Cboe;
+         IDictionary<VolSurfaceExpiration, IDictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>>> bbo = _parse_bbo("spx", venue, @"P:\Dropbox\Data\VolManager\TestData\processed_bbo_spx.csv");
          DateTime exp = DateTime.Parse("12/19/2015");
          DateTime theo_time = new DateTime(2015, 10, 13, 9, 0, 0);
          double cal_t = VolSurfaceUtilityFunctions.to_years(exp.Subtract(theo_time)),
                 biz_t = cal_t * 0.95,
                 rate = 0.3,
                 fwd = 2009.25;
-         IDictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>> k_map = bbo[new VolSurfaceExpiration() { expiration = exp, symbol = "spx" }];
+         IDictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>> k_map = bbo[new() { expiration = exp, symbol = "spx", venue = venue }];
          IDictionary<double, double> put_mid_prices, call_mid_prices;
          VolSurfaceCoreFunctions.get_filtered_bbo_mid(fit_settings, fwd, k_map, VolSurfaceEnums.MarketSide.Mid, out put_mid_prices, out call_mid_prices);
          int n_iters = 10;
@@ -51,7 +53,7 @@ namespace mbts.analytics {
          bts.Log.Testing.info(String.Format("MarketFitFunctionsTests.test_implied_vols puts = {0}, calls = {1}, runs = {2}, total time = {3}", put_mid_prices.Count, call_mid_prices.Count, n_iters, stop_watch.ElapsedMilliseconds));
       }
 
-      private static IDictionary<VolSurfaceExpiration, IDictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>>> _parse_bbo(string sym, string file_name) {
+      private static IDictionary<VolSurfaceExpiration, IDictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>>> _parse_bbo(string sym, Venue venue, string file_name) {
          IDictionary<VolSurfaceExpiration, IDictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>>> t_rv = new Dictionary<VolSurfaceExpiration, IDictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>>>();
          StreamReader sr = new StreamReader(file_name);
          while (!sr.EndOfStream) {
@@ -65,7 +67,7 @@ namespace mbts.analytics {
             bool is_call = toks[c++].ToLower().Contains("call");
             DateTime exp = DateTime.Parse(toks[c++]);
             double strike = Double.Parse(toks[c++]);
-            VolSurfaceExpiration vs_exp = new VolSurfaceExpiration() { symbol = sym, expiration = exp };
+            var vs_exp = VolSurfaceExpiration.Default.with(sym, exp, SecurityType.Option, venue);
             VolSurfaceMDUnit vs_bbo_unit = new VolSurfaceMDUnit(bid_price, bid_size, ask_price, ask_size, Double.NaN);
             IDictionary<double, Tuple<VolSurfaceMDUnit, VolSurfaceMDUnit>> k_map;
             if (!t_rv.TryGetValue(vs_exp, out k_map))

src/client/spark/Analytics/Tests/VolSurfaceModelTests.cs

@@ -4,13 +4,18 @@ using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
 
+using bts.api;
+using bts.collections;
+
+using mbts.spark;
+
 using NUnit.Framework;
 
 namespace mbts.analytics {
    [TestFixture]
    public class VolSurfaceModelTests {
       public static VolSurfaceInput create_sample_vs_input() {
-         VolSurfaceExpiration vs_exp = new VolSurfaceExpiration() { symbol = "spx", expiration = new DateTime(2016, 1, 15) };
+         var vs_exp = new VolSurfaceExpiration() { symbol = "spx", expiration = new DateTime(2026, 1, 15), venue = Venue.Cboe };
          IEnumerable<double> exp_strikes = _get_strikes(1000, 3000, 5);
          VolSurfaceExpirationAuxiliaryData exp_aux_data = _get_aux_data();
          VolPathParameters exp_volpath_parameters = _get_two_way_slope_volpath_parameters(0.1 / 100.0, -0.1 / 100.0);
@@ -47,8 +52,6 @@ namespace mbts.analytics {
                   put_rotation = _get_movement_operation(VolSurfaceEnums.VolCurveStrikeAxis.StandardStrike, VolSurfaceEnums.VolMovementTerms.Vol, Tuple.Create(-1.0, 1.0 / 100)),
                   full_rotation = _get_movement_operation(VolSurfaceEnums.VolCurveStrikeAxis.StandardStrike, VolSurfaceEnums.VolMovementTerms.Vol, Tuple.Create(-0.75, 0.5 / 100), Tuple.Create(0.75, -0.5 / 100)),
                   strangle = _get_movement_operation(VolSurfaceEnums.VolCurveStrikeAxis.StandardStrike, VolSurfaceEnums.VolMovementTerms.Vol, Tuple.Create(-0.75, 0.5 / 100), Tuple.Create(0.75, 0.5 / 100)),
-                  symbol_pricing_model_map = Enumerable.Repeat(Tuple.Create("spx", VolSurfaceEnums.VolSurfacePricingModel.BlackScholes), 1).ToList(),
-                  master_slave_relationships = Enumerable.Repeat(new VolSurfaceSymbolRelationship() { symbol = "spx", connected_symbols = new string[] { "spx" }.ToList() }, 1).ToList(),
                   vol_surface_model = VolSurfaceEnums.VolSurfaceModel.Polynomial,
                   volpath_settings = _get_volpath_settings(VolSurfaceEnums.VolSurfaceVolPathModel.TwoWaySlope),
                   wing_settings = _get_wing_settings(VolSurfaceEnums.VolCurveWingModel.ExplicitSlopeWithHorizontalAsymptotes, VolSurfaceEnums.VolCurveStrikeAxis.StandardStrike),
@@ -71,6 +74,10 @@ namespace mbts.analytics {
                }.ToList()
             }
          };
+         vs_input.surface_settings.core_settings.symbol_pricing_model_map.Add(
+            ("spx", Venue.Cboe), VolSurfaceEnums.VolSurfacePricingModel.BlackScholes);
+         vs_input.surface_settings.core_settings.master_slave_relationships.Add(
+            VolSurfaceSymbolRelationship.create(("spx", Venue.Cboe), ("spx", Venue.Cboe)));
          return vs_input;
       }
       /// The pattern for these tests that test the different
@@ -94,7 +101,7 @@ namespace mbts.analytics {
       /// 11.Test for Strangle
       [Test]
       public static void polynomial_standardstrike_add() {
-         VolSurfaceExpiration vs_exp = new VolSurfaceExpiration() { symbol = "spx", expiration = new DateTime(2016, 1, 15) };
+         var vs_exp = VolSurfaceExpiration.Default.with(symbol: "spx", expiration: new DateTime(2026, 1, 15), venue: Venue.Cboe);
          IEnumerable<double> exp_strikes = _get_strikes(1000, 3000, 5);
          VolSurfaceExpirationAuxiliaryData exp_aux_data = _get_aux_data();
          VolPathParameters exp_volpath_parameters = _get_two_way_slope_volpath_parameters(0.1 / 100.0, -0.1 / 100.0);
@@ -131,8 +138,6 @@ namespace mbts.analytics {
                   put_rotation = _get_movement_operation(VolSurfaceEnums.VolCurveStrikeAxis.StandardStrike, VolSurfaceEnums.VolMovementTerms.Vol, Tuple.Create(-1.0, 1.0 / 100)),
                   full_rotation = _get_movement_operation(VolSurfaceEnums.VolCurveStrikeAxis.StandardStrike, VolSurfaceEnums.VolMovementTerms.Vol, Tuple.Create(-0.75, 0.5 / 100), Tuple.Create(0.75, -0.5 / 100)),
                   strangle = _get_movement_operation(VolSurfaceEnums.VolCurveStrikeAxis.StandardStrike, VolSurfaceEnums.VolMovementTerms.Vol, Tuple.Create(-0.75, 0.5 / 100), Tuple.Create(0.75, 0.5 / 100)),
-                  symbol_pricing_model_map = Enumerable.Repeat(Tuple.Create("spx", VolSurfaceEnums.VolSurfacePricingModel.BlackScholes), 1).ToList(),
-                  master_slave_relationships = Enumerable.Repeat(new VolSurfaceSymbolRelationship() { symbol = "spx", connected_symbols = new string[] { "spx" }.ToList() }, 1).ToList(),
                   vol_surface_model = VolSurfaceEnums.VolSurfaceModel.Polynomial,
                   volpath_settings = _get_volpath_settings(VolSurfaceEnums.VolSurfaceVolPathModel.TwoWaySlope),
                   wing_settings = _get_wing_settings(VolSurfaceEnums.VolCurveWingModel.ExplicitSlopeWithHorizontalAsymptotes, VolSurfaceEnums.VolCurveStrikeAxis.StandardStrike),
@@ -155,6 +160,10 @@ namespace mbts.analytics {
                }.ToList()
             }
          };
+         vs_input.surface_settings.core_settings.symbol_pricing_model_map.Add(
+            ("spx", Venue.Cboe), VolSurfaceEnums.VolSurfacePricingModel.BlackScholes);
+         vs_input.surface_settings.core_settings.master_slave_relationships.Add(
+            VolSurfaceSymbolRelationship.create(("spx", Venue.Cboe), ("spx", Venue.Cboe)));
          _test_meat_points(vs_exp, vs_input, exp_aux_data, exp_strikes);
          _test_atm_vol_shift(vs_exp, vs_input, exp_aux_data, exp_strikes,
             new VolCurveMovementOperation() { atm_strike_vol_movement = 1.0, target_terms = VolSurfaceEnums.VolMovementTerms.Vol });
@@ -198,8 +207,7 @@ namespace mbts.analytics {
 
       /// Mostly for perf purposes, the data is not very consistent
       internal static VolSurfaceInput get_generic_vs_input(IEnumerable<VolSurfaceExpiration> vs_exps) {
-         HashSet<string> syms = new HashSet<string>();
-         vs_exps.ToList().ForEach(e => syms.Add(e.symbol));
+         var syms = vs_exps.Select(e => (e.symbol, e.venue)).ToHashSet();
          VolSurfaceExpirationAuxiliaryData exp_aux_data = _get_aux_data();
          VolPathParameters exp_volpath_parameters = _get_two_way_slope_volpath_parameters(0.1 / 100.0, -0.1 / 100.0);
          VolCurveCoreParameters exp_core_parameters = new VolCurveCoreParameters() {
@@ -235,13 +243,6 @@ namespace mbts.analytics {
                   put_rotation = _get_movement_operation(VolSurfaceEnums.VolCurveStrikeAxis.StandardStrike, VolSurfaceEnums.VolMovementTerms.Vol, Tuple.Create(-1.0, 1.0 / 100)),
                   full_rotation = _get_movement_operation(VolSurfaceEnums.VolCurveStrikeAxis.StandardStrike, VolSurfaceEnums.VolMovementTerms.Vol, Tuple.Create(-0.75, 0.5 / 100), Tuple.Create(0.75, -0.5 / 100)),
                   strangle = _get_movement_operation(VolSurfaceEnums.VolCurveStrikeAxis.StandardStrike, VolSurfaceEnums.VolMovementTerms.Vol, Tuple.Create(-0.75, 0.5 / 100), Tuple.Create(0.75, 0.5 / 100)),
-                  symbol_pricing_model_map = syms.Select(s => {
-                     VolSurfaceEnums.VolSurfacePricingModel model = VolSurfaceEnums.VolSurfacePricingModel.BlackScholes;
-                     if (s == "es" || s == "sp")
-                        model = VolSurfaceEnums.VolSurfacePricingModel.Whaley;
-                     return Tuple.Create(s, model);
-                  }).ToList(),
-                  master_slave_relationships = syms.Select(s => new VolSurfaceSymbolRelationship() { symbol = s, connected_symbols = Enumerable.Repeat<string>(s, 1).ToList() }).ToList(),
                   vol_surface_model = VolSurfaceEnums.VolSurfaceModel.Polynomial,
                   volpath_settings = _get_volpath_settings(VolSurfaceEnums.VolSurfaceVolPathModel.TwoWaySlope),
                   wing_settings = _get_wing_settings(VolSurfaceEnums.VolCurveWingModel.ExplicitSlopeWithHorizontalAsymptotes, VolSurfaceEnums.VolCurveStrikeAxis.StandardStrike)
@@ -256,6 +257,14 @@ namespace mbts.analytics {
                })).ToList()
             }
          };
+         vs_input.surface_settings.core_settings.symbol_pricing_model_map.AddRange(
+            syms.Select(s => new KeyValuePair<SymbolVenue, VolSurfaceEnums.VolSurfacePricingModel>(
+               s, s.symbol switch {
+                  "es" or "sp" => VolSurfaceEnums.VolSurfacePricingModel.Whaley,
+                  _ => VolSurfaceEnums.VolSurfacePricingModel.BlackScholes
+               })));
+         vs_input.surface_settings.core_settings.master_slave_relationships.AddRange(
+            syms.Select(s => VolSurfaceSymbolRelationship.create(s, s)));
          return vs_input;
       }
 

src/client/spark/Daemons/FakeMDDaemon.cs

@@ -7,7 +7,8 @@ using mbts.ui;
 using System;
 using System.Collections.Generic;
 using System.Composition;
-using System.Reactive.Linq;
+using System.Threading.Tasks;
+using System.Threading;
 
 using SecurityKey = System.UInt32;
 using MDHandle = mbts.co2.FakeMDConsumerHandle;
@@ -18,10 +19,7 @@ namespace bts {
    // publishes a fixed set of hard coded securities to the bus, remove the line
    // above and this class wont be created
    internal class FakeMDDaemon : IShellDaemon, IDisposable {
-      private readonly IBus _bus;
       private IDisposable _dispose;
-      private bool _published = false;
-      private IReadOnlyDictionary<uint, SecurityMD> _sec_map = _get_fake_secs();
 
       // Returns a bunch of hard coded SecurityMD
       private static Dictionary<uint, SecurityMD> _get_fake_secs() {
@@ -72,19 +70,14 @@ namespace bts {
       [ImportingConstructor]
       public FakeMDDaemon(IBus bus) {
          bts.Log.Gen.info("FakeMDDaemon started");
-         _bus = bus;
-         var observable = _bus.subscribe(SparkTags.SparkTimer).OnDispatcher().Subscribe(
-            d => _on_timer(d));
-      }
+         var cts = new CancellationTokenSource();
+         Task.Delay(TimeSpan.FromSeconds(5), cts.Token)
+            .ContinueWith(postFakeSecs, TaskContinuationOptions.OnlyOnRanToCompletion);
+         // 'Dispose' cancels the task to post securities
+         _dispose = Disposable.Create(cts.Cancel);
 
-      private void _on_timer(TimeSpan ts_from_start) {
-         if (!_published) {
-            if (ts_from_start.TotalSeconds >= 5) {
-               _published = true;
-               bts.Log.Gen.info("FakeMDDaemon publishing md");
-               _bus.post(SparkTags.FakeSecuritiesTag, _sec_map);
-            }
-         }
+         void postFakeSecs(Task delayTask) =>
+            bus.post(SparkTags.FakeSecuritiesTag, _get_fake_secs());
       }
 
       public void Dispose() =>

src/client/spark/Daemons/FakeSlideDataDaemon.cs

@@ -37,8 +37,23 @@ namespace bts.daemons {
                   s => get_fake_trade(to_sec_exp(s.security), 4200)).ToList();
                bus.post(SparkTags.TradeLogTag, fake_trades);
                var bbo_upd_refs = btc_secs.ToDictionary(s => s.security.Id, _ => 0UL);
-               var driver_sym_prices = btc_secs.ToImmutableDictionary(
-                  s => to_sec_exp(s.security), get_driver_price);
+               var driver_sym_prices_bd = ImmutableDictionary.CreateBuilder<SecurityExpiration, double>();
+               var sec_exps_index = new Dictionary<SecurityExpiration, List<SecurityImmutable>>();
+               foreach (var btc_sec in btc_secs) {
+                  var sec_exp = to_sec_exp(btc_sec.security);
+                  var existing_secs = sec_exps_index.GetOrAdd(sec_exp, _ => new(1));
+                  existing_secs.Add(btc_sec.security);
+                  if (driver_sym_prices_bd.ContainsKey(sec_exp)) {
+                     continue;
+                  }
+                  driver_sym_prices_bd.Add(sec_exp, get_driver_price(btc_sec));
+               }
+               foreach (var kv in sec_exps_index) {
+                  if (kv.Value.Count > 1) {
+                     Log.Md.warn($"Duplicate BTC driver secs detected in fake md: {string.Join(", ", kv.Value)}");
+                  }
+               }
+               var driver_sym_prices = driver_sym_prices_bd.ToImmutable();
 
                double get_driver_price(SecurityMD smd) {
                   var upd_ref = bbo_upd_refs[smd.security.Id];
@@ -64,7 +79,7 @@ namespace bts.daemons {
             }));
 
          static SecurityExpiration to_sec_exp(SecurityImmutable sec) =>
-            new(sec.Exchange, sec.ContractCase, sec.Symbol, sec.security_expiration() ?? DateTime.MinValue, sec.Id);
+            new(sec.Exchange, sec.ContractCase, sec.Symbol, sec.security_expiration() ?? DateTime.MinValue);
 
          ITradeLogRow get_fake_trade(
             SecurityExpiration sec_exp,

src/client/spark/Daemons/ThreadedBlackboardDaemon.cs

@@ -0,0 +1,40 @@
+#nullable enable
+namespace Spark.Daemons;
+
+using bts.api;
+
+using mbts.spark;
+using mbts.ui;
+
+using System;
+using System.Composition;
+using System.Threading;
+
+[Export(typeof(IShellDaemon))]
+class ThreadedBlackboardDaemon : IShellDaemon, IDisposable {
+   readonly Thread thread;
+   bool isRunning = true;
+
+   [ImportingConstructor]
+   public ThreadedBlackboardDaemon(IBus bus) {
+      thread = new(Start) {
+         IsBackground = true
+      };
+      thread.Start();
+
+      void Start() {
+         var interval = TimeSpan.FromMilliseconds(100);
+         DateTime start_time = DateTime.Now;
+         Thread.Sleep(interval);
+         while (isRunning) {
+            bus.post(SparkTags.SparkTimer, DateTime.Now - start_time);
+            Thread.Sleep(interval);
+         }
+      }
+   }
+
+   public void Dispose() {
+      isRunning = false;
+      thread.Join();
+   }
+}

src/client/spark/Modules/Analytics/VolSurface/View/VolSurfaceDriverGridTabControl.xaml

@@ -22,7 +22,7 @@
       BorderThickness="0"
       AutoGenerateColumns="False"
       CanUserResizeColumns="False"
-      ItemsSource="{Binding symbols}"
+      ItemsSource="{Binding slaves}"
       RowBackground="Transparent">
       <DataGrid.Resources>
         <Style TargetType="{x:Type DataGridCell}">
@@ -59,17 +59,21 @@
             <DataTemplate>
               <c:ComboCheckBox
                 Margin="4"
-                ItemsSource="{Binding driver.symbols}"
-                SelectedItems="{Binding masters}">
+                ItemsSource="{Binding master_symbol_items}">
                 <c:ComboCheckBox.HeaderContent>
                   <MultiBinding>
                     <MultiBinding.Converter>
                       <cv:SelectedItemsHeaderTextConverter ShowFraction="False" />
                     </MultiBinding.Converter>
-                    <Binding Path="mastersi" />
-                    <Binding Path="driver.symbols.Count" />
+                    <Binding Path="selected_master_symbols" />
+                    <Binding Path="master_symbol_items.Count" />
                   </MultiBinding>
                 </c:ComboCheckBox.HeaderContent>
+                <c:ComboCheckBox.ItemContainerStyle>
+                  <Style TargetType="c:ComboCheckBoxItem" BasedOn="{StaticResource {x:Type c:ComboCheckBoxItem}}">
+                    <Setter Property="IsSelected" Value="{Binding is_selected, Mode=TwoWay}" />
+                  </Style>
+                </c:ComboCheckBox.ItemContainerStyle>
               </c:ComboCheckBox>
             </DataTemplate>
           </DataGridTemplateColumn.CellTemplate>
@@ -79,16 +83,22 @@
           <DataGridTemplateColumn.CellTemplate>
             <DataTemplate>
               <c:ComboCheckBox
-                Margin="4" ItemsSource="{Binding driver.symbols}" SelectedItems="{Binding fits}">
+                Margin="4"
+                ItemsSource="{Binding fit_items}">
                 <c:ComboCheckBox.HeaderContent>
                   <MultiBinding>
                     <MultiBinding.Converter>
                       <cv:SelectedItemsHeaderTextConverter ShowFraction="False" />
                     </MultiBinding.Converter>
-                    <Binding Path="fitsi" />
-                    <Binding Path="driver.symbols.Count" />
+                    <Binding Path="selected_fits" />
+                    <Binding Path="fit_items.Count" />
                   </MultiBinding>
                 </c:ComboCheckBox.HeaderContent>
+                <c:ComboCheckBox.ItemContainerStyle>
+                  <Style TargetType="c:ComboCheckBoxItem" BasedOn="{StaticResource {x:Type c:ComboCheckBoxItem}}">
+                    <Setter Property="IsSelected" Value="{Binding is_selected, Mode=TwoWay}" />
+                  </Style>
+                </c:ComboCheckBox.ItemContainerStyle>
               </c:ComboCheckBox>
             </DataTemplate>
           </DataGridTemplateColumn.CellTemplate>

src/client/spark/Modules/Market/MarketView/MarketViewColumnType.cs

@@ -36,6 +36,7 @@ namespace mbts.spark {
       Gamma,
       Vega,
       Theta,
-      Rho
+      Rho,
+      Currency
    }
 }

src/client/spark/Modules/Market/MarketView/MarketViewRow.cs

@@ -207,6 +207,7 @@
          public SecurityMD security_md { get; }
          public SecurityImmutable security => security_md.security;
          public uint security_key => security_md.security.Id;
+         public string currency => security_md.security.Currency;
 
          public bool inside_bids => _inside_bids.value;
          public bool inside_asks => _inside_asks.value;

src/client/spark/Modules/Market/MarketView/MarketViewSettings.cs

@@ -93,6 +93,7 @@ namespace bts.shell.market_view {
 
       static class DefaultColumns {
          public static GridColumnSettings security => _mk(Columns.Security, ColHeader.Security, ColWidth.Security, h_align: HorizontalAlignment.Left);
+         public static GridColumnSettings currency => _mk(Columns.Currency, ColHeader.Currency, ColWidth.Currency, h_align: HorizontalAlignment.Left, is_visible: false);
          public static GridColumnSettings volume => _mk(Columns.Volume, ColHeader.Volume, ColWidth.Volume, h_align: HorizontalAlignment.Left);
          public static GridColumnSettings last => _mk(Columns.Last, ColHeader.Last, ColWidth.Price, h_align: HorizontalAlignment.Left);
          public static GridColumnSettings last_from_spread => _mk(Columns.LastSpd, "S", ColWidth.Price, is_visible: false, h_align: HorizontalAlignment.Left);

src/client/spark/Modules/Market/MarketView/MarketViewVM.cs

@@ -98,6 +98,7 @@ namespace mbts.spark {
          _sec_meta_store = sec_meta_store;
          _settings_hub = settings_hub;
          security_filter = new RegexFilterVM<Row>(r => r.security.ToString(), nameof(Row.security));
+         currency_filter =new RegexFilterVM<Row>(r => r.currency, nameof(Row.currency));
          // rows
          _wrapped_rows = _rows.WrapSelector(r => r.security_md, s => {
             var r = new Row(_bus, s, _orders_model, _position_model);
@@ -122,6 +123,7 @@ namespace mbts.spark {
          var price_converter = new TickDisplayConverter();
 
          add_col(ColKey.Security, r => r.security.ToString(), nameof(Row.security), security_filter, FilterType.Regex, ColWidth.Security);
+         add_col(ColKey.Currency, r => r.currency, nameof(Row.currency), currency_filter, FilterType.Regex, ColWidth.Currency);
          add_col(ColKey.Volume, r => r.volume_text, nameof(Row.volume), volume_filter, FilterType.ULongMinMax, ColWidth.Volume);
 
          // last
@@ -387,6 +389,7 @@ namespace mbts.spark {
       internal OrdersModel orders_model => _orders_model;
 
       public IDataGridColumnPersistentFilter<Row, string> security_filter { get; }
+      public IDataGridColumnPersistentFilter<Row, string> currency_filter { get; }
       public IDataGridColumnPersistentFilter<Row, DoubleMinMaxState> bid_price_filter { get; } = new DoubleMinMax<Row>(r => r.bid_price, nameof(Row.bid_price));
       public IDataGridColumnPersistentFilter<Row, DoubleMinMaxState> bid_qty_filter { get; } = new DoubleMinMax<Row>(r => r.bid_qty_display, nameof(Row.bid_qty_display));
       public IDataGridColumnPersistentFilter<Row, DoubleMinMaxState> dbid_qty_filter { get; } = new DoubleMinMax<Row>(r => r.dbid_qty_display, nameof(Row.dbid_qty_display));

src/client/spark/Modules/Trading/ManualFills/ManualFillRow.cs

@@ -1,4 +1,4 @@
-using bts.api;
+using bts.api;
 using bts.api.position;
 
 using mbts.spark;
@@ -112,8 +112,10 @@ namespace bts.shell.manual_fills {
       public TickDisplaySettingsImmutable quantity_display => _quantity_display.Value;
       public double price { get => _price; set => _set(ref _price, value); }
 
-      public DateTime time_entered =>
-         data.value.manual_fill.Timestamp.to_local_time();
+      public DateTime? time_entered =>
+         data.value.manual_fill.HasTimestamp
+            ? data.value.manual_fill.Timestamp.to_local_time()
+            : null;
 
       // is selected
       public bool is_selected {

src/client/spark/NMSheets/BlackboardStructure/NMSheetData.cs

@@ -53,14 +53,6 @@ namespace BCG
          _cached_atm_data = _assemble_atm_data_from_symbols(_sym_data_map);
       }
 
-      public static NMSheetData filter_data(NMSheetData data, HashSet<uint> restricted_seckey_map)
-      {
-         Dictionary<NMSheetSymbol, NMSheetSingleOptionData> main_data_map = new Dictionary<NMSheetSymbol, NMSheetSingleOptionData>();
-         foreach (KeyValuePair<NMSheetSymbol, NMSheetSingleOptionData> p in data._sym_data_map)
-            main_data_map.Add(p.Key, NMSheetSingleOptionData.filter_data(p.Value, restricted_seckey_map));
-         return new NMSheetData(main_data_map, data._bbo_map, data._complete_ticker_data, data._task_data);
-      }
-
       public NMSheetData filter_for_strikes_in_tasks(bool is_on_only = false)
       {
          // if (_task_data != null)

src/client/spark/NMSheets/BlackboardStructure/NMSheetDriverData.cs

@@ -1,4 +1,6 @@
-using System;
+using bts.api;
+
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
@@ -9,14 +11,16 @@ namespace BCG
    public class NMSheetDriverData
    {
       public readonly string driver_symbol;
+      public readonly Venue driver_venue;
       public readonly DateTime driver_expiration;
       public readonly double driver_live_price;
       public readonly double driver_price_at_computed_theo;
       public readonly bool is_frozen;
 
-      public NMSheetDriverData(string the_driver_symbol, DateTime the_driver_expiration, double the_driver_live_price, double the_driver_price_at_computed_theo, bool the_is_frozen)
+      public NMSheetDriverData(string the_driver_symbol, Venue driver_venue, DateTime the_driver_expiration, double the_driver_live_price, double the_driver_price_at_computed_theo, bool the_is_frozen)
       {
          driver_symbol = the_driver_symbol;
+         this.driver_venue = driver_venue;
          driver_expiration = the_driver_expiration;
          driver_live_price = the_driver_live_price;
          driver_price_at_computed_theo = the_driver_price_at_computed_theo;

src/client/spark/NMSheets/BlackboardStructure/NMSheetSingleOptionData.cs

@@ -63,25 +63,6 @@ namespace BCG {
          //                                                                   BlueDataUtility.get_row_value<double>(kvp.Value, "ask_price", Double.NaN));
       }
 
-      public static NMSheetSingleOptionData filter_data(NMSheetSingleOptionData data, HashSet<uint> seckey_map) {
-         ExpStrikeKeyedData<Triplet<mbts.spark.TheoPoint, uint, uint>> theo_data_map = new ExpStrikeKeyedData<Triplet<mbts.spark.TheoPoint, uint, uint>>();
-         foreach (Triplet<DateTime, double, Triplet<mbts.spark.TheoPoint, uint, uint>> t in data._theo_data_map.get_data_linearly())
-            if (seckey_map.Contains(t.third.second) || seckey_map.Contains(t.third.third))
-               theo_data_map.set_data(t.first, t.second, t.third);
-         HashSet<NMSheetExpiration> exp_map = new HashSet<NMSheetExpiration>(theo_data_map.get_expirations().Select(exp => new NMSheetExpiration(data._nmsheet_option.option_exchange, data._nmsheet_option.option_name, exp)));
-         NMSheetDataRequest req = NMSheetDataRequest.restrict_for_expirations(data._data_request, exp_map);
-         Dictionary<DateTime, NMSheetDriverData> driver_data = new Dictionary<DateTime, NMSheetDriverData>();
-         foreach (KeyValuePair<DateTime, NMSheetDriverData> p in data._option_exp_driver_data)
-            if (exp_map.Contains(new NMSheetExpiration(data._nmsheet_option, p.Key)))
-               driver_data.Add(p.Key, p.Value);
-         return new NMSheetSingleOptionData(data._nmsheet_option, req, theo_data_map,
-                                             driver_data, data._theo_shift_map, data._atm_data_map,
-                                             data._roll_over_data, data._position_map, data._presaved_aw2_theos, data._option_exp_driver_rows, data._sec_map, data._settings) {
-            _bbo_data_map = data._bbo_data_map,
-            _ticker_data = data._ticker_data,
-         };
-      }
-
       public static double theo(TheoPoint tp, PutOrCall cp, NMSheetSettings settings) {
          bool use_fwd_normalized = settings != null && settings.setting_layout.show_forward_mormalized_theos;
          if (use_fwd_normalized)
@@ -289,4 +270,4 @@ namespace BCG {
       }
       #endregion
    }
-}
+}

src/client/spark/NMSheets/DataProviders/NMSheetSingleOptionMainDataSource.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Linq;
@@ -49,9 +49,9 @@ namespace BCG {
          /// input
          _in_bb_main_sheet_request_tag = SparkTags.TC2Tags.NMSheetDataRequestTag(req_id.ToString());
          _in_bb_secmap_tag = SparkTags.TC2Tags.ExchangeSecurityMapTag;
-         _in_bb_live_theos_tag = SparkTags.TC2Tags.LiveTheosTag(_nmsheet_symbol.option_name);
-         _in_bb_live_driver_bbo_tag = SparkTags.OptionDriverBBOTag(option.option_name);
-         _in_bb_precomputed_sparse_theos_tag = SparkTags.SymbolInterpGreeksTag(_nmsheet_symbol.option_name);
+         _in_bb_live_theos_tag = SparkTags.TC2Tags.LiveTheosTag((_nmsheet_symbol.option_name, _nmsheet_symbol.option_exchange));
+         _in_bb_live_driver_bbo_tag = SparkTags.OptionDriverBBOTag((option.option_name, option.option_exchange));
+         _in_bb_precomputed_sparse_theos_tag = SparkTags.SymbolInterpGreeksTag((_nmsheet_symbol.option_name, _nmsheet_symbol.option_exchange));
          _in_bb_vol_shift_tag = SparkTags.TC2Tags.VolShiftTag(req_id.ToString());
          _in_bb_sheet_settings_tag = SparkTags.TC2Tags.NMSheetSettingsTag(req_id.ToString());
          _in_bb_main_sheet_visibility_tag = SparkTags.TC2Tags.NMSheetVisibilityTag(req_id.ToString());
@@ -96,9 +96,10 @@ namespace BCG {
          double dr_pr = d.get_driver();
          if (d.driver_security.HasValue)
             return new NMSheetDriverData(d.driver_security.symbol_or_default(),
+                                         d.driver_security.exchange_or_default(),
                                          d.driver_security.Value.security_expiration().Value,
                                          dr_pr, dr_pr, false);
-         return new NMSheetDriverData(String.Empty, DateTime.MinValue, dr_pr, dr_pr, false);
+         return new NMSheetDriverData(String.Empty, Venue.Unset, DateTime.MinValue, dr_pr, dr_pr, false);
       }
 
       private void _compute_and_fire() {
@@ -191,17 +192,16 @@ namespace BCG {
 
       private Dictionary<DateTime, NMSheetDriverData> _get_option_exp_driver_data_map(NMSheetDataRequest data_request,
                                                                                       ExpStrikeKeyedData<mbts.spark.TheoPoint> output_theo_data_map, ExpStrikeKeyedData<mbts.spark.TheoPoint> shared_live_theo_data_map,
-                                                                                      Dictionary<DateTime, Triplet<string, DateTime, double>> live_driver_map) {
+                                                                                      Dictionary<DateTime, Triplet<(string, Venue), DateTime, double>> live_driver_map) {
          Dictionary<DateTime, NMSheetDriverData> rv = new Dictionary<DateTime, NMSheetDriverData>();
          foreach (Triplet<NMSheetExpiration, bool, double> theo_request in data_request.expiration_request_list) {
             DateTime option_expiration = theo_request.first.option_expiration;
             /// live driver
-            Triplet<string, DateTime, double> live_driver_info;
-            live_driver_map.TryGetValue(option_expiration, out live_driver_info);
-            live_driver_info = live_driver_info ?? new Triplet<string, DateTime, double>(String.Empty, DateTime.MinValue, Double.NaN);
+            var live_driver_info = live_driver_map.GetValueOrDefault(option_expiration)
+               ?? new((string.Empty, Venue.Unset), DateTime.MinValue, double.NaN);
             /// computed driver
             double computed_driver_price = _get_driverprice_for_expiration(output_theo_data_map, option_expiration);
-            rv[option_expiration] = new NMSheetDriverData(live_driver_info.first, live_driver_info.second, live_driver_info.third, computed_driver_price, theo_request.second);
+            rv[option_expiration] = new(live_driver_info.first.Item1, live_driver_info.first.Item2, live_driver_info.second, live_driver_info.third, computed_driver_price, theo_request.second);
          }
          return rv;
       }
@@ -236,10 +236,11 @@ namespace BCG {
          var option_exp_roll_data = vs_nvinp
             ?.get_roll_map(exp =>
                exp.symbol == _nmsheet_symbol.option_name &&
+               exp.exchange == _nmsheet_symbol.option_exchange &&
                exp.security_type == SecurityType.Option)
-            ?.ToDictionary(kv => kv.Key.expiration, kv => kv.Value)
+            ?.ToDictionaryCrushed(kv => kv.Key.expiration, kv => kv.Value)
             ?? new();
-         IDictionary<DateTime, ATMData> atm_data = OptionATMDataSource.compute_atm_data(_nmsheet_symbol.option_name, _get_onefive(), theo_data, option_exp_roll_data);
+         IDictionary<DateTime, ATMData> atm_data = OptionATMDataSource.compute_atm_data(_get_onefive(), theo_data, option_exp_roll_data);
          if (sheet_settings.setting_atm.select_atm_nearest_strike_only)
             foreach (ATMData exp_atm_data in atm_data.Values)
                exp_atm_data.reduce_to_nearest_expiration();

src/client/spark/NMSheets/Outside/ExchangeSecurityMap.cs

@@ -48,8 +48,9 @@ namespace BCG
                   option_map_aux[k] = new[] { get_sec(call_id.Value), get_sec(put_id.Value) };
             }
          }
-         var fut_map_aux =
-            sc_data.exp_map.ToDictionary(kv => kv.Key, kv => key_sec_map_aux[kv.Value]);
+         var fut_map_aux = sc_data.Futures.Select(smd => smd.security).ToDictionary(
+            sec => new SecurityExpiration(sec.Exchange, SecurityType.Future, sec.Symbol, sec.Future.expiration_date()),
+            sec => sec.CloneToMutable());
          _key_sec_map = key_sec_map_aux;
          _option_map = option_map_aux;
          _fut_map = fut_map_aux;

src/client/spark/NMSheets/Outside/tc2/Calculator2/MainForm/ATM/OptionATMDataSource.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using mbts.ui;
@@ -33,7 +33,8 @@ namespace BCG {
          }
       }
 
-      internal static IDictionary<DateTime, ATMData> compute_atm_data(string option_symbol, double one_five, ExpStrikeKeyedData<mbts.spark.TheoPoint> theo_data, IDictionary<DateTime, double> opt_exp_rolls) {
+      internal static IDictionary<DateTime, ATMData> compute_atm_data(
+         double one_five, ExpStrikeKeyedData<TheoPoint> theo_data, IDictionary<DateTime, double> opt_exp_rolls) {
          IDictionary<DateTime, double> exp_vs_drivers = _get_computed_drivers(theo_data);
          IDictionary<DateTime, ATMData> rv = new Dictionary<DateTime, ATMData>();
          if (one_five != 0)

src/client/spark/NMSheets/UIComponents/MainSheet/NMSheetControl.cs

@@ -539,7 +539,7 @@ namespace BCG {
                //    string elec_equiv_symbol = ContractUtilities.get_electronic_equivelant(bts.o2.msgs.SecurityType.OPTION, contract.product);
                //    contract = new OptionContract(contract.exchange, elec_equiv_symbol, contract.expiration, contract.strike, contract.isCall);
                // }
-               PriceSpec price_spec = new PriceSpec(contract.product, driver_data.driver_symbol,
+               var price_spec = new PriceSpec(contract.product, contract.exchange, driver_data.driver_symbol, driver_data.driver_venue,
                   driver_data.driver_expiration, 0, (driver_data.is_frozen && initial_state.preset_mode == SpreadWindowMode.Greeks) ?
                      driver_data.driver_price_at_computed_theo : -1, -1);
                bts.Log.Ui.info(String.Format("NMSheetControls._add_spread_leg publishing AddSpreadLegMessage for contract: {0}",