Adding function get_options_across_exchanges to SystemContractData which speeds up fetching of options

src/client/spark/AW2/InterpTheosServerPublisher.cs

@@ -55,6 +55,11 @@ namespace mbts.spark {
          return rv;
       }
 
+      // this basically does 
+      // f(SystemContractData, 
+      //    IReadOnlyDictionary<string, OptionTheoInterpolationData>, 
+      //    IReadOnlyDictionary<string, OptionDriverPriceMap>) -> bts.api.orderman.PrecomputedTheos
+      // and publishes bts.api.orderman.PrecomputedTheos to the server
       private void _publish_data() {
          SystemContractData sc_data = _bb.read_state(SparkTags.SystemContractsTag, null);
          IReadOnlyDictionary<string, OptionTheoInterpolationData> all_theos =

src/client/spark/AW2/Misc/SecurityNormalizer.cs

@@ -42,6 +42,8 @@ namespace mbts.spark {
          if (symbol_theos is null) {
             return Enumerable.Empty<Tuple<SecurityImmutable, TheoPoint>>();
          }
+         mbts.spark.HighPerformanceTimer timer = new HighPerformanceTimer();
+         timer.start();
          var rv = new List<Tuple<SecurityImmutable, TheoPoint>>();
          foreach (DateTime exp in symbol_theos.get_expirations()) {
             Dictionary<double, Dictionary<double, TheoPoint>> exp_data =
@@ -51,19 +53,21 @@ namespace mbts.spark {
                foreach (var k_kv in exp_kv.Value) {
                   double strike = k_kv.Key;
                   TheoPoint tp = k_kv.Value;
-                  IEnumerable<SecurityMD> calls = sc_data.get_options(
-                     null, symbol_theos.symbol, exp, strike, PutOrCall.Call);
-                  IEnumerable<SecurityMD> puts = sc_data.get_options(
-                     null, symbol_theos.symbol, exp, strike, PutOrCall.Put);
-                  foreach (SecurityMD sec_md in calls) {
-                     rv.Add(Tuple.Create(sec_md.security, tp));
+                  IEnumerable<SecurityImmutable> calls = sc_data.get_options_across_exchanges(
+                     symbol_theos.symbol, exp, strike, PutOrCall.Call);
+                  IEnumerable<SecurityImmutable> puts = sc_data.get_options_across_exchanges(
+                     symbol_theos.symbol, exp, strike, PutOrCall.Put);
+                  foreach (SecurityImmutable sec in calls) {
+                     rv.Add(Tuple.Create(sec, tp));
                   }
-                  foreach (SecurityMD sec_md in puts) {
-                     rv.Add(Tuple.Create(sec_md.security, tp));
+                  foreach (SecurityImmutable sec in puts) {
+                     rv.Add(Tuple.Create(sec, tp));
                   }
                }
             }
          }
+         timer.stop();
+         bts.Log.Gen.info($"SecurityNormalizer.normalize_interp_theos size: {rv.Count} time: {timer.lastIntervalTime} ms");
          return rv;
       }
    }

src/client/spark/AW2/Misc/SystemContractData.cs

@@ -20,6 +20,11 @@ namespace mbts.spark {
          new Dictionary<SecurityExpiration, UInt32>();
       public readonly IReadOnlyDictionary<SecurityExpiration, HashSet<SecurityStrike>>
          strike_map = new Dictionary<SecurityExpiration, HashSet<SecurityStrike>>();
+      // this is to make the option fetching faster (the reason is that historically, we have
+      // a lot of options data without the sec key, and thuss we need functions to go from
+      // venue, symbol, exp, strike, cp to Security)
+      private BCG.ExchangeSymbolOptionContractKeyedData<SecurityImmutable> _option_map =
+         new BCG.ExchangeSymbolOptionContractKeyedData<SecurityImmutable>();
 
       private SystemContractData() { }
       public static readonly SystemContractData empty = new SystemContractData();
@@ -30,6 +35,7 @@ namespace mbts.spark {
 
       static void _add_strike(
             ref Dictionary<SecurityExpiration, Dictionary<double, CPKey>> exp_k_map,
+            ref BCG.ExchangeSymbolOptionContractKeyedData<SecurityImmutable> option_map,
             SecurityImmutable sec) {
          bts.Log.assert(sec.HasOption);
          SecurityExpiration sec_exp = new SecurityExpiration(sec.Exchange,
@@ -44,11 +50,15 @@ namespace mbts.spark {
             cp_key = new CPKey();
             k_map.Add(sec.Option.Strike, cp_key);
          }
+         var ex_sym_map = option_map.get_set(sec.Exchange, sec.Symbol);
          if (sec.Option.CallPut == PutOrCall.Put) {
             cp_key.pkey = sec.Id;
+            ex_sym_map.setValue(sec.Option.expiration_date(), sec.Option.Strike, false, sec);
          } else {
             cp_key.ckey = sec.Id;
+            ex_sym_map.setValue(sec.Option.expiration_date(), sec.Option.Strike, true, sec);
          }
+
       }
 
       internal SystemContractData(IReadOnlyDictionary<uint, SecurityMD> sec_map_arg) {
@@ -67,7 +77,7 @@ namespace mbts.spark {
                new SecurityExpiration(sec_tup, sec.security_expiration() ?? default, sec.Id);
             exp_map_aux[sec_exp] = sec.Id;
             if (sec.HasOption) {
-               _add_strike(ref strike_map_temp, sec);
+               _add_strike(ref strike_map_temp, ref _option_map, sec);
             }
          }
          Dictionary<SecurityExpiration, HashSet<SecurityStrike>> strike_map_aux =
@@ -276,11 +286,19 @@ namespace mbts.spark {
             var s = sc.security;
             return s.security_type() == SecurityType.Option &&
                   (exchange == null || s.Exchange == exchange) &&
+                  (String.IsNullOrEmpty(option_name) || s.Symbol == option_name) &&
                   (exp == null || s.earliest_expiration() == exp) &&
                   (strike == null || s.strike() == strike) &&
                   (cp == null || s.Option.CallPut == cp);
          });
       }
 
+      public IEnumerable<SecurityImmutable> get_options_across_exchanges(
+            string option_name, DateTime exp, double strike, PutOrCall cp) {
+         List<SecurityImmutable> rv = new List<SecurityImmutable>();
+         _option_map.get(option_name, exp, strike, cp == PutOrCall.Call, ref rv);
+         return rv;
+      }
+
    }
 }

src/client/spark/NMSheets/Outside/OptionContractKeyedData.cs

@@ -25,6 +25,53 @@ namespace BCG {
       }
    }
 
+   // Container that keys (exchange, symbol) -> OptionContractKeyedData<T>
+   public class ExchangeSymbolOptionContractKeyedData<T> {
+      private Dictionary<bts.api.Venue, Dictionary<string, OptionContractKeyedData<T>>> _map =
+         new Dictionary<bts.api.Venue, Dictionary<string, OptionContractKeyedData<T>>>();
+
+      public OptionContractKeyedData<T> get(bts.api.Venue venue, string symbol) {
+         Dictionary<string, OptionContractKeyedData<T>> sym_map = null;
+         if (!_map.TryGetValue(venue, out sym_map)) {
+            return null;
+         }
+         bts.Log.assert(sym_map != null);
+         OptionContractKeyedData<T> rv = null;
+         if (!sym_map.TryGetValue(symbol, out rv))
+            return null;
+         return rv;
+      }
+      public void set(bts.api.Venue venue, string symbol, OptionContractKeyedData<T> data) {
+         Dictionary<string, OptionContractKeyedData<T>> sym_map = null;
+         if (!_map.TryGetValue(venue, out sym_map)) {
+            sym_map = new Dictionary<string, OptionContractKeyedData<T>>();
+            _map.Add(venue, sym_map);
+         }
+         sym_map[symbol] = data;
+      }
+      public OptionContractKeyedData<T> get_set(bts.api.Venue venue, string symbol) {
+         var rv = get(venue, symbol);
+         if (rv == null) {
+            rv = new OptionContractKeyedData<T>();
+            set(venue, symbol, rv);
+         }
+         return rv;
+      }
+      // gets option keyed data independent of exchange
+      public void get(string option_name, DateTime exp, double strike, bool is_call,
+                        ref List<T> out_values) {
+         foreach (var kv in _map) {
+            OptionContractKeyedData<T> sym_data = null;
+            if (kv.Value.TryGetValue(option_name, out sym_data)) {
+               T t = sym_data.getValue(exp, strike, is_call);
+               if (t != null) {
+                  out_values.Add(t);
+               }
+            }
+         }
+      }
+   }
+
    public class OptionContractKeyedData<T> {
       private Dictionary<DateTime, Dictionary<double, CallPutValue<T>>> _map = new Dictionary<DateTime, Dictionary<double, CallPutValue<T>>>();
       public OptionContractKeyedData() { }