PerfLog recovery

.gitignore

@@ -28,5 +28,3 @@ src/server/sparkd/orderman/perf_analysis/trace_data/
 *.fsx.lock
 src/lib/orderman/perf_analysis/logs/latency_data/
 trace_data/
-/src/lib/orderman/perf_analysis/last_pretty_log.csv
-/src/lib/orderman/perf_analysis/log.csv

.vscode/launch.json

@@ -36,8 +36,8 @@
          "name": "SDK Launch",
          "type": "clr",
          "request": "launch",
-         "program": "${workspaceRoot}/build/Debug/bin/net471/spark_sdk.exe",         
-         "args": ["md","GLBX","Future","CL", "CLZ9", "--user=bts", "--pass="],
+         "program": "${workspaceRoot}/build/Debug/bin/net471/spark_sdk.exe",
+         "args": ["md","GLBX","Future","ZN", "ZNU8"],
          "cwd": "${workspaceRoot}",
          "stopAtEntry": false,
          "externalConsole": true,
@@ -48,7 +48,7 @@
          "type": "clr",
          "request": "launch",
          "program": "C:/Program Files/Microsoft Office/Office14/EXCEL.EXE",
-         "args": ["${workspaceRoot}/build/Debug/bin/net471/spark_excel-addin64-packed.xll"],
+         "args": ["${workspaceRoot}/build/Debug/bin/net471/spark_excel-addin64.xll"],
          "cwd": "${workspaceRoot}",
          "stopAtEntry": false,
          "externalConsole": true,
@@ -209,7 +209,7 @@
             "type": "cppdbg",
             "request": "launch",
             "program": "/home/bts/spark/build/ninja_debug/src/server/sparkd/sparkd",
-            "args": ["test","AuthToken"],
+            "args": ["test","Co2Tests"],
             "stopAtEntry": false,
             "cwd": "/home/bts/spark/build/ninja_debug/src/server/sparkd/",
             "environment": [],

CMakeLists.txt

@@ -39,9 +39,8 @@ else ()
   set (CMAKE_CXX_FLAGS_DEBUG  "${CMAKE_CXX_FLAGS_DEBUG} \
                                -fPIC -Wall -pedantic -Wextra -Werror \
                                -Wno-missing-braces -Wno-ignored-qualifiers \
-                               --std=c++17 -fdiagnostics-color")
+                               --std=c++17 -fdiagnostics-color -Wl,--no-undefined")
   set (CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_DEBUG} -fno-omit-frame-pointer -DNDEBUG -O3")
-  set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wno-undef")
   add_subdirectory(src/server/sparkd)
   add_subdirectory(src/lib)
   add_subdirectory(src/web)

doc/dev/dev_process.md

@@ -1,84 +0,0 @@
-# The BTS Development Process #
-
-## The Big Picture ##
-
-Our development work is organized in projects. A project is an effort
-to specifiy, develop and validate a related set of additions to our
-Spark platform. A project can be assigned to one more developers and a
-developer might concurrently be assigned to multiple
-projects. Projects typically start out with specification work,
-proceed to implementation and finish with validation efforts. At any
-given time there will be multiple projects being worked on
-concurrently. A project is concluded when the features it implements
-are merged into the product and deployed to customers.
-
-## The Details ##
-Typically project ideas come from sales interactions, opposition
-research and customer feedback, alltough on occasion development or
-operational requirements will also suggest potentially valuable
-ideas. Whatever the source of a project idea the final decision of
-which projects to pursue is always done by management. Once a project
-idea has been selected for implementation the corresponding project
-will go through several phases: inception, specification, development
-and merge. We detail each of these phases below.
-
-* Inception. In this phase we create the infrastructure for the
-project. Typically this means selecting a team, creating a project
-case in Jira (that is a case assigned to the team@bluetradesys.com
-Jira user) and possibly creating a new branch for the project. This
-phase is mostly mechanical and the responsibility of Pedro.
-
-* Specification. The specification phase turns a usually vague and
-ill-considered idea project idea into a concrete, implementable
-specification that attempts to maximize the value/cost ratio of the
-project. It is hard to exaggerate the impact of a this phase on the
-rest of the project. *Nothing* has more impact on our ability to
-quickly deliver value to customers than good specification
-work. Specification typically take the form of shared google docs,
-using a mix of text and UI mockups to capture both user requirements
-and suggested implementation strategies. Taha is responsible for this
-phase.
-
-* Development. Once a specification is complete, a team of developers
-will use a project specific branch to implement the
-specification. Periodically, typically once a week, progress on the
-project is assessed by Pedro, possibly with the help of others. This
-assessment takes the form of code reviews, demos, or just one-on-one
-conversations. To ensure development does not go off on sub-optimal
-tangents, developers are expected to continuously merge master into
-the project branch as well as encouraged to create Pull Requests (PRs)
-early in the project implementation, so that their work is kept
-visible and integrated with that of the rest of the group. During the
-implementation phase developers should reach out to Pedro for UI
-reviews, architecture and design concerns as well as specification
-trade off concerns.
-
-* Verification. Once code is complete think carefully about the best way
-to verify that it is good before it goes to production.  We don't have a
-dedicated QA cycle to flush out bugs so it's important that a change be
-fully tested before it's marked "Ready to Merge."  For any non-trivial
-change consider enlisting a 'test buddy' to help review the code and
-think of test cases that you may not have considered.  Stability from
-release to release is especially critical right now as we develop a
-reputation in the futures world.
-  - Code Reviews
-  - Unit tests
-  - System tests
-  - Manual tests
-
-* Merge. Once a project is considered complete by developers, a PR
-that includes Pedro should be created (unless, as mentioned above, an
-early PR was already created in the development phase). Whether the PR
-already existed or not, its description should be changed to start
-with the "Ready To Merge" statement. This marks the project as ready
-to be merged into master. Once a week, typically on Mondays, Pedro
-will go through every PR marked as "Ready To Merge", build it, check
-it out and if everything looks good merge it to master. At the end of
-the day the resulting master will be tagged with a new version
-number. This version will be go through some additional testing by
-Brian and will then be deployed by Karl and Pete to customers
-throughout the rest of the week.
-
-## Tools ##
-  * Pre-Merge checklist
-  * Code Review tutorial

doc/user/docs/release_notes.md

@@ -1,17 +1,4 @@
 # Release Notes
-## 1.4.3
-
-### Ladder Usability Improvements
-  - Controls found in Left Pane of the ladder can now be hidden via the checkboxes found under Ladder Form -> Settings button -> Left Pane tab
-  - The Left Pane is now scrollable with the mouse wheel when there isn't enough vertical space to accomodate the entire form
-  - The Left Pane can now grown and shrunk horizontally via the splitter between the left pane and the main ladder area
-### Performance Improvements
-  - Requoting and hedging latency has been reduced by up to 30%
-### Software Development Kit (SDK) Improvements
-  - The API now enforces authentication 
-### Bug Fixes
-  - The Market View form now honors filters when new securities are added
-  
 ## 1.4.2
 
 ### User Interface Improvements
@@ -19,18 +6,9 @@
 ### Bug Fixes
   - Shell no longer crashes when removing a security from the synthetics window
   - It is now possible to create a new synthetic using the same name as a previously deleted synthetic 
-### SDK Improvements
-  - Improved Client and Server SDK clarity and documentation
 ### Performance Improvements
   - Synthetic calculation speed was improved by 400%.
-### Risk / Entitlement System
-  - Risk Admin UI
-    - Spark users now have separate Trading, RiskAdmin, and/or MDAdmin permissions
-    - Spark users can now only add other users or accounts if permissioned to do so.
-  - Market Data Entitlement
-    - Automated market data reporting
-   
-
+  
 ## 1.4.1
 
 ### Tabs

src/client/spark/Admin/AdminModel.cs

@@ -6,7 +6,6 @@ using System.ComponentModel.Composition;
 using System.Linq;
 using System.Reactive.Linq;
 using System.Reactive.Subjects;
-using bts.api.core;
 
 namespace mbts.spark {
    [Export]

src/client/spark/Admin/AdminModelWrapper.cs

@@ -4,12 +4,11 @@ using System;
 using System.Collections.Generic;
 using System.Linq;
 
-namespace bts.api.core
+namespace mbts.spark 
 {
-   public class AdminModelWrapper
+   class AdminModelWrapper
    {
       readonly AdminDataImmutable? _admin_data;
-      public AdminDataImmutable? admin_data => _admin_data;
       readonly string _user_name;
       readonly List<BTSAccountImmutable> _allowed_accounts;
       readonly Dictionary<string, string> _allowed_account_dict = new Dictionary<string, string>();

src/client/spark/Admin/PermissionFilters.cs

@@ -4,6 +4,10 @@ using bts.api.core;
 using bts.api.orderman;
 using bts.api.position;
 using bts.spark;
+using mbts.collections;
+using mbts.spark;
+using mbts.spark.utilities;
+using mbts.utils;
 using System;
 using System.Collections.Generic;
 using System.Collections.Immutable;
@@ -14,14 +18,14 @@ using System.Reactive.Linq;
 
 namespace bts.api.core
 {
-   public static class PermissionFilters
+
+   static class PermissionFilters
    {
-      public static IObservable<Update<PositionUnitImmutable, int>> 
-         PermissionFilter(this IObservable<Update<PositionUnitImmutable, int>> source, IObservable<AdminModelWrapper> am)
+      public static IObservable<Update<PositionUnitImmutable, int>> PermissionFilter(this IObservable<Update<PositionUnitImmutable, int>> source, AdminModel am)
       {
          var pred = Observable.Create<Func<PositionUnitImmutable, bool>>( observer =>
          {
-            return am.Subscribe(x =>
+            return am.on_admin_data.Subscribe(x =>
             {
                Func<PositionUnitImmutable, bool> filter = (z => x.can_view_account(z.Account));
                observer.OnNext(filter);
@@ -31,14 +35,13 @@ namespace bts.api.core
          return source.Filter(pred);
       }
 
-      public static IObservable<Update<OrderRecordImmutable, uint>> 
-         PermissionFilter(this IObservable<Update<OrderRecordImmutable, uint>> source, IObservable<AdminModelWrapper> am)
+      public static IObservable<Update<OrderRecordImmutable, uint>> PermissionFilter(this IObservable<Update<OrderRecordImmutable, uint>> source, AdminModel am)
       {
          var pred = Observable.Create<Func<OrderRecordImmutable, bool>>(observer =>
         {
-           return am.Subscribe(x =>
+           return am.on_admin_data.Subscribe(x =>
            {
-              Func<OrderRecordImmutable, bool> filter = (z => x.can_view_account(z.Def.Account ?? string.Empty));
+              Func<OrderRecordImmutable, bool> filter = (z => x.can_view_account(z.account()));
               observer.OnNext(filter);
            });
         });
@@ -46,12 +49,11 @@ namespace bts.api.core
          return source.Filter(pred);
       }
 
-      public static IObservable<Update<FillImmutable, string>> 
-         PermissionFilter(this IObservable<Update<FillImmutable, string>> source, IObservable<AdminModelWrapper> am)
+      public static IObservable<Update<FillImmutable, string>> PermissionFilter(this IObservable<Update<FillImmutable, string>> source, AdminModel am)
       {
          var pred = Observable.Create<Func<FillImmutable, bool>>(observer =>
         {
-           return am.Subscribe(x =>
+           return am.on_admin_data.Subscribe(x =>
            {
               Func<FillImmutable, bool> filter = (z => x.can_view_account(z.Account));
               observer.OnNext(filter);

src/client/spark/Daemons/MDDaemon.cs

@@ -17,22 +17,31 @@ using bts.api.user;
 
 namespace bts {
    [Export(typeof(IInitializable))]
-   class MDDaemon : bts.md.MDSource, IObserver<List<SecurityAndHandle>>, IInitializable, IDisposable {
+   class MDSource : IObserver<List<SecurityAndHandle>>, IInitializable, IDisposable {
       readonly Lazy<IBus> _bus;
-      readonly Lazy<AuthTokenImmutable> _token;
+      readonly Lazy<AuthToken> _token;
       readonly Lazy<SyntheticSecurityService> _ss;
+      readonly MDConsumer _consumer = new MDConsumer();
+      readonly Co2RelayControl _relay;
       readonly Dictionary<SecurityKey, SecurityMD> _deleted_synthetics = new Dictionary<SecurityKey, SecurityMD>();
-
-      static uint get_naggle() => mbts.config.Configuration.get<bts.md.RelayOptions>().naggle_ms;
+      long _version = 0;
+      readonly ConcurrentDictionary<SecurityKey, SecurityMD> _exchange_sec_map = new ConcurrentDictionary<SecurityKey, SecurityMD>();
+      List<IDisposable> _dispose = new List<IDisposable>();
 
       [ImportingConstructor]
-      public MDDaemon(Lazy<IBus> bus, Lazy<AuthTokenImmutable> token, Lazy<SyntheticSecurityService> ss) : base(get_naggle(), true) {
+      public MDSource(
+         Lazy<IBus> bus,
+         Lazy<AuthToken> token, 
+         Lazy<SyntheticSecurityService> ss) {
          _bus = bus;
          _token = token;
-         _ss = ss;         
+         _ss = ss;
+         _relay = new Co2RelayControl(_consumer);
+         _relay.preload_securities((co2.MDRequest.Types.Flags)0xff).ignore();
+         _dispose.Add(_consumer.on_security.Subscribe(this));
       }
 
-      override public void initialize() {
+      public void initialize() {
          _relay.set_token(_token.Value);
          _dispose.Add(
             _ss.Value.subscribei_metadata()
@@ -57,9 +66,6 @@ namespace bts {
                // Post full dictionary of syntheics here
                bus.post(SparkTags.SyntheticstrategiesTag, t);
             }));
-         _dispose.Add( this.on_securities.Subscribe( (md) => {
-            bus.post(SparkTags.SecuritiesTag, md);
-         }));            
          Task.Run(run);
       }
 
@@ -68,5 +74,37 @@ namespace bts {
 
       public int priority => 55;
       IBus bus => _bus.Value;
+
+      public void OnCompleted() { }
+      public void OnError(Exception error) => Log.Md.error(error.ToString());
+      public void OnNext(List<SecurityAndHandle> values) {
+         if (values.Count > 0) {
+            foreach (var v in values) {
+               var smd = new SecurityMD(v.security, v.handle);
+               _exchange_sec_map.AddOrUpdate(smd.security.Id, smd, (_, __) => smd);
+            }
+            _version += 1;
+         }
+      }
+
+      public void Dispose() => Disposable.dispose(_dispose);
+      async Task run() {
+         long run_version = 0;
+         while (true) {
+            var exch = _exchange_sec_map;
+            long current_version = _version;
+            if (current_version > run_version) {
+               run_version = current_version;
+               try {
+                  Log.Md.info($"MDSource publishing {exch.Count} securities");
+                  bus.post(SparkTags.SecuritiesTag, exch);
+               } catch (Exception e) {
+                  Log.Md.error($"Error in publish_securities: {e}");
+               }
+            }
+            await Task.Delay(TimeSpan.FromSeconds(1));
+         }
+      }
+
    }
 }

src/client/spark/Daemons/ShellServices.cs

@@ -13,7 +13,10 @@ using System.ComponentModel.Composition;
 namespace Spark {
    public class ShellServices {
       [Export]
-      public ServiceContext context { get; } = new ServiceContext();
+      public static ServiceContext context { get; } = new ServiceContext();
+
+      [Export]
+      public UserService user_service => context.locate<UserService>();
 
       [Export("settings")]
       public PersistenceService settings_service => context.locate<PersistenceService>("settings");

src/client/spark/Modules/Market/MarketView/MarketViewRow.cs

@@ -35,7 +35,7 @@ namespace mbts.spark {
          readonly CellProperty<string> _working_bids_text, _working_asks_text;
          readonly MyOrderSummarizer _order_summarizer = new MyOrderSummarizer();
          readonly MDDisplayHelper.SizeFormatter _size_formatter = MDDisplayHelper.get_formatter(MarketSizeDisplay.get_default());
-         Property<uint> _bid_direct_size, _bid_merged_size, _ask_direct_size, _ask_merged_size;
+         uint _bid_direct_size, _bid_merged_size, _ask_direct_size, _ask_merged_size;
          string _bid_merged_size_text, _ask_merged_size_text;
          ulong _bbo_ref, _last_ref;
 
@@ -45,33 +45,28 @@ namespace mbts.spark {
             _security_text = smd.security.ToString();
             _security_key_text = security_key.ToString();
             string format_price(double price) => security.ContractSpec.format(price);
-            _position = new CellProperty<int>(this, nameof(position), 0);
+            _position = new CellProperty<int>(0);
             _bid_price = backing(nameof(bid_price), double.NaN);
             _ask_price = backing(nameof(ask_price), double.NaN);
             _last_price = backing(nameof(last_price), double.NaN);
-            _last_size = new CellProperty<uint>(this, nameof(last_size), 0);
+            _last_size = new CellProperty<uint>(0);
             _day_change = backing(nameof(day_change), double.NaN);
             _session_low = backing(nameof(session_low), double.NaN);
             _session_high = backing(nameof(session_high), double.NaN);
             _settlement = backing(nameof(settlement), double.NaN);
-            _volume = new CellProperty<ulong>(this, nameof(volume), 0);
-            _inside_bids = new CellProperty<bool>(this, nameof(inside_bids));
-            _inside_asks = new CellProperty<bool>(this, nameof(inside_asks));
-            _working_bid_volume = new CellProperty<int>(this, nameof(working_bid_volume));
-            _working_ask_volume = new CellProperty<int>(this, nameof(working_ask_volume));
-            //
-            _bid_direct_size = backing(nameof(bid_direct_size), default(uint));
-            _bid_merged_size = backing(nameof(bid_merged_size), default(uint));
-            _ask_direct_size = backing(nameof(ask_direct_size), default(uint));
-            _ask_merged_size = backing(nameof(ask_merged_size), default(uint));
+            _volume = new CellProperty<ulong>(0);
+            _inside_bids = new CellProperty<bool>();
+            _inside_asks = new CellProperty<bool>();
+            _working_bid_volume = new CellProperty<int>();
+            _working_ask_volume = new CellProperty<int>();
             //
             _working_orders = new ObservableFilterCollection<OrderRecordImmutable>(_orders, o => o.is_active());
             _working_bids = new ObservableFilterCollection<OrderRecordImmutable>(_working_orders, o => o.Def.Side == Side.Buy);
             _working_asks = new ObservableFilterCollection<OrderRecordImmutable>(_working_orders, o => o.Def.Side == Side.Sell);
-            _working_best_bid = new CellProperty<double>(this, nameof(working_best_bid), format_price, double.NaN);
-            _working_best_ask = new CellProperty<double>(this, nameof(working_best_ask), format_price, double.NaN);
-            _working_bids_text = new CellProperty<string>(this, nameof(working_bids_text), string.Empty);
-            _working_asks_text = new CellProperty<string>(this, nameof(working_asks_text), string.Empty);
+            _working_best_bid = new CellProperty<double>(format_price, double.NaN);
+            _working_best_ask = new CellProperty<double>(format_price, double.NaN);
+            _working_bids_text = new CellProperty<string>(string.Empty);
+            _working_asks_text = new CellProperty<string>(string.Empty);
             //
             default_command = new DelegateCommand(_show_ladder);
             bid_price_click = new DelegateCommand(() => _show_order_ticket(MarketViewView.Columns.Bid));
@@ -105,15 +100,15 @@ namespace mbts.spark {
          public double last_price => _last_price.Value;
          public uint last_size => _last_size.value;
          public double bid_price => _bid_price.Value;
-         public uint bid_direct_size => _bid_direct_size.Value;
-         public uint bid_implied_size => _bid_merged_size.Value - _bid_direct_size.Value;
-         public uint bid_merged_size => _bid_merged_size.Value;
-         public string bid_merged_size_text => _bid_merged_size_text ?? (_bid_merged_size_text = _size_formatter(_bid_merged_size.Value, bid_implied_size));
+         public uint bid_direct_size => _bid_direct_size;
+         public uint bid_implied_size => _bid_merged_size - _bid_direct_size;
+         public uint bid_merged_size => _bid_merged_size;
+         public string bid_merged_size_text => _bid_merged_size_text ?? (_bid_merged_size_text = _size_formatter(_bid_merged_size, bid_implied_size));
          public double ask_price => _ask_price.Value;
-         public uint ask_direct_size => _ask_direct_size.Value;
-         public uint ask_implied_size => _ask_merged_size.Value - _ask_direct_size.Value;
-         public uint ask_merged_size => _ask_merged_size.Value;
-         public string ask_merged_size_text => _ask_merged_size_text ?? (_ask_merged_size_text = _size_formatter(_ask_merged_size.Value, ask_implied_size));
+         public uint ask_direct_size => _ask_direct_size;
+         public uint ask_implied_size => _ask_merged_size - _ask_direct_size;
+         public uint ask_merged_size => _ask_merged_size;
+         public string ask_merged_size_text => _ask_merged_size_text ?? (_ask_merged_size_text = _size_formatter(_ask_merged_size, ask_implied_size));
          public ulong volume => _volume.value;
          public double settlement => _settlement.Value;
          public double day_change => _day_change.Value;
@@ -149,9 +144,9 @@ namespace mbts.spark {
             bool dirty = false;
             if (md.md.get_bbo(ref _bbo_ref, out NativePriceSize b, out NativePriceSize a)) {
                _bid_price.set(b.price);
-               refresh_size(ref b, _bid_direct_size, _bid_merged_size, ref _bid_merged_size_text, nameof(bid_merged_size_text));
+               refresh_size(ref b, ref _bid_direct_size, ref _bid_merged_size, ref _bid_merged_size_text, nameof(bid_merged_size_text));
                _ask_price.set(a.price);
-               refresh_size(ref a, _ask_direct_size, _ask_merged_size, ref _ask_merged_size_text, nameof(ask_merged_size_text));
+               refresh_size(ref a, ref _ask_direct_size, ref _ask_merged_size, ref _ask_merged_size_text, nameof(ask_merged_size_text));
                _inside_bids.on_next_distinct(_bid_price.Value >= _working_best_bid.value, ref dirty);
                _inside_asks.on_next_distinct(_ask_price.Value <= _working_best_ask.value, ref dirty);
             }
@@ -167,10 +162,10 @@ namespace mbts.spark {
             if (dirty)
                invalidated?.Invoke();
 
-            void refresh_size(ref NativePriceSize nps, Property<uint> direct, Property<uint> merged, ref string text, string text_prop) {
-               if (nps.size != merged.Value || nps.direct_size != direct.Value) {
-                  merged.set(nps.size);
-                  direct.set(nps.direct_size);
+            void refresh_size(ref NativePriceSize nps, ref uint direct, ref uint merged, ref string text, string text_prop) {
+               if (nps.size != merged || nps.direct_size != direct) {
+                  merged = nps.size;
+                  direct = nps.direct_size;
                   text = null;
                   dirty = true;
                   RaisePropertyChanged(text_prop);
@@ -271,15 +266,7 @@ namespace mbts.spark {
             _working_best_ask.on_next_distinct(_order_summarizer.best_working_ask, ref dirty);
             _inside_bids.on_next_distinct(_order_summarizer.best_working_bid >= bid_price, ref dirty);
             _inside_asks.on_next_distinct(_order_summarizer.best_working_ask <= ask_price, ref dirty);
-
-            RaisePropertyChanged(
-               // Implied might not be ever set, so it probably does not RaisePropertyChanged either
-               // This might be part of the filter funk?
-               //nameof(bid_implied_size),
-               //nameof(ask_implied_size),
-               nameof(working_bids_text), 
-               nameof(working_asks_text)
-            );
+            RaisePropertyChanged(nameof(working_bids_text), nameof(working_asks_text));
          }
          #endregion
 

src/client/spark/Modules/Trading/LadderWindowFactory.cs

@@ -61,12 +61,10 @@ namespace bts.shell {
          var view = new Lazy<UIElement>(() => {
             var model = new LadderModel(md, _orders_model, _md_auxiliary_model);
             var settings_manager = new WindowSettingsManager<LadderSettings>(
-               _settings_hub, state_settings == null || state_settings.use_defaults ? null : state_settings,
+               _settings_hub, state_settings,
                us => sanitize_ladder_settings(us.ladder),
                (us, v) => us.with(sanitize_ladder_settings(v)));
-            var init = state_settings ?? settings_manager.default_settings;
-            init.use_defaults |= state_settings == null;
-            var vm = new LadderVM(_bus, model, settings_manager, init);
+            var vm = new LadderVM(_bus, model, settings_manager);
             return new LadderView(vm);
          });
          string thread = Configuration.get<bts.config.toggles.LaddersOnThreads>().enabled ? $"Ladder{id:00}" : "";

src/client/spark/Orders/OrdersModel.cs

@@ -44,8 +44,8 @@ namespace mbts.spark {
          _dynamic_helper = dynamic_helper;
          var scheduler = ThreadPoolScheduler.Instance;
          var interval = TimeSpan.FromMilliseconds(100);
-         on_orders = _service.subscribei_orders().Naggle(interval, scheduler).PermissionFilter(admin_model.on_admin_data).Table().Cache();
-         on_positions = _position_service.subscribei_position_data().Naggle(interval, scheduler).PermissionFilter(admin_model.on_admin_data).Table().Cache();
+         on_orders = _service.subscribei_orders().Naggle(interval, scheduler).PermissionFilter(admin_model).Table().Cache();
+         on_positions = _position_service.subscribei_position_data().Naggle(interval, scheduler).PermissionFilter(admin_model).Table().Cache();
          working_orders = 
             on_orders.Scan(
                ImmutableDictionary<uint,OrderRecordImmutable>.Empty, 
@@ -55,7 +55,7 @@ namespace mbts.spark {
          on_order_tuples = order_record_tuple_source.subscribe_incremental_update();
          on_order_tuples_table = order_record_tuple_source.subscribe_incremental_table_update();
          on_order_securities = order_record_tuple_source.subscribe_order_securities();
-         on_fills = _service.subscribei_fills().Naggle(interval, scheduler).PermissionFilter(admin_model.on_admin_data).Table().Cache();
+         on_fills = _service.subscribei_fills().Naggle(interval, scheduler).PermissionFilter(admin_model).Table().Cache();
          _activity = new OrderActivitySummarizer(bus, on_orders);
          on_fill_tuples = FillTupleSource.create_observable(bus.subscribe(SparkTags.SecuritiesTag), on_orders, on_fills);
          _security_orders_demuxer = new TableUpdateDemuxer<OrderRecordImmutable, uint, uint>(o => o.Key, o => o.security_key());

src/client/spark/core/Bus/SparkCommands.cs

@@ -3,7 +3,6 @@ using bts.api.orderman;
 using bts.shell;
 using mbts.spark;
 using mbts.ui;
-using mbts;
 using System;
 using System.Collections.Generic;
 

src/client/spark/core/GlobalExtensions/GlobalExtensions.cs

@@ -218,7 +218,9 @@ namespace mbts
          private Func<T, T, int> _comparer_func;
          public DelegateComparer(Func<T, T, int> comparer_func) { _comparer_func = comparer_func; }
          public int Compare(T x, T y) { return _comparer_func(x, y); }
-      }      
+      }
+
+      public static void ignore(this Task t) { }
 
       public static Google.Protobuf.WellKnownTypes.Timestamp ToGoogleTime(this DateTime self) {
          return Google.Protobuf.WellKnownTypes.Timestamp.FromDateTime(self);

src/client/spark/core/Settings/UserPersistFactory.cs

@@ -22,12 +22,12 @@ namespace mbts.ui.settings
       readonly PersistenceService _persist;
 
       [ImportingConstructor]
-      public UISettingsFactory(AuthService login, [Import("settings")] PersistenceService persist) {
+      public UISettingsFactory(UserService login, [Import("settings")] PersistenceService persist) {
          _persist = persist;
          _users = get_users(login);
       }
 
-      async Task<User[]> get_users(AuthService login) {
+      async Task<User[]> get_users(UserService login) {
          var users = await login.get_users_async(null);
          if (users.success) return users.value.Users.ToArray();
          else return new User[0];

src/client/spark/core/ShellInterface/UISettings.cs

@@ -120,7 +120,7 @@ namespace mbts.persist
       public IObservable<IEnumerable<PersistResult<T>>> subscribe<T>(string ns, string pattern) {
          var re = new Regex(pattern);
          return _persist
-            .subscribe_ns(uns(ns))
+            .subscribe_ns(new NamespaceSubArgs() { Ns = uns(ns) })
             .Select(update =>
                update.added
                   .Where(item => re.IsMatch(item.Key))

src/client/spark/view/Controls/DIYGrid/CellProperty.cs

@@ -1,21 +1,16 @@
-using mbts.ui.shell;
-using System;
+using System;
 using System.Reactive.Linq;
 using System.Reactive.Subjects;
 
 namespace mbts.spark {
    class CellProperty<T> : ISubject<T, (T oldval, T newval)> {
       readonly Func<T, string> _format;
-      readonly IRaiseNotifyPropertyChanged _owner;
-      readonly string _prop_name;
       string _text;
 
-      public CellProperty(IRaiseNotifyPropertyChanged owner, string prop_name, T initial_value = default)
-         : this(owner, prop_name, t => t.ToString(), initial_value) { }
+      public CellProperty(T initial_value = default)
+         : this(t => t.ToString(), initial_value) { }
 
-      public CellProperty(IRaiseNotifyPropertyChanged owner, string prop_name, Func<T, string> format, T initial_value = default) {
-         _owner = owner;
-         _prop_name = prop_name;
+      public CellProperty(Func<T, string> format, T initial_value = default) {
          _format = format;
          value = initial_value;
       }
@@ -29,7 +24,6 @@ namespace mbts.spark {
          if (!value.Equals(this.value)) {
             OnNext(value);
             dirty = true;
-            _owner.raise_notify_property_changed(_prop_name);
          }
       }
 

src/client/spark/view/Converters/ResponsiveTextConverter.cs

@@ -1,31 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Globalization;
-using System.Linq;
-using System.Windows;
-using System.Windows.Data;
-
-namespace mbts.ui.converters {
-   class ResponsiveTextConverter : IValueConverter {
-
-      public string ShortText { get; set; }
-      public string LongText { get; set; }
-      public double Threshold { get; set; }
-
-
-      public object Convert(object value, Type targetType, object parameter, CultureInfo culture) {
-         if (value is double _input) {
-            if (_input < Threshold) {
-               return ShortText;
-            } else {
-               return LongText;
-            }
-         }
-         return LongText;
-      }
-
-      public object ConvertBack(object value, Type targetType, object parameter, CultureInfo culture) {
-         throw new NotImplementedException();
-      }
-   }
-}

src/client/spark/view/Shell/Login/LoginModel.cs

@@ -1,29 +1,39 @@
-using bts;
-using bts.api;
-using bts.api.core;
-using bts.api.persistence;
-using bts.api.user;
-using mbts.config;
-using System;
+using System;
 using System.Collections.Generic;
+using System.Collections.ObjectModel;
 using System.ComponentModel.Composition;
+using System.Diagnostics;
 using System.Linq;
+using System.Text;
+using System.Threading;
 using System.Threading.Tasks;
+using System.Windows;
 using System.Windows.Threading;
+using mbts.config;
+using mbts.ui.native;
+using mbts.utils;
+using bts.co2;
+using bts.api.user;
+using bts.api.persistence;
+using bts;
 
-namespace mbts.ui.shell {
+namespace mbts.ui.shell
+{
    // The LoginContainer participates in the startup Import/Export dependency system
    // Since the login form should not depend on *anything* it will be shown before this is created
    // Once this is created it access the global LoginModel and retrieves the username
    internal class LoginContainer
    {
       [ImportingConstructor]
-      public LoginContainer(ServiceContext context) {
+      public LoginContainer(UserService user, [Import("settings")] PersistenceService persist) {
          var model = LoginModel.instance;
-         model.provide_services(context);
+         model.provide_services(user, persist);
          // await_login pumps the dispatcher but doesn't allow control to return to MEF until the username and workspace are available
          model.await_login();
-         (UserName,UserId,Token) = model.get_user();
+         var u = model.get_user();
+         UserName = u.name;
+         UserId = u.id;
+         Token = u.token;
          Workspace = model.get_workspace();
          model.Dispose();
       }
@@ -34,19 +44,21 @@ namespace mbts.ui.shell {
       [Export("userid")]
       public uint UserId { get; private set; }
       [Export]
-      public AuthTokenImmutable Token { get; private set; }
+      public AuthToken Token { get; private set; }
 
       [Export("workspace")]
       public string Workspace { get; private set; }
    }
 
    public struct User {
-      public User(string name, uint id) {
+      public User(string name, uint id, AuthToken token) {
          this.name = name;
          this.id = id;
+         this.token = token;
       }
       public readonly string name;
       public readonly UInt32 id;
+      public readonly AuthToken token;
    }
 
    public class LoginModel : IDisposable
@@ -54,10 +66,11 @@ namespace mbts.ui.shell {
       // Singleton to help bootstrap the system
       private static LoginModel _instance = new LoginModel();
       public static LoginModel instance { get { return _instance; } }
+      private User _user;
       private string _workspace;
       private PreloginWorkspaceEnumerator _enum_workspace;
-
-      ServiceContext _ctx;
+      
+      UserService _logon;
       DispatcherFrame _frame = new DispatcherFrame();
 
       private LoginModel() {
@@ -69,14 +82,13 @@ namespace mbts.ui.shell {
       public string ErrorMessage { get; private set; }
       public bool HasFatalError { get; set; }
       public User[] Users { get; private set; }
-
-
+      
+      
       // The final username
-      public (string, uint, AuthTokenImmutable) get_user() {
-         Log.assert_d(!string.IsNullOrEmpty(_ctx.user));
-         return (_ctx.user, _ctx.user_id, _ctx.token.Value);
+      public User get_user() { 
+         Log.assert_d(!string.IsNullOrEmpty(_user.name)); 
+         return _user;
       }
-
       public string get_workspace() {
          return _workspace ?? "";
       }
@@ -100,13 +112,13 @@ namespace mbts.ui.shell {
          }
 
          // Wait for the hub to create the login doodad
-         _ctx = await _hub_src.Task;
+         (var users, var persist) = await _hub_src.Task;
+         _logon = users;
          // Populate the UI
-         _enum_workspace = new PreloginWorkspaceEnumerator(_ctx.locate<PersistenceService>("settings"));
-         var auth = _ctx.locate<AuthService>();
-         var response = await auth.get_users_async(new GetUsersRequest());
+         _enum_workspace = new PreloginWorkspaceEnumerator(persist);
+         var response = await _logon.get_users_async(new GetUsersRequest());
          if (response.success) {
-            Users = response.value.Users.Select(x => new User(x.Login,x.Id)).OrderBy(u => u.name).ToArray();
+            Users = response.value.Users.Select(x => new User(x.Login,x.Id, null)).OrderBy(u => u.name).ToArray();
          }
          else {
             ErrorMessage = "Logon: " + response.error_message;
@@ -117,37 +129,49 @@ namespace mbts.ui.shell {
 
       #endregion
 
-      public async Task<LoginResult> do_login(string username, string workspace, string password) {
+      public async Task<(bool success,string info)> do_login(string username, string workspace, string password) {
          if (Users.Select(u => u.name).Contains(username)) {
-            var result = await _ctx.login(username, password);
-            if (result.success) {
-               _workspace = workspace;
-               on_user_authenticated(_ctx.user, _ctx.token);
+            var response = await _logon.authenticate_login_async(new AuthenticateLoginRequest() {
+               Login = username,
+               Password = password ??""
+            });
+            if (response.success) {
+               if (response.value.IsValid) {
+                  _workspace = workspace;
+                  set_authenticated_user(new User(username, response.value.UserId, response.value.Token));
+                  return (true, "");
+               }
+               else {
+                  return (false, response.value.Reason);
+               }
+            }
+            else {
+               return (false, response.error_message);
             }
-            return result;
          }
-         return new LoginResult("Invalid username");
+         return (false, "Invalid username");
       }
 
-      private void on_user_authenticated(string username, AuthTokenImmutable? token) {
-         Log.assert(!string.IsNullOrEmpty(username));
-         Log.assert(Users.Select(u => u.name).Contains(username));
-         Environment.SetEnvironmentVariable("BTS_USER", username);
+      private void set_authenticated_user(User user) {
+         Log.assert(!string.IsNullOrEmpty(user.name));
+         Log.assert(Users.Select(u => u.name).Contains(user.name));
+         _user = user;
+         Environment.SetEnvironmentVariable("BTS_USER", user.name);
          _frame.Continue = false;
       }
 
       public void await_login() {
-         if (!_ctx.token.HasValue) {
+         if (string.IsNullOrEmpty(_user.name)) {
             Dispatcher.PushFrame(_frame);
          }
       }
 
       
       #region provide hub
-      internal void provide_services(ServiceContext ctx) {
-         _hub_src.TrySetResult(ctx);
+      public void provide_services(UserService u, PersistenceService p) {
+         _hub_src.TrySetResult((u,p));
       }
-      private TaskCompletionSource<ServiceContext> _hub_src = new TaskCompletionSource<ServiceContext>();
+      private TaskCompletionSource<(UserService, PersistenceService)> _hub_src = new TaskCompletionSource<(UserService, PersistenceService)>();
       #endregion
 
       #region Dispose

src/client/spark/view/Shell/Login/LoginViewModel.cs

@@ -138,15 +138,14 @@ namespace mbts.ui.shell
          if (selected_workspace == "New workspace...") {
             selected_workspace = null;
          }
-         var result = await _model.do_login(Name, selected_workspace, Password);
-         if (result.success) { VisualState = "Loading"; }
-         if (!result.success && !string.IsNullOrEmpty(result.reason)) {
-            WarningMessage = result.reason;
+         (bool success, string message) = await _model.do_login(Name, selected_workspace, Password);
+         if (success) { VisualState = "Loading"; }
+         if (!success && !string.IsNullOrEmpty(message)) {
+            WarningMessage = message;
             VisualState = "BadPassword";
          }
-         else if (!string.IsNullOrEmpty(result.reason)) {
-            // Login was successful but we have a message for the user
-            BTSMessageBox.Show(result.reason, "Login Message", BTSMessageBoxButtons.OK, MessageBoxImage.Information);
+         else if (!string.IsNullOrEmpty(message)) {
+            BTSMessageBox.Show(message, "Login Message", BTSMessageBoxButtons.OK, MessageBoxImage.Information);
          }
       }
       private void OnCancel(object unused) {

src/client/spark/view/Shell/Workspaces/SettingsManagerTool.cs

@@ -11,7 +11,7 @@ namespace bts.settings {
    public class SettingsManagerTool {
       public static void run() {
          var sc = new ServiceContext();
-         var users = sc.locate<AuthService>();
+         var users = sc.locate<UserService>();
          var persist = sc.locate<PersistenceService>("settings");
          var user_list = users.get_users(new GetUsersRequest());
          var user_names = user_list.Users.Select(u => u.Login);

src/client/spark/view/Spark/Core/SecurityMD.cs

@@ -6,7 +6,7 @@ using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
 
-namespace mbts {
+namespace mbts.spark {
    struct SecurityMD : IEquatable<SecurityMD> {
       public SecurityImmutable security { get; }
       public MDConsumerHandle md { get; }

src/client/spark/view/Spark/Ladder/LadderGridHost.cs

@@ -110,7 +110,7 @@ namespace mbts.spark {
       {
             var lgh = source as LadderGridHost;
             double width_adjustment = (double)(uint) e.NewValue - (double)(uint) e.OldValue;
-            lgh._adjust_column_width(lgh.columns.FirstOrDefault(c => c.column_type == ColumnType.Order), width_adjustment);    
+            lgh._adjust_column_width(lgh.columns.First(c => c.column_type == ColumnType.Order), width_adjustment);    
       }
 
       public uint bid_column_width
@@ -126,7 +126,7 @@ namespace mbts.spark {
       {
          var lgh = source as LadderGridHost;
          double width_adjustment = (double)(uint) e.NewValue - (double)(uint) e.OldValue;
-         lgh._adjust_column_width(lgh.columns.FirstOrDefault(c => c.column_type == ColumnType.Bid), width_adjustment);    
+         lgh._adjust_column_width(lgh.columns.First(c => c.column_type == ColumnType.Bid), width_adjustment);    
       }
 
       public uint price_column_width
@@ -142,7 +142,7 @@ namespace mbts.spark {
       {
             var lgh = source as LadderGridHost;
             double width_adjustment = (double)(uint) e.NewValue - (double)(uint) e.OldValue;
-            lgh._adjust_column_width(lgh.columns.FirstOrDefault(c => c.column_type == ColumnType.Price), width_adjustment);    
+            lgh._adjust_column_width(lgh.columns.First(c => c.column_type == ColumnType.Price), width_adjustment);    
       }
 
 
@@ -159,14 +159,14 @@ namespace mbts.spark {
       {
             var lgh = source as LadderGridHost;
             double width_adjustment = (double)(uint) e.NewValue - (double)(uint) e.OldValue;
-            lgh._adjust_column_width(lgh.columns.FirstOrDefault(c => c.column_type == ColumnType.Ask), width_adjustment);    
+            lgh._adjust_column_width(lgh.columns.First(c => c.column_type == ColumnType.Ask), width_adjustment);    
       }
 
       public static readonly DependencyProperty VolumeColumnWidthProperty =
          WPFHelper.create_dp<uint, LadderGridHost>(
             nameof(VolumeColumnWidth),
             FrameworkPropertyMetadataOptions.BindsTwoWayByDefault,
-            (o, oldval, newval) => o._adjust_column_width(o.columns.FirstOrDefault(c => c.column_type == ColumnType.Volume), (double)newval - (double)oldval),
+            (o, oldval, newval) => o._adjust_column_width(o.columns.First(c => c.column_type == ColumnType.Volume), (double)newval - (double)oldval),
             50u);
 
       public uint VolumeColumnWidth {
@@ -178,7 +178,7 @@ namespace mbts.spark {
          WPFHelper.create_dp<uint, LadderGridHost>(
             nameof(LastColumnWidth),
             FrameworkPropertyMetadataOptions.BindsTwoWayByDefault,
-            (o, oldval, newval) => o._adjust_column_width(o.columns.FirstOrDefault(c => c.column_type == ColumnType.Last), (double)newval - (double)oldval),
+            (o, oldval, newval) => o._adjust_column_width(o.columns.First(c => c.column_type == ColumnType.Last), (double)newval - (double)oldval),
             50u);
 
       public uint LastColumnWidth {
@@ -327,8 +327,6 @@ namespace mbts.spark {
       }
 
       void _adjust_column_width(Column column, double adjustment) {
-         if (column == null)
-            return;
          int new_col_width;
          switch (column.column_type) {
             case ColumnType.Order: new_col_width = (int)orders_column_width; break;
@@ -664,10 +662,22 @@ namespace mbts.spark {
          } else {
             init = false;
             prev_row_height = _settings.row_height;
+            // This is a general issue with default-settings:
+            // "Not every property in the settings object should be shared across windows."
+            // Column widths are one such property (they are changed from the actual Ladder window, not the settings dialog).
+            // Below is a hack which prevents overwriting current column widths; until we come up with a better solution this will suffice.
+            // TODO: fix this by moving such properties from the settings proto into a separate proto (or come up with a better workaround/solution)
+            copy_col_width(v => v.orders_column);
+            copy_col_width(v => v.bid_column);
+            copy_col_width(v => v.price_column);
+            copy_col_width(v => v.ask_column);
+            copy_col_width(v => v.volume_column.general);
+            copy_col_width(v => v.last_column);
+            void copy_col_width(Func<LadderSettings, LadderColumnSettings> select_col) =>
+               select_col(s).width = select_col(_settings).width;
          }
-         /// The viewmodel will always have the most up-to-date settings, so ignore <see cref="s"/> and just get it from <see cref="_vm"/>.
-         /// This is a workaround for the global settings-defaults issue: when defaults are updated, some properties are inadvertantly overwritten, such as column-widths, etc.
-         _settings = _vm.get_save_state();
+         //
+         _settings = s;
          _bid_vis_tup.glyph_context = glyph_context(s.bid_column);
          _ask_vis_tup.glyph_context = glyph_context(s.ask_column);
          _price_vis_tup.glyph_context = glyph_context(s.price_column);
@@ -688,19 +698,18 @@ namespace mbts.spark {
          // Renders the grid lines and column backgrounds
          _visualHost.AddVisualChild(_gridvis);
          // Orders column
-         SetCurrentValue(orders_column_widthProperty, s.orders_column.width);
+         orders_column_width = s.orders_column.width;
          _ordershost.update_settings(_price_list, _price_comparer, s.row_height, s.orders_column);
          foreach (var col in columns) {
             if (col.visual_tup != null) {
                if (col.settings.show) {
                   _visualHost.AddVisualChild(col.visual_tup.visual);
                   switch (col.column_type) {
-                     case ColumnType.Bid: SetCurrentValue(bid_column_widthProperty, col.settings.width); break;
-                     case ColumnType.Price: SetCurrentValue(price_column_widthProperty, col.settings.width); break;
-                     case ColumnType.Ask: SetCurrentValue(ask_column_widthProperty, col.settings.width); break;
-                     case ColumnType.Last: SetCurrentValue(LastColumnWidthProperty, col.settings.width); break;
+                     case ColumnType.Bid: bid_column_width = col.settings.width; break;
+                     case ColumnType.Price: price_column_width = col.settings.width; break;
+                     case ColumnType.Ask: ask_column_width = col.settings.width; break;
                      case ColumnType.Volume:
-                        SetCurrentValue(VolumeColumnWidthProperty, col.settings.width);
+                        VolumeColumnWidth = col.settings.width;
                         _volume_container.mark_dirty();
                         break;
                   }

src/client/spark/view/Spark/Ladder/LadderSettings.cs

@@ -8,7 +8,6 @@ using System.Windows;
 using bts.proto;
 using mbts.ui.spark;
 using System.Collections.Generic;
-using System.ComponentModel;
 
 namespace mbts.spark {
    // IMPORTANT: BE SURE TO ADD ANY NEW PROPERTIES TO THE EQUALS FUNCTION!
@@ -180,32 +179,6 @@ namespace mbts.spark {
    // Settings are liable to change in the near future, so an internal IEqualityComparer simply won't do as it risks becoming out of date.
    [ProtoContract]
    class LadderSettings : IEquatable<LadderSettings> {
-      const double _default_order_entry_panel_width = 110.0;
-
-      // Default proto constructor
-      // Added this so that when clients upgrade to a version with new settings,
-      // the default values are applied correctly; thx to John for this tip
-      public LadderSettings() {
-         day_visible = true;
-         high_visible = true;
-         low_visible = true;
-         open_visible = true;
-         settle_visible = true;
-         volume_visible = true;
-         account_visible = true;
-         tif_visible = true;
-         order_type_visible = true;
-         account_position_visible = true;
-         size_presets_visible = true;
-         cancel_buys_visible = true;
-         cancel_sells_visible = true;
-         cancel_all_visible = true;
-         auto_clear_visible = true;
-         clear_visible = true;
-         order_entry_panel_initial_width = _default_order_entry_panel_width;
-         use_defaults = true;
-      }
-
       public static LadderSettings Default {
          get {
             LadderColumnSettings
@@ -243,24 +216,7 @@ namespace mbts.spark {
                session_high_line = new LadderMarketBreakLine { color = BTSColors.MarketData.session_high, show = true },
                session_range_fill = new LadderMarketBreakLine { color = BTSColors.MarketData.session_range_fill, show = true },
                alternate_order_size = 0,
-               size_display = MarketSizeDisplay.get_default(),
-               day_visible = true,
-               high_visible = true,
-               low_visible = true,
-               open_visible = true,
-               settle_visible = true,
-               volume_visible = true,
-               account_visible = true,
-               tif_visible = true,
-               order_type_visible = true,
-               account_position_visible = true,
-               size_presets_visible = true,
-               cancel_buys_visible = true,
-               cancel_sells_visible = true,
-               cancel_all_visible = true,
-               auto_clear_visible = true,
-               clear_visible = true,
-               order_entry_panel_initial_width = _default_order_entry_panel_width
+               size_display = MarketSizeDisplay.get_default()
             };
          }
       }
@@ -322,45 +278,6 @@ namespace mbts.spark {
       [ProtoMember(26)]
       public MarketSizeDisplay size_display { get; set; }
 
-      // order entry panel items show/hide
-      [ProtoMember(27), DefaultValue(true)]
-      public bool day_visible { get; set; }
-      [ProtoMember(28), DefaultValue(true)]
-      public bool high_visible { get; set; }
-      [ProtoMember(29), DefaultValue(true)]
-      public bool low_visible { get; set; }
-      [ProtoMember(30), DefaultValue(true)]
-      public bool open_visible { get; set; }
-      [ProtoMember(31), DefaultValue(true)]
-      public bool settle_visible { get; set; }
-      [ProtoMember(32), DefaultValue(true)]
-      public bool volume_visible { get; set; }
-      [ProtoMember(33), DefaultValue(true)]
-      public bool account_visible { get; set; }
-      [ProtoMember(34), DefaultValue(true)]
-      public bool tif_visible { get; set; }
-      [ProtoMember(35), DefaultValue(true)]
-      public bool order_type_visible { get; set; }
-      [ProtoMember(36), DefaultValue(true)]
-      public bool account_position_visible { get; set; }
-      [ProtoMember(37), DefaultValue(true)]
-      public bool size_presets_visible { get; set; }
-      [ProtoMember(38), DefaultValue(true)]
-      public bool cancel_buys_visible { get; set; }
-      [ProtoMember(39), DefaultValue(true)]
-      public bool cancel_sells_visible { get; set; }
-      [ProtoMember(40), DefaultValue(true)]
-      public bool cancel_all_visible { get; set; }
-      [ProtoMember(41), DefaultValue(true)]
-      public bool auto_clear_visible { get; set; }
-      [ProtoMember(42), DefaultValue(true)]
-      public bool clear_visible { get; set; }
-      //
-      [ProtoMember(43), DefaultValue(_default_order_entry_panel_width)]
-      public double order_entry_panel_initial_width { get; set; }
-      [ProtoMember(44), DefaultValue(true)]
-      public bool use_defaults { get; set; }
-
       public bool Equals(LadderSettings other) =>
          other != null &&
          row_height == other.row_height &&
@@ -385,25 +302,7 @@ namespace mbts.spark {
          session_high_line.safe_equals(other.session_high_line) &&
          session_range_fill.safe_equals(other.session_range_fill) &&
          volume_column.safe_equals(other.volume_column) &&
-         size_display.safe_equals(other.size_display) &&
-         day_visible == other.day_visible &&
-         high_visible == other.high_visible &&
-         low_visible == other.low_visible &&
-         open_visible == other.open_visible &&
-         settle_visible == other.settle_visible &&
-         volume_visible == other.volume_visible &&
-         account_visible == other.account_visible &&
-         tif_visible == other.tif_visible &&
-         order_type_visible == other.order_type_visible &&
-         account_position_visible == other.account_position_visible &&
-         size_presets_visible == other.size_presets_visible &&
-         cancel_buys_visible == other.cancel_buys_visible &&
-         cancel_sells_visible == other.cancel_sells_visible &&
-         cancel_all_visible == other.cancel_all_visible &&
-         auto_clear_visible == other.auto_clear_visible &&
-         clear_visible == other.clear_visible &&
-         order_entry_panel_initial_width == other.order_entry_panel_initial_width &&
-         use_defaults == other.use_defaults;
+         size_display.safe_equals(other.size_display);
 
       public override bool Equals(object obj) => Equals(obj as LadderSettings);
 
@@ -431,43 +330,7 @@ namespace mbts.spark {
             EqualityComparer.safe_get_hash_code(session_high_line),
             EqualityComparer.safe_get_hash_code(session_range_fill),
             EqualityComparer.safe_get_hash_code(volume_column),
-            EqualityComparer.safe_get_hash_code(size_display),
-            day_visible.GetHashCode(),
-            high_visible.GetHashCode(),
-            low_visible.GetHashCode(),
-            open_visible.GetHashCode(),
-            settle_visible.GetHashCode(),
-            volume_visible.GetHashCode(),
-            account_visible.GetHashCode(),
-            tif_visible.GetHashCode(),
-            order_type_visible.GetHashCode(),
-            account_position_visible.GetHashCode(),
-            size_presets_visible.GetHashCode(),
-            cancel_buys_visible.GetHashCode(),
-            cancel_sells_visible.GetHashCode(),
-            cancel_all_visible.GetHashCode(),
-            auto_clear_visible.GetHashCode(),
-            clear_visible.GetHashCode(),
-            order_entry_panel_initial_width.GetHashCode(),
-            use_defaults.GetHashCode());
-
-      public struct WindowSpecifics {
-         public readonly double order_entry_panel_width;
-         public readonly uint orders_column_width, bid_column_width, price_column_width, ask_column_width, last_column_width, volume_column_width;
-
-         public WindowSpecifics(LadderSettings s)
-            : this(s.order_entry_panel_initial_width, s.orders_column.width, s.bid_column.width, s.price_column.width, s.ask_column.width, s.last_column.width, s.volume_column.general.width) { }
-
-         public WindowSpecifics(double order_entry_panel_width, uint orders_column_width, uint bid_column_width, uint price_column_width, uint ask_column_width, uint last_column_width, uint volume_column_width) {
-            this.order_entry_panel_width = order_entry_panel_width;
-            this.orders_column_width = orders_column_width;
-            this.bid_column_width = bid_column_width;
-            this.price_column_width = price_column_width;
-            this.ask_column_width = ask_column_width;
-            this.last_column_width = last_column_width;
-            this.volume_column_width = volume_column_width;
-         }
-      }
+            EqualityComparer.safe_get_hash_code(size_display));
    }
 
    static class LadderSettingsExtensions {
@@ -482,21 +345,5 @@ namespace mbts.spark {
 
       public static IEnumerable<LadderColumnSettings> visible_columns(this LadderSettings settings) =>
          settings.all_columns().Where(c => c.show);
-
-      public static LadderSettings.WindowSpecifics get_window_specifics(this LadderSettings settings) =>
-         new LadderSettings.WindowSpecifics(settings);
-
-      public static LadderSettings with_window_specifics(this LadderSettings source, LadderSettings.WindowSpecifics specifics) {
-         // must use clone bc we are applying settings that are specific to each individual LadderWindow, and should not be shared among all ladders (even if it is default settings).
-         source = SparkUtils.clone_proto(source);
-         source.order_entry_panel_initial_width = specifics.order_entry_panel_width;
-         source.orders_column.width = specifics.orders_column_width;
-         source.bid_column.width = specifics.bid_column_width;
-         source.price_column.width = specifics.price_column_width;
-         source.ask_column.width = specifics.ask_column_width;
-         source.last_column.width = specifics.last_column_width;
-         source.volume_column.general.width = specifics.volume_column_width;
-         return source;
-      }
    }
 }

src/client/spark/view/Spark/Ladder/LadderSettingsVM.cs

@@ -17,7 +17,6 @@ namespace mbts.spark {
       readonly Property<LadderSettings.BadgeStyle> _badge_style;
       readonly Property<uint> _alternate_order_size;
       readonly DelegateProperty<double> _row_height;
-      Func<double> _get_order_entry_panel_width;
       LadderSettings _current;
       LadderColumnSettingsVM
          _orders = new OrdersColumnSettingsVM(),
@@ -33,33 +32,14 @@ namespace mbts.spark {
          _session_low_mkt_line = new LadderMarketBreakLineVM("Session Low", 2, 1),
          _session_high_mkt_line = new LadderMarketBreakLineVM("Session High", 2, 1, false),
          _session_range_fill = new LadderMarketBreakLineVM("Session Range", 0, 0, false);
-      OrderEntryPanelItemVisibilityVM
-         _day_visibility = new OrderEntryPanelItemVisibilityVM("Day Change"),
-         _high_visibility = new OrderEntryPanelItemVisibilityVM("High"),
-         _low_visibility = new OrderEntryPanelItemVisibilityVM("Low"),
-         _open_visibility = new OrderEntryPanelItemVisibilityVM("Open"),
-         _settle_visibility = new OrderEntryPanelItemVisibilityVM("Settle"),
-         _volume_visibility = new OrderEntryPanelItemVisibilityVM("Volume"),
-         _account_visibility = new OrderEntryPanelItemVisibilityVM("Account"),
-         _tif_visibility = new OrderEntryPanelItemVisibilityVM("Time In Force"),
-         _order_type_visibility = new OrderEntryPanelItemVisibilityVM("Order Type"),
-         _account_position_visibility = new OrderEntryPanelItemVisibilityVM("Account Position"),
-         _size_preset_visibility = new OrderEntryPanelItemVisibilityVM("Size Presets"),
-         _auto_clear_visibility = new OrderEntryPanelItemVisibilityVM("Auto Clear"),
-         _clear_visibility = new OrderEntryPanelItemVisibilityVM("Clear"),
-         _cancel_buys_visibility = new OrderEntryPanelItemVisibilityVM("Cancel Buys"),
-         _cancel_sells_visibility = new OrderEntryPanelItemVisibilityVM("Cancel Sells"),
-         _cancel_all_visibility = new OrderEntryPanelItemVisibilityVM("Cancel All");
 
-      public LadderSettingsVM(bool is_synthetic, Func<double> get_left_panel_width) {
+      public LadderSettingsVM(bool is_synthetic) {
          this.is_synthetic = is_synthetic;
          size_presets = new ObservableEditableCollection<IMutableValueWrapperVM<uint>>(_create_size_preset_vm);
          size_presets.CollectionChanged += (s, e) => RaisePropertyChanged(nameof(size_presets_text));
          _badge_style = backing<LadderSettings.BadgeStyle>(nameof(badge_style));
          _badge_style.Changed += _badge_style_Changed;
          _alternate_order_size = backing<uint>(nameof(alternate_order_size));
-         _get_order_entry_panel_width = get_left_panel_width;
-         // be sure to initialize backings before this measure_row_height call
          _row_height = backing(nameof(row_height), () => LadderRender.measure_row_height(get_result()));
          _orders.font_size_changed += on_font_size_changed;
          _bid.font_size_changed += on_font_size_changed;
@@ -128,27 +108,6 @@ namespace mbts.spark {
          }
       }
 
-      public IEnumerable<OrderEntryPanelItemVisibilityVM> order_entry_panel_items {
-         get {
-            yield return _day_visibility;
-            yield return _high_visibility;
-            yield return _low_visibility;
-            yield return _open_visibility;
-            yield return _settle_visibility;
-            yield return _volume_visibility;
-            yield return _account_visibility;
-            yield return _tif_visibility;
-            yield return _order_type_visibility;
-            yield return _account_position_visibility;
-            yield return _size_preset_visibility;
-            yield return _clear_visibility;
-            yield return _auto_clear_visibility;
-            yield return _cancel_buys_visibility;
-            yield return _cancel_sells_visibility;
-            yield return _cancel_all_visibility;
-         }
-      }
-
       public string key => GridOrLadderSettingsSelectorKeys.Ladder;
 
       public void load(LadderSettings settings) {
@@ -169,26 +128,6 @@ namespace mbts.spark {
          _session_low_mkt_line.assign_from(_current.session_low_line);
          _session_high_mkt_line.assign_from(_current.session_high_line);
          _session_range_fill.assign_from(_current.session_range_fill);
-         _day_visibility.show = settings.day_visible;
-         _high_visibility.show = settings.high_visible;
-         _low_visibility.show = settings.low_visible;
-         _open_visibility.show = settings.open_visible;
-         _settle_visibility.show = settings.settle_visible;
-         _volume_visibility.show = settings.volume_visible;
-         _account_visibility.show = settings.account_visible;
-         _tif_visibility.show = settings.tif_visible;
-         _order_type_visibility.show = settings.order_type_visible;
-         _account_position_visibility.show = settings.account_position_visible;
-         _size_preset_visibility.show = settings.size_presets_visible;
-         _clear_visibility.show = settings.clear_visible;
-         _auto_clear_visibility.show = settings.auto_clear_visible;
-         _cancel_buys_visibility.show = settings.cancel_buys_visible;
-         _cancel_sells_visibility.show = settings.cancel_sells_visible;
-         _cancel_all_visibility.show = settings.cancel_all_visible;
-         // do not read the order entry panel width here from settings
-         // because it could have changed from the UI
-         // there might be a better way of doing this, like how
-         // does the widths of the ladder grid column pull this off?
       }
 
       public LadderSettings get_result() {
@@ -210,23 +149,6 @@ namespace mbts.spark {
          clone.session_high_line = _session_high_mkt_line.get_settings();
          clone.session_range_fill = _session_range_fill.get_settings();
          clone.row_height = LadderRender.measure_row_height(clone);
-         clone.day_visible = _day_visibility.show;
-         clone.high_visible = _high_visibility.show;
-         clone.low_visible = _low_visibility.show;
-         clone.open_visible = _open_visibility.show;
-         clone.settle_visible = _settle_visibility.show;
-         clone.volume_visible = _volume_visibility.show;
-         clone.account_visible = _account_visibility.show;
-         clone.tif_visible = _tif_visibility.show;
-         clone.order_type_visible = _order_type_visibility.show;
-         clone.account_position_visible = _account_position_visibility.show;
-         clone.size_presets_visible = _size_preset_visibility.show;
-         clone.auto_clear_visible = _auto_clear_visibility.show;
-         clone.clear_visible = _clear_visibility.show;
-         clone.cancel_buys_visible = _cancel_buys_visibility.show;
-         clone.cancel_sells_visible = _cancel_sells_visibility.show;
-         clone.cancel_all_visible = _cancel_all_visibility.show;
-         clone.order_entry_panel_initial_width = _get_order_entry_panel_width();
          return clone;
       }
 
@@ -354,19 +276,6 @@ namespace mbts.spark {
             };
       }
 
-      public class OrderEntryPanelItemVisibilityVM : ViewModelBase {
-         readonly Property<bool> _show;
-
-         public OrderEntryPanelItemVisibilityVM(string name) {
-            this.name = name;
-            _show = backing(nameof(show), true);
-         }
-
-         public string name { get; }
-         public bool show { get => _show.Value; set => _show.Value = value; }
-
-      }
-
       public class VolumeColumnVM : ViewModelBase {
          readonly DelegateProperty<bool> _any_visible;
          public VolumeColumnVM() {

src/client/spark/view/Spark/Ladder/LadderSettingsView.xaml

@@ -140,7 +140,6 @@
           <Setter Property="VerticalContentAlignment" Value="Top"/>
         </Style>
       </TabControl.ItemContainerStyle>
-      <!-- Fonts and Colors Tab -->
       <TabItem Header="Fonts and Colors" VerticalAlignment="Stretch">
         <Grid>
           <Grid.RowDefinitions>
@@ -233,7 +232,6 @@
           </WrapPanel>
         </Grid>
       </TabItem>
-      <!-- Market Break Lines Tab -->
       <TabItem Header="Market Break Lines" VerticalAlignment="Stretch">
         <Grid>
           <Grid.RowDefinitions>
@@ -352,7 +350,6 @@
           </ItemsControl>
         </Grid>
       </TabItem>
-      <!-- Market Size Tab -->
       <TabItem Header="Market Size Display">
         <Grid DataContext="{Binding size_display}">
           <Grid.RowDefinitions>
@@ -443,7 +440,6 @@
           </GroupBox>
         </Grid>
       </TabItem>
-      <!-- Volume Display Tab -->
       <TabItem Header="Volume Display" VerticalAlignment="Stretch" Visibility="{Binding is_synthetic, Converter={StaticResource FalseVisibilityConverter}}">
         <Grid>
           <Grid.RowDefinitions>
@@ -561,7 +557,6 @@
           </WrapPanel>
         </Grid>
       </TabItem>
-      <!-- Size Presets Tab-->
       <TabItem Header="Size Presets" VerticalAlignment="Stretch">
         <Grid HorizontalAlignment="Left">
           <Grid.RowDefinitions>
@@ -597,63 +592,6 @@
           </Button>
         </Grid>
       </TabItem>
-      <!-- Order Entry Panel Item Control Visbility -->
-      <TabItem Header="Left Panel" VerticalAlignment="Stretch">
-        <Grid HorizontalAlignment="Left" Grid.IsSharedSizeScope="True">
-          <Grid.RowDefinitions>
-            <RowDefinition Height="*"/>
-            <RowDefinition Height="{Binding Source={StaticResource VerticalPush}}"/>
-            <RowDefinition Height="Auto"/>
-          </Grid.RowDefinitions>
-          <ItemsControl Grid.Row="0" ItemsSource="{Binding order_entry_panel_items}" Grid.IsSharedSizeScope="True">
-            <ItemsControl.ItemsPanel>
-              <ItemsPanelTemplate>
-                <UniformGrid Columns="2" Focusable="False"/>
-              </ItemsPanelTemplate>
-            </ItemsControl.ItemsPanel>
-            <ItemsControl.ItemTemplate>
-              <DataTemplate>
-                <Grid>
-                  <Grid.ColumnDefinitions>
-                    <ColumnDefinition Width="Auto" SharedSizeGroup="A" />
-                  </Grid.ColumnDefinitions>
-                  <Border Margin="0,5,5,0">
-                    <CheckBox Content="{Binding name}" FlowDirection="RightToLeft" IsChecked="{Binding show}">
-                      <CheckBox.Resources>
-                        <Style TargetType="{x:Type Path}">
-                          <Setter Property="FlowDirection" Value="LeftToRight" />
-                        </Style>
-                      </CheckBox.Resources>
-                    </CheckBox>
-                  </Border>
-                </Grid>
-              </DataTemplate>
-            </ItemsControl.ItemTemplate>
-          </ItemsControl>
-          <!--
-          <ItemsControl ItemsSource="{Binding order_entry_panel_items}">
-            <ItemsControl.ItemsPanel>
-              <ItemsPanelTemplate>
-                <UniformGrid Columns="2" Focusable="False"/>
-              </ItemsPanelTemplate>
-            </ItemsControl.ItemsPanel>
-            <ItemsControl.ItemTemplate>
-              <DataTemplate>
-                <Grid>
-                  <Grid.ColumnDefinitions>
-                    <ColumnDefinition Width="Auto"/>
-                    <ColumnDefinition Width="52"/>
-                  </Grid.ColumnDefinitions>
-                  <TextBlock Grid.Column="0" VerticalAlignment="Center" Text="{Binding name}"/>
-                  <CheckBox Grid.Column="1" HorizontalAlignment="Center" VerticalAlignment="Center" IsChecked="{Binding show}"/>
-                </Grid>
-              </DataTemplate>
-            </ItemsControl.ItemTemplate>
-          </ItemsControl>
-          -->
-        </Grid>
-      </TabItem>
-
     </TabControl>
   </Grid>
 </UserControl>

src/client/spark/view/Spark/Ladder/LadderVM.cs

@@ -20,23 +20,18 @@ using System.Windows.Media;
 
 namespace mbts.spark {
    class LadderVM : DisposableViewModelBase, IWillLetYouKnowWhenIDispose, IKeyListener {
-      static readonly Brush _transparent_status_brush = BTSColors.UI.transparent;
-
       readonly IBus _bus;
       readonly LadderModel _model;
-      readonly Property<double> _high, _low, _change, _open, _settlement, _position, _left_panel_width;
-      readonly Property<uint> _orders_column_width, _bid_column_width, _price_column_width, _ask_column_width, _last_column_width, _volume_column_width;
+      readonly Property<double> _high, _low, _change, _open, _settlement, _position;
       readonly Property<ulong> _volume;
-      readonly Property<bool> _auto_clear, _show_day, _show_high, _show_low, _show_open, _show_settle, _show_volume, _show_account,
-         _show_tif, _show_order_type, _show_account_position, _show_size_preset, _show_auto_clear, _show_clear, _show_cancel_buys,
-         _show_cancel_sells, _show_cancel_all;
+      readonly Property<bool> _auto_clear;
       readonly Property<Brush> _bid_background, _bid_foreground, _ask_background, _ask_foreground;
       IWindowSettingsManager<LadderSettings> _settings_manager;
       ulong _misc_time = 0;
       IEqualityComparer<double> _equality;
       Dictionary<double, LadderOrderVM> _slot_price_vs_orders;
 
-      public LadderVM(IBus bus, LadderModel model, IWindowSettingsManager<LadderSettings> settings_manager, LadderSettings init) {
+      public LadderVM(IBus bus, LadderModel model, IWindowSettingsManager<LadderSettings> settings_manager) {
          _bus = bus;
          _model = model;
          _equality = TickUtils.tick_equality_comparer(_model.security.ContractSpec);
@@ -49,22 +44,13 @@ namespace mbts.spark {
          _change = backing(nameof(change), double.NaN);
          _open = backing(nameof(open), double.NaN);
          _settlement = backing(nameof(settlement), double.NaN);
-         // init window-specific settings: these properties should not observe settings updates; they are independent from the settings dialog, so it doesn't make sense that the settings dialog should cause them to change.
-         var ws = init.get_window_specifics();
-         _orders_column_width = backing(nameof(orders_column_width), ws.orders_column_width);
-         _bid_column_width = backing(nameof(bid_column_width), ws.bid_column_width);
-         _price_column_width = backing(nameof(price_column_width), ws.price_column_width);
-         _ask_column_width = backing(nameof(ask_column_width), ws.ask_column_width);
-         _last_column_width = backing(nameof(last_column_width), ws.last_column_width);
-         _volume_column_width = backing(nameof(volume_column_width), ws.volume_column_width);
-         _left_panel_width = backing(nameof(left_panel_width), ws.order_entry_panel_width);
          // remember: account_positions needs to subscribe to selected_account; both the sub and account_positions need to be disposed
          var account_positions = new KeyDrivenObservableSeries<string, double>(a => _model.order_gateway.security_total_position(security.Id, a));
          _position = backing(nameof(position), account_positions.OnDispatcher());
          _volume = backing(nameof(volume), 0ul);
          _auto_clear = backing(nameof(auto_clear), false);
          _status = backing(nameof(status_message), "");
-         _status_brush = backing(nameof(status_background), _transparent_status_brush);
+         _status_brush = backing(nameof(status_background), (Brush)BTSColors.UI.spark_menu_background);
          // brush props
          _bid_background = brush_prop(nameof(bid_background), s => s.bid_column.background.rgb_brush());
          _bid_foreground = brush_prop(nameof(bid_foreground), s => s.bid_column.foreground.rgb_brush());
@@ -72,27 +58,6 @@ namespace mbts.spark {
          _ask_foreground = brush_prop(nameof(ask_foreground), s => s.ask_column.foreground.rgb_brush());
          Property<Brush> brush_prop(string name, Func<LadderSettings, Brush> get_brush) =>
             backing(name, settings_manager.subscribe_current().OnDispatcher(), get_brush);
-         // bool visibility props
-         _show_day = bool_prop(nameof(show_day), s => s.day_visible);
-         _show_high = bool_prop(nameof(show_high), s => s.high_visible);
-         _show_low = bool_prop(nameof(show_low), s => s.low_visible);
-         _show_open = bool_prop(nameof(show_open), s => s.open_visible);
-         _show_settle = bool_prop(nameof(show_settle), s => s.settle_visible);
-         _show_volume = bool_prop(nameof(show_volume), s => s.volume_visible);
-         _show_account = bool_prop(nameof(show_account), s => s.account_visible);
-         _show_tif = bool_prop(nameof(show_tif), s => s.tif_visible);
-         _show_order_type = bool_prop(nameof(show_order_type), s => s.order_type_visible);
-         _show_account_position = bool_prop(nameof(show_account_position), s => s.account_position_visible);
-         _show_size_preset = bool_prop(nameof(show_size_preset), s => s.size_presets_visible);
-         _show_auto_clear = bool_prop(nameof(show_auto_clear), s => s.auto_clear_visible);
-         _show_clear = bool_prop(nameof(show_clear), s => s.clear_visible);
-         _show_cancel_buys = bool_prop(nameof(show_cancel_buys), s => s.cancel_buys_visible);
-         _show_cancel_sells = bool_prop(nameof(show_cancel_sells), s => s.cancel_sells_visible);
-         _show_cancel_all = bool_prop(nameof(show_cancel_all), s => s.cancel_all_visible);
-
-         Property<bool> bool_prop(string name, Func<LadderSettings, bool> get_bool) =>
-            backing(name, settings_manager.subscribe_current().OnDispatcher(), get_bool);
-
          //
          position_click_command = new DelegateCommand(() => size_calculator.value = DoubleHelper.safe_convert_to_uint(Math.Abs(position)));
          send_order_command = new DelegateCommand<(double price, Side? side, bool size_from_alternate)>(send_order);
@@ -102,11 +67,11 @@ namespace mbts.spark {
          settings_command = new DialogCommand(() => {
          var vm = new WindowSettingsDialogVM<LadderSettings, LadderSettingsVM>(
                settings_manager, s => s.ladder, UserSettings.FileExtensions.LADDER,
-               s => { 
-                  var svm = new LadderSettingsVM(security.ContractCase == Security.ContractOneofCase.Synthetic, () => left_panel_width);
+               s => {
+                  var svm = new LadderSettingsVM(security.ContractCase == Security.ContractOneofCase.Synthetic);
                   svm.load(s);
                   return svm;
-               }, s => s.with_window_specifics(get_save_state().get_window_specifics()));
+               });
             var dialog = new ui.spark.SimpleGridSettingsDialog {
                DataContext = vm,
                Title = $"Ladder Settings - '{security}'"
@@ -237,7 +202,7 @@ namespace mbts.spark {
             await Task.Delay(TimeSpan.FromSeconds(5));
             if (_status.Value == a.message) {
                _status.set("");
-               _status_brush.set(_transparent_status_brush);
+               _status_brush.set(BTSColors.UI.spark_menu_background);
             }
          }
       }
@@ -268,37 +233,16 @@ namespace mbts.spark {
       public Brush ask_background => _ask_background.Value;
       public Brush ask_foreground => _ask_foreground.Value;
       public DialogCommand settings_command { get; }
-      public bool show_day { get => _show_day.Value; set => _show_day.set(value); }
-      public bool show_high { get => _show_high.Value; set => _show_high.set(value); }
-      public bool show_low { get => _show_low.Value; set => _show_low.set(value); }
-      public bool show_open { get => _show_open.Value; set => _show_open.set(value); }
-      public bool show_settle { get => _show_settle.Value; set => _show_settle.set(value); }
-      public bool show_volume { get => _show_volume.Value; set => _show_volume.set(value); }
-      public bool show_account { get => _show_account.Value; set => _show_account.set(value); }
-      public bool show_tif { get => _show_tif.Value; set => _show_tif.set(value); }
-      public bool show_order_type { get => _show_order_type.Value; set => _show_order_type.set(value); }
-      public bool show_account_position { get => _show_account_position.Value; set => _show_account_position.set(value); }
-      public bool show_size_preset { get => _show_size_preset.Value; set => _show_size_preset.set(value); }
-      public bool show_auto_clear { get => _show_auto_clear.Value; set => _show_auto_clear.set(value); }
-      public bool show_clear { get => _show_clear.Value; set => _show_clear.set(value); }
-      public bool show_cancel_buys { get => _show_cancel_buys.Value; set => _show_cancel_buys.set(value); }
-      public bool show_cancel_sells { get => _show_cancel_sells.Value; set => _show_cancel_sells.set(value); }
-      public bool show_cancel_all { get => _show_cancel_all.Value; set => _show_cancel_all.set(value); }
-      public double left_panel_width { get => _left_panel_width.Value; set => _left_panel_width.set(value); }
-      public uint orders_column_width { get => _orders_column_width.Value; set => _orders_column_width.set(value); }
-      public uint bid_column_width { get => _bid_column_width.Value; set => _bid_column_width.set(value); }
-      public uint price_column_width { get => _price_column_width.Value; set => _price_column_width.set(value); }
-      public uint ask_column_width { get => _ask_column_width.Value; set => _ask_column_width.set(value); }
-      public uint last_column_width { get => _last_column_width.Value; set => _last_column_width.set(value); }
-      public uint volume_column_width { get => _volume_column_width.Value; set => _volume_column_width.set(value); }
       internal LadderModel model => _model;
       internal IObservable<LadderSettings> settings => _settings_manager.subscribe_current();
 
       public LadderSettings get_save_state() {
-         var state = _settings_manager.current_settings();
-         state = state.with_window_specifics(new LadderSettings.WindowSpecifics(left_panel_width, orders_column_width, bid_column_width, price_column_width, ask_column_width, last_column_width, volume_column_width));
-         state.use_defaults = _settings_manager.instance_settings == null;
-         return state;
+         // TODO: save an actual OrderDefinition to preserve order params.
+         if (_settings_manager.instance_settings == null)
+            return null;
+         var settings_clone = SparkUtils.clone_proto(_settings_manager.instance_settings);
+         settings_clone.default_order_type = order_properties_vm.selected_order_type?.order_type ?? settings_clone.default_order_type;
+         return settings_clone;
       }
 
       public void key_down(KeyEventArgs e) {
@@ -355,7 +299,6 @@ namespace mbts.spark {
                _change.set(mid - md.settlement_price);
             }
             _volume.set(volume);
-            _misc_time = _model.md.misc_timestamp;
          }
       }
 

src/client/spark/view/Spark/Ladder/LadderView.xaml

@@ -1,19 +1,16 @@
-<UserControl
-  x:Class="mbts.spark.LadderView"
-  x:ClassModifier="internal"
-  xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
-  xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
-  xmlns:mbts="clr-namespace:mbts.ui"
-  xmlns:controls="clr-namespace:mbts.ui.controls"
-  xmlns:spark="clr-namespace:mbts.spark"
-  xmlns:uag="clr-namespace:mbts.ui.spark.algo"
-  xmlns:converters="clr-namespace:mbts.ui.converters"
-  xmlns:txt="clr-namespace:mbts.ui.utility.text"
-  Height="600"
-  Width="420"
-  Background="{x:Static mbts:BTSColors.common_background_brush}"
-  TextElement.FontSize="{x:Static mbts:FontSizes.Small}"
->
+<UserControl x:Class="mbts.spark.LadderView"
+             xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
+             xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
+             xmlns:mbts="clr-namespace:mbts.ui"
+             xmlns:controls="clr-namespace:mbts.ui.controls"
+             xmlns:spark="clr-namespace:mbts.spark"
+             xmlns:uag="clr-namespace:mbts.ui.spark.algo"
+             xmlns:converters="clr-namespace:mbts.ui.converters"
+             xmlns:txt="clr-namespace:mbts.ui.utility.text"
+             Height="600"
+             Width="420"
+             Background="{x:Static mbts:BTSColors.common_background_brush}"
+             TextElement.FontSize="{x:Static mbts:FontSizes.Small}">
   <UserControl.Resources>
     <ResourceDictionary>
       <ResourceDictionary.MergedDictionaries>
@@ -30,13 +27,6 @@
       </converters:BooleanToObjectConverter>
       <converters:EnumCharacterCasingConverter x:Key="casing_converter"/>
       <converters:DoubleTextConverter x:Key="PositionTextConverter" Format="0" NaNText="-"/>
-      <converters:GridLengthConverter x:Key="GridLengthPixelConverter" UnitType="Pixel" />
-      <converters:ResponsiveTextConverter x:Key="DayTextConverter" LongText="Day:" ShortText="D:" Threshold="99" />
-      <converters:ResponsiveTextConverter x:Key="HighTextConverter" LongText="High:" ShortText="H:" Threshold="99"  />
-      <converters:ResponsiveTextConverter x:Key="LowTextConverter" LongText="Low:" ShortText="L:" Threshold="99" />
-      <converters:ResponsiveTextConverter x:Key="OpenTextConverter" LongText="Open:" ShortText="O:" Threshold="99" />
-      <converters:ResponsiveTextConverter x:Key="SettleTextConverter" LongText="Settle:" ShortText="S:" Threshold="99" />
-      <converters:ResponsiveTextConverter x:Key="VolumeTextConverter" LongText="Volume:" ShortText="V:" Threshold="99" />
       <Style TargetType="GridViewColumnHeader">
         <Setter Property="Background" Value="{mbts:BTSColor header}"/>
         <Setter Property="Foreground" Value="{mbts:BTSColor header_text}"/>
@@ -44,7 +34,6 @@
         <Setter Property="BorderThickness" Value="0,0,1,0"/>
         <Setter Property="HorizontalContentAlignment" Value="Center" />
         <Setter Property="Height" Value="23" />
-        <Setter Property="MinWidth" Value="30" />
         <Setter Property="Template">
           <Setter.Value>
             <ControlTemplate TargetType="{x:Type GridViewColumnHeader}">
@@ -68,8 +57,8 @@
                     </Border>
                   </Grid>
                 </Border>
-                <Canvas MinWidth="{Binding RelativeSource={RelativeSource TemplatedParent}, Path=MinWidth, Mode=OneTime}">
-                  <Thumb x:Name="PART_HeaderGripper" DragDelta="Thumb_DragDelta">
+                <Canvas>
+                  <Thumb x:Name="PART_HeaderGripper">
                     <Thumb.Style>
                       <Style TargetType="{x:Type Thumb}">
                         <Setter Property="Canvas.Right" Value="-9"/>
@@ -134,7 +123,7 @@
       <Style x:Key="LadderMenuButton" TargetType="Button">
         <Setter Property="BorderThickness" Value="1"/>
         <Setter Property="HorizontalAlignment" Value="Stretch"/>
-        <Setter Property="Margin" Value="2,0,2,5"/>
+        <Setter Property="Margin" Value="5,0,5,5"/>
         <Setter Property="Padding" Value="2"/>
       </Style>
       <Style x:Key="CancelBuys" TargetType="Button" BasedOn="{StaticResource LadderMenuButton}">
@@ -255,7 +244,8 @@
                 Background="{mbts:BTSColor UI.Button.normal_background}"
                 Padding="{TemplateBinding Padding}"
                 SnapsToDevicePixels="True"
-                Margin="2,0,2,10">
+                VerticalAlignment="Bottom"
+                Margin="0,0,0,12">
                 <ContentPresenter 
                   HorizontalAlignment="Center"
                   VerticalAlignment="Center"
@@ -277,147 +267,104 @@
   </UserControl.Resources>
   <AdornerDecorator>
     <Grid x:Name="_root">
-      <Grid.ColumnDefinitions>
-        <ColumnDefinition Width="{Binding left_panel_width, Mode=TwoWay, Converter={StaticResource GridLengthPixelConverter}}" 
-                          MinWidth="60" MaxWidth="150"/>
-        <ColumnDefinition/>
-      </Grid.ColumnDefinitions>
+    <Grid.ColumnDefinitions>
+      <ColumnDefinition Width="112"/>
+      <ColumnDefinition Width="2"/>
+      <ColumnDefinition/>
+    </Grid.ColumnDefinitions>
+    <Grid.RowDefinitions>
+      <RowDefinition Height="Auto" />
+      <RowDefinition Height="*" />
+    </Grid.RowDefinitions>
+    <Grid RowSpan="2" Background="{mbts:BTSColor UI.spark_menu_background}">
       <Grid.RowDefinitions>
-        <RowDefinition Height="Auto" />
-        <RowDefinition Height="*" />
+        <RowDefinition Height="Auto"/>
+        <RowDefinition/>
       </Grid.RowDefinitions>
-      <ScrollViewer Grid.Row="0" Grid.RowSpan="2" VerticalScrollBarVisibility="Hidden">
-        <Grid RowSpan="2" Background="{mbts:BTSColor UI.spark_menu_background}">
+      <StackPanel Margin="5,0">
+        <StackPanel.Resources>
+          <Style TargetType="Button" BasedOn="{StaticResource {x:Type Button}}">
+            <Setter Property="HorizontalAlignment" Value="Stretch" />
+          </Style>
+        </StackPanel.Resources>
+        <Grid Margin="8,4,0,0">
           <Grid.RowDefinitions>
-            <RowDefinition Height="*"/>
-            <RowDefinition/>
+            <RowDefinition Height="Auto"/>
+            <RowDefinition Height="4"/>
+            <RowDefinition Height="Auto"/>
+            <RowDefinition Height="4"/>
+            <RowDefinition Height="Auto"/>
+            <RowDefinition Height="4"/>
+            <RowDefinition Height="Auto"/>
+            <RowDefinition Height="4"/>
+            <RowDefinition Height="Auto"/>
+            <RowDefinition Height="4"/>
+            <RowDefinition Height="Auto"/>
+            <RowDefinition Height="Auto"/>
           </Grid.RowDefinitions>
-          <StackPanel Margin="2,0,5,0">
-            <StackPanel.Resources>
-              <Style TargetType="Button" BasedOn="{StaticResource {x:Type Button}}">
-                <Setter Property="HorizontalAlignment" Value="Stretch" />
-              </Style>
-            </StackPanel.Resources>
-            <Grid Margin="2,4,2,0">
-              <Grid.RowDefinitions>
-                <RowDefinition Height="Auto"/>
-                <RowDefinition Height="Auto"/>
-                <RowDefinition Height="Auto"/>
-                <RowDefinition Height="Auto"/>
-                <RowDefinition Height="Auto"/>
-                <RowDefinition Height="Auto"/>
-                <RowDefinition Height="Auto"/>
-              </Grid.RowDefinitions>
-              <Grid.ColumnDefinitions>
-                <ColumnDefinition Width="Auto"/>
-                <ColumnDefinition Width="6"/>
-                <ColumnDefinition Width="Auto"/>
-              </Grid.ColumnDefinitions>
-              <!-- Day -->
-              <TextBlock Grid.Column="0" Grid.Row="0"
-                     Text="{Binding left_panel_width, Converter={StaticResource DayTextConverter}}"
-                     Margin="0,0,0,4"
-                     Visibility="{Binding show_day, Converter={StaticResource BooleanToVisibilityConverter}}"/>
-              <TextBlock Grid.Column="2" Grid.Row="0" FontWeight="Bold"
-                     Margin="0,0,0,4"
-                     Foreground="{Binding change, Converter={StaticResource NumericSignBrushConverter}}" 
-                     Visibility="{Binding show_day, Converter={StaticResource BooleanToVisibilityConverter}}">
-                <TextBlock.Text>
-                  <MultiBinding Converter="{StaticResource TickDisplayConverter}" Mode="OneWay">
-                    <Binding Path="change"/>
-                    <Binding Path="security.ContractSpec"/>
-                  </MultiBinding>
-                </TextBlock.Text>
-              </TextBlock>
-              <!-- High -->
-              <TextBlock Grid.Column="0" Grid.Row="1"
-                     Text="{Binding left_panel_width, Converter={StaticResource HighTextConverter}}"
-                     Margin="0,0,0,4"
-                     Visibility="{Binding show_high, Converter={StaticResource BooleanToVisibilityConverter}}"/>
-              <TextBlock Grid.Column="2" Grid.Row="1" Foreground="{mbts:BTSColor UI.default_text}" FontWeight="Bold"
-                     Margin="0,0,0,4"
-                     Visibility="{Binding show_high, Converter={StaticResource BooleanToVisibilityConverter}}">
-                <TextBlock.Text>
-                  <MultiBinding Converter="{StaticResource TickDisplayConverter}" Mode="OneWay">
-                    <Binding Path="high"/>
-                    <Binding Path="security.ContractSpec"/>
-                  </MultiBinding>
-                </TextBlock.Text>
-              </TextBlock>
-              <!-- Low -->
-              <TextBlock Grid.Column="0" Grid.Row="2"
-                     Text="{Binding left_panel_width, Converter={StaticResource LowTextConverter}}"
-                     Margin="0,0,0,4"
-                     Visibility="{Binding show_low, Converter={StaticResource BooleanToVisibilityConverter}}"/>
-              <TextBlock Grid.Column="2" Grid.Row="2" Foreground="{mbts:BTSColor UI.default_text}" FontWeight="Bold"
-                     Margin="0,0,0,4"
-                     Visibility="{Binding show_low, Converter={StaticResource BooleanToVisibilityConverter}}">
-                <TextBlock.Text>
-                  <MultiBinding Converter="{StaticResource TickDisplayConverter}" Mode="OneWay">
-                    <Binding Path="low"/>
-                    <Binding Path="security.ContractSpec"/>
-                  </MultiBinding>
-                </TextBlock.Text>
-              </TextBlock>
-              <!-- Open -->
-              <TextBlock Grid.Column="0" Grid.Row="3"
-                     Text="{Binding left_panel_width, Converter={StaticResource OpenTextConverter}}"
-                     Margin="0,0,0,4"
-                     Visibility="{Binding show_open, Converter={StaticResource BooleanToVisibilityConverter}}"/>
-              <TextBlock Grid.Column="2" Grid.Row="3" FontWeight="Bold"
-                     Margin="0,0,0,4"
-                     Visibility="{Binding show_open, Converter={StaticResource BooleanToVisibilityConverter}}">
-                <TextBlock.Text>
-                  <MultiBinding Converter="{StaticResource TickDisplayConverter}" Mode="OneWay">
-                    <Binding Path="open"/>
-                    <Binding Path="security.ContractSpec"/>
-                  </MultiBinding>
-                </TextBlock.Text>
-              </TextBlock>
-              <!-- Settle -->
-              <TextBlock Grid.Column="0" Grid.Row="4"
-                     Text="{Binding left_panel_width, Converter={StaticResource SettleTextConverter}}"
-                     Margin="0,0,0,4"
-                     Visibility="{Binding show_settle, Converter={StaticResource BooleanToVisibilityConverter}}"/>
-              <TextBlock Grid.Column="2" Grid.Row="4" FontWeight="Bold"
-                     Margin="0,0,0,4"
-                     Visibility="{Binding show_settle, Converter={StaticResource BooleanToVisibilityConverter}}">
-                <TextBlock.Text>
-                  <MultiBinding Converter="{StaticResource TickDisplayConverter}" Mode="OneWay">
-                    <Binding Path="settlement"/>
-                    <Binding Path="security.ContractSpec"/>
-                  </MultiBinding>
-                </TextBlock.Text>
-              </TextBlock>
-              <!-- Volume -->
-              <TextBlock Grid.Column="0" Grid.Row="5"
-                     Text="{Binding left_panel_width, Converter={StaticResource VolumeTextConverter}}"
-                     Margin="0,0,0,4"
-                     Visibility="{Binding show_volume, Converter={StaticResource BooleanToVisibilityConverter}}"/>
-              <TextBlock Grid.Column="2" Grid.Row="5" Text="{Binding volume, Mode=OneWay}" FontWeight="Bold"
-                     Margin="0,0,0,4"
-                     Visibility="{Binding show_volume, Converter={StaticResource BooleanToVisibilityConverter}}"/>
-            </Grid>
-            <!-- Account combobox -->
-            <ComboBox DataContext="{Binding order_properties_vm}" Margin="2,10,2,2" 
-                  ItemsSource="{Binding accounts}" SelectedItem="{Binding selected_account}"
-                  Visibility="{Binding RelativeSource={RelativeSource FindAncestor, AncestorType={x:Type UserControl}},
-                              Path = DataContext.show_account,
-                              Converter={StaticResource BooleanToVisibilityConverter}}"/>
-            <!-- TIF combobox-->
-            <ComboBox DataContext="{Binding order_properties_vm}" ItemsSource="{Binding time_in_forces}" 
-                  SelectedItem="{Binding selected_time_in_force}" Margin="2"
-                  Visibility="{Binding RelativeSource={RelativeSource FindAncestor, AncestorType={x:Type UserControl}},
-                              Path = DataContext.show_tif,
-                              Converter={StaticResource BooleanToVisibilityConverter}}">
-              <ComboBox.ItemTemplate>
-                <DataTemplate>
-                  <TextBlock Text="{Binding Converter={StaticResource casing_converter}, ConverterParameter=Upper}" />
-                </DataTemplate>
-              </ComboBox.ItemTemplate>
-            </ComboBox>
-            <!-- Order Type combobox -->
-            <uag:OrderDetailsPopupControl
+          <Grid.ColumnDefinitions>
+            <ColumnDefinition Width="Auto"/>
+            <ColumnDefinition Width="6"/>
+            <ColumnDefinition Width="Auto"/>
+          </Grid.ColumnDefinitions>
+          <TextBlock Grid.Column="0" Grid.Row="0" Text="Day"/>
+          <TextBlock Grid.Column="0" Grid.Row="2" Text="High"/>
+          <TextBlock Grid.Column="0" Grid.Row="4" Text="Low"/>
+          <TextBlock Grid.Column="0" Grid.Row="6" Text="Open"/>
+          <TextBlock Grid.Column="0" Grid.Row="8" Text="Settle"/>
+          <TextBlock Grid.Column="0" Grid.Row="10" Text="Volume"/>
+
+          <TextBlock Grid.Column="2" Grid.Row="0" Foreground="{Binding change, Converter={StaticResource NumericSignBrushConverter}}" FontWeight="Bold">
+            <TextBlock.Text>
+              <MultiBinding Converter="{StaticResource TickDisplayConverter}" Mode="OneWay">
+                <Binding Path="change"/>
+                <Binding Path="security.ContractSpec"/>
+              </MultiBinding>
+            </TextBlock.Text>
+          </TextBlock>
+          <TextBlock Grid.Column="2" Grid.Row="2" Foreground="{mbts:BTSColor UI.default_text}" FontWeight="Bold">
+            <TextBlock.Text>
+              <MultiBinding Converter="{StaticResource TickDisplayConverter}" Mode="OneWay">
+                <Binding Path="high"/>
+                <Binding Path="security.ContractSpec"/>
+              </MultiBinding>
+            </TextBlock.Text>
+          </TextBlock>
+          <TextBlock Grid.Column="2" Grid.Row="4" Foreground="{mbts:BTSColor UI.default_text}" FontWeight="Bold">
+            <TextBlock.Text>
+              <MultiBinding Converter="{StaticResource TickDisplayConverter}" Mode="OneWay">
+                <Binding Path="low"/>
+                <Binding Path="security.ContractSpec"/>
+              </MultiBinding>
+            </TextBlock.Text>
+          </TextBlock>
+          <TextBlock Grid.Column="2" Grid.Row="6" FontWeight="Bold">
+            <TextBlock.Text>
+              <MultiBinding Converter="{StaticResource TickDisplayConverter}" Mode="OneWay">
+                <Binding Path="open"/>
+                <Binding Path="security.ContractSpec"/>
+              </MultiBinding>
+            </TextBlock.Text>
+          </TextBlock>
+          <TextBlock Grid.Column="2" Grid.Row="8" FontWeight="Bold">
+            <TextBlock.Text>
+              <MultiBinding Converter="{StaticResource TickDisplayConverter}" Mode="OneWay">
+                <Binding Path="settlement"/>
+                <Binding Path="security.ContractSpec"/>
+              </MultiBinding>
+            </TextBlock.Text>
+          </TextBlock>
+          <TextBlock Grid.Column="2" Grid.Row="10" Text="{Binding volume, Mode=OneWay}" FontWeight="Bold" />
+        </Grid>
+        <ComboBox DataContext="{Binding order_properties_vm}" Margin="2,10,2,2" ItemsSource="{Binding accounts}" SelectedItem="{Binding selected_account}"/>
+        <ComboBox DataContext="{Binding order_properties_vm}" ItemsSource="{Binding time_in_forces}" SelectedItem="{Binding selected_time_in_force}" Margin="2">
+          <ComboBox.ItemTemplate>
+            <DataTemplate>
+                <TextBlock Text="{Binding Converter={StaticResource casing_converter}, ConverterParameter=Upper}" />
+              </DataTemplate>           
+          </ComboBox.ItemTemplate>
+        </ComboBox>
+        <uag:OrderDetailsPopupControl
           DataContext="{Binding order_properties_vm}"
           Margin="2"
           ExpandDirection="Left"
@@ -425,25 +372,19 @@
           IsOpen="{Binding selected_order_type.editable_properties_open, FallbackValue=False}"
           ItemsSource="{Binding order_types}"
           SelectedItem="{Binding selected_order_type}"
-          Visibility="{Binding RelativeSource={RelativeSource FindAncestor, AncestorType={x:Type UserControl}},
-                      Path = DataContext.show_order_type,
-                      Converter={StaticResource BooleanToVisibilityConverter}}"
         />
-            <!-- Position into qty button -->
-            <Button
+        <Button
           Margin="2"
           Command="{Binding position_click_command}"
           Content="{Binding position, Converter={StaticResource PositionTextConverter}}"
           FontWeight="Bold"
           Foreground="{Binding position, Converter={StaticResource NumericSignBrushConverter}}"
-          Visibility="{Binding show_account_position, Converter={StaticResource BooleanToVisibilityConverter}}"
         >
-              <Button.ToolTip>
-                <TextBlock Text="{Binding order_properties_vm.selected_account, StringFormat='{}{0} total position (click to copy to size)'}"/>
-              </Button.ToolTip>
-            </Button>
-            <!-- Qty textbox -->
-            <TextBox
+          <Button.ToolTip>
+            <TextBlock Text="{Binding order_properties_vm.selected_account, StringFormat='{}{0} total position (click to copy to size)'}"/>
+          </Button.ToolTip>
+        </Button>
+        <TextBox
           mbts:TextBoxBehavior.InputFilter="{mbts:RegexInputFilter Regex={x:Static Member=Integer, MemberType={mbts:Nested txt:TextUtils.RegexFilters}}}"
           mbts:TextBoxBehavior.IncrementDecrementOnMouseClick="True"
           HorizontalContentAlignment="Right"
@@ -451,64 +392,42 @@
           Text="{Binding size_calculator.value, Mode=TwoWay, UpdateSourceTrigger=PropertyChanged}"
           ContextMenu="{x:Null}"
         />
-            <!-- Qty quick buttons -->
-            <ItemsControl ItemsSource="{Binding size_calculator.buttons}" Focusable="False"
-                      Visibility="{Binding show_size_preset, Converter={StaticResource BooleanToVisibilityConverter}}"
-                      >
-              <ItemsControl.ItemsPanel>
-                <ItemsPanelTemplate>
-                  <UniformGrid Columns="2" Focusable="False"/>
-                </ItemsPanelTemplate>
-              </ItemsControl.ItemsPanel>
-              <ItemsControl.ItemTemplate>
-                <DataTemplate>
-                  <Button Margin="2" Command="{Binding Path=DataContext.size_calculator.click, RelativeSource={RelativeSource AncestorType=ItemsControl}}" CommandParameter="{Binding}" Content="{Binding}" Focusable="False"/>
-                </DataTemplate>
-              </ItemsControl.ItemTemplate>
-            </ItemsControl>
-            <!-- Clear button -->
-            <Button Margin="2" HorizontalAlignment="Stretch" Command="{Binding size_calculator.clear}"
-                    Visibility="{Binding show_clear, Converter={StaticResource BooleanToVisibilityConverter}}">Clear</Button>
-            <!-- Auto Clear button - reset to defaults after order -->
-            <CheckBox Margin="2" IsChecked="{Binding auto_clear}"
-                  Visibility="{Binding show_auto_clear, Converter={StaticResource BooleanToVisibilityConverter}}">
-              Auto Clear
-            </CheckBox>
-            <StackPanel Margin="0,17,0,0" VerticalAlignment="Top">
-              <!-- Cancel Buttons -->
-              <Button Command="{Binding cancel_command}" CommandParameter="Bids" Style="{StaticResource CancelBuys}"
-                  Visibility="{Binding show_cancel_buys, Converter={StaticResource BooleanToVisibilityConverter}}"/>
-              <Button Command="{Binding cancel_command}" CommandParameter="Asks" Style="{StaticResource CancelSells}"
-                  Visibility="{Binding show_cancel_sells, Converter={StaticResource BooleanToVisibilityConverter}}"/>
-              <Button Command="{Binding cancel_command}" CommandParameter="All" Style="{StaticResource CancelAll}"
-                  Visibility="{Binding show_cancel_all, Converter={StaticResource BooleanToVisibilityConverter}}"/>
-            </StackPanel>
-            <StackPanel Grid.Row="1" Margin="0,17,0,0" VerticalAlignment="Bottom">
-              <!-- Settings Button -->
-              <Button Command="{Binding settings_command}" Style="{StaticResource SettingsButton}"/>
-            </StackPanel>
-          </StackPanel>
-        </Grid>
-      </ScrollViewer>
-      <!-- Ladder Column Headers -->
-      <Border Grid.Column="2" HorizontalAlignment="Stretch">
-        <GridViewHeaderRowPresenter x:Name="_column_headers_presenter" AllowsColumnReorder="False" />
-      </Border>
-      <!-- LadderGridHost -->
-      <spark:LadderGridHost
-        x:Name="_ladder"
-        Grid.Column="2"
-        Grid.Row="1" 
-        Command="{Binding send_order_command}" 
-        Orders="{Binding orders}"
-        orders_column_width="{Binding orders_column_width, Mode=TwoWay}"
-        bid_column_width="{Binding bid_column_width, Mode=TwoWay}"
-        price_column_width="{Binding price_column_width, Mode=TwoWay}"
-        ask_column_width="{Binding ask_column_width, Mode=TwoWay}"
-        LastColumnWidth="{Binding last_column_width, Mode=TwoWay}"
-        VolumeColumnWidth="{Binding volume_column_width, Mode=TwoWay}"
-      >
-        <spark:LadderGridHost.Resources>
+        <ItemsControl ItemsSource="{Binding size_calculator.buttons}" Focusable="False">
+          <ItemsControl.ItemsPanel>
+            <ItemsPanelTemplate>
+              <UniformGrid Columns="2" Focusable="False"/>
+            </ItemsPanelTemplate>
+          </ItemsControl.ItemsPanel>
+          <ItemsControl.ItemTemplate>
+            <DataTemplate>
+              <Button Margin="2" Command="{Binding Path=DataContext.size_calculator.click, RelativeSource={RelativeSource AncestorType=ItemsControl}}" CommandParameter="{Binding}" Content="{Binding}" Focusable="False"/>
+            </DataTemplate>
+          </ItemsControl.ItemTemplate>
+        </ItemsControl>
+        <Button Margin="2" HorizontalAlignment="Stretch" Command="{Binding size_calculator.clear}">Clear</Button>
+        <!-- reset to defaults after order -->
+        <CheckBox Margin="2" IsChecked="{Binding auto_clear}">Auto Clear</CheckBox>
+      </StackPanel>
+      <Grid Grid.Row="1">
+        <Grid.RowDefinitions>
+          <RowDefinition Height="Auto"/>
+          <RowDefinition/>
+        </Grid.RowDefinitions>
+        <StackPanel Margin="0,17,0,0" VerticalAlignment="Top">
+          <Button Command="{Binding cancel_command}" CommandParameter="Bids" Style="{StaticResource CancelBuys}"/>
+          <Button Command="{Binding cancel_command}" CommandParameter="Asks" Style="{StaticResource CancelSells}"/>
+          <Button Command="{Binding cancel_command}" CommandParameter="All" Style="{StaticResource CancelAll}"/>
+        </StackPanel>
+        <Button Grid.Row="1" HorizontalContentAlignment="Stretch" Command="{Binding settings_command}" Style="{StaticResource SettingsButton}" />
+      </Grid>
+    </Grid>
+    <Border Grid.Column="2" HorizontalAlignment="Stretch">
+      <GridViewHeaderRowPresenter x:Name="_column_headers_presenter" AllowsColumnReorder="False"/>
+    </Border>
+    <spark:LadderGridHost x:Name="_ladder" Grid.Column="2" Grid.Row="1" 
+      Command="{Binding send_order_command}" 
+      Orders="{Binding orders}">
+      <spark:LadderGridHost.Resources>
         <Style TargetType="controls:Badge">
           <Setter Property="Background" Value="{Binding background}" />
           <Setter Property="BorderBrush" Value="{Binding border}" />
@@ -548,7 +467,7 @@
         </Style>
       </spark:LadderGridHost.Resources>
     </spark:LadderGridHost>
-      <spark:RecenterCountdown Grid.Column="2" Grid.Row="1" Margin="0,0,18,0"
+    <spark:RecenterCountdown Grid.Column="2" Grid.Row="1" Margin="0,0,18,0"
                              VerticalAlignment="Bottom" HorizontalAlignment="Right" 
                              Width="30" Height="30"
                              Mode="Pie" Delay="0:0:10"
@@ -564,19 +483,14 @@
         <Pen Thickness="0" Brush="MediumPurple" />
       </spark:RecenterCountdown.PieStroke>
     </spark:RecenterCountdown>
-      <!-- Since the GridSplitter will share a cell with other elements,
-      we add it in toward the end, but we still want the ladder status
-      message to have the topmost Z order
-      -->
-      <GridSplitter Grid.Column="0" Grid.RowSpan="2" Width="5" />
-      <!-- Ladder Status message -->                        
-      <Button x:Name="_status" Grid.ColumnSpan="4" Grid.Row="1" VerticalAlignment="Bottom" HorizontalAlignment="Left" Command="{Binding status_click_command}">
-        <Button.Template>
-          <ControlTemplate>
-            <TextBlock Text="{Binding status_message}" Background="{Binding status_background}" Padding="5,0" />
-          </ControlTemplate>
-        </Button.Template>
-      </Button>
-    </Grid>
+                            
+    <Button x:Name="_status" Grid.ColumnSpan="4" Grid.Row="1" VerticalAlignment="Bottom" HorizontalAlignment="Left" Command="{Binding status_click_command}">
+      <Button.Template>
+        <ControlTemplate>
+          <TextBlock Text="{Binding status_message}" Background="{Binding status_background}" Padding="5,0" />
+        </ControlTemplate>
+      </Button.Template>
+    </Button>
+  </Grid>
   </AdornerDecorator>
 </UserControl>

src/client/spark/view/Spark/Ladder/LadderView.xaml.cs

@@ -3,11 +3,10 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Windows;
 using System.Windows.Controls;
-using System.Windows.Controls.Primitives;
 using System.Windows.Data;
 
 namespace mbts.spark {
-   partial class LadderView : UserControl {
+   public partial class LadderView : UserControl {
       internal LadderView(LadderVM vm) {
          InitializeComponent();
          _column_headers_presenter.Columns = new GridViewColumnCollection();
@@ -22,31 +21,6 @@ namespace mbts.spark {
          this.vm = DataContext as LadderVM;
       }
 
-      // I found this on StackOverflow (and the XAML) and thought it was a neat
-      // way of enforcing a MinWidth on the columns. This is so that the columns
-      // cannot be resized to nothing with the mouse.
-      private void Thumb_DragDelta(object sender, System.Windows.Controls.Primitives.DragDeltaEventArgs e) {
-         if (sender != null) {
-            DependencyObject parent = sender as DependencyObject;
-            if (parent != null) {
-               try {
-                  do {
-                     parent = System.Windows.Media.VisualTreeHelper.GetParent(parent as DependencyObject);
-                  } while (parent.GetType() != typeof(Canvas));
-
-                  Canvas canvas = parent as Canvas;
-                  if (canvas != null) {
-                     if (canvas.ActualWidth + e.HorizontalChange < canvas.MinWidth) {
-                        e.Handled = true;
-                     }
-                  }
-               } catch {
-               }
-            }
-         }
-      }
-
-
       internal LadderVM vm { get; private set; }
 
       protected override void OnDragOver(DragEventArgs e) {
@@ -66,31 +40,31 @@ namespace mbts.spark {
             switch (type) {
                case LadderGridHost.ColumnType.Order:
                   header = "Orders";
-                  width_prop = nameof(LadderVM.orders_column_width);
+                  width_prop = nameof(LadderGridHost.orders_column_width);
                   break;
                case LadderGridHost.ColumnType.Bid:
                   header = "Bid";
-                  width_prop = nameof(LadderVM.bid_column_width);
+                  width_prop = nameof(LadderGridHost.bid_column_width);
                   break;
                case LadderGridHost.ColumnType.Price:
                   header = "Price";
-                  width_prop = nameof(LadderVM.price_column_width);
+                  width_prop = nameof(LadderGridHost.price_column_width);
                   break;
                case LadderGridHost.ColumnType.Ask:
                   header = "Ask";
-                  width_prop = nameof(LadderVM.ask_column_width);
+                  width_prop = nameof(LadderGridHost.ask_column_width);
                   break;
                case LadderGridHost.ColumnType.Volume:
                   header = "Volume";
-                  width_prop = nameof(LadderVM.volume_column_width);
+                  width_prop = nameof(LadderGridHost.VolumeColumnWidth);
                   break;
                case LadderGridHost.ColumnType.Last:
                   header = "Last";
-                  width_prop = nameof(LadderVM.last_column_width);
+                  width_prop = nameof(LadderGridHost.LastColumnWidth);
                   break;
                default: throw ExceptionBuilder.Argument.unexpected_case(type, nameof(type));
             }
-            return _create(host, header, $"DataContext.{width_prop}");
+            return _create(host, header, width_prop);
          }
 
          static GridViewColumn _create(LadderGridHost host, string header, string width_prop) {
@@ -103,6 +77,5 @@ namespace mbts.spark {
             return gvc;
          }
       }
-
    }
 }

src/client/spark/view/Spark/PAndL/PnLSummaryRow.cs

@@ -15,7 +15,7 @@ namespace mbts.spark {
    class PnLSummaryRow : ViewModelBase, IEquatable<PnLSummaryRow>, DIYGrid.IRowVM {
       static readonly SignedDataStore<Brush> _brushes;
       readonly PnLDoubleBrushCell _start_pnl, _day_pnl, _total_pnl;
-      readonly CellProperty<double> _pnl_price, _weighted_average_buy, _weighted_average_sell;
+      readonly PnLPriceCell _pnl_price, _weighted_average_buy, _weighted_average_sell;
       (string account, Venue? exchange, string symbol, SecurityImmutable? security, SecurityMD? sec_md, SecurityType? sec_type) key;
       IEnumerable<FillImmutable> _fills;
       double _driver;
@@ -38,9 +38,9 @@ namespace mbts.spark {
          day_short_quantity = day_sell_quant;
          total_net_position = net_pos;
          start_net_position = sod_pos;
-         _pnl_price = new CellProperty<double>(this, nameof(pnl_price), price => this.security.spec_or_default().format(price), 0);
-         _weighted_average_buy = new CellProperty<double>(this, nameof(weighted_average_buy), price => this.security.spec_or_default().format(price), wavg_buy);
-         _weighted_average_sell = new CellProperty<double>(this, nameof(weighted_average_sell), price => this.security.spec_or_default().format(price), wavg_sell);
+         _pnl_price = new PnLPriceCell(this, nameof(pnl_price));
+         _weighted_average_buy = new PnLPriceCell(this, nameof(weighted_average_buy), wavg_buy);
+         _weighted_average_sell = new PnLPriceCell(this, nameof(weighted_average_sell), wavg_sell);
          _fills = fills;
 
          // set the initial pnl and pnl_price when there has been a
@@ -193,13 +193,19 @@ namespace mbts.spark {
       class PnLDoubleBrushCell : CellProperty<double> {
          Brush _background;
 
-         public PnLDoubleBrushCell(PnLSummaryRow owner, string prop) : base(owner, prop, d => double.IsNaN(d) ? string.Empty : d.ToString("N2"), double.NaN) =>
+         public PnLDoubleBrushCell(PnLSummaryRow owner, string prop) : base(d => double.IsNaN(d) ? string.Empty : d.ToString("N2"), double.NaN) =>
             _value_set += _ => {
                _background = null;
+               owner.RaisePropertyChanged(prop);
             };
 
          public Brush background => _background ?? (_background = _brushes.get(value));
       }
 
+      class PnLPriceCell : CellProperty<double> {
+         public PnLPriceCell(PnLSummaryRow owner, string prop, double init = 0)
+            : base(price => owner.security.spec_or_default().format(price), init) =>
+            _value_set += _ => owner.RaisePropertyChanged(prop);
+      }
    }
 }

src/client/spark/view/Spark/Utilities/WindowSettingsDialogVM.cs

@@ -17,9 +17,7 @@ namespace mbts.spark {
       readonly DelegateProperty<IEnumerable<DialogSkirtItem>> _left_skirt_items;
       readonly List<IDisposable> _resources = new List<IDisposable>(2); // initialize with 2 capacity bc the only potential disposables are the instance/default vms
 
-      public WindowSettingsDialogVM(IWindowSettingsManager<T> settings_manager, Func<UserSettings, T> get_factory_settings, string settings_file_ext, Func<T, TVM> vm_factory, Func<T, T> merge_with_current = null) {
-         if (merge_with_current == null)
-            merge_with_current = s => s;
+      public WindowSettingsDialogVM(IWindowSettingsManager<T> settings_manager, Func<UserSettings, T> get_factory_settings, string settings_file_ext, Func<T, TVM> vm_factory) {
          // by default, each new window will use their own instance settings
          _use_default = backing(nameof(use_default), settings_manager.instance_settings == null);
          default_vm = vm_factory(settings_manager.default_settings);
@@ -61,14 +59,15 @@ namespace mbts.spark {
             _resources.Add(dispose_dvm);
          // local methods
          void apply() {
-            var result = merge_with_current(instance_vm.get_result());
             if (!use_default) {
-               settings_manager.instance_settings = result;
+               settings_manager.instance_settings = instance_vm.get_result();
             } else {
                settings_manager.instance_settings = null;
+
                // copy the current instance settings and persist
-               if (!result.safe_equals(settings_manager.default_settings))
-                  settings_manager.save_default_settings(result).ignore();
+               var my_default = instance_vm.get_result();
+               if (!my_default.safe_equals(settings_manager.default_settings))
+                  settings_manager.save_default_settings(my_default).ignore();
             }
          }
 

src/client/spark_api/api/ApiNativeMethods.cs

@@ -64,7 +64,7 @@ namespace bts.api.core {
       internal static extern void api_client_send(IntPtr nchannel, byte[] buffer, int length, raw_pb_callback p, IntPtr extra);
 
       [DllImport("libspark")]
-      internal static extern IntPtr api_client_subscribe(IntPtr client, string method, string qualifier, raw_subscription_callback cb, IntPtr extra);
+      internal static extern IntPtr api_client_subscribe(IntPtr client, string method_and_qualifier, raw_subscription_callback cb, IntPtr extra);
       [DllImport("libspark")]
       internal static extern void api_client_unsubscribe(IntPtr client, IntPtr subscription);
 
@@ -96,7 +96,7 @@ namespace bts.api.core {
       }
 
       [DllImport("libspark")]
-      internal static extern IntPtr api_server_create_table(IntPtr server, string method, string qualifier);
+      internal static extern IntPtr api_server_create_table(IntPtr server, string method_and_qualifier);
       [DllImport("libspark")]
       internal static extern void api_server_update_table(IntPtr table, [In] StreamUpdate[] changes, long change_count);
       [DllImport("libspark")]

src/client/spark_api/api/ApiServer.cs

@@ -17,6 +17,15 @@ namespace bts.api.core
 
    internal abstract class ApiServer : IApiServer
    {
+      static JsonFormatter _formatter = new JsonFormatter(new JsonFormatter.Settings(true));
+      static Regex _fmt_cleanup = new Regex(@"[\s_+]", RegexOptions.Compiled | RegexOptions.Multiline);
+      public static string format_qualifier<T>(string method, T arg) where T : IMessage {
+         if (arg != null) {
+            string arg_fmt = _fmt_cleanup.Replace(_formatter.Format(arg), "");
+            return method + arg_fmt;
+         }
+         return method;
+      }
       public abstract void dispatch(RequestHeader header, Stream request, Action<byte[]> on_response);
 
       protected void respond<T>(int request_id, Task<T> task, Action<byte[]> on_response) where T : IMessage, IImmutableMessage {

src/client/spark_api/api/ApiUtils.cs

@@ -1,68 +1,68 @@
-using bts.api.core;
-using Google.Protobuf;
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.IO;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace bts.api {
-   internal static class ApiUtils {
-
-      public static byte[] serialize<T>(this RequestHeader header, T body) where T : IImmutableMessage {
-         int body_size = body?.CalculateSize() ?? 0;
-         int header_size = header.CalculateSize() + 1; // 1 byte header length
-         var buffer = new byte[header_size + body_size];
+using Google.Protobuf;
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace bts.api {
+   internal static class ApiUtils {
+
+
+      public static byte[] serialize<T>(this RequestHeader header, T body) where T : IImmutableMessage {
+         int body_size = body?.CalculateSize() ?? 0;
+         int header_size = header.CalculateSize() + 1; // 1 byte header length
+         var buffer = new byte[header_size+body_size];
          using (var stream = new MemoryStream(buffer)) {
             checked { stream.WriteByte((byte)header_size); }
             header.WriteTo(stream);
             if (body != null) body.WriteTo(stream);
-         }
-
-         return buffer;
-      }
-
-      public struct DeserializeResult<THeader, TBody> {
-         public THeader header;
-         public TBody body;
-      }
-
-      private static DeserializeResult<THeader, TBody> mkresult<THeader, TBody>(THeader header, TBody body) {
-         return new DeserializeResult<THeader, TBody>() { header = header, body = body }; }
-
-      public static unsafe DeserializeResult<THeader, TBody> deserialize<THeader, TBody>(byte* buffer, long buffer_length) where THeader : IMessage, new() where TBody : class, IMessage, new() {
-         var header_size = buffer[0];
-         Debug.Assert(header_size <= buffer_length);
-         THeader header = new THeader();
-         header.MergeFrom(new UnmanagedMemoryStream(buffer + 1, header_size - 1));
-         TBody body = new TBody();
-         if (buffer_length > header_size) {
-            body.MergeFrom(new UnmanagedMemoryStream(buffer + header_size, buffer_length - header_size));
-         }
-         return mkresult(header, body);
-      }
-
-      public static byte[] serialize<T>(this ResponseHeader header, T body) where T : IImmutableMessage {
-         int header_size = 1 + header.CalculateSize();
-         int body_size = body == null ? 0 : body.CalculateSize();
+         }
+
+         return buffer;
+      }
+
+      public struct DeserializeResult<THeader,TBody> {
+         public THeader header;
+         public TBody body;
+      }
+
+      private static DeserializeResult<THeader, TBody> mkresult<THeader,TBody>(THeader header, TBody body) {
+         return new DeserializeResult<THeader, TBody>() { header = header, body = body }; }
+
+      public static unsafe DeserializeResult<THeader, TBody> deserialize<THeader,TBody>(byte* buffer, long buffer_length) where THeader : IMessage,new() where TBody : class,IMessage,new() {
+         var header_size = buffer[0];
+         Debug.Assert(header_size <= buffer_length);
+         THeader header = new THeader();
+         header.MergeFrom(new UnmanagedMemoryStream(buffer+1, header_size - 1));
+         TBody body = new TBody();
+         if (buffer_length > header_size) {
+            body.MergeFrom(new UnmanagedMemoryStream(buffer + header_size, buffer_length - header_size));
+         }
+         return mkresult(header, body);
+      }
+
+      public static byte[] serialize<T>(this ResponseHeader header, T body) where T : IImmutableMessage {
+         int header_size = 1 + header.CalculateSize();
+         int body_size = body == null ? 0 : body.CalculateSize();
          var buffer = new byte[header_size + body_size];
-         using (var stream = new MemoryStream(buffer)) {
-            checked { stream.WriteByte((byte)header_size); }
-            header.WriteTo(stream);
-            if (body != null) body.WriteTo(stream);
+         using (var stream = new MemoryStream(buffer)) {
+            checked { stream.WriteByte((byte)header_size); }
+            header.WriteTo(stream);
+            if (body != null) body.WriteTo(stream);
          }
          return buffer;
-      }
-      public static byte[] serialize(this ResponseHeader header) {
-         int header_size = 1 + header.CalculateSize();
-         var buffer = new byte[header_size];
-         using (var stream = new MemoryStream(buffer)) {
-            checked { stream.WriteByte((byte)header_size); }
-            header.WriteTo(stream);
-         }
-         return buffer;
-      }
-   }
-}
+      }
+      public static byte[] serialize(this ResponseHeader header) {
+         int header_size = 1 + header.CalculateSize();
+         var buffer = new byte[header_size];
+         using (var stream = new MemoryStream(buffer)) {
+            checked { stream.WriteByte((byte)header_size); }
+            header.WriteTo(stream);
+         }
+         return buffer;
+      }
+   }
+}

src/client/spark_api/api/Channels/QuickBusClientChannel.cs

@@ -35,7 +35,6 @@ namespace bts.api.core {
          var header = new RequestHeader() {
             Method = method_id,
             RequestId = id,
-            Token = _context.token.HasValue ? new AuthToken() { Token = _context.token?.Token } : null
          };
          var buffer = header.serialize(request);
          var tcs = new TaskCompletionSource<Result<Response>>();
@@ -57,99 +56,60 @@ namespace bts.api.core {
          return tcs.Task;
       }
 
-      public override IObservable<RawUpdate<TOut>> subscribe<TParse,TOut>(int method_id, string method, string qualifier, MessageParser<TParse> parser, Func<TParse,TOut> converter) {
-         var result = Observable.Create<RawUpdate<TOut>>(async observer => {
-            IntPtr temp_subscription = IntPtr.Zero;
-            void unsubscribe_temp_sub() {
-               // Unsubscribe late to keep qb refcount > 0 continuously
-               if (temp_subscription != IntPtr.Zero) ApiNativeMethods.api_client_unsubscribe(_client, temp_subscription);
-            }
-            while (true) {
-               var response = await check_stream_access(method_id, qualifier);
-               if (response.success) {
-                  if (response.value.Authorized) {
-                     // success. Break loop and subscribe
-                     break;
+      public unsafe override IObservable<RawUpdate<TOut>> subscribe<TParse,TOut>(string method_and_qualifier, MessageParser<TParse> parser, Func<TParse,TOut> converter) {
+         var result = Observable.Create<RawUpdate<TOut>>(observer => {
+            bool first_update = true;
+            // Memory optimization. There is only one update per subscription.
+            RawUpdate<TOut> the_update = new RawUpdate<TOut>();
+            ApiNativeMethods.raw_subscription_callback cb = (event_type, changes, change_count, unused) => {
+               the_update.Clear();
+               try {
+                  if (event_type == ApiNativeMethods.EventType.TABLE_UPDATED) {
+                     foreach (var change in changes) {
+                        switch (change.type) {
+                           case ApiNativeMethods.ChangeType.REMOVE:
+                              the_update.removed_keys.Add(change.key);
+                              break;
+                           case ApiNativeMethods.ChangeType.REPLACE:
+                              the_update.changed_items.Add(converter(parser.ParseFrom(new UnmanagedMemoryStream(change.buffer, change.buffer_len))));
+                              the_update.changed_keys.Add(change.key);
+                              break;
+                           case ApiNativeMethods.ChangeType.CREATE:
+                              the_update.added_items.Add(converter(parser.ParseFrom(new UnmanagedMemoryStream(change.buffer, change.buffer_len))));
+                              the_update.added_keys.Add(change.key);
+                              break;
+                        }
+                     }
+                     observer.OnNext(the_update);
                   }
-                  else {
-                     // Not authorized. return empty stream
-                     unsubscribe_temp_sub();
-                     observer.OnCompleted();
-                     return System.Reactive.Disposables.Disposable.Empty;
+                  else if (event_type == ApiNativeMethods.EventType.TABLE_CREATED) {
+                     if (!first_update) {
+                        observer.OnNext(the_update.MarkAsReset());
+                     }
+                     first_update = false;
                   }
                }
-               else {
-                  // Wait for stream to fire once before trying again
-                  unsubscribe_temp_sub();
-                  var tcs = new TaskCompletionSource<bool>();
-                  ApiNativeMethods.raw_subscription_callback cb = (event_type, changes, change_count, unused) => {
-                     tcs.TrySetResult(true);
-                  };
-                  temp_subscription = ApiNativeMethods.api_client_subscribe(_client, method, qualifier, cb, IntPtr.Zero);
-                  await tcs.Task;
-                  GC.KeepAlive(cb);
+               catch (Exception e) {
+                  ApiNativeMethods.api_client_log_error($"Exception in subscription callback for sub: '{method_and_qualifier}' \n--\n '{e}'");
+                  // Sleep to let the log message get asyncronously written to disk
+                  Thread.Sleep(100);
+                  Environment.Exit(-1);
                }
-            }
-
-            // Authorized. Stream events
-            unsubscribe_temp_sub();
-            return SubscribeNative(method, qualifier, parser, converter, observer);
+            };
+            var subscription = ApiNativeMethods.api_client_subscribe(_client, method_and_qualifier, cb, IntPtr.Zero);
+            return FinalizeableAnonymousDisposable.Create((disposing) => {
+               ApiNativeMethods.api_client_unsubscribe(_client, subscription);
+               subscription = IntPtr.Zero;
+               GC.KeepAlive(cb);
+               cb = null;
+               // Cannot (ever) call logging from the finalizer thread
+               // if (!disposing) {
+               //    ApiNativeMethods.api_client_log_error($"Subscription has been terminated because the returned IDisposable has been finalized. Subscription: {method_and_qualifier}");
+               // }
+            });
          });
          return result;
       }
-      private Task<Result<StreamAccessResponse>> check_stream_access(int method_id, string qualifier) {
-         var request = new StreamAccessRequest() { Qualifier = qualifier ?? "" };
-         return send<StreamAccessRequest, StreamAccessResponse>(method_id, request);
-      }
-
-      private unsafe IDisposable SubscribeNative<TParse, TOut>(string method, string qualifier, MessageParser<TParse> parser, Func<TParse, TOut> converter, IObserver<RawUpdate<TOut>> observer) where TParse : IMessage<TParse> {
-         bool first_update = true;
-         // Memory optimization. There is only one update per subscription.
-         RawUpdate<TOut> the_update = new RawUpdate<TOut>();
-         ApiNativeMethods.raw_subscription_callback cb = (event_type, changes, change_count, unused) => {
-            the_update.Clear();
-            try {
-               if (event_type == ApiNativeMethods.EventType.TABLE_UPDATED) {
-                  foreach (var change in changes) {
-                     switch (change.type) {
-                        case ApiNativeMethods.ChangeType.REMOVE:
-                           the_update.removed_keys.Add(change.key);
-                           break;
-                        case ApiNativeMethods.ChangeType.REPLACE:
-                           the_update.changed_items.Add(converter(parser.ParseFrom(new UnmanagedMemoryStream(change.buffer, change.buffer_len))));
-                           the_update.changed_keys.Add(change.key);
-                           break;
-                        case ApiNativeMethods.ChangeType.CREATE:
-                           the_update.added_items.Add(converter(parser.ParseFrom(new UnmanagedMemoryStream(change.buffer, change.buffer_len))));
-                           the_update.added_keys.Add(change.key);
-                           break;
-                     }
-                  }
-                  observer.OnNext(the_update);
-               }
-               else if (event_type == ApiNativeMethods.EventType.TABLE_CREATED) {
-                  if (!first_update) {
-                     observer.OnNext(the_update.MarkAsReset());
-                  }
-                  first_update = false;
-               }
-            }
-            catch (Exception e) {
-               ApiNativeMethods.api_client_log_error($"Exception in subscription callback for sub: '{method}/{qualifier}' \n--\n '{e}'");
-               // Sleep to let the log message get asyncronously written to disk
-               Thread.Sleep(100);
-               Environment.Exit(-1);
-            }
-         };
-         var subscription = ApiNativeMethods.api_client_subscribe(_client, method, qualifier, cb, IntPtr.Zero);
-         return FinalizeableAnonymousDisposable.Create((disposing) => {
-            ApiNativeMethods.api_client_unsubscribe(_client, subscription);
-            subscription = IntPtr.Zero;
-            GC.KeepAlive(cb);
-            cb = null;
-         });
-      }
-
       private class FinalizeableAnonymousDisposable : IDisposable {
          private Action<bool> _action;
          public FinalizeableAnonymousDisposable(Action<bool> dispose) {

src/client/spark_api/api/ClientChannel.cs

@@ -5,7 +5,7 @@ using System.Threading.Tasks;
 namespace bts.api.core {
    internal abstract class ClientChannel : IDisposable
    {
-      protected readonly ServiceContext _context;
+      readonly ServiceContext _context;
 
       public ClientChannel(ServiceContext context) {
          _context = context;
@@ -13,7 +13,7 @@ namespace bts.api.core {
       public abstract void Dispose();
       public abstract Task<Result<Response>> send<Request, Response>(int method_id, Request request)
          where Request: IImmutableMessage where Response : class,IMessage,new();
-      public abstract IObservable<RawUpdate<TOut>> subscribe<TParse,TOut>(int method_id, string method, string qualifier, MessageParser<TParse> parser, Func<TParse,TOut> converter=null) 
+      public abstract IObservable<RawUpdate<TOut>> subscribe<TParse,TOut>(string method_and_qualifier, MessageParser<TParse> parser, Func<TParse,TOut> converter=null) 
          where TParse : IMessage<TParse>;
 
       public abstract bool is_sync_call_ok();

src/client/spark_api/api/Result.cs

@@ -37,12 +37,7 @@ namespace bts.api.core
       public T value {
          get {
             if (success) return _result;
-            else {
-               if (typeof(T).IsValueType) { throw new ApiException(error_message); }
-               else {
-                  return default(T);// always null
-               }
-            }
+            else throw new ApiException(error_message);
          }
       }
       public bool success { get { return _code == ApiError.Success; } }

src/client/spark_api/api/ServerChannel.cs

@@ -34,8 +34,8 @@ namespace bts.api.core {
       }
       public Action<RequestHeader, Stream, Action<byte[]>> on_request;
 
-      public IntPtr create_table(string method, string qualifier) {
-         return ApiNativeMethods.api_server_create_table(_server, method, qualifier);
+      public IntPtr create_table(string method_and_qualifier) {
+         return ApiNativeMethods.api_server_create_table(_server, method_and_qualifier);
       }
 
       public void Dispose() {

src/client/spark_api/api/ServiceContext.cs

@@ -4,14 +4,12 @@ using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
 using System.Reflection;
-using bts.api.user;
 
-namespace bts.api.core {
+namespace bts.api.core
+{
    public class ServiceContext : IDisposable
    {
       IntPtr _qbus;
-      AuthToken _token;
-
       public ServiceContext() {
          _qbus = ApiNativeMethods.api_create_quickbus(null, 0);
       }
@@ -24,6 +22,7 @@ namespace bts.api.core {
          Type[] arg_types = new[] { typeof(ClientChannel) };
          ConstructorInfo ctor = typeof(T).GetConstructors(BindingFlags.NonPublic | BindingFlags.Instance).First(ci => ci.GetParameters().SingleOrDefault()?.ParameterType == typeof(ClientChannel));
          return (T)ctor.Invoke(new object[] { channel });
+         //return (T)Activator.CreateInstance(typeof(T), channel);
       }
 
       internal ServerChannel publish<T>(T service, string service_type, string qualifier) where T : IApiServer {
@@ -32,38 +31,6 @@ namespace bts.api.core {
          return ch;
       }
 
-      public AuthTokenImmutable? token => _token ?? (AuthTokenImmutable?)null;
-      public string user { get; private set; }
-      public uint user_id { get; private set; }
-      public bool is_authenticated => token.HasValue;
-
-      public async Task<LoginResult> login(string username, string password) {
-         var auth_service = locate<AuthService>();
-         var result = await auth_service.authenticate_login_async(new AuthenticateLoginRequest() {
-            Login = username ?? "",
-            Password = password ?? ""
-         });
-         if (result.success) {
-            if (result.value.IsValid) {
-               _token = result.value.Token;
-               user = username;
-               user_id = result.value.UserId;
-               return new LoginResult();
-            }
-            else {
-               return new LoginResult(result.value.Reason);
-            }
-         }
-         else {
-            return new LoginResult(result.error_message);
-         }
-      }
-      public void login(AuthTokenImmutable token) {
-         this.user = "";
-         this.user_id = 0;
-         this._token = token.CloneToMutable();
-      }
-
       #region IDisposable Support
 
       protected virtual void Dispose(bool disposing) {
@@ -85,11 +52,4 @@ namespace bts.api.core {
       }
       #endregion
    }
-
-   public class LoginResult {
-      public LoginResult() { success = true; }
-      public LoginResult(string reason) { success = false; this.reason = reason; }
-      public bool success { get; }
-      public string reason { get; }
-   }
 }

src/client/spark_api/md/Authenticator.cs

@@ -0,0 +1,23 @@
+using System;
+using System.Threading.Tasks;
+using bts.api;
+using bts.api.core;
+
+namespace bts.api
+{
+   public class Authenticator
+   {
+      readonly static Lazy<user.UserService> _auth = new Lazy<user.UserService>(() => Locator.locate<user.UserService>(), true);
+      user.AuthToken _token = null;
+      public Authenticator() { }
+
+      public user.AuthToken last_token => _token;
+      public async Task<user.AuthToken> login(string login, string password)
+      {
+         var res = await _auth.Value.authenticate_login_async(new user.AuthenticateLoginRequest { Login = login, Password = password });
+         if (res.success)
+            _token = res.value.Token;            
+         return _token;
+      }
+   }
+}

src/client/spark_api/md/LazyAsync.cs

@@ -2,7 +2,7 @@
 using System;
 using System.Threading.Tasks;
 
-namespace bts.utils
+namespace bts
 {
    public class AsyncLazy<T> : Lazy<Task<T>>
    {

src/client/spark_api/md/Locator.cs

@@ -11,15 +11,6 @@ namespace bts.api
       public static T locate<T>(string qualifier="")  where T : class, IApiClient
       {
          return _context.Value.locate<T>(qualifier);
-      }
-      static public async Task<LoginResult> login(string username, string password)
-      {
-         return await _context.Value.login(username, password);
-      }
-      static public void login_with_token(AuthTokenImmutable token)
-      {
-         _context.Value.login(token);
       }      
-      static public AuthTokenImmutable? token => _context.Value.token;
    }
 }

src/client/spark_api/md/MDBook.cs

@@ -1,59 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-
-using bts.api;
-using mbts;
-using mbts.co2;
-
-namespace bts.md
-{
-   interface IMDUpdate
-   {
-      bool update(SecurityMD smd, ref ulong last_update_time);
-   }
-
-   public interface IMDData 
-   {
-      SecurityImmutable security { get; } 
-   } 
-   
-   public interface IMDBook : IMDData
-   {
-      List<PriceSize> bids { get; }
-      List<PriceSize> asks { get; }
-   }
-
-   class MDBook : IMDBook, IMDUpdate
-   {
-      public MDBook()
-      {
-      }
-      public SecurityImmutable security { get; private set; }
-      
-      public List<PriceSize> bids { get; private set; }
-      public List<PriceSize> asks { get; private set; }
-            
-      bool IMDUpdate.update(SecurityMD smd, ref ulong last_update_time)
-      {
-         this.security = smd.security;
-         NativePriceSize[] bids = null, asks = null;
-         if (smd.md.get_depth(ref last_update_time, ref bids, ref asks, NativePriceHelpers.COCONUT_MAX_DEPTH)) 
-         {            
-            this.bids = bids.Select(x => new PriceSize(x.size, x.direct_size, x.implied_size, x.price)).ToList();
-            this.asks = asks.Select(x => new PriceSize(x.size, x.direct_size, x.implied_size, x.price)).ToList();
-            return true;
-         }
-         return false;
-      }
-
-      public override string ToString()
-      {         
-         return security.ToString() + Environment.NewLine + 
-               "Bid BidSz BidISz" + Environment.NewLine + 
-               string.Join("\n", bids.Select(x => x.ToString())) + Environment.NewLine +
-               "Ask AskSz AskISz" + Environment.NewLine +
-               string.Join("\n", asks.Select(x => x.ToString()));
-      }
-   }
-}

src/client/spark_api/md/MDCombined.cs

@@ -1,61 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using bts.api;
-using mbts;
-
-namespace bts.md
-{
-   public interface IMDCombined : IMDData, IMDMisc, IMDLast, IMDBook
-   {
-   }
-
-   class MDCombined : IMDCombined, IMDUpdate
-   {
-      MDBook _book = new MDBook();
-      MDLast _last = new MDLast();
-      MDMisc _misc = new MDMisc();
-      public MDCombined()
-      {
-      }
-      public SecurityImmutable security { get; private set; }
-
-      public ulong total_volume => _misc.total_volume;
-      public double settlement_price => _misc.settlement_price;
-      public DateTime settlement_date => _misc.settlement_date;
-      public double previous_settlement_price => _misc.previous_settlement_price;
-      public DateTime previous_settlement_date => _misc.previous_settlement_date;
-      
-      public double open_price => _misc.open_price;
-      public DateTime open_date => _misc.open_date;
-      public double close_price => _misc.close_price;
-      public DateTime close_date => _misc.close_date;
-      public double session_high => _misc.session_high;
-      public double session_low => _misc.session_low;
-      public double fixing_price => _misc.fixing_price;
-
-      public PriceSize last => _last.last;
-      public ulong total_volume2 => _last.total_volume;
-      public DateTime last_time => _last.last_time;
-
-      public List<PriceSize> bids => _book.bids;
-      public List<PriceSize> asks => _book.asks;
-      
-
-      bool IMDUpdate.update(SecurityMD smd, ref ulong last_update_time)
-      {
-         this.security = smd.security;
-         var bl = last_update_time;
-         bool b = ((IMDUpdate)_book).update(smd, ref bl);
-
-         var ll = last_update_time;
-         bool l = ((IMDUpdate)_last).update(smd, ref ll);
-
-         var ml = last_update_time;
-         bool m = ((IMDUpdate)_misc).update(smd, ref ml);
-
-         last_update_time = new[] { bl, ll, ml }.Max();
-         return b || l || m;
-      }
-   }   
-}

src/client/spark_api/md/MDLast.cs

@@ -1,40 +0,0 @@
-using System;
-using bts.api;
-using mbts;
-using mbts.co2;
-
-namespace bts.md
-{
-   public interface IMDLast : IMDData
-   {
-      PriceSize last{ get; }
-      ulong total_volume { get; }
-      DateTime last_time { get; }
-   }
-
-   class MDLast : IMDLast, IMDUpdate
-   {
-      public MDLast()
-      {
-      }
-      public SecurityImmutable security { get; private set; }
-
-      public PriceSize last { get; private set; }
-      public ulong total_volume { get; private set; }
-      public DateTime last_time { get; private set; }
-
-      bool IMDUpdate.update(SecurityMD smd, ref ulong last_update_time)
-      {
-         this.security = smd.security;
-         NativePriceSize x; DateTime ts; ulong vol;
-         if (smd.md.get_last_trade(ref last_update_time, out x, out ts, out vol))
-         {
-            last = new PriceSize(x.size, x.direct_size, x.implied_size, x.price);
-            total_volume = vol;
-            last_time = ts;
-            return true;
-         }
-         return false;
-      }
-   }   
-}

src/client/spark_api/md/MDMisc.cs

@@ -1,67 +0,0 @@
-using System;
-using bts.api;
-using mbts;
-
-namespace bts.md
-{
-   public interface IMDMisc : IMDData
-   {
-      ulong total_volume { get; }
-      double settlement_price { get; }
-      DateTime settlement_date { get; }
-      double previous_settlement_price { get; }
-      DateTime previous_settlement_date { get; }
-
-      double open_price { get; }
-      DateTime open_date { get; }
-      double close_price { get; }
-      DateTime close_date { get; }
-      double session_high { get; }
-      double session_low { get; }
-      double fixing_price { get; }
-   }
-
-   class MDMisc : IMDMisc, IMDUpdate
-   {
-      public MDMisc()
-      {
-      }
-      public SecurityImmutable security { get; private set; }
-
-      public ulong total_volume { get; private set; }
-      public double settlement_price { get; private set; }
-      public DateTime settlement_date { get; private set; }
-      public double previous_settlement_price { get; private set; }
-      public DateTime previous_settlement_date { get; private set; }
-      
-      public double open_price { get; private set; }
-      public DateTime open_date { get; private set; }
-      public double close_price { get; private set; }
-      public DateTime close_date { get; private set; }
-      public double session_high { get; private set; }
-      public double session_low { get; private set; }
-      public double fixing_price { get; private set; }
-
-      bool IMDUpdate.update(SecurityMD smd, ref ulong last_update_time)
-      {
-         this.security = smd.security;
-         if (smd.md.misc_timestamp > last_update_time)
-         {
-            total_volume = smd.md.total_volume;
-            settlement_price = smd.md.settlement_price;
-            settlement_date = smd.md.settlement_date;
-            previous_settlement_price = smd.md.previous_settlement_price;
-            previous_settlement_date = smd.md.previous_settlement_date;
-
-            open_price = smd.md.open_price;
-            open_date = smd.md.open_date;
-            close_price = smd.md.close_price;
-            close_date = smd.md.close_date;
-            session_high = smd.md.session_high;
-            session_low = smd.md.session_low;
-            fixing_price = smd.md.fixing_price;
-         }
-         return false;
-      }
-   }   
-}

src/client/spark_api/md/MDSource.cs

@@ -1,90 +0,0 @@
-using bts.api;
-using bts.api.synthetic;
-using mbts;
-using mbts.co2;
-using System;
-using System.Collections.Generic;
-using System.Reactive.Linq;
-using System.Threading.Tasks;
-using System.Linq;
-using SecurityKey = System.UInt32;
-using System.Collections.Concurrent;
-using System.Threading;
-using bts.api.user;
-using System.Reactive.Subjects;
-
-namespace bts.md
-{
-   class MDSource : IObserver<List<SecurityAndHandle>>, IDisposable
-   {   
-      protected readonly MDConsumer _consumer = new MDConsumer();
-      protected readonly Co2RelayControl _relay;
-      protected long _version = 0;
-      protected readonly ConcurrentDictionary<SecurityKey, SecurityMD> _exchange_sec_map = new ConcurrentDictionary<SecurityKey, SecurityMD>();
-      protected List<IDisposable> _dispose = new List<IDisposable>();
-
-      public MDSource(uint naggle_ms, bool defer_init = false)
-      {         
-         _relay = new Co2RelayControl(_consumer, naggle_ms);
-         _relay.preload_securities((co2.MDRequest.Types.Flags)0xff).ignore();
-         _dispose.Add(_consumer.on_security.Subscribe(this));
-         if(!defer_init)
-            initialize();
-      }
-
-      readonly BehaviorSubject<ConcurrentDictionary<SecurityKey, SecurityMD>> _on_securities = 
-         new BehaviorSubject<ConcurrentDictionary<SecurityKey, SecurityMD>>( new ConcurrentDictionary<SecurityKey, SecurityMD>() );
-      public IObservable<ConcurrentDictionary<SecurityKey, SecurityMD>> on_securities => _on_securities;
-
-      virtual public void initialize()
-      {
-         Task.Run(run);
-      }
-
-      public void set_token(AuthTokenImmutable token)
-      {
-         _relay.set_token(token);
-      }
-
-      public void OnCompleted() { }
-      public void OnError(Exception error) => Log.Md.error(error.ToString());
-      public void OnNext(List<SecurityAndHandle> values)
-      {
-         if (values.Count > 0)
-         {
-            foreach (var v in values)
-            {
-               var smd = new SecurityMD(v.security, v.handle);
-               _exchange_sec_map.AddOrUpdate(smd.security.Id, smd, (_, __) => smd);
-            }
-            _version += 1;
-         }
-      }
-
-      public void Dispose() => Disposable.dispose(_dispose);
-      async protected Task run()
-      {
-         long run_version = 0;
-         while (true)
-         {
-            var exch = _exchange_sec_map;
-            long current_version = _version;
-            if (current_version > run_version)
-            {
-               run_version = current_version;
-               try
-               {
-                  Log.Md.info($"MDSource publishing {exch.Count} securities");
-                  _on_securities.OnNext(exch);
-               }
-               catch (Exception e)
-               {
-                  Log.Md.error($"Error in publish_securities: {e}");
-               }
-            }
-            await Task.Delay(TimeSpan.FromSeconds(1));
-         }
-      }
-
-   }
-}

src/client/spark_api/md/MarketDataClient.cs

@@ -11,224 +11,103 @@ using System.Reactive.Subjects;
 using bts.api;
 using bts.api.core;
 using bts.co2;
-using mbts;
-using SecurityKey = System.UInt32;
 
-namespace bts.md
+namespace bts.api.md
 {
-
-   public class MarketDataClient : IDisposable 
+   using security_key = System.UInt32;
+   public class MarketDataClient : IDisposable
    {
-      static uint naggle_ms = 100;      
-      readonly static bts.md.MDSource _subscription = new MDSource(naggle_ms);
-
-      readonly ConcurrentDictionary<SecurityKey, SecurityMD> _data = new ConcurrentDictionary<SecurityKey, SecurityMD>();
-      readonly List<IDisposable> _disp = new List<IDisposable>();
-      readonly uint _default_pump_ms;
-      public MarketDataClient(uint default_pump_ms = 250, AuthTokenImmutable? token = null)
+      class Data
       {
-         _default_pump_ms = default_pump_ms;
-         if (token != null)
-            set_auth_token(token);
-
-         _disp.Add(_subscription.on_securities.Subscribe( md =>
-         {
-            foreach (var entry in md)
-               _data.AddOrUpdate(entry.Key, entry.Value, (_, __) => entry.Value);
-         }));
-      }
-      public IEnumerable<SecurityImmutable> securities => _data.Values.Select(x => x.security);
-      public IObservable<IEnumerable<SecurityImmutable>> on_securities
-      {
-         get { return _subscription.on_securities.Select(x => x.Values.Select(xx => xx.security)); }
+         public long last_update_time;
+         public ChannelUpdate data;
       }
 
-      public void set_auth_token(AuthTokenImmutable? toke)
-      {
-         if(toke.HasValue)
-            _subscription.set_token(toke.Value);
-      }
-      public IObservable< IEnumerable<IMDBook> > subscribe_book(Func<SecurityImmutable, bool> filter,
-                                                                TimeSpan? update_interval = null, IScheduler scheduler = null)
-      {
-         return _subscribe<MDBook>(filter, update_interval, scheduler);
-      }
-      public IObservable< IEnumerable<IMDLast> > subscribe_last(Func<SecurityImmutable, bool> filter,
-                                                                TimeSpan? update_interval = null, IScheduler scheduler = null)
-      {
-         return _subscribe<MDLast>(filter, update_interval, scheduler);
-      }
-      public IObservable< IEnumerable<IMDMisc> > subscribe_misc(Func<SecurityImmutable, bool> filter,
-                                                                TimeSpan? update_interval = null, IScheduler scheduler = null)
-      {
-         return _subscribe<MDMisc>(filter, update_interval, scheduler);
-      }
-      public IObservable< IEnumerable<IMDCombined> > subscribe_combined(Func<SecurityImmutable, bool> filter,
-                                                                        TimeSpan? update_interval = null, IScheduler scheduler = null)
-      {
-         return _subscribe<MDCombined>(filter, update_interval, scheduler);
-      }
+      const UInt32 _all_flags = (uint)(
+            MDRequest.Types.Flags.Security | 
+            MDRequest.Types.Flags.Bbo |
+            MDRequest.Types.Flags.Direct | 
+            MDRequest.Types.Flags.Implied | 
+            MDRequest.Types.Flags.LastTrade);
       
-      IObservable<IEnumerable<T>> _subscribe<T>(Func<SecurityImmutable, bool> filter, 
-                                                TimeSpan? update_interval = null, IScheduler scheduler = null) where T : IMDData, IMDUpdate, new()
+      readonly ConcurrentDictionary<security_key, Data> _data = new ConcurrentDictionary<security_key, Data>();
+      readonly ConcurrentDictionary<security_key, string> _security_key_to_desc= new ConcurrentDictionary<security_key, string>();
+      readonly ConcurrentDictionary<string, security_key> _security_desc_to_key = new ConcurrentDictionary<string, security_key>();      
+      readonly MDRequest _subscription;
+      readonly Venue _venue;
+
+      readonly static Lazy<MarketDataRelay> _md = new Lazy<MarketDataRelay>(() => Locator.locate<MarketDataRelay>(), true);
+      readonly static ConcurrentDictionary<api.Venue, MarketDataClient> _venue_cache = new ConcurrentDictionary<api.Venue, api.md.MarketDataClient>();
+      static readonly List<Product> _all_products;
+      static MarketDataClient()
       {
-         List<Slot<T>> slots = new List<Slot<T>>();  int data_size = 0;
-         return timer_pump<IEnumerable<T>>((last_timer_tick, observer) =>
-         {
-            //if the number of securities has changed, or this is the first call
-            if (data_size != _data.Count)
-            {               
-               slots = _data.Values.Where(x => filter(x.security)).Select(xx => new Slot<T>(xx)).ToList();
-               data_size = _data.Count;
-            }
-            List<T> updates = new List<T>();
-            foreach (var slot in slots)
-            {
-               bool did_update = slot.data.update(slot.secmd, ref slot.last_update_time);
-               //if (slot.last_update_time > last_timer_tick)
-               if(did_update)
-                  updates.Add(slot.data);
-            }
-            if(updates.Count > 0)
-               observer.OnNext(updates);
-         },
-         update_interval, scheduler);
+         _all_products = _md.Value.request_products(new VoidMessage()).Products.ToList();
       }
 
-      public IObservable<IEnumerable<IMDBook>> subscribe_book(IEnumerable<SecurityKey> ids, TimeSpan? update_interval = null,
-                                                              IScheduler scheduler = null)
+      public static IEnumerable<api.Venue> cached_exchanges => _venue_cache.Keys;
+      public static bool create_or_get(api.Venue exchange, out MarketDataClient md, user.AuthToken token = null)
       {
-         return _subscribe<MDBook>(ids, update_interval, scheduler);
-      }
-      public IObservable<IEnumerable<IMDLast>> subscribe_last(IEnumerable<SecurityKey> ids, TimeSpan? update_interval = null,
-                                                              IScheduler scheduler = null)
-      {
-         return _subscribe<MDLast>(ids, update_interval, scheduler);
-      }
-      public IObservable<IEnumerable<IMDMisc>> subscribe_misc(IEnumerable<SecurityKey> ids, TimeSpan? update_interval = null,
-                                                              IScheduler scheduler = null)
-      {
-         return _subscribe<MDMisc>(ids, update_interval, scheduler);
-      }
-      public IObservable<IEnumerable<IMDCombined>> subscribe_combined(IEnumerable<SecurityKey> ids, TimeSpan? update_interval = null,
-                                                                      IScheduler scheduler = null)
-      {
-         return _subscribe<MDCombined>(ids, update_interval, scheduler);
-      }
-
-      IObservable<IEnumerable<T>> _subscribe<T>(IEnumerable<SecurityKey> ids, TimeSpan? update_interval = null, 
-                                                IScheduler scheduler = null) where T : IMDData, IMDUpdate, new()
-      {
-         List<Slot<T>> slots = ids.Where(ii => _data.TryGetValue(ii, out SecurityMD _)).Select(x =>
+         if (!_venue_cache.TryGetValue(exchange, out md))
          {
-            _data.TryGetValue(x, out SecurityMD smd);
-            return new Slot<T>(smd);
-         }).ToList();
-         return timer_pump<IEnumerable<T>>((last_timer_tick, observer) =>
-         {
-            List<T> updates = new List<T>();
-            foreach (var slot in slots)
-            {
-               bool did_update = slot.data.update(slot.secmd, ref slot.last_update_time);
-               //if (slot.last_update_time > last_timer_tick)
-               if(did_update)
-                  updates.Add(slot.data);                  
-            }
-            if(updates.Count > 0)
-               observer.OnNext(updates);
-         },
-         update_interval, scheduler);
-      }      
-
-      class Slot<T> where T : IMDData, IMDUpdate, new()
-      {
-         public Slot(SecurityMD md)
-         {
-            secmd = md;
-            last_update_time = 0;
-            data = new T();
+            _venue_cache[exchange] = md = new MarketDataClient(exchange, token);
+            return true;
          }
-         public SecurityMD secmd;
-         public ulong last_update_time;
-         public T data;
-      }            
-
-      IObservable<T> timer_pump<T>(Action<ulong, IObserver<T>> tick, TimeSpan? interval, IScheduler scheduler = null)
-      {
-         if(!interval.HasValue)
-            interval = TimeSpan.FromMilliseconds(_default_pump_ms);
-         ulong last_update_time = 0;
-         return Observable.Create<T>(observer =>
-         {            
-            return Observable.Timer(interval.Value, interval.Value, scheduler ?? Scheduler.Default).Subscribe(_ =>
-            {
-               tick(last_update_time, observer);
-               last_update_time = (ulong)DateTime.UtcNow.Ticks;
-            });
-         });
+         return false;
       }      
 
-      public void Dispose()
+      public MarketDataClient(bts.api.Venue exchange = Venue.Unset, user.AuthToken token = null) 
       {
-         _disp.ForEach(x => x.Dispose());
+         _venue = exchange;
+         _subscription = load_request(exchange, token);
       }
 
-      #region security search
-
-      public SecurityImmutable? lookup(SecurityKey key)
+      public void set_auth_token(user.AuthToken toke)
       {
-         if (_data.TryGetValue(key, out SecurityMD value))
-            return value.security;
+         _subscription.Token = toke?.Token ?? string.Empty;
+      }
+
+      MDRequest load_request(bts.api.Venue exchange, user.AuthToken token)
+      {
+         var req = new MDRequest() { Token = token?.Token ?? string.Empty } ;
+         var prods = _all_products.Where(x=>x.Exchange == exchange || exchange == bts.api.Venue.Unset);
+         foreach(var prod in prods)
+            req.Channels.Add(new SecurityChannel() { Exchange = prod.Exchange, Type = prod.Type, Symbol = prod.Symbol, Flags = 0 });
+         return req;
+      }
+
+      public Security lookup(security_key key)
+      {
+         Data value;
+         if(_data.TryGetValue(key, out value))
+            return value.data.Security;
          else
             return null;
       }
 
-      public bool lookup_security_desc(SecurityKey key, out string security_desc)
+      public bool lookup_security_desc(security_key key, out string security_desc)
       {
-         security_desc = null;
-         var sec = lookup(key);
-         if (sec.HasValue)
-         {
-            security_desc = sec.Value.glbx_desc();
-            return true;
-         }
-         return false;
+         return _security_key_to_desc.TryGetValue(key, out security_desc);
       }
 
-      public bool lookup_security_key_from_desc(string security_desc, out SecurityKey key)
+      public bool lookup_security_key_from_desc(string security_desc, out security_key key)
       {
-         key = 0;
-         var fnd = find(x => x.glbx_desc() == security_desc).TryFirst();
-         if (fnd.HasValue)
-         {
-            key = fnd.Value.security.Id;
-            return true;
-         }
-         return false;
+         return _security_desc_to_key.TryGetValue(security_desc, out key);
       }
 
-      IEnumerable<SecurityMD> find(Func<SecurityImmutable, bool> filter)
+      public Security find_security(Func<Security, bool> filter)
       {
-         Func<SecurityMD, bool> filt = (SecurityMD secmd) => filter(secmd.security);
-         return _data.Values.Where(filt);
+         return _data.Values.Select(x => x.data.Security).Where(xx => filter(xx)).FirstOrDefault();
       }
 
-      public SecurityImmutable? find_security(Func<SecurityImmutable, bool> filter)
+      public async Task<Security> find_security_async(Func<Security, bool> filter, TimeSpan? timeout = null)
       {
-         var fnd = find(filter).TryFirst();
-         return fnd?.security;
-      }
-
-      public async Task<SecurityImmutable?> find_security_async(Func<SecurityImmutable, bool> filter, TimeSpan? timeout = null)
-      {
-         if (!timeout.HasValue)
-         {
-            timeout = TimeSpan.FromMilliseconds(10000);
+         if(!timeout.HasValue) {
+            timeout = TimeSpan.FromMilliseconds(int.MaxValue-1);
          }
          CancellationTokenSource src = new CancellationTokenSource(timeout.Value);
-         SecurityImmutable? security = await Task.Run<SecurityImmutable?>(async () =>
+         Security security = await Task.Run<Security>(async () =>
         {
-           SecurityImmutable? sec = null;
+           Security sec = null;
            do
            {
               sec = find_security(filter);
@@ -238,11 +117,195 @@ namespace bts.md
            while (sec == null);
            return sec;
         }, src.Token);
-
+                 
          return security ?? throw new KeyNotFoundException("Can't find security for filter"); ;
       }
 
+      public IObservable<ChannelUpdate> subscribe(Product product, 
+                                                  TimeSpan? update_interval = null, IScheduler scheduler = null)
+      {
+         return subscribe(product, _all_flags, update_interval, scheduler);
+      }
 
-      #endregion
+
+      public IObservable<ChannelUpdate> subscribe(Product product, uint flags, 
+                                                  TimeSpan? update_interval = null, IScheduler scheduler = null)
+      {
+         check_subscriptions(flags, x =>
+            (x.Exchange == product.Exchange && x.Type == product.Type && x.Symbol == product.Symbol));
+         return _subscribe( 
+            x => (x.Exchange == product.Exchange && ((SecurityImmutable)x).security_type() == product.Type && x.Symbol == product.Symbol), 
+            update_interval,
+            scheduler);
+      }
+
+      IObservable<ChannelUpdate> _subscribe(Func<Security, bool> filter, 
+                                            TimeSpan? update_interval = null, IScheduler scheduler = null)
+      {
+         Data[] slots = new Data[0]; long local_security_count = 0;
+         return timer_pump<Data>((last_update, observer) =>
+        {
+           //if the number of securities has changed, or this is the first call
+           if (slots.Length == 0 || local_security_count != _data.Count)
+              slots = _data.Values.Where(xx => filter(xx.data.Security)).ToArray();
+           foreach (var slot in slots)
+           {
+              Data data;
+              if (_data.TryGetValue(slot.data.Security.Id, out data))
+              {
+                 if (data.last_update_time > last_update)
+                    observer.OnNext(data);
+              }
+           }
+        },
+         update_interval, scheduler).Select(x => x.data);
+      }
+
+      /// <summary>
+      /// Poll for updates for a security on a specified update_interval, on the specified scheduler with default flags
+      /// </summary>
+      /// <param name="security"></param>
+      /// <param name="update_interval"></param>
+      /// <param name="scheduler"></param>
+      /// <returns></returns>             
+      public IObservable<ChannelUpdate> subscribe(Security security, 
+                                                  TimeSpan? update_interval = null, IScheduler scheduler = null)
+      {
+         return subscribe(security, _all_flags, update_interval, scheduler);
+      }      
+
+      /// <summary>
+      /// Poll for updates for a security on a specified update_interval, on the specified scheduler
+      /// </summary>
+      /// <param name="security"></param>
+      /// <param name="flags"></param>
+      /// <param name="update_interval"></param>
+      /// <param name="scheduler"></param>
+      /// <returns></returns>
+      public IObservable<ChannelUpdate> subscribe(Security security, uint flags, 
+                                                  TimeSpan? update_interval = null, IScheduler scheduler = null)
+      {
+         check_subscriptions(flags, x => 
+            (x.Exchange == security.Exchange && x.Type == ((SecurityImmutable)security).security_type() && security.Symbol == security.Symbol));
+         return timer_pump<Data>( (last_update, observer) =>
+         {
+            Data data;
+            if (_data.TryGetValue(security.Id, out data))
+            {
+               if (data.last_update_time > last_update)
+                  observer.OnNext(data);
+            }
+         },
+         update_interval, scheduler).Select(x => x.data);
+      }
+      
+      IObservable<T> timer_pump<T>(Action<long, IObserver<T>> tick, TimeSpan? interval, IScheduler scheduler = null)
+      {
+         if(!interval.HasValue)
+            interval = TimeSpan.FromMilliseconds(25);
+         long last_update_time = 0;
+         return Observable.Create<T>(observer =>
+         {            
+            return Observable.Timer(interval.Value, interval.Value, scheduler ?? Scheduler.Default).Subscribe(_ =>
+            {
+               tick(last_update_time, observer);
+               last_update_time = DateTime.UtcNow.Ticks;
+            });
+         });
+      }
+
+      bool check_subscriptions(uint flags, Func<SecurityChannel, bool> selector)
+      {
+         foreach (var sub in _subscription.Channels)
+         {
+            if (selector(sub))
+            {
+               sub.Flags |= flags;
+               return true;
+            }
+         }
+         return false;
+      }
+
+      /// <summary>
+      /// Start pumping market data for all subscribed channels and securities. Call this without await
+      /// </summary>
+      /// <param name="interval"></param>
+      /// <returns></returns>
+      public async Task start_pump(TimeSpan? interval = null)
+      {
+         var rep = await _md.Value.request_names_async(new VoidMessage()); 
+         int count = 0;
+         do
+         {
+            foreach (var s in rep.value.Securities)
+            {
+               foreach (var ss in s.Names)
+               {
+                  _security_desc_to_key[ss.Value] = ss.Key;
+                  _security_key_to_desc[ss.Key] = ss.Value;
+               }
+            }
+         }
+         while (!rep.success && count++ < 5);
+         while (true)
+         {
+            var reply = await _md.Value.requestmd_async(_subscription);
+            if (reply.success)
+            {
+               foreach (var channel in reply.value.Channels)
+               {
+                  long now = DateTime.UtcNow.Ticks;
+                  foreach (var update in channel.Updates)
+                  {
+                     Data data;
+                     if (_data.TryGetValue(update.Security.Id, out data))
+                     {
+                        //mergefrom concats repeated fields                        
+                        if (update.Direct != null)
+                           data.data.Direct = null;
+                        //this can tear, but the book won't tear, so maybe it's ok
+                        data.data.MergeFrom(update);
+                        data.last_update_time = now;
+                     }
+                     else
+                     {
+                        _data[update.Security.Id] = new Data() { last_update_time = now, data = update };
+                     }
+                  }
+               }
+               await Task.Delay(interval ?? TimeSpan.FromMilliseconds(100), _cts.Token);
+            }
+         }
+      }
+
+      Task _pump_task;
+      public void run_pump(TimeSpan? interval = null)
+      {
+         if (_pump_task == null)
+         {
+            _pump_task = this.start_pump(interval)
+               .ContinueWith(ex =>
+               {
+                  System.Diagnostics.Debug.Assert(ex == null, ex.ToString());
+               }
+               , TaskContinuationOptions.OnlyOnFaulted);
+         }
+      }
+
+      public async void run_pump_async(TimeSpan? interval = null)
+      {
+         await this.start_pump(interval);
+      }
+
+
+      readonly CancellationTokenSource _cts = new CancellationTokenSource();
+      public void Dispose()
+      {
+         _cts.Cancel();
+         _cts.Dispose();
+         if (_md.IsValueCreated)
+            _md.Value.Dispose();
+      }
    }
 }

src/client/spark_api/md/PriceSize.cs

@@ -1,24 +0,0 @@
-using bts.api;
-using mbts.co2;
-
-namespace bts.md
-{
-   public class PriceSize 
-   {      
-      public uint size { get; }
-      public uint direct_size { get; }
-      public uint implied_size { get; }
-      public double price { get; }
-      public PriceSize(uint sz, uint direct_sz, uint implied_sz, double prx)
-      {
-         size = sz;
-         direct_size = direct_sz;
-         implied_size = implied_sz;
-         price = prx;
-      }
-      public override string ToString()
-      {
-         return $"{price} {size}  {implied_size} ";
-      }
-   }
-}

src/client/spark_api/utils/APIExtensions.cs

@@ -1,34 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Threading.Tasks;
-
-namespace mbts
-{
-   static class APIExtensions
-   {
-      public static T? TryFirst<T>(this IEnumerable<T> src) where T : struct
-      {
-         foreach (var v in src)
-            return v;
-         return null;
-      }
-
-      public static DateTime lldatetime(this ulong ll)
-      {
-         // Warning: Do not remove these consts, or try to simplify this method.
-         // System.DateTime() requires bits 63-64 to be empty in the "DateTime(long ticks)" constructor.
-         // The only way to set the DateTimeKind is through the "DateTime(long ticks, DateTimeKind kind)" constructor.
-         // (Yes, it is annoying. Internally, System.DateTime stores the DateTimeKind in bits 63-64 - in the exact format we already have!!!)
-         const ulong ticks_mask = 0x3FFFFFFFFFFFFFFF;
-         const int kind_shift = 62;
-         return new DateTime((long)(ll & ticks_mask), (DateTimeKind)(ll >> kind_shift));
-      }
-      
-      public static string fmt(this string input, params object[] vals)
-      {
-         return string.Format(input, vals);
-      }
-
-      public static void ignore(this Task t) { }         
-   }
-}

src/client/spark_core/Co2/Co2NativeMethods.cs

@@ -13,9 +13,14 @@ namespace mbts.co2
    {
       internal static unsafe T deserialize<T>(byte* buffer, ulong length) where T : new() {
          var stream = new System.IO.UnmanagedMemoryStream(buffer, (long)length);
-         IMessage result = (IMessage)new T();
-         result.MergeFrom(stream);
-         return (T)result;
+         if (typeof(IMessage).IsAssignableFrom(typeof(T))) {
+            IMessage result = (IMessage)new T();
+            result.MergeFrom(stream);
+            return (T)result;
+         }
+         else {
+            return ProtoBuf.Serializer.Deserialize<T>(stream);
+         }
       }
 
       [DllImport("libspark")]
@@ -76,7 +81,7 @@ namespace mbts.co2
       [DllImport("libspark")]
       internal unsafe static extern IntPtr add_relay_symbol(IntPtr relay, UserSymbol symbol, bts.co2.MDRequest.Types.Flags flags);
       [DllImport("libspark")]
-      internal unsafe static extern IntPtr set_relay_token(IntPtr relay, byte[] token, long token_size);
+      internal unsafe static extern IntPtr set_relay_token(IntPtr relay, string user_name);
 
       [DllImport("libspark")]
       internal unsafe static extern void post_on_scan_security_thread(Action callback);

src/client/spark_core/Co2/Co2RelayControl.cs

@@ -14,16 +14,17 @@ namespace mbts.co2
    {
       MDConsumer _consumer;
       IntPtr _relay;
-      public Co2RelayControl(MDConsumer consumer, uint naggle_ms) {
+      public Co2RelayControl(MDConsumer consumer) {
          _consumer = consumer;
          Log.assert(consumer.native_pointer() != IntPtr.Zero);
-         _relay = Co2NativeMethods.start_private_relay(consumer.native_pointer(), naggle_ms);
+         _relay = 
+            Co2NativeMethods.start_private_relay(consumer.native_pointer(),
+               config.Configuration.get<bts.md.RelayOptions>().naggle_ms);
          Log.assert(_relay != IntPtr.Zero);
       }
 
-      public void set_token(AuthTokenImmutable token) {
-         var buffer = token.Token.ToByteArray();
-         Co2NativeMethods.set_relay_token(_relay, buffer, buffer.LongLength);
+      public void set_token(AuthToken token) {
+         Co2NativeMethods.set_relay_token(_relay, token.Token);
       }
 
       public async Task preload_securities(MDRequest.Types.Flags flags=MDRequest.Types.Flags.Security) {

src/client/spark_core/Co2/MDConsumer.cs

@@ -9,7 +9,6 @@ using System.Runtime.InteropServices;
 using System.Text;
 using System.Threading.Tasks;
 using security_key = System.UInt32;
-using bts;
 
 namespace mbts.co2
 {

src/client/spark_core/utils/GlobalExtensions.cs

@@ -6,6 +6,9 @@ using System.Text;
 
 namespace mbts {
    public static class GlobalExtensions {
+      public static string fmt(this string input, params object[] vals) {
+         return string.Format(input, vals);
+      }
       public static string Repeat(this string s, uint count) {
          if (count == 0) return "";
          if (count == 1) return s;
@@ -24,6 +27,15 @@ namespace mbts {
          return range.Skip(index).First();
       }
 
+      public static DateTime lldatetime(this ulong ll) {
+         // Warning: Do not remove these consts, or try to simplify this method.
+         // System.DateTime() requires bits 63-64 to be empty in the "DateTime(long ticks)" constructor.
+         // The only way to set the DateTimeKind is through the "DateTime(long ticks, DateTimeKind kind)" constructor.
+         // (Yes, it is annoying. Internally, System.DateTime stores the DateTimeKind in bits 63-64 - in the exact format we already have!!!)
+         const ulong ticks_mask = 0x3FFFFFFFFFFFFFFF;
+         const int kind_shift = 62;
+         return new DateTime((long)(ll & ticks_mask), (DateTimeKind)(ll >> kind_shift));
+      }
       
       //this overloaded declaration, (as opposed to one function with comp = null) allows observable_collection.sync_with to be passed into functions as a parameterless Action
       public static void sync_with<T>(this IList<T> dest, IEnumerable<T> src) { dest.sync_with(src, EqualityComparer<T>.Default); }
@@ -58,6 +70,11 @@ namespace mbts {
          }
       }
 
-
+      public static T? TryFirst<T>(this IEnumerable<T> src) where T : struct
+      {
+         foreach(var v in src)
+            return v;
+         return null;
+      }
    }
 }

src/client/spark_excel/extensions/OrderService_Excel.cs

@@ -17,7 +17,7 @@ namespace bts.api.orderman
 {
    public partial class OrderService_Excel 
    {
-      readonly static Lazy<user.AuthService> _users = new Lazy<user.AuthService>(() => Locator.locate<user.AuthService>(), true);
+      readonly static Lazy<user.UserService> _users = new Lazy<user.UserService>(() => Locator.locate<user.UserService>(), true);
       readonly static Lazy<admin.AdminService> _admin = new Lazy<admin.AdminService>(() => Locator.locate<admin.AdminService>(), true);
 
       [ExcelFunction(Category = "BTS", Description = "Get All Accessible Accounts")]
@@ -26,7 +26,7 @@ namespace bts.api.orderman
          try
          {
             List<string> accts = new List<string>();
-            var admin = await _admin.Value.get_filtered_admin_data_async(new FilteredAdminDataRequest());
+            var admin = await _admin.Value.get_filtered_admin_data_async(new FilteredAdminDataRequest { Token = login_token });
             if (admin.success)
             {
                foreach (var acc in admin.value.Accounts)

src/client/spark_excel/rtd/formatting/BookMDFormatter.cs

@@ -1,22 +1,21 @@
 using System;
-using bts.md;
 
 namespace bts.excel.rtd
 {
    class BookMDFormatter : IExcelFormatter
    {
-      readonly IMDBook _update;
-      public BookMDFormatter(IMDBook upd)
+      readonly co2.ChannelUpdate _update;
+      public BookMDFormatter(co2.ChannelUpdate upd)
       {
          _update = upd;
       }
 
-      public object[,] format(bool include_headers = false)
+      public static double format_md(long mantissa, int exponent)
       {
-         return format(_update, include_headers);
+         return mantissa * Math.Pow(10, exponent);
       }
 
-      static public object[,] format(IMDBook update, bool include_headers = false)
+      public object[,] format(bool include_headers = false)
       {
          const int cols = 5;
          const int rows = 30;
@@ -25,21 +24,21 @@ namespace bts.excel.rtd
             for (int j = 0; j < cols; j++)
                ret[i, j] = string.Empty;
 
-         if (update != null)
+         if (_update.Direct != null)
          {
             int idx = 0;
-            for (int j = update.asks.Count - 1; j >= 0 && idx < rows; j--)
+            for (int j = _update.Direct.Asks.Count - 1; j >= 0 && idx < rows; j--)
             {
-               var ask = update.asks[j];
-               ret[idx, 1] = ask.price;
-               ret[idx, 2] = ask.size;
+               var ask = _update.Direct.Asks[j];
+               ret[idx, 1] = format_md(ask.Mantissa, ask.Exponent);
+               ret[idx, 2] = ask.Size;
                idx++;
             }
-            for (int j = 0; j < update.bids.Count && idx < rows; j++)
+            for (int j = 0; j < _update.Direct.Bids.Count && idx < rows; j++)
             {
-               var bid = update.bids[j];
-               ret[idx, 1] = bid.price;
-               ret[idx, 0] = bid.size;
+               var bid = _update.Direct.Bids[j];
+               ret[idx, 1] = format_md(bid.Mantissa, bid.Exponent);
+               ret[idx, 0] = bid.Size;
                idx++;
             }
          }

src/client/spark_excel/rtd/formatting/FormatterFactory.cs

@@ -1,5 +1,4 @@
 using System;
-using bts.md;
 
 namespace bts.excel.rtd
 {
@@ -7,7 +6,7 @@ namespace bts.excel.rtd
    {
       public static IExcelFormatter make_formatter(object data)
       {
-         if(data is IMDBook upd)
+         if(data is co2.ChannelUpdate upd)
             return new BookMDFormatter(upd);
          else
          {

src/client/spark_excel/rtd/formatting/LastMDFormatter.cs

@@ -1,35 +0,0 @@
-using System;
-using bts.md;
-
-namespace bts.excel.rtd
-{
-   class LastMDFormatter : IExcelFormatter
-   {
-      readonly IMDLast _update;
-      public LastMDFormatter(IMDLast upd)
-      {
-         _update = upd;
-      }
-
-      public object[,] format(bool include_headers = false)
-      {
-         return format(_update, include_headers);
-      }
-
-      static public object[,] format(IMDLast update, bool include_headers = false)
-      {
-         const int cols = 3;
-         const int rows = 1;
-         var ret = new object[rows, cols];
-         
-         if (update != null)
-         {
-            ret[0, 0] = update.last?.price;
-            ret[0, 1] = update.last?.size;
-            ret[0, 1] = update.last_time;
-         }
-         return ret;
-      }
-   }
-
-}

src/client/spark_excel/rtd/formatting/MiscMDFormatter.cs

@@ -1,22 +1,16 @@
 using System;
-using bts.md;
 
 namespace bts.excel.rtd
 {
    class MiscMDFormatter : IExcelFormatter
    {
-      readonly IMDMisc _update;
-      public MiscMDFormatter(IMDMisc upd)
+      readonly co2.ChannelUpdate _update;
+      public MiscMDFormatter(co2.ChannelUpdate upd)
       {
          _update = upd;
       }
 
       public object[,] format(bool include_headers = false)
-      {
-         return format(_update, include_headers);
-      }
-
-      static public object[,] format(IMDMisc update, bool include_headers = false)
       {
          return new object[0, 0];
       }

src/client/spark_excel/rtd/formatting/TopOfBookMDFormatter.cs

@@ -1,22 +1,17 @@
+
 using System;
-using bts.md;
 
 namespace bts.excel.rtd
 {
    class TopOfBookMDFormatter : IExcelFormatter
    {
-      readonly IMDBook _update;
-      public TopOfBookMDFormatter(IMDBook upd)
+      readonly co2.ChannelUpdate _update;
+      public TopOfBookMDFormatter(co2.ChannelUpdate upd)
       {
          _update = upd;
       }
 
       public object[,] format(bool include_headers = false)
-      {
-         return format(_update, include_headers);
-      }
-
-      static public object[,] format(IMDBook update, bool include_headers = false)
       {
          const int cols = 4;
          const int rows = 1;
@@ -25,22 +20,22 @@ namespace bts.excel.rtd
             for (int j = 0; j < cols; j++)
                ret[i, j] = string.Empty;
          
-         if (update != null)
+         if (_update.Direct != null)
          {
             int idx = 0;
-            for (int j = 0; j < update.asks.Count && idx < rows; j++)
+            for (int j = 0; j < _update.Direct.Asks.Count && idx < rows; j++)
             {
-               var ask = update.asks[j];
-               ret[idx, 2] = ask.price;
-               ret[idx, 3] = ask.size;
+               var ask = _update.Direct.Asks[j];
+               ret[idx, 2] = BookMDFormatter.format_md(ask.Mantissa, ask.Exponent);
+               ret[idx, 3] = ask.Size;
                idx++;
             }
             idx = 0;
-            for (int j = 0; j < update.bids.Count && idx < rows; j++)
+            for (int j = 0; j < _update.Direct.Bids.Count && idx < rows; j++)
             {
-               var bid = update.bids[j];
-               ret[idx, 1] = bid.price;
-               ret[idx, 0] = bid.size;
+               var bid = _update.Direct.Bids[j];
+               ret[idx, 1] = BookMDFormatter.format_md(bid.Mantissa, bid.Exponent);
+               ret[idx, 0] = bid.Size;
                idx++;
             }
          }

src/client/spark_excel/rtd/formatting/TopicMDFormatter.cs

@@ -1,95 +1,86 @@
 
 using System;
-using bts.md;
 
 namespace bts.excel.rtd
 {
    class TopicMDFormatter : IExcelFormatter
    {
-      readonly IMDCombined _update;
+      readonly co2.ChannelUpdate _update;
       readonly string _topic;
-      public TopicMDFormatter(IMDCombined upd, string topic)
+      public TopicMDFormatter(co2.ChannelUpdate upd, string topic)
       {
          _update = upd;
          _topic = topic;
       }
 
       public object[,] format(bool include_headers = false)
-      {
-         return format(_update, _topic, include_headers);
-      }
-
-      static public object[,] format(IMDCombined update, string topic, bool include_headers = false)
       {
          var retval = new object[1, 1];
-         if(update == null || string.IsNullOrEmpty(topic))
-            return retval;
-
-         switch (topic.ToLower())
+         switch (_topic.ToLower())
          {
             case "bid":
             case "bidprice":
-               retval[0, 0] = update.bids?[0]?.price; 
+               retval[0, 0] = BookMDFormatter.format_md(_update?.Direct?.Bids?[0]?.Mantissa ?? 0, _update?.Direct?.Bids?[0]?.Exponent ?? 0);
                break;
             case "ask":
             case "askprice":
-               retval[0, 0] = update.asks?[0]?.price;
+               retval[0, 0] = BookMDFormatter.format_md(_update?.Direct?.Asks?[0]?.Mantissa ?? 0, _update?.Direct?.Asks?[0]?.Exponent ?? 0);
                break;
             case "bidsize":
             case "bidqty":
-               retval[0, 0] = update.bids?[0]?.size;
+               retval[0, 0] = _update?.Direct?.Bids?[0]?.Size;
                break;
             case "asksize":
             case "askqty":
-               retval[0, 0] = update.asks?[0]?.size;
+               retval[0, 0] = _update?.Direct?.Asks?[0]?.Size;
                break;
             case "impliedbid":
             case "impliedbidprice":
-               retval[0, 0] = update.bids?[0]?.price; //TODO is this the first price with impsz > 0?
+               retval[0, 0] = BookMDFormatter.format_md(_update?.Implied?.Bids?[0]?.Mantissa ?? 0, _update?.Implied?.Bids?[0]?.Exponent ?? 0);
                break;
             case "impliedask":
             case "impliedaskprice":
-               retval[0, 0] = update.asks?[0]?.price;
+               retval[0, 0] = BookMDFormatter.format_md(_update?.Implied?.Asks?[0]?.Mantissa ?? 0, _update?.Implied?.Asks?[0]?.Exponent ?? 0);
                break;
             case "impliedbidsize":
             case "impliedbidqty":
-               retval[0, 0] = update.bids?[0]?.implied_size;
+               retval[0, 0] = _update?.Implied?.Bids?[0]?.Size;
                break;
             case "impliedasksize":
             case "impliedaskqty":
-               retval[0, 0] = update.asks?[0]?.implied_size;
+               retval[0, 0] = _update?.Implied?.Asks?[0]?.Size;
                break;
             case "settlement":
             case "settlementprice":
-               retval[0, 0] = update.settlement_price;
+               retval[0, 0] = BookMDFormatter.format_md(_update?.Misc?.Settlement?.Mantissa ?? 0, _update?.Misc?.Settlement?.Exponent ?? 0);
                break;
             case "previoussettlement":
             case "previoussettlementprice":
-               retval[0, 0] = update.previous_settlement_price;
+               retval[0, 0] = BookMDFormatter.format_md(_update?.Misc?.PreviousSettlementPrice?.Mantissa ?? 0, _update?.Misc?.PreviousSettlementPrice?.Exponent ?? 0);
                break;
             case "open":
             case "openprice":
-               retval[0, 0] = update.open_price;
+               retval[0, 0] = _update?.Misc?.OpenPrice;
                break;
             case "close":
             case "closeprice":
-               retval[0, 0] = update.close_price;
+               retval[0, 0] = _update?.Misc?.ClosePrice;
                break;
             case "high":
             case "highprice":
-               retval[0, 0] = update.session_high;
+               retval[0, 0] = _update?.Misc?.SessionHigh;
                break;
             case "low":
             case "lowprice":
-               retval[0, 0] = update.session_low;
+               retval[0, 0] = _update?.Misc?.SessionLow;
                break;
             case "last":
             case "lastprice":
-               retval[0, 0] = update?.last?.price;
+               retval[0, 0] = BookMDFormatter.format_md(_update?.LastTrade?.Mantissa ?? 0, _update?.LastTrade?.Exponent ?? 0);
                break;
             case "lastsize":
             case "lastqty":
-               retval[0, 0] = update?.last?.size;
+               retval[0, 0] = _update?.LastTrade?.Size;
                break;               
          }
          return retval;

src/client/spark_excel/rtd/subscriptions/MDSubscribe.cs

@@ -7,26 +7,26 @@ using ExcelDna.Integration.Rtd;
 using ExcelDna.Registration.Utils;
 using System.Reactive.Linq;
 using bts.api;
-using bts.md;
+using bts.api.md;
 using bts.api.orderman;
 
 namespace bts.excel.rtd
 {
    public class MDSubscribe
    {
+      static Authenticator _auth = new Authenticator();
       [ExcelFunction(Category = "BTS", Description = "Log in to BTS Spark")]
       async public static void spark_login(string login, string password, ExcelAsyncHandle handle) 
       {
          try
          {
-            var res = await Locator.login(login, password);
-            if (res.success && Locator.token.HasValue)
+            var toke = await _auth.login(login, password);
+            foreach (var ven in MarketDataClient.cached_exchanges)
             {
-               _md.set_auth_token(Locator.token);
-               handle.SetResult("Login Successful");
+               MarketDataClient.create_or_get(ven, out MarketDataClient md);
+               md?.set_auth_token(toke);
             }
-            else
-               handle.SetResult("Login Error: Could not log in check user/pass " + res.reason);
+            handle.SetResult(toke?.Token ?? "Could not log in check user/pass");
          }
          catch(Exception ex)
          {
@@ -34,10 +34,12 @@ namespace bts.excel.rtd
          }
       }
 
-      static MarketDataClient _md = new MarketDataClient();
       public static MarketDataClient market_data(api.Venue exchange)
       {
-         return _md;
+         var newc = MarketDataClient.create_or_get(exchange, out MarketDataClient md);
+         if(newc)
+            md.run_pump_async(TimeSpan.FromMilliseconds(333));
+         return md;
       }
 
       static public object spark_lookup_security_desc(string exchange, long security_key)
@@ -49,7 +51,7 @@ namespace bts.excel.rtd
          return desc ?? string.Empty;
       }
 
-      static IObservable<IEnumerable<object>> md_observable(string contract, string symbol, string security_type, string exchange)
+      static IObservable<object> md_observable(string contract, string symbol, string security_type, string exchange)
       {
          security_type = string.IsNullOrEmpty(security_type) ? SecurityType.Future.ToString() : security_type;
          exchange = string.IsNullOrEmpty(exchange) ? Security.ExchangeDataOneofCase.Glbx.ToString() : exchange;
@@ -58,18 +60,18 @@ namespace bts.excel.rtd
          Venue ven;
          SecurityType sectype;
          if (!Utilities.parse_venue(exchange, out ven))
-            return Observable.Return(new [] {"Can't Parse Exchange: " + exchange ?? string.Empty});
+            return Observable.Return("Can't Parse Exchange: " + exchange ?? string.Empty);
          if (!Utilities.parse_security_type(security_type, out sectype))
-            return Observable.Return(new [] {"Can't Parse Security Type: " + security_type ?? string.Empty});
+            return Observable.Return("Can't Parse Security Type: " + security_type ?? string.Empty);
          if (string.IsNullOrEmpty(symbol))
-            return Observable.Return(new []{"Can't Parse Symbol: " + symbol ?? string.Empty});
+            return Observable.Return("Can't Parse Symbol: " + symbol ?? string.Empty);
          if (string.IsNullOrEmpty(contract))
-            return Observable.Return(new []{"Can't Parse Contract: " + contract ?? string.Empty});
+            return Observable.Return("Can't Parse Contract: " + contract ?? string.Empty);
 
-         Func<SecurityImmutable, bool> where = (x => x.Exchange == ven && x.Symbol == symbol && x.security_type() == sectype &&
-                                                x.Glbx.SecurityDesc == contract);
+         Product prod = new Product() { Exchange = ven, Type = sectype, Symbol = symbol };
+         Func<Security, bool> filter = (x => x.Glbx.SecurityDesc == contract); //TODO GLBX
 
-         var stream = market_data(ven).subscribe_combined(where);
+         var stream =  market_data(ven).subscribe(prod).Where(x => filter(x.Security));
          return stream;
       }      
 
@@ -83,11 +85,10 @@ namespace bts.excel.rtd
 
          var param = new object[] { contract, symbol, security_type, exchange };
          return ObservableRtdUtil.Observe(nameof(spark_md_book), param, 
-                                          () => md_observable(contract, symbol, security_type, exchange).Select(data =>
+                                          () => md_observable(contract, symbol, security_type, exchange).Select(x =>
                                           {
-                                             var x = data.FirstOrDefault();
-                                             if (x != null && x is IMDBook upd)
-                                                return BookMDFormatter.format(upd);
+                                             if (x is co2.ChannelUpdate upd)
+                                                return new BookMDFormatter(upd).format();
                                              return x;
                                           }));
          // security_type = string.IsNullOrEmpty(security_type) ? SecurityType.Future.ToString() : security_type;
@@ -110,33 +111,13 @@ namespace bts.excel.rtd
       {
          var param = new object[] { contract, symbol, security_type, exchange };
          return ObservableRtdUtil.Observe(nameof(spark_md_top_of_book), param,
-                                          () => md_observable(contract, symbol, security_type, exchange).Select(data =>
+                                          () => md_observable(contract, symbol, security_type, exchange).Select(x =>
                                           {
-                                             var x = data.FirstOrDefault();
-                                             if (x != null && x is IMDBook upd)
-                                                return TopOfBookMDFormatter.format(upd);
+                                             if (x is co2.ChannelUpdate upd)
+                                                return new TopOfBookMDFormatter(upd).format();
                                              return x;
                                           }));
-      }
-
-      [ExcelFunction(Category = "BTS", Description = "Subscribe to BTS Spark last trade market data")]
-      public static object spark_md_last(
-                                    [ExcelArgument("Required. The specific contract, GCZ8 or ZNU8-ZNZ8, etc")] string contract,
-                                    [ExcelArgument("Required. The symbol, GC or ZN, etc")] string symbol,
-                                    [ExcelArgument("Optional. The type, Future or Spread, default = Future")] string security_type = null,
-                                    [ExcelArgument("Optional. The exchange, default = GLBX")] string exchange = null)
-      {
-         var param = new object[] { contract, symbol, security_type, exchange };
-         return ObservableRtdUtil.Observe(nameof(spark_md_last), param,
-                                          () => md_observable(contract, symbol, security_type, exchange).Select(data =>
-                                          {
-                                             var x = data.FirstOrDefault();
-                                             if (x != null && x is IMDLast upd)
-                                                return LastMDFormatter.format(upd);
-                                             return x;
-                                          }));
-
-      }
+      }      
 
       [ExcelFunction(Category = "BTS", Description = "Subscribe to BTS Spark miscellaneous market data")]
       public static object spark_md_misc(
@@ -147,17 +128,16 @@ namespace bts.excel.rtd
       {
          var param = new object[] { contract, symbol, security_type, exchange };
          return ObservableRtdUtil.Observe(nameof(spark_md_misc), param,
-                                          () => md_observable(contract, symbol, security_type, exchange).Select(data =>
+                                          () => md_observable(contract, symbol, security_type, exchange).Select(x =>
                                           {
-                                             var x = data.FirstOrDefault();
-                                             if (x != null && x is IMDMisc upd)
-                                                return MiscMDFormatter.format(upd);
+                                             if (x is co2.ChannelUpdate upd)
+                                                return new MiscMDFormatter(upd).format();
                                              return x;
                                           }));
 
       }      
       
-      [ExcelFunction(Category = "BTS", Description = "Subscribe to BTS Spark topic based market data")]
+      [ExcelFunction(Category = "BTS", Description = "Subscribe to BTS Spark top of book market data")]
       public static object spark_md(
                                     [ExcelArgument("Required. The 'Topic' string, 'bid', 'ask', etc")] string topic,
                                     [ExcelArgument("Required. The specific contract, GCZ8 or ZNU8-ZNZ8, etc")] string contract,
@@ -167,14 +147,12 @@ namespace bts.excel.rtd
       {
          var param = new object[] { topic, contract, symbol, security_type, exchange };
          return ObservableRtdUtil.Observe(nameof(spark_md), param,
-                                          () => md_observable(contract, symbol, security_type, exchange).Select(data =>
+                                          () => md_observable(contract, symbol, security_type, exchange).Select(x =>
                                           {
-                                             var x = data.FirstOrDefault();
-                                             if (x != null && x is IMDCombined upd)
-                                                return TopicMDFormatter.format(upd, topic);
+                                             if (x is co2.ChannelUpdate upd)
+                                                return new TopicMDFormatter(upd, topic).format();
                                              return x;
                                           }));
       }           
-      
    }
 }

src/client/spark_sdk/samples/MarketDataSample.cs

@@ -9,7 +9,8 @@ using System.Reactive.Concurrency;
 
 using bts.api;
 using bts.api.core;
-using bts.md;
+using bts.co2;
+using bts.api.md;
 
 namespace bts.sdk
 {
@@ -23,13 +24,10 @@ namespace bts.sdk
          _md = client;
       }
 
-      void print_update(IEnumerable<IMDData> updates)
+      void print_update(ChannelUpdate update)
       {
-         foreach (var update in updates)
-         {
-            Console.WriteLine(update.ToString());
-            Console.WriteLine();
-         }
+         Console.WriteLine(update.ToString());
+         Console.WriteLine();
       }
       
       public static Venue parse_venue(string exch)
@@ -48,14 +46,12 @@ namespace bts.sdk
          return st;         
       }
 
-      public void print_product(string exch, string sectype, string symbol)
+      public void print_channel(string exch, string sectype, string symbol)
       {
          Venue ven = parse_venue(exch);
          SecurityType st = parse_security_type(sectype);
          Product prod = new Product() { Exchange = ven, Type = st, Symbol = symbol };
-         Func<SecurityImmutable, bool> where = (x => x.Exchange == ven && x.Symbol == symbol && x.security_type() == st);
-
-         _disps.Add(_md.subscribe_book(where).Subscribe(x =>
+         _disps.Add(_md.subscribe(prod).Subscribe(x =>
         {
             print_update(x);
         }));
@@ -65,11 +61,11 @@ namespace bts.sdk
       {
          Venue ven = parse_venue(exch);
          SecurityType st = parse_security_type(sectype);
+         Product prod = new Product() { Exchange = ven, Type = st, Symbol = symbol };
+         Func<Security, bool> filter = (x => x.Glbx.SecurityDesc == security_desc);
 
-         Func<SecurityImmutable, bool> where = (x => x.Exchange == ven && x.Symbol == symbol && x.security_type() == st &&
-                                                x.Glbx.SecurityDesc == security_desc);
-
-         _disps.Add(_md.subscribe_book(where).Subscribe(x =>
+         //if you have a security_id, we could make a more specifics subscription, but this is a command line driver....      
+         _disps.Add(_md.subscribe(prod).Where(x => filter(x.Security)).Subscribe(x =>
          {
             print_update(x);
          }));

src/client/spark_sdk/samples/OrderSample.cs

@@ -7,7 +7,7 @@ using System.Threading.Tasks;
 using System.Reactive.Linq;
 
 using bts.api;
-using bts.md;
+using bts.api.md;
 using bts.api.core;
 using bts.api.orderman;
 using bts.api.user;
@@ -27,7 +27,7 @@ namespace bts.sdk
       }
 
       readonly static Lazy<OrderService> _orders = new Lazy<OrderService>(() => Locator.locate<OrderService>(), true);
-      readonly static Lazy<AuthService> _users = new Lazy<AuthService>(() => Locator.locate<AuthService>(), true);
+      readonly static Lazy<UserService> _users = new Lazy<UserService>(() => Locator.locate<UserService>(), true);
       readonly static Lazy<AdminService> _admin = new Lazy<AdminService>(() => Locator.locate<AdminService>(), true);
 
       void print_update<T>(T update) 
@@ -36,8 +36,8 @@ namespace bts.sdk
          Console.WriteLine();
       }
 
-      public async Task<Result<Response>> send_order(string user, string account ,
-            string exch, string sectype, string symbol, string security_desc, string qty, string price = null)
+      public async Task<Result<Response>> send_order(string exch, string sectype, string symbol, string security_desc, string qty, 
+                                                     string price = null, string user = null, string account = null)
       {
          Venue ven = MarketDataSample.parse_venue(exch);
          SecurityType st = MarketDataSample.parse_security_type(sectype);
@@ -58,7 +58,7 @@ namespace bts.sdk
          else
             cr.NewOrder.CustomDef = new CustomOrderDefinition() { Market = new MarketOrderDefinition() };
          var sec = await _md.find_security_async(x => x.Glbx.SecurityDesc == security_desc.ToUpper());
-         cr.NewOrder.SecurityKey = sec.Value.Id;
+         cr.NewOrder.SecurityKey = sec.Id;
          Result<Response> resp = await _orders.Value.create_order_async(cr);
          print_update(resp);
          if(resp.value != null)

src/client/spark_sdk/samples/SDKMain.cs

@@ -6,54 +6,46 @@ using System.Reactive.Disposables;
 
 using bts.api;
 using bts.api.core;
-using bts.md;
+using bts.api.md;
 
 namespace bts.sdk
 {
    
    public class SDKMain
    {
-      static async Task<LoginResult> login(string user, string pass)
-      {
-         return await Locator.login(user, pass);
-      }
       private static void user_test()
-      {   
-         var s = Locator.locate<api.user.AuthService>();
+      {
+         var ctx = new ServiceContext();
+
+         var s = ctx.locate<api.user.UserService>();
          var r = new api.user.GetUsersRequest();
          var resp = s.get_users(r);
          Console.WriteLine("{0}", resp);
       }      
-      public async static Task Main(string[] args)
-      {         
+      public static void Main(string[] args)
+      {
          Log.ClientSdk.info("Welcome to the BTS Client Side SDK example");
-
-         string u = args.FirstOrDefault(x => x.Contains("--user="));
-         string p = args.FirstOrDefault(x => x.Contains("--pass="));
-         u = u?.Replace("--user=", string.Empty) ?? string.Empty;
-         p = p?.Replace("--pass=", string.Empty) ?? string.Empty;
-
-         var lr = await login(u, p);
-         if (!lr.success)
-         {
-            Console.WriteLine("Login failure: " + lr.reason);
-            help();
-         }
-
          if (args.FirstOrDefault() == "users")
          {
             user_test();
          }
          else
          {
-            uint ms = 100;
-            string pu = args.FirstOrDefault(x => x.Contains("--pump="));
-            if (pu != null)
-               ms = uint.Parse(p.Replace("--pump=", string.Empty));
-
-            MarketDataClient client = new MarketDataClient(ms, Locator.token);
+            MarketDataClient client = new MarketDataClient();
             CompositeDisposable disp = new CompositeDisposable(client);
-            if (args.Length > 0 && args.FirstOrDefault().Contains("md"))
+            if (args.FirstOrDefault() == "ui")
+            {
+               if (args.Length >= 6)
+               {
+                  using (var ui = new ui.UISample(client))
+                  {
+                     ui.simple_price_ui(args[1], args[2], args[3], args[4], args[5]);
+                  }
+               }
+               else
+                  ui_help();
+            }
+            else if (args.Length > 0 && args.FirstOrDefault().Contains("md"))
             {
                var s = new MarketDataSample(client);
                if (args.FirstOrDefault() == "md")
@@ -64,7 +56,7 @@ namespace bts.sdk
                   }
                   else if (args.Length >= 4)
                   {
-                     s.print_product(args[1], args[2], args[3]);
+                     s.print_channel(args[1], args[2], args[3]);
                   }
                   else
                      md_help();
@@ -76,24 +68,33 @@ namespace bts.sdk
             {
                OrderSample os = new OrderSample(client);
                disp.Add(os);
-               order_sample(os, u, args);
+               order_sample(os, args);
             }
             else
                help();
 
+            int ms = 100;
+            string p = args.FirstOrDefault(x => x.Contains("--pump="));
+            if (p != null)
+               ms = int.Parse(p.Replace("--pump=", string.Empty));
+            client.run_pump(TimeSpan.FromMilliseconds(ms));
             wait_for_stop(disp);
          }
       }
 
-      static async void order_sample(OrderSample os, string user, string[] args)
+      static async void order_sample(OrderSample os, string[] args)
       {
-         if (args.Length >= 10) //--user and --pass are in args[]
+         if (args.Length >= 9)
          {
-            await os.send_order(user, args[1], args[2], args[3], args[4], args[5], args[6], args[7]);
+            await os.send_order(args[1], args[2], args[3], args[4], args[5], args[6], args[7], args[8]);
          }         
-         else if (args.Length >= 9)
+         else if (args.Length >= 7)
          {
-            await os.send_order(user, args[1], args[2], args[3], args[4], args[5], args[6]);
+            await os.send_order(args[1], args[2], args[3], args[4], args[5], args[6]);
+         }
+         else if (args.Length >= 6)
+         {
+            await os.send_order(args[1], args[2], args[3], args[4], args[5]);
          }
          else
             order_help();         
@@ -118,10 +119,9 @@ namespace bts.sdk
          Console.WriteLine();
          Console.WriteLine("Commands:");
          Console.WriteLine("   md <exchange> <security_type> <symbol> [<security_desc>]");
-         Console.WriteLine("   order <account> <exchange> <security_type> <symbol> <security_desc> <signed_quantity> [<price>]");
+         Console.WriteLine("   ui <exchange> <security_type> <symbol> <security_desc> <poll_interval_in_ms>");
+         Console.WriteLine("   order <exchange> <security_type> <symbol> <security_desc> <signed_quantity> [<price>]");
          Console.WriteLine("Arguments:");
-         Console.WriteLine("   --user=<bts user name>");
-         Console.WriteLine("   --pass=<password>");
          Console.WriteLine("   --pump=<pump_interval_in_ms=100 default>");
          Environment.Exit(1);
       }
@@ -130,9 +130,10 @@ namespace bts.sdk
       {
          Console.WriteLine("order <exchange> <security_type> <symbol> <security_desc> <signed_quantity> [<price>] [<user>] [<account>]");
          Console.WriteLine("This command sends a limit or market DAY order for the specified security_desc.  Leave off the price for a market order");
-         Console.WriteLine("   example: order \"ACC A1\" GLBX Future ES ESM8 10 2650.0");
-         Console.WriteLine("            order \"ACC A1\" SIM Future ES SIM:ESM8 -10 2650.0");
-         Console.WriteLine("            order \"ACC A1\" GLBX Future ES ESM8 10 (this is a market order)");
+         Console.WriteLine("   example: order GLBX Future ES ESM8 -2 2650.25");
+         Console.WriteLine("            order GLBX Future ES ESM8 -2");
+         Console.WriteLine("            order GLBX Future ES ESM8 10 2650.0 bts \"ACC A1\"");
+         Console.WriteLine("            order GLBX Future ES ESM8 10 2650.0");
          Environment.Exit(1);
       }      
 
@@ -141,11 +142,19 @@ namespace bts.sdk
          Console.WriteLine("md <exchange> <security_type> <symbol>");
          Console.WriteLine("This command subscribes to market data for an exchange-type-symbol channel and writes to console.");
          Console.WriteLine("Marketdata is read off the wire on the same thread as is used to read and print");
-         Console.WriteLine("   example: md GLBX Future ES --user=bts --pass=password");
-         Console.WriteLine("            md GLBX Future ES ESZ8 --user=bts --pass=password");
-         Console.WriteLine("            md GLBX Spread ES --user=bts --pass=password");
+         Console.WriteLine("   example: md GLBX Future ES");
+         Console.WriteLine("            md GLBX Future ES ESZ8");
+         Console.WriteLine("            md GLBX Spread ES");
          Environment.Exit(1);
       }
 
+      static void ui_help()
+      {
+         Console.WriteLine("ui <exchange> <security_type> <symbol> <security_desc> <poll_interval_in_ms>");
+         Console.WriteLine("This command subscribes to market data for a specific security and writes to a UI on a specified refresh interval (in ms).");
+         Console.WriteLine("Marketdata is read off the wire on one thread and posted to the GUI on another through Observable.Timer()");
+         Console.WriteLine("   example: ui GLBX Future ES ESZ8 25");
+         Environment.Exit(1);
+      }
    }
 }

src/client/spark_sdk/samples/UISample.cs

@@ -2,7 +2,7 @@ using System;
 using System.Linq;
 
 using bts.api;
-using bts.md;
+using bts.api.md;
 
 namespace bts.sdk.ui
 {

src/client/spark_sdk/spark_sdk.csproj

@@ -18,7 +18,6 @@
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
     <WarningsAsErrors />
     <Version>1.0.0</Version>
-    <LangVersion>7.2</LangVersion>
   </PropertyGroup>
   
   <ItemGroup>

src/client/spark_sdk/uisamples/SparkSDKUI.cs

@@ -11,7 +11,7 @@ using System.Threading.Tasks;
 using System.Windows.Forms;
 
 using bts.api;
-using bts.md;
+using bts.api.md;
 
 namespace bts.sdk.ui
 {

src/client/spark_test/api/ApiTests.cs

@@ -34,6 +34,25 @@ namespace mbts.api.Test {
          Disposable.dispose(ref _context);
       }
 
+      [Test,Ignore("Timeout")]
+      public void test_format_qualifier() {
+         var simple = new QualifierArgument() { Simple = new SimpleQualifier() { Symbol = "ES" } };
+         var s1 = _mtest.format_qualifier(simple);
+         var s2 = _ntest.format_qualifier(simple);
+         Assert.AreEqual(s1.Str, s2.Str);
+
+         var complex = new ComplexQualifier() {
+            Name = "name",
+         };
+         complex.Probability.AddRange(new[] { 1.0, 2.0 });
+         complex.Symbols.Add(simple.Simple);
+         complex.Symbols.Add(new SimpleQualifier() { Symbol = "$50\"+" });
+         var carg = new QualifierArgument() { Complex = complex };
+         var c1 = _mtest.format_qualifier(carg);
+         var c2 = _ntest.format_qualifier(carg);
+         Assert.AreEqual(c1.Str, c2.Str);
+
+      }
       [Test, Ignore("Times out")]
       public void test_empty_reply_is_not_null() {
          foreach (var svc in test_services()) {
@@ -108,6 +127,19 @@ namespace mbts.api.Test {
 
       public ApiTestServiceServer(ServiceContext context) : base(context, "c#") { _context = context; }
 
+      public override Task<UnitTestResult> format_qualifier(QualifierArgument request) {
+         UnitTestResult result = new UnitTestResult();
+         switch (request.QualifierCase) {
+            case QualifierArgument.QualifierOneofCase.Simple:
+               result.Str = format_qualifier("format_qualifier", request.Simple);
+               break;
+            case QualifierArgument.QualifierOneofCase.Complex:
+               result.Str = format_qualifier("format_qualifier", request.Complex);
+               break;
+         }
+         return Task.FromResult(result);
+      }
+
       public override Task<UnitTestResult> run_hello_service(EmptyRequest request) {
          stop_hello_service(null);
          _hello = new HelloServiceServer(_context);

src/client/sparkcmd/Cmd/ServiceTestCommand.cs

@@ -4,7 +4,7 @@ using bts.api.synthetic;
 using bts.test;
 using ManyConsole;
 using System;
-using System.Threading;
+using System.Linq;
 
 namespace bts {
    public class ServiceTest : ConsoleCommand {
@@ -86,7 +86,7 @@ namespace bts {
       private static void user_test() {
          var ctx = new ServiceContext();
 
-         var s = ctx.locate<api.user.AuthService>();
+         var s = ctx.locate<api.user.UserService>();
          var r = new api.user.GetUsersRequest();
          var resp = s.get_users(r);
          Console.WriteLine("{0}", resp);
@@ -96,13 +96,8 @@ namespace bts {
          var ctx = new ServiceContext();
 
          var s = ctx.locate<api.admin.AdminService>();
-         var d = s.subscribei_admin_data().Subscribe(Console.WriteLine);
-         //var resp = s.get_admin_data(new VoidMessage());
-         //Console.WriteLine("{0}", resp);
-
-         while (true) {
-            Thread.Sleep(100);
-         }
+         var resp = s.get_admin_data(new VoidMessage());
+         Console.WriteLine("{0}", resp);
       }
    }
 }

src/data_files/CMakeDeps.cmake

@@ -1 +1 @@
-set(sources files/Onixs.BTS.lic;files/cert_secure_logon.txt;files/perf_trace/compiled.css;files/perf_trace/index.html;files/perf_trace/jquery-3.3.1.min.js;files/perf_trace/main.js;files/perf_trace/perf_log.sql;files/perf_trace/perf_trace_analysis.sql;files/perf_trace/plotly-latest.min.js)
+set(sources files/Onixs.BTS.lic;files/cert_secure_logon.txt;files/perf_trace/compiled.css;files/perf_trace/index.html;files/perf_trace/jquery-3.3.1.min.js;files/perf_trace/main.js;files/perf_trace/plotly-latest.min.js)

src/data_files/files/perf_trace/main.js

@@ -1,6 +1,4 @@
 $( document ).ready(function (){
-  $.ajaxSetup({ timeout: 30000 });
-
   let urlParams = new URLSearchParams(window.location.search);
   num_rows = urlParams.get('r');
   let show_rows = urlParams.get('showrows');
@@ -113,9 +111,6 @@ function ajax_err (jqXHR, textStatus,errorThrown) {
 
    console.log(textStatus);
    console.log(errorThrown);
-
-   console.log(jqXHR);
-
 }
 
 function first_upper (str)

src/data_files/files/perf_trace/perf_log.sql

@@ -1,183 +0,0 @@
-create table if not exists
-session (
-   version TEXT, 
-   host TEXT, 
-   build_type TEXT, 
-   session_start TIMESTAMP PRIMARY KEY
-);
-
-create table if not exists
-trace (
-   session_start TIMESTAMP,
-   ts INTEGER,
-   hts timestamp,
-   channel TEXT,
-   entry_kind TEXT,
-   txt_1 TEXT,
-   uint64_1 INTEGER,
-   thread TEXT,
-   cor_event INTEGER,
-   sn INTEGER,
-   uint32_1 INTEGER
-);
-
-drop view if exists recent_trace;
-create view recent_trace as
-select * from trace order by rowid desc limit 140123;
-
-create index if not exists trace_idx on trace(session_start,channel,sn,ts);
-
-drop view if exists ordered_trace;
-create view ordered_trace as
-select
-   recent_trace.*,
-   row_number() over (order by session_start,ts) entry_rank 
-from
-   recent_trace;
-
-drop view if exists delta_trace;
-create view delta_trace as
-select
-   current.*,
-   (current.ts - previous.ts) / 1000.0 delta
-from
-   ordered_trace current,
-   ordered_trace previous
-where
-   current.entry_rank = previous.entry_rank+1;
-
-drop view if exists log;
-create view log as
-select
-   delta_trace.*,
-   case cor_event>0 when 1 then (ts-cor_event)/1000.0 else 0 end scope_delta,
-   case cor_event>0 when 1 then "end of " || entry_kind else entry_kind end scoped_entry
-from
-   delta_trace;
-   
-drop view if exists events;
-create view events as
-select
-   fix_reads.session_start,
-   "hedge" event_type,
-   fix_reads.ts start,
-   fix_writes.ts stop,
-   (fix_writes.ts - fix_reads.ts) / 1000.0 duration,
-   row_number() over (order by fix_reads.session_start,fix_reads.ts,fix_writes.ts) event_rank ,
-   fix_reads.channel,
-   fix_reads.sn
-from
-   (select session_start, channel, sn, ts from recent_trace where entry_kind='fix_msg_r') fix_reads,
-   (select session_start, channel, sn, ts from recent_trace where entry_kind='before_fix_socket_w') fix_writes
-where
-   fix_reads.session_start = fix_writes.session_start and
-   fix_reads.channel = fix_writes.channel and
-   fix_reads.sn = fix_writes.sn 
-union   
-select
-   md_reads.session_start,
-   "requote" event_type,
-   md_reads.ts start,
-   fix_writes.ts stop,
-   (fix_writes.ts - md_reads.ts) / 1000.0 duration,
-   row_number() over (order by md_reads.session_start,md_reads.ts,fix_writes.ts) event_rank ,
-   md_reads.channel,
-   md_reads.sn
-from
-   (select session_start, channel, sn, ts from recent_trace where entry_kind='md') md_reads,
-   (select session_start, channel, sn, ts from recent_trace where entry_kind='before_fix_socket_w') fix_writes
-where
-   md_reads.session_start = fix_writes.session_start and
-   md_reads.channel = fix_writes.channel and
-   md_reads.sn = fix_writes.sn;
-   
-drop view if exists event_log;
-create view event_log as
-select
-   log.*,
-   event_type,
-   event_rank,
-   events.duration event_duration,
-   events.start event_start,
-   events.stop event_stop
-from
-   log,
-   events
-where
-   log.session_start = events.session_start and
-   log.ts >= events.start and log.ts <= events.stop;
-
-drop view if exists pretty_log;
-create view pretty_log as
-select
-   scoped_entry entry_kind,
-   delta,
-   scope_delta	,
-   event_duration,
-   event_type,
-   event_rank,
-   hts,
-   ts,
-   session_start,
-   entry_rank
-from
-  event_log;
-  
-drop view if exists last_pretty_log;
-create view last_pretty_log as
-select * from pretty_log where session_start=(select max(session_start) from session);
-
--- drop view if exists summary;
--- create view summary as
--- select session_start,event_type,count(*) count, avg(duration) avg, max(duration) max, min(duration) min, median(duration) median, stdev(duration) stdev from events group by session_start,event_type;
-
-
-drop view if exists hedge_trace;
-create view hedge_trace as 
-select 
-   session.version, 
-   session.host, 
-   session.build_type, 
-   session.session_start, 
-   recent_trace.hts trace_time, 
-   recent_trace.uint64_1 order_key, 
-   hedge_events.duration total,
-   hedge_events.event_rank rank
-from  
-   recent_trace, 
-   (select * from events where event_type='hedge') hedge_events,
-   session 
-where 
-   entry_kind='NativeOrder::on_activate' and 
-   recent_trace.cor_event = 0 and 
-   recent_trace.channel = hedge_events.channel and 
-   recent_trace.session_start = hedge_events.session_start and 
-   recent_trace.sn = hedge_events.sn and 
-   recent_trace.session_start = session.session_start;
-
-drop view if exists requote_trace;
-create view requote_trace as  
-select  
-   session.version,  
-   session.host,  
-   session.build_type,  
-   session.session_start,  
-   recent_trace.hts trace_time,  
-   recent_trace.uint64_1 order_key,  
-   requote_events.duration total,
-   requote_events.event_rank rank
-from   
-   recent_trace,  
-   (select * from events where event_type='requote') requote_events,  
-   session  
-where  
-   entry_kind='NativeOrder::on_update' and  
-   recent_trace.cor_event = 0 and 
-   recent_trace.channel = requote_events.channel and  
-   recent_trace.session_start = requote_events.session_start and  
-   recent_trace.sn = requote_events.sn and  
-   recent_trace.session_start = session.session_start     ;
-
-delete from trace where 
-rowid in (select rowid from trace order by rowid desc limit -1 offset 10000123);
-

src/data_files/files/perf_trace/perf_trace_analysis.sql

@@ -1,21 +0,0 @@
-drop table if exists pretty_log_table;
-create table pretty_log_table as select * from pretty_log;
-
-create view if not exists broken_down_hedges as
-select 
-   socket_r.event_id,
-   socket_r.event_duration,
-   (om.ts - socket_r.ts) / 1000.0 fix_parsing,
-   (jf.ts - om.ts) / 1000.0 om,
-   (socket_w.ts - jf.ts) / 1000.0 fix_gen
- from 
-   (select event_id,event_duration,ts from pretty_log_table where entry_kind = 'fix_socket_r') socket_r,
-   (select event_id, ts from pretty_log_table where entry_kind='NativeOrder::on_exe_report') om,
-   (select event_id, ts from pretty_log_table where entry_kind='ExecutionGateway::send_fix') jf,
-   (select event_id, ts from pretty_log_table where entry_kind='before_fix_socket_w') socket_w
-where
-    socket_r.event_id = om.event_id and om.event_id = jf.event_id and jf.event_id = socket_w.event_id
-	and event_duration < 100
-order by socket_r.event_id;	
-
-	

src/lib/CMakeDeps.cmake

@@ -1 +1 @@
-set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp;utils/stats.cpp;utils/tests/AuthTokenTests.cpp;utils/thread_manager.cpp)
+set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/services/UserService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/Executor.conf;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;mercury/actors/IPollableActor.cpp;mercury/broker/Broker.cpp;mercury/broker/BrokerProxy.cpp;mercury/broker/ILocalBroker.cpp;mercury/broker/IRemoteBroker.cpp;mercury/common/mercury_common.cpp;mercury/events/TimerEvent.cpp;mercury/executor/Executor.cpp;mercury/topics/Topic.cpp;metrics/metrics.cpp;metrics/system_metrics.cpp;orderman/actors/AsyncFactory.cpp;orderman/actors/ExecutionGateway.cpp;orderman/actors/FillStore.cpp;orderman/actors/MDSupplementarySource.cpp;orderman/actors/OrderStore.cpp;orderman/actors/SODPosition.cpp;orderman/actors/SecuritySrc.cpp;orderman/actors/ServiceBridge.cpp;orderman/actors/order_manager/OrderBuilder.cpp;orderman/actors/order_manager/OrderManager.cpp;orderman/actors/order_manager/OrderServicesData.cpp;orderman/events/CancelRejectEvent.cpp;orderman/events/NewExecutionReportEvent.cpp;orderman/limiter/LimitStore.cpp;orderman/limiter/Limiter.cpp;orderman/limiter/ProductLimitNode.cpp;orderman/limiter/ProductLimitTree.cpp;orderman/limiter/SecurityLimitNode.cpp;orderman/limiter/SecurityLimitTree.cpp;orderman/limiter/tests/limit_tests.cpp;orderman/order_factory/DynamiLine truncated

src/lib/api/apicore.proto

@@ -12,14 +12,9 @@ message PingPong {
 // RequestHeader
 // T Request
 
-message AuthToken {
-   bytes token = 1;
-}
-
 message RequestHeader {
    int32 request_id = 1;
    int32 method = 2;
-   AuthToken token = 3;
 }
 
 enum ApiError {
@@ -37,12 +32,6 @@ message ResponseHeader {
    ApiError code = 2;
 }
 
-message StreamAccessRequest {
-   string qualifier = 1;
-}
-message StreamAccessResponse {
-   bool authorized = 1;
-}
 
 // Some Useful buffers for use as arguments/returns
 

src/lib/api/apigateway.proto

@@ -1,7 +1,6 @@
 syntax="proto3";
 package bts.api.gateway;
 option csharp_namespace="bts.api.gateway";
-import "lib/api/apicore.proto";
 
 message GatewayLogin {
     string username = 1;
@@ -10,13 +9,13 @@ message GatewayLogin {
 message GatewayLoginResult {
     bool is_valid = 1;
     string reason = 2;
-    api.AuthToken token = 3;
+    string token = 3;
     bool can_create = 4;
 }
 message GatewayCallHeader {
    int32 request_id = 1;
    int32 method = 2;
-   api.AuthToken token = 3;
+   string token = 3;
    string service_type = 4;
    string service_qualifier = 5;
 }

src/lib/api/core/ApiClientInterop.cpp

@@ -181,16 +181,11 @@ void safe_call_subscription_callback(raw_subscription_callback cb, EventType typ
 }
 
 
-bts::qbus::Topic stream_topic(const std::string& base_topic, const char* method, const char* qualifier) {
-   if (qualifier == nullptr || strlen(qualifier) == 0) {
-      return qbus::Topic(base_topic + "-" + method);
-   }
-   else {
-      return qbus::Topic(base_topic + "-" + method + "/" + qualifier);
-   }
+bts::qbus::Topic stream_topic(const std::string& base_topic, const char* method_and_qualifier) {
+   return qbus::Topic(base_topic + "-" + method_and_qualifier);
 }
 
-CAPI TableSubscription api_client_subscribe(ApiClient* pclient, const char* method, const char* qualifier, raw_subscription_callback cb, void* extra) {
+CAPI TableSubscription api_client_subscribe(ApiClient* pclient, const char* method_and_qualifier, raw_subscription_callback cb, void* extra) {
    bts_assert(pclient);
    static_assert((int)qbus::TableEvent::EventType::TABLE_CREATED == EventType::TABLE_CREATED, "Keep these Enums in sync");
    static_assert((int)qbus::TableEvent::EventType::TABLE_DESTROYED == EventType::TABLE_DESTROYED, "");
@@ -199,7 +194,7 @@ CAPI TableSubscription api_client_subscribe(ApiClient* pclient, const char* meth
    static_assert((int)qbus::Change::Type::REPLACE == StreamUpdate::REPLACE, "");
    static_assert((int)qbus::Change::Type::CREATE == StreamUpdate::CREATE, "");
 
-   auto topic = stream_topic(pclient->service_name, method, qualifier);
+   auto topic = stream_topic(pclient->service_name, method_and_qualifier);
    LOG(Api,Diag,"api subscribe: {}" ,topic.value());
 
    auto result = new qbus::TableSubscriber(pclient->qbus, equal_topic(topic), [cb, extra](const qbus::TableEvent& ev) {

src/lib/api/core/ApiClientInterop.h

@@ -21,7 +21,7 @@ enum EventType {
 
 typedef void* TableSubscription;
 typedef void(*raw_subscription_callback)(EventType event_type, const StreamUpdate* changes, int count, void* extra);
-CAPI TableSubscription api_client_subscribe(ApiClient* pclient, const char* method, const char* qualifier, raw_subscription_callback cb, void* extra);
+CAPI TableSubscription api_client_subscribe(ApiClient* pclient, const char* method_and_qualifier, raw_subscription_callback cb, void* extra);
 CAPI void api_client_unsubscribe(ApiClient* pclient, TableSubscription ptable);
 
 }}//namespace

src/lib/api/core/ApiServerInterop.cpp

@@ -9,7 +9,7 @@ namespace api {
 // defined in ApiClientInterop.h
 std::string base_topic(const char* type_name, const char* service_qualifier);
 bts::qbus::RPCServiceName rpc_topic(const std::string& base_topic);
-bts::qbus::Topic stream_topic(const std::string& base_topic, const char* method, const char* qualifier);
+bts::qbus::Topic stream_topic(const std::string& base_topic, const char* method_and_qualifier);
 
 struct ApiServerCore {
    ApiServerCore(Quickbus& qbus, const std::string topic, const qbus::OnRPCRequest& handler) :
@@ -51,9 +51,9 @@ CAPI void api_server_call_callback(ReplyHandler** ppvhandler, const uint8_t* buf
    *pphandler = nullptr;
 }
 
-CAPI TableHandle api_server_create_table(ApiServerCore* pserver, const char* method, const char* qualifier) {
+CAPI TableHandle api_server_create_table(ApiServerCore* pserver, const char* method_and_qualifier) {
    bts_assert(pserver);
-   auto topic = stream_topic(pserver->topic, method, qualifier);
+   auto topic = stream_topic(pserver->topic, method_and_qualifier);
    LOG(Api,Info,"Create table: {}",topic.value());
    return new qbus::TablePublisher(pserver->qbus, topic);
 }

src/lib/api/core/ApiServerInterop.h

@@ -12,7 +12,7 @@ CAPI ApiServerCore* api_server_create(Quickbus* pbus, const char* type_name, con
 CAPI void api_server_call_callback(ReplyHandler** pphandler, const uint8_t* buffer, int length);
 
 typedef void* TableHandle;
-CAPI TableHandle api_server_create_table(ApiServerCore* pserver, const char* method, const char* qualifier);
+CAPI TableHandle api_server_create_table(ApiServerCore* pserver, const char* method_and_qualifier);
 CAPI void api_server_update_table(TableHandle htable, StreamUpdate* raw_changes, int change_count);
 
 }}//namespace

src/lib/api/public/ApiUtils.h

@@ -18,5 +18,22 @@ namespace api {
       else return "";
    }
 
+   template <class T>
+   const std::string format_qualifier(const char* type_name, const T& t) {
+      google::protobuf::util::JsonPrintOptions options;
+      options.add_whitespace = false;
+      options.always_print_primitive_fields = true;
+      std::string result;
+      auto status = google::protobuf::util::MessageToJsonString(t, &result, options);
+      assert(status.ok());
+      result.erase( std::remove_if(result.begin(), result.end(), [](char c) {
+         return 
+            c == '+' || 
+            c == '_';
+      }), result.end());
+      return type_name + result;
+   }
+
+
 }}//namespace
 

src/lib/api/public/ClientChannel.cpp

@@ -3,7 +3,7 @@
 namespace bts {
 namespace api {
 
-ClientChannel::ClientChannel(const ServiceContext& ctx) : _id(1), _context(ctx)
+ClientChannel::ClientChannel() : _id(1)
 {
 }
 

src/lib/api/public/ClientChannel.h

@@ -4,7 +4,7 @@
 #include "Result.h"
 #include "ApiException.h"
 #include "RawUpdate.h"
-#include "ServiceContext.h"
+//#include "Update.h"
 #include <rxcpp/rx.hpp>
 #include <functional>
 #include <google/protobuf/io/zero_copy_stream_impl_lite.h>
@@ -12,21 +12,19 @@
 namespace bts {
 namespace api {
 
+
 class ClientChannel
 {
    int _id;
-   ServiceContext _context;
 public:
-   ClientChannel(const ServiceContext&);
+   ClientChannel();
    virtual ~ClientChannel();
 
    template <typename Request, typename Response>
-   void send(int method_id, const Request& request, std::function<void(const Result<Response>)> on_response) {
+   void send(int method_id, const Request& request, std::function<void(Result<Response>)> on_response) {
       RequestHeader header;
       header.set_method(method_id);
       header.set_request_id(++_id);
-      auto ptoken = _context.token();
-      if (ptoken) header.mutable_token()->CopyFrom(*ptoken);
       int header_size = header.ByteSize()+1;
       assert(header_size < 128);
 
@@ -64,57 +62,42 @@ public:
 
 
    template <typename T>
-   rxcpp::observable<RawUpdate<std::shared_ptr<const T>>> subscribe(int method_id, const char* method, const std::string& qualifier) {
+   rxcpp::observable<RawUpdate<std::shared_ptr<const T>>> subscribe(const std::string& method_and_qualifier) {
       return rxcpp::observable<>::create<RawUpdate<std::shared_ptr<const T>>>([=](rxcpp::subscriber<RawUpdate<std::shared_ptr<const T>>> observer) {
-         StreamAccessRequest request;
-         request.set_qualifier(qualifier);
-         send<StreamAccessRequest,StreamAccessResponse>(method_id, request, [=](const Result<StreamAccessResponse>& response) {
-            // There is a race on server startup. If the server is not yet we can't return a dead stream
-            // if (!response) {
-            //    observer.on_error(
-            //       std::make_exception_ptr(
-            //          ApiException(response.error_message())));
-            // }
-            if (response && !response.value()->authorized()) {
-               observer.on_completed();
-            }
-            else {
-               bool first_update = true;
-               auto cb = [observer, first_update](EventType event_type, const StreamUpdate* changes, int change_count) mutable {
-                  if (event_type == EventType::TABLE_UPDATED) {
-                     std::vector<std::shared_ptr<const T>> added_items;
-                     std::vector<int> added_keys,
-                                    removed_keys;
-                     for (size_t x = 0; x < change_count; ++x) {
-                        const StreamUpdate& change = changes[x];
-                        if (change.type != StreamUpdate::REMOVE) {
-                           added_keys.reserve(change_count);
-                           added_items.reserve(change_count);
-                           added_keys.push_back(change.key);
-                           google::protobuf::io::ArrayInputStream stream(change.buffer, change.buffer_len);
-                           auto item = std::make_shared<T>();
-                           if (item->ParseFromZeroCopyStream(&stream)) {
-                              added_items.push_back(move(item));
-                           }
-                        }
-                        if (change.type != StreamUpdate::CREATE) {
-                           removed_keys.reserve(change_count);
-                           removed_keys.push_back(change.key);
-                        }
+         bool first_update = true;
+         auto cb = [observer, first_update](EventType event_type, const StreamUpdate* changes, int change_count) mutable {
+            if (event_type == EventType::TABLE_UPDATED) {
+               std::vector<std::shared_ptr<const T>> added_items;
+               std::vector<int> added_keys,
+                                removed_keys;
+               for (size_t x = 0; x < change_count; ++x) {
+                  const StreamUpdate& change = changes[x];
+                  if (change.type != StreamUpdate::REMOVE) {
+                     added_keys.reserve(change_count);
+                     added_items.reserve(change_count);
+                     added_keys.push_back(change.key);
+                     google::protobuf::io::ArrayInputStream stream(change.buffer, change.buffer_len);
+                     auto item = std::make_shared<T>();
+                     if (item->ParseFromZeroCopyStream(&stream)) {
+                        added_items.push_back(move(item));
                      }
-                     observer.on_next(RawUpdate<std::shared_ptr<const T>>(added_items, added_keys, removed_keys));
                   }
-                  else if (event_type == EventType::TABLE_CREATED) {
-                     if (!first_update) {
-                        observer.on_next(RawUpdate<std::shared_ptr<const T>>::Reset());
-                     }
-                     first_update = false;
+                  if (change.type != StreamUpdate::CREATE) {
+                     removed_keys.reserve(change_count);
+                     removed_keys.push_back(change.key);
                   }
-               };
-               observer.add(
-                  subscribe_core(method, qualifier.c_str(), cb));
+               }
+               observer.on_next(RawUpdate<std::shared_ptr<const T>>(added_items, added_keys, removed_keys));
             }
-         });
+            else if (event_type == EventType::TABLE_CREATED) {
+               if (!first_update) {
+                  observer.on_next(RawUpdate<std::shared_ptr<const T>>::Reset());
+               }
+               first_update = false;
+            }
+         };
+         observer.add(
+            subscribe_core(method_and_qualifier, cb));
       });
    }
    
@@ -125,7 +108,7 @@ protected:
    typedef std::function<void()> Unsubscribe;
    typedef std::function<void(EventType,const StreamUpdate*,int)> OnSubscriptionCB;
    virtual void send_core(const std::string& request_buffer, OnReply on_response) = 0;
-   virtual Unsubscribe subscribe_core(const char* method, const char* qualifier, OnSubscriptionCB cb) = 0;
+   virtual Unsubscribe subscribe_core(const std::string& method_and_qualifier, OnSubscriptionCB cb) = 0;
 
 private:
    ClientChannel(const ClientChannel& other);// = delete;

src/lib/api/public/QuickbusClientChannel.cpp

@@ -4,8 +4,7 @@
 namespace bts {
 namespace api {
 
-QuickbusClientChannel::QuickbusClientChannel(Quickbus& bus, const ServiceContext& context, const char* type_name, const char* service_qualifier) :
-   ClientChannel(context),
+QuickbusClientChannel::QuickbusClientChannel(Quickbus& bus, const char* type_name, const char* service_qualifier) :
    _pclient(api_client_create(&bus, type_name, service_qualifier))
 {
 }
@@ -28,14 +27,14 @@ void QuickbusClientChannel::send_core(const std::string& request_envalope, OnRep
    api_client_send(_pclient, request_envalope.c_str(), request_envalope.size(), trampoline, heap_on_response);
 }
 
-ClientChannel::Unsubscribe QuickbusClientChannel::subscribe_core(const char* method, const char* qualifier, OnSubscriptionCB cb) {
+ClientChannel::Unsubscribe QuickbusClientChannel::subscribe_core(const std::string & method_and_qualifier, OnSubscriptionCB cb) {
    raw_subscription_callback trampoline = [](EventType event_type, const StreamUpdate* changes, int count, void* extra) {
       OnSubscriptionCB* heap_cb = static_cast<OnSubscriptionCB*>(extra);
       assert(heap_cb != nullptr);
       (*heap_cb)(event_type, changes, count);
    };
    OnSubscriptionCB* heap_cb = new OnSubscriptionCB(cb);
-   auto subscription = api_client_subscribe(_pclient, method, qualifier, trampoline, heap_cb);
+   auto subscription = api_client_subscribe(_pclient, method_and_qualifier.c_str(), trampoline, heap_cb);
    return [this,subscription,heap_cb]() { 
       api_client_unsubscribe(_pclient, subscription);
       delete heap_cb;

src/lib/api/public/QuickbusClientChannel.h

@@ -6,20 +6,19 @@
 namespace bts {
 namespace api {
 
-class ServiceContext;
 class QuickbusClientChannel : public ClientChannel
 {
    ApiClient* _pclient;
 
 public:
-   QuickbusClientChannel(Quickbus&, const ServiceContext&, const char* type_name, const char* service_qualifier);
+   QuickbusClientChannel(Quickbus&, const char* type_name, const char* service_qualifier);
    ~QuickbusClientChannel();
 
 private:
    BTS_NO_COPY(QuickbusClientChannel);
    
    virtual void send_core(const std::string& request_envalope, OnReply on_response) override;
-   virtual Unsubscribe subscribe_core(const char* method, const char* qualifier, OnSubscriptionCB cb) override;
+   virtual Unsubscribe subscribe_core(const std::string& method_and_qualifier, OnSubscriptionCB cb) override;
    virtual bool is_sync_call_ok() override;
 
 };