Brian tested manually

Merged in ui_permissions (pull request #82)

Ui permissions

Approved-by: Eric Harding <eric.harding@bluetradesys.com>

.gitignore

@@ -24,3 +24,5 @@ project.lock.json
 src/server/sparkd/orderman/perf_analysis/data
 src/server/sparkd/orderman/perf_analysis/logs/
 src/server/sparkd/orderman/perf_analysis/trace_data/
+src/lib/orderman/perf_analysis/logs/latency_data/
+trace_data/

.vscode/launch.json

@@ -42,7 +42,18 @@
          "stopAtEntry": false,
          "externalConsole": true,
          "requireExactSource": false
-      },        
+      },     
+      {
+         "name": "Client Tests",
+         "type": "clr",
+         "request": "launch",
+         "program": "${workspaceRoot}/build/Debug/bin/net471/sparkcmd.exe",
+         "args": ["test","test_filter"],
+         "cwd": "${workspaceRoot}",
+         "stopAtEntry": false,
+         "externalConsole": true,
+         "requireExactSource": false
+      },           
 		{
             "name": "Client Attach",
             "type": "clr",

CMakeLists.txt

@@ -40,7 +40,7 @@ else ()
                                -fPIC -Wall -pedantic -Wextra -Werror \
                                -Wno-missing-braces -Wno-ignored-qualifiers \
                                --std=c++17 -fdiagnostics-color")
-  set (CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_DEBUG} -DNDEBUG -O3")
+  set (CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_DEBUG} -fno-omit-frame-pointer -DNDEBUG -O3")
   add_subdirectory(src/server/sparkd)
   add_subdirectory(src/lib)
   add_subdirectory(src/web)

doc/dev/dev_process.html

@@ -0,0 +1,301 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en">
+<head>
+<!-- 2018-10-08 Mon 18:50 -->
+<meta http-equiv="Content-Type" content="text/html;charset=utf-8" />
+<meta name="viewport" content="width=device-width, initial-scale=1" />
+<title>&lrm;</title>
+<meta name="generator" content="Org mode" />
+<meta name="author" content="ppinto" />
+<style type="text/css">
+ <!--/*--><![CDATA[/*><!--*/
+  .title  { text-align: center;
+             margin-bottom: .2em; }
+  .subtitle { text-align: center;
+              font-size: medium;
+              font-weight: bold;
+              margin-top:0; }
+  .todo   { font-family: monospace; color: red; }
+  .done   { font-family: monospace; color: green; }
+  .priority { font-family: monospace; color: orange; }
+  .tag    { background-color: #eee; font-family: monospace;
+            padding: 2px; font-size: 80%; font-weight: normal; }
+  .timestamp { color: #bebebe; }
+  .timestamp-kwd { color: #5f9ea0; }
+  .org-right  { margin-left: auto; margin-right: 0px;  text-align: right; }
+  .org-left   { margin-left: 0px;  margin-right: auto; text-align: left; }
+  .org-center { margin-left: auto; margin-right: auto; text-align: center; }
+  .underline { text-decoration: underline; }
+  #postamble p, #preamble p { font-size: 90%; margin: .2em; }
+  p.verse { margin-left: 3%; }
+  pre {
+    border: 1px solid #ccc;
+    box-shadow: 3px 3px 3px #eee;
+    padding: 8pt;
+    font-family: monospace;
+    overflow: auto;
+    margin: 1.2em;
+  }
+  pre.src {
+    position: relative;
+    overflow: visible;
+    padding-top: 1.2em;
+  }
+  pre.src:before {
+    display: none;
+    position: absolute;
+    background-color: white;
+    top: -10px;
+    right: 10px;
+    padding: 3px;
+    border: 1px solid black;
+  }
+  pre.src:hover:before { display: inline;}
+  /* Languages per Org manual */
+  pre.src-asymptote:before { content: 'Asymptote'; }
+  pre.src-awk:before { content: 'Awk'; }
+  pre.src-C:before { content: 'C'; }
+  /* pre.src-C++ doesn't work in CSS */
+  pre.src-clojure:before { content: 'Clojure'; }
+  pre.src-css:before { content: 'CSS'; }
+  pre.src-D:before { content: 'D'; }
+  pre.src-ditaa:before { content: 'ditaa'; }
+  pre.src-dot:before { content: 'Graphviz'; }
+  pre.src-calc:before { content: 'Emacs Calc'; }
+  pre.src-emacs-lisp:before { content: 'Emacs Lisp'; }
+  pre.src-fortran:before { content: 'Fortran'; }
+  pre.src-gnuplot:before { content: 'gnuplot'; }
+  pre.src-haskell:before { content: 'Haskell'; }
+  pre.src-hledger:before { content: 'hledger'; }
+  pre.src-java:before { content: 'Java'; }
+  pre.src-js:before { content: 'Javascript'; }
+  pre.src-latex:before { content: 'LaTeX'; }
+  pre.src-ledger:before { content: 'Ledger'; }
+  pre.src-lisp:before { content: 'Lisp'; }
+  pre.src-lilypond:before { content: 'Lilypond'; }
+  pre.src-lua:before { content: 'Lua'; }
+  pre.src-matlab:before { content: 'MATLAB'; }
+  pre.src-mscgen:before { content: 'Mscgen'; }
+  pre.src-ocaml:before { content: 'Objective Caml'; }
+  pre.src-octave:before { content: 'Octave'; }
+  pre.src-org:before { content: 'Org mode'; }
+  pre.src-oz:before { content: 'OZ'; }
+  pre.src-plantuml:before { content: 'Plantuml'; }
+  pre.src-processing:before { content: 'Processing.js'; }
+  pre.src-python:before { content: 'Python'; }
+  pre.src-R:before { content: 'R'; }
+  pre.src-ruby:before { content: 'Ruby'; }
+  pre.src-sass:before { content: 'Sass'; }
+  pre.src-scheme:before { content: 'Scheme'; }
+  pre.src-screen:before { content: 'Gnu Screen'; }
+  pre.src-sed:before { content: 'Sed'; }
+  pre.src-sh:before { content: 'shell'; }
+  pre.src-sql:before { content: 'SQL'; }
+  pre.src-sqlite:before { content: 'SQLite'; }
+  /* additional languages in org.el's org-babel-load-languages alist */
+  pre.src-forth:before { content: 'Forth'; }
+  pre.src-io:before { content: 'IO'; }
+  pre.src-J:before { content: 'J'; }
+  pre.src-makefile:before { content: 'Makefile'; }
+  pre.src-maxima:before { content: 'Maxima'; }
+  pre.src-perl:before { content: 'Perl'; }
+  pre.src-picolisp:before { content: 'Pico Lisp'; }
+  pre.src-scala:before { content: 'Scala'; }
+  pre.src-shell:before { content: 'Shell Script'; }
+  pre.src-ebnf2ps:before { content: 'ebfn2ps'; }
+  /* additional language identifiers per "defun org-babel-execute"
+       in ob-*.el */
+  pre.src-cpp:before  { content: 'C++'; }
+  pre.src-abc:before  { content: 'ABC'; }
+  pre.src-coq:before  { content: 'Coq'; }
+  pre.src-groovy:before  { content: 'Groovy'; }
+  /* additional language identifiers from org-babel-shell-names in
+     ob-shell.el: ob-shell is the only babel language using a lambda to put
+     the execution function name together. */
+  pre.src-bash:before  { content: 'bash'; }
+  pre.src-csh:before  { content: 'csh'; }
+  pre.src-ash:before  { content: 'ash'; }
+  pre.src-dash:before  { content: 'dash'; }
+  pre.src-ksh:before  { content: 'ksh'; }
+  pre.src-mksh:before  { content: 'mksh'; }
+  pre.src-posh:before  { content: 'posh'; }
+  /* Additional Emacs modes also supported by the LaTeX listings package */
+  pre.src-ada:before { content: 'Ada'; }
+  pre.src-asm:before { content: 'Assembler'; }
+  pre.src-caml:before { content: 'Caml'; }
+  pre.src-delphi:before { content: 'Delphi'; }
+  pre.src-html:before { content: 'HTML'; }
+  pre.src-idl:before { content: 'IDL'; }
+  pre.src-mercury:before { content: 'Mercury'; }
+  pre.src-metapost:before { content: 'MetaPost'; }
+  pre.src-modula-2:before { content: 'Modula-2'; }
+  pre.src-pascal:before { content: 'Pascal'; }
+  pre.src-ps:before { content: 'PostScript'; }
+  pre.src-prolog:before { content: 'Prolog'; }
+  pre.src-simula:before { content: 'Simula'; }
+  pre.src-tcl:before { content: 'tcl'; }
+  pre.src-tex:before { content: 'TeX'; }
+  pre.src-plain-tex:before { content: 'Plain TeX'; }
+  pre.src-verilog:before { content: 'Verilog'; }
+  pre.src-vhdl:before { content: 'VHDL'; }
+  pre.src-xml:before { content: 'XML'; }
+  pre.src-nxml:before { content: 'XML'; }
+  /* add a generic configuration mode; LaTeX export needs an additional
+     (add-to-list 'org-latex-listings-langs '(conf " ")) in .emacs */
+  pre.src-conf:before { content: 'Configuration File'; }
+
+  table { border-collapse:collapse; }
+  caption.t-above { caption-side: top; }
+  caption.t-bottom { caption-side: bottom; }
+  td, th { vertical-align:top;  }
+  th.org-right  { text-align: center;  }
+  th.org-left   { text-align: center;   }
+  th.org-center { text-align: center; }
+  td.org-right  { text-align: right;  }
+  td.org-left   { text-align: left;   }
+  td.org-center { text-align: center; }
+  dt { font-weight: bold; }
+  .footpara { display: inline; }
+  .footdef  { margin-bottom: 1em; }
+  .figure { padding: 1em; }
+  .figure p { text-align: center; }
+  .inlinetask {
+    padding: 10px;
+    border: 2px solid gray;
+    margin: 10px;
+    background: #ffffcc;
+  }
+  #org-div-home-and-up
+   { text-align: right; font-size: 70%; white-space: nowrap; }
+  textarea { overflow-x: auto; }
+  .linenr { font-size: smaller }
+  .code-highlighted { background-color: #ffff00; }
+  .org-info-js_info-navigation { border-style: none; }
+  #org-info-js_console-label
+    { font-size: 10px; font-weight: bold; white-space: nowrap; }
+  .org-info-js_search-highlight
+    { background-color: #ffff00; color: #000000; font-weight: bold; }
+  .org-svg { width: 90%; }
+  /*]]>*/-->
+</style>
+<script type="text/javascript">
+/*
+@licstart  The following is the entire license notice for the
+JavaScript code in this tag.
+
+Copyright (C) 2012-2018 Free Software Foundation, Inc.
+
+The JavaScript code in this tag is free software: you can
+redistribute it and/or modify it under the terms of the GNU
+General Public License (GNU GPL) as published by the Free Software
+Foundation, either version 3 of the License, or (at your option)
+any later version.  The code is distributed WITHOUT ANY WARRANTY;
+without even the implied warranty of MERCHANTABILITY or FITNESS
+FOR A PARTICULAR PURPOSE.  See the GNU GPL for more details.
+
+As additional permission under GNU GPL version 3 section 7, you
+may distribute non-source (e.g., minimized or compacted) forms of
+that code without the copy of the GNU GPL normally required by
+section 4, provided you include this license notice and a URL
+through which recipients can access the Corresponding Source.
+
+
+@licend  The above is the entire license notice
+for the JavaScript code in this tag.
+*/
+<!--/*--><![CDATA[/*><!--*/
+ function CodeHighlightOn(elem, id)
+ {
+   var target = document.getElementById(id);
+   if(null != target) {
+     elem.cacheClassElem = elem.className;
+     elem.cacheClassTarget = target.className;
+     target.className = "code-highlighted";
+     elem.className   = "code-highlighted";
+   }
+ }
+ function CodeHighlightOff(elem, id)
+ {
+   var target = document.getElementById(id);
+   if(elem.cacheClassElem)
+     elem.className = elem.cacheClassElem;
+   if(elem.cacheClassTarget)
+     target.className = elem.cacheClassTarget;
+ }
+/*]]>*///-->
+</script>
+</head>
+<body>
+<div id="content">
+<p>
+We are going to experiment with our development process, getting away
+from cycles and towards more continuous development. The main idea is
+to get rid of the prod and QA branches and instead deploy directly
+from master, which we keep deployment ready at all times. Development
+occurs on private branches, each branch mapped to a separately
+deploy-able project. Each of projects will be managed more or less
+independently and will include specification, development and
+validation stages. At any particular time we might have anywhere from
+1 to 6 projects in flight each of which continuously pulls from master,
+but does not push back to master until the code is ready to be
+deployed to production. The point at which a project is ready to be
+deployed then takes roughly the place of the QA phase of our old cycle
+mode. At this time, we will do a PR, and go through a series of
+validation steps, to make sure the work meets our standards. This
+validation includes:
+</p>
+
+<ul class="org-ul">
+<li>Code Reviews</li>
+<li>Acceptance Testing (including UI and business tests)</li>
+<li>Automated System testing (to be developed)</li>
+<li>A mini-qa, focused on the feature, by someone not directly involved
+in its implementation</li>
+<li>If possible, deployment to a beta customer.</li>
+</ul>
+
+<p>
+Once these QA gates are passed, we will update the release notes, tag
+a new version, integrate to master and then ideally immediately deploy
+to a customer.
+</p>
+
+<p>
+The main goal with this change is to reduce the latency from feature
+specification to production roll out. We also hope to improve
+throughput by making more efficient use of QA time and reducing idle
+cycles. We expect people to participate in more than one project
+depending on their area of expertise and work load. This does not mean
+people will work on two things at the same time, rather that people
+come into and leave projects as they free up and not when the project
+starts or ends.
+</p>
+
+<p>
+This new scheme will come into full effect when we finish 1.3
+although to some degree 1.3 was already setup like this. At the
+moment, we have 3 major projects, vap/settlments/implied, grid
+settings and speed work, which are in branches of prod. We will merge
+these back to prod and then master this week. We have 3 other projects
+that are in master, the excel API, the server side SDK and the MIC
+administration and enforcement components. These we hope to stabilize
+as soon as possible so that we can merge them to prod and deploy them
+along with the rest. Once we have all this, prod and master should be
+identical and we will delete prod and operate exclusively out of
+master. Please avoid committing anything new to master so we can
+complete this transition as soon as possible.
+</p>
+
+<p>
+Let me know if you have any question
+</p>
+</div>
+<div id="postamble" class="status">
+<p class="author">Author: ppinto</p>
+<p class="date">Created: 2018-10-08 Mon 18:50</p>
+<p class="validation"><a href="http://validator.w3.org/check?uri=referer">Validate</a></p>
+</div>
+</body>
+</html>

doc/dev/dev_process.org

@@ -0,0 +1,55 @@
+We are going to experiment with our development process, getting away
+from cycles and towards more continuous development. The main idea is
+to get rid of the prod and QA branches and instead deploy directly
+from master, which we keep deployment ready at all times. Development
+occurs on private branches, each branch mapped to a separately
+deploy-able project. Each of projects will be managed more or less
+independently and will include specification, development and
+validation stages. At any particular time we might have anywhere from
+1 to 6 projects in flight each of which continuously pulls from master,
+but does not push back to master until the code is ready to be
+deployed to production. The point at which a project is ready to be
+deployed then takes roughly the place of the QA phase of our old cycle
+mode. At this time, we will do a PR, and go through a series of
+validation steps, to make sure the work meets our standards. This
+validation includes:
+
+- Code Reviews
+- Acceptance Testing (including UI and business tests)
+- Automated System testing (to be developed)
+- A mini-qa, focused on the feature, by someone not directly involved
+  in its implementation
+- If possible, deployment to a beta customer.
+
+Once these QA gates are passed, we will update the release notes, tag
+a new version, integrate to master and then ideally immediately deploy
+to a customer.
+
+The main goal with this change is to reduce the latency from feature
+specification to production roll out. We also hope to improve
+throughput by making more efficient use of QA time and reducing idle
+cycles. We expect people to participate in more than one project
+depending on their area of expertise and work load. This does not mean
+people will work on two things at the same time, rather that people
+come into and leave projects as they free up and not when the project
+starts or ends.
+
+This new scheme will come into full effect when we finish 1.3
+although to some degree 1.3 was already setup like this. At the
+moment, we have 3 major projects, vap/settlments/implied, grid
+settings and speed work, which are in branches of prod. We will merge
+these back to prod and then master this week. We have 3 other projects
+that are in master, the excel API, the server side SDK and the MIC
+administration and enforcement components. These we hope to stabilize
+as soon as possible so that we can merge them to prod and deploy them
+along with the rest. Once we have all this, prod and master should be
+identical and we will delete prod and operate exclusively out of
+master. Please avoid committing anything new to master so we can
+complete this transition as soon as possible.
+
+Let me know if you have any question
+
+
+
+
+

ops_notes.txt

@@ -1,3 +1,6 @@
+1.3.4
+- Changed default logging config for Windows.  If Logging config is overridden for a system it should be adjusted to override only linux.
+
 1.1.0
 - With this release we dropped wild cards from limits, so any user
   that has limits expressed in terms of wild cards will have to have

release_notes.txt

@@ -1,3 +1,24 @@
+1.3.4
+- Significantly reduced client memory use
+- Improved OrderBook performance
+- Improved workspace load time
+
+1.3.3
+- Reduced server memory usage
+
+1.3.2
+- Improved OrderBook performance
+- Fix some memory leaks when switching workspaces
+
+1.3.1
+- Improved server side logging
+- Fixed leaking of market data subscriptions on market view form closing
+
+1.3.0
+- Significantly improved order execution latency
+- Added latency monitoring tools
+- Provided core binding controls to system admins so as to reduce latency jitter
+
 1.1.22
 - Orderbook: Orders in Error status can now be recreated
 - Orderbook: Orders containing child orders will now be collapsed by default.

src/client/spark/Admin/AdminModel.cs

@@ -11,18 +11,22 @@ namespace mbts.spark {
    [Export]
    class AdminModel : IDisposable {
       readonly string _user_name;
+      public string user_name => _user_name;
+      
       readonly BehaviorSubject<IReadOnlyDictionary<uint, string>>
          _accounts_subject = new BehaviorSubject<IReadOnlyDictionary<uint, string>>(new Dictionary<uint, string>()),
-         _users_subject = new BehaviorSubject<IReadOnlyDictionary<uint, string>>(new Dictionary<uint, string>());
+         _users_subject = new BehaviorSubject<IReadOnlyDictionary<uint, string>>(new Dictionary<uint, string>());               
       IReadOnlyDictionary<uint, string>
          _account_map = new Dictionary<uint, string>(),
          _user_map = new Dictionary<uint, string>();
-      IDisposable _dispose;
+      readonly BehaviorSubject<AdminModelWrapper> _wrapper_subject = new BehaviorSubject<AdminModelWrapper>(null);         
+      IDisposable _dispose;      
 
       [ImportingConstructor]
       AdminModel(AdminService service, [Import("username")]string user_name) {
          _user_name = user_name;
          _dispose = service.subscribei_admin_data().Select(update => {
+            _wrapper_subject.OnNext(new AdminModelWrapper(update.all.FirstOrDefault(), _user_name));
             IEnumerable<BTSUserImmutable> active_users =
                from admin_data in update.all
                from bts_user in admin_data.Users
@@ -52,9 +56,10 @@ namespace mbts.spark {
       }
 
       public IObservable<IReadOnlyDictionary<uint, string>> on_users => _users_subject.AsObservable();
-      public IObservable<IReadOnlyDictionary<uint, string>> on_accounts => _accounts_subject.AsObservable();
+      public IObservable<IReadOnlyDictionary<uint, string>> on_accounts => _accounts_subject.AsObservable();      
       public IObservable<IReadOnlyList<UserTuple>> on_user_tuples => on_users.Select(d => d.Select(kv => new UserTuple(kv.Key, kv.Value)).ToList());
       public IObservable<IReadOnlyList<AccountTuple>> on_account_tuples => on_accounts.Select(d => d.Select(kv => new AccountTuple(kv.Key, kv.Value)).ToList());
+      public IObservable<AdminModelWrapper> on_admin_data => _wrapper_subject.AsObservable();
 
       public UserTuple? get_current_user() {
          foreach (var kv in _user_map) {

src/client/spark/Admin/AdminModelWrapper.cs

@@ -0,0 +1,65 @@
+using bts;
+using bts.api.admin;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace mbts.spark 
+{
+   class AdminModelWrapper
+   {
+      readonly AdminDataImmutable? _admin_data;
+      readonly string _user_name;
+      readonly List<BTSAccountImmutable> _allowed_accounts;
+      readonly Dictionary<string, string> _allowed_account_dict = new Dictionary<string, string>();
+      public AdminModelWrapper(AdminDataImmutable? admin_data = null, string user_name = null)
+      {
+         _admin_data = admin_data;
+         _user_name = user_name;
+         _allowed_accounts = search_admin(u => u.Name == _user_name).Select(x => x.Item1).ToList();
+         foreach(var val in _allowed_accounts)
+            _allowed_account_dict[val.Name] = val.Name;
+      }
+      public IEnumerable<Tuple<BTSAccountImmutable, TradingPermissionUnitImmutable>> get_trading_permissions(UInt32 user_id)
+      {
+         return search_admin(u => u.Id == user_id);
+      }         
+      public IEnumerable<BTSAccountImmutable> viewable_accounts()
+      {
+         return _allowed_accounts;
+      }
+      public bool can_view_account(string account)
+      {
+         return _allowed_account_dict.ContainsKey(account);
+      }
+
+      IEnumerable<Tuple<BTSAccountImmutable, TradingPermissionUnitImmutable>> search_admin(Func<BTSUserImmutable, bool> predicate)
+      {
+         var rv = new List<Tuple<BTSAccountImmutable, TradingPermissionUnitImmutable>>();
+         if (_admin_data.HasValue)
+         {
+            var users = _admin_data.Value.Users.Where(predicate);
+            if (users != null && users.Count() == 1)
+            {
+               var user = users.ElementAt(0);
+               if (user.HasUserData && user.UserData.HasAccountPermissions)
+               {
+                  foreach (var kv in user.UserData.AccountPermissions.AccountPermissions)
+                  {
+                     var account_id = kv.Key;
+                     var accounts = _admin_data.Value.Accounts.Where(a => a.Id == account_id);
+                     if (accounts != null && accounts.Count() == 1)
+                     {
+                        var account = accounts.ElementAt(0);
+                        var tp = kv.Value.TradingPermission;
+                        var rv_val = new TradingPermissionUnitImmutable(tp);
+                        rv.Add(Tuple.Create(account, rv_val));
+                     }
+                  }
+               }
+            }
+         }
+         return rv;
+      }
+   }
+}

src/client/spark/Admin/PermissionFilters.cs

@@ -0,0 +1,66 @@
+using bts;
+using bts.api;
+using bts.api.core;
+using bts.api.orderman;
+using bts.api.position;
+using bts.spark;
+using mbts.collections;
+using mbts.spark;
+using mbts.spark.utilities;
+using mbts.utils;
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Collections.Specialized;
+using System.Linq;
+using System.Reactive.Concurrency;
+using System.Reactive.Linq;
+
+namespace bts.api.core
+{
+
+   static class PermissionFilters
+   {
+      public static IObservable<Update<PositionUnitImmutable, int>> PermissionFilter(this IObservable<Update<PositionUnitImmutable, int>> source, AdminModel am)
+      {
+         var pred = Observable.Create<Func<PositionUnitImmutable, bool>>( observer =>
+         {
+            return am.on_admin_data.Subscribe(x =>
+            {
+               Func<PositionUnitImmutable, bool> filter = (z => x.can_view_account(z.Account));
+               observer.OnNext(filter);
+            });
+         });
+
+         return source.Filter(pred);
+      }
+
+      public static IObservable<Update<OrderRecordImmutable, uint>> PermissionFilter(this IObservable<Update<OrderRecordImmutable, uint>> source, AdminModel am)
+      {
+         var pred = Observable.Create<Func<OrderRecordImmutable, bool>>(observer =>
+        {
+           return am.on_admin_data.Subscribe(x =>
+           {
+              Func<OrderRecordImmutable, bool> filter = (z => x.can_view_account(z.account()));
+              observer.OnNext(filter);
+           });
+        });
+
+         return source.Filter(pred);
+      }
+
+      public static IObservable<Update<FillImmutable, string>> PermissionFilter(this IObservable<Update<FillImmutable, string>> source, AdminModel am)
+      {
+         var pred = Observable.Create<Func<FillImmutable, bool>>(observer =>
+        {
+           return am.on_admin_data.Subscribe(x =>
+           {
+              Func<FillImmutable, bool> filter = (z => x.can_view_account(z.Account));
+              observer.OnNext(filter);
+           });
+        });
+
+         return source.Filter(pred);
+      }
+   }
+}

src/client/spark/Daemons/SecurityPickerDaemon.cs

@@ -53,8 +53,8 @@ namespace bts {
          }
          //
          void handle_bad_icon(string icon_error_desc) {
-            Log.Api.warn($"{nameof(SecurityPickerDaemon)} received a message '{message.title}' with {icon_error_desc}, will use {nameof(Icons.AdminFrozen)}.");
-            icon = Icons.AdminFrozen;
+            Log.Api.warn($"{nameof(SecurityPickerDaemon)} received a message '{message.title}' with {icon_error_desc}, will use {nameof(Icons.Admin)}.");
+            icon = Icons.Admin;
          }
          //
          var cpv = new ContractPickerView(vm);

src/client/spark/Images/Icons.cs

@@ -16,11 +16,6 @@ namespace mbts.ui {
          ToolsGeometryKey = nameof(ToolsGeometryKey),
          DraggableGeometryKey = nameof(DraggableGeometryKey);
 
-      static Icons() {
-         // TODO: probably do this for all icons, only reason for holding back is the uncertainity of any unexpected impacts
-         AdminFrozen = Admin.frozen();
-      }
-
       public static Geometry PlayGeometry => Geometry.Parse(_get_geometric_path(PlayGeometryKey).FirstOrDefault() ?? string.Empty);
 
       static readonly string base_uri = "pack://application:,,,/Images/Icons/";
@@ -32,40 +27,43 @@ namespace mbts.ui {
 
       [Obsolete("Please only add new vector icons using icons.xaml")]
       private static ImageSource load_image(string filename) {
-         return new BitmapImage(uri(filename));
+         return new BitmapImage(uri(filename)).frozen();
+      }
+
+      private static ImageSource load_resource_and_freeze(string resource_name) {
+         return ((DrawingImage)_resources[resource_name]).frozen();
       }
 
 #pragma warning disable CS0618 // Type or member is obsolete
       public static ImageSource Logo { get { return load_image("logo.ico"); } }
-      public static ImageSource Admin { get { return (DrawingImage)_resources["Admin"]; } }
-      public static ImageSource AdminFrozen { get; }
-      public static ImageSource Analytics { get { return (DrawingImage)_resources["Analytics"]; } }
-      public static ImageSource Sheets { get { return (DrawingImage)_resources["Sheets"]; } }
-      public static ImageSource Trades { get { return (DrawingImage)_resources["Trades"]; } }
-      public static ImageSource Depth { get { return (DrawingImage)_resources["Depth"]; } }
-      public static ImageSource Ladder { get { return (DrawingImage)_resources["Ladder"]; } }
+      public static ImageSource Admin { get { return load_resource_and_freeze("Admin"); } }
+      public static ImageSource Analytics { get { return load_resource_and_freeze("Analytics"); } }
+      public static ImageSource Sheets { get { return load_resource_and_freeze("Sheets"); } }
+      public static ImageSource Trades { get { return load_resource_and_freeze("Trades"); } }
+      public static ImageSource Depth { get { return load_resource_and_freeze("Depth"); } }
+      public static ImageSource Ladder { get { return load_resource_and_freeze("Ladder"); } }
       public static ImageSource SheetsRed { get { return load_image("SheetsRed.png"); } }
       public static ImageSource SheetsGreen { get { return load_image("SheetsGreen.png"); } }
-      public static ImageSource Spreads { get { return (DrawingImage)_resources["Spreads"]; } }
+      public static ImageSource Spreads { get { return load_resource_and_freeze("Spreads"); } }
       public static ImageSource BlueSpreads { get { return load_image("BlueSpread.ico"); } }
-      public static ImageSource Reports { get { return (DrawingImage)_resources["Reports"]; } }
-      public static ImageSource Risk { get { return (DrawingImage)_resources["Risk"]; } }
-      public static ImageSource Eye { get { return (DrawingImage)_resources["Eye"]; } }
-      public static ImageSource Coins { get { return (DrawingImage)_resources["Coins"]; } }
+      public static ImageSource Reports { get { return load_resource_and_freeze("Reports"); } }
+      public static ImageSource Risk { get { return load_resource_and_freeze("Risk"); } }
+      public static ImageSource Eye { get { return load_resource_and_freeze("Eye"); } }
+      public static ImageSource Coins { get { return load_resource_and_freeze("Coins"); } }
 
 
       public static ImageSource DataCopy { get { return load_image("DataCopy.ico"); } }
       public static ImageSource Refresh { get { return load_image("Refresh.png"); } }
       public static ImageSource FavoriteWindow { get { return load_image("FavoriteWindow.png"); } }
 
-      public static ImageSource Browser => (DrawingImage)_resources["Browser"];
+      public static ImageSource Browser => load_resource_and_freeze("Browser");
 
       public static ImageSource Date { get { return load_image("Date.png"); } }
 
-      public static ImageSource Cube3d { get { return (DrawingImage)_resources["Cube"]; } }
+      public static ImageSource Cube3d { get { return load_resource_and_freeze("Cube"); } }
       public static ImageSource Algo { get { return Cube3d; } }
       public static ImageSource Quote { get { return Math; } }
-      public static ImageSource Math { get { return (DrawingImage)_resources["Surface"]; } }
+      public static ImageSource Math { get { return load_resource_and_freeze("Surface"); } }
       public static ImageSource Abacus { get { return load_image("Abacus.png"); } }
 
       public static ImageSource Delete { get { return load_image("Delete.png"); } }
@@ -77,58 +75,58 @@ namespace mbts.ui {
 #pragma warning restore CS0618
 
       // Vector icons
-      public static ImageSource Bookmarks { get { return (DrawingImage)_resources["Bookmarks"]; } }
-      public static ImageSource Window { get { return (DrawingImage)_resources["Window"]; } }
-      public static ImageSource ActiveWindow { get { return (DrawingImage)_resources["ActiveWindow"]; } }
-      public static ImageSource Trash { get { return (DrawingImage)_resources["Trash"]; } }
-      public static ImageSource Trash2 { get { return (DrawingImage)_resources["Trash2"]; } }
-      public static ImageSource Crucible { get { return (DrawingImage)_resources["Crucible"]; } }
+      public static ImageSource Bookmarks { get { return load_resource_and_freeze("Bookmarks"); } }
+      public static ImageSource Window { get { return load_resource_and_freeze("Window"); } }
+      public static ImageSource ActiveWindow { get { return load_resource_and_freeze("ActiveWindow"); } }
+      public static ImageSource Trash { get { return load_resource_and_freeze("Trash"); } }
+      public static ImageSource Trash2 { get { return load_resource_and_freeze("Trash2"); } }
+      public static ImageSource Crucible { get { return load_resource_and_freeze("Crucible"); } }
 
-      public static ImageSource Accounting { get { return (DrawingImage)_resources["Accounting"]; } }
-      public static ImageSource Add { get { return (DrawingImage)_resources["Add"]; } }
-      public static ImageSource AddEncircled { get { return (DrawingImage)_resources["AddEncircled"]; } }
-      public static ImageSource Beaker { get { return (DrawingImage)_resources["Beaker"]; } }
-      public static ImageSource Book { get { return (DrawingImage)_resources["Book"]; } }
+      public static ImageSource Accounting { get { return load_resource_and_freeze("Accounting"); } }
+      public static ImageSource Add { get { return load_resource_and_freeze("Add"); } }
+      public static ImageSource AddEncircled { get { return load_resource_and_freeze("AddEncircled"); } }
+      public static ImageSource Beaker { get { return load_resource_and_freeze("Beaker"); } }
+      public static ImageSource Book { get { return load_resource_and_freeze("Book"); } }
 
       public static ImageSource Orders => create_icon(OrdersGeometryKey);
-      public static ImageSource AddOrders { get { return (DrawingImage)_resources["AddOrders"]; } }
-      public static ImageSource CancelOrders { get { return (DrawingImage)_resources["CancelOrders"]; } }
-      public static ImageSource CancelBuys { get { return (DrawingImage)_resources["CancelBuys"]; } }
-      public static ImageSource CancelSells { get { return (DrawingImage)_resources["CancelSells"]; } }
-      public static ImageSource CancelSelected { get { return (DrawingImage)_resources["CancelSelected"]; } }
+      public static ImageSource AddOrders { get { return load_resource_and_freeze("AddOrders"); } }
+      public static ImageSource CancelOrders { get { return load_resource_and_freeze("CancelOrders"); } }
+      public static ImageSource CancelBuys { get { return load_resource_and_freeze("CancelBuys"); } }
+      public static ImageSource CancelSells { get { return load_resource_and_freeze("CancelSells"); } }
+      public static ImageSource CancelSelected { get { return load_resource_and_freeze("CancelSelected"); } }
       
-      public static ImageSource LadderCancelOrders { get { return (DrawingImage)_resources["LadderCancelOrders"]; } }
-      public static ImageSource LadderCancelBuys { get { return (DrawingImage)_resources["LadderCancelBuys"]; } }
-      public static ImageSource LadderCancelSells { get { return (DrawingImage)_resources["LadderCancelSells"]; } }
+      public static ImageSource LadderCancelOrders { get { return load_resource_and_freeze("LadderCancelOrders"); } }
+      public static ImageSource LadderCancelBuys { get { return load_resource_and_freeze("LadderCancelBuys"); } }
+      public static ImageSource LadderCancelSells { get { return load_resource_and_freeze("LadderCancelSells"); } }
 
-      public static ImageSource Synthetics { get { return (DrawingImage)_resources["Synthetics"]; } } 
-      public static ImageSource AddSynthetic { get { return (DrawingImage)_resources["AddSynthetic"]; } }
-      public static ImageSource RemoveSynthetic { get { return (DrawingImage)_resources["RemoveSynthetic"]; } }
-      public static ImageSource ModifySynthetic { get { return (DrawingImage)_resources["ModifySynthetic"]; } }
-      public static ImageSource CloneSynthetic { get { return (DrawingImage)_resources["CloneSynthetic"]; } }
+      public static ImageSource Synthetics { get { return load_resource_and_freeze("Synthetics"); } } 
+      public static ImageSource AddSynthetic { get { return load_resource_and_freeze("AddSynthetic"); } }
+      public static ImageSource RemoveSynthetic { get { return load_resource_and_freeze("RemoveSynthetic"); } }
+      public static ImageSource ModifySynthetic { get { return load_resource_and_freeze("ModifySynthetic"); } }
+      public static ImageSource CloneSynthetic { get { return load_resource_and_freeze("CloneSynthetic"); } }
 
-      public static ImageSource ModifyOrder { get { return (DrawingImage)_resources["ModifyOrder"]; } }
-      public static ImageSource Clock { get { return (DrawingImage)_resources["Clock"]; } }
-      public static ImageSource Defrag { get { return (DrawingImage)_resources["Defrag"]; } }
-      public static ImageSource Display { get { return (DrawingImage)_resources["Display"]; } }
+      public static ImageSource ModifyOrder { get { return load_resource_and_freeze("ModifyOrder"); } }
+      public static ImageSource Clock { get { return load_resource_and_freeze("Clock"); } }
+      public static ImageSource Defrag { get { return load_resource_and_freeze("Defrag"); } }
+      public static ImageSource Display { get { return load_resource_and_freeze("Display"); } }
       public static ImageSource Draggable => create_icon(DraggableGeometryKey);
-      public static ImageSource Export => (DrawingImage)_resources["Export"];
-      public static ImageSource Filter { get { return (DrawingImage)_resources["Filter"]; } }
-      public static ImageSource FilterActive { get { return (DrawingImage)_resources["FilterActive"]; } }
-      public static ImageSource FilterError { get { return (DrawingImage)_resources["FilterError"]; } }
+      public static ImageSource Export => load_resource_and_freeze("Export");
+      public static ImageSource Filter { get { return load_resource_and_freeze("Filter"); } }
+      public static ImageSource FilterActive { get { return load_resource_and_freeze("FilterActive"); } }
+      public static ImageSource FilterError { get { return load_resource_and_freeze("FilterError"); } }
       public static ImageSource Gear => create_icon(GearGeometryKey);
-      public static ImageSource Import => (DrawingImage)_resources["Import"];
-      public static ImageSource Knight { get { return (DrawingImage)_resources["Knight"]; } }
+      public static ImageSource Import => load_resource_and_freeze("Import");
+      public static ImageSource Knight { get { return load_resource_and_freeze("Knight"); } }
       public static ImageSource Payment => create_icon(PaymentGeometryKey);
-      public static ImageSource Spawn { get { return (DrawingImage)_resources["Spawn"]; } }
-      public static ImageSource Target { get { return (DrawingImage)_resources["Target"]; } }
-      public static ImageSource Tetris { get { return (DrawingImage)_resources["Tetris"]; } }
+      public static ImageSource Spawn { get { return load_resource_and_freeze("Spawn"); } }
+      public static ImageSource Target { get { return load_resource_and_freeze("Target"); } }
+      public static ImageSource Tetris { get { return load_resource_and_freeze("Tetris"); } }
       public static ImageSource Tools => create_icon(ToolsGeometryKey);
-      public static ImageSource ViewSettings => (DrawingImage)_resources["ViewSettings"];
-      public static ImageSource Wrench { get { return (DrawingImage)_resources["Wrench"]; } }
-      public static ImageSource RightArrow { get { return ((DrawingImage)_resources["RightArrow"]); } }
-      public static ImageSource DownArrow { get { return ((DrawingImage)_resources["DownArrow"]); } }
-      public static ImageSource Support => (DrawingImage)_resources["SupportIcon"];
+      public static ImageSource ViewSettings => load_resource_and_freeze("ViewSettings");
+      public static ImageSource Wrench { get { return load_resource_and_freeze("Wrench"); } }
+      public static ImageSource RightArrow { get { return (load_resource_and_freeze("RightArrow")); } }
+      public static ImageSource DownArrow { get { return (load_resource_and_freeze("DownArrow")); } }
+      public static ImageSource Support => load_resource_and_freeze("SupportIcon");
 
       public static ImageSource create_icon(string geometric_path_key) =>
          create_icon(geometric_path_key, Brushes.Black);

src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesVM.cs

@@ -121,7 +121,7 @@ namespace mbts.spark {
 
          public Row(TimeAndSalesVM parent, SecurityImmutable security, TradeMessageImmutable trade) : base(security) {
             _trade_key = trade.Key;
-            time = trade.Time.lldatetime();
+            time = ((ulong)trade.Time).lldatetime().ToLocalTime();
             price = trade.Mantissa * Math.Pow(10, trade.Exponent);
             size = trade.Size;
             aggressor_side = trade.AggressorSide;

src/client/spark/Modules/Settings/SettingsShellModule.cs

@@ -42,7 +42,7 @@ namespace bts.ui {
 
          _bus = bus;
       }
-      public ImageSource icon => Icons.AdminFrozen;
+      public ImageSource icon => Icons.Admin;
 
       public string name => "Settings";
 

src/client/spark/Modules/Trading/OrderBook/OrderBookCell.cs

@@ -47,7 +47,7 @@ namespace mbts.spark {
             where T : struct =>
             new OrderBookCellStore<T?>(row, column, prop_name, get_value, select_other, format, CompareUtils.nullable_comparison(compare), EqualityComparer.CreateNullable<T>(EqualityComparer<T>.Default.Equals));
 
-         class OrderBookCellStore<T> : IOrderBookCellStore<T> {
+         class OrderBookCellStore<T> : IOrderBookCellStore<T>, INotifyPropertyChanged {
             readonly OrderBookRowVM _row;
             readonly ICVOrderBookColumn _column;
             readonly string _prop_name;
@@ -58,6 +58,10 @@ namespace mbts.spark {
             readonly IEqualityComparer<T> _eq_comparer;
             OrderBookCell<T> _current;
 
+            // Property changed handler to avoid WPF binding leak
+            // https://blog.jetbrains.com/dotnet/2014/09/04/fighting-common-wpf-memory-leaks-with-dotmemory/
+            public event PropertyChangedEventHandler PropertyChanged = (_,__) => { };
+
             public OrderBookCellStore(
               OrderBookRowVM row,
               ICVOrderBookColumn column,

src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs

@@ -15,7 +15,7 @@ using OrderStatus = bts.api.orderman.OrderStatus;
 
 namespace mbts.spark {
    partial class OrderBookRowVM : DisposableViewModelBase, IEquatable<OrderBookRowVM>, DIYGrid.IRowVM<OrderRecordTuple>, DIYGrid.IActionableRowVM {
-      public readonly PredicateAnimator<OrderBookRowVM, OrderRecordTuple> animator;
+      //public readonly PredicateAnimator<OrderBookRowVM, OrderRecordTuple> animator;
 
       const string _unknown_security_price_format_string = "0.#########";
       const int _unknown_security_price_num_fractional_digits = 9;
@@ -30,31 +30,66 @@ namespace mbts.spark {
       string _symbol;
       OrderBookRowVM _parent;
       OrderBookVM.BranchFilterSource _branch_filter_source;
-      IDisposable _branch_filter_sub;
       // Core
-      IOrderBookCellStore<uint> _order_key_store;
-      IOrderBookCellStore<uint> _parent_order_key_store;
-      IOrderBookCellStore<uint> _security_key_store;
-      IOrderBookCellStore<SecurityImmutable?> _security_store;
-      IOrderBookCellStore<Venue> _exchange_store;
-      IOrderBookCellStore<string> _symbol_store;
+      IOrderBookCellStore<uint> _order_key_store_backing;
+      IOrderBookCellStore<uint> _parent_order_key_store_backing;
+      IOrderBookCellStore<uint> _security_key_store_backing;
+      IOrderBookCellStore<SecurityImmutable?> _security_store_backing;
+      IOrderBookCellStore<Venue> _exchange_store_backing;
+      IOrderBookCellStore<string> _symbol_store_backing;
       // Def
-      IOrderBookCellStore<CustomOrderDefinition.CustomOneofCase> _order_type_store;
-      IOrderBookCellStore<Side> _side_store;
-      IOrderBookCellStore<string> _user_store;
-      IOrderBookCellStore<string> _account_store;
-      IOrderBookCellStore<TimeInForce> _time_in_force_store;
-      IOrderBookCellStore<DateTime> _time_created_store;
+      IOrderBookCellStore<CustomOrderDefinition.CustomOneofCase> _order_type_store_backing;
+      IOrderBookCellStore<Side> _side_store_backing;
+      IOrderBookCellStore<string> _user_store_backing;
+      IOrderBookCellStore<string> _account_store_backing;
+      IOrderBookCellStore<TimeInForce> _time_in_force_store_backing;
+      IOrderBookCellStore<DateTime> _time_created_store_backing;
       // State
-      IOrderBookCellStore<DateTime> _last_change_store;
-      IOrderBookCellStore<OrderStatus> _status_store;
-      IOrderBookCellStore<double> _price_store;
-      IOrderBookCellStore<uint> _quantity_store;
-      IOrderBookCellStore<uint> _filled_quantity_store;
-      IOrderBookCellStore<double> _last_price_store;
-      IOrderBookCellStore<double> _avg_price_store;
-      IOrderBookCellStore<string> _last_msg_store;
-      IOrderBookCellStore<OrderSubType> _order_sub_type_store;
+      IOrderBookCellStore<DateTime> _last_change_store_backing;
+      IOrderBookCellStore<OrderStatus> _status_store_backing;
+      IOrderBookCellStore<double> _price_store_backing;
+      IOrderBookCellStore<uint> _quantity_store_backing;
+      IOrderBookCellStore<uint> _filled_quantity_store_backing;
+      IOrderBookCellStore<double> _last_price_store_backing;
+      IOrderBookCellStore<double> _avg_price_store_backing;
+      IOrderBookCellStore<string> _last_msg_store_backing;
+      IOrderBookCellStore<OrderSubType> _order_sub_type_store_backing;
+      // Core
+      IOrderBookCellStore<uint> _order_key_store => _order_key_store_backing ?? (_order_key_store_backing = CellFactory.create(this, _book_vm.columns.order_key_column, nameof(order_key), r => r._order.Key, r => r._order_key_store));
+      IOrderBookCellStore<uint> _parent_order_key_store => _parent_order_key_store_backing ?? (_parent_order_key_store_backing = CellFactory.create(this, _book_vm.columns.parent_key_column, nameof(parent_order_key), r => r._order.ParentKey, r => r._parent_order_key_store));
+      IOrderBookCellStore<uint> _security_key_store => _security_key_store_backing ?? (_security_key_store_backing = CellFactory.create(this, _book_vm.columns.security_key_column, nameof(security_key), r => r._order.security_key(), r => r._security_key_store));
+      IOrderBookCellStore<SecurityImmutable?> _security_store => _security_store_backing ?? (_security_store_backing = CellFactory.create_nullable(this, _book_vm.columns.security_column, nameof(security), r => r._security, r => r._security_store, v => v.display_name()));
+      IOrderBookCellStore<Venue> _exchange_store => _exchange_store_backing ?? (_exchange_store_backing = CellFactory.create(this, _book_vm.columns.exchange_column, nameof(exchange), r => r._security.HasValue ? _security.Value.Exchange : Venue.Unset, r => r._exchange_store));
+      IOrderBookCellStore<string> _symbol_store => _symbol_store_backing ?? (_symbol_store_backing = CellFactory.create(this, _book_vm.columns.symbol_column, nameof(symbol), r => r._symbol, r => r._symbol_store, v => !String.IsNullOrEmpty(v) ? v : "unknown symbol"));
+      // Def
+      IOrderBookCellStore<CustomOrderDefinition.CustomOneofCase> _order_type_store => _order_type_store_backing ?? (_order_type_store_backing = CellFactory.create(this, _book_vm.columns.order_type_column, nameof(order_type), r => r._order.order_type(), r => r._order_type_store));
+      IOrderBookCellStore<Side> _side_store => _side_store_backing ?? (_side_store_backing = CellFactory.create(this, _book_vm.columns.side_column, nameof(side), r => r._order.side(), r => r._side_store));
+      IOrderBookCellStore<string> _user_store => _user_store_backing ?? (_user_store_backing = CellFactory.create(this, _book_vm.columns.user_column, nameof(user), r => _order.user(), r => r._user_store));
+      IOrderBookCellStore<string> _account_store => _account_store_backing ?? (_account_store_backing = CellFactory.create(this, _book_vm.columns.account_column, nameof(account), r => r._order.account(), r => r._account_store));
+      IOrderBookCellStore<TimeInForce> _time_in_force_store => _time_in_force_store_backing ?? (_time_in_force_store_backing = CellFactory.create(this, _book_vm.columns.time_in_force_column, nameof(time_in_force), r => r._order.time_in_force(), r => r._time_in_force_store));
+      IOrderBookCellStore<DateTime> _time_created_store => _time_created_store_backing ?? (_time_created_store_backing = CellFactory.create(this, _book_vm.columns.time_created_column, nameof(time_created), r => r._order.created(), r => r._time_created_store, format_date));
+      // State
+      IOrderBookCellStore<DateTime> _last_change_store => _last_change_store_backing ?? (_last_change_store_backing = CellFactory.create(this, _book_vm.columns.last_change_column, nameof(last_change), r => _order.last_change(), r => r._last_change_store, format_date));
+      IOrderBookCellStore<OrderStatus> _status_store => _status_store_backing ?? (_status_store_backing = CellFactory.create(this, _book_vm.columns.status_column, nameof(status), r => _order.status(), r => r._status_store));
+      IOrderBookCellStore<double> _price_store => _price_store_backing ?? (_price_store_backing = CellFactory.create(this, _book_vm.columns.price_column, nameof(price), r => _order.price(), r => r._price_store, format_price));
+      IOrderBookCellStore<uint> _quantity_store => _quantity_store_backing ?? (_quantity_store_backing = CellFactory.create(this, _book_vm.columns.qty_column, nameof(quantity), r => _order.quantity(), r => r._quantity_store));
+      IOrderBookCellStore<uint> _filled_quantity_store => _filled_quantity_store_backing ?? (_filled_quantity_store_backing = CellFactory.create(this, _book_vm.columns.filled_qty_column, nameof(filled_quantity), r => _order.filled(), r => r._filled_quantity_store));
+      IOrderBookCellStore<double> _last_price_store => _last_price_store_backing ?? (_last_price_store_backing = CellFactory.create(this, _book_vm.columns.last_price_column, nameof(last_price), r => _order.last_price(), r => r._last_price_store, format_price));
+      IOrderBookCellStore<double> _avg_price_store => _avg_price_store_backing ?? (_avg_price_store_backing = CellFactory.create(this, _book_vm.columns.avg_price_column, nameof(avg_price), r => _order.avg_price(), r => r._avg_price_store, format_price));
+      IOrderBookCellStore<string> _last_msg_store => _last_msg_store_backing ?? (_last_msg_store_backing = CellFactory.create(this, _book_vm.columns.last_message_column, nameof(last_msg), r => _order.last_msg(), r => r._last_msg_store));
+      IOrderBookCellStore<OrderSubType> _order_sub_type_store => _order_sub_type_store_backing ?? (_order_sub_type_store_backing = CellFactory.create(this, _book_vm.columns.sub_type_column, nameof(order_sub_type), r => r._order.sub_type(), r => r._order_sub_type_store, format_sub_type));
+
+      static string format_date(DateTime v) => $"{v:M/d} {v:h:mm:ss.fff tt}";
+      string format_price(double v) => _security.HasValue ?
+         _security.Value.ContractSpec.format(v) :
+         Math.Round(v, _unknown_security_price_num_fractional_digits, MidpointRounding.AwayFromZero).ToString(_unknown_security_price_format_string);
+      static string format_sub_type(OrderSubType v) {
+         switch (v) {
+            case OrderSubType.HedgingLeg: return "H";
+            case OrderSubType.QuotingLeg: return "Q";
+            default: return string.Empty;
+         }
+      }
 
       public OrderBookRowVM(IBus bus, OrderBookVM source_vm, OrdersModel model, OrderRecordImmutable record, SecurityMD? s) {
          _book_vm = source_vm;
@@ -62,43 +97,7 @@ namespace mbts.spark {
          _order = record;
          _security_md = s;
          _symbol = _security_md.HasValue ? _security_md.Value.security.Symbol : null;
-         animator = new PredicateAnimator<OrderBookRowVM,OrderRecordTuple>(this);
-         // cells
-         cell_stores = new DIYGrid.IGridCellStore[] {
-            _security_store = CellFactory.create_nullable(this, _book_vm.columns.security_column, nameof(security), r => r._security, r => r._security_store, v => v.display_name()),
-            _security_key_store = CellFactory.create(this, _book_vm.columns.security_key_column, nameof(security_key), r => r._order.security_key(), r => r._security_key_store),
-            _parent_order_key_store = CellFactory.create(this, _book_vm.columns.parent_key_column, nameof(parent_order_key), r => r._order.ParentKey, r => r._parent_order_key_store),
-            _order_key_store = CellFactory.create(this, _book_vm.columns.order_key_column, nameof(order_key), r => r._order.Key, r => r._order_key_store),
-            _symbol_store = CellFactory.create(this, _book_vm.columns.symbol_column, nameof(symbol), r => r._symbol, r => r._symbol_store, v => !String.IsNullOrEmpty(v) ? v : "unknown symbol"),            
-            _time_created_store = CellFactory.create(this, _book_vm.columns.time_created_column, nameof(time_created), r => r._order.created(), r => r._time_created_store, format_date),
-            _user_store = CellFactory.create(this, _book_vm.columns.user_column, nameof(user), r => _order.user(), r => r._user_store),
-            _account_store = CellFactory.create(this, _book_vm.columns.account_column, nameof(account), r => r._order.account(), r => r._account_store),
-            _exchange_store = CellFactory.create(this, _book_vm.columns.exchange_column, nameof(exchange), r => r._security.HasValue ? _security.Value.Exchange : Venue.Unset, r => r._exchange_store),
-            _time_in_force_store = CellFactory.create(this, _book_vm.columns.time_in_force_column, nameof(time_in_force), r => r._order.time_in_force(), r => r._time_in_force_store),
-            _order_type_store = CellFactory.create(this, _book_vm.columns.order_type_column, nameof(order_type), r => r._order.order_type(), r => r._order_type_store),
-            _order_sub_type_store = CellFactory.create(this, _book_vm.columns.sub_type_column, nameof(order_sub_type), r => r._order.sub_type(), r => r._order_sub_type_store, format_sub_type),
-            _side_store = CellFactory.create(this, _book_vm.columns.side_column, nameof(side), r => r._order.side(), r => r._side_store),
-            _last_change_store = CellFactory.create(this, _book_vm.columns.last_change_column, nameof(last_change), r => _order.last_change(), r => r._last_change_store, format_date),
-            _price_store = CellFactory.create(this, _book_vm.columns.price_column, nameof(price), r => _order.price(), r => r._price_store, format_price),
-            _quantity_store = CellFactory.create(this, _book_vm.columns.qty_column, nameof(quantity), r => _order.quantity(), r => r._quantity_store),
-            _filled_quantity_store = CellFactory.create(this, _book_vm.columns.filled_qty_column, nameof(filled_quantity), r => _order.filled(), r => r._filled_quantity_store),
-            _avg_price_store = CellFactory.create(this, _book_vm.columns.avg_price_column, nameof(avg_price), r => _order.avg_price(), r => r._avg_price_store, format_price),
-            _last_price_store = CellFactory.create(this, _book_vm.columns.last_price_column, nameof(last_price), r => _order.last_price(), r => r._last_price_store, format_price),
-            _status_store = CellFactory.create(this, _book_vm.columns.status_column, nameof(status), r => _order.status(), r => r._status_store),
-            _last_msg_store = CellFactory.create(this, _book_vm.columns.last_message_column, nameof(last_msg), r => _order.last_msg(), r => r._last_msg_store),
-         };
-         //
-         string format_price(double v) => _security.HasValue ? _security.Value.ContractSpec.format(v) :
-            Math.Round(v, _unknown_security_price_num_fractional_digits, MidpointRounding.AwayFromZero).ToString(_unknown_security_price_format_string);
-         string format_date(DateTime v) => $"{v:M/d} {v:h:mm:ss.fff tt}";
-         string format_sub_type(OrderSubType v) {
-            switch (v) {
-               case OrderSubType.HedgingLeg: return "H";
-               case OrderSubType.QuotingLeg: return "Q";
-               default: return string.Empty;
-            }
-         }
-         //
+         //animator = new PredicateAnimator<OrderBookRowVM,OrderRecordTuple>(this);
          _visible = backing(nameof(is_visible), true);
          _expanded = backing(nameof(is_expanded), false);
          _depth_level = backing(nameof(depth_level), () => {
@@ -114,15 +113,11 @@ namespace mbts.spark {
          _show_order_modify_cmd = new ObservantCommand(() => bus.post(SparkTags.ShowModifyOrderTag, _order.Key));
          _show_order_modify_cmd.OnNext(status.value == OrderStatus.Working);
          row_passes_filter = _book_vm.columns.filter(this);
-         _passes_filter = backing(nameof(passes_filter),
-            Observable.FromEvent<Action<IObservable<bool>>, IObservable<bool>>(
-               cb => _branch_filter_source_changed += cb, cb => _branch_filter_source_changed -= cb)
-         .SelectMany(o => o));
+         _passes_filter = backing(nameof(passes_filter), row_passes_filter);
       }
 
       public event Action invalidated;
       public event Action<(OrderRecordTuple old_val, OrderRecordTuple new_val)> updated;
-      event Action<IObservable<bool>> _branch_filter_source_changed;
 
       public OrderRecordImmutable order => _order;
       public OrderMetadata order_metadata => _model.factory.get_metadata(order_type.value);
@@ -217,7 +212,31 @@ namespace mbts.spark {
       }
 
       public ObservableCollection<RowAnimation> row_animations { get; } = new ObservableCollection<RowAnimation>();
-      public IReadOnlyList<DIYGrid.IGridCellStore> cell_stores { get; }
+
+      DIYGrid.IGridCellStore[] _cell_stores_backing;
+      public IReadOnlyList<DIYGrid.IGridCellStore> cell_stores => _cell_stores_backing ?? (_cell_stores_backing = new DIYGrid.IGridCellStore[] {
+         _security_store,
+         _security_key_store,
+         _parent_order_key_store,
+         _order_key_store,
+         _symbol_store,
+         _time_created_store,
+         _user_store,
+         _account_store,
+         _exchange_store,
+         _time_in_force_store,
+         _order_type_store,
+         _order_sub_type_store,
+         _side_store,
+         _last_change_store,
+         _price_store,
+         _quantity_store,
+         _filled_quantity_store,
+         _avg_price_store,
+         _last_price_store,
+         _status_store,
+         _last_msg_store
+      });
 
       public static Stack<OrderBookRowVM> lineage(OrderBookRowVM node) {
          var rv = new Stack<OrderBookRowVM>();
@@ -228,11 +247,15 @@ namespace mbts.spark {
       }
 
       public void join_branch_filter_source(OrderBookVM.BranchFilterSource new_branch_filter_source) {
+         if (_branch_filter_source != null) {
+            _branch_filter_source.passes_filter_changed -= _passes_filter.set;
+         }
          _branch_filter_source = new_branch_filter_source;
          _branch_filter_source.set_row_passes_filter(_order.Key, row_passes_filter);
-         _branch_filter_source_changed?.Invoke(_branch_filter_source);
          foreach (var child_row in _child_rows)
             child_row.join_branch_filter_source(_branch_filter_source);
+         _passes_filter.set(_branch_filter_source.passes_filter);
+         _branch_filter_source.passes_filter_changed += _passes_filter.set;
       }
 
       /// <summary>
@@ -311,7 +334,13 @@ namespace mbts.spark {
       public override int GetHashCode() => (int)_order.Key;
       public override string ToString() => $"{order_key} {last_change} {parent_order_key}";
 
-      protected override void Dispose(bool disposing) => Disposable.dispose(ref _branch_filter_sub);
+      protected override void Dispose(bool disposing) {
+         // idempotent
+         if (_branch_filter_source != null) {
+            _branch_filter_source.passes_filter_changed -= _passes_filter.set;
+            _branch_filter_source = null;
+         }
+      }
 
       static IEnumerable<OrderBookRowVM> _descendents(OrderBookRowVM root) {
          foreach (var child in root.child_orders) {

src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs

@@ -1,4 +1,5 @@
-using bts.api;
+using bts;
+using bts.api;
 using bts.api.core;
 using bts.api.orderman;
 using bts.shell;
@@ -27,10 +28,7 @@ namespace mbts.spark {
    partial class OrderBookVM : DisposableViewModelBase, IHandleRowSelection {
       readonly LiveSortingColumKeys _live_sorting_columns;
       readonly Property<OrderBookRowVM> _current_row;
-      readonly Subject<(IEnumerable<SecurityImmutable> added, IEnumerable<SecurityImmutable> removed)> _order_securities_subject =
-         new Subject<(IEnumerable<SecurityImmutable> added, IEnumerable<SecurityImmutable> removed)>();
       OrderBookStartupData _state;
-
       Dictionary<uint, OrderBookRowVM> _order_rows_cache = new Dictionary<uint, OrderBookRowVM>();
       Dictionary<uint, BranchFilterSource> _branch_filter_roots = new Dictionary<uint, BranchFilterSource>();
       ObservableCollection<OrderBookRowVM> _order_rows = new ObservableCollection<OrderBookRowVM>();
@@ -39,7 +37,8 @@ namespace mbts.spark {
       SetMap<uint, uint> _security_orders = new SetMap<uint, uint>();
 
       public OrderBookVM(IObservable<string> owner_title, IBus bus, INotificationService notification_service, OrdersModel model) {
-         columns = new OrderBookColumnCollection(bus, model, _order_securities_subject);
+         var order_securities_subject = new Subject<(IEnumerable<SecurityImmutable> added, IEnumerable<SecurityImmutable> removed)>();
+         columns = new OrderBookColumnCollection(bus, model, order_securities_subject);
          rows_view = CollectionViewSource.GetDefaultView(_order_rows);
          rows_view.Filter = obj => obj is OrderBookRowVM vm && vm.is_visible && vm.passes_filter;
          columns.filter_changed += () => {
@@ -71,11 +70,12 @@ namespace mbts.spark {
 
          _dispose = Disposable.Create(
             bus.subscribe(SparkTags.SecuritiesTag).OnDispatcher().Subscribe(on_securities),
-            model.on_orders.OnDispatcher().AddRemoveOnly().Subscribe(on_orders),
+            model.on_orders.OnDispatcher().Subscribe(on_orders),
             grid_settings_manager,
-            columns);
+            columns,
+            order_securities_subject);
 
-         void on_orders(LegacyTableUpdate<OrderRecordImmutable, uint> table) {
+         void on_orders(TableUpdate<OrderRecordImmutable, uint> table) {
             // clear everything (most likely the server restarted)
             if (table.removed.Count() == _order_rows.Count && !table.added.Any()) {
                _selected_rows.Clear();
@@ -86,18 +86,14 @@ namespace mbts.spark {
                _parent_to_children.clear();
                _security_orders.clear();
                missing_security_orders.clear();
+               Log.Ui.warn("OrderBook received reset message.");
                return;
             }
-            var changed_keys = new HashSet<uint>();
             var added_securities = new List<SecurityImmutable>(1);
             var removed_securities = new List<SecurityImmutable>(1);
 
             foreach (var o in table.removed) {
-               // if state has it, then it wasn't actually removed, just changed
-               if (table.state.TryGetValue(o.Key, out OrderRecordImmutable changed_order)) {
-                  changed_keys.Add(o.Key);
-                  _order_rows_cache[o.Key].update_order(changed_order);
-               } else if (_order_rows_cache.TryPopValue(o.Key, out OrderBookRowVM row)) {
+               if (_order_rows_cache.TryPopValue(o.Key, out OrderBookRowVM row)) {
                   row.Dispose();
                   _order_rows.Remove(row);
                   if (_security_orders.removes_key(o.Def.SecurityKey, o.Key)) {
@@ -124,10 +120,10 @@ namespace mbts.spark {
                   missing_security_orders.remove(o.Key);
                }
             }
+            foreach (var ch in table.changes)
+               _order_rows_cache[ch.New.Key].update_order(ch.New);
             var orphans = new Dictionary<uint, OrderBookRowVM>();
             foreach (var o in table.added) {
-               if (changed_keys.Contains(o.Key))
-                  continue;
                SecurityMD? security_md;
                if (security_dict.TryGetValue(o.Def.SecurityKey, out SecurityMD smd)) {
                   security_md = smd;
@@ -170,7 +166,7 @@ namespace mbts.spark {
                _order_rows.Add(row);
             }
             if (added_securities.Count > 0 || removed_securities.Count > 0)
-               _order_securities_subject.OnNext((added_securities, removed_securities));
+               order_securities_subject.OnNext((added_securities, removed_securities));
 
             void try_add_children(OrderBookRowVM row) {
                if (_parent_to_children.try_lookup(row.order.Key, out ISet<uint> children))
@@ -206,7 +202,7 @@ namespace mbts.spark {
                }
             }
             if (added_securities.Count > 0)
-               _order_securities_subject.OnNext((added_securities, new List<SecurityImmutable>(0)));
+               order_securities_subject.OnNext((added_securities, new List<SecurityImmutable>(0)));
          }
       }
 
@@ -243,6 +239,9 @@ namespace mbts.spark {
       internal void remove_branch_filter_root(uint key) =>
          _branch_filter_roots.Remove(key);
 
+      protected override void Dispose(bool disposing) =>
+         Disposable.dispose(_order_rows);
+
       static GridSettingsManager<OrderBookRowVM, OrderRecordTuple, INotificationServiceDataCache, OrderRecordTuplePredicate> _create_grid_settings_manager(
          IObservable<string> owner_title, IBus bus, INotificationService notification_service, OrdersModel model,
          ObservableCollection<OrderBookRowVM> rows, ICollectionView rows_view, ObservableCollection<DIYGrid.ColumnBase> columns) {
@@ -297,24 +296,21 @@ namespace mbts.spark {
          }
       }
 
-      internal class BranchFilterSource : IObservable<bool> {
+      internal class BranchFilterSource {
          readonly HashSet<uint> _orders_pass_filter = new HashSet<uint>();
-         event Action<bool> _passes_filter_changed;
+
+         public event Action<bool> passes_filter_changed;
+
+         public bool passes_filter => _orders_pass_filter.Count > 0;
 
          public void set_row_passes_filter(uint key, bool value) {
             if (value) {
                if (_orders_pass_filter.Add(key) && _orders_pass_filter.Count == 1)
-                  _passes_filter_changed?.Invoke(value);
+                  passes_filter_changed?.Invoke(value);
             } else if (_orders_pass_filter.Remove(key) && _orders_pass_filter.Count == 0) {
-               _passes_filter_changed?.Invoke(value);
+               passes_filter_changed?.Invoke(value);
             }
          }
-
-         public IDisposable Subscribe(IObserver<bool> observer) {
-            _passes_filter_changed += observer.OnNext;
-            observer.OnNext(_orders_pass_filter.Count > 0);
-            return Disposable.Create(() => _passes_filter_changed -= observer.OnNext);
-         }
       }
 
       class LiveSortingColumKeys : HashSet<Columns> {

src/client/spark/Modules/Trading/OrderBook/OrderBookView.xaml.cs

@@ -15,11 +15,9 @@ namespace mbts.spark {
          CVRowTemplateKey = "CVRowTemplateKey";
 
       public const double RowHeight = 17;
-      OrderBookVM _vm;
 
       public OrderBookView(OrderBookVM vm) {
          InitializeComponent();
-         _vm = vm;
          DataContext = vm;
       }
    }

src/client/spark/Orders/OrderTypeVMs/LeanOrderVM.cs

@@ -24,13 +24,15 @@ namespace mbts.spark
             this.child_order_types = new InlineUserProperty<IReadOnlyCollection<CustomOneofCase>>(child_order_types, true);
             cross_threshold = new InlineUserProperty<bts.spark.QuantityMeasure>(
                               new bts.spark.QuantityMeasure() { Ratio = 1.0, Absolute = 1}, false);
-            max_chase_ticks = new InlineUserProperty<uint>(10, false);
+            max_chase_ticks = new InlineUserProperty<uint>(2, false);
+            trade_toward_behavior = new InlineUserProperty<LeanOrderDefinition.Types.OnToward>(LeanOrderDefinition.Types.OnToward.Cancel, false);
          }
          public readonly InlineUserProperty<CustomOneofCase> selected_child_order_type;
          public readonly InlineUserProperty<IReadOnlyCollection<CustomOneofCase>> child_order_types;
          
          public readonly InlineUserProperty<bts.spark.QuantityMeasure> cross_threshold;
          public readonly InlineUserProperty<uint> max_chase_ticks;
+         public readonly InlineUserProperty<LeanOrderDefinition.Types.OnToward> trade_toward_behavior;
       }
 
       class LeanOrderVM : OrderTypeVMBase<LeanOrderDefinition>, IHierarchicalOrderTypeVM, IDisposable 
@@ -44,7 +46,8 @@ namespace mbts.spark
          IDisposable _dispose, _child_order_errors_sub;
          
          public bts.spark.QuantityMeasure cross_threshold { get; set; } = new bts.spark.QuantityMeasure() { Ratio = 1.0, Absolute = 1 };
-         public uint max_chase_ticks { get; set; } = 10;
+         public uint max_chase_ticks { get; set; } = 2;
+         public LeanOrderDefinition.Types.OnToward trade_toward_behavior { get; set; } = LeanOrderDefinition.Types.OnToward.Cancel;      
 
          public LeanOrderVM(OrdersModel model, IObservable<SecurityMD> security_md_source)
             : this(model, security_md_source, security_md_source.Select(smd =>
@@ -77,6 +80,11 @@ namespace mbts.spark
                      max_chase_ticks = p.max_chase_ticks.value;
                      RaisePropertyChanged(nameof(max_chase_ticks));
                   }
+                  if (p.trade_toward_behavior.force_override)
+                  {
+                     trade_toward_behavior = p.trade_toward_behavior.value;
+                     RaisePropertyChanged(nameof(trade_toward_behavior));
+                  }
                   _inline_props = p;
                   _validate();
                })
@@ -94,10 +102,13 @@ namespace mbts.spark
                var i = factory.lean_order();
                i.CrossThreshold = cross_threshold;
                i.MaxChaseTicks = max_chase_ticks;
+               i.TradeTowardBehavior = trade_toward_behavior;
                child_order.assign_to(i.Child);
                value.Lean = i;
             }
-         }         
+         }
+
+         public IEnumerable<LeanOrderDefinition.Types.OnToward> trade_toward_types => EnumHelper.get_values_except<LeanOrderDefinition.Types.OnToward>();         
 
          public ICollection<IOrderTypeVM> child_order_types 
          {

src/client/spark/Orders/OrdersModel.cs

@@ -32,7 +32,7 @@ namespace mbts.spark {
       IDisposable _dispose;
 
       [ImportingConstructor]
-      private OrdersModel(IBus bus, OrderFactory factory, OrderService service, PositionService position_service, SpreaderModel spreader, ErrorReporter error) {
+      private OrdersModel(IBus bus, OrderFactory factory, OrderService service, PositionService position_service, SpreaderModel spreader, ErrorReporter error, AdminModel admin_model) {
          _bus = bus;
          _factory = factory;
          _service = service;
@@ -41,15 +41,15 @@ namespace mbts.spark {
          _error = error;
          var scheduler = ThreadPoolScheduler.Instance;
          var interval = TimeSpan.FromMilliseconds(100);
-         on_orders = _service.subscribei_orders().Naggle(interval, scheduler).Table().Cache();
-         on_positions = _position_service.subscribei_position_data().Naggle(interval, scheduler).Table().Cache();
+         on_orders = _service.subscribei_orders().Naggle(interval, scheduler).PermissionFilter(admin_model).Table().Cache();
+         on_positions = _position_service.subscribei_position_data().Naggle(interval, scheduler).PermissionFilter(admin_model).Table().Cache();
          working_orders = 
             on_orders.Scan(
                ImmutableDictionary<uint,OrderRecordImmutable>.Empty, 
                (workers, table) => _update_working_orders(workers, table))
                .Replay(1).RefCount();
          on_order_tuples = OrderRecordTupleSource.create_observable(bus.subscribe(SparkTags.SecuritiesTag), on_orders);
-         on_fills = _service.subscribei_fills().Naggle(interval, scheduler).Table().Cache();
+         on_fills = _service.subscribei_fills().Naggle(interval, scheduler).PermissionFilter(admin_model).Table().Cache();
          _activity = new OrderActivitySummarizer(bus, on_orders);
          on_fill_tuples = FillTupleSource.create_observable(bus.subscribe(SparkTags.SecuritiesTag), on_orders, on_fills);
          _security_orders_demuxer = new TableUpdateDemuxer<OrderRecordImmutable, uint, uint>(o => o.Key, o => o.security_key());

src/client/spark/core/Bus/SparkCommands.cs

@@ -70,7 +70,7 @@ namespace bts.spark
       public static SparkCommand ShowWorkspaces =>
          SparkCommand.create("Workspaces", Icons.FavoriteWindow, SparkTags.ShowWorkspaces);
       public static SparkCommand ShowSettings =>
-         SparkCommand.create("Settings", Icons.AdminFrozen, SparkTags.ShowSettings);
+         SparkCommand.create("Settings", Icons.Admin, SparkTags.ShowSettings);
 
       // Support
       public static SparkCommand ShowManual =>
@@ -113,12 +113,12 @@ namespace bts.spark
 
       // TODO: remove this method, eventually want all windows to use window_settings_command()
       public static SparkCommand change_securities_command(Func<(PartialCommand<SecurityCriteria>, SecurityCriteria)> get_param) =>
-         SparkCommand.create("Settings", Icons.AdminFrozen, SparkTags.ShowAndEditPickMultiple, get_param);
+         SparkCommand.create("Settings", Icons.Admin, SparkTags.ShowAndEditPickMultiple, get_param);
 
       public static PartialCommand<SecurityCriteria> change_securities_callback(string window_name) =>
-         SparkCommand.create(window_name, Icons.AdminFrozen, new SparkTags.SparkTag<SecurityCriteria>($"Change {window_name} securities callback"));
+         SparkCommand.create(window_name, Icons.Admin, new SparkTags.SparkTag<SecurityCriteria>($"Change {window_name} securities callback"));
 
       public static SparkCommand window_settings_command(IDialogCommand launch_settings_editor) =>
-         new SparkCommandWrapper("Settings", Icons.AdminFrozen, launch_settings_editor);
+         new SparkCommandWrapper("Settings", Icons.Admin, launch_settings_editor);
    }
 }

src/client/spark/core/Collections/ObservableAutoWrapper.cs

@@ -6,6 +6,7 @@ using System.Collections.Generic;
 using NUnit.Framework;
 using mbts.collections.extensions;
 using System.Collections;
+using bts.collections;
 
 namespace mbts.ui.utility
 {

src/client/spark/core/GlobalExtensions/GenericCollectionExtensions.cs

@@ -666,44 +666,6 @@ namespace mbts.collections.extensions {
       }
    }
 
-   public static class ListExtensions
-   {
-      #region RemoveWhere
-      public static void RemoveWhere<T>(this IList<T> list, Func<T, bool> where) { foreach (var r in list.Where(where).ToList()) list.Remove(r); }
-
-      public static void RemoveWhere<T>(this IList<T> list, Func<T, bool> where, Action<T> on_remove) {
-         int count = list.Count;
-         for (int i = 0; i < count; i++) {
-            var t = list[i];
-            if (where(t)) {
-               list.RemoveAt(i--);
-               --count;
-               on_remove(t);
-            }
-         }
-      }
-
-      private static Random rng = new Random();
-
-      public static void Shuffle<T>(this IList<T> list) {
-         int n = list.Count;
-         while (n > 1) {
-            n--;
-            int k = rng.Next(n + 1);
-            T value = list[k];
-            list[k] = list[n];
-            list[n] = value;
-         }
-      }
-
-      #endregion
-
-      public static void RemoveAt<T>(this IList<T> list, int index, int count) {
-         for (int i = 0; i < count; i++)
-            list.RemoveAt(index);
-      }
-   }
-
    public static class DictionaryExtensions
    {
       #region GetValueOrDefault

src/client/spark/core/Settings/ISavedWindowFactory.cs

@@ -29,6 +29,7 @@ namespace mbts.ui.settings
       }
 
       public Task<IShellWindow> restore(WindowState window) {
+         Log.Ui.info("Loading window '{0}'".fmt(window.title));
          bool changed;
          int count = 0;
          do {

src/client/spark/core/Utility/KeyDrivenObservable.cs

@@ -66,15 +66,73 @@ namespace mbts.utils {
 
       class AnonymousConnectable<TResult> : IConnectableObservable<TResult> {
          readonly IObservable<TSource> _source;
-         readonly KeyDrivenObservableSeries<TSource, TResult> _driver;
+         readonly Func<TSource, IObservable<TResult>> _select_next;
+         List<LazySubscription> _lazy_subs = new List<LazySubscription>();
+         KeyDrivenObservableSeries<TSource, TResult> _driver;
+         IDisposable _driver_sub;
+         bool _is_connected = false;
 
          public AnonymousConnectable(IObservable<TSource> source, Func<TSource, IObservable<TResult>> select_next) {
             _source = source;
-            _driver = new KeyDrivenObservableSeries<TSource, TResult>(select_next);
+            _select_next = select_next;
          }
 
-         public IDisposable Connect() => _source.Subscribe(_driver);
-         public IDisposable Subscribe(IObserver<TResult> observer) => _driver.Subscribe(observer);
+         // idempotent
+         public IDisposable Connect() {
+            lock (this) {
+               if (!_is_connected) {
+                  _driver = new KeyDrivenObservableSeries<TSource, TResult>(_select_next);
+                  foreach (var ls in _lazy_subs)
+                     ls.attach();
+                  _lazy_subs.Clear();
+                  _driver_sub = _source.Subscribe(_driver);
+                  _is_connected = true;
+               }
+            }
+            return Disposable.Create(_deconnect);
+         }
+
+         void _deconnect() {
+            lock (this) {
+               if (_is_connected) {
+                  _driver_sub.Dispose();
+                  _driver.Dispose();
+                  _is_connected = false;
+               }
+            }
+         }
+
+         public IDisposable Subscribe(IObserver<TResult> observer) {
+            lock (this) {
+               if (_is_connected)
+                  return _driver.Subscribe(observer);
+               var ls = new LazySubscription(this, observer);
+               _lazy_subs.Add(ls);
+               return ls;
+            }
+            throw new InvalidOperationException();
+         }
+
+         class LazySubscription : IDisposable {
+            readonly IObserver<TResult> _observer;
+            readonly AnonymousConnectable<TResult> _owner;
+            IDisposable _sub;
+
+            public LazySubscription(AnonymousConnectable<TResult> owner, IObserver<TResult> observer) {
+               _owner = owner;
+               _observer = observer;
+            }
+
+            public void attach() =>
+               _sub = _owner._driver.Subscribe(_observer);
+
+            public void Dispose() {
+               lock (_owner) {
+                  Disposable.dispose(ref _sub);
+                  _owner._lazy_subs.Remove(this);
+               }
+            }
+         }
       }
    }
 

src/client/spark/core/Utility/ViewModelBase.cs

@@ -446,9 +446,6 @@ namespace mbts.ui.shell
       public IEnumerable<string> AllErrors { get { return _errors.Values; } }
       #endregion
 
-      protected Property<T> backing<T>(string name, T default_value, bool validate_now = false, params Func<T, bool>[] validation_steps) =>
-          backing(name, default_value, validation_steps.Select(v => new PropertyValidation<T>(v)).ToLArray(), validate_now);
-
       protected Property<T> backing<T>(string name, T default_value, IEnumerable<PropertyValidation<T>> validation_steps, bool validate_now = false) {
          void run_validation(T val) {
             foreach (var pv in validation_steps) {

src/client/spark/view/Controls/DIYGrid/DIYGrid.cs

@@ -168,7 +168,7 @@ namespace mbts.ui.controls {
                o.Resources[ActiveSelectionHighlightKey] = v;
                o.Resources[InactiveSelectionHighlightKey] = new SolidColorBrush(v.Color) { Opacity = 0.2 }.frozen();
                o.Resources[SelectionStrokeKey] = new Pen(o.SelectionHighlight.Color.to_brush().frozen(), 1).frozen();
-               foreach (var col in o.Columns.OfType<ISelectionHighlightListener>())
+               foreach (var col in o.Columns.empty_if_null().OfType<ISelectionHighlightListener>())
                   col.on_selection_fill(v);
             }, (o, v) => v as SolidColorBrush ?? DefaultSelectionHighlight,
             DefaultSelectionHighlight);
@@ -634,7 +634,7 @@ namespace mbts.ui.controls {
          (o, v) => {
             if (DoubleHelper.IsNaNOrInfinity(v))
                v = 0;
-            if (!o._visible_columns.is_null_or_empty()) {
+            if (!o._visible_columns.is_null_or_empty() && o.Columns != null) {
                double max_font_size = o.Columns.Max(c => c.font_size);
                double min_row_height = o.RowPadding.Top + max_font_size + o.RowPadding.Bottom;
                if (v < min_row_height)

src/client/spark/view/Controls/DIYGrid/Settings/GridSettingsManager.cs

@@ -31,7 +31,7 @@ namespace mbts.ui {
       readonly ObservableCollection<TRow> _rows;
       readonly ICollectionView _rows_view;
       readonly ObservableCollection<ColumnBase> _columns;
-      readonly ObservableAutoWrapper<PredicateAnimator<TRow, TRowData>, TRow> _animators;
+      //readonly ObservableAutoWrapper<PredicateAnimator<TRow, TRowData>, TRow> _animators;
       readonly Property<GridSettings> _grid_settings;
       readonly DelegateProperty<FontFamily> _font_family;
       readonly DelegateProperty<FontWeight> _font_weight;
@@ -59,18 +59,18 @@ namespace mbts.ui {
          _font_size = backing(nameof(font_size), () => grid_settings.font_size, nameof(font_size));
          _font_color = backing(nameof(font_color), () => grid_settings.font_color.rgb_solid_brush().frozen(), nameof(grid_settings));
          _selection_highlight = backing(nameof(selection_highlight), () => grid_settings.selection_highlight.to_brush().frozen(), nameof(grid_settings));
-         _animators = new ObservableAutoWrapper<PredicateAnimator<TRow, TRowData>, TRow>(_rows = rows, create_animator);
+         //_animators = new ObservableAutoWrapper<PredicateAnimator<TRow, TRowData>, TRow>(_rows = rows, create_animator);
          _resources = new List<IDisposable> {
             ObservableAutoWrapper.connect<SortDescriptionCollection, SortDescription, string>(
                rows_view.SortDescriptions, live_sorting_props = ((ICollectionViewLiveShaping)rows_view).LiveSortingProperties, sd => sd.PropertyName),
             (launch_settings_editor = new DialogCommand(create_settings_view))
          };
          //
-         PredicateAnimator<TRow, TRowData> create_animator(TRow r) {
-            var pa = new PredicateAnimator<TRow, TRowData>(r);
-            pa.set_predicate_animations(_animation_defs);
-            return pa;
-         }
+         //PredicateAnimator<TRow, TRowData> create_animator(TRow r) {
+         //   var pa = new PredicateAnimator<TRow, TRowData>(r);
+         //   pa.set_predicate_animations(_animation_defs);
+         //   return pa;
+         //}
          Window create_settings_view() {
             var vm = new GridSettingsVM<TRowData, TPredicateBuilder>(owner_title, _bus, _orders_model, create_predicate_vm, create_column_predicate_vm);
             // need to refresh before launching the settings-view bc users can chanve some grid-column-settings properties directly from the grid (e.g. width, is_collapsed, etc.)
@@ -134,9 +134,9 @@ namespace mbts.ui {
                _animation_defs.add(pa.predicate_builder.create_predicate(_arg), PredicateAnimations.flat_definitions<TRow, TRowData>(pa.row_animations, pa.cell_animations, lookup_column));
             }
          }
-         foreach (var a in _animators) {
-            a.set_predicate_animations(_animation_defs);
-         }
+         //foreach (var a in _animators) {
+         //   a.set_predicate_animations(_animation_defs);
+         //}
          // ** in the unfortunate event that we botch backwards compatability, and the columns don't bear a 1:1 relationship with the column-settings
          // don't add insult to injury by crashing **
          if (grid_settings.columns != null && _columns != null) {

src/client/spark/view/Controls/Order/OrderDetailsPopupControl.xaml

@@ -222,6 +222,8 @@
                           <RowDefinition Height="4" />
                           <RowDefinition Height="Auto"/>
                           <RowDefinition Height="4" />
+                          <RowDefinition Height="Auto"/>
+                          <RowDefinition Height="4" />                          
                           <RowDefinition/>
                         </Grid.RowDefinitions>
                         <Grid.ColumnDefinitions>
@@ -233,9 +235,11 @@
                         <uag:QuantityMeasureControl Grid.Row="1" Grid.Column="2" QuantityMeasure="{Binding cross_threshold}"/>
                         <TextBlock Grid.Row="3" VerticalAlignment="Center">Max Chase Ticks</TextBlock>
                         <c:TickUpDownUInt Grid.Row="3" Grid.Column="2" Value="{Binding max_chase_ticks}"/>
-                        <TextBlock Grid.Row="5">Child</TextBlock>
+                        <TextBlock Grid.Row="5">On Toward</TextBlock>
+                        <ComboBox Grid.Row="5" Grid.Column="2" ItemsSource="{Binding trade_toward_types}" ItemTemplate="{StaticResource EnumDescriptionTemplate}" SelectedItem="{Binding trade_toward_behavior}"/>                        
+                        <TextBlock Grid.Row="7">Child</TextBlock>
                         <uag:OrderDetailsPopupControl
-                          Grid.Row="5" Grid.Column="2"
+                          Grid.Row="7" Grid.Column="2"
                           IsOpen="{Binding child_order.editable_properties_open, FallbackValue=False}"
                           ItemsSource="{Binding child_order_types}"
                           SelectedItem="{Binding child_order}"

src/client/spark/view/Controls/SparkMenu/SparkMenu.cs

@@ -1,6 +1,7 @@
 using bts.api;
 using bts.spark;
 using System.Collections.Generic;
+using System.Collections.ObjectModel;
 using System.Linq;
 using System.Windows;
 using System.Windows.Controls;
@@ -14,9 +15,10 @@ namespace mbts.ui.controls {
          SetCurrentValue(FocusableProperty, false);
       }
 
-      
+
       public SparkMenu(IEnumerable<SparkCommand> commands, IBus bus) : this() =>
-         ItemsSource = commands.Select(c => new SparkCommandVM(c, bus)).ToList();
+         // ObservableCollection to avoid WPF Binding Leak
+         ItemsSource = new ObservableCollection<SparkCommandVM>(commands.Select(c => new SparkCommandVM(c, bus)));
 
       public bool ShowText {
          get { return (bool)GetValue(ShowTextProperty); }

src/client/spark/view/Shell/FloatingWindow.xaml.cs

@@ -154,6 +154,7 @@ namespace mbts.ui.shell
          foreach (TabDocument document in _tabs.Items.OfType<TabDocument>().ToList()) {
             if (document != null && document.ViewModel.CloseCommand.CanExecute(this)) {
                document.ViewModel.CloseCommand.Execute(this);
+               document.ViewModel.Closing -= TabClosing;
             }
          }
       }

src/client/spark/view/Shell/HwndContentHost.cs

@@ -6,6 +6,7 @@ using System.Windows.Interop;
 using System.Runtime.InteropServices;
 using mbts.ui.native;
 using System.Windows;
+using System.Windows.Threading;
 
 namespace mbts.ui.shell
 {
@@ -82,7 +83,14 @@ namespace mbts.ui.shell
          return _hwnd;
       }
 
-      protected override void DestroyWindowCore(HandleRef hwnd) { }
+      protected override void DestroyWindowCore(HandleRef hwnd) {
+         // Improve GC performance by keeping windows out of the Finalizer queue
+         if (hwnd.Wrapper is WPFPanel p) {
+            p.BeginInvoke((Action)(() => {
+               p.Dispose();
+            }));
+         }
+      }
 
       [DllImport("user32.dll", SetLastError=true)]
       static extern IntPtr SetParent(HandleRef hwndChild, HandleRef hwndParent);

src/client/spark/view/Shell/ViewModels/ShellThreadFactory.cs

@@ -10,6 +10,7 @@ using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using System.Windows.Threading;
+using System.Windows.Interop;
 
 namespace mbts.ui.shell
 {
@@ -19,9 +20,8 @@ namespace mbts.ui.shell
       class UIThreadData : IDisposable {
          public Dispatcher dispatcher;
          public Thread thread;
-         public List<ISlowWindow> windows = new List<ISlowWindow>();
-         public System.Windows.Interop.MSG last_msg;
-         public void Dispose() { lock(windows) Disposable.dispose(windows); }
+         public MSG last_msg;
+         public void Dispose() {}
       }
       int _count = 0;
       ConcurrentDictionary<string, UIThreadData> _shared_threads = new ConcurrentDictionary<string, UIThreadData>();
@@ -38,7 +38,7 @@ namespace mbts.ui.shell
          uid.thread = new Thread(() => {
             uid.dispatcher = Dispatcher.CurrentDispatcher;
             uid.dispatcher.UnhandledException += BTSApplicationBootstrapper.CurrentDispatcher_HandleClipboardError;
-            System.Windows.Interop.ComponentDispatcher.ThreadFilterMessage += (ref System.Windows.Interop.MSG m,ref bool h) => ComponentDispatcher_ThreadFilterMessage(uid, ref m, ref h);
+            ComponentDispatcher.ThreadFilterMessage += (ref MSG m,ref bool h) => ComponentDispatcher_ThreadFilterMessage(uid, ref m, ref h);
             try {
                Dispatcher.Run();
             }
@@ -62,7 +62,7 @@ namespace mbts.ui.shell
          return uid;
       }
 
-      private void ComponentDispatcher_ThreadFilterMessage(UIThreadData d, ref System.Windows.Interop.MSG msg, ref bool handled) {
+      private void ComponentDispatcher_ThreadFilterMessage(UIThreadData d, ref MSG msg, ref bool handled) {
          d.last_msg = msg;
 
          // We definitely don't need to reload themes on background threads
@@ -75,7 +75,6 @@ namespace mbts.ui.shell
 
       private Dispatcher attach_window(UIThreadData uid, ISlowWindow window) {
          uid.dispatcher.BeginInvoke(DispatcherPriority.Send, (Action)(() => {
-            lock(uid.windows) uid.windows.Add(window);
             window.thread_init();
          }));
          return uid.dispatcher;

src/client/spark/view/Shell/ViewModels/ShellWindowVM.cs

@@ -29,8 +29,8 @@ namespace mbts.ui.shell
       public ShellWindowVM(IShellWindow model, IWindowManager wm, IShellThreadFactory threads, string host = "") {
          Log.assert(model != null);
          _model = model;
-         _model.TitleChanged += () => { if (!HasCustomTitle) RaisePropertyChanged("Title"); };
-         _model.IconChanged += () => { _icon = null; RaisePropertyChanged("Icon"); };
+         _model.TitleChanged += title_changed;
+         _model.IconChanged += icon_changed;
          _wm = wm;
          host_id = host;
 
@@ -46,6 +46,16 @@ namespace mbts.ui.shell
          _edit_mode = backing(nameof(EditMode), false);
       }
 
+      void title_changed() {
+         if (!HasCustomTitle) {
+            RaisePropertyChanged("Title");
+         }
+      }
+      void icon_changed() {
+         _icon = null;
+         RaisePropertyChanged("Icon");
+      }
+
       public string host_id { get; set; }
       public Dispatcher Dispatcher { get { return _dispatcher; } }
 
@@ -208,6 +218,9 @@ namespace mbts.ui.shell
 
 
       public void Dispose() {
+         _model.TitleChanged -= title_changed;
+         _model.IconChanged -= icon_changed;
+
          if (IsShared) {
             // Shared threads continue running
             _dispatcher.Invoke(() => Disposable.dispose(ref _model));
@@ -221,6 +234,7 @@ namespace mbts.ui.shell
             Disposable.dispose(ref _model);
          }
 
+         _ui = null;
          Disposable.dispose(ref _hwnd_host);
       }
 

src/client/spark/view/Shell/WindowManager.cs

@@ -31,11 +31,13 @@ namespace mbts.ui.shell {
       readonly Lazy<IBookmarks> _bookmarks;
       IDisposable _dispose;
       object _lock = new object();
+      readonly IBus _bus;
 
       [ImportingConstructor]
       public WindowManager(
          IShellThreadFactory threads, Lazy<ISettingsCollector> settings, 
          ISavedWindowFactory factory, Lazy<IBookmarks> bookmarks, IBus bus) {
+         _bus = bus;
          _threads = threads;
          _window_settings = settings;
          _window_factory = factory;
@@ -92,11 +94,15 @@ namespace mbts.ui.shell {
       public IReadOnlyList<TabDocument> list_documents() => _documents.Values.ToList();
 
       public async Task add(IShellWindow window) {
-         if (!Application.Current.Dispatcher.CheckAccess())
-            await Application.Current.Dispatcher.InvokeAsync(() => add(window));
-         Log.Ui.info($"Opening window {window.diagnostic_name()}.");
-         lock (_lock) {
-            _documents[window] = _create_document(window);
+         if (window != null) {
+            if (!Application.Current.Dispatcher.CheckAccess())
+               await Application.Current.Dispatcher.InvokeAsync(() => add(window));
+            Log.Ui.info($"Opening window {window.diagnostic_name()}.");
+            lock (_lock) {
+               _documents[window] = _create_document(window);
+            }
+            // Prevent the bus from holding on to a reference to the window
+            _bus.post(SparkTags.ShowWindow, null);
          }
       }
 

src/client/spark/view/Shell/Workspaces/SettingsManagerTool.cs

@@ -0,0 +1,37 @@
+using bts.api.core;
+using bts.api.persistence;
+using bts.api.user;
+using bts.test;
+using mbts.persist;
+using mbts.ui.shell;
+using System;
+using System.Linq;
+
+namespace bts.settings {
+   public class SettingsManagerTool {
+      public static void run() {
+         var sc = new ServiceContext();
+         var users = sc.locate<UserService>();
+         var persist = sc.locate<PersistenceService>("settings");
+         var user_list = users.get_users(new GetUsersRequest());
+         var user_names = user_list.Users.Select(u => u.Login);
+         var from = ConsoleEx.prompt("From User", user_names);
+         var from_persist = new UserUISettings(user_list.Users.First(u => u.Login == from).Id, persist, null);
+
+         var workspaces = from_persist.query<WorkspaceDefinition>(WorkspacesModel.WORKSPACE_NAMESPACE, ".*").Result;
+         var selected_ws = ConsoleEx.prompt("Choose workspace:", workspaces.Select(w => w.key));
+
+         var to = ConsoleEx.prompt("To User", user_names);
+
+         var name = ConsoleEx.input_prompt("Rename workspace?", selected_ws);
+
+         var confirm = ConsoleEx.prompt($"copy workspace '{selected_ws}' from {from} to {to} and name it '{name}'. Confirm?", "Yes", "No");
+         if (confirm == "Yes") {
+            var ws_blob = workspaces.First(w => w.key == selected_ws).value;
+            var to_persist = new UserUISettings(user_list.Users.First(u => u.Login == to).Id, persist, null);
+            to_persist.set(WorkspacesModel.WORKSPACE_NAMESPACE, name, ws_blob).Wait();
+            Console.WriteLine("Done");
+         }
+      }
+   }
+}

src/client/spark/view/Shell/Workspaces/WorkspacesModel.cs

@@ -222,16 +222,15 @@ namespace mbts.ui.shell
             }
          }
          if (def.windows != null) {
-            foreach (var window in def.windows) {
-               Log.Ui.info("Loading window '{0}'".fmt(window.title));
-               var new_shell_window = await _window_restore.restore(window);
-               // user can close the launcher from the taskbar while a workspace is loading
-               if (_closing) {
-                  return;
-               } else if (new_shell_window != null) {
-                  await _wm.add(new_shell_window);
+            async Task restore(WindowState window) {
+               var w = await _window_restore.restore(window);
+               if (!_closing) {
+                  await _wm.add(w);
                }
             }
+            var loading = def.windows.Select(restore);
+            await Task.WhenAll(loading);
+            if (_closing) return;
          }
          sw.Stop();
          Log.Ui.info($"Time to load workspace '{name}': {sw.Elapsed}");

src/client/spark/view/Spark/Ladder/LadderGridHost.cs

@@ -73,6 +73,10 @@ namespace mbts.spark {
          };
          RecenterCommand = new DelegateCommand(_scroll_to_center);
          DataContextChanged += (s, e) => _connect_vm();
+         Unloaded += (s, e) => {
+            Disposable.dispose(ref _settings_sub);
+            Disposable.dispose(ref _tick_sub);
+         };
       }
 
       enum ColumnType { Order, Bid, Price, Ask, Last, None }

src/client/spark/view/Spark/SparkWindowView.xaml.cs

@@ -16,6 +16,7 @@ using System.Windows.Media.Imaging;
 using System.Windows.Navigation;
 using System.Windows.Shapes;
 using mbts.ui;
+using System.Collections.ObjectModel;
 
 namespace bts.view {
    public partial class SparkWindowView : UserControl {
@@ -36,7 +37,8 @@ namespace bts.view {
             _menu.ShowText = false;
             _grid.Children.Add(_menu);
             if (ContextMenu != null) {
-               ContextMenu.ItemsSource = commands.Select(c => new SparkCommandVM(c, bus));
+               // ObservableCollection to avoid WPF Binding Leak
+               ContextMenu.ItemsSource = new ObservableCollection<SparkCommandVM>(commands.Select(c => new SparkCommandVM(c, bus)));
             }
          } else {
             expand_button.Visibility = Visibility.Collapsed;

src/client/spark_api/api/Update.cs

@@ -129,6 +129,7 @@ namespace bts.api.core {
       public IEnumerable<T> removed { get; }
 
       public TKey key(T t) { return _key(t); }
+      internal Func<T, TKey> key_extractor => _key;
 
 #if DEBUG
       public override string ToString() {
@@ -347,5 +348,72 @@ namespace bts.api.core {
          return source.Select(t => new LegacyTableUpdate<T, TKey>(t));
       }
 
+      public static IObservable<Update<T, TKey>> Filter<T, TKey>(this IObservable<Update<T, TKey>> source, IObservable<Func<T, bool>> predicate) where TKey : IEquatable<TKey>
+      {
+         return Observable.Create<Update<T, TKey>>(observer =>
+         {
+            Update<T, TKey> unfiltered_cache = null, filtered_cache = null;
+            Func<T, bool> pred_cache = null;
+            Log.DataFlow.diag($"Create Filter {typeof(T).Name}");
+
+            return source.CombineLatest(predicate, (upd, pred) => (upd, pred)).Subscribe( tup =>
+            {
+               bool pred_changed = !object.ReferenceEquals(tup.pred, pred_cache) && (pred_cache != null);
+               pred_cache = tup.pred;
+               unfiltered_cache = tup.upd;
+               if (pred_changed && filtered_cache != null)
+               {
+                  //clear out existing
+                  var clear = new Update<T, TKey>(
+                     (IReadOnlyCollection<T>)Array.Empty<T>(), 
+                     Enumerable.Empty<T>(),
+                     Enumerable.Empty<Change<T>>(),
+                     filtered_cache.all, unfiltered_cache.key_extractor);
+                  observer.OnNext(clear);
+
+                  var refilter = unfiltered_cache.all.Where(tup.pred).ToList();
+                  var readd = new Update<T, TKey>(
+                     refilter, refilter, 
+                     Enumerable.Empty<Change<T>>(),
+                     Enumerable.Empty<T>(), unfiltered_cache.key_extractor);
+                  observer.OnNext(readd);
+                  filtered_cache = readd;
+               }
+               else
+               {
+                  IReadOnlyCollection<T> all; IEnumerable<T> added, removed; IEnumerable<Change<T>> changed;
+                  apply_predicate(tup.pred, tup.upd, out all, out added, out changed, out removed);
+                  var next = new Update<T, TKey>(all, added, changed, removed, tup.upd.key_extractor);
+                  observer.OnNext(next);
+                  filtered_cache = next;
+               }
+            }, 
+            () => observer.OnCompleted());
+         });
+      }
+
+      private static void apply_predicate<T, TKey>(Func<T, bool> predicate, Update<T, TKey> cache, 
+                                                   out IReadOnlyCollection<T> all, out IEnumerable<T> added, 
+                                                   out IEnumerable<Change<T>> changed, out IEnumerable<T> removed) where TKey : IEquatable<TKey>
+      {                 
+         var adds = new List<T>();
+         adds.AddRange(cache.added.Where(predicate));
+         var rems = new List<T>();
+         rems.AddRange(cache.removed.Where(predicate));
+         var diffs = new List<Change<T>>();
+         foreach (var change in cache.changes)
+         {
+            if(!predicate(change.Old) && predicate(change.New))
+               adds.Add(change.New);
+            else if(predicate(change.Old) && !predicate(change.New))
+               rems.Add(change.Old);
+            else if(predicate(change.Old) && predicate(change.New))
+               diffs.Add(change);
+         }
+         all = cache.all.Where(predicate).ToList();
+         added = adds;
+         changed = diffs;
+         removed = rems;
+      }
    }
 }

src/client/spark_core/Co2/Co2NativeMethods.cs

@@ -113,5 +113,7 @@ namespace mbts.co2
       [DllImport("libspark")]
       internal static extern void co2_get_zero_and_nan_book(ref NativePriceLevel nps);
 
+      [DllImport("libspark")]
+      internal static extern void co2_get_sample_datetime(ref utils.LLDateTime now, ref utils.LLDateTime utc_now);
    }
 }

src/client/spark_core/Collections/Extensions/ListExtensions.cs

@@ -0,0 +1,40 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace bts.collections {
+   public static class ListExtensions {
+      public static void RemoveWhere<T>(this IList<T> list, Func<T, bool> where) { foreach (var r in list.Where(where).ToList()) list.Remove(r); }
+
+      public static void RemoveWhere<T>(this IList<T> list, Func<T, bool> where, Action<T> on_remove) {
+         int count = list.Count;
+         for (int i = 0; i < count; i++) {
+            var t = list[i];
+            if (where(t)) {
+               list.RemoveAt(i--);
+               --count;
+               on_remove(t);
+            }
+         }
+      }
+
+      private static Random rng = new Random();
+
+      public static void Shuffle<T>(this IList<T> list) {
+         int n = list.Count;
+         while (n > 1) {
+            n--;
+            int k = rng.Next(n + 1);
+            T value = list[k];
+            list[k] = list[n];
+            list[n] = value;
+         }
+      }
+
+      public static void RemoveAt<T>(this IList<T> list, int index, int count) {
+         for (int i = 0; i < count; i++)
+            list.RemoveAt(index);
+      }
+   }
+}

src/client/spark_core/Collections/SortedTree.cs

@@ -27,7 +27,7 @@ namespace bts.collections {
 
       // Insert (return: true) or overwrite (return false) a new value
       // NOTE: you should use AddAll if you are adding multiple values
-      bool Insert(T item) {
+      public bool Insert(T item) {
          var item_id = id(item);
          if (!_index.TryGetValue(item_id, out Node itemnode)) {
             // Item is new
@@ -57,7 +57,7 @@ namespace bts.collections {
             }
          }
       }
-      void Remove(T item) {
+      public void Remove(T item) {
          var item_id = id(item);
          if (_index.TryGetValue(item_id, out Node itemnode)) {
             if (_index.TryGetValue(parent(item), out Node parentnode)) {
@@ -75,7 +75,7 @@ namespace bts.collections {
       }
 
       // Note: This method does not handle replacements correctly. 
-      void AddAll(IEnumerable<T> items) {
+      public void AddAll(IEnumerable<T> items) {
          // When adding multiple items we build the index first to avoid the children before parents problem
          foreach (var item in items) {
             //_index[id(item)] = new Node(item, _comparer);
@@ -103,6 +103,27 @@ namespace bts.collections {
             }
          }
       }
+
+      Node parentmost(T item) {
+         if (_index.TryGetValue(parent(item), out Node pnode)) {
+            return parentmost(pnode.value);
+         }
+         else if (_index.TryGetValue(id(item), out Node mynode)) {
+            return mynode;
+         }
+         return null;
+      }
+
+
+      public IEnumerable<T> EnumerateGroup(T item) {
+         var pnode = parentmost(item);
+         if (pnode != null) {
+            yield return pnode.value;
+            foreach (var child in iterate(pnode.children)) {
+               yield return child;
+            }
+         }
+      }
       public IEnumerator<T> GetEnumerator() {
          return iterate(_tree).GetEnumerator();
       }

src/client/spark_core/TestRunner/ConsoleEx.cs

@@ -1,4 +1,6 @@
 using System;
+using System.Collections.Generic;
+using System.Linq;
 
 namespace bts.test {
    public static class ConsoleEx {
@@ -21,17 +23,34 @@ namespace bts.test {
       }
 
       public static T prompt<T>(string prompt, params T[] options) {
-         Console.WriteLine(prompt);
-         for (int x = 1; x <= options.Length; ++x) {
+         return ConsoleEx.prompt(prompt, (IEnumerable<T>)options);
+      }
+      public static T prompt<T>(string prompt, IEnumerable<T> options) {
+         WriteLine(prompt, ConsoleColor.Yellow);
+         for (int x = 0; x < options.Count(); ++x) {
             Write($"{x}. ", ConsoleColor.Yellow);
-            WriteLine(options[x].ToString());
+            WriteLine(options.ElementAt(x).ToString());
          }
-         int response;
-         do {
-            Int32.TryParse(Console.ReadKey().ToString(), out response);
-            if (response > options.Length) response = 0;
-         } while (response == 0);
-         return options[response];
+         int response = 0;
+         bool success = false;
+         while(!success) {
+            var key = Console.ReadKey();
+            success = Int32.TryParse(key.KeyChar.ToString(), out response);
+            if (response > options.Count()) success = false;
+         }
+         var value = options.ElementAt(response);
+         Console.WriteLine(" >> " + value.ToString());
+         Console.WriteLine();
+         return value;
+      }
+
+      public static string input_prompt(string prompt, string default_result) {
+         Write(prompt + " (", ConsoleColor.Yellow);
+         Write(default_result, ConsoleColor.White);
+         Write(") ", ConsoleColor.Yellow);
+         var result = Console.ReadLine();
+         if (string.IsNullOrWhiteSpace(result)) return default_result;
+         return result;
       }
 
 

src/client/spark_core/TestRunner/InProcessTestRunner.cs

@@ -43,6 +43,9 @@ namespace bts.test {
             if (ue.InnerException is AssertFailture af) {
                result.Message = $"Assertion Failure:\n{af.Message}\nLocation:{af._filename}:{af._line}";
             }
+            else if (ue.InnerException != null) {
+               result.Message = $"Unexpected error: {ue.InnerException}";
+            }
             else {
                result.Message = $"Unexpected error: {ue.Message}\n\n{ue.StackTrace}";
             }

src/client/spark_core/utils/GlobalExtensions.cs

@@ -1,7 +1,6 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
-using System.Runtime.InteropServices;
 using System.Text;
 
 namespace mbts {
@@ -24,20 +23,14 @@ namespace mbts {
          return range.Skip(index).First();
       }
 
-      [DllImport("libspark")]
-      private static extern void ll_to_parts(UInt64 ll_time, out int year, out int month, out int day, out int hour, out int minute, out int second, out int ms);
-
       public static DateTime lldatetime(this ulong ll) {
-         int year, month, day, hour, minute, second, ms;
-         ll_to_parts(ll, out year, out month, out day, out hour, out minute, out second, out ms);
-         if (year > 0) {
-            return new DateTime(year, month, day, hour, minute, second, ms, DateTimeKind.Utc).ToLocalTime();
-         }
-         return DateTime.MinValue;
+         // Warning: Do not remove these consts, or try to simplify this method.
+         // System.DateTime() requires bits 63-64 to be empty in the "DateTime(long ticks)" constructor.
+         // The only way to set the DateTimeKind is through the "DateTime(long ticks, DateTimeKind kind)" constructor.
+         // (Yes, it is annoying. Internally, System.DateTime stores the DateTimeKind in bits 63-64 - in the exact format we already have!!!)
+         const ulong ticks_mask = 0x3FFFFFFFFFFFFFFF;
+         const int kind_shift = 62;
+         return new DateTime((long)(ll & ticks_mask), (DateTimeKind)(ll >> kind_shift));
       }
-
-      public static DateTime lldatetime(this long ll) { return ((ulong)ll).lldatetime(); }
-
    }
-}
-
+}

src/client/spark_core/utils/LLDateTime.cs

@@ -18,6 +18,7 @@ namespace mbts.utils
       }
 
       public static implicit operator DateTime (LLDateTime time) { return time.to_datetime(); }
+      public static implicit operator LLDateTime (DateTime time) { return new LLDateTime((ulong)time.Ticks); }
 
       public static LLDateTime now() {
          return new LLDateTime(BTSNativeMethods.ll_now());

src/client/spark_core/utils/MathUtils.cs

@@ -56,6 +56,11 @@ namespace bts.utils {
          if (value > max) return max;
          return value;
       }
+      public static long within_range(long min, long max, long value) {
+         if (value < min) return min;
+         if (value > max) return max;
+         return value;
+      }
       public static double within_range(double min, double max, double value) {
          if (value < min) return min;
          if (value > max) return max;

src/client/spark_test/Co2/Co2Tests.cs

@@ -112,7 +112,7 @@ namespace mbts.co2.tests
          Assert.IsTrue(last.price == 4.2);
          Assert.IsTrue(last.size == 101);
          Assert.IsTrue(entry.total_volume == 12345);
-         Assert.IsTrue(compare_date(entry.last_trade_time, DateTime.Now));
+         Assert.IsTrue(compare_date(entry.last_trade_time, DateTime.UtcNow));
          //// Settlement
          Assert.AreEqual(entry.settlement_price, 3.3);
          Assert.IsTrue(entry.settlement_date == new DateTime(2015, 9, 3));
@@ -251,6 +251,25 @@ namespace mbts.co2.tests
          Assert.IsTrue(double.IsNaN(npl.bid.price));
       }
 
+      [Test]
+      public void lldatetime_test() {
+         utils.LLDateTime now = DateTime.MinValue, utc_now = DateTime.MinValue;
+         Co2NativeTestMethods.co2_get_sample_datetime(ref now, ref utc_now);
+         DateTime dt_now = now;
+         DateTime dt_utc_now = utc_now;
+         Assert.AreEqual(dt_now.ToShortDateString(), DateTime.Now.ToShortDateString());
+         Assert.AreEqual(dt_utc_now.ToShortDateString(), DateTime.UtcNow.ToShortDateString());
+         void assert_about_equal(int a, int b) {
+            Assert.That(Math.Abs(a - b) <= 1);
+         }
 
+         assert_about_equal(dt_now.Hour, DateTime.Now.Hour);
+         assert_about_equal(dt_now.Minute, DateTime.Now.Minute);
+         assert_about_equal(dt_now.Second, DateTime.Now.Second);
+
+         assert_about_equal(dt_utc_now.Hour, DateTime.UtcNow.Hour);
+         assert_about_equal(dt_utc_now.Minute, DateTime.UtcNow.Minute);
+         assert_about_equal(dt_utc_now.Second, DateTime.UtcNow.Second);
+      }
    }
 }

src/client/spark_test/SortedTree/SortedTreeTests.cs

@@ -1,6 +1,5 @@
 using bts.collections;
-using mbts.collections.extensions;
-using NUnit.Framework;
+using bts.test;
 using System;
 using System.Collections;
 using System.Collections.Generic;
@@ -9,7 +8,7 @@ using System.Text;
 using System.Threading.Tasks;
 
 namespace bts.tests {
-   [TestFixture]
+   [TestGroup]
    class SortedTreeTests {
 
       class Order {
@@ -92,5 +91,30 @@ namespace bts.tests {
          }
       }
 
+      [Test]
+      public void test_enumerate_group() {
+         var comparer = Comparer<Order>.Create((l, r) => l.name.CompareTo(r.name));
+         var input = orders.ToList();
+         input.Shuffle();
+         var tree = new SortedTree<Order, int>(comparer, id, pid, input);
+         Assert.SequenceEqual(tree, orders);
+
+         var children_of_a = tree.EnumerateGroup(orders.First(o => o.name == 'A'));
+         var also_children_of_a = tree.EnumerateGroup(orders.First(o => o.name == 'C'));
+         Assert.AreEqual(3, children_of_a.Count());
+         Assert.AreEqual(3, also_children_of_a.Count());
+         Assert.SequenceEqual(children_of_a, also_children_of_a);
+
+         var children_of_d = tree.EnumerateGroup(orders.First(o => o.name == 'F'));
+         var also_children_of_d = tree.EnumerateGroup(orders.First(o => o.name == 'E'));
+         Assert.AreEqual(children_of_d.First().name, 'D');
+         Assert.AreEqual(also_children_of_d.First().name, 'D');
+         Assert.SequenceEqual(children_of_d, also_children_of_d);
+         Assert.SequenceEqual(children_of_d.Select(o => o.name), new [] { 'D', 'E', 'F' });
+
+         var children_of_g = tree.EnumerateGroup(orders.First(o => o.name == 'G'));
+         Assert.SequenceEqual(children_of_g.Select(o => o.name), new [] { 'G' });
+      }
+
    }
 }

src/client/spark_test/api/ApiUpdateTests.cs

@@ -6,6 +6,7 @@ using System.Reactive.Linq;
 using System.Threading.Tasks;
 using System.Linq;
 using System.Reactive.Concurrency;
+using System.Reactive.Subjects;
 
 namespace mbts.api.Test {
    [TestGroup]
@@ -104,6 +105,121 @@ namespace mbts.api.Test {
 
       }
 
+      void validate_filter(IObservable<RawUpdate<Item>> raw, IObservable<Update<Item, int>> filtered, Func<Item, bool> pred)
+      {
+         var total = raw.ToEnumerable().Count();
+
+         // Validate that update is right at each point
+         for (var count = 1; count < total; ++count)
+         {
+            var orig = raw.Take(count).reconstitute(id).ToEnumerable().Last();            
+            var next = filtered.Take(count).ToEnumerable().Last();
+            Assert.That(next.all.All(pred));
+            Assert.That(next.added.All(pred));
+            Assert.That(next.removed.All(pred));
+            Assert.That(next.changed.All(pred));            
+         }
+
+         // Do it again with TableUpdate
+         for (var count = 1; count < total; ++count)
+         {
+            var orig = raw.Take(count).reconstitute(id).Table().ToEnumerable().Last();
+            var next = filtered.Take(count).ToEnumerable().Last();
+            Assert.That(next.added.All(pred));
+            Assert.That(next.removed.All(pred));
+            Assert.That(next.changed.All(pred));
+         }
+      }
+
+
+      void validate_filter_sub(IObservable<RawUpdate<Item>> raw, IObservable<Update<Item, int>> filtered, BehaviorSubject<Func<Item, bool>> pred)
+      {
+         raw.reconstitute(x => x.id).CombineLatest(filtered, (r, f) => Tuple.Create(r, f)).Subscribe(tup =>
+         {
+            Assert.That(tup.Item2.all.All(pred.Value));
+            Assert.That(tup.Item2.added.All(pred.Value));
+            Assert.That(!tup.Item2.removed.Any(pred.Value)); 
+            Assert.That(tup.Item2.changed.All(pred.Value));
+
+            Dictionary<int, Item> rawd = tup.Item1.added.ToDictionary(x => x.id);
+            Dictionary<int, Item> fd = tup.Item2.added.ToDictionary(x => x.id);
+            foreach (var kv in rawd)
+            {
+               if(pred.Value(kv.Value))
+                  Assert.That(fd.ContainsKey(kv.Key));
+               else 
+                  Assert.That(!fd.ContainsKey(kv.Key));
+            }
+         });
+      }
+
+
+      [Test]
+      public void test_filter()
+      {
+         var raw = Observable.Create<RawUpdate<Item>>(observer =>
+         {
+            RawUpdate<Item> rup = new RawUpdate<Item>();
+            for (int x = 0; x < 50; ++x)
+            {
+               rup.Clear();
+               var item = _item_pool[x];
+               rup.added_items.Add(item);
+               rup.added_keys.Add(item.id);
+               observer.OnNext(rup);
+            }
+            var rand = new Random();
+            for (int r = 0; r < rand.Next(4, 30); r += rand.Next(1, 2))
+            {
+               rup.Clear();
+               rup.removed_keys.Add(_item_pool[r].id);
+               observer.OnNext(rup);
+            }
+            rup.Clear();
+            for (int c = 31; c < rand.Next(32, 50); c += rand.Next(1, 2))
+            {               
+               var ch = new Item(_item_pool[c].id, _item_pool[c].name + "_change");
+               rup.changed_items.Add(ch);
+               rup.changed_keys.Add(ch.id);               
+            }
+            observer.OnNext(rup);
+
+            observer.OnCompleted(); 
+            return Disposable.empty;
+         });
+
+         Func<Item, bool> evens = (x => x.id % 2 == 0);
+         Func<Item, bool> odds = (x => x.id % 2 == 1);
+         var predeven = Observable.Create<Func<Item, bool>>(observer =>
+         {            
+            observer.OnNext(evens);            
+            observer.OnCompleted();
+            return Disposable.empty;
+         });
+
+         var filtered = raw.reconstitute(x=> x.id).Filter(predeven);
+         validate_filter(raw, filtered, evens);
+
+         var predodd = Observable.Create<Func<Item, bool>>(observer =>
+         {
+            observer.OnNext(odds);
+            observer.OnCompleted();
+            return Disposable.empty;
+         });
+
+         filtered = raw.reconstitute(x => x.id).Filter(predodd);
+         validate_filter(raw, filtered, odds);
+
+
+
+         var sub= new BehaviorSubject<Func<Item, bool>>(evens);
+         filtered = raw.reconstitute(x => x.id).Filter(sub);
+         validate_filter_sub(raw, filtered, sub);
+
+         sub.OnNext(odds);
+         sub.OnCompleted();
+      }      
+
       [Test]
       public async Task test_add() {
          // Add 1 item 10x

src/client/sparkcmd/Cmd/SettingsManagerCommand.cs

@@ -0,0 +1,20 @@
+using ManyConsole;
+using System.Reflection;
+
+namespace bts {
+
+   public class SettinsManagerCommand : ConsoleCommand {
+      public SettinsManagerCommand() {
+         IsCommand("SettingsManager", "Copy workspaces around");
+      }
+
+      public override int Run(string[] remainingArguments) {
+         var asm = Assembly.Load("spark");
+         var type = asm.GetType("bts.settings.SettingsManagerTool");
+         var run = type.GetMethod("run", BindingFlags.Static | BindingFlags.Public);
+         run.Invoke(null, new object[0]);
+         return 0;
+      }
+   }
+
+}

src/data_files/CMakeDeps.cmake

@@ -1 +1 @@
-set(sources /home/ppinto/Dropbox/bts/spark/src/data_files/files/Onixs.BTS.lic;/home/ppinto/Dropbox/bts/spark/src/data_files/files/Onixs.BTS.lic.orig;/home/ppinto/Dropbox/bts/spark/src/data_files/files/cert_secure_logon.txt)
+set(sources files/Onixs.BTS.lic;files/cert_secure_logon.txt;files/perf_trace/compiled.css;files/perf_trace/index.html;files/perf_trace/jquery-3.3.1.min.js;files/perf_trace/main.js;files/perf_trace/plotly-latest.min.js)

src/data_files/CMakeLists.txt

@@ -4,7 +4,7 @@ project(data_files)
 include(${CMAKE_SOURCE_DIR}/tools/safe_dep_glob.cmake)
 
 # DEPENDENCIES
-file (GLOB_RECURSE To_Copy ABSOLUTE ${CMAKE_CURRENT_SOURCE_DIR} files/*)
+file (GLOB_RECURSE To_Copy RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} files/*)
 update_deps_file("${To_Copy}")
 # END DEPENDENCIES
 

src/data_files/files/perf_trace/index.html

@@ -0,0 +1,63 @@
+<html>
+
+<head>
+    <title>BTS Spark Socket To Socket Requote Times</title>
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <link rel="stylesheet" href="/perf_trace/compiled.css">
+    <script src="https://cdn.polyfill.io/v2/polyfill.js?features=es6"></script>
+    <style type="text/css">
+        .summary-block {
+            margin: 2em 1em;
+        }
+        .summary-block > ul {
+            margin: 0em 1em;
+            display:inline-block;
+            vertical-align: top;
+        }
+        .histogram-div {
+            width: 550px;
+            height:350px;
+            display:inline-block
+        }
+        .clear {
+            clear:both;
+        }
+    </style>
+</head>
+
+<body>
+    <div>
+        <div class="columns is-fullheight">
+            <div class="column is-narrow is-launcher-sidebar-menu">
+                <aside class="menu">
+                    <ul class="menu-list">
+                        <li>&nbsp;&nbsp;</li>
+                    </ul>
+                </aside>
+            </div>
+            <div class="column is-main-content">
+                <div class="columns is-fullheight">
+                    <!--div class="column is-narrow is-sidebar-menu">
+                    </div-->
+                    <div class="column is-main-content">
+                        <!--div class="tabs is-boxed">
+                            <ul>
+                                <li class="is-active"><a>Stats</a></li>
+                            </ul>
+                        </div-->
+                        <div style="margin:2em 0 0 2em;"><!--div class="is-tab-content" is-tab-content is shorter than the screen to accommodate tabs at the top -->
+                            <h1 class="is-size-2">Socket To Socket Requote Times</h1>
+                            <div id="output"></div>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+    </div>
+    <script src="/perf_trace/jquery-3.3.1.min.js"></script>
+    <script src="/perf_trace/plotly-latest.min.js"></script>
+    <script src="/perf_trace/main.js"></script>
+</body>
+
+</html>

src/data_files/files/perf_trace/jquery-3.3.1.min.js

@@ -0,0 +1,2 @@
+/*! jQuery v3.3.1 | (c) JS Foundation and other contributors | jquery.org/license */
+!function(e,t){"use strict";"object"==typeof module&&"object"==typeof module.exports?module.exports=e.document?t(e,!0):function(e){if(!e.document)throw new Error("jQuery requires a window with a document");return t(e)}:t(e)}("undefined"!=typeof window?window:this,function(e,t){"use strict";var n=[],r=e.document,i=Object.getPrototypeOf,o=n.slice,a=n.concat,s=n.push,u=n.indexOf,l={},c=l.toString,f=l.hasOwnProperty,p=f.toString,d=p.call(Object),h={},g=function e(t){return"function"==typeof t&&"number"!=typeof t.nodeType},y=function e(t){return null!=t&&t===t.window},v={type:!0,src:!0,noModule:!0};function m(e,t,n){var i,o=(t=t||r).createElement("script");if(o.text=e,n)for(i in v)n[i]&&(o[i]=n[i]);t.head.appendChild(o).parentNode.removeChild(o)}function x(e){return null==e?e+"":"object"==typeof e||"function"==typeof e?l[c.call(e)]||"object":typeof e}var b="3.3.1",w=function(e,t){return new w.fn.init(e,t)},T=/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g;w.fn=w.prototype={jquery:"3.3.1",constructor:w,length:0,toArray:function(){return o.call(this)},get:function(e){return null==e?o.call(this):e<0?this[e+this.length]:this[e]},pushStack:function(e){var t=w.merge(this.constructor(),e);return t.prevObject=this,t},each:function(e){return w.each(this,e)},map:function(e){return this.pushStack(w.map(this,function(t,n){return e.call(t,n,t)}))},slice:function(){return this.pushStack(o.apply(this,arguments))},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},eq:function(e){var t=this.length,n=+e+(e<0?t:0);return this.pushStack(n>=0&&n<t?[this[n]]:[])},end:function(){return this.prevObject||this.constructor()},push:s,sort:n.sort,splice:n.splice},w.extend=w.fn.extend=function(){var e,t,n,r,i,o,a=arguments[0]||{},s=1,u=arguments.length,l=!1;for("boolean"==typeof a&&(l=a,a=arguments[s]||{},s++),"object"==typeof a||g(a)||(a={}),s===u&&(a=this,s--);s<u;s++)if(null!=(e=arguments[s]))for(t in e)n=a[t],a!==(r=e[t])&&(l&&r&&(w.isPlainObject(r)||(i=Array.isArray(r)))?(i?(i=!1,o=n&&Array.isArray(n)?n:[]):o=n&&w.isPlainObject(n)?n:{},a[t]=w.extend(l,o,r)):void 0!==r&&(a[t]=r));return a},w.extend({expando:"jQuery"+("3.3.1"+Math.random()).replace(/\D/g,""),isReady:!0,error:function(e){throw new Error(e)},noop:function(){},isPlainObject:function(e){var t,n;return!(!e||"[object Object]"!==c.call(e))&&(!(t=i(e))||"function"==typeof(n=f.call(t,"constructor")&&t.constructor)&&p.call(n)===d)},isEmptyObject:function(e){var t;for(t in e)return!1;return!0},globalEval:function(e){m(e)},each:function(e,t){var n,r=0;if(C(e)){for(n=e.length;r<n;r++)if(!1===t.call(e[r],r,e[r]))break}else for(r in e)if(!1===t.call(e[r],r,e[r]))break;return e},trim:function(e){return null==e?"":(e+"").replace(T,"")},makeArray:function(e,t){var n=t||[];return null!=e&&(C(Object(e))?w.merge(n,"string"==typeof e?[e]:e):s.call(n,e)),n},inArray:function(e,t,n){return null==t?-1:u.call(t,e,n)},merge:function(e,t){for(var n=+t.length,r=0,i=e.length;r<n;r++)e[i++]=t[r];return e.length=i,e},grep:function(e,t,n){for(var r,i=[],o=0,a=e.length,s=!n;o<a;o++)(r=!t(e[o],o))!==s&&i.push(e[o]);return i},map:function(e,t,n){var r,i,o=0,s=[];if(C(e))for(r=e.length;o<r;o++)null!=(i=t(e[o],o,n))&&s.push(i);else for(o in e)null!=(i=t(e[o],o,n))&&s.push(i);return a.apply([],s)},guid:1,support:h}),"function"==typeof Symbol&&(w.fn[Symbol.iterator]=n[Symbol.iterator]),w.each("Boolean Number String Function Array Date RegExp Object Error Symbol".split(" "),function(e,t){l["[object "+t+"]"]=t.toLowerCase()});function C(e){var t=!!e&&"length"in e&&e.length,n=x(e);return!g(e)&&!y(e)&&("array"===n||0===t||"number"==typeof t&&t>0&&t-1 in e)}var E=function(e){var t,n,r,i,o,a,s,u,l,c,f,p,d,h,g,y,v,m,x,b="sizzle"+1*new Date,w=e.document,T=0,C=0,E=ae(),k=ae(),S=ae(),D=function(e,t){return e===t&&(f=!0),0},N={}.hasOwnProperty,A=[],j=A.pop,q=A.push,L=A.push,H=A.slice,O=function(e,t){for(var n=0,r=e.length;n<r;n++)if(e[n]===t)return n;return-1},P="checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|ismap|loop|multiple|open|readonly|required|scoped",M="[\\x20\\t\\r\\n\\f]",R="(?:\\\\.|[\\w-]|[^\0-\\xa0])+",I="\\["+M+"*("+R+")(?:"+M+"*([*^$|!~]?=)"+M+"*(?:'((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\"|("+R+"))|)"+M+"*\\]",W=":("+R+")(?:\\((('((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\")|((?:\\\\.|[^\\\\()[\\]]|"+I+")*)|.*)\\)|)",$=new RegExp(M+"+","g"),B=new RegExp("^"+M+"+|((?:^|[^\\\\])(?:\\\\.)*)"+M+"+$","g"),F=new RegExp("^"+M+"*,"+M+"*"),_=new RegExp("^"+M+"*([>+~]|"+M+")"+M+"*"),z=new RegExp("="+M+"*([^\\]'\"]*?)"+M+"*\\]","g"),X=new RegExp(W),U=new RegExp("^"+R+"$"),V={ID:new RegExp("^#("+R+")"),CLASS:new RegExp("^\\.("+R+")"),TAG:new RegExp("^("+R+"|[*])"),ATTR:new RegExp("^"+I),PSEUDO:new RegExp("^"+W),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+M+"*(even|odd|(([+-]|)(\\d*)n|)"+M+"*(?:([+-]|)"+M+"*(\\d+)|))"+M+"*\\)|)","i"),bool:new RegExp("^(?:"+P+")$","i"),needsContext:new RegExp("^"+M+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)Line truncated

src/data_files/files/perf_trace/main.js

@@ -0,0 +1,164 @@
+$( document ).ready(function (){
+  let urlParams = new URLSearchParams(window.location.search);
+  num_rows = urlParams.get('r');
+  let show_rows = urlParams.get('showrows');
+
+  let trace_data_url = "/perf_trace_data";
+  if(num_rows != null) {
+    trace_data_url += "/" + num_rows;
+  }
+
+    $.ajax({
+      url: trace_data_url,
+      dataType: 'json',
+      context: document.body,
+      error: ajax_err,
+      crossDomain: true
+    }).done(function(result) {
+      if(show_rows != null) {
+        list_rows(show_rows, result);
+      }
+      else {
+        if(result.length > 0){
+          let summary_info = "<h1 class='is-size-3'><b>Host:</b> " + result[0].host + "</h1>";
+          $("#output").html(summary_info);
+          $(result).each(display_session);
+        }
+      }
+
+    });
+
+});
+
+function list_rows(show_rows, data) {
+  $.each(data, function(index, value){
+    if(value.session_name === show_rows){
+      let summary_info = "<table class='table bts-data-table'>";
+      summary_info += "<thead><tr><th>Trace Time</th>" +
+                        "<th>Order Key</th>" +
+                        "<th>MD</th>" +
+                        "<th>Spreader</th>" +
+                        "<th>Fix</th>" +
+                        "<th>Total</th></tr></thead><tbody>";
+
+      $.each(value.rows, function(key, val) {
+        summary_info += "<tr>";
+        console.log(val);
+
+        summary_info += "<td>" + val.trace_time + "</td>" +
+                        "<td>" + val.order_key + "</td>" +
+                        "<td>" + val.md + "</td>" +
+                        "<td>" + val.spreader + "</td>" +
+                        "<td>" + val.fix + "</td>" +
+                        "<td>" + val.total + "</td>";
+
+
+        summary_info += "</tr>";
+      });
+      summary_info += "</tbody></table>";
+      $("#output").html(summary_info);
+      return false;
+    }
+
+  });
+}
+
+
+function ajax_err (jqXHR, textStatus,errorThrown) {
+   $("#output").html("<h3 class='is-size-3 has-text-danger'>Error loading perf_trace_data</h3>" +
+                    "<p>" + textStatus + "</p>" +
+                    "<pre>" + errorThrown + "</pre>"
+                      );
+
+   console.log(textStatus);
+   console.log(errorThrown);
+}
+
+function first_upper (str)
+{
+  let ret_str = str[0].toUpperCase() + str.substring(1, str.length);
+  return ret_str
+}
+
+function generate_histogram(id, data)
+{
+  var trace = {
+      x: data,
+      type: 'histogram',
+      marker: {
+          color: 'rgb(65,105,170)'
+        },
+    };
+
+  var plot_data = [trace];
+
+  Plotly.newPlot(id, plot_data);
+}
+
+function display_session (index, value) {
+   //console.log(value);
+   let session_info = "<ul class=''>";
+   session_info += "<li id='" + value.session_name + "' class='is-size-5'><b>Session:</b> " + value.session_name + "</li>";
+   session_info += "<li><b>Build Type:</b> " + value.build_type + "</li>";
+   session_info += "<li><b>Version:</b> " + value.version + "</li>";
+   if(num_rows != null) {
+      session_info += "<li><a href='?showrows=" + value.session_name + "&r=" + num_rows + "'><b>Rows:</b> " + value.n_rows + "</a></li>";
+   }
+   else {
+      session_info += "<li><a href='?showrows=" + value.session_name + "'><b>Rows:</b> " + value.n_rows + "</a></li>";
+   }
+
+   $(value.summaries).each(function(i,v){
+      if(v.value === "total") {
+         session_info += "<li class='summary-block'><ul>";
+         session_info += "<li><b>" + first_upper(v.value) + "</b></li>";
+         session_info += "<li>Min: " + v.min + "</li>";
+         session_info += "<li>Median: " + v.median + "</li>";
+         session_info += "<li>Max: " + v.max + "</li>";
+         session_info += "<li>99%: " + v.ninenty_nine + "</li>";
+         session_info += "<li>Standard Deviation: " + v.stdev + "</li>";
+         session_info += "</ul>";
+         session_info += "<div id='" + value.session_name + "-" + v.value + "' class='histogram-div'></div>"
+         session_info += "</li>";
+         return false
+      }
+   });
+
+   $(value.summaries).each(function(i,v){
+      if(v.value !== "total") {
+         session_info += "<li class='summary-block'><ul>";
+         session_info += "<li><b>" + first_upper(v.value) + "</b></li>";
+         session_info += "<li>Min: " + v.min + "</li>";
+         session_info += "<li>Median: " + v.median + "</li>";
+         session_info += "<li>Max: " + v.max + "</li>";
+         session_info += "<li>99%: " + v.ninenty_nine + "</li>";
+         session_info += "<li>Standard Deviation: " + v.stdev + "</li>";
+         session_info += "</ul>";
+         session_info += "<div id='" + value.session_name + "-" + v.value + "' class='histogram-div'></div>"
+         session_info += "</li>";
+      }
+   });
+
+   session_info += "</ul><hr class='clear'>";
+   $("#output").append(session_info);
+
+   let all_data = {};
+
+   $(value.rows).each(function(i, row) {
+      $.each(row, function(name, v) {
+          if(all_data[name] === undefined){
+            all_data[name] = [v];
+          }
+          else {
+            all_data[name].push(v);
+          }
+      });
+   });
+
+   //console.log(all_data);
+
+   $(value.summaries).each(function(i,v){
+      generate_histogram(value.session_name + "-" + v.value, all_data[v.value]);
+   });
+
+}

src/data_files/files/perf_trace/plotly-latest.min.js

@@ -0,0 +1,7 @@
+/**
+* plotly.js v1.41.3
+* Copyright 2012-2018, Plotly, Inc.
+* All rights reserved.
+* Licensed under the MIT license
+*/
+!function(t){if("object"==typeof exports&&"undefined"!=typeof module)module.exports=t();else if("function"==typeof define&&define.amd)define([],t);else{("undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof self?self:this).Plotly=t()}}(function(){return function(){return function t(e,r,n){function i(o,s){if(!r[o]){if(!e[o]){var l="function"==typeof require&&require;if(!s&&l)return l(o,!0);if(a)return a(o,!0);var c=new Error("Cannot find module '"+o+"'");throw c.code="MODULE_NOT_FOUND",c}var u=r[o]={exports:{}};e[o][0].call(u.exports,function(t){return i(e[o][1][t]||t)},u,u.exports,t,e,r,n)}return r[o].exports}for(var a="function"==typeof require&&require,o=0;o<n.length;o++)i(n[o]);return i}}()({1:[function(t,e,r){"use strict";var n=t("../src/lib"),i={"X,X div":"direction:ltr;font-family:'Open Sans', verdana, arial, sans-serif;margin:0;padding:0;","X input,X button":"font-family:'Open Sans', verdana, arial, sans-serif;","X input:focus,X button:focus":"outline:none;","X a":"text-decoration:none;","X a:hover":"text-decoration:none;","X .crisp":"shape-rendering:crispEdges;","X .user-select-none":"-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;-o-user-select:none;user-select:none;","X svg":"overflow:hidden;","X svg a":"fill:#447adb;","X svg a:hover":"fill:#3c6dc5;","X .main-svg":"position:absolute;top:0;left:0;pointer-events:none;","X .main-svg .draglayer":"pointer-events:all;","X .cursor-default":"cursor:default;","X .cursor-pointer":"cursor:pointer;","X .cursor-crosshair":"cursor:crosshair;","X .cursor-move":"cursor:move;","X .cursor-col-resize":"cursor:col-resize;","X .cursor-row-resize":"cursor:row-resize;","X .cursor-ns-resize":"cursor:ns-resize;","X .cursor-ew-resize":"cursor:ew-resize;","X .cursor-sw-resize":"cursor:sw-resize;","X .cursor-s-resize":"cursor:s-resize;","X .cursor-se-resize":"cursor:se-resize;","X .cursor-w-resize":"cursor:w-resize;","X .cursor-e-resize":"cursor:e-resize;","X .cursor-nw-resize":"cursor:nw-resize;","X .cursor-n-resize":"cursor:n-resize;","X .cursor-ne-resize":"cursor:ne-resize;","X .cursor-grab":"cursor:-webkit-grab;cursor:grab;","X .modebar":"position:absolute;top:2px;right:2px;z-index:1001;background:rgba(255,255,255,0.7);","X .modebar--hover":"opacity:0;-webkit-transition:opacity 0.3s ease 0s;-moz-transition:opacity 0.3s ease 0s;-ms-transition:opacity 0.3s ease 0s;-o-transition:opacity 0.3s ease 0s;transition:opacity 0.3s ease 0s;","X:hover .modebar--hover":"opacity:1;","X .modebar-group":"float:left;display:inline-block;box-sizing:border-box;margin-left:8px;position:relative;vertical-align:middle;white-space:nowrap;","X .modebar-group:first-child":"margin-left:0px;","X .modebar-btn":"position:relative;font-size:16px;padding:3px 4px;cursor:pointer;line-height:normal;box-sizing:border-box;","X .modebar-btn svg":"position:relative;top:2px;","X .modebar-btn path":"fill:rgba(0,31,95,0.3);","X .modebar-btn.active path,X .modebar-btn:hover path":"fill:rgba(0,22,72,0.5);","X .modebar-btn.modebar-btn--logo":"padding:3px 1px;","X .modebar-btn.modebar-btn--logo path":"fill:#447adb !important;","X [data-title]:before,X [data-title]:after":"position:absolute;-webkit-transform:translate3d(0, 0, 0);-moz-transform:translate3d(0, 0, 0);-ms-transform:translate3d(0, 0, 0);-o-transform:translate3d(0, 0, 0);transform:translate3d(0, 0, 0);display:none;opacity:0;z-index:1001;pointer-events:none;top:110%;right:50%;","X [data-title]:hover:before,X [data-title]:hover:after":"display:block;opacity:1;","X [data-title]:before":"content:'';position:absolute;background:transparent;border:6px solid transparent;z-index:1002;margin-top:-12px;border-bottom-color:#69738a;margin-right:-6px;","X [data-title]:after":"content:attr(data-title);background:#69738a;color:white;padding:8px 10px;font-size:12px;line-height:12px;white-space:nowrap;margin-right:-18px;border-radius:2px;","X .select-outline":"fill:none;stroke-width:1;shape-rendering:crispEdges;","X .select-outline-1":"stroke:white;","X .select-outline-2":"stroke:black;stroke-dasharray:2px 2px;",Y:"font-family:'Open Sans';position:fixed;top:50px;right:20px;z-index:10000;font-size:10pt;max-width:180px;","Y p":"margin:0;","Y .notifier-note":"min-width:180px;max-width:250px;border:1px solid #fff;z-index:3000;margin:0;background-color:#8c97af;background-color:rgba(140,151,175,0.9);color:#fff;padding:10px;overflow-wrap:break-word;word-wrap:break-word;-ms-hyphens:auto;-webkit-hyphens:auto;hyphens:auto;","Y .notifier-close":"color:#fff;opacity:0.8;float:right;padding:0 5px;background:none;border:none;font-size:20px;font-weight:bold;line-height:20px;","Y .notifier-close:hover":"color:#444;text-decoration:none;cursor:pointer;"};for(var a in i){var o=a.replace(/^,/," ,").replace(/X/g,".js-plotly-plot .plotly").replace(/Y/g,".plotly-notifier");n.addStyleRule(o,i[a])}},{"../src/lib":693}],2:[function(t,e,r){"use strict";e.exports={undo:{width:857.1,height:1e3,path:"m857 350q0-87-34-166t-91-137-137-92-166-34q-96 0Line truncated

src/lib/CMakeDeps.cmake

@@ -1 +1 @@
-set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/services/UserService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/Executor.conf;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cpp_config.conf;config/defaults/dynamic_orders.conf;config/defaults/edge.conf;config/defaults/expiration_display_offset.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cpp_config.proto;config/protos/dynamic_orders.proto;config/protos/environment.proto;config/protos/expiration_display_offset.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/orders_fills_cleanup.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPriceLantencyTester.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;mercury/actors/Actor.cpp;mercury/broker/AsyncEventBroker.cpp;mercury/broker/CoreEventBroker.cpp;mercury/broker/IEventBroker.cpp;mercury/events/TimerEvent.cpp;mercury/mercury_utils/Executor.cpp;mercury/mercury_utils/IPollableActor.cpp;mercury/mercury_utils/mercury_common.cpp;mercury/synopsis/SynopsisBuilder.cpp;mercury/topics/ITopic.cpp;metrics/metrics.cpp;metrics/system_metrics.cpp;orderman/actors/AsyncFactory.cpp;orderman/actors/ExecutionGateway.cpp;orderman/actors/FillStore.cpp;orderman/actors/OrderStore.cpp;orderman/actors/SODPosition.cpp;orderman/actors/SecuritySrc.cpp;orderman/actors/ServiceBridge.cpp;orderman/actors/order_manager/OrderBuilder.cpp;orderman/actors/order_manager/OrderManager.cpp;orderman/actors/order_manager/OrderServicesData.cpp;orderman/events/CancelRejectEvent.cpp;orderman/events/NewExecutionReportEvent.cpp;orderman/limiter/LimitStore.cpp;orderman/limiter/Limiter.cpp;orderman/limiter/ProductLimitNode.cpp;orderman/limiter/ProductLimitTree.cpp;orderman/limiter/SecurityLimitNode.cpp;orderman/limiter/SecurityLimitTree.cpp;orderman/limiter/tests/limit_tests.cpp;orderman/order_factory/OrLine truncated
+set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/services/UserService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/Executor.conf;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cpp_config.conf;config/defaults/dynamic_orders.conf;config/defaults/edge.conf;config/defaults/expiration_display_offset.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cpp_config.proto;config/protos/dynamic_orders.proto;config/protos/expiration_display_offset.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPriceLantencyTester.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;mercury/actors/IPollableActor.cpp;mercury/broker/Broker.cpp;mercury/broker/BrokerProxy.cpp;mercury/broker/ILocalBroker.cpp;mercury/broker/IRemoteBroker.cpp;mercury/common/mercury_common.cpp;mercury/events/TimerEvent.cpp;mercury/executor/Executor.cpp;mercury/topics/Topic.cpp;metrics/metrics.cpp;metrics/system_metrics.cpp;orderman/actors/AsyncFactory.cpp;orderman/actors/ExecutionGateway.cpp;orderman/actors/FillStore.cpp;orderman/actors/OrderStore.cpp;orderman/actors/SODPosition.cpp;orderman/actors/SecuritySrc.cpp;orderman/actors/ServiceBridge.cpp;orderman/actors/order_manager/OrderBuilder.cpp;orderman/actors/order_manager/OrderManager.cpp;orderman/actors/order_manager/OrderServicesData.cpp;orderman/events/CancelRejectEvent.cpp;orderman/events/NewExecutionReportEvent.cpp;orderman/limiter/LimitStore.cpp;orderman/limiter/Limiter.cpp;orderman/limiter/ProductLimitNode.cpp;orderman/limiter/ProductLimitTree.cpp;orderman/limiter/SecurityLimitNode.cpp;orderman/limiter/SecurityLimitTree.cpp;ordLine truncated

src/lib/CMakeLists.txt

@@ -103,6 +103,7 @@ else ()
     boost_chrono
     boost_date_time
     boost_atomic
+    SQLiteCpp
     sqlite3
     curl
     uuid

src/lib/api/services/OrderService.proto

@@ -109,6 +109,15 @@ message TickOrderDefinition {
 message LeanOrderDefinition {
   spark.QuantityMeasure cross_threshold = 1;
   uint32 max_chase_ticks = 2;
+  /*deprecated*/ bool detach_on_toward = 4;
+  enum OnToward
+  {
+     NO_CHANGE = 0; //continue to lean on initial inside market
+     CANCEL = 1; //cancel algo and order
+     DETACH = 2; //cancel algo but leave order working
+     REPRICE = 3; //lean on new inside market
+  }
+  OnToward trade_toward_behavior = 5;
   CustomOrderDefinition child = 3;
 }
 message JoinOrderDefinition {

src/lib/config/ConfigPropertyEvaluator.cpp

@@ -234,7 +234,7 @@ bool ConfigPropertyEvaluator::get_value(const std::string& property, Value* out_
    tokenizer<char_separator<char> >::iterator iter = tokens.begin (), end = tokens.end ();
 
    // The first token is the key name
-   const proto::Message* key = _config.try_get_message(*iter);
+   const proto::Message* key = _config.try_get_message_or_default(*iter);
    if (!key) return false;
 
    // 2) Drill down using reflection to find the named property

src/lib/config/ConfigurationState.cpp

@@ -616,6 +616,19 @@ const proto::Message* ConfigurationState::try_get_message(const std::string& pro
    return nullptr;
 }
 
+const proto::Message* ConfigurationState::try_get_message_or_default(const std::string& property_name) {
+   const auto pmsg = try_get_message(property_name);
+   if (pmsg) {
+      return pmsg;
+   }
+   // If we can't find a value we should return a default instance
+   const auto* descriptor = get_descriptor_by_name(property_name);
+   if (descriptor) {
+      return proto::MessageFactory::generated_factory()->GetPrototype(descriptor);
+   }
+   return nullptr;
+}
+
 const proto::Message& ConfigurationState::get_message(const proto::Descriptor* property_desc)
 {
    if (!is_valid_key_descriptor(property_desc))

src/lib/config/ConfigurationState.h

@@ -59,6 +59,7 @@ namespace internal {
 
       const proto::Message& get_message(const proto::Descriptor* property_desc);
       const proto::Message* try_get_message(const std::string& property_name);
+      const proto::Message* try_get_message_or_default(const std::string& property_name);
       
       bool is_initialized() { return _is_initialized; }
 

src/lib/config/config.cpp

@@ -76,16 +76,7 @@ extern "C" BTS_EXPORT const char* get_buffer(const char* key_name, int& out_leng
 
 extern "C" BTS_EXPORT const proto::Message* get_named(const char* key_name)
 {
-   const auto* pmessage = config_state.try_get_message(key_name);
-   if (pmessage) {
-      return pmessage;
-   }
-   // If we can't find a value we should return a default instance
-   const auto* descriptor = config_state.get_descriptor_by_name(key_name);
-   if (descriptor) {
-      return proto::MessageFactory::generated_factory()->GetPrototype(descriptor);
-   }
-   return nullptr;
+   return config_state.try_get_message_or_default(key_name);
 }
 
 BTS_EXPORT std::string get_macro_string(const char* name) { return config_state.get_macro_string(name); }

src/lib/config/defaults/Symbols.conf

@@ -26,6 +26,26 @@ SymbolList {
     }
 }
 
+Symbols {
+    rule: "DeploymentType.type == 'DEMO' and System.platform == 'windows'"
+    value {
+        sim {
+            future: @SymbolList.symbol
+            spread: @SymbolList.symbol
+        }
+    }
+}
+
+Symbols {
+    rule: "DeploymentType.type == 'PROD' and System.platform == 'windows'"
+    value {
+        glbx {
+            future: @SymbolList.symbol
+            spread: @SymbolList.symbol
+        }
+    }
+}
+
 Symbols {
     value {
         glbx {

src/lib/config/defaults/logging.conf

@@ -8,6 +8,17 @@ Logging {
    }
 }
 
+Logging {
+    rule: "System.platform == 'windows'"
+    value {
+      console { default_level: "Error" }
+      syslog  { default_level: "Info" }
+      file  { default_level: "Info" }
+      initial_queue_size: 1024
+      max_queue_growth: 100
+    }
+}
+
 Logging
 {
    value

src/lib/config/defaults/orderman_cfg.conf

@@ -0,0 +1,34 @@
+OrdermanCfg {
+   rule: "DeploymentType.type == 'dev'"
+   value {
+      executor {
+         thread_name: "oman_orders"
+	 poll: false 
+	 hosted_actor: "MDSource"
+	 hosted_actor: "OrderManager"
+	 hosted_actor: "ExecutionGateway"
+      }
+      executor {
+         thread_name: "oman_slow"
+	 poll: false 
+	 hosted_actor: "*"
+      }
+   }
+}
+
+OrdermanCfg {
+   value {
+      executor {
+         thread_name: "oman_orders"
+	 poll: true 
+	 hosted_actor: "MDSource"
+	 hosted_actor: "OrderManager"
+	 hosted_actor: "ExecutionGateway"
+      }
+      executor {
+         thread_name: "oman_slow"
+	 poll: false 
+	 hosted_actor: "*"
+      }
+   }
+}

src/lib/config/defaults/threads.conf

@@ -0,0 +1,7 @@
+Threads {
+   value {
+      thread_group {name_rx: "mdp3feed" core: 0}
+      thread_group {name_rx: "oman_orders" core: 1}
+      thread_group {name_rx: ".*" exclude:true core:0 core:1}
+   }
+}

src/lib/config/protos/MarketData.proto

@@ -75,7 +75,7 @@ message CMEOptions
       // Defines amount of time Feed Engine keeps cycling before going to sleep when no useful activity can be done
       optional uint32 spin_time = 4 [default = 1];
       optional uint32 channels_per_thread = 5 [default = 3];
-      repeated uint32 affinity = 10;
+      optional string thread_group = 10 [default=""]; // if present this will be used to retrieve affinity settings from Threads config key
       optional uint32 udp_socket_buffer_size = 11 [default = 33554432]; //32mb
    }
    enum BookType {

src/lib/config/protos/environment.proto

@@ -1,8 +0,0 @@
-syntax='proto2';
-package bts.config;
-
-message Deployment {
-  // Acceptable values:
-  // DEV_CHAPEL_HILL, DEV_CHICAGO, DEMO_GT, QA, CUSTOMER
-  optional string type = 1 [default="DEV_CHAPEL_HILL"];  
-}

src/lib/config/protos/mercury_cfg.proto

@@ -0,0 +1,7 @@
+syntax='proto2';
+
+package bts.config;
+
+message  Mercury {
+  optional uint32 queue_size = 1 [default=1000];
+}

src/lib/config/protos/orderman_cfg.proto

@@ -0,0 +1,11 @@
+syntax='proto2';
+package bts.config;
+
+message OrdermanCfg {
+  message Executor {
+    required string thread_name = 1;
+    optional bool poll = 2 [ default = false];
+    repeated string hosted_actor = 3;
+  }
+  repeated Executor executor = 1;
+}

src/lib/config/protos/threads.proto

@@ -0,0 +1,13 @@
+syntax='proto2';
+package bts.config;
+
+message ThreadGroupCfg {
+  required string name_rx = 1	[default = "*"];
+  optional bool exclude = 2	[default = false];
+  repeated uint32 core = 3; 
+}
+
+
+message Threads {
+  repeated ThreadGroupCfg thread_group = 1; 
+}

src/lib/jetfix/codec/FIXEncodingStream.h

@@ -18,7 +18,7 @@ class FIXEncodingStream
 {
 public:
    FIXEncodingStream(config::JetFIX::Connection const &cfg,
-                     uint32_t initial_capacity = 100*1024*10)
+                     uint32_t initial_capacity = 1024*10)
       :
       _msg_header_template (),
       _header_template_size (),

src/lib/jetfix/codec/FIXFragment.h

@@ -30,7 +30,7 @@ public:
       _owns_buffer (false)
    {}
 
-   explicit FIXFragment(uint32_t initial_capacity=100*1024)
+   explicit FIXFragment(uint32_t initial_capacity=100)
       :
       _buffer (new FIXBuffer (initial_capacity)),
       _checksum (0),

src/lib/jetfix/codec/FIXMsg.h

@@ -11,7 +11,7 @@
 #include "lib/jetfix/codec/IFIXMsg.h"
 #include "lib/jetfix/common/InMsgBuffer.h"
 #include "lib/utils/meta_programming.h"
-#include "lib/orderman/orderman_utils/TickToTradeCorID.h"
+#include "lib/orderman/orderman_utils/PerfTrace.h"
 
 namespace bts {
 namespace jetfix {

src/lib/jetfix/common/InMsgBuffer.h

@@ -6,7 +6,7 @@
 #include <vector>
 #include "lib/utils/bts_assert.h"
 #include "lib/jetfix/codec/MessageTypes.h"
-#include "lib/orderman/orderman_utils/TickToTradeCorID.h"
+#include "lib/orderman/orderman_utils/PerfTrace.h"
 
 namespace bts {
 namespace jetfix {

src/lib/jetfix/connection/AsyncReader.cpp

@@ -7,7 +7,8 @@
 #include "lib/utils/bts_assert.h"
 #include "lib/os/os.h"
 #include "lib/tracing/bts_trace.h"
-#include "lib/orderman/orderman_utils/TickToTradeCorID.h"
+#include "lib/orderman/orderman_utils/PerfTrace.h"
+#include "lib/utils/thread_manager.h"
 
 #include <fstream>
 
@@ -112,8 +113,8 @@ AsyncReader::AsyncReader (IConnection &connection,
 
 void AsyncReader::start ()
 {
-   me->reader_thread = std::thread ([this] () {
-         os::set_thread_name(me->connection_name+"_asyncr");
+   auto thread_name = fmt::format ("jf_ar_{}", me->connection_name);
+   me->reader_thread = threading::make_thread (thread_name,[this] () {
          dispatch();
       });
 }

src/lib/jetfix/connection/IConnection.h

@@ -1,7 +1,7 @@
 #pragma once
 
 #include <array>
-#include "lib/orderman/orderman_utils/TickToTradeCorID.h"
+#include "lib/orderman/orderman_utils/PerfTrace.h"
 
 namespace bts {
 namespace jetfix {

src/lib/jetfix/connection/SocketConnection.cpp

@@ -1,11 +1,12 @@
 #include "SocketConnection.h"
 #include "lib/jetfix/store/session_name.h"
 #include "lib/logging/logging.h"
-#include "lib/metrics/metrics.h"
+#include "lib/orderman/orderman_utils/PerfTraceLog.h"
 #include "lib/tracing/bts_trace.h"
 #include "lib/utils/TimeStamp.h"
 #include "lib/utils/bts_assert.h"
 #include "lib/utils/helper_macros.h"
+#include "lib/orderman/orderman_utils/PerfTraceLog.h"
 
 #include <arpa/inet.h>
 #include <fstream>
@@ -19,29 +20,15 @@ namespace bts {
 namespace jetfix {
 
 SocketConnection::SocketConnection(bts::config::JetFIX::Connection const &cfg,
-                                   metrics::MetricsContainer &stats)
+                                   PerfTraceLog &perf_log)
     : _fd(-1), _cfg(cfg), _log(cfg.log_fix_stream()), _in_log(), _out_log(),
-      _stats(stats), _tick_to_order(), _fill_to_order() {
+      _perf_log(perf_log) {
    if (_log) {
       _in_log.open(std::string("/tmp/fix_raw_read_log_") + session_name(cfg),
                    std::ios_base::app);
       _out_log.open(std::string("/tmp/fix_raw_write_log_") + session_name(cfg),
                     std::ios_base::app);
    }
-   std::vector<metrics::Label> labels;
-   labels.push_back({"connection", cfg.name()});
-
-   std::array<double, 30> buckets;
-   auto i = 1;
-   for (auto &b : buckets)
-      b = (i *= 2);
-
-   _tick_to_order = std::make_shared<metrics::HistogramMetric<30>>(
-       "spark_tick_to_order_ns", buckets, labels);
-   _fill_to_order = std::make_shared<metrics::HistogramMetric<30>>(
-       "spark_fill_to_order_ns", buckets, labels);
-   _stats.publish(_tick_to_order);
-   _stats.publish(_fill_to_order);
 }
 
 SocketConnection::~SocketConnection() {
@@ -122,46 +109,56 @@ int SocketConnection::write(WriteVec const &vec, size_t vec_size,
 
    auto trace_cpy = trace;
    trace_cpy.fix_socket_w = utils::TimeStamp::now();
-   tracepoint(bts_trace, trace_order, (char *)"SocketConnection.pre_write", 0,
-              trace.md_sn, 0);
+
+   utils::TimeStamp::Interval s2s;
+   if (trace.md.to_uint64()) {
+      s2s = (trace_cpy.fix_socket_w - trace_cpy.md);
+      tracepoint(bts_trace, trace_order,
+                 (char *)"SocketConnection.tick_to_order.pre_write", 0,
+                 trace.md_sn, s2s.microseconds());
+   } else {
+      s2s = (trace_cpy.fix_socket_w - trace_cpy.fix_socket_r);
+      tracepoint(bts_trace, trace_order,
+                 (char *)"SocketConnection.fill_to_order.pre_write", 0,
+                 trace.md_sn, s2s.microseconds());
+   }
+
    auto result = ::writev(_fd, iv, (int)vec_size);
 
    if (trace.md.to_uint64()) {
-      auto s2s = (trace_cpy.fix_socket_w - trace_cpy.md);
       LOG(Latency, Info,
           "perf_event:tick_to_order s2s_us:{} fix_socket_r:{} md:{} "
           "md_src:{} spreader_on_book:{} spreader_modify_order:{} "
-          "spreader_on_child_order_update:{} " 
-          "order_update:{} exe_gw:{} fix_socket_w:{} md_sn:{} order_key:{}",
+          "spreader_on_child_order_update:{} "
+          "order_update:{} exe_gw:{} fix_socket_w:{} md_sn:{} order_key:{} "
+          "send_fix_buffer:{}",
           s2s.microseconds(), trace_cpy.fix_socket_r.to_uint64(),
           trace_cpy.md.to_uint64(), trace_cpy.md_src.to_uint64(),
           trace_cpy.spreader_on_book.to_uint64(),
           trace_cpy.spreader_modify_order.to_uint64(),
-          trace_cpy.spreader_on_child_order_update.to_uint64(), 
-          trace_cpy.order_update.to_uint64(),
-          trace_cpy.exe_gw.to_uint64(), trace_cpy.fix_socket_w.to_uint64(),
-          trace_cpy.md_sn,
-          trace_cpy.order_key);
-      _tick_to_order->observe(s2s.nanoseconds());
+          trace_cpy.spreader_on_child_order_update.to_uint64(),
+          trace_cpy.order_update.to_uint64(), trace_cpy.exe_gw.to_uint64(),
+          trace_cpy.fix_socket_w.to_uint64(), trace_cpy.md_sn,
+          trace_cpy.order_key, trace.send_fix_buffer.to_uint64());
+      _perf_log.log(trace_cpy);
    }
 
    if (trace.fix_socket_r.to_uint64()) {
-      auto s2s = (trace_cpy.fix_socket_w - trace_cpy.fix_socket_r);
       LOG(Latency, Info,
           "perf_event:fill_to_order s2s_us:{} fix_socket_r:{} md:{} md_src:{} "
           "spreader_on_book:{} spreader_modify_order:{} "
-          "spreader_on_child_order_update:{} " 
-          "order_update:{} exe_gw:{} fix_socket_w:{} md_sn:{} order_key:{}",
+          "spreader_on_child_order_update:{} "
+          "order_update:{} exe_gw:{} fix_socket_w:{} md_sn:{} order_key:{} "
+          "send_fix_buffer:{} ",
           s2s.microseconds(), trace_cpy.fix_socket_r.to_uint64(),
           trace_cpy.md.to_uint64(), trace_cpy.md_src.to_uint64(),
           trace_cpy.spreader_on_book.to_uint64(),
           trace_cpy.spreader_modify_order.to_uint64(),
-          trace_cpy.spreader_on_child_order_update.to_uint64(), 
-          trace_cpy.order_update.to_uint64(),
-          trace_cpy.exe_gw.to_uint64(), trace_cpy.fix_socket_w.to_uint64(),
-          trace_cpy.md_sn,
-          trace.order_key);
-      _fill_to_order->observe(s2s.nanoseconds());
+          trace_cpy.spreader_on_child_order_update.to_uint64(),
+          trace_cpy.order_update.to_uint64(), trace_cpy.exe_gw.to_uint64(),
+          trace_cpy.fix_socket_w.to_uint64(), trace_cpy.md_sn, trace.order_key,
+          trace.send_fix_buffer.to_uint64());
+      _perf_log.log(trace_cpy);
    }
 
    tracepoint(bts_trace, trace_order, (char *)"SocketConnection.post_write", 0,

src/lib/jetfix/connection/SocketConnection.h

@@ -7,14 +7,14 @@
 #include "lib/metrics/metrics.h"
 
 namespace bts {
-namespace metrics {class MetricsContainer;}
+class PerfTraceLog;
 
 namespace jetfix {
 
 class SocketConnection : public IConnection
 {
 public:
-   SocketConnection(bts::config::JetFIX::Connection const &cfgl,metrics::MetricsContainer &stats);
+   SocketConnection(bts::config::JetFIX::Connection const &cfgl,PerfTraceLog &log);
    ~SocketConnection();
    bool connect () override;
    void shutdown () override;
@@ -27,9 +27,7 @@ private:
    bool _log;
    std::ofstream _in_log;
    std::ofstream _out_log;
-   metrics::MetricsContainer &_stats;
-   std::shared_ptr<metrics::HistogramMetric<30>> _tick_to_order;
-   std::shared_ptr<metrics::HistogramMetric<30>> _fill_to_order;
+   PerfTraceLog &_perf_log;
 };
 
 }}

src/lib/jetfix/gateway/CoreGateway.cpp

@@ -13,6 +13,7 @@
 #include "lib/orderman/orderman_utils/orderman_common.h"
 #include "lib/tracing/bts_trace.h"
 #include <boost/lockfree/queue.hpp>
+#include "lib/utils/thread_manager.h"
 
 #include <atomic>
 #include <boost/date_time/posix_time/posix_time.hpp>
@@ -121,12 +122,17 @@ struct CoreGateway::PImpl {
    uint32_t connection_attempts;
 };
 
+std::thread CoreGateway::make_timer_thread() {
+   auto thread_name = fmt::format ("jf_tmr_{}", me->cfg.name());
+   return threading::make_thread(thread_name, std::bind(&CoreGateway::call_time, this));
+}
+
 CoreGateway::CoreGateway(config::JetFIX::Connection const &cfg,
                          ICoreGatewayObserver &observer,
                          ConnectionFactory const &conn_factory,
                          StoreFactory const &store_factory)
     : me(new PImpl(cfg, observer, conn_factory, store_factory)) {
-   me->timer_thread = std::thread(std::bind(&CoreGateway::call_time, this));
+   me->timer_thread = make_timer_thread();
 }
 
 CoreGateway::~CoreGateway() {
@@ -202,7 +208,6 @@ void CoreGateway::dispatch_events() {
 }
 
 void CoreGateway::call_time() {
-   os::set_thread_name(fmt::format("{0}_timer", me->cfg.name()).c_str());
    while (!me->destroy) {
       std::this_thread::sleep_for(10ms);
       auto ts = std::make_shared<std::string>(

src/lib/jetfix/gateway/CoreGateway.h

@@ -67,7 +67,8 @@ private:
    void transition (State new_state);
    void check_state_invariants ();
    const char* state_name (State st);
-
+   std::thread make_timer_thread();
+   
    std::unique_ptr<PImpl> me;
 };
 

src/lib/jetfix/gateway/JetFIXGateway.h

@@ -12,6 +12,7 @@
 #include "lib/config/protos/jetfix.pb.h"
 
 namespace bts {
+class PerfTraceLog;
 namespace jetfix {
 
 // A parsing layer over CoreGateway
@@ -19,13 +20,12 @@ template <typename... Messages>
 class JetFIXGateway : ICoreGatewayObserver
 {
 public:
-   JetFIXGateway(config::JetFIX::Connection const &cfg, 
-                 metrics::MetricsContainer &stats)
+   JetFIXGateway(config::JetFIX::Connection const &cfg, PerfTraceLog &log)
       :
       _core (cfg,*this,
                
-             [&stats] (config::JetFIX::Connection const &cfg)
-             {return new SocketConnection (cfg,stats);},
+             [&log] (config::JetFIX::Connection const &cfg)
+             {return new SocketConnection (cfg,log);},
              
              [] (config::JetFIX::Connection const &cfg)
              { return new FIXFileStore (cfg);}),

src/lib/jetfix/store/FIXFileStore.cpp

@@ -9,6 +9,7 @@
 #include "lib/utils/bts_chrono.h"
 #include "lib/utils/datetime.h"
 #include "lib/utils/file_utils.h"
+#include "lib/utils/thread_manager.h"
 
 #include <boost/date_time.hpp>
 #include <boost/date_time/gregorian/gregorian.hpp>
@@ -204,7 +205,9 @@ FIXFileStore::FIXFileStore(config::JetFIX::Connection const &cfg,
          }
       }
       me->fix_store.open(fix_log_p, std::ofstream::app);
-      me->writer_thread = std::thread(std::bind(&FIXFileStore::write, this));
+      auto thread_name = fmt::format("jf_store_{}", me->cfg.name().c_str());
+      me->writer_thread = threading::make_thread(
+          thread_name, std::bind(&FIXFileStore::write, this));
    }
 }
 
@@ -270,7 +273,6 @@ void FIXFileStore::set_last_seq_num(uint32_t last_seq_num) {
 
 void FIXFileStore::write() {
    Event ev;
-   os::set_thread_name(fmt::format("{0}_store", me->cfg.name()).c_str());
 
    while (true)
       if (me->event_queue.try_pop(ev)) {

src/lib/logging/LoggerHub.cpp

@@ -4,6 +4,7 @@
 #include "lib/os/os.h"
 #include "lib/utils/bts_assert.h"
 #include "lib/utils/bts_chrono.h"
+#include "lib/utils/thread_manager.h"
 
 #include <algorithm>
 #include <atomic>
@@ -25,7 +26,7 @@ struct LoggerHub::PImpl {
    PImpl(std::string const &logger_name)
        : lock(), sinks(), loggers(), level_names(), subsystem_names(), worker(),
          done(false), logger_initial_queue_size(), logger_max_queue_growth(),
-         logger_name(logger_name) {}
+         logger_name(logger_name), thread_cfg() {}
 
    std::mutex lock;
    std::vector<SinkRec> sinks;
@@ -37,6 +38,7 @@ struct LoggerHub::PImpl {
    size_t logger_initial_queue_size;
    size_t logger_max_queue_growth;
    std::string logger_name;
+   config::ThreadGroupCfg thread_cfg;
 };
 
 LoggerHub::LoggerHub(std::string const &logger_name)
@@ -49,7 +51,6 @@ LoggerHub::LoggerHub(std::string const &logger_name)
 
 void LoggerHub::config(const LoggingConfig &cfg) {
    std::lock_guard<std::mutex> l(me->lock);
-
    me->logger_initial_queue_size = cfg.logger_initial_queue_size;
    me->logger_max_queue_growth = cfg.logger_max_queue_growth;
 
@@ -74,11 +75,14 @@ void LoggerHub::config(const LoggingConfig &cfg) {
    }
    rebuild_logger_filters();
    update_logger_queue_limits();
+   me->thread_cfg = cfg.thread_cfg;
 }
 
 void LoggerHub::start() {
    bts_assert(!me->worker);
-   me->worker = std::make_unique<std::thread>(&LoggerHub::do_work, this);
+   me->worker = std::make_unique<std::thread>(
+      threading::make_thread(me->logger_name.c_str(),
+                             std::bind(&LoggerHub::do_work, this),me->thread_cfg));
 }
 
 LoggerHub::~LoggerHub() {
@@ -96,7 +100,6 @@ Logger &LoggerHub::make_logger() {
 }
 
 void LoggerHub::do_work() {
-   os::set_thread_name(me->logger_name.c_str());
    while (!me->done) {
       size_t consumed = 0;
       {

src/lib/logging/LoggerHub.h

@@ -33,5 +33,4 @@ private:
    Filter get_base_filter() const;
 };
 
-void start_logging_hub();
 }}

src/lib/logging/LoggingConfig.h

@@ -4,18 +4,22 @@
 #include "Common.h"
 #include <vector>
 #include <map>
+#include "lib/config/protos/threads.pb.h"
 
 namespace bts {
 namespace log {
 
 struct LoggingConfig {
-   LoggingConfig() : logger_initial_queue_size(120000),
-                     logger_max_queue_growth(4),
-                     sinks(),
-                     level_names(),
-                     subsystem_names(),
-                     deployment_type("dev")
-   {}
+   LoggingConfig() :
+      logger_initial_queue_size(120000),
+      logger_max_queue_growth(4),
+      sinks(),
+      level_names(),
+      subsystem_names(),
+      deployment_type("dev"),
+      thread_cfg()
+   {
+   }
 
    size_t logger_initial_queue_size = 0;
    size_t logger_max_queue_growth = 0;
@@ -23,6 +27,7 @@ struct LoggingConfig {
    std::map<Level,std::string> level_names;
    std::map<Subsystem,std::string> subsystem_names;
    std::string deployment_type;
+   config::ThreadGroupCfg thread_cfg;
 };
 
 }}

src/lib/logging/logging.cpp

@@ -9,6 +9,7 @@
 #include "lib/utils/file_utils.h"
 #include "lib/utils/Dispatcher.h"
 #include "lib/config/protos/system.pb.h"
+#include "lib/utils/thread_manager.h"
 
 #include <google/protobuf/text_format.h>
 #include <google/protobuf/io/zero_copy_stream_impl.h>
@@ -86,8 +87,8 @@ Filter get_filter (LoggingConfig &cfg,
 }
 
 void configure_logging(LoggingConfig const&cfg) {_default_logger_hub.config(cfg);}
-
-void configure_logging(bts::config::Logging const&cfg)
+void configure_logging(bts::config::Logging const&cfg) {configure_logging(cfg,bts::config::ThreadGroupCfg ());}
+void configure_logging(bts::config::Logging const&cfg, bts::config::ThreadGroupCfg const&thread_cfg)
 {
    LoggingConfig real_cfg;
    real_cfg.deployment_type = config::get<config::DeploymentType>().type();
@@ -124,6 +125,7 @@ void configure_logging(bts::config::Logging const&cfg)
       std::make_pair (syslog_sink(real_cfg),get_filter(real_cfg,cfg.syslog()));
    real_cfg.sinks["file"] =
       std::make_pair (file_sink(real_cfg),get_filter(real_cfg,cfg.file()));
+   real_cfg.thread_cfg = thread_cfg;
    configure_logging(real_cfg);
 }
 
@@ -136,19 +138,20 @@ boost::filesystem::path log_cache_file_path ()
    return result;
 }
 
-LoggingConfigPoller::LoggingConfigPoller (bts::config::Logging const&cfg)
+LoggingConfigPoller::LoggingConfigPoller (bts::config::Logging const&cfg, bts::config::ThreadGroupCfg const&thread_cfg)
    : _poller(utils::Dispatcher::background(), 1000, [=]() { poll_once(); }), 
    _last_ts(std::time(nullptr)),
    _log_cache_file(log_cache_file_path()),
    _done(false)
 {
-   init(cfg);
+   init(cfg,thread_cfg);
 }
 
-LoggingConfigPoller::LoggingConfigPoller () : LoggingConfigPoller(config::get<config::Logging>())
+LoggingConfigPoller::LoggingConfigPoller () : LoggingConfigPoller(config::get<config::Logging>(),
+                                                                  threading::get_thread_cfg("logger"))
 { }
 
-void LoggingConfigPoller::init (bts::config::Logging const&cfg)
+void LoggingConfigPoller::init (bts::config::Logging const&cfg,bts::config::ThreadGroupCfg const&thread_cfg)
 {
    auto parent = _log_cache_file.parent_path();
    if (!boost::filesystem::is_directory(parent)) 
@@ -160,7 +163,7 @@ void LoggingConfigPoller::init (bts::config::Logging const&cfg)
 
 
    // initialize logging
-   configure_logging(cfg);
+   configure_logging(cfg,thread_cfg);
    _poller.start();
 }
 

src/lib/logging/logging.h

@@ -10,7 +10,7 @@
 
 
 namespace bts {
-namespace config {class Logging;}
+namespace config {class Logging; class ThreadGroupCfg;}
 
 // Please keep this list in sync with Log.cs
 constexpr log::Level Diag     {0};
@@ -55,17 +55,18 @@ struct LoggerHandle {
 
 extern thread_local LoggerHandle _thread_logger;
 void configure_logging(LoggingConfig const&cfg);
+void configure_logging(bts::config::Logging const&cfg,bts::config::ThreadGroupCfg const&);
 void configure_logging(bts::config::Logging const&cfg);
 void start_default_logging_thread();
 
 class LoggingConfigPoller {
 public:
    LoggingConfigPoller ();
-   LoggingConfigPoller (bts::config::Logging const&cfg);
+   LoggingConfigPoller (bts::config::Logging const&cfg, bts::config::ThreadGroupCfg const&thread_cfg);
   ~LoggingConfigPoller ();
 
 private:
-   void init(bts::config::Logging const&cfg);
+   void init(bts::config::Logging const&cfg, bts::config::ThreadGroupCfg const&thread_cfg);
    void poll_once();
    utils::DispatcherTimer _poller;
    std::time_t _last_ts;

src/lib/md/Diag/ShmDumper.cpp

@@ -76,8 +76,7 @@ string dump_security(const FixedSecurity& security) {
    res << "'main_frac': " << (int)security.cme_main_fraction << ", ";
    res << "'sub_fract': " << (int)security.cme_sub_fraction << ", ";
    res << "'price_format': " << (int)security.cme_price_display_format << ", ";
-
-   res << "'market_segment': " << security.cme_market_segment_id << ", ";
+   res << "'market_segment': " << (int)security.cme_market_segment_id << ", ";
 
    res << "'price_break': " << security.price_break << ", ";
    res << "'low_tick': " << security.low_tick << ", ";

src/lib/md/test/Co2ManagedTestHelper.cpp

@@ -141,6 +141,10 @@ extern "C" BTS_EXPORT void co2_get_zero_and_nan_book(PriceLevel& price) {
    price = { PriceSize::NO_MARKET, PriceSize::from_double<7>(0,1) };
 }
 
+extern "C" BTS_EXPORT void co2_get_sample_datetime(datetime::LLDateTime& now, datetime::LLDateTime& utc_now) {
+	now = datetime::ll_local_now();
+	utc_now = datetime::ll_now();
+}
 
 }}}//namespace
 #endif

src/lib/mercury/actors/Actor.cpp

@@ -1,7 +0,0 @@
-#include "Actor.h"
-
-namespace bts {
-namespace mercury {
-
-
-}}

src/lib/mercury/actors/Actor.h

@@ -3,30 +3,32 @@
 #include <boost/container/flat_set.hpp>
 #include <memory>
 
-#include "lib/utils/bts_assert.h"
-#include "lib/utils/warn_ctrl.h"
-#include "lib/mercury/broker/AsyncEventBroker.h"
+#include "lib/mercury/common/mercury_common.h"
 #include "lib/mercury/events/Stream.h"
 #include "lib/mercury/events/Update.h"
 #include "lib/mercury/topics/Topic.h"
+#include "lib/mercury/actors/ActorBroker.h"
+
+#include "lib/tracing/bts_trace.h"
+#include "lib/utils/bts_assert.h"
+#include "lib/utils/warn_ctrl.h"
 
 namespace bts {
 namespace mercury {
 
-// Helper class to ease subscription/publication of events Client code
-// should derive from this class and implement an init() method which
-// signals to the class that the event dispatch framework is ready to
-// process event subscription and publication.
+// Helper class to ease subscription/publication of events. Client
+// code should derive from this class and implement an init() method
+// which signals to the class that the event dispatch framework is
+// ready to process event subscription and publication.
 
-GCC_DIAG_OFF("-Wnon-virtual-dtor")
-class Actor : public std::enable_shared_from_this<Actor> {
+class Actor {
  public:
-   using SubHandle = InternalSubID;
+   using SubHandle = LocalSubID;
 
-   Actor() : _broker(nullptr), _active_streams(), _active_subs() {}
+   Actor() : _broker(nullptr), _active_streams(), _active_subs(), _broker_id() {}
    virtual ~Actor() {}
 
-   // Override this to kick off actor. 
+   // Override this to kick off actor.
    virtual void init() = 0;
 
    // Override for custom actor names, used in logging messages.
@@ -42,30 +44,29 @@ class Actor : public std::enable_shared_from_this<Actor> {
       using TopicType = Topic<EV>;
       using UpdateType = Update<EV>;
 
-      auto self = shared_from_this();
-      TopicFilter filter = [tag, self](const ITopic &t) {
+      TopicFilter filter = [tag, this](const ITopic &t) {
          auto typed_topic = dynamic_cast<TopicType const *>(&t);
          if (!typed_topic)
             return false;
          else {
             auto match = typed_topic->tag() == tag;
             if (match)
-               LOG(Mercury, Diag, "{} on event broker {} matched sub to {}",
-                   self->name(), self->broker_id().value(),
+               LOG(Mercury, Info, "{} on event broker {} matched sub to {}",
+                   name(), _broker_id,
                    TopicType(tag).to_str());
             return match;
          }
       };
 
-      InternalEventHandler handler = [this, cb, tag](
-          const StreamID &sid, EventType type, EventPayload const *payload) {
+      LocalEventHandler handler = [this, cb, tag](const StreamID &sid,
+                                                  EventType type,
+                                                  Notification const *payload) {
          if (!this->_broker)
             return;
 
-         LOG(Mercury, Diag,
-             "{} on event broker {} received update to {}. Payload: {}", name(),
-             broker_id().value(), TopicType(tag).to_str(), (size_t)payload);
-
+         tracepoint(bts_trace, trace_event, utils::type_name(typeid(*this)),
+                    utils::type_name<EV>(), _broker_id, sid.value(),
+                    0);
          if (payload)
             cb(UpdateType{sid, type, &(payload->unwrap<typename EV::Array>())});
          else
@@ -76,38 +77,36 @@ class Actor : public std::enable_shared_from_this<Actor> {
 
       LOG(Mercury, Info,
           "{} on event broker {} subscribing to {} with subid {}", name(),
-          broker_id().value(), TopicType(tag).to_str(), result.value());
+          _broker_id, TopicType(tag).to_str(), result.value());
 
       return result;
    }
-   
+
    // subscribe to all events of a specific type
    template <class EV, class CB> SubHandle subscribe(CB const &cb) {
       bts_assert(_broker);
       using TopicType = Topic<EV>;
       using UpdateType = Update<EV>;
-      auto self = shared_from_this();
 
-      TopicFilter filter = [self](const ITopic &t) {
+      TopicFilter filter = [this](const ITopic &t) {
          auto typed_topic = dynamic_cast<TopicType const *>(&t);
          auto match = typed_topic != nullptr;
          if (match)
-            LOG(Mercury, Diag, "{} on event broker {} matched sub to {}",
-                self->name(), self->broker_id().value(),
+            LOG(Mercury, Info, "{} on event broker {} matched sub to {}",
+                name(), _broker_id,
                 std::string(utils::type_name<EV>()));
          return match;
       };
 
-      InternalEventHandler handler = [this, cb](
-          const StreamID &sid, EventType type, EventPayload const *payload) {
+      LocalEventHandler handler = [this, cb](const StreamID &sid,
+                                             EventType type,
+                                             Notification const *payload) {
          if (!this->_broker)
             return;
 
-         LOG(Mercury, Diag,
-             "{} on event broker {} received update to {}. Payload: {}", name(),
-             broker_id().value(), std::string(utils::type_name<EV>()),
-             (size_t)payload);
-
+         tracepoint(bts_trace, trace_event, utils::type_name(typeid(*this)),
+                    utils::type_name<EV>(), _broker_id, sid.value(),
+                    0);
          if (payload)
             cb(UpdateType{sid, type, &(payload->unwrap<typename EV::Array>())});
          else
@@ -115,9 +114,10 @@ class Actor : public std::enable_shared_from_this<Actor> {
       };
 
       auto result = _subscribe(filter, handler);
-      LOG(Mercury, Info, "{} on event broker {} subscribing to all instances "
-                         "of type {} with subid {}",
-          name(), broker_id().value(), std::string(utils::type_name<EV>()),
+      LOG(Mercury, Info,
+          "{} on event broker {} subscribing to all instances "
+          "of type {} with subid {}",
+          name(), _broker_id, std::string(utils::type_name<EV>()),
           result.value());
       return result;
    }
@@ -130,8 +130,7 @@ class Actor : public std::enable_shared_from_this<Actor> {
       using TopicType = Topic<EV>;
       using UpdateType = Update<EV>;
 
-      auto self = shared_from_this();
-      TopicFilter filter = [self, user_filter](const ITopic &t) {
+      TopicFilter filter = [this, user_filter](const ITopic &t) {
          auto typed_topic = dynamic_cast<TopicType const *>(&t);
          if (typed_topic == nullptr)
             return false;
@@ -139,11 +138,14 @@ class Actor : public std::enable_shared_from_this<Actor> {
             return user_filter(*typed_topic);
       };
 
-      InternalEventHandler handler = [this, cb](
-          const StreamID &sid, EventType type, EventPayload const *payload) {
+      LocalEventHandler handler = [this, cb](const StreamID &sid,
+                                             EventType type,
+                                             Notification const *payload) {
          if (!this->_broker)
             return;
-
+         tracepoint(bts_trace, trace_event, utils::type_name(typeid(*this)),
+                    utils::type_name<EV>(), _broker_id, sid.value(),
+                    0);
          if (payload)
             cb(UpdateType{sid, type, &(payload->unwrap<typename EV::Array>())});
          else
@@ -153,7 +155,7 @@ class Actor : public std::enable_shared_from_this<Actor> {
       auto result = _subscribe(filter, handler);
       LOG(Mercury, Info,
           "{} on event broker {} subscribing to filter on {} with subid {}",
-          name(), broker_id().value(), std::string(utils::type_name<EV>()),
+          name(), _broker_id, std::string(utils::type_name<EV>()),
           result.value());
       return result;
    }
@@ -161,7 +163,7 @@ class Actor : public std::enable_shared_from_this<Actor> {
    // unsubscribe
    void unsubscribe(SubHandle id) {
       bts_assert(_broker);
-      _broker->internal_unsubscribe(id);
+      _broker->unsubscribe(id);
       _active_subs.erase(id);
    }
 
@@ -170,26 +172,26 @@ class Actor : public std::enable_shared_from_this<Actor> {
       bts_assert(_broker);
       using TopicType = Topic<EV>;
       auto topic = std::make_shared<TopicType>(tag);
-      auto sid = _broker->internal_advertise(topic);
+      auto sid = _broker->advertise(topic);
 
       LOG(Mercury, Info,
           "{} on event broker {} creating stream for {} with sid {}", name(),
-          broker_id().value(), TopicType(tag).to_str(), sid.value());
+          _broker_id, TopicType(tag).to_str(), sid.value());
 
       _active_streams.insert(sid);
       return Stream<EV>{sid};
    }
 
-   // create a stream of events of a specific type 
+   // create a stream of events of a specific type
    template <class EV> Stream<EV> make_stream() {
       bts_assert(_broker);
       using TopicType = Topic<EV>;
       auto topic = std::make_shared<TopicType>();
-      auto sid = _broker->internal_advertise(topic);
+      auto sid = _broker->advertise(topic);
 
       LOG(Mercury, Info,
           "{} on event broker {} creating stream for {} with sid {}", name(),
-          broker_id().value(), TopicType().to_str(), sid.value());
+          _broker_id, TopicType().to_str(), sid.value());
 
       _active_streams.insert(sid);
       return Stream<EV>{sid};
@@ -199,56 +201,37 @@ class Actor : public std::enable_shared_from_this<Actor> {
    // event with the args passed
    template <class EV, typename... Args>
    void publish(Stream<EV> const &stream, Args &&... args) {
+      bts_assert_d(_broker);
 
-      LOG(Mercury, Diag, "{} on event broker {} publishing {} on stream {}",
-          name(), broker_id().value(), std::string(utils::type_name<EV>()),
-          stream.sid.value());
+      tracepoint(bts_trace, trace_event, utils::type_name(typeid(*this)),
+                 utils::type_name<EV>(), _broker_id,
+                 stream.sid.value(), 1);
 
-      bts_assert(_broker);
-      auto msg = _broker->alloc_internal_notification_msg(stream.sid);
-      auto &array = msg.template wrap_in_place<typename EV::Array>();
-      array.emplace_back(std::forward<Args>(args)...);
-      _broker->internal_notify(msg);
+      _broker->notify(stream, std::forward<Args>(args)...);
    }
 
    // publish a sequence of events to a previously created stream
    template <class EV, class C>
    void publish_batch(Stream<EV> const &stream, C const &c) {
-
-      LOG(Mercury, Info, "{} on event broker {} publishing a batch of {} {} "
-                         " on stream {}",
-          name(), broker_id().value(), c.size(),
-          std::string(utils::type_name<EV>()), stream.sid.value());
-
       bts_assert(_broker);
-      auto msg = _broker->alloc_internal_notification_msg(stream.sid);
-      auto &array = msg.template wrap_in_place<typename EV::Array>();
-      array.reserve(c.size());
-      for (auto &item : c)
-         array.emplace_back(item);
-      _broker->internal_notify(msg);
+      _broker->notify_batch(stream,c);
    }
 
    // destroy a previously created stream
    template <class EV> void destroy_stream(Stream<EV> const &stream) {
       bts_assert(_broker);
       _active_streams.erase(stream.sid);
-      _broker->internal_unadvertise(stream.sid);
+      _broker->unadvertise(stream.sid);
    }
 
-   // queue a lambda for execution in the Actors thread
-   void post_action(std::function<void()> const &action) {
-      _broker->post_action(action);
+   void schedule_action (std::function<void()> const&action) {
+      bts_assert(_broker);
+      _broker->schedule_action(action);
    }
 
  protected:
-   EventBrokerID broker_id() {
-      bts_assert(_broker);
-      return _broker->id();
-   }
-
    friend class Executor;
-   void activate(AsyncEventBroker &broker) {
+   void activate(ActorBroker &broker) {
       bts_assert(!_broker);
       _broker = &broker;
       LOG(Mercury, Info, "Initializing {}", name());
@@ -260,27 +243,27 @@ class Actor : public std::enable_shared_from_this<Actor> {
       finit();
 
       for (auto sid : _active_streams)
-         _broker->internal_unadvertise(sid);
+         _broker->unadvertise(sid);
       _active_streams.clear();
 
       for (auto h : _active_subs)
-         _broker->internal_unsubscribe(h);
+         _broker->unsubscribe(h);
       _active_subs.clear();
 
       _broker = nullptr;
    }
 
  private:
-   AsyncEventBroker *_broker;
+   ActorBroker *_broker;
    boost::container::flat_set<StreamID> _active_streams;
    boost::container::flat_set<SubHandle> _active_subs;
+   BrokerID::Value _broker_id;
 
-   SubHandle _subscribe(TopicFilter const &f, InternalEventHandler const &h) {
-      auto handle = _broker->internal_subscribe(f, h);
+   SubHandle _subscribe(TopicFilter const &f, LocalEventHandler const &h) {
+      auto handle = _broker->subscribe(f, h);
       _active_subs.insert(handle);
       return handle;
    }
 };
-GCC_DIAG_RESTORE()
-}
-}
+} // namespace mercury
+} // namespace bts

src/lib/mercury/actors/ActorBroker.h

@@ -0,0 +1,69 @@
+#pragma once
+
+#include "lib/mercury/common/mercury_common.h"
+#include "lib/mercury/broker/BrokerProxy.h"
+#include "lib/mercury/events/Stream.h"
+
+namespace bts {
+namespace mercury {
+
+class ActorBroker {
+public:
+   ActorBroker (BrokerProxy &proxy) : _proxy(proxy),_proxy_id(_proxy.id()),_current_suid(0) {}
+
+   BrokerID id() {return _proxy_id;}
+
+   LocalSubID subscribe(TopicFilter const&f, LocalEventHandler const &h) {
+      _current_suid = LocalSubID(_current_suid.value() + 1);
+      _proxy.local_subscribe(_current_suid,f,h);
+      return _current_suid;
+   }
+
+   void unsubscribe(LocalSubID suid) {
+      _proxy.local_unsubscribe(suid);
+   }
+
+   StreamID advertise (TopicPtr const&t) {
+      auto sid = gen_stream_id();
+      _proxy.advertise(_proxy_id,sid,t);
+      return sid;
+   }
+    
+   void unadvertise (StreamID sid) {
+      _proxy.unadvertise(_proxy_id,sid);
+   }
+   
+   template <class EV, typename... Args>
+   void notify (Stream<EV> const &stream, Args &&... args)  {
+      BrokerProxy::BrokerMsg *m;
+      Notification *notification;
+      _proxy.alloc_notify(&m,&notification);
+      auto &array = notification->template wrap_in_place<typename EV::Array>();
+      array.emplace_back(std::forward<Args>(args)...);
+      _proxy.send_notify(_proxy_id, stream.sid, SubID(0), *m);
+   }
+
+   template <class EV, class C>
+   void notify_batch(Stream<EV> const &stream, C const &c) {
+      BrokerProxy::BrokerMsg *msg;
+      Notification *notification;
+      _proxy.alloc_notify(&msg,&notification);
+      auto &array = notification->template wrap_in_place<typename EV::Array>();
+      array.reserve(c.size());
+      for (auto &item : c)
+         array.emplace_back(item);
+      _proxy.send_notify(_proxy_id, stream.sid, SubID(0), *msg);
+   }
+
+   void schedule_action (std::function<void()> const&action) {
+      _proxy.schedule_action(action);
+   }
+
+private:
+   BrokerProxy &_proxy;
+   BrokerID _proxy_id;
+   LocalSubID _current_suid;
+};
+
+
+}}

src/lib/mercury/actors/ActorPtr.h

@@ -1,11 +0,0 @@
-#pragma once
-
-#include <memory>
-
-namespace bts {
-namespace mercury {
-
-class Actor;
-using ActorPtr = std::shared_ptr<Actor>;
-
-}}