Reduce log spamming further.

Update release notes.
Reduced log spamming in order manager.
2018-10-15 17:33:40 +00:00 · 2018-10-15 14:42:43 +00:00 · 2018-10-15 14:38:44 +00:00 · 2018-10-14 18:29:00 -04:00 · 2018-10-14 18:28:56 -04:00 · 2018-10-11 15:26:14 -05:00
181 changed files with 4287 additions and 5777 deletions
@@ -24,3 +24,5 @@ project.lock.json
 src/server/sparkd/orderman/perf_analysis/data
 src/server/sparkd/orderman/perf_analysis/logs/
 src/server/sparkd/orderman/perf_analysis/trace_data/
+src/lib/orderman/perf_analysis/logs/latency_data/
+trace_data/
@@ -40,7 +40,7 @@ else ()
                               -fPIC -Wall -pedantic -Wextra -Werror \
                               -Wno-missing-braces -Wno-ignored-qualifiers \
                               --std=c++17 -fdiagnostics-color")
-  set (CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_DEBUG} -DNDEBUG -O3")
+  set (CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_DEBUG} -fno-omit-frame-pointer -DNDEBUG -O3")
  add_subdirectory(src/server/sparkd)
  add_subdirectory(src/lib)
  add_subdirectory(src/web)
@@ -0,0 +1,301 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en">
+<head>
+<!-- 2018-10-08 Mon 18:50 -->
+<meta http-equiv="Content-Type" content="text/html;charset=utf-8" />
+<meta name="viewport" content="width=device-width, initial-scale=1" />
+<title>&lrm;</title>
+<meta name="generator" content="Org mode" />
+<meta name="author" content="ppinto" />
+<style type="text/css">
+ <!--/*--><![CDATA[/*><!--*/
+  .title  { text-align: center;
+             margin-bottom: .2em; }
+  .subtitle { text-align: center;
+              font-size: medium;
+              font-weight: bold;
+              margin-top:0; }
+  .todo   { font-family: monospace; color: red; }
+  .done   { font-family: monospace; color: green; }
+  .priority { font-family: monospace; color: orange; }
+  .tag    { background-color: #eee; font-family: monospace;
+            padding: 2px; font-size: 80%; font-weight: normal; }
+  .timestamp { color: #bebebe; }
+  .timestamp-kwd { color: #5f9ea0; }
+  .org-right  { margin-left: auto; margin-right: 0px;  text-align: right; }
+  .org-left   { margin-left: 0px;  margin-right: auto; text-align: left; }
+  .org-center { margin-left: auto; margin-right: auto; text-align: center; }
+  .underline { text-decoration: underline; }
+  #postamble p, #preamble p { font-size: 90%; margin: .2em; }
+  p.verse { margin-left: 3%; }
+  pre {
+    border: 1px solid #ccc;
+    box-shadow: 3px 3px 3px #eee;
+    padding: 8pt;
+    font-family: monospace;
+    overflow: auto;
+    margin: 1.2em;
+  }
+  pre.src {
+    position: relative;
+    overflow: visible;
+    padding-top: 1.2em;
+  }
+  pre.src:before {
+    display: none;
+    position: absolute;
+    background-color: white;
+    top: -10px;
+    right: 10px;
+    padding: 3px;
+    border: 1px solid black;
+  }
+  pre.src:hover:before { display: inline;}
+  /* Languages per Org manual */
+  pre.src-asymptote:before { content: 'Asymptote'; }
+  pre.src-awk:before { content: 'Awk'; }
+  pre.src-C:before { content: 'C'; }
+  /* pre.src-C++ doesn't work in CSS */
+  pre.src-clojure:before { content: 'Clojure'; }
+  pre.src-css:before { content: 'CSS'; }
+  pre.src-D:before { content: 'D'; }
+  pre.src-ditaa:before { content: 'ditaa'; }
+  pre.src-dot:before { content: 'Graphviz'; }
+  pre.src-calc:before { content: 'Emacs Calc'; }
+  pre.src-emacs-lisp:before { content: 'Emacs Lisp'; }
+  pre.src-fortran:before { content: 'Fortran'; }
+  pre.src-gnuplot:before { content: 'gnuplot'; }
+  pre.src-haskell:before { content: 'Haskell'; }
+  pre.src-hledger:before { content: 'hledger'; }
+  pre.src-java:before { content: 'Java'; }
+  pre.src-js:before { content: 'Javascript'; }
+  pre.src-latex:before { content: 'LaTeX'; }
+  pre.src-ledger:before { content: 'Ledger'; }
+  pre.src-lisp:before { content: 'Lisp'; }
+  pre.src-lilypond:before { content: 'Lilypond'; }
+  pre.src-lua:before { content: 'Lua'; }
+  pre.src-matlab:before { content: 'MATLAB'; }
+  pre.src-mscgen:before { content: 'Mscgen'; }
+  pre.src-ocaml:before { content: 'Objective Caml'; }
+  pre.src-octave:before { content: 'Octave'; }
+  pre.src-org:before { content: 'Org mode'; }
+  pre.src-oz:before { content: 'OZ'; }
+  pre.src-plantuml:before { content: 'Plantuml'; }
+  pre.src-processing:before { content: 'Processing.js'; }
+  pre.src-python:before { content: 'Python'; }
+  pre.src-R:before { content: 'R'; }
+  pre.src-ruby:before { content: 'Ruby'; }
+  pre.src-sass:before { content: 'Sass'; }
+  pre.src-scheme:before { content: 'Scheme'; }
+  pre.src-screen:before { content: 'Gnu Screen'; }
+  pre.src-sed:before { content: 'Sed'; }
+  pre.src-sh:before { content: 'shell'; }
+  pre.src-sql:before { content: 'SQL'; }
+  pre.src-sqlite:before { content: 'SQLite'; }
+  /* additional languages in org.el's org-babel-load-languages alist */
+  pre.src-forth:before { content: 'Forth'; }
+  pre.src-io:before { content: 'IO'; }
+  pre.src-J:before { content: 'J'; }
+  pre.src-makefile:before { content: 'Makefile'; }
+  pre.src-maxima:before { content: 'Maxima'; }
+  pre.src-perl:before { content: 'Perl'; }
+  pre.src-picolisp:before { content: 'Pico Lisp'; }
+  pre.src-scala:before { content: 'Scala'; }
+  pre.src-shell:before { content: 'Shell Script'; }
+  pre.src-ebnf2ps:before { content: 'ebfn2ps'; }
+  /* additional language identifiers per "defun org-babel-execute"
+       in ob-*.el */
+  pre.src-cpp:before  { content: 'C++'; }
+  pre.src-abc:before  { content: 'ABC'; }
+  pre.src-coq:before  { content: 'Coq'; }
+  pre.src-groovy:before  { content: 'Groovy'; }
+  /* additional language identifiers from org-babel-shell-names in
+     ob-shell.el: ob-shell is the only babel language using a lambda to put
+     the execution function name together. */
+  pre.src-bash:before  { content: 'bash'; }
+  pre.src-csh:before  { content: 'csh'; }
+  pre.src-ash:before  { content: 'ash'; }
+  pre.src-dash:before  { content: 'dash'; }
+  pre.src-ksh:before  { content: 'ksh'; }
+  pre.src-mksh:before  { content: 'mksh'; }
+  pre.src-posh:before  { content: 'posh'; }
+  /* Additional Emacs modes also supported by the LaTeX listings package */
+  pre.src-ada:before { content: 'Ada'; }
+  pre.src-asm:before { content: 'Assembler'; }
+  pre.src-caml:before { content: 'Caml'; }
+  pre.src-delphi:before { content: 'Delphi'; }
+  pre.src-html:before { content: 'HTML'; }
+  pre.src-idl:before { content: 'IDL'; }
+  pre.src-mercury:before { content: 'Mercury'; }
+  pre.src-metapost:before { content: 'MetaPost'; }
+  pre.src-modula-2:before { content: 'Modula-2'; }
+  pre.src-pascal:before { content: 'Pascal'; }
+  pre.src-ps:before { content: 'PostScript'; }
+  pre.src-prolog:before { content: 'Prolog'; }
+  pre.src-simula:before { content: 'Simula'; }
+  pre.src-tcl:before { content: 'tcl'; }
+  pre.src-tex:before { content: 'TeX'; }
+  pre.src-plain-tex:before { content: 'Plain TeX'; }
+  pre.src-verilog:before { content: 'Verilog'; }
+  pre.src-vhdl:before { content: 'VHDL'; }
+  pre.src-xml:before { content: 'XML'; }
+  pre.src-nxml:before { content: 'XML'; }
+  /* add a generic configuration mode; LaTeX export needs an additional
+     (add-to-list 'org-latex-listings-langs '(conf " ")) in .emacs */
+  pre.src-conf:before { content: 'Configuration File'; }
+
+  table { border-collapse:collapse; }
+  caption.t-above { caption-side: top; }
+  caption.t-bottom { caption-side: bottom; }
+  td, th { vertical-align:top;  }
+  th.org-right  { text-align: center;  }
+  th.org-left   { text-align: center;   }
+  th.org-center { text-align: center; }
+  td.org-right  { text-align: right;  }
+  td.org-left   { text-align: left;   }
+  td.org-center { text-align: center; }
+  dt { font-weight: bold; }
+  .footpara { display: inline; }
+  .footdef  { margin-bottom: 1em; }
+  .figure { padding: 1em; }
+  .figure p { text-align: center; }
+  .inlinetask {
+    padding: 10px;
+    border: 2px solid gray;
+    margin: 10px;
+    background: #ffffcc;
+  }
+  #org-div-home-and-up
+   { text-align: right; font-size: 70%; white-space: nowrap; }
+  textarea { overflow-x: auto; }
+  .linenr { font-size: smaller }
+  .code-highlighted { background-color: #ffff00; }
+  .org-info-js_info-navigation { border-style: none; }
+  #org-info-js_console-label
+    { font-size: 10px; font-weight: bold; white-space: nowrap; }
+  .org-info-js_search-highlight
+    { background-color: #ffff00; color: #000000; font-weight: bold; }
+  .org-svg { width: 90%; }
+  /*]]>*/-->
+</style>
+<script type="text/javascript">
+/*
+@licstart  The following is the entire license notice for the
+JavaScript code in this tag.
+
+Copyright (C) 2012-2018 Free Software Foundation, Inc.
+
+The JavaScript code in this tag is free software: you can
+redistribute it and/or modify it under the terms of the GNU
+General Public License (GNU GPL) as published by the Free Software
+Foundation, either version 3 of the License, or (at your option)
+any later version.  The code is distributed WITHOUT ANY WARRANTY;
+without even the implied warranty of MERCHANTABILITY or FITNESS
+FOR A PARTICULAR PURPOSE.  See the GNU GPL for more details.
+
+As additional permission under GNU GPL version 3 section 7, you
+may distribute non-source (e.g., minimized or compacted) forms of
+that code without the copy of the GNU GPL normally required by
+section 4, provided you include this license notice and a URL
+through which recipients can access the Corresponding Source.
+
+
+@licend  The above is the entire license notice
+for the JavaScript code in this tag.
+*/
+<!--/*--><![CDATA[/*><!--*/
+ function CodeHighlightOn(elem, id)
+ {
+   var target = document.getElementById(id);
+   if(null != target) {
+     elem.cacheClassElem = elem.className;
+     elem.cacheClassTarget = target.className;
+     target.className = "code-highlighted";
+     elem.className   = "code-highlighted";
+   }
+ }
+ function CodeHighlightOff(elem, id)
+ {
+   var target = document.getElementById(id);
+   if(elem.cacheClassElem)
+     elem.className = elem.cacheClassElem;
+   if(elem.cacheClassTarget)
+     target.className = elem.cacheClassTarget;
+ }
+/*]]>*///-->
+</script>
+</head>
+<body>
+<div id="content">
+<p>
+We are going to experiment with our development process, getting away
+from cycles and towards more continuous development. The main idea is
+to get rid of the prod and QA branches and instead deploy directly
+from master, which we keep deployment ready at all times. Development
+occurs on private branches, each branch mapped to a separately
+deploy-able project. Each of projects will be managed more or less
+independently and will include specification, development and
+validation stages. At any particular time we might have anywhere from
+1 to 6 projects in flight each of which continuously pulls from master,
+but does not push back to master until the code is ready to be
+deployed to production. The point at which a project is ready to be
+deployed then takes roughly the place of the QA phase of our old cycle
+mode. At this time, we will do a PR, and go through a series of
+validation steps, to make sure the work meets our standards. This
+validation includes:
+</p>
+
+<ul class="org-ul">
+<li>Code Reviews</li>
+<li>Acceptance Testing (including UI and business tests)</li>
+<li>Automated System testing (to be developed)</li>
+<li>A mini-qa, focused on the feature, by someone not directly involved
+in its implementation</li>
+<li>If possible, deployment to a beta customer.</li>
+</ul>
+
+<p>
+Once these QA gates are passed, we will update the release notes, tag
+a new version, integrate to master and then ideally immediately deploy
+to a customer.
+</p>
+
+<p>
+The main goal with this change is to reduce the latency from feature
+specification to production roll out. We also hope to improve
+throughput by making more efficient use of QA time and reducing idle
+cycles. We expect people to participate in more than one project
+depending on their area of expertise and work load. This does not mean
+people will work on two things at the same time, rather that people
+come into and leave projects as they free up and not when the project
+starts or ends.
+</p>
+
+<p>
+This new scheme will come into full effect when we finish 1.3
+although to some degree 1.3 was already setup like this. At the
+moment, we have 3 major projects, vap/settlments/implied, grid
+settings and speed work, which are in branches of prod. We will merge
+these back to prod and then master this week. We have 3 other projects
+that are in master, the excel API, the server side SDK and the MIC
+administration and enforcement components. These we hope to stabilize
+as soon as possible so that we can merge them to prod and deploy them
+along with the rest. Once we have all this, prod and master should be
+identical and we will delete prod and operate exclusively out of
+master. Please avoid committing anything new to master so we can
+complete this transition as soon as possible.
+</p>
+
+<p>
+Let me know if you have any question
+</p>
+</div>
+<div id="postamble" class="status">
+<p class="author">Author: ppinto</p>
+<p class="date">Created: 2018-10-08 Mon 18:50</p>
+<p class="validation"><a href="http://validator.w3.org/check?uri=referer">Validate</a></p>
+</div>
+</body>
+</html>
@@ -0,0 +1,55 @@
+We are going to experiment with our development process, getting away
+from cycles and towards more continuous development. The main idea is
+to get rid of the prod and QA branches and instead deploy directly
+from master, which we keep deployment ready at all times. Development
+occurs on private branches, each branch mapped to a separately
+deploy-able project. Each of projects will be managed more or less
+independently and will include specification, development and
+validation stages. At any particular time we might have anywhere from
+1 to 6 projects in flight each of which continuously pulls from master,
+but does not push back to master until the code is ready to be
+deployed to production. The point at which a project is ready to be
+deployed then takes roughly the place of the QA phase of our old cycle
+mode. At this time, we will do a PR, and go through a series of
+validation steps, to make sure the work meets our standards. This
+validation includes:
+
+- Code Reviews
+- Acceptance Testing (including UI and business tests)
+- Automated System testing (to be developed)
+- A mini-qa, focused on the feature, by someone not directly involved
+  in its implementation
+- If possible, deployment to a beta customer.
+
+Once these QA gates are passed, we will update the release notes, tag
+a new version, integrate to master and then ideally immediately deploy
+to a customer.
+
+The main goal with this change is to reduce the latency from feature
+specification to production roll out. We also hope to improve
+throughput by making more efficient use of QA time and reducing idle
+cycles. We expect people to participate in more than one project
+depending on their area of expertise and work load. This does not mean
+people will work on two things at the same time, rather that people
+come into and leave projects as they free up and not when the project
+starts or ends.
+
+This new scheme will come into full effect when we finish 1.3
+although to some degree 1.3 was already setup like this. At the
+moment, we have 3 major projects, vap/settlments/implied, grid
+settings and speed work, which are in branches of prod. We will merge
+these back to prod and then master this week. We have 3 other projects
+that are in master, the excel API, the server side SDK and the MIC
+administration and enforcement components. These we hope to stabilize
+as soon as possible so that we can merge them to prod and deploy them
+along with the rest. Once we have all this, prod and master should be
+identical and we will delete prod and operate exclusively out of
+master. Please avoid committing anything new to master so we can
+complete this transition as soon as possible.
+
+Let me know if you have any question
+
+
+
+
+
@@ -1,3 +1,12 @@
+1.3.1
+- Improved server side logging
+- Fixed leaking of market data subscriptions on market view form closing
+
+1.3.0
+- Significantly improved order execution latency
+- Added latency monitoring tools
+- Provided core binding controls to system admins so as to reduce latency jitter
+
 1.1.22
 - Orderbook: Orders in Error status can now be recreated
 - Orderbook: Orders containing child orders will now be collapsed by default.
@@ -121,7 +121,7 @@ namespace mbts.spark {

         public Row(TimeAndSalesVM parent, SecurityImmutable security, TradeMessageImmutable trade) : base(security) {
            _trade_key = trade.Key;
-            time = trade.Time.lldatetime();
+            time = ((ulong)trade.Time).lldatetime().ToLocalTime();
            price = trade.Mantissa * Math.Pow(10, trade.Exponent);
            size = trade.Size;
            aggressor_side = trade.AggressorSide;
@@ -24,13 +24,15 @@ namespace mbts.spark
            this.child_order_types = new InlineUserProperty<IReadOnlyCollection<CustomOneofCase>>(child_order_types, true);
            cross_threshold = new InlineUserProperty<bts.spark.QuantityMeasure>(
                              new bts.spark.QuantityMeasure() { Ratio = 1.0, Absolute = 1}, false);
-            max_chase_ticks = new InlineUserProperty<uint>(10, false);
+            max_chase_ticks = new InlineUserProperty<uint>(2, false);
+            trade_toward_behavior = new InlineUserProperty<LeanOrderDefinition.Types.OnToward>(LeanOrderDefinition.Types.OnToward.Cancel, false);
         }
         public readonly InlineUserProperty<CustomOneofCase> selected_child_order_type;
         public readonly InlineUserProperty<IReadOnlyCollection<CustomOneofCase>> child_order_types;
         
         public readonly InlineUserProperty<bts.spark.QuantityMeasure> cross_threshold;
         public readonly InlineUserProperty<uint> max_chase_ticks;
+         public readonly InlineUserProperty<LeanOrderDefinition.Types.OnToward> trade_toward_behavior;
      }

      class LeanOrderVM : OrderTypeVMBase<LeanOrderDefinition>, IHierarchicalOrderTypeVM, IDisposable 
@@ -44,7 +46,8 @@ namespace mbts.spark
         IDisposable _dispose, _child_order_errors_sub;
         
         public bts.spark.QuantityMeasure cross_threshold { get; set; } = new bts.spark.QuantityMeasure() { Ratio = 1.0, Absolute = 1 };
-         public uint max_chase_ticks { get; set; } = 10;
+         public uint max_chase_ticks { get; set; } = 2;
+         public LeanOrderDefinition.Types.OnToward trade_toward_behavior { get; set; } = LeanOrderDefinition.Types.OnToward.Cancel;      

         public LeanOrderVM(OrdersModel model, IObservable<SecurityMD> security_md_source)
            : this(model, security_md_source, security_md_source.Select(smd =>
@@ -77,6 +80,11 @@ namespace mbts.spark
                     max_chase_ticks = p.max_chase_ticks.value;
                     RaisePropertyChanged(nameof(max_chase_ticks));
                  }
+                  if (p.trade_toward_behavior.force_override)
+                  {
+                     trade_toward_behavior = p.trade_toward_behavior.value;
+                     RaisePropertyChanged(nameof(trade_toward_behavior));
+                  }
                  _inline_props = p;
                  _validate();
               })
@@ -94,10 +102,13 @@ namespace mbts.spark
               var i = factory.lean_order();
               i.CrossThreshold = cross_threshold;
               i.MaxChaseTicks = max_chase_ticks;
+               i.TradeTowardBehavior = trade_toward_behavior;
               child_order.assign_to(i.Child);
               value.Lean = i;
            }
-         }         
+         }
+
+         public IEnumerable<LeanOrderDefinition.Types.OnToward> trade_toward_types => EnumHelper.get_values_except<LeanOrderDefinition.Types.OnToward>();         

         public ICollection<IOrderTypeVM> child_order_types 
         {
@@ -66,15 +66,73 @@ namespace mbts.utils {

      class AnonymousConnectable<TResult> : IConnectableObservable<TResult> {
         readonly IObservable<TSource> _source;
-         readonly KeyDrivenObservableSeries<TSource, TResult> _driver;
+         readonly Func<TSource, IObservable<TResult>> _select_next;
+         List<LazySubscription> _lazy_subs = new List<LazySubscription>();
+         KeyDrivenObservableSeries<TSource, TResult> _driver;
+         IDisposable _driver_sub;
+         bool _is_connected = false;

         public AnonymousConnectable(IObservable<TSource> source, Func<TSource, IObservable<TResult>> select_next) {
            _source = source;
-            _driver = new KeyDrivenObservableSeries<TSource, TResult>(select_next);
+            _select_next = select_next;
         }

-         public IDisposable Connect() => _source.Subscribe(_driver);
-         public IDisposable Subscribe(IObserver<TResult> observer) => _driver.Subscribe(observer);
+         // idempotent
+         public IDisposable Connect() {
+            lock (this) {
+               if (!_is_connected) {
+                  _driver = new KeyDrivenObservableSeries<TSource, TResult>(_select_next);
+                  foreach (var ls in _lazy_subs)
+                     ls.attach();
+                  _lazy_subs.Clear();
+                  _driver_sub = _source.Subscribe(_driver);
+                  _is_connected = true;
+               }
+            }
+            return Disposable.Create(_deconnect);
+         }
+
+         void _deconnect() {
+            lock (this) {
+               if (_is_connected) {
+                  _driver_sub.Dispose();
+                  _driver.Dispose();
+                  _is_connected = false;
+               }
+            }
+         }
+
+         public IDisposable Subscribe(IObserver<TResult> observer) {
+            lock (this) {
+               if (_is_connected)
+                  return _driver.Subscribe(observer);
+               var ls = new LazySubscription(this, observer);
+               _lazy_subs.Add(ls);
+               return ls;
+            }
+            throw new InvalidOperationException();
+         }
+
+         class LazySubscription : IDisposable {
+            readonly IObserver<TResult> _observer;
+            readonly AnonymousConnectable<TResult> _owner;
+            IDisposable _sub;
+
+            public LazySubscription(AnonymousConnectable<TResult> owner, IObserver<TResult> observer) {
+               _owner = owner;
+               _observer = observer;
+            }
+
+            public void attach() =>
+               _sub = _owner._driver.Subscribe(_observer);
+
+            public void Dispose() {
+               lock (_owner) {
+                  Disposable.dispose(ref _sub);
+                  _owner._lazy_subs.Remove(this);
+               }
+            }
+         }
      }
   }

@@ -222,6 +222,8 @@
                          <RowDefinition Height="4" />
                          <RowDefinition Height="Auto"/>
                          <RowDefinition Height="4" />
+                          <RowDefinition Height="Auto"/>
+                          <RowDefinition Height="4" />                          
                          <RowDefinition/>
                        </Grid.RowDefinitions>
                        <Grid.ColumnDefinitions>
@@ -233,9 +235,11 @@
                        <uag:QuantityMeasureControl Grid.Row="1" Grid.Column="2" QuantityMeasure="{Binding cross_threshold}"/>
                        <TextBlock Grid.Row="3" VerticalAlignment="Center">Max Chase Ticks</TextBlock>
                        <c:TickUpDownUInt Grid.Row="3" Grid.Column="2" Value="{Binding max_chase_ticks}"/>
-                        <TextBlock Grid.Row="5">Child</TextBlock>
+                        <TextBlock Grid.Row="5">On Toward</TextBlock>
+                        <ComboBox Grid.Row="5" Grid.Column="2" ItemsSource="{Binding trade_toward_types}" ItemTemplate="{StaticResource EnumDescriptionTemplate}" SelectedItem="{Binding trade_toward_behavior}"/>                        
+                        <TextBlock Grid.Row="7">Child</TextBlock>
                        <uag:OrderDetailsPopupControl
-                          Grid.Row="5" Grid.Column="2"
+                          Grid.Row="7" Grid.Column="2"
                          IsOpen="{Binding child_order.editable_properties_open, FallbackValue=False}"
                          ItemsSource="{Binding child_order_types}"
                          SelectedItem="{Binding child_order}"
@@ -0,0 +1,37 @@
+using bts.api.core;
+using bts.api.persistence;
+using bts.api.user;
+using bts.test;
+using mbts.persist;
+using mbts.ui.shell;
+using System;
+using System.Linq;
+
+namespace bts.settings {
+   public class SettingsManagerTool {
+      public static void run() {
+         var sc = new ServiceContext();
+         var users = sc.locate<UserService>();
+         var persist = sc.locate<PersistenceService>("settings");
+         var user_list = users.get_users(new GetUsersRequest());
+         var user_names = user_list.Users.Select(u => u.Login);
+         var from = ConsoleEx.prompt("From User", user_names);
+         var from_persist = new UserUISettings(user_list.Users.First(u => u.Login == from).Id, persist, null);
+
+         var workspaces = from_persist.query<WorkspaceDefinition>(WorkspacesModel.WORKSPACE_NAMESPACE, ".*").Result;
+         var selected_ws = ConsoleEx.prompt("Choose workspace:", workspaces.Select(w => w.key));
+
+         var to = ConsoleEx.prompt("To User", user_names);
+
+         var name = ConsoleEx.input_prompt("Rename workspace?", selected_ws);
+
+         var confirm = ConsoleEx.prompt($"copy workspace '{selected_ws}' from {from} to {to} and name it '{name}'. Confirm?", "Yes", "No");
+         if (confirm == "Yes") {
+            var ws_blob = workspaces.First(w => w.key == selected_ws).value;
+            var to_persist = new UserUISettings(user_list.Users.First(u => u.Login == to).Id, persist, null);
+            to_persist.set(WorkspacesModel.WORKSPACE_NAMESPACE, name, ws_blob).Wait();
+            Console.WriteLine("Done");
+         }
+      }
+   }
+}
@@ -113,5 +113,7 @@ namespace mbts.co2
      [DllImport("libspark")]
      internal static extern void co2_get_zero_and_nan_book(ref NativePriceLevel nps);

+      [DllImport("libspark")]
+      internal static extern void co2_get_sample_datetime(ref utils.LLDateTime now, ref utils.LLDateTime utc_now);
   }
 }
@@ -1,4 +1,6 @@
 using System;
+using System.Collections.Generic;
+using System.Linq;

 namespace bts.test {
   public static class ConsoleEx {
@@ -21,17 +23,34 @@ namespace bts.test {
      }

      public static T prompt<T>(string prompt, params T[] options) {
-         Console.WriteLine(prompt);
-         for (int x = 1; x <= options.Length; ++x) {
+         return ConsoleEx.prompt(prompt, (IEnumerable<T>)options);
+      }
+      public static T prompt<T>(string prompt, IEnumerable<T> options) {
+         WriteLine(prompt, ConsoleColor.Yellow);
+         for (int x = 0; x < options.Count(); ++x) {
            Write($"{x}. ", ConsoleColor.Yellow);
-            WriteLine(options[x].ToString());
+            WriteLine(options.ElementAt(x).ToString());
         }
-         int response;
-         do {
-            Int32.TryParse(Console.ReadKey().ToString(), out response);
-            if (response > options.Length) response = 0;
-         } while (response == 0);
-         return options[response];
+         int response = 0;
+         bool success = false;
+         while(!success) {
+            var key = Console.ReadKey();
+            success = Int32.TryParse(key.KeyChar.ToString(), out response);
+            if (response > options.Count()) success = false;
+         }
+         var value = options.ElementAt(response);
+         Console.WriteLine(" >> " + value.ToString());
+         Console.WriteLine();
+         return value;
+      }
+
+      public static string input_prompt(string prompt, string default_result) {
+         Write(prompt + " (", ConsoleColor.Yellow);
+         Write(default_result, ConsoleColor.White);
+         Write(") ", ConsoleColor.Yellow);
+         var result = Console.ReadLine();
+         if (string.IsNullOrWhiteSpace(result)) return default_result;
+         return result;
      }


@@ -43,6 +43,9 @@ namespace bts.test {
            if (ue.InnerException is AssertFailture af) {
               result.Message = $"Assertion Failure:\n{af.Message}\nLocation:{af._filename}:{af._line}";
            }
+            else if (ue.InnerException != null) {
+               result.Message = $"Unexpected error: {ue.InnerException}";
+            }
            else {
               result.Message = $"Unexpected error: {ue.Message}\n\n{ue.StackTrace}";
            }
@@ -1,7 +1,6 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
-using System.Runtime.InteropServices;
 using System.Text;

 namespace mbts {
@@ -24,20 +23,14 @@ namespace mbts {
         return range.Skip(index).First();
      }

-      [DllImport("libspark")]
-      private static extern void ll_to_parts(UInt64 ll_time, out int year, out int month, out int day, out int hour, out int minute, out int second, out int ms);
-
      public static DateTime lldatetime(this ulong ll) {
-         int year, month, day, hour, minute, second, ms;
-         ll_to_parts(ll, out year, out month, out day, out hour, out minute, out second, out ms);
-         if (year > 0) {
-            return new DateTime(year, month, day, hour, minute, second, ms, DateTimeKind.Utc).ToLocalTime();
-         }
-         return DateTime.MinValue;
+         // Warning: Do not remove these consts, or try to simplify this method.
+         // System.DateTime() requires bits 63-64 to be empty in the "DateTime(long ticks)" constructor.
+         // The only way to set the DateTimeKind is through the "DateTime(long ticks, DateTimeKind kind)" constructor.
+         // (Yes, it is annoying. Internally, System.DateTime stores the DateTimeKind in bits 63-64 - in the exact format we already have!!!)
+         const ulong ticks_mask = 0x3FFFFFFFFFFFFFFF;
+         const int kind_shift = 62;
+         return new DateTime((long)(ll & ticks_mask), (DateTimeKind)(ll >> kind_shift));
      }
-
-      public static DateTime lldatetime(this long ll) { return ((ulong)ll).lldatetime(); }
-
   }
-}
-
+}
@@ -18,6 +18,7 @@ namespace mbts.utils
      }

      public static implicit operator DateTime (LLDateTime time) { return time.to_datetime(); }
+      public static implicit operator LLDateTime (DateTime time) { return new LLDateTime((ulong)time.Ticks); }

      public static LLDateTime now() {
         return new LLDateTime(BTSNativeMethods.ll_now());
@@ -56,6 +56,11 @@ namespace bts.utils {
         if (value > max) return max;
         return value;
      }
+      public static long within_range(long min, long max, long value) {
+         if (value < min) return min;
+         if (value > max) return max;
+         return value;
+      }
      public static double within_range(double min, double max, double value) {
         if (value < min) return min;
         if (value > max) return max;
@@ -112,7 +112,7 @@ namespace mbts.co2.tests
         Assert.IsTrue(last.price == 4.2);
         Assert.IsTrue(last.size == 101);
         Assert.IsTrue(entry.total_volume == 12345);
-         Assert.IsTrue(compare_date(entry.last_trade_time, DateTime.Now));
+         Assert.IsTrue(compare_date(entry.last_trade_time, DateTime.UtcNow));
         //// Settlement
         Assert.AreEqual(entry.settlement_price, 3.3);
         Assert.IsTrue(entry.settlement_date == new DateTime(2015, 9, 3));
@@ -251,6 +251,25 @@ namespace mbts.co2.tests
         Assert.IsTrue(double.IsNaN(npl.bid.price));
      }

+      [Test]
+      public void lldatetime_test() {
+         utils.LLDateTime now = DateTime.MinValue, utc_now = DateTime.MinValue;
+         Co2NativeTestMethods.co2_get_sample_datetime(ref now, ref utc_now);
+         DateTime dt_now = now;
+         DateTime dt_utc_now = utc_now;
+         Assert.AreEqual(dt_now.ToShortDateString(), DateTime.Now.ToShortDateString());
+         Assert.AreEqual(dt_utc_now.ToShortDateString(), DateTime.UtcNow.ToShortDateString());
+         void assert_about_equal(int a, int b) {
+            Assert.That(Math.Abs(a - b) <= 1);
+         }

+         assert_about_equal(dt_now.Hour, DateTime.Now.Hour);
+         assert_about_equal(dt_now.Minute, DateTime.Now.Minute);
+         assert_about_equal(dt_now.Second, DateTime.Now.Second);
+
+         assert_about_equal(dt_utc_now.Hour, DateTime.UtcNow.Hour);
+         assert_about_equal(dt_utc_now.Minute, DateTime.UtcNow.Minute);
+         assert_about_equal(dt_utc_now.Second, DateTime.UtcNow.Second);
+      }
   }
 }
@@ -0,0 +1,20 @@
+using ManyConsole;
+using System.Reflection;
+
+namespace bts {
+
+   public class SettinsManagerCommand : ConsoleCommand {
+      public SettinsManagerCommand() {
+         IsCommand("SettingsManager", "Copy workspaces around");
+      }
+
+      public override int Run(string[] remainingArguments) {
+         var asm = Assembly.Load("spark");
+         var type = asm.GetType("bts.settings.SettingsManagerTool");
+         var run = type.GetMethod("run", BindingFlags.Static | BindingFlags.Public);
+         run.Invoke(null, new object[0]);
+         return 0;
+      }
+   }
+
+}
@@ -1 +1 @@
-set(sources /home/ppinto/Dropbox/bts/spark/src/data_files/files/Onixs.BTS.lic;/home/ppinto/Dropbox/bts/spark/src/data_files/files/Onixs.BTS.lic.orig;/home/ppinto/Dropbox/bts/spark/src/data_files/files/cert_secure_logon.txt)
+set(sources files/Onixs.BTS.lic;files/cert_secure_logon.txt;files/perf_trace/compiled.css;files/perf_trace/index.html;files/perf_trace/jquery-3.3.1.min.js;files/perf_trace/main.js;files/perf_trace/plotly-latest.min.js)
@@ -4,7 +4,7 @@ project(data_files)
 include(${CMAKE_SOURCE_DIR}/tools/safe_dep_glob.cmake)

 # DEPENDENCIES
-file (GLOB_RECURSE To_Copy ABSOLUTE ${CMAKE_CURRENT_SOURCE_DIR} files/*)
+file (GLOB_RECURSE To_Copy RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} files/*)
 update_deps_file("${To_Copy}")
 # END DEPENDENCIES

@@ -0,0 +1,63 @@
+<html>
+
+<head>
+    <title>BTS Spark Socket To Socket Requote Times</title>
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <link rel="stylesheet" href="/perf_trace/compiled.css">
+    <script src="https://cdn.polyfill.io/v2/polyfill.js?features=es6"></script>
+    <style type="text/css">
+        .summary-block {
+            margin: 2em 1em;
+        }
+        .summary-block > ul {
+            margin: 0em 1em;
+            display:inline-block;
+            vertical-align: top;
+        }
+        .histogram-div {
+            width: 550px;
+            height:350px;
+            display:inline-block
+        }
+        .clear {
+            clear:both;
+        }
+    </style>
+</head>
+
+<body>
+    <div>
+        <div class="columns is-fullheight">
+            <div class="column is-narrow is-launcher-sidebar-menu">
+                <aside class="menu">
+                    <ul class="menu-list">
+                        <li>&nbsp;&nbsp;</li>
+                    </ul>
+                </aside>
+            </div>
+            <div class="column is-main-content">
+                <div class="columns is-fullheight">
+                    <!--div class="column is-narrow is-sidebar-menu">
+                    </div-->
+                    <div class="column is-main-content">
+                        <!--div class="tabs is-boxed">
+                            <ul>
+                                <li class="is-active"><a>Stats</a></li>
+                            </ul>
+                        </div-->
+                        <div style="margin:2em 0 0 2em;"><!--div class="is-tab-content" is-tab-content is shorter than the screen to accommodate tabs at the top -->
+                            <h1 class="is-size-2">Socket To Socket Requote Times</h1>
+                            <div id="output"></div>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+    </div>
+    <script src="/perf_trace/jquery-3.3.1.min.js"></script>
+    <script src="/perf_trace/plotly-latest.min.js"></script>
+    <script src="/perf_trace/main.js"></script>
+</body>
+
+</html>
@@ -0,0 +1,2 @@
+/*! jQuery v3.3.1 | (c) JS Foundation and other contributors | jquery.org/license */
+!function(e,t){"use strict";"object"==typeof module&&"object"==typeof module.exports?module.exports=e.document?t(e,!0):function(e){if(!e.document)throw new Error("jQuery requires a window with a document");return t(e)}:t(e)}("undefined"!=typeof window?window:this,function(e,t){"use strict";var n=[],r=e.document,i=Object.getPrototypeOf,o=n.slice,a=n.concat,s=n.push,u=n.indexOf,l={},c=l.toString,f=l.hasOwnProperty,p=f.toString,d=p.call(Object),h={},g=function e(t){return"function"==typeof t&&"number"!=typeof t.nodeType},y=function e(t){return null!=t&&t===t.window},v={type:!0,src:!0,noModule:!0};function m(e,t,n){var i,o=(t=t||r).createElement("script");if(o.text=e,n)for(i in v)n[i]&&(o[i]=n[i]);t.head.appendChild(o).parentNode.removeChild(o)}function x(e){return null==e?e+"":"object"==typeof e||"function"==typeof e?l[c.call(e)]||"object":typeof e}var b="3.3.1",w=function(e,t){return new w.fn.init(e,t)},T=/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g;w.fn=w.prototype={jquery:"3.3.1",constructor:w,length:0,toArray:function(){return o.call(this)},get:function(e){return null==e?o.call(this):e<0?this[e+this.length]:this[e]},pushStack:function(e){var t=w.merge(this.constructor(),e);return t.prevObject=this,t},each:function(e){return w.each(this,e)},map:function(e){return this.pushStack(w.map(this,function(t,n){return e.call(t,n,t)}))},slice:function(){return this.pushStack(o.apply(this,arguments))},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},eq:function(e){var t=this.length,n=+e+(e<0?t:0);return this.pushStack(n>=0&&n<t?[this[n]]:[])},end:function(){return this.prevObject||this.constructor()},push:s,sort:n.sort,splice:n.splice},w.extend=w.fn.extend=function(){var e,t,n,r,i,o,a=arguments[0]||{},s=1,u=arguments.length,l=!1;for("boolean"==typeof a&&(l=a,a=arguments[s]||{},s++),"object"==typeof a||g(a)||(a={}),s===u&&(a=this,s--);s<u;s++)if(null!=(e=arguments[s]))for(t in e)n=a[t],a!==(r=e[t])&&(l&&r&&(w.isPlainObject(r)||(i=Array.isArray(r)))?(i?(i=!1,o=n&&Array.isArray(n)?n:[]):o=n&&w.isPlainObject(n)?n:{},a[t]=w.extend(l,o,r)):void 0!==r&&(a[t]=r));return a},w.extend({expando:"jQuery"+("3.3.1"+Math.random()).replace(/\D/g,""),isReady:!0,error:function(e){throw new Error(e)},noop:function(){},isPlainObject:function(e){var t,n;return!(!e||"[object Object]"!==c.call(e))&&(!(t=i(e))||"function"==typeof(n=f.call(t,"constructor")&&t.constructor)&&p.call(n)===d)},isEmptyObject:function(e){var t;for(t in e)return!1;return!0},globalEval:function(e){m(e)},each:function(e,t){var n,r=0;if(C(e)){for(n=e.length;r<n;r++)if(!1===t.call(e[r],r,e[r]))break}else for(r in e)if(!1===t.call(e[r],r,e[r]))break;return e},trim:function(e){return null==e?"":(e+"").replace(T,"")},makeArray:function(e,t){var n=t||[];return null!=e&&(C(Object(e))?w.merge(n,"string"==typeof e?[e]:e):s.call(n,e)),n},inArray:function(e,t,n){return null==t?-1:u.call(t,e,n)},merge:function(e,t){for(var n=+t.length,r=0,i=e.length;r<n;r++)e[i++]=t[r];return e.length=i,e},grep:function(e,t,n){for(var r,i=[],o=0,a=e.length,s=!n;o<a;o++)(r=!t(e[o],o))!==s&&i.push(e[o]);return i},map:function(e,t,n){var r,i,o=0,s=[];if(C(e))for(r=e.length;o<r;o++)null!=(i=t(e[o],o,n))&&s.push(i);else for(o in e)null!=(i=t(e[o],o,n))&&s.push(i);return a.apply([],s)},guid:1,support:h}),"function"==typeof Symbol&&(w.fn[Symbol.iterator]=n[Symbol.iterator]),w.each("Boolean Number String Function Array Date RegExp Object Error Symbol".split(" "),function(e,t){l["[object "+t+"]"]=t.toLowerCase()});function C(e){var t=!!e&&"length"in e&&e.length,n=x(e);return!g(e)&&!y(e)&&("array"===n||0===t||"number"==typeof t&&t>0&&t-1 in e)}var E=function(e){var t,n,r,i,o,a,s,u,l,c,f,p,d,h,g,y,v,m,x,b="sizzle"+1*new Date,w=e.document,T=0,C=0,E=ae(),k=ae(),S=ae(),D=function(e,t){return e===t&&(f=!0),0},N={}.hasOwnProperty,A=[],j=A.pop,q=A.push,L=A.push,H=A.slice,O=function(e,t){for(var n=0,r=e.length;n<r;n++)if(e[n]===t)return n;return-1},P="checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|ismap|loop|multiple|open|readonly|required|scoped",M="[\\x20\\t\\r\\n\\f]",R="(?:\\\\.|[\\w-]|[^\0-\\xa0])+",I="\\["+M+"*("+R+")(?:"+M+"*([*^$|!~]?=)"+M+"*(?:'((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\"|("+R+"))|)"+M+"*\\]",W=":("+R+")(?:\\((('((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\")|((?:\\\\.|[^\\\\()[\\]]|"+I+")*)|.*)\\)|)",$=new RegExp(M+"+","g"),B=new RegExp("^"+M+"+|((?:^|[^\\\\])(?:\\\\.)*)"+M+"+$","g"),F=new RegExp("^"+M+"*,"+M+"*"),_=new RegExp("^"+M+"*([>+~]|"+M+")"+M+"*"),z=new RegExp("="+M+"*([^\\]'\"]*?)"+M+"*\\]","g"),X=new RegExp(W),U=new RegExp("^"+R+"$"),V={ID:new RegExp("^#("+R+")"),CLASS:new RegExp("^\\.("+R+")"),TAG:new RegExp("^("+R+"|[*])"),ATTR:new RegExp("^"+I),PSEUDO:new RegExp("^"+W),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+M+"*(even|odd|(([+-]|)(\\d*)n|)"+M+"*(?:([+-]|)"+M+"*(\\d+)|))"+M+"*\\)|)","i"),bool:new RegExp("^(?:"+P+")$","i"),needsContext:new RegExp("^"+M+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)Line truncated
@@ -0,0 +1,164 @@
+$( document ).ready(function (){
+  let urlParams = new URLSearchParams(window.location.search);
+  num_rows = urlParams.get('r');
+  let show_rows = urlParams.get('showrows');
+
+  let trace_data_url = "/perf_trace_data";
+  if(num_rows != null) {
+    trace_data_url += "/" + num_rows;
+  }
+
+    $.ajax({
+      url: trace_data_url,
+      dataType: 'json',
+      context: document.body,
+      error: ajax_err,
+      crossDomain: true
+    }).done(function(result) {
+      if(show_rows != null) {
+        list_rows(show_rows, result);
+      }
+      else {
+        if(result.length > 0){
+          let summary_info = "<h1 class='is-size-3'><b>Host:</b> " + result[0].host + "</h1>";
+          $("#output").html(summary_info);
+          $(result).each(display_session);
+        }
+      }
+
+    });
+
+});
+
+function list_rows(show_rows, data) {
+  $.each(data, function(index, value){
+    if(value.session_name === show_rows){
+      let summary_info = "<table class='table bts-data-table'>";
+      summary_info += "<thead><tr><th>Trace Time</th>" +
+                        "<th>Order Key</th>" +
+                        "<th>MD</th>" +
+                        "<th>Spreader</th>" +
+                        "<th>Fix</th>" +
+                        "<th>Total</th></tr></thead><tbody>";
+
+      $.each(value.rows, function(key, val) {
+        summary_info += "<tr>";
+        console.log(val);
+
+        summary_info += "<td>" + val.trace_time + "</td>" +
+                        "<td>" + val.order_key + "</td>" +
+                        "<td>" + val.md + "</td>" +
+                        "<td>" + val.spreader + "</td>" +
+                        "<td>" + val.fix + "</td>" +
+                        "<td>" + val.total + "</td>";
+
+
+        summary_info += "</tr>";
+      });
+      summary_info += "</tbody></table>";
+      $("#output").html(summary_info);
+      return false;
+    }
+
+  });
+}
+
+
+function ajax_err (jqXHR, textStatus,errorThrown) {
+   $("#output").html("<h3 class='is-size-3 has-text-danger'>Error loading perf_trace_data</h3>" +
+                    "<p>" + textStatus + "</p>" +
+                    "<pre>" + errorThrown + "</pre>"
+                      );
+
+   console.log(textStatus);
+   console.log(errorThrown);
+}
+
+function first_upper (str)
+{
+  let ret_str = str[0].toUpperCase() + str.substring(1, str.length);
+  return ret_str
+}
+
+function generate_histogram(id, data)
+{
+  var trace = {
+      x: data,
+      type: 'histogram',
+      marker: {
+          color: 'rgb(65,105,170)'
+        },
+    };
+
+  var plot_data = [trace];
+
+  Plotly.newPlot(id, plot_data);
+}
+
+function display_session (index, value) {
+   //console.log(value);
+   let session_info = "<ul class=''>";
+   session_info += "<li id='" + value.session_name + "' class='is-size-5'><b>Session:</b> " + value.session_name + "</li>";
+   session_info += "<li><b>Build Type:</b> " + value.build_type + "</li>";
+   session_info += "<li><b>Version:</b> " + value.version + "</li>";
+   if(num_rows != null) {
+      session_info += "<li><a href='?showrows=" + value.session_name + "&r=" + num_rows + "'><b>Rows:</b> " + value.n_rows + "</a></li>";
+   }
+   else {
+      session_info += "<li><a href='?showrows=" + value.session_name + "'><b>Rows:</b> " + value.n_rows + "</a></li>";
+   }
+
+   $(value.summaries).each(function(i,v){
+      if(v.value === "total") {
+         session_info += "<li class='summary-block'><ul>";
+         session_info += "<li><b>" + first_upper(v.value) + "</b></li>";
+         session_info += "<li>Min: " + v.min + "</li>";
+         session_info += "<li>Median: " + v.median + "</li>";
+         session_info += "<li>Max: " + v.max + "</li>";
+         session_info += "<li>99%: " + v.ninenty_nine + "</li>";
+         session_info += "<li>Standard Deviation: " + v.stdev + "</li>";
+         session_info += "</ul>";
+         session_info += "<div id='" + value.session_name + "-" + v.value + "' class='histogram-div'></div>"
+         session_info += "</li>";
+         return false
+      }
+   });
+
+   $(value.summaries).each(function(i,v){
+      if(v.value !== "total") {
+         session_info += "<li class='summary-block'><ul>";
+         session_info += "<li><b>" + first_upper(v.value) + "</b></li>";
+         session_info += "<li>Min: " + v.min + "</li>";
+         session_info += "<li>Median: " + v.median + "</li>";
+         session_info += "<li>Max: " + v.max + "</li>";
+         session_info += "<li>99%: " + v.ninenty_nine + "</li>";
+         session_info += "<li>Standard Deviation: " + v.stdev + "</li>";
+         session_info += "</ul>";
+         session_info += "<div id='" + value.session_name + "-" + v.value + "' class='histogram-div'></div>"
+         session_info += "</li>";
+      }
+   });
+
+   session_info += "</ul><hr class='clear'>";
+   $("#output").append(session_info);
+
+   let all_data = {};
+
+   $(value.rows).each(function(i, row) {
+      $.each(row, function(name, v) {
+          if(all_data[name] === undefined){
+            all_data[name] = [v];
+          }
+          else {
+            all_data[name].push(v);
+          }
+      });
+   });
+
+   //console.log(all_data);
+
+   $(value.summaries).each(function(i,v){
+      generate_histogram(value.session_name + "-" + v.value, all_data[v.value]);
+   });
+
+}
@@ -0,0 +1,7 @@
+/**
+* plotly.js v1.41.3
+* Copyright 2012-2018, Plotly, Inc.
+* All rights reserved.
+* Licensed under the MIT license
+*/
+!function(t){if("object"==typeof exports&&"undefined"!=typeof module)module.exports=t();else if("function"==typeof define&&define.amd)define([],t);else{("undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof self?self:this).Plotly=t()}}(function(){return function(){return function t(e,r,n){function i(o,s){if(!r[o]){if(!e[o]){var l="function"==typeof require&&require;if(!s&&l)return l(o,!0);if(a)return a(o,!0);var c=new Error("Cannot find module '"+o+"'");throw c.code="MODULE_NOT_FOUND",c}var u=r[o]={exports:{}};e[o][0].call(u.exports,function(t){return i(e[o][1][t]||t)},u,u.exports,t,e,r,n)}return r[o].exports}for(var a="function"==typeof require&&require,o=0;o<n.length;o++)i(n[o]);return i}}()({1:[function(t,e,r){"use strict";var n=t("../src/lib"),i={"X,X div":"direction:ltr;font-family:'Open Sans', verdana, arial, sans-serif;margin:0;padding:0;","X input,X button":"font-family:'Open Sans', verdana, arial, sans-serif;","X input:focus,X button:focus":"outline:none;","X a":"text-decoration:none;","X a:hover":"text-decoration:none;","X .crisp":"shape-rendering:crispEdges;","X .user-select-none":"-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;-o-user-select:none;user-select:none;","X svg":"overflow:hidden;","X svg a":"fill:#447adb;","X svg a:hover":"fill:#3c6dc5;","X .main-svg":"position:absolute;top:0;left:0;pointer-events:none;","X .main-svg .draglayer":"pointer-events:all;","X .cursor-default":"cursor:default;","X .cursor-pointer":"cursor:pointer;","X .cursor-crosshair":"cursor:crosshair;","X .cursor-move":"cursor:move;","X .cursor-col-resize":"cursor:col-resize;","X .cursor-row-resize":"cursor:row-resize;","X .cursor-ns-resize":"cursor:ns-resize;","X .cursor-ew-resize":"cursor:ew-resize;","X .cursor-sw-resize":"cursor:sw-resize;","X .cursor-s-resize":"cursor:s-resize;","X .cursor-se-resize":"cursor:se-resize;","X .cursor-w-resize":"cursor:w-resize;","X .cursor-e-resize":"cursor:e-resize;","X .cursor-nw-resize":"cursor:nw-resize;","X .cursor-n-resize":"cursor:n-resize;","X .cursor-ne-resize":"cursor:ne-resize;","X .cursor-grab":"cursor:-webkit-grab;cursor:grab;","X .modebar":"position:absolute;top:2px;right:2px;z-index:1001;background:rgba(255,255,255,0.7);","X .modebar--hover":"opacity:0;-webkit-transition:opacity 0.3s ease 0s;-moz-transition:opacity 0.3s ease 0s;-ms-transition:opacity 0.3s ease 0s;-o-transition:opacity 0.3s ease 0s;transition:opacity 0.3s ease 0s;","X:hover .modebar--hover":"opacity:1;","X .modebar-group":"float:left;display:inline-block;box-sizing:border-box;margin-left:8px;position:relative;vertical-align:middle;white-space:nowrap;","X .modebar-group:first-child":"margin-left:0px;","X .modebar-btn":"position:relative;font-size:16px;padding:3px 4px;cursor:pointer;line-height:normal;box-sizing:border-box;","X .modebar-btn svg":"position:relative;top:2px;","X .modebar-btn path":"fill:rgba(0,31,95,0.3);","X .modebar-btn.active path,X .modebar-btn:hover path":"fill:rgba(0,22,72,0.5);","X .modebar-btn.modebar-btn--logo":"padding:3px 1px;","X .modebar-btn.modebar-btn--logo path":"fill:#447adb !important;","X [data-title]:before,X [data-title]:after":"position:absolute;-webkit-transform:translate3d(0, 0, 0);-moz-transform:translate3d(0, 0, 0);-ms-transform:translate3d(0, 0, 0);-o-transform:translate3d(0, 0, 0);transform:translate3d(0, 0, 0);display:none;opacity:0;z-index:1001;pointer-events:none;top:110%;right:50%;","X [data-title]:hover:before,X [data-title]:hover:after":"display:block;opacity:1;","X [data-title]:before":"content:'';position:absolute;background:transparent;border:6px solid transparent;z-index:1002;margin-top:-12px;border-bottom-color:#69738a;margin-right:-6px;","X [data-title]:after":"content:attr(data-title);background:#69738a;color:white;padding:8px 10px;font-size:12px;line-height:12px;white-space:nowrap;margin-right:-18px;border-radius:2px;","X .select-outline":"fill:none;stroke-width:1;shape-rendering:crispEdges;","X .select-outline-1":"stroke:white;","X .select-outline-2":"stroke:black;stroke-dasharray:2px 2px;",Y:"font-family:'Open Sans';position:fixed;top:50px;right:20px;z-index:10000;font-size:10pt;max-width:180px;","Y p":"margin:0;","Y .notifier-note":"min-width:180px;max-width:250px;border:1px solid #fff;z-index:3000;margin:0;background-color:#8c97af;background-color:rgba(140,151,175,0.9);color:#fff;padding:10px;overflow-wrap:break-word;word-wrap:break-word;-ms-hyphens:auto;-webkit-hyphens:auto;hyphens:auto;","Y .notifier-close":"color:#fff;opacity:0.8;float:right;padding:0 5px;background:none;border:none;font-size:20px;font-weight:bold;line-height:20px;","Y .notifier-close:hover":"color:#444;text-decoration:none;cursor:pointer;"};for(var a in i){var o=a.replace(/^,/," ,").replace(/X/g,".js-plotly-plot .plotly").replace(/Y/g,".plotly-notifier");n.addStyleRule(o,i[a])}},{"../src/lib":693}],2:[function(t,e,r){"use strict";e.exports={undo:{width:857.1,height:1e3,path:"m857 350q0-87-34-166t-91-137-137-92-166-34q-96 0Line truncated
@@ -1 +1 @@
-set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/services/UserService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/Executor.conf;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cpp_config.conf;config/defaults/dynamic_orders.conf;config/defaults/edge.conf;config/defaults/expiration_display_offset.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cpp_config.proto;config/protos/dynamic_orders.proto;config/protos/environment.proto;config/protos/expiration_display_offset.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/orders_fills_cleanup.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPriceLantencyTester.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;mercury/actors/Actor.cpp;mercury/broker/AsyncEventBroker.cpp;mercury/broker/CoreEventBroker.cpp;mercury/broker/IEventBroker.cpp;mercury/events/TimerEvent.cpp;mercury/mercury_utils/Executor.cpp;mercury/mercury_utils/IPollableActor.cpp;mercury/mercury_utils/mercury_common.cpp;mercury/synopsis/SynopsisBuilder.cpp;mercury/topics/ITopic.cpp;metrics/metrics.cpp;metrics/system_metrics.cpp;orderman/actors/AsyncFactory.cpp;orderman/actors/ExecutionGateway.cpp;orderman/actors/FillStore.cpp;orderman/actors/OrderStore.cpp;orderman/actors/SODPosition.cpp;orderman/actors/SecuritySrc.cpp;orderman/actors/ServiceBridge.cpp;orderman/actors/order_manager/OrderBuilder.cpp;orderman/actors/order_manager/OrderManager.cpp;orderman/actors/order_manager/OrderServicesData.cpp;orderman/events/CancelRejectEvent.cpp;orderman/events/NewExecutionReportEvent.cpp;orderman/limiter/LimitStore.cpp;orderman/limiter/Limiter.cpp;orderman/limiter/ProductLimitNode.cpp;orderman/limiter/ProductLimitTree.cpp;orderman/limiter/SecurityLimitNode.cpp;orderman/limiter/SecurityLimitTree.cpp;orderman/limiter/tests/limit_tests.cpp;orderman/order_factory/OrLine truncated
+set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/services/UserService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/Executor.conf;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cpp_config.conf;config/defaults/dynamic_orders.conf;config/defaults/edge.conf;config/defaults/expiration_display_offset.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cpp_config.proto;config/protos/dynamic_orders.proto;config/protos/expiration_display_offset.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPriceLantencyTester.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;mercury/actors/IPollableActor.cpp;mercury/broker/Broker.cpp;mercury/broker/BrokerProxy.cpp;mercury/broker/ILocalBroker.cpp;mercury/broker/IRemoteBroker.cpp;mercury/common/mercury_common.cpp;mercury/events/TimerEvent.cpp;mercury/executor/Executor.cpp;mercury/topics/Topic.cpp;metrics/metrics.cpp;metrics/system_metrics.cpp;orderman/actors/AsyncFactory.cpp;orderman/actors/ExecutionGateway.cpp;orderman/actors/FillStore.cpp;orderman/actors/OrderStore.cpp;orderman/actors/SODPosition.cpp;orderman/actors/SecuritySrc.cpp;orderman/actors/ServiceBridge.cpp;orderman/actors/order_manager/OrderBuilder.cpp;orderman/actors/order_manager/OrderManager.cpp;orderman/actors/order_manager/OrderServicesData.cpp;orderman/events/CancelRejectEvent.cpp;orderman/events/NewExecutionReportEvent.cpp;orderman/limiter/LimitStore.cpp;orderman/limiter/Limiter.cpp;orderman/limiter/ProductLimitNode.cpp;orderman/limiter/ProductLimitTree.cpp;orderman/limiter/SecurityLimitNode.cpp;orderman/limiter/SecurityLimitTree.cpp;orderman/limiter/tests/limit_tests.Line truncated
@@ -103,6 +103,7 @@ else ()
    boost_chrono
    boost_date_time
    boost_atomic
+    SQLiteCpp
    sqlite3
    curl
    uuid
@@ -109,6 +109,15 @@ message TickOrderDefinition {
 message LeanOrderDefinition {
  spark.QuantityMeasure cross_threshold = 1;
  uint32 max_chase_ticks = 2;
+  /*deprecated*/ bool detach_on_toward = 4;
+  enum OnToward
+  {
+     NO_CHANGE = 0; //continue to lean on initial inside market
+     CANCEL = 1; //cancel algo and order
+     DETACH = 2; //cancel algo but leave order working
+     REPRICE = 3; //lean on new inside market
+  }
+  OnToward trade_toward_behavior = 5;
  CustomOrderDefinition child = 3;
 }
 message JoinOrderDefinition {
@@ -234,7 +234,7 @@ bool ConfigPropertyEvaluator::get_value(const std::string& property, Value* out_
   tokenizer<char_separator<char> >::iterator iter = tokens.begin (), end = tokens.end ();

   // The first token is the key name
-   const proto::Message* key = _config.try_get_message(*iter);
+   const proto::Message* key = _config.try_get_message_or_default(*iter);
   if (!key) return false;

   // 2) Drill down using reflection to find the named property
@@ -616,6 +616,19 @@ const proto::Message* ConfigurationState::try_get_message(const std::string& pro
   return nullptr;
 }

+const proto::Message* ConfigurationState::try_get_message_or_default(const std::string& property_name) {
+   const auto pmsg = try_get_message(property_name);
+   if (pmsg) {
+      return pmsg;
+   }
+   // If we can't find a value we should return a default instance
+   const auto* descriptor = get_descriptor_by_name(property_name);
+   if (descriptor) {
+      return proto::MessageFactory::generated_factory()->GetPrototype(descriptor);
+   }
+   return nullptr;
+}
+
 const proto::Message& ConfigurationState::get_message(const proto::Descriptor* property_desc)
 {
   if (!is_valid_key_descriptor(property_desc))
@@ -59,6 +59,7 @@ namespace internal {

      const proto::Message& get_message(const proto::Descriptor* property_desc);
      const proto::Message* try_get_message(const std::string& property_name);
+      const proto::Message* try_get_message_or_default(const std::string& property_name);
      
      bool is_initialized() { return _is_initialized; }

@@ -76,16 +76,7 @@ extern "C" BTS_EXPORT const char* get_buffer(const char* key_name, int& out_leng

 extern "C" BTS_EXPORT const proto::Message* get_named(const char* key_name)
 {
-   const auto* pmessage = config_state.try_get_message(key_name);
-   if (pmessage) {
-      return pmessage;
-   }
-   // If we can't find a value we should return a default instance
-   const auto* descriptor = config_state.get_descriptor_by_name(key_name);
-   if (descriptor) {
-      return proto::MessageFactory::generated_factory()->GetPrototype(descriptor);
-   }
-   return nullptr;
+   return config_state.try_get_message_or_default(key_name);
 }

 BTS_EXPORT std::string get_macro_string(const char* name) { return config_state.get_macro_string(name); }
@@ -26,6 +26,26 @@ SymbolList {
    }
 }

+Symbols {
+    rule: "DeploymentType.type == 'DEMO' and System.platform == 'windows'"
+    value {
+        sim {
+            future: @SymbolList.symbol
+            spread: @SymbolList.symbol
+        }
+    }
+}
+
+Symbols {
+    rule: "DeploymentType.type == 'PROD' and System.platform == 'windows'"
+    value {
+        glbx {
+            future: @SymbolList.symbol
+            spread: @SymbolList.symbol
+        }
+    }
+}
+
 Symbols {
    value {
        glbx {
@@ -0,0 +1,34 @@
+OrdermanCfg {
+   rule: "DeploymentType.type == 'dev'"
+   value {
+      executor {
+         thread_name: "oman_orders"
+	 poll: false 
+	 hosted_actor: "MDSource"
+	 hosted_actor: "OrderManager"
+	 hosted_actor: "ExecutionGateway"
+      }
+      executor {
+         thread_name: "oman_slow"
+	 poll: false 
+	 hosted_actor: "*"
+      }
+   }
+}
+
+OrdermanCfg {
+   value {
+      executor {
+         thread_name: "oman_orders"
+	 poll: true 
+	 hosted_actor: "MDSource"
+	 hosted_actor: "OrderManager"
+	 hosted_actor: "ExecutionGateway"
+      }
+      executor {
+         thread_name: "oman_slow"
+	 poll: false 
+	 hosted_actor: "*"
+      }
+   }
+}
@@ -0,0 +1,7 @@
+Threads {
+   value {
+      thread_group {name_rx: "mdp3feed" core: 0}
+      thread_group {name_rx: "oman_orders" core: 1}
+      thread_group {name_rx: ".*" exclude:true core:0 core:1}
+   }
+}
@@ -75,7 +75,7 @@ message CMEOptions
      // Defines amount of time Feed Engine keeps cycling before going to sleep when no useful activity can be done
      optional uint32 spin_time = 4 [default = 1];
      optional uint32 channels_per_thread = 5 [default = 3];
-      repeated uint32 affinity = 10;
+      optional string thread_group = 10 [default=""]; // if present this will be used to retrieve affinity settings from Threads config key
      optional uint32 udp_socket_buffer_size = 11 [default = 33554432]; //32mb
   }
   enum BookType {
@@ -1,8 +0,0 @@
-syntax='proto2';
-package bts.config;
-
-message Deployment {
-  // Acceptable values:
-  // DEV_CHAPEL_HILL, DEV_CHICAGO, DEMO_GT, QA, CUSTOMER
-  optional string type = 1 [default="DEV_CHAPEL_HILL"];  
-}
@@ -0,0 +1,11 @@
+syntax='proto2';
+package bts.config;
+
+message OrdermanCfg {
+  message Executor {
+    required string thread_name = 1;
+    optional bool poll = 2 [ default = false];
+    repeated string hosted_actor = 3;
+  }
+  repeated Executor executor = 1;
+}
@@ -0,0 +1,13 @@
+syntax='proto2';
+package bts.config;
+
+message ThreadGroupCfg {
+  required string name_rx = 1	[default = "*"];
+  optional bool exclude = 2	[default = false];
+  repeated uint32 core = 3; 
+}
+
+
+message Threads {
+  repeated ThreadGroupCfg thread_group = 1; 
+}
@@ -11,7 +11,7 @@
 #include "lib/jetfix/codec/IFIXMsg.h"
 #include "lib/jetfix/common/InMsgBuffer.h"
 #include "lib/utils/meta_programming.h"
-#include "lib/orderman/orderman_utils/TickToTradeCorID.h"
+#include "lib/orderman/orderman_utils/PerfTrace.h"

 namespace bts {
 namespace jetfix {
@@ -6,7 +6,7 @@
 #include <vector>
 #include "lib/utils/bts_assert.h"
 #include "lib/jetfix/codec/MessageTypes.h"
-#include "lib/orderman/orderman_utils/TickToTradeCorID.h"
+#include "lib/orderman/orderman_utils/PerfTrace.h"

 namespace bts {
 namespace jetfix {
@@ -7,7 +7,8 @@
 #include "lib/utils/bts_assert.h"
 #include "lib/os/os.h"
 #include "lib/tracing/bts_trace.h"
-#include "lib/orderman/orderman_utils/TickToTradeCorID.h"
+#include "lib/orderman/orderman_utils/PerfTrace.h"
+#include "lib/utils/thread_manager.h"

 #include <fstream>

@@ -112,8 +113,8 @@ AsyncReader::AsyncReader (IConnection &connection,

 void AsyncReader::start ()
 {
-   me->reader_thread = std::thread ([this] () {
-         os::set_thread_name(me->connection_name+"_asyncr");
+   auto thread_name = fmt::format ("jf_ar_{}", me->connection_name);
+   me->reader_thread = threading::make_thread (thread_name,[this] () {
         dispatch();
      });
 }
@@ -1,7 +1,7 @@
 #pragma once

 #include <array>
-#include "lib/orderman/orderman_utils/TickToTradeCorID.h"
+#include "lib/orderman/orderman_utils/PerfTrace.h"

 namespace bts {
 namespace jetfix {
@@ -1,11 +1,12 @@
 #include "SocketConnection.h"
 #include "lib/jetfix/store/session_name.h"
 #include "lib/logging/logging.h"
-#include "lib/metrics/metrics.h"
+#include "lib/orderman/orderman_utils/PerfTraceLog.h"
 #include "lib/tracing/bts_trace.h"
 #include "lib/utils/TimeStamp.h"
 #include "lib/utils/bts_assert.h"
 #include "lib/utils/helper_macros.h"
+#include "lib/orderman/orderman_utils/PerfTraceLog.h"

 #include <arpa/inet.h>
 #include <fstream>
@@ -19,29 +20,15 @@ namespace bts {
 namespace jetfix {

 SocketConnection::SocketConnection(bts::config::JetFIX::Connection const &cfg,
-                                   metrics::MetricsContainer &stats)
+                                   PerfTraceLog &perf_log)
    : _fd(-1), _cfg(cfg), _log(cfg.log_fix_stream()), _in_log(), _out_log(),
-      _stats(stats), _tick_to_order(), _fill_to_order() {
+      _perf_log(perf_log) {
   if (_log) {
      _in_log.open(std::string("/tmp/fix_raw_read_log_") + session_name(cfg),
                   std::ios_base::app);
      _out_log.open(std::string("/tmp/fix_raw_write_log_") + session_name(cfg),
                    std::ios_base::app);
   }
-   std::vector<metrics::Label> labels;
-   labels.push_back({"connection", cfg.name()});
-
-   std::array<double, 30> buckets;
-   auto i = 1;
-   for (auto &b : buckets)
-      b = (i *= 2);
-
-   _tick_to_order = std::make_shared<metrics::HistogramMetric<30>>(
-       "spark_tick_to_order_ns", buckets, labels);
-   _fill_to_order = std::make_shared<metrics::HistogramMetric<30>>(
-       "spark_fill_to_order_ns", buckets, labels);
-   _stats.publish(_tick_to_order);
-   _stats.publish(_fill_to_order);
 }

 SocketConnection::~SocketConnection() {
@@ -122,46 +109,56 @@ int SocketConnection::write(WriteVec const &vec, size_t vec_size,

   auto trace_cpy = trace;
   trace_cpy.fix_socket_w = utils::TimeStamp::now();
-   tracepoint(bts_trace, trace_order, (char *)"SocketConnection.pre_write", 0,
-              trace.md_sn, 0);
+
+   utils::TimeStamp::Interval s2s;
+   if (trace.md.to_uint64()) {
+      s2s = (trace_cpy.fix_socket_w - trace_cpy.md);
+      tracepoint(bts_trace, trace_order,
+                 (char *)"SocketConnection.tick_to_order.pre_write", 0,
+                 trace.md_sn, s2s.microseconds());
+   } else {
+      s2s = (trace_cpy.fix_socket_w - trace_cpy.fix_socket_r);
+      tracepoint(bts_trace, trace_order,
+                 (char *)"SocketConnection.fill_to_order.pre_write", 0,
+                 trace.md_sn, s2s.microseconds());
+   }
+
   auto result = ::writev(_fd, iv, (int)vec_size);

   if (trace.md.to_uint64()) {
-      auto s2s = (trace_cpy.fix_socket_w - trace_cpy.md);
      LOG(Latency, Info,
          "perf_event:tick_to_order s2s_us:{} fix_socket_r:{} md:{} "
          "md_src:{} spreader_on_book:{} spreader_modify_order:{} "
-          "spreader_on_child_order_update:{} " 
-          "order_update:{} exe_gw:{} fix_socket_w:{} md_sn:{} order_key:{}",
+          "spreader_on_child_order_update:{} "
+          "order_update:{} exe_gw:{} fix_socket_w:{} md_sn:{} order_key:{} "
+          "send_fix_buffer:{}",
          s2s.microseconds(), trace_cpy.fix_socket_r.to_uint64(),
          trace_cpy.md.to_uint64(), trace_cpy.md_src.to_uint64(),
          trace_cpy.spreader_on_book.to_uint64(),
          trace_cpy.spreader_modify_order.to_uint64(),
-          trace_cpy.spreader_on_child_order_update.to_uint64(), 
-          trace_cpy.order_update.to_uint64(),
-          trace_cpy.exe_gw.to_uint64(), trace_cpy.fix_socket_w.to_uint64(),
-          trace_cpy.md_sn,
-          trace_cpy.order_key);
-      _tick_to_order->observe(s2s.nanoseconds());
+          trace_cpy.spreader_on_child_order_update.to_uint64(),
+          trace_cpy.order_update.to_uint64(), trace_cpy.exe_gw.to_uint64(),
+          trace_cpy.fix_socket_w.to_uint64(), trace_cpy.md_sn,
+          trace_cpy.order_key, trace.send_fix_buffer.to_uint64());
+      _perf_log.log(trace_cpy);
   }

   if (trace.fix_socket_r.to_uint64()) {
-      auto s2s = (trace_cpy.fix_socket_w - trace_cpy.fix_socket_r);
      LOG(Latency, Info,
          "perf_event:fill_to_order s2s_us:{} fix_socket_r:{} md:{} md_src:{} "
          "spreader_on_book:{} spreader_modify_order:{} "
-          "spreader_on_child_order_update:{} " 
-          "order_update:{} exe_gw:{} fix_socket_w:{} md_sn:{} order_key:{}",
+          "spreader_on_child_order_update:{} "
+          "order_update:{} exe_gw:{} fix_socket_w:{} md_sn:{} order_key:{} "
+          "send_fix_buffer:{} ",
          s2s.microseconds(), trace_cpy.fix_socket_r.to_uint64(),
          trace_cpy.md.to_uint64(), trace_cpy.md_src.to_uint64(),
          trace_cpy.spreader_on_book.to_uint64(),
          trace_cpy.spreader_modify_order.to_uint64(),
-          trace_cpy.spreader_on_child_order_update.to_uint64(), 
-          trace_cpy.order_update.to_uint64(),
-          trace_cpy.exe_gw.to_uint64(), trace_cpy.fix_socket_w.to_uint64(),
-          trace_cpy.md_sn,
-          trace.order_key);
-      _fill_to_order->observe(s2s.nanoseconds());
+          trace_cpy.spreader_on_child_order_update.to_uint64(),
+          trace_cpy.order_update.to_uint64(), trace_cpy.exe_gw.to_uint64(),
+          trace_cpy.fix_socket_w.to_uint64(), trace_cpy.md_sn, trace.order_key,
+          trace.send_fix_buffer.to_uint64());
+      _perf_log.log(trace_cpy);
   }

   tracepoint(bts_trace, trace_order, (char *)"SocketConnection.post_write", 0,
@@ -7,14 +7,14 @@
 #include "lib/metrics/metrics.h"

 namespace bts {
-namespace metrics {class MetricsContainer;}
+class PerfTraceLog;

 namespace jetfix {

 class SocketConnection : public IConnection
 {
 public:
-   SocketConnection(bts::config::JetFIX::Connection const &cfgl,metrics::MetricsContainer &stats);
+   SocketConnection(bts::config::JetFIX::Connection const &cfgl,PerfTraceLog &log);
   ~SocketConnection();
   bool connect () override;
   void shutdown () override;
@@ -27,9 +27,7 @@ private:
   bool _log;
   std::ofstream _in_log;
   std::ofstream _out_log;
-   metrics::MetricsContainer &_stats;
-   std::shared_ptr<metrics::HistogramMetric<30>> _tick_to_order;
-   std::shared_ptr<metrics::HistogramMetric<30>> _fill_to_order;
+   PerfTraceLog &_perf_log;
 };

 }}
@@ -13,6 +13,7 @@
 #include "lib/orderman/orderman_utils/orderman_common.h"
 #include "lib/tracing/bts_trace.h"
 #include <boost/lockfree/queue.hpp>
+#include "lib/utils/thread_manager.h"

 #include <atomic>
 #include <boost/date_time/posix_time/posix_time.hpp>
@@ -121,12 +122,17 @@ struct CoreGateway::PImpl {
   uint32_t connection_attempts;
 };

+std::thread CoreGateway::make_timer_thread() {
+   auto thread_name = fmt::format ("jf_tmr_{}", me->cfg.name());
+   return threading::make_thread(thread_name, std::bind(&CoreGateway::call_time, this));
+}
+
 CoreGateway::CoreGateway(config::JetFIX::Connection const &cfg,
                         ICoreGatewayObserver &observer,
                         ConnectionFactory const &conn_factory,
                         StoreFactory const &store_factory)
    : me(new PImpl(cfg, observer, conn_factory, store_factory)) {
-   me->timer_thread = std::thread(std::bind(&CoreGateway::call_time, this));
+   me->timer_thread = make_timer_thread();
 }

 CoreGateway::~CoreGateway() {
@@ -202,7 +208,6 @@ void CoreGateway::dispatch_events() {
 }

 void CoreGateway::call_time() {
-   os::set_thread_name(fmt::format("{0}_timer", me->cfg.name()).c_str());
   while (!me->destroy) {
      std::this_thread::sleep_for(10ms);
      auto ts = std::make_shared<std::string>(
@@ -67,7 +67,8 @@ private:
   void transition (State new_state);
   void check_state_invariants ();
   const char* state_name (State st);
-
+   std::thread make_timer_thread();
+   
   std::unique_ptr<PImpl> me;
 };

@@ -12,6 +12,7 @@
 #include "lib/config/protos/jetfix.pb.h"

 namespace bts {
+class PerfTraceLog;
 namespace jetfix {

 // A parsing layer over CoreGateway
@@ -19,13 +20,12 @@ template <typename... Messages>
 class JetFIXGateway : ICoreGatewayObserver
 {
 public:
-   JetFIXGateway(config::JetFIX::Connection const &cfg, 
-                 metrics::MetricsContainer &stats)
+   JetFIXGateway(config::JetFIX::Connection const &cfg, PerfTraceLog &log)
      :
      _core (cfg,*this,
               
-             [&stats] (config::JetFIX::Connection const &cfg)
-             {return new SocketConnection (cfg,stats);},
+             [&log] (config::JetFIX::Connection const &cfg)
+             {return new SocketConnection (cfg,log);},
             
             [] (config::JetFIX::Connection const &cfg)
             { return new FIXFileStore (cfg);}),
@@ -9,6 +9,7 @@
 #include "lib/utils/bts_chrono.h"
 #include "lib/utils/datetime.h"
 #include "lib/utils/file_utils.h"
+#include "lib/utils/thread_manager.h"

 #include <boost/date_time.hpp>
 #include <boost/date_time/gregorian/gregorian.hpp>
@@ -204,7 +205,9 @@ FIXFileStore::FIXFileStore(config::JetFIX::Connection const &cfg,
         }
      }
      me->fix_store.open(fix_log_p, std::ofstream::app);
-      me->writer_thread = std::thread(std::bind(&FIXFileStore::write, this));
+      auto thread_name = fmt::format("jf_store_{}", me->cfg.name().c_str());
+      me->writer_thread = threading::make_thread(
+          thread_name, std::bind(&FIXFileStore::write, this));
   }
 }

@@ -270,7 +273,6 @@ void FIXFileStore::set_last_seq_num(uint32_t last_seq_num) {

 void FIXFileStore::write() {
   Event ev;
-   os::set_thread_name(fmt::format("{0}_store", me->cfg.name()).c_str());

   while (true)
      if (me->event_queue.try_pop(ev)) {
@@ -4,6 +4,7 @@
 #include "lib/os/os.h"
 #include "lib/utils/bts_assert.h"
 #include "lib/utils/bts_chrono.h"
+#include "lib/utils/thread_manager.h"

 #include <algorithm>
 #include <atomic>
@@ -25,7 +26,7 @@ struct LoggerHub::PImpl {
   PImpl(std::string const &logger_name)
       : lock(), sinks(), loggers(), level_names(), subsystem_names(), worker(),
         done(false), logger_initial_queue_size(), logger_max_queue_growth(),
-         logger_name(logger_name) {}
+         logger_name(logger_name), thread_cfg() {}

   std::mutex lock;
   std::vector<SinkRec> sinks;
@@ -37,6 +38,7 @@ struct LoggerHub::PImpl {
   size_t logger_initial_queue_size;
   size_t logger_max_queue_growth;
   std::string logger_name;
+   config::ThreadGroupCfg thread_cfg;
 };

 LoggerHub::LoggerHub(std::string const &logger_name)
@@ -49,7 +51,6 @@ LoggerHub::LoggerHub(std::string const &logger_name)

 void LoggerHub::config(const LoggingConfig &cfg) {
   std::lock_guard<std::mutex> l(me->lock);
-
   me->logger_initial_queue_size = cfg.logger_initial_queue_size;
   me->logger_max_queue_growth = cfg.logger_max_queue_growth;

@@ -74,11 +75,14 @@ void LoggerHub::config(const LoggingConfig &cfg) {
   }
   rebuild_logger_filters();
   update_logger_queue_limits();
+   me->thread_cfg = cfg.thread_cfg;
 }

 void LoggerHub::start() {
   bts_assert(!me->worker);
-   me->worker = std::make_unique<std::thread>(&LoggerHub::do_work, this);
+   me->worker = std::make_unique<std::thread>(
+      threading::make_thread(me->logger_name.c_str(),
+                             std::bind(&LoggerHub::do_work, this),me->thread_cfg));
 }

 LoggerHub::~LoggerHub() {
@@ -96,7 +100,6 @@ Logger &LoggerHub::make_logger() {
 }

 void LoggerHub::do_work() {
-   os::set_thread_name(me->logger_name.c_str());
   while (!me->done) {
      size_t consumed = 0;
      {
@@ -33,5 +33,4 @@ private:
   Filter get_base_filter() const;
 };

-void start_logging_hub();
 }}
@@ -4,18 +4,22 @@
 #include "Common.h"
 #include <vector>
 #include <map>
+#include "lib/config/protos/threads.pb.h"

 namespace bts {
 namespace log {

 struct LoggingConfig {
-   LoggingConfig() : logger_initial_queue_size(120000),
-                     logger_max_queue_growth(4),
-                     sinks(),
-                     level_names(),
-                     subsystem_names(),
-                     deployment_type("dev")
-   {}
+   LoggingConfig() :
+      logger_initial_queue_size(120000),
+      logger_max_queue_growth(4),
+      sinks(),
+      level_names(),
+      subsystem_names(),
+      deployment_type("dev"),
+      thread_cfg()
+   {
+   }

   size_t logger_initial_queue_size = 0;
   size_t logger_max_queue_growth = 0;
@@ -23,6 +27,7 @@ struct LoggingConfig {
   std::map<Level,std::string> level_names;
   std::map<Subsystem,std::string> subsystem_names;
   std::string deployment_type;
+   config::ThreadGroupCfg thread_cfg;
 };

 }}
@@ -9,6 +9,7 @@
 #include "lib/utils/file_utils.h"
 #include "lib/utils/Dispatcher.h"
 #include "lib/config/protos/system.pb.h"
+#include "lib/utils/thread_manager.h"

 #include <google/protobuf/text_format.h>
 #include <google/protobuf/io/zero_copy_stream_impl.h>
@@ -86,8 +87,8 @@ Filter get_filter (LoggingConfig &cfg,
 }

 void configure_logging(LoggingConfig const&cfg) {_default_logger_hub.config(cfg);}
-
-void configure_logging(bts::config::Logging const&cfg)
+void configure_logging(bts::config::Logging const&cfg) {configure_logging(cfg,bts::config::ThreadGroupCfg ());}
+void configure_logging(bts::config::Logging const&cfg, bts::config::ThreadGroupCfg const&thread_cfg)
 {
   LoggingConfig real_cfg;
   real_cfg.deployment_type = config::get<config::DeploymentType>().type();
@@ -124,6 +125,7 @@ void configure_logging(bts::config::Logging const&cfg)
      std::make_pair (syslog_sink(real_cfg),get_filter(real_cfg,cfg.syslog()));
   real_cfg.sinks["file"] =
      std::make_pair (file_sink(real_cfg),get_filter(real_cfg,cfg.file()));
+   real_cfg.thread_cfg = thread_cfg;
   configure_logging(real_cfg);
 }

@@ -136,19 +138,20 @@ boost::filesystem::path log_cache_file_path ()
   return result;
 }

-LoggingConfigPoller::LoggingConfigPoller (bts::config::Logging const&cfg)
+LoggingConfigPoller::LoggingConfigPoller (bts::config::Logging const&cfg, bts::config::ThreadGroupCfg const&thread_cfg)
   : _poller(utils::Dispatcher::background(), 1000, [=]() { poll_once(); }), 
   _last_ts(std::time(nullptr)),
   _log_cache_file(log_cache_file_path()),
   _done(false)
 {
-   init(cfg);
+   init(cfg,thread_cfg);
 }

-LoggingConfigPoller::LoggingConfigPoller () : LoggingConfigPoller(config::get<config::Logging>())
+LoggingConfigPoller::LoggingConfigPoller () : LoggingConfigPoller(config::get<config::Logging>(),
+                                                                  threading::get_thread_cfg("logger"))
 { }

-void LoggingConfigPoller::init (bts::config::Logging const&cfg)
+void LoggingConfigPoller::init (bts::config::Logging const&cfg,bts::config::ThreadGroupCfg const&thread_cfg)
 {
   auto parent = _log_cache_file.parent_path();
   if (!boost::filesystem::is_directory(parent)) 
@@ -160,7 +163,7 @@ void LoggingConfigPoller::init (bts::config::Logging const&cfg)


   // initialize logging
-   configure_logging(cfg);
+   configure_logging(cfg,thread_cfg);
   _poller.start();
 }

@@ -10,7 +10,7 @@


 namespace bts {
-namespace config {class Logging;}
+namespace config {class Logging; class ThreadGroupCfg;}

 // Please keep this list in sync with Log.cs
 constexpr log::Level Diag     {0};
@@ -55,17 +55,18 @@ struct LoggerHandle {

 extern thread_local LoggerHandle _thread_logger;
 void configure_logging(LoggingConfig const&cfg);
+void configure_logging(bts::config::Logging const&cfg,bts::config::ThreadGroupCfg const&);
 void configure_logging(bts::config::Logging const&cfg);
 void start_default_logging_thread();

 class LoggingConfigPoller {
 public:
   LoggingConfigPoller ();
-   LoggingConfigPoller (bts::config::Logging const&cfg);
+   LoggingConfigPoller (bts::config::Logging const&cfg, bts::config::ThreadGroupCfg const&thread_cfg);
  ~LoggingConfigPoller ();

 private:
-   void init(bts::config::Logging const&cfg);
+   void init(bts::config::Logging const&cfg, bts::config::ThreadGroupCfg const&thread_cfg);
   void poll_once();
   utils::DispatcherTimer _poller;
   std::time_t _last_ts;
@@ -76,8 +76,7 @@ string dump_security(const FixedSecurity& security) {
   res << "'main_frac': " << (int)security.cme_main_fraction << ", ";
   res << "'sub_fract': " << (int)security.cme_sub_fraction << ", ";
   res << "'price_format': " << (int)security.cme_price_display_format << ", ";
-
-   res << "'market_segment': " << security.cme_market_segment_id << ", ";
+   res << "'market_segment': " << (int)security.cme_market_segment_id << ", ";

   res << "'price_break': " << security.price_break << ", ";
   res << "'low_tick': " << security.low_tick << ", ";
@@ -141,6 +141,10 @@ extern "C" BTS_EXPORT void co2_get_zero_and_nan_book(PriceLevel& price) {
   price = { PriceSize::NO_MARKET, PriceSize::from_double<7>(0,1) };
 }

+extern "C" BTS_EXPORT void co2_get_sample_datetime(datetime::LLDateTime& now, datetime::LLDateTime& utc_now) {
+	now = datetime::ll_local_now();
+	utc_now = datetime::ll_now();
+}

 }}}//namespace
 #endif
@@ -1,7 +0,0 @@
-#include "Actor.h"
-
-namespace bts {
-namespace mercury {
-
-
-}}
@@ -3,30 +3,32 @@
 #include <boost/container/flat_set.hpp>
 #include <memory>

-#include "lib/utils/bts_assert.h"
-#include "lib/utils/warn_ctrl.h"
-#include "lib/mercury/broker/AsyncEventBroker.h"
+#include "lib/mercury/common/mercury_common.h"
 #include "lib/mercury/events/Stream.h"
 #include "lib/mercury/events/Update.h"
 #include "lib/mercury/topics/Topic.h"
+#include "lib/mercury/actors/ActorBroker.h"
+
+#include "lib/tracing/bts_trace.h"
+#include "lib/utils/bts_assert.h"
+#include "lib/utils/warn_ctrl.h"

 namespace bts {
 namespace mercury {

-// Helper class to ease subscription/publication of events Client code
-// should derive from this class and implement an init() method which
-// signals to the class that the event dispatch framework is ready to
-// process event subscription and publication.
+// Helper class to ease subscription/publication of events. Client
+// code should derive from this class and implement an init() method
+// which signals to the class that the event dispatch framework is
+// ready to process event subscription and publication.

-GCC_DIAG_OFF("-Wnon-virtual-dtor")
-class Actor : public std::enable_shared_from_this<Actor> {
+class Actor {
 public:
-   using SubHandle = InternalSubID;
+   using SubHandle = LocalSubID;

-   Actor() : _broker(nullptr), _active_streams(), _active_subs() {}
+   Actor() : _broker(nullptr), _active_streams(), _active_subs(), _broker_id() {}
   virtual ~Actor() {}

-   // Override this to kick off actor. 
+   // Override this to kick off actor.
   virtual void init() = 0;

   // Override for custom actor names, used in logging messages.
@@ -42,30 +44,29 @@ class Actor : public std::enable_shared_from_this<Actor> {
      using TopicType = Topic<EV>;
      using UpdateType = Update<EV>;

-      auto self = shared_from_this();
-      TopicFilter filter = [tag, self](const ITopic &t) {
+      TopicFilter filter = [tag, this](const ITopic &t) {
         auto typed_topic = dynamic_cast<TopicType const *>(&t);
         if (!typed_topic)
            return false;
         else {
            auto match = typed_topic->tag() == tag;
            if (match)
-               LOG(Mercury, Diag, "{} on event broker {} matched sub to {}",
-                   self->name(), self->broker_id().value(),
+               LOG(Mercury, Info, "{} on event broker {} matched sub to {}",
+                   name(), _broker_id,
                   TopicType(tag).to_str());
            return match;
         }
      };

-      InternalEventHandler handler = [this, cb, tag](
-          const StreamID &sid, EventType type, EventPayload const *payload) {
+      LocalEventHandler handler = [this, cb, tag](const StreamID &sid,
+                                                  EventType type,
+                                                  Notification const *payload) {
         if (!this->_broker)
            return;

-         LOG(Mercury, Diag,
-             "{} on event broker {} received update to {}. Payload: {}", name(),
-             broker_id().value(), TopicType(tag).to_str(), (size_t)payload);
-
+         tracepoint(bts_trace, trace_event, utils::type_name(typeid(*this)),
+                    utils::type_name<EV>(), _broker_id, sid.value(),
+                    0);
         if (payload)
            cb(UpdateType{sid, type, &(payload->unwrap<typename EV::Array>())});
         else
@@ -76,38 +77,36 @@ class Actor : public std::enable_shared_from_this<Actor> {

      LOG(Mercury, Info,
          "{} on event broker {} subscribing to {} with subid {}", name(),
-          broker_id().value(), TopicType(tag).to_str(), result.value());
+          _broker_id, TopicType(tag).to_str(), result.value());

      return result;
   }
-   
+
   // subscribe to all events of a specific type
   template <class EV, class CB> SubHandle subscribe(CB const &cb) {
      bts_assert(_broker);
      using TopicType = Topic<EV>;
      using UpdateType = Update<EV>;
-      auto self = shared_from_this();

-      TopicFilter filter = [self](const ITopic &t) {
+      TopicFilter filter = [this](const ITopic &t) {
         auto typed_topic = dynamic_cast<TopicType const *>(&t);
         auto match = typed_topic != nullptr;
         if (match)
-            LOG(Mercury, Diag, "{} on event broker {} matched sub to {}",
-                self->name(), self->broker_id().value(),
+            LOG(Mercury, Info, "{} on event broker {} matched sub to {}",
+                name(), _broker_id,
                std::string(utils::type_name<EV>()));
         return match;
      };

-      InternalEventHandler handler = [this, cb](
-          const StreamID &sid, EventType type, EventPayload const *payload) {
+      LocalEventHandler handler = [this, cb](const StreamID &sid,
+                                             EventType type,
+                                             Notification const *payload) {
         if (!this->_broker)
            return;

-         LOG(Mercury, Diag,
-             "{} on event broker {} received update to {}. Payload: {}", name(),
-             broker_id().value(), std::string(utils::type_name<EV>()),
-             (size_t)payload);
-
+         tracepoint(bts_trace, trace_event, utils::type_name(typeid(*this)),
+                    utils::type_name<EV>(), _broker_id, sid.value(),
+                    0);
         if (payload)
            cb(UpdateType{sid, type, &(payload->unwrap<typename EV::Array>())});
         else
@@ -115,9 +114,10 @@ class Actor : public std::enable_shared_from_this<Actor> {
      };

      auto result = _subscribe(filter, handler);
-      LOG(Mercury, Info, "{} on event broker {} subscribing to all instances "
-                         "of type {} with subid {}",
-          name(), broker_id().value(), std::string(utils::type_name<EV>()),
+      LOG(Mercury, Info,
+          "{} on event broker {} subscribing to all instances "
+          "of type {} with subid {}",
+          name(), _broker_id, std::string(utils::type_name<EV>()),
          result.value());
      return result;
   }
@@ -130,8 +130,7 @@ class Actor : public std::enable_shared_from_this<Actor> {
      using TopicType = Topic<EV>;
      using UpdateType = Update<EV>;

-      auto self = shared_from_this();
-      TopicFilter filter = [self, user_filter](const ITopic &t) {
+      TopicFilter filter = [this, user_filter](const ITopic &t) {
         auto typed_topic = dynamic_cast<TopicType const *>(&t);
         if (typed_topic == nullptr)
            return false;
@@ -139,11 +138,14 @@ class Actor : public std::enable_shared_from_this<Actor> {
            return user_filter(*typed_topic);
      };

-      InternalEventHandler handler = [this, cb](
-          const StreamID &sid, EventType type, EventPayload const *payload) {
+      LocalEventHandler handler = [this, cb](const StreamID &sid,
+                                             EventType type,
+                                             Notification const *payload) {
         if (!this->_broker)
            return;
-
+         tracepoint(bts_trace, trace_event, utils::type_name(typeid(*this)),
+                    utils::type_name<EV>(), _broker_id, sid.value(),
+                    0);
         if (payload)
            cb(UpdateType{sid, type, &(payload->unwrap<typename EV::Array>())});
         else
@@ -153,7 +155,7 @@ class Actor : public std::enable_shared_from_this<Actor> {
      auto result = _subscribe(filter, handler);
      LOG(Mercury, Info,
          "{} on event broker {} subscribing to filter on {} with subid {}",
-          name(), broker_id().value(), std::string(utils::type_name<EV>()),
+          name(), _broker_id, std::string(utils::type_name<EV>()),
          result.value());
      return result;
   }
@@ -161,7 +163,7 @@ class Actor : public std::enable_shared_from_this<Actor> {
   // unsubscribe
   void unsubscribe(SubHandle id) {
      bts_assert(_broker);
-      _broker->internal_unsubscribe(id);
+      _broker->unsubscribe(id);
      _active_subs.erase(id);
   }

@@ -170,26 +172,26 @@ class Actor : public std::enable_shared_from_this<Actor> {
      bts_assert(_broker);
      using TopicType = Topic<EV>;
      auto topic = std::make_shared<TopicType>(tag);
-      auto sid = _broker->internal_advertise(topic);
+      auto sid = _broker->advertise(topic);

      LOG(Mercury, Info,
          "{} on event broker {} creating stream for {} with sid {}", name(),
-          broker_id().value(), TopicType(tag).to_str(), sid.value());
+          _broker_id, TopicType(tag).to_str(), sid.value());

      _active_streams.insert(sid);
      return Stream<EV>{sid};
   }

-   // create a stream of events of a specific type 
+   // create a stream of events of a specific type
   template <class EV> Stream<EV> make_stream() {
      bts_assert(_broker);
      using TopicType = Topic<EV>;
      auto topic = std::make_shared<TopicType>();
-      auto sid = _broker->internal_advertise(topic);
+      auto sid = _broker->advertise(topic);

      LOG(Mercury, Info,
          "{} on event broker {} creating stream for {} with sid {}", name(),
-          broker_id().value(), TopicType().to_str(), sid.value());
+          _broker_id, TopicType().to_str(), sid.value());

      _active_streams.insert(sid);
      return Stream<EV>{sid};
@@ -199,56 +201,37 @@ class Actor : public std::enable_shared_from_this<Actor> {
   // event with the args passed
   template <class EV, typename... Args>
   void publish(Stream<EV> const &stream, Args &&... args) {
+      bts_assert_d(_broker);

-      LOG(Mercury, Diag, "{} on event broker {} publishing {} on stream {}",
-          name(), broker_id().value(), std::string(utils::type_name<EV>()),
-          stream.sid.value());
+      tracepoint(bts_trace, trace_event, utils::type_name(typeid(*this)),
+                 utils::type_name<EV>(), _broker_id,
+                 stream.sid.value(), 1);

-      bts_assert(_broker);
-      auto msg = _broker->alloc_internal_notification_msg(stream.sid);
-      auto &array = msg.template wrap_in_place<typename EV::Array>();
-      array.emplace_back(std::forward<Args>(args)...);
-      _broker->internal_notify(msg);
+      _broker->notify(stream, std::forward<Args>(args)...);
   }

   // publish a sequence of events to a previously created stream
   template <class EV, class C>
   void publish_batch(Stream<EV> const &stream, C const &c) {
-
-      LOG(Mercury, Info, "{} on event broker {} publishing a batch of {} {} "
-                         " on stream {}",
-          name(), broker_id().value(), c.size(),
-          std::string(utils::type_name<EV>()), stream.sid.value());
-
      bts_assert(_broker);
-      auto msg = _broker->alloc_internal_notification_msg(stream.sid);
-      auto &array = msg.template wrap_in_place<typename EV::Array>();
-      array.reserve(c.size());
-      for (auto &item : c)
-         array.emplace_back(item);
-      _broker->internal_notify(msg);
+      _broker->notify_batch(stream,c);
   }

   // destroy a previously created stream
   template <class EV> void destroy_stream(Stream<EV> const &stream) {
      bts_assert(_broker);
      _active_streams.erase(stream.sid);
-      _broker->internal_unadvertise(stream.sid);
+      _broker->unadvertise(stream.sid);
   }

-   // queue a lambda for execution in the Actors thread
-   void post_action(std::function<void()> const &action) {
-      _broker->post_action(action);
+   void schedule_action (std::function<void()> const&action) {
+      bts_assert(_broker);
+      _broker->schedule_action(action);
   }

 protected:
-   EventBrokerID broker_id() {
-      bts_assert(_broker);
-      return _broker->id();
-   }
-
   friend class Executor;
-   void activate(AsyncEventBroker &broker) {
+   void activate(ActorBroker &broker) {
      bts_assert(!_broker);
      _broker = &broker;
      LOG(Mercury, Info, "Initializing {}", name());
@@ -260,27 +243,27 @@ class Actor : public std::enable_shared_from_this<Actor> {
      finit();

      for (auto sid : _active_streams)
-         _broker->internal_unadvertise(sid);
+         _broker->unadvertise(sid);
      _active_streams.clear();

      for (auto h : _active_subs)
-         _broker->internal_unsubscribe(h);
+         _broker->unsubscribe(h);
      _active_subs.clear();

      _broker = nullptr;
   }

 private:
-   AsyncEventBroker *_broker;
+   ActorBroker *_broker;
   boost::container::flat_set<StreamID> _active_streams;
   boost::container::flat_set<SubHandle> _active_subs;
+   BrokerID::Value _broker_id;

-   SubHandle _subscribe(TopicFilter const &f, InternalEventHandler const &h) {
-      auto handle = _broker->internal_subscribe(f, h);
+   SubHandle _subscribe(TopicFilter const &f, LocalEventHandler const &h) {
+      auto handle = _broker->subscribe(f, h);
      _active_subs.insert(handle);
      return handle;
   }
 };
-GCC_DIAG_RESTORE()
-}
-}
+} // namespace mercury
+} // namespace bts
@@ -0,0 +1,69 @@
+#pragma once
+
+#include "lib/mercury/common/mercury_common.h"
+#include "lib/mercury/broker/BrokerProxy.h"
+#include "lib/mercury/events/Stream.h"
+
+namespace bts {
+namespace mercury {
+
+class ActorBroker {
+public:
+   ActorBroker (BrokerProxy &proxy) : _proxy(proxy),_proxy_id(_proxy.id()),_current_suid(0) {}
+
+   BrokerID id() {return _proxy_id;}
+
+   LocalSubID subscribe(TopicFilter const&f, LocalEventHandler const &h) {
+      _current_suid = LocalSubID(_current_suid.value() + 1);
+      _proxy.local_subscribe(_current_suid,f,h);
+      return _current_suid;
+   }
+
+   void unsubscribe(LocalSubID suid) {
+      _proxy.local_unsubscribe(suid);
+   }
+
+   StreamID advertise (TopicPtr const&t) {
+      auto sid = gen_stream_id();
+      _proxy.advertise(_proxy_id,sid,t);
+      return sid;
+   }
+    
+   void unadvertise (StreamID sid) {
+      _proxy.unadvertise(_proxy_id,sid);
+   }
+   
+   template <class EV, typename... Args>
+   void notify (Stream<EV> const &stream, Args &&... args)  {
+      BrokerProxy::BrokerMsg *m;
+      Notification *notification;
+      _proxy.alloc_notify(&m,&notification);
+      auto &array = notification->template wrap_in_place<typename EV::Array>();
+      array.emplace_back(std::forward<Args>(args)...);
+      _proxy.send_notify(_proxy_id, stream.sid, SubID(0), *m);
+   }
+
+   template <class EV, class C>
+   void notify_batch(Stream<EV> const &stream, C const &c) {
+      BrokerProxy::BrokerMsg *msg;
+      Notification *notification;
+      _proxy.alloc_notify(&msg,&notification);
+      auto &array = notification->template wrap_in_place<typename EV::Array>();
+      array.reserve(c.size());
+      for (auto &item : c)
+         array.emplace_back(item);
+      _proxy.send_notify(_proxy_id, stream.sid, SubID(0), *msg);
+   }
+
+   void schedule_action (std::function<void()> const&action) {
+      _proxy.schedule_action(action);
+   }
+
+private:
+   BrokerProxy &_proxy;
+   BrokerID _proxy_id;
+   LocalSubID _current_suid;
+};
+
+
+}}
@@ -1,11 +0,0 @@
-#pragma once
-
-#include <memory>
-
-namespace bts {
-namespace mercury {
-
-class Actor;
-using ActorPtr = std::shared_ptr<Actor>;
-
-}}
@@ -65,7 +65,6 @@ template <typename EV, typename KE = KeyExtractor> class ContentSubscription {
         _pre_sub(pscb) {}

   template <class CBT> SubID sub(Key k, CBT const &cb) {
-      bts_assert(!_dispatching);
      _pre_sub(k);
      auto sid = SubID{k, _current_sub_id++};
      auto &k_subs = _my_subs[k];
@@ -78,7 +77,6 @@ template <typename EV, typename KE = KeyExtractor> class ContentSubscription {
   }

   void unsub(SubID sid) {
-      bts_assert(!_dispatching);
      auto &subs = _my_subs[sid.k];
      auto it =
          std::find_if(subs.begin(), subs.end(), [sid](auto const &subscriber) {
@@ -105,6 +103,7 @@ template <typename EV, typename KE = KeyExtractor> class ContentSubscription {
   Actor *_actor;
   Actor::SubHandle _sub;
   Subscriptions _my_subs;
+   SubList _aux_sub_list;
   bool _dispatching;
   uint32_t _current_sub_id;
   std::function<void(Key k)> _pre_sub;
@@ -115,7 +114,8 @@ template <typename EV, typename KE = KeyExtractor> class ContentSubscription {
      if (u.events)
         for (auto const &ev : *(u.events)) {
            auto k = ke(ev);
-            for (auto const &sub : _my_subs[k])
+            _aux_sub_list = _my_subs[k]; // we copy so we can allow for unsub/sub during dispatch 
+            for (auto const &sub : _aux_sub_list)
               sub.cb(ev);
         }
      _dispatching = false;
@@ -0,0 +1,59 @@
+#pragma once
+
+#include "lib/mercury/actors/Actor.h"
+#include "lib/mercury/actors/IPollableActor.h"
+#include "lib/mercury/events/Event.h"
+
+#include "lib/tracing/bts_trace.h"
+#include "lib/utils/bts_chrono.h"
+#include "lib/utils/datetime.h"
+#include "lib/utils/helper_macros.h"
+
+#include <memory>
+
+namespace bts {
+namespace mercury {
+
+struct PingEvent : public Event<PingEvent> {
+   using SynopsisPolicy = NoSynopsis;
+   PingEvent(utils::TimeStamp src) : src(src) {}
+   utils::TimeStamp src;
+};
+
+class Ping : public Actor, public mercury::IPollableActor {
+ public:
+   void init() override {
+      _stream = make_stream<PingEvent>();
+      _i = 0;
+   }
+
+ private:
+   void on_poll() override {
+      _i++;
+      if (_i % 100000 == 0) {
+         auto now = utils::TimeStamp::now();
+         tracepoint(bts_trace, trace_order, "ping_send", 0, now.to_uint64(),0);
+         publish(_stream, now);
+      }
+   }
+
+   mercury::Stream<PingEvent> _stream;
+   uint64_t _i;
+};
+
+class Pong : public Actor {
+ public:
+   void init() override {
+      subscribe<PingEvent>([](auto const &m) {
+            if (m.events) {
+               auto then = (*m.events)[0].src;
+               auto d = utils::TimeStamp::now() - then;
+               UNUSED(d);
+               tracepoint(bts_trace, trace_order, "ping_receive", 0,then.to_uint64(),d.nanoseconds());
+            }
+         });
+   }
+};
+
+} // namespace mercury
+} // namespace bts
@@ -1,9 +1,9 @@
 #pragma once

-#include "lib/utils/bts_chrono.h"
-#include "lib/utils/datetime.h"
 #include "lib/mercury/actors/Actor.h"
 #include "lib/mercury/events/TimerEvent.h"
+#include "lib/utils/bts_chrono.h"
+#include "lib/utils/datetime.h"

 #include <boost/date_time/posix_time/posix_time.hpp>
 #include <memory>
@@ -12,7 +12,7 @@ namespace bts {
 namespace mercury {

 // a standard actor providing periodic time events at a
-// fixed 10ms frequency 
+// fixed 10ms frequency
 class Timer : public Actor, public mercury::IPollableActor {
 public:
   void init() override {
@@ -37,5 +37,5 @@ class Timer : public Actor, public mercury::IPollableActor {
   mercury::Stream<TimerEvent> _stream;
   TimeStamp _last_ts;
 };
-}
-}
+} // namespace mercury
+} // namespace bts
@@ -1,9 +0,0 @@
-#include "AsyncEventBroker.h"
-
-namespace bts {
-namespace mercury {
-
-std::atomic<uint64_t> AsyncEventBroker::_msg_id;
-
-}}
-
@@ -1,339 +0,0 @@
-#pragma once
-
-#include <atomic>
-#include <boost/container/flat_map.hpp>
-#include <boost/pool/object_pool.hpp>
-#include <memory>
-#include <typeindex>
-#include <boost/lockfree/queue.hpp>
-
-#include "lib/mercury/broker/CoreEventBroker.h"
-#include "lib/mercury/mercury_utils/FatWrapper.h"
-
-namespace bts {
-namespace mercury {
-
-class AsyncEventBroker : public IEventBroker {
- private:
-   struct BrokerMsg {
-      uint64_t msg_id;
-      EventBrokerID broker_src_id;
-      utils::FatWrapper<127 - 8 - 4> wrapper;
-      void (*handle_msg)(AsyncEventBroker *self, BrokerMsg *m);
-      bool is_external;
-   };
-
- public:
-   AsyncEventBroker()
-       : _core_broker(), _queue(5000), _internal_pool(), _external_pool(),
-         _external_pool_mtx(), _current_suid(0) {}
-
-   void
-   init_neighbours(std::initializer_list<IEventBrokerPtr> const &neighbours) {
-      for (auto const &n : neighbours)
-         _core_broker.add_neighbour(n);
-   }
-
-   // ieventbroker methods, can be called from multiple threads
-   EventBrokerID id() const override { return _core_broker.id(); }
-
-   void advertise(EventBrokerID eid, StreamID sid, TopicPtr const &t) override {
-      post_external_msg<Advertisement>(on_advertisement, eid, sid, t);
-   }
-
-   void unadvertise(EventBrokerID eid, StreamID sid) override {
-      post_external_msg<Unadvertisement>(on_unadvertisement, eid, sid);
-   }
-
-   void subscribe(EventBrokerID eid, SubID suid, StreamID sid) override {
-      post_external_msg<Subscription>(on_subscription, eid, suid, sid);
-   }
-
-   void unsubscribe(EventBrokerID eid, SubID suid, StreamID sid) override {
-      post_external_msg<Unsubscription>(on_unsubscription, eid, suid, sid);
-   }
-
-   void notify(EventBrokerID eid, StreamID sid, SubID suid,
-               EventPayload const &payload) override {
-      post_external_msg<Notification>(on_notification, eid, sid, suid, payload);
-   }
-   // End IEventBrokerPtr
-
-   // can be called from outside
-   void post_action(std::function<void()> const &action) {
-      post_external_msg<Action>(on_action, EventBrokerID(0), action);
-   }
-
-   // Internal API, i.e. API intended for use by a single thread, the
-   // same thread calling do_work.
-   struct InternalNotificationMsg {
-    public:
-      template <class T, typename... Args> T &wrap_in_place(Args &&... args) {
-         auto &n = _msg->wrapper.unwrap<InternalNotification>();
-         return n.payload.wrap_in_place<T>(std::forward<Args>(args)...);
-      }
-      BrokerMsg *_msg;
-   };
-
-   InternalNotificationMsg alloc_internal_notification_msg(StreamID sid) {
-      auto msg = _internal_pool.construct();
-      msg->is_external = false;
-      msg->handle_msg = on_internal_notification;
-      msg->wrapper.wrap_in_place<InternalNotification>(sid);
-      return InternalNotificationMsg{msg};
-   }
-
-   void internal_notify(InternalNotificationMsg msg) {
-      msg._msg->broker_src_id = id();
-      enqueue(msg._msg);
-   }
-
-   InternalSubID internal_subscribe(TopicFilter const &f,
-                                    InternalEventHandler const &cb) {
-      _current_suid = InternalSubID(_current_suid.value() + 1);
-      post_internal_msg<InternalSubscription>(on_internal_subscription,
-                                              _current_suid, f, cb);
-      return _current_suid;
-   }
-
-   void internal_unsubscribe(InternalSubID suid) {
-      post_internal_msg<InternalUnsubscription>(on_internal_unsubscription,
-                                                suid);
-   }
-
-   StreamID internal_advertise(TopicPtr const &t) {
-      auto sid = gen_stream_id();
-      post_external_msg<Advertisement>(on_advertisement, id(), sid, t);
-      return sid;
-   }
-
-   void internal_unadvertise(StreamID sid) {
-      post_external_msg<Unadvertisement>(on_unadvertisement, id(), sid);
-   }
-
-   bool do_work() {
-      BrokerMsg *m;
-      auto count = 0;
-      for(;;) {
-         m = nullptr;
-         _queue.pop(m);
-         if (m) {
-            LOG(Mercury, Diag, "{} dequeued msg {}:{} ", id().value(), m->broker_src_id.value(), m->msg_id);
-            m->handle_msg(this, m);
-            free_msg(m);
-            count ++;
-         }
-         else
-            break;
-      } 
-      return count;
-   }
-
- private:
-   void add_neighbour(IEventBrokerPtr const &n) override {
-      post_external_msg<NeighbourAddition>(on_neighbour_addition,
-                                           EventBrokerID(0), n);
-   }
-
-   void remove_neighbour(EventBrokerID n) override {
-      post_external_msg<NeighbourRemoval>(on_neighbour_removal,
-                                          EventBrokerID(0), n);
-   }
-
-   using MsgHandler = void(AsyncEventBroker *, BrokerMsg *);
-
-   using Pool = boost::object_pool<BrokerMsg>;
-   using Queue = boost::lockfree::queue<BrokerMsg *>;
-   using DispatchMap =
-       boost::container::flat_map<std::type_index,
-                                  std::function<void(BrokerMsg *)>>;
-   CoreEventBroker _core_broker;
-   Queue _queue;
-   Pool _internal_pool;
-   Pool _external_pool;
-   std::atomic_flag _external_pool_mtx = ATOMIC_FLAG_INIT;
-   DispatchMap _dispatch_map;
-   InternalSubID _current_suid;
-   static std::atomic<uint64_t> _msg_id;
-
-   void free_msg(BrokerMsg *m) {
-      m->wrapper.clear();
-      if (m->is_external) {
-         while (_external_pool_mtx.test_and_set(std::memory_order_acquire))
-            ;
-         _external_pool.destroy(m);
-         _external_pool_mtx.clear(std::memory_order_release);
-      } else
-         _internal_pool.destroy(m);
-   }
-
-   template <class T, typename... Args>
-   void post_external_msg(MsgHandler handler, EventBrokerID eid,
-                          Args &&... args) {
-
-      while (_external_pool_mtx.test_and_set(std::memory_order_acquire))
-         ;
-      auto msg = _external_pool.construct();
-      _external_pool_mtx.clear(std::memory_order_release);
-
-      msg->broker_src_id = eid;
-      msg->is_external = true;
-      msg->handle_msg = handler;
-      msg->wrapper.wrap_in_place<T>(eid, std::forward<Args>(args)...);
-      enqueue(msg);
-   }
-
-   template <class T, typename... Args>
-   void post_internal_msg(MsgHandler handler, Args &&... args) {
-      auto msg = _internal_pool.construct();
-      msg->broker_src_id = _core_broker.id();
-      msg->handle_msg = handler;
-      msg->is_external = false;
-      msg->wrapper.wrap_in_place<T>(std::forward<Args>(args)...);
-      enqueue(msg);
-   }
-
-   void enqueue(BrokerMsg *msg) {
-      auto mid = _msg_id.fetch_add(1);
-      msg->msg_id = mid;
-      LOG(Mercury, Diag, "{} enqueing msg {}:{}", id().value(), msg->broker_src_id.value(), mid);
-      _queue.push(msg);
-   }
-
-   struct NeighbourAddition {
-      NeighbourAddition(EventBrokerID, const IEventBrokerPtr &src) : ptr(src) {}
-      IEventBrokerPtr ptr;
-   };
-
-   struct NeighbourRemoval {
-      NeighbourRemoval(EventBrokerID, EventBrokerID id) : id(id) {}
-      EventBrokerID id;
-   };
-
-   struct Advertisement {
-      Advertisement(EventBrokerID id, StreamID sid, TopicPtr topic)
-          : id(id), sid(sid), topic(topic) {}
-      EventBrokerID id;
-      StreamID sid;
-      TopicPtr topic;
-   };
-
-   struct Unadvertisement {
-      Unadvertisement(EventBrokerID id, StreamID sid) : id(id), sid(sid) {}
-      EventBrokerID id;
-      StreamID sid;
-   };
-
-   struct Subscription {
-      Subscription(EventBrokerID id, SubID suid, StreamID sid)
-          : id(id), suid(suid), sid(sid) {}
-      EventBrokerID id;
-      SubID suid;
-      StreamID sid;
-   };
-
-   struct Unsubscription {
-      Unsubscription(EventBrokerID id, SubID suid, StreamID sid)
-          : id(id), suid(suid), sid(sid) {}
-      EventBrokerID id;
-      SubID suid;
-      StreamID sid;
-   };
-
-   struct Notification {
-      Notification(EventBrokerID id, StreamID sid, SubID suid,
-                   EventPayload const &payload)
-          : id(id), sid(sid), suid(suid), payload(payload) {}
-      EventBrokerID id;
-      StreamID sid;
-      SubID suid;
-      EventPayload payload;
-   };
-
-   struct InternalNotification {
-      InternalNotification(StreamID sid) : sid(sid), payload() {}
-      StreamID sid;
-      EventPayload payload;
-   };
-
-   struct InternalSubscription {
-      InternalSubscription(InternalSubID suid, TopicFilter const &filter,
-                           InternalEventHandler const &handler)
-          : suid(suid), filter(filter), handler(handler) {}
-      InternalSubID suid;
-      TopicFilter filter;
-      InternalEventHandler handler;
-   };
-
-   struct InternalUnsubscription {
-      InternalUnsubscription(InternalSubID suid) : suid(suid) {}
-      InternalSubID suid;
-   };
-
-   struct Action {
-      Action(EventBrokerID, std::function<void()> const &action)
-          : action(action) {}
-      std::function<void()> action;
-   };
-
-   static void on_neighbour_addition(AsyncEventBroker *self, BrokerMsg *m) {
-      self->_core_broker.add_neighbour(
-          m->wrapper.unwrap<NeighbourAddition>().ptr);
-   }
-
-   static void on_neighbour_removal(AsyncEventBroker *self, BrokerMsg *m) {
-      self->_core_broker.remove_neighbour(
-          m->wrapper.unwrap<NeighbourRemoval>().id);
-   }
-
-   static void on_advertisement(AsyncEventBroker *self, BrokerMsg *m) {
-      auto const &adv = m->wrapper.unwrap<Advertisement>();
-      self->_core_broker.advertise(adv.id, adv.sid, adv.topic);
-   }
-
-   static void on_unadvertisement(AsyncEventBroker *self, BrokerMsg *m) {
-      auto const &uadv = m->wrapper.unwrap<Unadvertisement>();
-      self->_core_broker.unadvertise(uadv.id, uadv.sid);
-   }
-
-   static void on_subscription(AsyncEventBroker *self, BrokerMsg *m) {
-      auto const &sub = m->wrapper.unwrap<Subscription>();
-      self->_core_broker.subscribe(sub.id, sub.suid, sub.sid);
-   }
-
-   static void on_unsubscription(AsyncEventBroker *self, BrokerMsg *m) {
-      auto const &usub = m->wrapper.unwrap<Unsubscription>();
-      self->_core_broker.unsubscribe(usub.id, usub.suid, usub.sid);
-   }
-
-   static void on_notification(AsyncEventBroker *self, BrokerMsg *m) {
-      auto const &notify = m->wrapper.unwrap<Notification>();
-      self->_core_broker.notify(notify.id, notify.sid, notify.suid,
-                                notify.payload);
-   }
-
-   static void on_internal_notification(AsyncEventBroker *self, BrokerMsg *m) {
-      auto const &inotify = m->wrapper.unwrap<InternalNotification>();
-      self->_core_broker.notify(self->_core_broker.id(), inotify.sid, SubID(0),
-                                inotify.payload);
-   }
-
-   static void on_internal_subscription(AsyncEventBroker *self, BrokerMsg *m) {
-      auto const &isub = m->wrapper.unwrap<InternalSubscription>();
-      self->_core_broker.internal_subscribe(isub.suid, isub.filter,
-                                            isub.handler);
-   }
-
-   static void on_internal_unsubscription(AsyncEventBroker *self,
-                                          BrokerMsg *m) {
-      auto const &iunsub = m->wrapper.unwrap<InternalUnsubscription>();
-      self->_core_broker.internal_unsubscribe(iunsub.suid);
-   }
-
-   static void on_action(AsyncEventBroker *, BrokerMsg *m) {
-      auto const &action = m->wrapper.unwrap<Action>();
-      action.action();
-   }
-};
-} // namespace mercury
-} // namespace bts
@@ -1,9 +1,9 @@
-#include "CoreEventBroker.h"
+#include "lib/mercury/broker/Broker.h"

-#include "lib/utils/Puid.h"
-#include "lib/utils/bts_assert.h"
 #include "lib/mercury/synopsis/ISynopsisBuilder.h"
 #include "lib/mercury/topics/ITopic.h"
+#include "lib/utils/Puid.h"
+#include "lib/utils/bts_assert.h"

 #include <algorithm>
 #include <boost/container/flat_map.hpp>
@@ -13,7 +13,7 @@ namespace mercury {

 template <class K, class V> using Map = boost::container::flat_map<K, V>;

-struct CoreEventBroker::NoReenter {
+struct Broker::NoReenter {
   NoReenter(bool &inside) : inside(inside) {
      bts_assert(!inside);
      inside = true;
@@ -22,47 +22,47 @@ struct CoreEventBroker::NoReenter {
   bool &inside;
 };

-struct CoreEventBroker::InternalSubFilter {
+struct Broker::LocalSubFilter {
   TopicFilter matches;
-   InternalEventHandler handler;
+   LocalEventHandler handler;
 };

-struct CoreEventBroker::StreamRec {
-   StreamRec(CoreEventBroker &owner, IEventBrokerPtr const &src, StreamID sid,
+struct Broker::StreamRec {
+   StreamRec(Broker &owner, IRemoteBroker &src, StreamID sid,
             TopicPtr const &topic)
-       : owner(owner), local(false), src(src), sid(sid), topic(topic),
-         internal_subs(), external_subs(), sub_id(),
+       : owner(owner), local(false), src(&src), sid(sid), topic(topic),
+         local_subs(), external_subs(), sub_id(),
         synopsis_builder(topic->make_synopsis_builder()),
         topic_tag(topic->to_str()) {}

-   StreamRec(CoreEventBroker &owner, StreamID sid, TopicPtr const &topic)
-       : owner(owner), local(true), src(), sid(sid), topic(topic),
-         internal_subs(), external_subs(), sub_id(),
+   StreamRec(Broker &owner, StreamID sid, TopicPtr const &topic)
+       : owner(owner), local(true), src(), sid(sid), topic(topic), local_subs(),
+         external_subs(), sub_id(),
         synopsis_builder(topic->make_synopsis_builder()),
         topic_tag(topic->to_str()) {}

   ~StreamRec() {
-      for (auto &isub : internal_subs)
+      for (auto &isub : local_subs)
         isub.on_event(sid, EventType::END_STREAM, nullptr);
   }

-   struct InternalSubRec {
-      InternalSubID sub_id;
-      InternalEventHandler on_event;
+   struct LocalSubRec {
+      LocalSubID sub_id;
+      LocalEventHandler on_event;
   };

   struct SubRec {
      SubID sub_id;
-      IEventBrokerPtr neighbour;
+      IRemoteBroker *neighbour;
   };

-   CoreEventBroker &owner;
+   Broker &owner;
   const bool local;
-   const IEventBrokerPtr src;
+   IRemoteBroker *const src;
   const StreamID sid;
   const TopicPtr topic;

-   std::vector<InternalSubRec> internal_subs;
+   std::vector<LocalSubRec> local_subs;
   std::vector<SubRec> external_subs;

   SubID sub_id;
@@ -71,11 +71,11 @@ struct CoreEventBroker::StreamRec {

   bool is_local() const { return local; }
   bool is_active() const {
-      return is_local() || internal_subs.size() || external_subs.size();
+      return is_local() || local_subs.size() || external_subs.size();
   }
-   EventPayload *get_synopsis() { return synopsis_builder->get_synopsis(); }
+   Notification *get_synopsis() { return synopsis_builder->get_synopsis(); }

-   void add_external_sub(IEventBrokerPtr const &node, SubID subid) {
+   void add_external_sub(IRemoteBroker *const &node, SubID subid) {
      auto sender = node->id();
      bts_assert(std::find_if(external_subs.begin(), external_subs.end(),
                              [sender](auto const &sub_rec) {
@@ -95,21 +95,22 @@ struct CoreEventBroker::StreamRec {
         // we were already active so if we have a synopsis forward it
         auto synopsis = synopsis_builder->get_synopsis();
         if (synopsis) {
-            LOG(Mercury, Info, "CoreEventBroker {} sending synopsis for sid: "
-                               "{} to broker {} (sub_id:{} topic: {})",
+            LOG(Mercury, Info,
+                "Broker {} sending synopsis for sid: "
+                "{} to broker {} (sub_id:{} topic: {})",
                owner.id().value(), sid.value(), sender.value(), subid.value(),
                topic_tag);
            node->notify(owner.id(), sid, subid, *synopsis);
         } else
            LOG(Mercury, Info,
-                "CoreEventBroker {} does not have synopsis for sid: "
+                "Broker {} does not have synopsis for sid: "
                "{} for broker {} (sub_id:{} topic:{})",
                owner.id().value(), sid.value(), sender.value(), subid.value(),
                topic_tag);
      }
   }

-   void remove_external_sub(EventBrokerID sender, SubID subid) {
+   void remove_external_sub(BrokerID sender, SubID subid) {
      auto it = std::find_if(external_subs.begin(), external_subs.end(),
                             [sender](auto const &sub_rec) {
                                return sub_rec.neighbour->id() == sender;
@@ -121,33 +122,32 @@ struct CoreEventBroker::StreamRec {
         forward_unsub();
   }

-   void add_internal_sub(InternalEventHandler const &handler,
-                         InternalSubID sub_id) {
+   void add_local_sub(LocalEventHandler const &handler, LocalSubID sub_id) {
      auto it =
-          std::find_if(internal_subs.begin(), internal_subs.end(),
+          std::find_if(local_subs.begin(), local_subs.end(),
                       [sub_id](auto &sub) { return sub.sub_id == sub_id; });
-      bts_assert(it == internal_subs.end());
+      bts_assert(it == local_subs.end());
      auto was_active = is_active();
-      internal_subs.push_back(InternalSubRec{sub_id, handler});
+      local_subs.push_back(LocalSubRec{sub_id, handler});
      handler(sid, EventType::BEGIN_STREAM, get_synopsis());
      if (!was_active)
         forward_sub();
   }

-   void remove_internal_sub(InternalSubID sub_id) {
+   void remove_local_sub(LocalSubID sub_id) {
      auto it =
-          std::find_if(internal_subs.begin(), internal_subs.end(),
+          std::find_if(local_subs.begin(), local_subs.end(),
                       [sub_id](auto &sub) { return sub.sub_id == sub_id; });
-      bts_assert(it != internal_subs.end());
+      bts_assert(it != local_subs.end());
      it->on_event(sid, EventType::END_SUB, nullptr);
-      internal_subs.erase(it);
+      local_subs.erase(it);
      if (!is_active())
         forward_unsub();
   }

-   void notify(EventPayload const &payload) {
+   void notify(Notification const &payload) {
      // notify local subs
-      for (auto &isub : internal_subs)
+      for (auto &isub : local_subs)
         isub.on_event(sid, EventType::NEW_EVENT, &payload);

      // fwd to remote subs
@@ -160,14 +160,14 @@ struct CoreEventBroker::StreamRec {

   void forward_sub() {
      bts_assert(!is_local());
-      bts_assert(internal_subs.size() || external_subs.size());
+      bts_assert(local_subs.size() || external_subs.size());

-      sub_id = owner.gen_external_sub_id();
+      sub_id = owner.gen_remote_sub_id();

      LOG(Mercury, Info,
-          "CoreEventBroker {} issuing subscription for topic: {} sid: {} "
+          "Broker {} issuing subscription for topic: {} sid: {} "
          "to broker {} (sub_id:{})",
-          owner.id().value(),topic_tag , sid.value(), src->id().value(),
+          owner.id().value(), topic_tag, sid.value(), src->id().value(),
          sub_id.value());

      src->subscribe(owner.id(), sub_id, sid);
@@ -176,49 +176,54 @@ struct CoreEventBroker::StreamRec {

   void forward_unsub() {
      bts_assert(!is_local());
-      bts_assert(internal_subs.empty() && external_subs.empty());
-      LOG(Mercury, Info, "CoreEventBroker {} canceling subscription for {} "
-                         "from {} (sub_id: {})",
+      bts_assert(local_subs.empty() && external_subs.empty());
+      LOG(Mercury, Info,
+          "Broker {} canceling subscription for {} "
+          "from {} (sub_id: {})",
          owner.id().value(), sid.value(), src->id().value(), sub_id.value());
      src->unsubscribe(owner.id(), sub_id, sid);
   }
 };

-struct CoreEventBroker::PImpl {
+struct Broker::PImpl {
   PImpl()
-       : streams(), neighbours(), internal_subs(),
+       : streams(), neighbours(), local_subs(),
         my_id(utils::gen_id("event_broker")), last_external_sub_id(),
         inside(false) {}

   Map<StreamID, std::unique_ptr<StreamRec>> streams;
-   Map<EventBrokerID, IEventBrokerPtr> neighbours;
-   Map<InternalSubID, InternalSubFilter> internal_subs;
-   EventBrokerID my_id;
+   Map<BrokerID, IRemoteBroker *> neighbours;
+   Map<LocalSubID, LocalSubFilter> local_subs;
+   BrokerID my_id;
   SubID last_external_sub_id;
   bool inside;
 };

-CoreEventBroker::CoreEventBroker() : me(new PImpl()) {}
-CoreEventBroker::~CoreEventBroker() {}
+Broker::Broker() : me(new PImpl()) {}
+Broker::~Broker() {}

-EventBrokerID CoreEventBroker::id() const { return me->my_id; }
+BrokerID Broker::id() const { return me->my_id; }

-void CoreEventBroker::add_neighbour(IEventBrokerPtr const &neighbour) {
-   LOG(Mercury, Info, "CoreEventBroker {} add_neighbour:  {}",
-       me->my_id.value(), neighbour->id().value());
+void Broker::add_neighbour(IRemoteBroker &neighbour) {
+   LOG(Mercury, Info, "Broker {} add_neighbour:  {}", me->my_id.value(),
+       neighbour.id().value());
   NoReenter nr(me->inside);

-   auto &n = me->neighbours[neighbour->id()];
+   auto &n = me->neighbours[neighbour.id()];
   bts_assert(!n);
-   n = neighbour;
+   n = &neighbour;

+   // Since we assume direct connections between all brokers (i.e. no
+   // forwarding of notifications is needed), we only need to
+   // advertise our own streams to this new neighbour
   for (auto const &s : me->streams)
-      n->advertise(me->my_id, s.first, s.second->topic);
+      if (s.second->is_local())
+         n->advertise(me->my_id, s.first, s.second->topic);
 }

-void CoreEventBroker::remove_neighbour(EventBrokerID removed_neighbour_id) {
-   LOG(Mercury, Info, "CoreEventBroker {} remove_neighbour broker {} ",
-       id().value(), removed_neighbour_id.value());
+void Broker::remove_neighbour(BrokerID removed_neighbour_id) {
+   LOG(Mercury, Info, "Broker {} remove_neighbour broker {} ", id().value(),
+       removed_neighbour_id.value());
   NoReenter nr(me->inside);

   auto &n_rec = me->neighbours[removed_neighbour_id];
@@ -247,7 +252,7 @@ void CoreEventBroker::remove_neighbour(EventBrokerID removed_neighbour_id) {
   // unadvertise orphan streams
   for (auto &sid : its_streams) {
      LOG(Mercury, Info,
-          "CoreEventBroker {} sending unadvertisement for stream owned by dead "
+          "Broker {} sending unadvertisement for stream owned by dead "
          "node: {} sid: {}",
          id().value(), removed_neighbour_id.value(), sid.value());
      _unadvertise(removed_neighbour_id, sid);
@@ -258,7 +263,7 @@ void CoreEventBroker::remove_neighbour(EventBrokerID removed_neighbour_id) {
      auto sub_id = p.first;
      auto sid = p.second;
      LOG(Mercury, Info,
-          "CoreEventBroker {} canceling subscription for stream requested by "
+          "Broker {} canceling subscription for stream requested by "
          "dead node: {} sid: {} (sub_id:{})",
          id().value(), removed_neighbour_id.value(), sid.value(),
          sub_id.value());
@@ -269,10 +274,9 @@ void CoreEventBroker::remove_neighbour(EventBrokerID removed_neighbour_id) {
   me->neighbours.erase(removed_neighbour_id);
 }

-void CoreEventBroker::advertise(EventBrokerID sender, StreamID sid,
-                                TopicPtr const &topic) {
+void Broker::advertise(BrokerID sender, StreamID sid, TopicPtr const &topic) {
   LOG(Mercury, Info,
-       "CoreEventBroker {} received advertisement from broker {}  "
+       "Broker {} received advertisement from broker {}  "
       "sid: {} topic: {}",
       id().value(), sender.value(), sid.value(), topic->to_str());
   NoReenter nr(me->inside);
@@ -285,72 +289,69 @@ void CoreEventBroker::advertise(EventBrokerID sender, StreamID sid,
      // external advertisement, must come from a known neighbour
      auto const &node = me->neighbours[sender];
      bts_assert(node);
-      srec = std::make_unique<StreamRec>(*this, node, sid, topic);
-   } else
+      srec = std::make_unique<StreamRec>(*this, *node, sid, topic);
+   } else {
      srec = std::make_unique<StreamRec>(*this, sid, topic);
-
-   // forward to neighbours
-   for (auto &kvp : me->neighbours)
-      if (kvp.first != sender) {
+      bts_assert(srec->is_local());
+      // only advertisments for streams we produce will be forwarded
+      // to our neighbours (since all nodes are directly
+      // interconnected)
+      for (auto &kvp : me->neighbours) {
         LOG(Mercury, Info,
-             "CoreEventBroker {} forwarding advertisement for stream {} owned "
+             "Broker {} forwarding advertisement for stream {} owned "
             "by broker {}  to: {} sid: {} ",
             id().value(), topic->to_str(), sender.value(), kvp.first.value(),
             sid.value());
         kvp.second->advertise(me->my_id, sid, topic);
      }
+   }

-   // match to existing internal subscriptions
+   // match to existing local subscriptions
   bool found = false;
-   for (auto const &kvp : me->internal_subs) {
-      auto internal_sub_id = kvp.first;
+   for (auto const &kvp : me->local_subs) {
+      auto local_sub_id = kvp.first;
      auto const &f = kvp.second;
      if (f.matches(*topic)) {
         found = true;
-         LOG(Mercury, Info,
-             "CoreEventBroker {} matched internal sub {} to topic {} sid {}",
-             id().value(), internal_sub_id, topic->to_str(), sid.value());
-         srec->add_internal_sub(f.handler, internal_sub_id);
+         LOG(Mercury, Info, "Broker {} matched local sub {} to topic {} sid {}",
+             id().value(), local_sub_id, topic->to_str(), sid.value());
+         srec->add_local_sub(f.handler, local_sub_id);
      }
   }
   if (!found)
      LOG(Mercury, Info,
-          "CoreEventBroker {} has no internal sub matches to topic {} sid {}",
-          id().value(), topic->to_str(), sid.value());
+          "Broker {} has no local sub matches to topic {} sid {}", id().value(),
+          topic->to_str(), sid.value());
 }

-void CoreEventBroker::unadvertise(EventBrokerID sender, StreamID sid) {
+void Broker::unadvertise(BrokerID sender, StreamID sid) {
   LOG(Mercury, Info,
-       "CoreEventBroker {} received unadvertisement from broker {}  sid: {}",
+       "Broker {} received unadvertisement from broker {}  sid: {}",
       id().value(), sender.value(), sid.value());
   NoReenter nr(me->inside);
   _unadvertise(sender, sid);
 }

-void CoreEventBroker::notify(EventBrokerID sender, StreamID sid, SubID ex_subid,
-                             EventPayload const &payload) {
-   LOG(Mercury, Diag,
-       "CoreEventBroker {} received notify from broker {}  sid: {} sub_id:{}",
-       id().value(), sender.value(), sid.value(), ex_subid.value());
+#ifndef NDEBUG
+void Broker::notify(BrokerID sender, StreamID sid, SubID ex_subid,
+                    Notification const &payload) {
   NoReenter nr(me->inside);
-
+#else
+void Broker::notify(BrokerID , StreamID sid, SubID ex_subid,
+                    Notification const &payload) {
+#endif
   auto &srec = me->streams[sid];
-   if (!srec)
-      LOG(Mercury, Diag,
-          "Will crash CoreEventBroker {} received notify from broker {}  sid: {} sub_id:{}",
-          id().value(), sender.value(), sid.value(), ex_subid.value());
-      
-   bts_assert(srec);
-   bts_assert((srec->is_local() && sender == id()) ||
-              (!srec->is_local() && srec->src->id() == sender));
+   bts_assert_d(srec);
+   bts_assert_d((srec->is_local() && sender == id()) ||
+                (!srec->is_local() && srec->src->id() == sender));
   if (srec->is_local() || srec->sub_id == ex_subid)
      srec->notify(payload);
 }

-void CoreEventBroker::subscribe(EventBrokerID sender, SubID ex_subid,
-                                StreamID sid) {
-   LOG(Mercury, Info, "CoreEventBroker {} received subscribe from broker {}  "
-                      "sid: {} sub_id:{}",
+void Broker::subscribe(BrokerID sender, SubID ex_subid, StreamID sid) {
+   LOG(Mercury, Info,
+       "Broker {} received subscribe from broker {}  "
+       "sid: {} sub_id:{}",
       id().value(), sender.value(), sid.value(), ex_subid.value());
   NoReenter nr(me->inside);

@@ -362,33 +363,32 @@ void CoreEventBroker::subscribe(EventBrokerID sender, SubID ex_subid,
   if (it != me->streams.end())
      it->second->add_external_sub(node, ex_subid);
   else
-      LOG(Mercury, Info, "CoreEventBroker {} received external subscribe for "
-                         "dead stream {}. ignoring",
+      LOG(Mercury, Info,
+          "Broker {} received external subscribe for "
+          "dead stream {}. ignoring",
          id().value(), sid.value());
 }

-void CoreEventBroker::unsubscribe(EventBrokerID sender, SubID ex_subid,
-                                  StreamID sid) {
-   LOG(Mercury, Info, "CoreEventBroker {} received unsubscribe from broker {} "
-                      "sid: {} sub_id:{}",
+void Broker::unsubscribe(BrokerID sender, SubID ex_subid, StreamID sid) {
+   LOG(Mercury, Info,
+       "Broker {} received unsubscribe from broker {} "
+       "sid: {} sub_id:{}",
       id().value(), sender.value(), sid.value(), ex_subid.value());
   NoReenter nr(me->inside);
   _unsubscribe(sender, ex_subid, sid);
 }

-void CoreEventBroker::internal_subscribe(InternalSubID in_subid,
-                                         TopicFilter const &f,
-                                         InternalEventHandler const &cb) {
-   LOG(Mercury, Info,
-       "CoreEventBroker {} received internal_subscribe (sub_id: {})",
+void Broker::local_subscribe(LocalSubID in_subid, TopicFilter const &f,
+                             LocalEventHandler const &cb) {
+   LOG(Mercury, Info, "Broker {} received local_subscribe (sub_id: {})",
       id().value(), in_subid.value());
   NoReenter nr(me->inside);

   // no duplicates
-   bts_assert(me->internal_subs.find(in_subid) == me->internal_subs.end());
+   bts_assert(me->local_subs.find(in_subid) == me->local_subs.end());

-   // add to internal sub list
-   auto &sf = me->internal_subs[in_subid];
+   // add to local sub list
+   auto &sf = me->local_subs[in_subid];
   sf.matches = f;
   sf.handler = cb;

@@ -399,74 +399,77 @@ void CoreEventBroker::internal_subscribe(InternalSubID in_subid,
      auto &srec = kvp.second;
      if (sf.matches(*srec->topic)) {
         found = true;
-         LOG(Mercury, Info,
-             "CoreEventBroker {} matched internal sub {} to topic {} sid {}",
+         LOG(Mercury, Info, "Broker {} matched local sub {} to topic {} sid {}",
             id().value(), in_subid, srec->topic->to_str(), kvp.first.value());
-         srec->add_internal_sub(cb, in_subid);
+         srec->add_local_sub(cb, in_subid);
      }
   }
   if (!found)
-      LOG(Mercury, Info,
-          "CoreEventBroker {} found no topics to match internal sub {}",
+      LOG(Mercury, Info, "Broker {} found no topics to match local sub {}",
          id().value(), in_subid.value());
 }

-void CoreEventBroker::internal_unsubscribe(InternalSubID in_subid) {
-   LOG(Mercury, Info,
-       "CoreEventBroker {} received internal_unsubscribe (sub_id: {})",
+void Broker::local_unsubscribe(LocalSubID in_subid) {
+   LOG(Mercury, Info, "Broker {} received local_unsubscribe (sub_id: {})",
       id().value(), in_subid.value());
   NoReenter nr(me->inside);

   // must be a previously entered sub
-   auto it = me->internal_subs.find(in_subid);
-   bts_assert(it != me->internal_subs.end());
+   auto it = me->local_subs.find(in_subid);
+   bts_assert(it != me->local_subs.end());
   auto &matches = it->second.matches;

-   // for each stream that matched this filter remove the corresponding internal
+   // for each stream that matched this filter remove the corresponding local
   // sub
   for (auto &kvp : me->streams) {
      auto &srec = kvp.second;
      if (matches(*srec->topic))
-         srec->remove_internal_sub(in_subid);
+         srec->remove_local_sub(in_subid);
   }
 }

-SubID CoreEventBroker::gen_external_sub_id() {
+SubID Broker::gen_remote_sub_id() {
   me->last_external_sub_id = SubID(me->last_external_sub_id.value() + 1);
   return me->last_external_sub_id;
 }

-void CoreEventBroker::_unadvertise(EventBrokerID sender, StreamID sid) {
+void Broker::_unadvertise(BrokerID sender, StreamID sid) {
   auto it = me->streams.find(sid);
   bts_assert(it != me->streams.end());

   auto &srec = it->second;
+
+   // either the stream is local and therefore the sender is us or the
+   // stream is not local and the sender is in srec
   bts_assert((sender == id() && srec->is_local()) ||
              (!srec->is_local() && sender == srec->src->id()));

-   auto topic_str = srec->topic->to_str();
-
   // remove from our index
   me->streams.erase(it);

-   for (auto &kvp : me->neighbours)
-      if (kvp.first != sender) {
-         LOG(Mercury, Info, "CoreEventBroker {} forwarding unadvertisement for "
-                            "stream {} owned "
-                            "by broker {}  to: {} sid: {}",
-             id().value(), topic_str, sender.value(), kvp.first.value(),
-             sid.value());
-         kvp.second->unadvertise(me->my_id, sid);
-      }
+   // We only need to send unadvertisements for streams we own, since
+   // we assume direct connections between all brokers.
+   if (srec->is_local()) {
+      auto topic_str = srec->topic->to_str();
+      for (auto &kvp : me->neighbours)
+         if (kvp.first != sender) {
+            LOG(Mercury, Info,
+                "Broker {} forwarding unadvertisement for "
+                "stream {} owned "
+                "by broker {}  to: {} sid: {}",
+                id().value(), topic_str, sender.value(), kvp.first.value(),
+                sid.value());
+            kvp.second->unadvertise(me->my_id, sid);
+         }
+   }
 }

-void CoreEventBroker::_unsubscribe(EventBrokerID sender, SubID sub_id,
-                                   StreamID sid) {
+void Broker::_unsubscribe(BrokerID sender, SubID sub_id, StreamID sid) {
   auto it = me->streams.find(sid);
   if (it != me->streams.end()) {
      auto &srec = it->second;
      srec->remove_external_sub(sender, sub_id);
   }
 }
-}
-}
+} // namespace mercury
+} // namespace bts
@@ -0,0 +1,47 @@
+#pragma once
+
+#include <memory>
+#include "lib/mercury/broker/ILocalBroker.h"
+
+namespace bts {
+namespace mercury {
+
+class Broker : public ILocalBroker 
+{
+public:
+   Broker();
+   virtual ~Broker();
+
+   BrokerID id() const override;
+   
+   void add_neighbour(IRemoteBroker &) override;
+   void remove_neighbour(BrokerID) override;
+
+   void advertise(BrokerID, StreamID, TopicPtr const &) override;
+   void unadvertise(BrokerID, StreamID) override;
+
+   void subscribe(BrokerID, SubID, StreamID) override;
+   void unsubscribe(BrokerID, SubID, StreamID) override;
+
+   void notify(BrokerID, StreamID, SubID, Notification const &) override;
+
+   void local_subscribe(LocalSubID, TopicFilter const &,
+                           LocalEventHandler const &cb) override;
+   void local_unsubscribe(LocalSubID) override;
+
+private:
+   struct NoReenter;
+   struct StreamRec;
+   struct LocalSubRec;
+   struct SubRec;
+   struct LocalSubFilter;
+   struct PImpl;
+
+   SubID gen_remote_sub_id();
+   void _unadvertise(BrokerID,StreamID) ;
+   void _unsubscribe (BrokerID,SubID,StreamID) ;
+
+   std::unique_ptr<PImpl> me;
+};
+
+}}
@@ -0,0 +1,99 @@
+#include "lib/mercury/broker/BrokerProxy.h"
+
+namespace bts {
+namespace mercury {
+
+BrokerProxy::BrokerProxy(ILocalBroker &real_broker)
+   :  _real_broker(real_broker), _id(real_broker.id()), _outgoing(1000), _incoming(1000), _action_pool()
+{}
+
+BrokerProxy::~BrokerProxy() {}
+
+
+void BrokerProxy::add_neighbour(IRemoteBroker &n) {
+   auto m = _action_pool.construct();
+   m->msg_type = BrokerMsgType::ADD_NEIGHBOUR;
+   m->neighbour = &n;
+   send(*m);
+}
+
+void BrokerProxy::remove_neighbour(BrokerID id) {
+   auto m = _action_pool.construct();
+   m->msg_type = BrokerMsgType::REMOVE_NEIGHBOUR;
+   m->broker_id = id;
+   send(*m);
+}
+
+void BrokerProxy::advertise(BrokerID bid, StreamID sid, TopicPtr const &t) {
+   auto m = _action_pool.construct();
+   m->msg_type = BrokerMsgType::ADVERTISE;
+   m->broker_id = bid;
+   m->stream_id = sid;
+   m->topic = t;
+   send(*m);
+}
+
+void BrokerProxy::unadvertise(BrokerID bid, StreamID sid) {
+   auto m = _action_pool.construct();
+   m->msg_type = BrokerMsgType::UNADVERTISE;
+   m->broker_id = bid;
+   m->stream_id = sid;
+   send(*m);
+}
+
+void BrokerProxy::subscribe(BrokerID bid, SubID sub_id, StreamID sid) {
+   auto m = _action_pool.construct();
+   m->msg_type = BrokerMsgType::SUBSCRIBE;
+   m->broker_id = bid;
+   m->stream_id = sid;
+   m->sub_id = sub_id;
+   send(*m);
+}
+
+void BrokerProxy::unsubscribe(BrokerID bid, SubID sub_id, StreamID sid) {
+   auto m = _action_pool.construct();
+   m->msg_type = BrokerMsgType::UNSUBSCRIBE;
+   m->broker_id = bid;
+   m->stream_id = sid;
+   m->sub_id = sub_id;
+   send(*m);
+}
+
+void BrokerProxy::notify(BrokerID bid, StreamID sid, SubID sub_id,
+                         Notification const &n) {
+   auto m = _action_pool.construct();
+   m->msg_type = BrokerMsgType::NOTIFY;
+   m->broker_id = bid;
+   m->stream_id = sid;
+   m->sub_id = sub_id;
+   m->notification = n;
+   send(*m);
+}
+
+void BrokerProxy::local_subscribe(LocalSubID lsid, TopicFilter const &tf,
+                                  LocalEventHandler const &handler) {
+   auto m = _action_pool.construct();
+   m->msg_type = BrokerMsgType::LOCAL_SUBSCRIBE;
+   m->local_sub_id = lsid;
+   m->filter = tf;
+   m->local_handler = handler;
+   send(*m);
+}
+
+void BrokerProxy::local_unsubscribe(LocalSubID lsid) {
+   auto m = _action_pool.construct();
+   m->msg_type = BrokerMsgType::LOCAL_UNSUBSCRIBE;
+   m->local_sub_id = lsid;
+   send(*m);
+}
+
+void BrokerProxy::schedule_action (std::function<void()> const&action)
+{
+   auto m = _action_pool.construct();
+   m->msg_type = BrokerMsgType::EXECUTE_ACTION;
+   m->action = action;
+   send(*m);
+}
+
+} // namespace mercury
+} // namespace bts
@@ -0,0 +1,138 @@
+#pragma once
+
+#include "lib/mercury/broker/ILocalBroker.h"
+#include "lib/mercury/common/mercury_common.h"
+#include "lib/mercury/topics/Topic.h"
+
+#include <boost/lockfree/spsc_queue.hpp>
+#include <boost/pool/object_pool.hpp>
+#include <memory>
+
+namespace bts {
+namespace mercury {
+
+class BrokerProxy : public ILocalBroker {
+ protected:
+   friend class ActorBroker;
+   enum class BrokerMsgType {
+      ADD_NEIGHBOUR,
+      REMOVE_NEIGHBOUR,
+      ADVERTISE,
+      UNADVERTISE,
+      SUBSCRIBE,
+      UNSUBSCRIBE,
+      NOTIFY,
+      LOCAL_SUBSCRIBE,
+      LOCAL_UNSUBSCRIBE,
+      EXECUTE_ACTION
+   };
+
+   struct BrokerMsg {
+      BrokerMsgType msg_type;
+      BrokerID broker_id;
+      StreamID stream_id;
+      LocalSubID local_sub_id;
+      Notification notification;
+      TopicPtr topic;
+      SubID sub_id;
+      TopicFilter filter;
+      LocalEventHandler local_handler;
+      IRemoteBroker *neighbour;
+      std::function<void()> action;
+   };
+
+ public:
+   BrokerProxy(ILocalBroker &real_broker);
+   virtual ~BrokerProxy();
+
+   // ILocalBroker
+   BrokerID id() const override { return _id; }
+   void add_neighbour(IRemoteBroker &) override;
+   void remove_neighbour(BrokerID) override;
+   void advertise(BrokerID, StreamID, TopicPtr const &) override;
+   void unadvertise(BrokerID, StreamID) override;
+   void subscribe(BrokerID, SubID, StreamID) override;
+   void unsubscribe(BrokerID, SubID, StreamID) override;
+   void notify(BrokerID, StreamID, SubID, Notification const &) override;
+   void local_subscribe(LocalSubID, TopicFilter const &,
+                        LocalEventHandler const &cb) override;
+   void local_unsubscribe(LocalSubID) override;
+   // ILocalBroker
+
+   void schedule_action(std::function<void()> const &action);
+
+   // these are just for performance, we want to allow for
+   // notifications to be initialized in place but this class does not
+   // know how to do that, so we need to expose the notification
+   // member to the outside.
+   void alloc_notify(BrokerMsg **msg, Notification **n) {
+      auto m = _action_pool.construct();
+      *msg = m;
+      *n = &m->notification;
+   }
+   void send_notify(BrokerID bid, StreamID sid, SubID suid, BrokerMsg &m) {
+      m.msg_type = BrokerMsgType::NOTIFY;
+      m.broker_id = bid;
+      m.stream_id = sid;
+      m.sub_id = suid;
+      send(m);
+   }
+
+   void do_work() {
+      _outgoing.consume_all([this](BrokerMsg *m) {
+         switch (m->msg_type) {
+         case BrokerMsgType::ADD_NEIGHBOUR:
+            _real_broker.add_neighbour(*m->neighbour);
+            break;
+         case BrokerMsgType::REMOVE_NEIGHBOUR:
+            _real_broker.remove_neighbour(m->broker_id);
+            break;
+         case BrokerMsgType::ADVERTISE:
+            _real_broker.advertise(m->broker_id, m->stream_id, m->topic);
+            break;
+         case BrokerMsgType::UNADVERTISE:
+            _real_broker.unadvertise(m->broker_id, m->stream_id);
+            break;
+         case BrokerMsgType::SUBSCRIBE:
+            _real_broker.subscribe(m->broker_id, m->sub_id, m->stream_id);
+            break;
+         case BrokerMsgType::UNSUBSCRIBE:
+            _real_broker.unsubscribe(m->broker_id, m->sub_id, m->stream_id);
+            break;
+         case BrokerMsgType::NOTIFY:
+            _real_broker.notify(m->broker_id, m->stream_id, m->sub_id,
+                                m->notification);
+            break;
+         case BrokerMsgType::LOCAL_SUBSCRIBE:
+            _real_broker.local_subscribe(m->local_sub_id, m->filter,
+                                         m->local_handler);
+            break;
+         case BrokerMsgType::LOCAL_UNSUBSCRIBE:
+            _real_broker.local_unsubscribe(m->local_sub_id);
+            break;
+         case BrokerMsgType::EXECUTE_ACTION:
+            m->action();
+            break;
+         default:
+            bts_assert(false);
+            break;
+         }
+         _incoming.push(m);
+      });
+   }
+
+ private:
+   ILocalBroker &_real_broker;
+   const BrokerID _id;
+   boost::lockfree::spsc_queue<BrokerMsg *> _outgoing;
+   boost::lockfree::spsc_queue<BrokerMsg *> _incoming;
+   boost::object_pool<BrokerMsg> _action_pool;
+
+   void send(BrokerMsg &m) {
+      _outgoing.push(&m);
+      _incoming.consume_all([this](BrokerMsg *m) { _action_pool.destroy(m); });
+   }
+};
+
+} // namespace mercury
+} // namespace bts
@@ -1,46 +0,0 @@
-#pragma once
-
-#include <memory>
-#include "lib/mercury/broker/IEventBroker.h"
-
-namespace bts {
-namespace mercury {
-
-class CoreEventBroker : public IEventBroker
-{
-public:
-   CoreEventBroker();
-    ~CoreEventBroker();
-
-   EventBrokerID id() const override;
-   
-   void add_neighbour(IEventBrokerPtr  const &) override;
-   void remove_neighbour(EventBrokerID) override;
-
-   void advertise(EventBrokerID,StreamID,TopicPtr const&) override;
-   void unadvertise(EventBrokerID,StreamID) override;
-
-   void subscribe(EventBrokerID,SubID,StreamID) override;
-   void unsubscribe(EventBrokerID,SubID,StreamID) override;
-
-   void notify(EventBrokerID,StreamID,SubID,EventPayload const&) override;
-
-   void internal_subscribe(InternalSubID,TopicFilter const&,InternalEventHandler const&cb);
-   void internal_unsubscribe(InternalSubID);
-
-private:
-   struct NoReenter;
-   struct StreamRec;
-   struct InternalSubRec;
-   struct SubRec;
-   struct InternalSubFilter;
-   struct PImpl;
-
-   SubID gen_external_sub_id();
-   void _unadvertise(EventBrokerID,StreamID) ;
-   void _unsubscribe (EventBrokerID,SubID,StreamID) ;
-
-   std::unique_ptr<PImpl> me;
-};
-
-}}
@@ -1,8 +0,0 @@
-#include "IEventBroker.h"
-
-namespace bts {
-namespace mercury {
-
-IEventBroker::~IEventBroker(){}
-
-}}
@@ -1,29 +0,0 @@
-#pragma once
-
-#include <memory>
-#include "lib/mercury/mercury_utils/mercury_common.h"
-#include "lib/mercury/topics/ITopic.h"
-
-namespace bts {
-namespace mercury {
-
-class IEventBroker
-{
-public:
-   virtual EventBrokerID id() const = 0;
-
-   virtual void add_neighbour(IEventBrokerPtr  const &) = 0;
-   virtual void remove_neighbour(EventBrokerID) = 0;
-
-   virtual void advertise(EventBrokerID,StreamID,TopicPtr const&) = 0;
-   virtual void unadvertise(EventBrokerID,StreamID) = 0;
-
-   virtual void subscribe(EventBrokerID,SubID,StreamID) = 0;
-   virtual void unsubscribe(EventBrokerID,SubID,StreamID) = 0;
-
-   virtual void notify(EventBrokerID, StreamID,SubID,EventPayload const&) = 0;
-
-   virtual ~IEventBroker() ;
-};
-
-}}
@@ -0,0 +1,6 @@
+#include "lib/mercury/broker/ILocalBroker.h"
+
+namespace bts {
+namespace mercury {
+ILocalBroker::~ILocalBroker() {}
+}}
@@ -0,0 +1,16 @@
+#pragma once
+#include "lib/mercury/broker/IRemoteBroker.h"
+
+namespace bts {
+namespace mercury {
+
+class ILocalBroker : public IRemoteBroker {
+ public:
+   virtual void local_subscribe(LocalSubID, TopicFilter const &,
+                                LocalEventHandler const &cb) = 0;
+   virtual void local_unsubscribe(LocalSubID) = 0;
+   virtual ~ILocalBroker();
+};
+
+} // namespace mercury
+} // namespace bts
@@ -0,0 +1,6 @@
+#include "lib/mercury/broker/IRemoteBroker.h"
+
+namespace bts {
+namespace mercury {
+IRemoteBroker::~IRemoteBroker() {}
+}}
@@ -0,0 +1,28 @@
+#pragma once
+
+#include "lib/mercury/common/mercury_common.h"
+#include "lib/mercury/topics/Topic.h"
+#include <memory>
+
+namespace bts {
+namespace mercury {
+
+class IRemoteBroker {
+ public:
+   virtual BrokerID id() const = 0;
+
+   virtual void add_neighbour(IRemoteBroker  &) = 0;
+   virtual void remove_neighbour(BrokerID) = 0;
+
+   virtual void advertise(BrokerID, StreamID, TopicPtr const &) = 0;
+   virtual void unadvertise(BrokerID, StreamID) = 0;
+
+   virtual void subscribe(BrokerID, SubID, StreamID) = 0;
+   virtual void unsubscribe(BrokerID, SubID, StreamID) = 0;
+
+   virtual void notify(BrokerID, StreamID, SubID, Notification const &) = 0;
+
+   virtual ~IRemoteBroker();
+};
+} // namespace mercury
+} // namespace bts
@@ -6,7 +6,7 @@
 #include "lib/utils/bts_assert.h"

 namespace bts {
-namespace utils {
+namespace mercury {

 template <class T,size_t N>
 void fat_wrapper_copy(std::array<uint8_t,N> *dst,
@@ -31,11 +31,10 @@ void fat_wrapper_destroy(std::array<uint8_t,N> &dst)
   }
 }

-// A type safe(ish) wrapper for arbitrary types with the purpose of
-// making it possible to use small object optimization to avoid heap
-// allocations. This would not be needed if boost::variant wasn't a
-// slow, complex POS. std::variant might do trick when its widely
-// available.
+// A type safe(ish) wrapper for arbitrary types making it possible to
+// use small object optimization to avoid heap allocations. This would
+// not be needed if boost::variant wasn't a slow, complex
+// POS. std::variant might do trick when its widely available.
 template<size_t SZ>
 class FatWrapper
 {
@@ -104,8 +103,8 @@ public:
   template<class T>
   T &unwrap()
   {
-      bts_assert(!empty());
-      bts_assert(*_wrapped_type == typeid(T));
+      bts_assert_d(!empty());
+      bts_assert_d(*_wrapped_type == typeid(T));
      if (sizeof(T) <= BufferSize) 
         return *((T*) _buffer.data());
      else
@@ -0,0 +1,33 @@
+#pragma once
+
+#include <memory>
+#include <array>
+#include <functional>
+#include "lib/mercury/common/FatWrapper.h"
+#include "lib/utils/ValueWrapper.h"
+
+namespace bts {
+namespace mercury {
+
+class ITopic;
+using TopicFilter = std::function<bool(const ITopic&)> ;
+
+using StreamID = utils::ValueWrapper<struct StreamIDTag,uint32_t>;
+using SubID = utils::ValueWrapper<struct SubIDTag,uint32_t>;
+using Notification = FatWrapper<80>;
+using LocalSubID = utils::ValueWrapper<struct LocalSubIDTag,uint32_t>;
+using BrokerID = utils::ValueWrapper<struct BrokerIDTag,uint32_t>;
+
+enum class EventType {
+   BEGIN_STREAM,
+   END_STREAM,
+   NEW_EVENT,
+   END_SUB
+};
+using LocalEventHandler = std::function<void(const StreamID &, EventType, Notification const*)>;
+
+StreamID gen_stream_id();
+
+using TimeStampStr = utils::ValueWrapper<struct TimeStampStrTag,std::array<char, sizeof("20121204-14:44:02.000")-1>> ;
+
+}}
@@ -1,11 +1,11 @@
 #pragma once

-#include <memory>
-#include "lib/orderman/orderman_utils/orderman_common.h"
-#include <boost/container/small_vector.hpp>
-#include "lib/mercury/synopsis/NoSynopsis.h"
 #include "lib/mercury/synopsis/CacheAll.h"
 #include "lib/mercury/synopsis/CacheLast.h"
+#include "lib/mercury/synopsis/NoSynopsis.h"
+#include "lib/orderman/orderman_utils/orderman_common.h"
+#include <boost/container/small_vector.hpp>
+#include <memory>

 namespace bts {
 namespace mercury {
@@ -14,11 +14,11 @@ namespace mercury {
 // any given event. User events should derive from Event and use the
 // curiously recurrent template pattern (yeah, its a thing, google it).
 template <typename BaseEvent,
-          typename TArray = boost::container::small_vector<BaseEvent,1>>
-class Event
-{
-public:
+          typename TArray = boost::container::small_vector<BaseEvent, 1>>
+class Event {
+ public:
   using Array = TArray;
 };

-}}
+} // namespace mercury
+} // namespace bts
@@ -1,17 +1,15 @@
 #pragma once

+#include "lib/mercury/common/mercury_common.h"
 #include <memory>
-#include "lib/mercury/mercury_utils/mercury_common.h"

 namespace bts {
 namespace mercury {

-template <class EV> class Stream 
-{
-public:
+template <class EV> class Stream {
+ public:
   StreamID sid;
 };

-
-}}
-
+} // namespace mercury
+} // namespace bts
@@ -2,7 +2,7 @@

 #include <memory>
 #include "lib/mercury/events/Event.h"
-#include "lib/mercury/mercury_utils/mercury_common.h"
+#include "lib/mercury/common/mercury_common.h"
 #include "lib/utils/bts_chrono.h"
 #include "lib/mercury/synopsis/NoSynopsis.h"

@@ -1,7 +1,7 @@
 #pragma once

 #include <memory>
-#include "lib/mercury/mercury_utils/mercury_common.h"
+#include "lib/mercury/common/mercury_common.h"
 #include <boost/container/small_vector.hpp>

 namespace bts {
@@ -0,0 +1,48 @@
+#include "lib/mercury/executor/Executor.h"
+#include "lib/mercury/broker/Broker.h"
+#include "lib/mercury/broker/BrokerProxy.h"
+#include "lib/mercury/actors/Actor.h"
+#include "lib/mercury/actors/IPollableActor.h"
+
+namespace bts {
+namespace mercury {
+
+Executor::Executor() :
+   _broker(),
+   _proxies(),
+   _actor_broker(make_proxy()),
+   _actors() {}
+
+Executor::~Executor() {
+   for (auto &a : _actors)
+      a->deactivate();
+}
+ 
+void Executor::init_neighbours(std::vector<Executor*> const&neighbours)
+{
+   std::vector<IRemoteBroker*> broker_neighbours;
+   for (auto e : neighbours)
+      if (e != this) 
+         broker_neighbours.push_back(&e->make_proxy());
+   for (auto n : broker_neighbours)
+      _broker.add_neighbour(*n);
+}
+
+
+void Executor::activate(Actor *actor)
+{
+   _actors.push_back(std::unique_ptr<Actor> (actor));
+   auto pollable = dynamic_cast<IPollableActor*> (actor);
+   if (pollable)
+      _pollable_actors.push_back(pollable);
+   actor->activate(_actor_broker);}
+   
+
+BrokerProxy& Executor::make_proxy()
+{
+   _proxies.push_back(std::make_unique<BrokerProxy>(_broker));
+   return *_proxies.back();
+}
+
+} // namespace mercury
+} // namespace bts
@@ -0,0 +1,43 @@
+#pragma once
+
+#include "lib/mercury/actors/IPollableActor.h"
+#include "lib/mercury/broker/Broker.h"
+#include "lib/mercury/broker/BrokerProxy.h"
+#include "lib/mercury/actors/ActorBroker.h"
+
+#include <memory>
+#include <vector>
+
+namespace bts {
+namespace mercury {
+class Actor;
+class BrokerProxy;
+class Executor {
+ public:
+   Executor();
+   ~Executor();
+   void init_neighbours(std::vector<Executor *> const &neighbours);
+
+   template <class A, typename... Args> void spawn(Args &&... args) {
+      activate(new A(std::forward<Args>(args)...));
+   }
+
+   void do_work() {
+      for (auto p : _pollable_actors)
+         p->on_poll();
+      for (auto &bp : _proxies)
+         bp->do_work();
+   }
+
+   void activate(Actor *actor);
+   BrokerProxy &make_proxy();
+ private:
+
+   Broker _broker;
+   std::vector<std::unique_ptr<BrokerProxy>> _proxies;
+   ActorBroker _actor_broker;
+   std::vector<IPollableActor *> _pollable_actors;
+   std::vector<std::unique_ptr<Actor>> _actors;
+};
+} // namespace mercury
+} // namespace bts
@@ -1,7 +0,0 @@
-#include "Executor.h"
-
-namespace bts {
-namespace mercury {
-
-
-}}
@@ -1,57 +0,0 @@
-#pragma once
-
-#include <atomic>
-#include <memory>
-
-#include "lib/logging/logging.h"
-#include "lib/mercury/actors/Actor.h"
-#include "lib/mercury/actors/ActorPtr.h"
-#include "lib/mercury/broker/AsyncEventBroker.h"
-#include "lib/mercury/mercury_utils/IPollableActor.h"
-
-namespace bts {
-namespace mercury {
-
-class Executor {
-public:
-   Executor(AsyncEventBroker &broker)
-      : _broker(broker),
-        _actors(),
-        _pollable_actors()
-   {}
-   
-   // Destruction is not thread safe, it is the responsability of the
-   // caller to ensure that no concurrent calls to do_work() or any
-   // other method are not being made at this point.
-   ~Executor() {
-      for (auto actor : _actors)
-         actor->deactivate();
-   }
-
-   void activate(ActorPtr const &actor) {
-      _broker.post_action([actor,this]() {
-            _actors.push_back(actor);
-            auto pollable = dynamic_cast<IPollableActor*> (actor.get());
-            if (pollable)
-               this->_pollable_actors.push_back(pollable);
-            actor->activate(this->_broker);
-         });
-   }
-
-   template <class A, typename... Args> void spawn(Args &&... args) {
-      activate(std::make_shared<A>(std::forward<Args>(args)...));
-   }
-
-   bool do_work() {
-      for (auto pollable : _pollable_actors)
-         pollable->on_poll();
-      return _broker.do_work();
-   }
-
-private:
-   AsyncEventBroker &_broker;
-   std::vector<ActorPtr> _actors;
-   std::vector<IPollableActor *> _pollable_actors;
-};
-}
-}
@@ -1,39 +0,0 @@
-#pragma once
-
-#include <memory>
-#include <array>
-#include <functional>
-#include "lib/mercury/mercury_utils/FatWrapper.h"
-#include "lib/utils/ValueWrapper.h"
-
-namespace bts {
-namespace mercury {
-
-class ITopic;
-using TopicFilter = std::function<bool(const ITopic&)> ;
-
-class IEventBroker;
-using IEventBrokerPtr = std::shared_ptr<IEventBroker>  ;
-
-using StreamID = utils::ValueWrapper<struct StreamIDTag,uint64_t>;
-using SubID = utils::ValueWrapper<struct SubIDTag,uint64_t>;
-using EventPayload = utils::FatWrapper<80>;
-
-using InternalSubID = utils::ValueWrapper<struct InternalSubIDTag,uint64_t>;
-
-using EventTypeID = utils::ValueWrapper<struct EventTypeIDTag,uint64_t>;
-
-using EventBrokerID = utils::ValueWrapper<struct EventBrokerIDTag,uint64_t>;
-enum class EventType {
-   BEGIN_STREAM,
-   END_STREAM,
-   NEW_EVENT,
-   END_SUB
-};
-using InternalEventHandler = std::function<void(const StreamID &, EventType, EventPayload const*)>;
-
-StreamID gen_stream_id();
-
-using TimeStampStr = utils::ValueWrapper<struct TimeStampStrTag,std::array<char, sizeof("20121204-14:44:02.000")-1>> ;
-
-}}
@@ -1,5 +1,6 @@
 #pragma once

+#include "lib/mercury/synopsis/ISynopsisBuilder.h"
 #include <memory>

 namespace bts {
@@ -7,26 +8,22 @@ namespace mercury {

 // Cache all previously published events and use that
 // as the synopsis
-template <class EV>
-class CacheAll : public ISynopsisBuilder
-{
-public:
+template <class EV> class CacheAll : public ISynopsisBuilder {
+ public:
   CacheAll() : _synopsis(), _empty(true) {}

-   EventPayload* get_synopsis () override
-   {
+   Notification *get_synopsis() override {
      if (!_empty)
         return &_synopsis;
      else
         return nullptr;
   }
-   
-   void update(EventPayload const&pl) override
-   {
+
+   void update(Notification const &pl) override {
      if (_synopsis.empty())
         _synopsis = pl;
      else {
-         auto const &update = pl.unwrap<typename EV::Array> ();
+         auto const &update = pl.unwrap<typename EV::Array>();
         auto &previous = _synopsis.unwrap<typename EV::Array>();
         for (auto const &ev : update)
            previous.emplace_back(ev);
@@ -34,11 +31,12 @@ public:
      _empty = false;
   }

-   void clear_synopsis() override {_empty = true;};
+   void clear_synopsis() override { _empty = true; };

-private:
-   EventPayload _synopsis;
+ private:
+   Notification _synopsis;
   bool _empty;
 };

-}}
+} // namespace mercury
+} // namespace bts
@@ -1,34 +1,35 @@
 #pragma once

-#include <memory>
+#include "lib/mercury/common/mercury_common.h"
 #include "lib/mercury/synopsis/ISynopsisBuilder.h"
-#include "lib/mercury/mercury_utils/mercury_common.h"
+#include <memory>

 namespace bts {
 namespace mercury {

 // Cache last event published and use that as the synopsis

-class CacheLast : public ISynopsisBuilder
-{
-public:
+class CacheLast : public ISynopsisBuilder {
+ public:
   CacheLast() : _synopsis(), _empty(true) {}

-   EventPayload* get_synopsis () override
-   {
+   Notification *get_synopsis() override {
      if (!_empty)
         return &_synopsis;
      else
         return nullptr;
   }
-   
-   void update(EventPayload const&pl) override {_synopsis = pl; _empty= false;}
-   void clear_synopsis() override {_empty = true;}

-private:
-   EventPayload _synopsis;
+   void update(Notification const &pl) override {
+      _synopsis = pl;
+      _empty = false;
+   }
+   void clear_synopsis() override { _empty = true; }
+
+ private:
+   Notification _synopsis;
   bool _empty;
 };

-
-}}
+} // namespace mercury
+} // namespace bts
@@ -1,20 +1,20 @@
 #pragma once

+#include "lib/mercury/common/mercury_common.h"
 #include <memory>
-#include "lib/mercury/mercury_utils/mercury_common.h"

 namespace bts {
 namespace mercury {

-class ISynopsisBuilder
-{
-public:
-   virtual EventPayload* get_synopsis() = 0;
-   virtual void update(EventPayload const&) = 0;
+class ISynopsisBuilder {
+ public:
+   virtual Notification *get_synopsis() = 0;
+   virtual void update(Notification const &) = 0;
   virtual void clear_synopsis() = 0;
   virtual ~ISynopsisBuilder() {}
 };

-using SynopsisBuilderPtr = std::shared_ptr<ISynopsisBuilder> ;
+using SynopsisBuilderPtr = std::shared_ptr<ISynopsisBuilder>;

-}}
+} // namespace mercury
+} // namespace bts
@@ -1,19 +0,0 @@
-#pragma once
-
-#include <memory>
-#include "lib/mercury/mercury_utils/mercury_common.h"
-
-namespace bts {
-namespace mercury {
-
-class ISynopsisBuilder
-{
-public:
-   EventPayload* get_synopsis() {return nullptr;}
-   void update(EventPayload const&) {};
-   void reset() {};
-};
-
-using SynopsisBuilderPtr = std::shared_ptr<ISynopsisBuilder> ;
-
-}}
@@ -6,19 +6,19 @@
 namespace bts {
 namespace mercury {

-// assume each event as a key and use as a synopsis the last event
+// assume each event has a key and use as a synopsis the last event
 // published for each key
 template <class EV> class MergeByKey : public ISynopsisBuilder {
 public:
-   using RefConstKey =  decltype(((EV *)nullptr)->key());
-   using ConstKey =  typename std::remove_reference<RefConstKey>::type;
+   using RefConstKey = decltype(((EV *)nullptr)->key());
+   using ConstKey = typename std::remove_reference<RefConstKey>::type;

   using Key = typename std::remove_const<ConstKey>::type;
   using TMap = std::unordered_map<Key, EV>;

   MergeByKey() : _synopsis(), _empty(), _dirty() { clear_synopsis(); }

-   EventPayload *get_synopsis() override {
+   Notification *get_synopsis() override {
      if (!_empty) {
         if (_dirty) {
            // regen synopsis from merged map
@@ -33,7 +33,7 @@ template <class EV> class MergeByKey : public ISynopsisBuilder {
         return nullptr;
   }

-   void update(EventPayload const &pl) override {
+   void update(Notification const &pl) override {
      _dirty = true;
      _empty = false;
      auto const &update = pl.unwrap<typename EV::Array>();
@@ -49,10 +49,10 @@ template <class EV> class MergeByKey : public ISynopsisBuilder {
   };

 private:
-   EventPayload _synopsis;
+   Notification _synopsis;
   TMap _merged;
   bool _empty;
   bool _dirty;
 };
-}
-}
+} // namespace mercury
+} // namespace bts
@@ -1,22 +1,19 @@
 #pragma once

-#include <memory>
+#include "lib/mercury/common/mercury_common.h"
 #include "lib/mercury/synopsis/ISynopsisBuilder.h"
-#include "lib/mercury/mercury_utils/mercury_common.h"
+#include <memory>

 namespace bts {
 namespace mercury {

 // do not keep a synopsis
-
-class NoSynopsis : public ISynopsisBuilder
-{
-public:
-   EventPayload* get_synopsis() override {return nullptr;}
-   void update(EventPayload const&) override {};
-   void clear_synopsis() override {};
+class NoSynopsis : public ISynopsisBuilder {
+ public:
+   Notification *get_synopsis() override { return nullptr; }
+   void update(Notification const &) override{};
+   void clear_synopsis() override{};
 };

-
-}}
-
+} // namespace mercury
+} // namespace bts
@@ -1,6 +0,0 @@
-#include "SynopsisBuilder.h"
-
-namespace bts {
-namespace mercury {
-
-}}
@@ -1,17 +0,0 @@
-#pragma once
-
-#include <memory>
-#include "lib/mercury/mercury_utils/mercury_common.h"
-
-namespace bts {
-namespace mercury {
-
-class SynopsisBuilder
-{
-public:
-   EventPayload* get_synopsis();
-   void update(EventPayload const&);
-   void reset();
-};
-
-}}
Author	SHA1	Message	Date
Pedro Pinto	0013d529da	Reduce log spamming further.	2018-10-15 17:33:40 +00:00
Pedro Pinto	3c72760a3f	Update release notes.	2018-10-15 14:42:43 +00:00
Pedro Pinto	8844f60cd0	Reduced log spamming in order manager.	2018-10-15 14:38:44 +00:00
Pedro Pinto	62e01e43e2	Merge branch 'prod' of bitbucket.org:bluetradesys/spark into prod	2018-10-14 18:29:00 -04:00
Pedro Pinto	7953cc1209	Docker ptr	2018-10-14 18:28:56 -04:00
John Colvin	5dd2c14fba	mview - dispose security subscription	2018-10-11 15:26:14 -05:00
Pedro Pinto	dcf116fe82	Updated release notes.	2018-10-11 18:28:10 +00:00
Pedro Pinto	9c571cc16d	Moved externals pointer	2018-10-11 17:53:00 +00:00
Pedro Pinto	aa8e32ee43	Updated external subrepos	2018-10-11 17:43:39 +00:00
Michael Harding	4d0a793ecc	SPARK-1773: rows in table	2018-10-11 11:44:30 -04:00
Michael Harding	659bf6b208	SPARK-1773: update paths	2018-10-11 09:36:19 -04:00
Pedro Pinto	33312a4bf8	Fixed up build with respect to the perf_trace web page.	2018-10-11 12:46:12 +00:00
Pedro Pinto	929a93b1f7	Updated docker	2018-10-11 12:11:00 +00:00
Pedro Pinto	f49e41c5c0	Fixed a few bugs in perf state json generation.	2018-10-11 12:11:00 +00:00
Michael Harding	87f99d42bf	SPARK-1773: Add Histogram and rows	2018-10-10 16:13:29 -04:00
Michael Harding	e4f5531855	Add perf_trace page	2018-10-09 17:43:42 -04:00
Pedro Pinto	d62bde8158	Added placeholder for perf trace web page.	2018-10-09 20:28:56 +00:00
Pedro Pinto	53ea5ddf81	Merged	2018-10-09 19:27:54 +00:00
Pedro Pinto	9e0b95a1ad	SPARK-1772, Serving perf traces via json	2018-10-09 17:43:18 +00:00
Pedro Pinto	5325c372ba	Added a few docs	2018-10-08 19:08:49 +00:00
Pedro Pinto	50abbd6aea	Added a few more benchmarks.	2018-10-08 14:11:26 +00:00
Pedro Pinto	f324504df1	Added locking to perf trace log.	2018-10-08 14:10:56 +00:00
Pedro Pinto	931d5dad01	SPARK-1771, Logging perf metrics in a DB	2018-10-05 20:08:45 -04:00
Pedro Pinto	ef094b891a	SPARK-1771, Logging perf metrics in a DB	2018-10-05 23:05:27 +00:00
Pedro Pinto	d25d91038d	clang format.	2018-10-05 15:37:44 +00:00
Pedro Pinto	700f1d97fd	Renamed TickToOrderCorID.h to PerfTrace.h	2018-10-05 15:37:09 +00:00
Michael Harding	5b1cf890e8	Merged in spark1538-risk-admin-cloning (pull request #79 ) SPARK-1538: Spark Admin - Cloning Approved-by: Eric Harding <eric.harding@bluetradesys.com>	2018-10-04 19:30:19 +00:00
Michael Harding	c254248886	SPARK-1538: Clones incoming	2018-10-04 15:00:33 -04:00
Pedro Pinto	5aaa5a3360	Ran clang-format	2018-10-04 10:13:29 -04:00
Pedro Pinto	54d31c345a	Logged non-routable orders at Error level.	2018-10-03 19:23:09 +00:00
Pedro Pinto	ae3e721f62	Changed MDP3 producer to honor new thread configuration.	2018-10-03 15:45:29 +00:00
Pedro Pinto	0833dce43d	Added some tools to quickly analyze latency stats.	2018-10-03 14:53:36 +00:00
Pedro Pinto	47b2e917c1	Tweaked threading defaults.	2018-10-02 16:30:56 +00:00
Pedro Pinto	c2864d1715	Improved performance.	2018-10-02 16:24:00 +00:00
Pedro Pinto	2769de6f7e	Added central control of threading.	2018-10-02 16:24:00 +00:00
Pedro Pinto	ef3b39361a	Cleaning up mercury.	2018-10-02 16:24:00 +00:00
Pedro Pinto	fa2a2e2107	Rewrite of mercury for better performance.	2018-10-02 16:24:00 +00:00
Pedro Pinto	3005dfdc0f	Perf analysys script tweak.	2018-10-02 16:22:04 +00:00
Pedro Pinto	4fd8d9dc5b	Fixed typo in comment.	2018-10-02 16:22:04 +00:00
Pedro Pinto	c948b981cf	More performance tracing work.	2018-10-02 16:22:04 +00:00
Pedro Pinto	8e51b05941	More tweaks to improve performance tracing.	2018-10-02 16:22:04 +00:00
Pedro Pinto	75568e6c75	Made thread allocation configurable for order management.	2018-10-02 16:22:04 +00:00
Pedro Pinto	6e3c880ea2	Changed mercury to assume direct broker connections. This is needed to reduce latency as we (optionally) move the execution gateway to its on thread.	2018-10-02 16:20:00 +00:00
Pedro Pinto	ded957a0ef	SPARK-1686, Towards better server side performance.	2018-10-02 16:20:00 +00:00
Eric Harding	05741ceab0	SPARK-1742 test can fail when time moves foward	2018-10-01 14:56:35 -04:00
John Colvin	9e3b3879d0	Merged in lldatetime (pull request #75 ) SPARK-1726 change DateTime.cpp to match c# format Approved-by: Eric Harding <eric.harding@bluetradesys.com>	2018-09-25 15:32:54 +00:00
Kevin Darby	a1b189fcb1	repenting my sins against the proto gods	2018-09-21 12:46:42 -04:00
Michael Harding	7a48e5dd5c	SPARK-1725: not too many characters	2018-09-21 11:26:09 -04:00
John Colvin	2842aea933	lldatetime - TimeAndSales rows show local time	2018-09-20 15:35:48 -04:00
John Colvin	f97d156020	in datetime.cpp, ptime_to_ll should use KindUnspecified by default	2018-09-20 15:13:20 -04:00
Eric Harding	d8922be0ad	Add udp.py to tools	2018-09-20 14:35:43 -04:00
John Colvin	5634ada367	cautionary comment in lldatetime(this ulong) extension	2018-09-20 13:43:56 -04:00
John Colvin	94d249edbd	use ticks_per_day instead of (micros_per_day * ticks_per_micro)	2018-09-20 13:21:06 -04:00
John Colvin	1c29ac0ec5	missed a couple changes in last commit	2018-09-20 13:18:02 -04:00
John Colvin	e7e280bad8	lldatetime pr edits, add test_lldatetime() to Co2Tests	2018-09-20 13:10:48 -04:00
Eric Harding	7ca11c3eee	SPARK-1718 Fix trailing comma after deleted users.	2018-09-20 10:27:52 -04:00
Eric Harding	caf6ac029c	Fix ancient config bug where you can't reference a non-overloaded key in a rule or macro.	2018-09-20 10:27:52 -04:00
Eric Harding	4c0fa81963	Remove unused environment.proto	2018-09-20 10:27:52 -04:00
Kevin Darby	720c1595e0	Merged in more_lean (pull request #76 ) made the 'toward' behavior for lean an enum	2018-09-19 22:14:31 +00:00
Kevin Darby	bedd0051a9	made the 'toward' behavior on lean an enum	2018-09-19 16:36:25 -04:00
Jake Norbut	09bed1a713	SPARK-1728 - Symbol list is now restrictued according to DeploymentType for the gui; when DEMO, only sim; when PROD, not sim; will require upkeep as exchanges are added	2018-09-19 15:26:25 -05:00
John Colvin	ebb53c5035	LLDateTime conversion pr edits	2018-09-19 14:20:33 -04:00
Eric Harding	243b82a6d7	Primitive settingsmanager	2018-09-19 13:31:45 -04:00
Taha Afzal	bd4f840d27	Setting last_price to avg_price for SOD records	2018-09-19 12:17:22 -04:00
John Colvin	de893d2175	SPARK-1726 change DateTime.cpp to match c# format	2018-09-19 11:49:38 -04:00
Taha Afzal	33658f13f6	SPARK-1722 orders cleanup weekend bug	2018-09-18 13:28:20 -04:00
Kevin Darby	469ee033b2	added a 'detach on toward' boolean to Lean, reasonable in it's own right, but so K. Loftus can race us against Chalupa.	2018-09-15 15:29:12 -04:00
Eric Harding	451fb67195	Fix market_segment_id in shared memory dumper. (uint8_t and std::cout disagree)	2018-09-11 12:44:52 -04:00