PerfLog recovery

SPARK-1512 Allow the same name for a new spread when previous one was deleted

Approved-by: Jake Norbut <jake.norbut@bluetradesys.com>
Approved-by: Pedro Pinto <pedro.pinto+atlassian@bluetradesys.com>

.vscode/launch.json

@@ -209,8 +209,7 @@
             "type": "cppdbg",
             "request": "launch",
             "program": "/home/bts/spark/build/ninja_debug/src/server/sparkd/sparkd",
-            // "args": ["test","Co2RelayTests"],
-            "args": ["md", "run", "10"],
+            "args": ["test","Co2Tests"],
             "stopAtEntry": false,
             "cwd": "/home/bts/spark/build/ninja_debug/src/server/sparkd/",
             "environment": [],

CMakeLists.txt

@@ -39,7 +39,7 @@ else ()
   set (CMAKE_CXX_FLAGS_DEBUG  "${CMAKE_CXX_FLAGS_DEBUG} \
                                -fPIC -Wall -pedantic -Wextra -Werror \
                                -Wno-missing-braces -Wno-ignored-qualifiers \
-                               --std=c++17 -fdiagnostics-color")
+                               --std=c++17 -fdiagnostics-color -Wl,--no-undefined")
   set (CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_DEBUG} -fno-omit-frame-pointer -DNDEBUG -O3")
   add_subdirectory(src/server/sparkd)
   add_subdirectory(src/lib)
@@ -51,7 +51,6 @@ else ()
   add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/server
     COMMAND mkdir -p ${CMAKE_BINARY_DIR}/server
     COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
-    COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/defaults ${CMAKE_BINARY_DIR}/server
     COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/web ${CMAKE_BINARY_DIR}/server
     COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
     COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server

doc/dev/architecture.org

@@ -25,3 +25,47 @@
 ***** FillStore
 ***** SecuritySrc
 * Benchmarks
+* OrderMan
+** OrderServices
+*** callbacks
+**** standard
+- update
+- cancel
+- crack
+- activate
+- book
+- settlement
+- child order state
+- timer
+**** low level
+- exe reports
+- cancel rejects
+- gateway status
+*** services
+**** standard
+- secdb
+- get time
+- schedule actions
+- order builder
+  - activate
+  - update
+  - cancel 
+  - crack
+**** low level
+- governor
+- send fix
+- get time str
+- limiter
+** Internalize plan
+- change order initiated update/cancel/crack/activate/report_state to
+  use direct method calls
+- need to watch out for reentrancy
+- need to also report state externally 
+- also want to separate active from non active orders in map to speed
+  up look ups
+- want to use direct references in orders to avoid lookups
+- want to change how time is captured, the trace is too big to copy
+  around
+
+
+

doc/user/docs/Help.md

@@ -1,5 +1,9 @@
 #Help
 
+<div>
+   <iframe width="560" height="315" src="https://www.youtube.com/embed/yGQfr_puT3w" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
+</div>
+
 ##Report a Problem
    If there is any issue at all, you can click here
    and it’ll bring up a “Report a Problem” window, which you can type

doc/user/docs/Launcher.md

@@ -1,5 +1,9 @@
 #Module Launcher
 
+<div>
+    <iframe width="560" height="315" src="https://www.youtube.com/embed/OxVqQBKFIDw" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
+</div>
+
 -  The module launcher is the first item that will come up after
     logging in to Spark.
 -  This is your central control for all modules
@@ -9,7 +13,7 @@
 
 ![](images/image38.png)
 
--  If you right click anywhere on the launcher you can modify
+-  If you right-click anywhere on the launcher you can modify
     various display settings including:
       -  Vertical layout, instead of horizontal
       -  Showing the text of the individual modules
@@ -25,5 +29,5 @@
 Some common features within BTS Spark
 
 -  All Windows are tabbed so they can be combined or separated to organize your workspace
--  Some Spark windows have a menu on the left hand side where you can launch related modules or change settings. You can see the menu item names either by hovering your mouse over the icon, or clicking on the right edge of the menu (a blue bar will appear).
+-  Some Spark windows have a menu on the left-hand side where you can launch related modules or change settings. You can see the menu item names either by hovering your mouse over the icon or clicking on the right edge of the menu (a blue bar will appear).
 -   Some table columns have filtering capabilities, where if you hover over the column header, it will display a filter icon, which enables you to filter the table based on the information in that column.

doc/user/docs/Market.md

@@ -9,6 +9,10 @@
 
 ##Market View
 
+<div>
+    <iframe width="560" height="315" src="https://www.youtube.com/embed/Y54S1uscT_g" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
+</div>
+
 -  This provides a grid of outright, spread, and synthetic products
     showing real-time market data all in one place
 -  You can create multiple market view grids and customize the
@@ -29,7 +33,7 @@
     and all securities will be displayed.  This will show up in the
     selected securities window as “&lt;symbol&gt; future at
     &lt;exchange&gt;”
--  Synthetic’s can be displayed by clicking on exchange “BTS”, then
+-  Synthetics can be displayed by clicking on exchange “BTS”, then
     synthetic, symbol, and finally under the securities window.
 -  Below is an example of an outright, spread, all futures, and
     synthetic chosen:
@@ -98,10 +102,10 @@
             either the Bid or Ask price and volume.
          -  Highlighting a single security and clicking on the order ticket
             ![](images/image23.png) icon
-            on the left hand browser window of the market view
+            on the left-hand browser window of the market view
 
 -  Launch a ladder for 1 security
-      -  You can double click on any security in the market view, besides
+      -  You can double-click on any security in the market view, besides
          the Bid Qty, Bid Price, Ask Price, or Ask Qty columns to launch a
          ladder for that Security.
 
@@ -110,7 +114,7 @@
       -  If you hold down the Ctrl key and select multiple securities, you
          can then click on the ladder icon
          ![](images/image69.png) on
-         the left hand side to bring up the selected ladders all at
+         the left-hand side to bring up the selected ladders all at
          once.
 
 -  Launching a Fills window for 1 or many securities
@@ -118,7 +122,7 @@
       -  This is done by either highlighting 1 or many securities and then
          clicking on the Fills icon
          ![](images/image20.png) on
-         the left hand side to bring up a fill window for the selected
+         the left-hand side to bring up a fill window for the selected
          securities.  
 
          -  Please note - If there haven’t been any fills for the day in that
@@ -129,14 +133,14 @@
       -  This is accomplished by highlighting 1 or many securities and then
          clicking on the time and sales icon
          ![](images/image49.png) on
-         the left hand side
+         the left-hand side
 
 -  Launching the synthetic window to create a synthetic product
 
       -  This is accomplished by highlighting either 1 or 2 securities and
          clicking on the Synthetic icon
          ![](images/image34.png) on
-         the lower left hand side
+         the lower left-hand side
 
          -  Please note - If only 1 security is chosen, a create synthetic
             window will appear, and you will be required to add another leg from
@@ -144,7 +148,11 @@
 
 ##Time and Sales
 
--  Provides a display of real time fills as they happen for 1 or many
+<div>
+    <iframe width="560" height="315" src="https://www.youtube.com/embed/nEkSGacjt0s" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
+</div>
+
+-  Provides a display of real-time fills as they happen for 1 or many
     securities
 -  You can create multiple time and sale windows and customize the
     security selection to fit your needs

doc/user/docs/My-Windows.md

@@ -1,5 +1,9 @@
 #My Windows
 
+<div>
+   <iframe width="560" height="315" src="https://www.youtube.com/embed/pjm1JJ9Q3jM" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
+</div>
+
 This is the housing of the windows that you have manually named and
 creates a place where you can quickly and easily launch windows
 which you’ve customized.
@@ -18,9 +22,9 @@ section of the module launcher.
 
 ![](images/image15.png)
 
-Right clicking on a tab that has already had its default title modified
+Right-clicking on a tab that has already had its default title modified
 will show the following which will allow you to rename the modified
-title or remove it from the my windows, which will then restore the
+title or remove it from the My Windows, which will then restore the
 default standard title.
 
 ![](images/image37.png)

doc/user/docs/Settings.md

@@ -1,6 +1,10 @@
 
 #Settings
 
+<div>
+   <iframe width="560" height="315" src="https://www.youtube.com/embed/yGQfr_puT3w" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
+</div>
+
 ##Workspaces
 
 This is where your workspaces can be saved or switched

doc/user/docs/Spreader.md

@@ -1,14 +1,35 @@
 
 #Spreader
 
+<div>
+    <iframe width="560" height="315" src="https://www.youtube.com/embed/XEZpuPAgMC0" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
+</div>
+
+<div>
+    <iframe width="560" height="315" src="https://www.youtube.com/embed/EVFK2yNQFS4" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
+</div>
+
+
+<div>
+    <iframe width="560" height="315" src="https://www.youtube.com/embed/d5ywlHen6yo" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
+</div>
+
+<div>
+<iframe width="560" height="315" src="https://www.youtube.com/embed/Mmw8PNJA0Dw" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
+</div>
+
+<div>
+    <iframe width="560" height="315" src="https://www.youtube.com/embed/QK-bBSH6Rc0" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
+</div>
+
 -  Launching the spreader from the module launcher will bring up the
     synthetic management browser which appears like below:
 
 ![](images/image35.png)
 
--  From this window it’ll show any synthetics that you have already
+-  From this window, it will show any synthetics that you have already
     created as well as various other options of both managing the
-    synthetics and other tasks all from the left hand side pane within
+    synthetics and other tasks all from the left-hand side pane within
     the window, including:
 
     -  Adding a new synthetic ![](images/image8.png)
@@ -30,7 +51,7 @@
     -  From the main module launcher, you can go to spreader, then
         synthetics, then click add
     -  Highlighting one or more securities in a market view and clicking
-        on create synthetic
+        on Create Synthetic
     -  Cloning an existing security
 
 -  An empty synthetic browser will look like below without any legs
@@ -43,10 +64,10 @@
 -  Upon adding legs, you can then go through each of the parameters to
     customize how you would like the spread to be worked in the
     market
--  We’ll start at the top of create synthetic window and work our way
+-  We will start at the top of the Create Synthetic window and work our way
     down:
 
-    -  At the very top of the create synthetic window you will have to
+    -  At the very top of the Create Synthetic window you will have to
         name your synthetic product.  By default, it will auto-populate
         based on the symbols you have chosen:
 
@@ -73,16 +94,15 @@
         exchange
     -  Furthermore, if you are ever unsure of what a parameter’s function
         is and how it is controlled you can hover over the parameter’s title
-        and a “?” will be shown.  Clicking on this will expand built in
+        and a “?” will be shown.  Clicking on this will expand built-in
         details of how that parameter functions.  Below is an example of the
         throttle details displayed:
 
     ![](images/image56.png)
 
-    -  Main parameters - these are the top 4 rows in the create synthetic
+    -  Main parameters - these are the top 4 rows in the Create Synthetic
         window which all synthetics will be required to have set.
-         Everything below these are custom to a traders needs, however the
-        top 4 are the main foundational aspects used for the calculation of
+         Everything below these can be customized to a trader's needs, however, the top 4 are the main foundational aspects used for the calculation of
         the synthetic:
 
     ![](images/image10.png)
@@ -95,7 +115,7 @@
         be priced, by default, it’ll be based on the same as the quantity of
         the synthetic produced, but you can change it to modify the balance
         of the price, giving more weighting to one leg over another.
-    -  Enable quoting - This parameter is in the form of a check box,
+    -  Enable quoting - This parameter is in the form of a checkbox,
         where if the box is checked the leg will be quoted versus not
         checked, it will not be shown in the market.  If it is not actively
         quoted in the market, the leg will still be used for calculation of
@@ -107,7 +127,7 @@
          If your Hedge Ratio is 3 on the NQ leg, then you’ll work the ES
         order based on where you could fill 6 (2 (Qty of NQ) \* 3 (Hedge
         Ratio)).
-    -  The next 6 parameters are all completely custom to a traders needs
+    -  The next 6 parameters are all completely custom to a trader's needs
         and can further define how a synthetic is worked.  As previously
         mentioned, if you hover over any of these parameters a “?” will be
         displayed and click on such will display further details on how this
@@ -118,7 +138,7 @@
     -  Quote
 
         -  Throttle - This can be enabled or disabled based on if the box is
-            checked per leg.  It’s main function is to reduce re-quoting updates
+            checked per leg.  Its main function is to reduce re-quoting updates
             by defining a region on either side of the working quote order,
             inside which quote updates will not be sent:
 
@@ -148,8 +168,8 @@
 
         ![](images/image70.png)
 
-        -  Time Based Cross - Basically the exact same as ticks away cross,
-            however this is based on time (in ms) on when to automatically cross
+        -  Time Based Cross - Basically the same as Ticks Away Cross,
+            however, this is based on time (in ms) on when to automatically cross
             (fill) the market if they get too far away from the opposite inside
             market.  If a legged hedge order’s price gets X amount of “ticks”
             away from the opposite inside market, it is re-priced to cross the

doc/user/docs/Trading.md

@@ -9,7 +9,11 @@
 
 ##Order Book
 
--  This provides list of all orders a user has placed, by default it
+<div>
+    <iframe width="560" height="315" src="https://www.youtube.com/embed/ygMFkjBBGeU" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
+</div>
+
+-  This provides a list of all orders a user has placed, by default it
     will display only working orders
 -  You can create multiple order books filtered however you would
     like
@@ -44,13 +48,13 @@
 -  Anytime
 -  Today
 -  Since Open
--  A specific user defined period of time
+-  A specific user-defined period of time
 
 ![](images/image25.png)
 
--  The left hand box within an order book provides users with further
+-  The left-hand box within an order book provides users with further
     functionality to quickly perform various tasks with live orders as
-    well as launch other modules.  It’s appears like the below:
+    well as launch other modules.  It appears like the below:
 
 ![](images/image4.png)
 
@@ -62,7 +66,7 @@
 -  ![](images/image73.png)Cancel
     selected - clicking on this will cancel the selected orders you have
     highlighted in the order book.  You can choose more than one by
-    holding hold the Ctrl key when click on each row of orders
+    holding hold the Ctrl key when clicking on each row of orders
 -  ![](images/image74.png)Cancel
     buys - clicking on this will cancel all the buy orders you have in
     the order book
@@ -75,8 +79,8 @@
 
 ##Fills
 
--  This provides list of all fills a user has had for a given time
-    period, by default it will open up with “since open” time frame
+-  This provides a list of all fills a user has had for a given time
+    period, by default it will open up with “since open” timeframe
     selected showing you fills on the day thus far
 -  You can create multiple fill windows, filtered however you
     like
@@ -85,8 +89,12 @@
 
 ##P&amp;L
 
+<div>
+    <iframe width="560" height="315" src="https://www.youtube.com/embed/xj6cEQ6By34" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
+</div>
+
 -  This provides a running ledger of a users account(s) profit and
-    loss broken down per symbol
+    loss, broken down per symbol
 -  You can create multiple P&L windows, filtered however you would
     like
 -  All P&L windows are brought up as tabs and can be combined with all
@@ -101,6 +109,10 @@
 
 ##Ladder
 
+<div>
+    <iframe width="560" height="315" src="https://www.youtube.com/embed/Q0qsyc1nvYo" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
+</div>
+
 -  The ladder was designed to be the most logically produced ladder on
     the market, engineered and designed by professional traders for
     professional traders
@@ -112,8 +124,8 @@
 
 ![](images/image36.png)
 
--  The ladder consist of 2 major parts, the Ladder itself and the
-    information and control section on the left hand side
+-  The ladder consists of 2 major parts, the Ladder itself and the
+    information and control section on the left-hand side
 
 -  The information section on the left contains the following: &nbsp; ![](images/image1.png)
       -  Day - showing the current price of the security on the day
@@ -126,17 +138,17 @@
 
 
 -  The control section right underneath the information section on the
-    left hand side offers the following:
+    left-hand side offers the following:
 
 ![](images/image62.png)
 
--  Account in which the orders will flow into when trading this
+-  The account in which the orders will flow into when trading this
     security
--  Length of time the order will remain in market, either DAY or
+-  Length of time the order will remain in the market, either DAY or
     GTC
--  Order type, by default orders will go in as Limit, however all of
+-  Order type, by default, orders will go in as Limit, however all of
     the following are offered out of the box.  The top 5 are all fairly
-    standard order types, however the bottom 5 are all specialized
+    standard order types, however, the bottom 5 are all specialized
     algos.  They are only offered by BTS and require specialized
     parameters.  Please see below for each order type:
 
@@ -147,8 +159,8 @@
 
       -  Limit
 
-         -  This is a buy or sell order designed to take place on the exact
-            price level you have entered the order at.
+         -  This is a buy or sell order designed to take place on the
+            price level where you entered the order.
 
       -  Stop
 
@@ -218,8 +230,7 @@
             market flips and has at least threshold size as defined.
          -  “Flip” refers to the event where the best offer becomes the best
             bid or the market trades away from you, i.e. the line moves up in
-            the GUI.  This is all true in the reverse way as well if
-            selling.
+            the GUI.  The reverse is true if selling.
 
       ![](images/image41.png)
 
@@ -241,7 +252,7 @@
 
 -  You can manually type in a quantity here as well
 -  Furthermore, if you have a value in the volume bar, you can
-    increase this size by right clicking and the value will increase
+    increase this size by right-clicking and the value will increase
     incrementally by 1.  The same is true if you would like to decrease
     this value by left click and the value will decrease.
 -  By having the auto clear box checked, it will bring the value back
@@ -252,12 +263,12 @@
 -   The bottom section of the control area of a ladder has 4 aspects
     to it
 
-      -  Cancel Buys - which will cancel all buys currently working on the
+      -  Cancel Buys, which will cancel all buys currently working on the
          ladder
-      -  Cancel Sells - which will cancel all sells currently working on the
+      -  Cancel Sells, which will cancel all sells currently working on the
          ladder
-      -  Cancel All - which will cancel all orders on the ladder
-      -  Settings - this will launch the ladder settings window which
+      -  Cancel All, which will cancel all orders on the ladder
+      -  Settings, which will launch the ladder settings window which
          appears like below
 
 ![](images/image71.png)
@@ -269,27 +280,26 @@
 
 -  There are 5 main columns
 
-      -  Orders - which display current orders working in the market
-      -  Bid - which displays the current bid volumes at each price
+      -  Orders, which display current orders working in the market
+      -  Bid, which displays the current bid volumes at each price
          level
-      -  Price - which displays the price levels of the symbol shown
-      -  Ask - which displays the current ask volumes at each price
+      -  Price, which displays the price levels of the symbol shown
+      -  Ask, which displays the current ask volumes at each price
          level
-      -  Last Trade - which displays where the price level and volume of
+      -  Last Trade, which displays the price level and volume
          where the last trade occurred
 
 -  Functionality of the ladder
 
       -  To enter an order directly by clicking in the ladder, left click on
          either the bid or ask volume with a volume greater than 0 on the
-         predefined value box on the left hand section
+         predefined value box on the left-hand section
       -  A badge will be displayed within the orders column of the value you
          have chosen and the quantity filled as displayed below
 
 ![](images/image46.png)
 
--  To change the price level you wish to work the order, you can right
-    click and hold on the badge to move it up or down the ladder for the
+-  To change the price level you wish to work the order, you can right-click and hold on the badge to move it up or down the ladder for the
     desired price
 -  To cancel the order, left click on the badge
 -  Furthermore, you can click down on the scroll wheel of your mouse
@@ -298,6 +308,10 @@
 
 ## Order Ticket
 
+<div>
+    <iframe width="560" height="315" src="https://www.youtube.com/embed/bdC5pZ6T35I" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
+</div>
+
 -  The order ticket was designed to provide another means other than
     the ladder to enter orders into the market
 -  You can bring up an order ticket directly from the module launcher,
@@ -307,7 +321,7 @@
 
 ![](images/image27.png)
 
--  On the top left you have the price for the order as well as the
+-  On the top left, you have the price for the order as well as the
     volume
 -  Beneath are the predefined volumes as well as a clear button to
     clear out the volume

doc/user/docs/index.md

@@ -1,3 +1,3 @@
 
-Click on the headings on the the left to expand their contents. If you need specific information quickly use the Search button in the top right of this page.
+Click on the headings on the left to expand their contents. If you need specific information quickly use the Search button in the top right of this page.
 

doc/user/docs/release_notes.md

@@ -1,4 +1,24 @@
 # Release Notes
+## 1.4.2
+
+### User Interface Improvements
+  - When creating a synthetic from the Market View window, securities are now added to the Create Synthetic dialog in the same sort order as they appear in grid
+### Bug Fixes
+  - Shell no longer crashes when removing a security from the synthetics window
+  - It is now possible to create a new synthetic using the same name as a previously deleted synthetic 
+### Performance Improvements
+  - Synthetic calculation speed was improved by 400%.
+  
+## 1.4.1
+
+### Tabs
+  - Tabs now shrink and expand to make better use of horizontal space
+### Bug Fixes
+  - Fixed a bug that could cause excessive CPU usage when a connection broke
+  - Fixed a layout bug in hedging/requoting performance reports
+  - Fixed account name mismatching bug for old fills
+### Absolute Position limits
+  - Positions can now be netted or absolute for purposes of limit enforcing
 
 ## 1.4.0
 
@@ -6,7 +26,7 @@
   - Improved requoting and hedging performance
   - Improved order book form performance
 ### Server Side SDK
-  - Added beta version of the server side SDK
+  - Added a beta version of the server side SDK
 ### Front End
   - Added status bars to all UI grids
 ### Reporting
@@ -18,7 +38,7 @@
   - Further speed enhancements
 ### Conditional (new quoting parameter)
   - Reduces re-quoting updates by only quoting if N ticks away from the opposite inside market.
-  - Quotes are cancelled when it gets further away.
+  - Quotes are canceled when it gets further away.
   - "Price Type" dictates if this logic applies to the quoting market or to the synthetic spread market.
 
 ## 1.3.7
@@ -42,7 +62,7 @@
 ### Synthetic creation option to choose Merged or Direct
   - The ability to choose Merged or Direct per leg when creating a synthetic
 ### Volume at price
-  - Within a ladder you can now see a volume at price graph
+  - Within a ladder, you can now see a volume at price graph
   - This is configurable from a ladder's settings as well
 ### Customized settings in all windows
  - All windows now contain a settings page where grid settings (font color, size, etc) are applied
@@ -60,7 +80,7 @@
 
 ### Order Book
   - Many visual facets of the Order Book grid can now be customized
-    - The Order Book settings page can be access from the gears icon in the Order Book toolbar or from the right-click menu
+    - The Order Book settings page can be accessed from the gears icon in the Order Book toolbar or from the right-click menu
   - The Expand/Collapse All button has been separated into two different buttons on the toolbar
 
 ### Sound Alerts
@@ -96,13 +116,13 @@
 ### Synthetics
   - Hedge Ratio parameter has been renamed to Lean Qty Factor
   - Spreader now includes a Hidden mode parameter
-    - A quoting leg can be hidden, ie an order will not sent out until the market crosses the computed quoting price. When that happens, the quote is sent out as an IOC order.
+    - A quoting leg can be hidden, ie an order will not be sent out until the market crosses the computed quoting price. When that happens, the quote is sent out as an IOC order.
 
 ### Algos
   - Trigger orders now can contain a Trigger Exit parameter based on time or price; once the exit condition has been met the Trigger order will cancel
   - IOC/FAK order type added
   - Cover order type added
-  - Cancelling a synthetic order no longer cancels any working hedge orders
+  - Canceling a synthetic order no longer cancels any working hedge orders
 
 ### General
   - A manual page has been added to the launcher Help menu
@@ -110,18 +130,18 @@
   - Various cash per point values updated
   - Time-based grid filters now have tick up/down buttons
   - Numerous performance improvements and stability enhancements
-  - Automated position and orderbook maintenance during daily recycle
-    - Individual fills be compacted into start of day position records
+  - Automated position and Order Book maintenance during daily recycle
+    - Individual fills will be compacted into the Start of Day position records
     - Orders from the day before the recycle will be deleted
 
 
 ##1.1
 
 ###1.1.22
-- Orderbook: Orders in Error status can now be recreated
-- Orderbook: Orders containing child orders will now be collapsed by default.
-- Orderbook: Expand / Collapse All button added to the right-click menu and window action pane, denoted by ▲ icon
-- Orderbook: Symbol column added to allow for filtering based on group symbol
+- Order Book: Orders in Error status can now be recreated
+- Order Book: Orders containing child orders will now be collapsed by default.
+- Order Book: Expand / Collapse All button added to the right-click menu and window action pane, denoted by ▲ icon
+- Order Book: Symbol column added to allow for filtering based on group symbol
 - Time filters now have a quick Set Last Hour hyperlink which will set the Period Begin filter to (Now - 1 hour)
 
 ###1.1.20
@@ -140,15 +160,15 @@
 - Fixed occasional crash when filtering fills
 - Fixed several cash per point multipliers
 - Colored time of sales row with aggressor side
-- Iceberg orders now cancel (rathern than error) when a child is canceled
+- Iceberg orders now cancel (rather than an error) when a child is canceled
 
 ###1.1.0
 - Added per user self match protection, location and self match protection FIX tag settings
 - Implemented a new "Trigger Order" algo
 - Corrected tick formatting for a number of products
-- Added the hability to recreate canceled orders
-- Added leg fills to facilitate futured based PnL
-- PnL report now has PnL columns calculated based on your start of day, current day, and total position
+- Added the ability to recreate canceled orders
+- Added leg fills to facilitate futures based PnL
+- PnL report now has PnL columns calculated based on your Start of Day, current day, and total position
 - Start, Day and Total PnL columns are painted with a red or green background color based on if the pnl value is negative or positive respectively
 - Corrected cash per point values, used in pnl calculations, for a number of products
-- PnL report will now remember any customizations you make to filtering and/or sorting criteria
+- PnL report will now remember any customizations you make to filtering and/or sorting criteria

ops_notes.txt

@@ -1,3 +1,10 @@
+1.4.2
+- Be sure the system has a bts user before deployment.  bts is the superuser, he has all permissions and can see all market data 
+- After deployment re-permission all existing users for trading, riskadmin, and/or md admin permissions as appropriate
+- Also note which users can add other users and accounts and permission them appropriately (Can Add?)
+- Permission  users for appropriate market data entitlements; this has monetary consequences so ask Kevin if you aren't sure.
+- MIC reporting needs to be added to audit Jenkins job (https://bluetradesys.atlassian.net/browse/SPARK-1900)
+
 1.3.5
 - Change in market data wire format. Old shell will get only direct market data instead of merged which may not match what the servers is driving from.
 

src/client/spark/Admin/AdminModel.cs

@@ -36,7 +36,7 @@ namespace mbts.spark {
             IEnumerable<BTSAccountImmutable> active_accounts =
                from admin_data in update.all
                from bts_account in admin_data.Accounts
-               where bts_account.IsPresent
+               where bts_account.IsPresent && _wrapper_subject.Value.can_view_account(bts_account.Name)
                join account_id in (
                   from bts_user in active_users
                   from account_id in bts_user.UserData.AccountPermissions.AccountPermissions.Keys

src/client/spark/Admin/AdminModelWrapper.cs

@@ -16,26 +16,26 @@ namespace mbts.spark
       {
          _admin_data = admin_data;
          _user_name = user_name ?? string.Empty;
-         _allowed_accounts = search_admin(u => u.Name == _user_name).Select(x => x.Item1).ToList();
-         foreach(var val in _allowed_accounts)
+         _allowed_accounts = search_admin(u => u.Name == _user_name).Where(x =>
+         {
+            return x.Item2.HasPermissions || x.Item3.HasPermissions;
+         }).Select(xx => xx.Item1).ToList();
+         foreach (var val in _allowed_accounts)
             _allowed_account_dict[val.Name] = val.Name;
       }
-      public IEnumerable<Tuple<BTSAccountImmutable, TradingPermissionUnitImmutable>> get_trading_permissions(UInt32 user_id)
-      {
-         return search_admin(u => u.Id == user_id);
-      }         
       public IEnumerable<BTSAccountImmutable> viewable_accounts()
       {
          return _allowed_accounts;
       }
       public bool can_view_account(string account)
       {
-         return _allowed_account_dict.ContainsKey(account) || _user_name.ToLower() == "bts";
+         return _allowed_account_dict.ContainsKey(account) || _user_name.ToLower().Equals("bts");
       }
 
-      IEnumerable<Tuple<BTSAccountImmutable, TradingPermissionUnitImmutable>> search_admin(Func<BTSUserImmutable, bool> predicate)
+      IEnumerable<Tuple<BTSAccountImmutable, TradingPermissionUnit, AdminPermissionUnit>> 
+         search_admin(Func<BTSUserImmutable, bool> predicate)
       {
-         var rv = new List<Tuple<BTSAccountImmutable, TradingPermissionUnitImmutable>>();
+         var rv = new List<Tuple<BTSAccountImmutable, TradingPermissionUnit, AdminPermissionUnit>>();
          if (_admin_data.HasValue)
          {
             var users = _admin_data.Value.Users.Where(predicate);
@@ -51,9 +51,7 @@ namespace mbts.spark
                      if (accounts != null && accounts.Count() == 1)
                      {
                         var account = accounts.ElementAt(0);
-                        var tp = kv.Value.TradingPermission;
-                        var rv_val = new TradingPermissionUnitImmutable(tp);
-                        rv.Add(Tuple.Create(account, rv_val));
+                        rv.Add(Tuple.Create(account, kv.Value.TradingPermission, kv.Value.AdminPermission));
                      }
                   }
                }

src/client/spark/Daemons/MDDaemon.cs

@@ -49,7 +49,7 @@ namespace bts {
             .Subscribe(t => {
                var deleted_synthetic_keys = new HashSet<uint>(t.Where(kv => kv.Value.Deleted).Select(kv => kv.Key));
                // add back 'undeleted' synthetics
-               foreach (var kvp in _deleted_synthetics) {
+               foreach (var kvp in _deleted_synthetics.ToList()) {
                   if (!deleted_synthetic_keys.Contains(kvp.Key)) {
                      _exchange_sec_map.TryAdd(kvp.Key, kvp.Value);
                      _deleted_synthetics.Remove(kvp.Key);

src/client/spark/Modules/Market/MarketView/MarketViewWindow.cs

@@ -65,7 +65,7 @@ namespace mbts.spark {
                })
             ),
             SparkCommands.ShowTimeAndSales.complete(_vm.selected_securities_observable(), _ => true, secs => secs.Count > 0 ? new SecurityCriteria(secs) : _state.security_criteria),
-            SparkCommands.ShowSyntheticCreate.complete(_vm.selected_securities_observable(), secs => secs.Count > 0 && secs.Count <= 24, secs => new SecurityCriteria(secs)),
+            SparkCommands.ShowSyntheticCreate.complete(_vm.sorted_selected_securities_observable(), secs => secs.Count > 0 && secs.Count <= 24, secs => new SecurityCriteria(secs)),
             SparkCommands.change_securities_command(() => (update_securities_cmd, _state.security_criteria)),
             // wrap the grid settings view factory in a dialogCommand here since the vm is destroyed/recreated when the security criteria changes
             SparkCommands.window_settings_command(new DialogCommand(() => {

src/client/spark/Modules/Spreader/ShowSyntheticEditorHandler.cs

@@ -6,6 +6,7 @@ using mbts.ui;
 using mbts.ui.spark;
 using mbts.ui.utility;
 using System;
+using System.Collections;
 using System.Collections.Generic;
 using System.ComponentModel.Composition;
 using System.Linq;
@@ -43,8 +44,23 @@ namespace bts.shell {
             name + "(1)";
       }
 
-      async void _show_synthetic_create(SecurityCriteria security_criteria) =>
-         _show_synthetic_create((await security_criteria.securities(_bus)).ToList());
+      async void _show_synthetic_create(SecurityCriteria security_criteria) {
+         var found_securities = await security_criteria.securities(_bus);
+         var respect_clause_security_ordering_is_possible = security_criteria.clauses.Any(clause => clause.security_keys != null && clause.security_keys.Count > 0);
+         if (!respect_clause_security_ordering_is_possible) {
+            _show_synthetic_create(found_securities.Values.ToList());
+         } else {
+            List<SecurityImmutable> ordered_security_list = new List<SecurityImmutable>();
+            foreach (var clause in security_criteria.clauses) {
+               foreach (var sec_key in clause.security_keys) {
+                  if (found_securities.TryGetValue(sec_key, out SecurityImmutable found_result)) { 
+                     ordered_security_list.Add(found_result);
+                  }
+               }
+               _show_synthetic_create(ordered_security_list);
+            }
+         }
+      }
 
       void _show_synthetic_create(IReadOnlyCollection<SecurityImmutable> securities) {
          if (securities.Count == 1) {

src/client/spark/Orders/OrderRecordTupleSource.cs

@@ -95,7 +95,7 @@ namespace mbts.spark {
             /// Warning: <see cref="_security_md_cache"/> must NOT be reassigned with <see cref="securities"/>; the latter is always the same object.
             /// This would mean that the <see cref="GlobalExtensions.difference{T}(IEnumerable{T}, IEnumerable{T}, out IEnumerable{T}, out IEnumerable{T})"/>
             /// call above would literally be calculating the difference between the keys of the same dictionary!
-            foreach (var k in removed_keys) {
+            foreach (var k in removed_keys.ToList()) {
                if (_security_orders.try_lookup(k, out ISet<uint> order_keys)) {
                   SecurityMD old_sec_md = _security_md_cache[k];
                   removed_securities.Add(k, old_sec_md);
@@ -109,7 +109,7 @@ namespace mbts.spark {
                }
                _security_md_cache.Remove(k);
             }
-            foreach (var k in added_keys) {
+            foreach (var k in added_keys.ToList()) {
                SecurityMD new_sec_md = securities[k];
                if (_security_orders.try_lookup(k, out ISet<uint> order_keys)) {
                   added_securities.Add(k, new_sec_md);

src/client/spark/Orders/OrderTypeVMs/LeanOrderVM.cs

@@ -25,7 +25,7 @@ namespace mbts.spark
             cross_threshold = new InlineUserProperty<bts.spark.QuantityMeasure>(
                               new bts.spark.QuantityMeasure() { Ratio = 1.0, Absolute = 1}, false);
             max_chase_ticks = new InlineUserProperty<uint>(2, false);
-            trade_toward_behavior = new InlineUserProperty<LeanOrderDefinition.Types.OnToward>(LeanOrderDefinition.Types.OnToward.Cancel, false);
+            trade_toward_behavior = new InlineUserProperty<LeanOrderDefinition.Types.OnToward>(LeanOrderDefinition.Types.OnToward.NoChange, false);
          }
          public readonly InlineUserProperty<OrderTemplateHeader> selected_child_order_type;
          public readonly InlineUserProperty<IReadOnlyCollection<OrderTemplateHeader>> child_order_types;
@@ -47,7 +47,7 @@ namespace mbts.spark
          
          public bts.spark.QuantityMeasure cross_threshold { get; set; } = new bts.spark.QuantityMeasure() { Ratio = 1.0, Absolute = 1 };
          public uint max_chase_ticks { get; set; } = 2;
-         public LeanOrderDefinition.Types.OnToward trade_toward_behavior { get; set; } = LeanOrderDefinition.Types.OnToward.Cancel;      
+         public LeanOrderDefinition.Types.OnToward trade_toward_behavior { get; set; } = LeanOrderDefinition.Types.OnToward.NoChange;      
 
          public LeanOrderVM(OrdersModel model, IObservable<SecurityMD> security_md_source, string id, string display_name)
             : this(model, security_md_source, security_md_source.Select(smd =>

src/client/spark/Spark.csproj

@@ -82,9 +82,4 @@
   </ItemGroup>
   <Target Name="diagnostic" AfterTargets="PreBuildEvent">
   </Target>
-  <Target Name="copy_defaults" AfterTargets="PostBuildEvent">
-  <!-- this feels like the wrong place to copy defaults but it works alright -->
-    <!-- <Message Importance="High" Text="copy defaults :: @(ConfigDefaults)" /> -->
-     <Copy SourceFiles="@(ConfigDefaults)" DestinationFolder="$(OutputPath)\defaults\" SkipUnchangedFiles="true" />
-  </Target>
 </Project>

src/client/spark/view/Controls/UniformTabGrid/UniformTabGrid.cs

@@ -0,0 +1,50 @@
+using bts;
+using mbts.spark.filters;
+using mbts.ui.controls.adorners;
+using System.Collections.Generic;
+using System.Linq;
+using System.Windows;
+using System.Windows.Controls;
+using System.Windows.Controls.Primitives;
+
+namespace mbts.ui.controls {
+   public class UniformTabGrid : UniformGrid {
+      public UniformTabGrid () {
+         this.IsItemsHost = true;
+         this.Rows = 1;
+      }
+ 
+      protected override Size MeasureOverride(Size constraint) {
+         var child_tabs = this.Children.OfType<TabItem>();
+         var child_tabs_max_width = child_tabs.Sum(t => t.MaxWidth);
+         // I noticed when debugging that this can sometimes be infinty,
+         // especially when a TabItem has not been set with a MaxWidth.
+         if (!double.IsInfinity(child_tabs_max_width)) {
+            // if the window width is greater than the total tab width, then 
+            // there is more than enough space for all the tabs. If this
+            // is set to the default of Stretch and the window has only 1
+            // tab or few tabs, the tab appears in the middle, which is not good.
+            // i.e. we want tabs to added left to right, so align left
+            //
+            // When the window is less than the total tab width, then align
+            // can be set to stretch b/c the UniformGrid will take care of
+            // sizing each tab accordingly.
+            this.HorizontalAlignment = (constraint.Width > child_tabs_max_width)
+               ? HorizontalAlignment.Left
+               : HorizontalAlignment.Stretch;
+ 
+            // this bit here is necessary because when the window is expanded
+            // beyond the maxwidth of all the tabs, we switch to Left and the
+            // headers snap to some smaller than MaxWidth size, which is not
+            // the behavior we want
+            foreach (var child in child_tabs) {
+               child.Width = this.HorizontalAlignment == HorizontalAlignment.Left
+                  ? child.MaxWidth
+                  : double.NaN;
+            }
+         }
+ 
+         return base.MeasureOverride(constraint);
+      }
+   }
+}

src/client/spark/view/Shell/FloatingWindow.xaml

@@ -5,6 +5,7 @@
          xmlns:d="http://schemas.microsoft.com/expression/blend/2008" 
          xmlns:diag="clr-namespace:System.Diagnostics;assembly=WindowsBase"
          xmlns:ui="clr-namespace:mbts.ui"
+         xmlns:c="clr-namespace:mbts.ui.controls"
          mc:Ignorable="d"  AllowDrop="True" 
          Style="{StaticResource BTSFloatingWindow}"
          d:DesignHeight="300" d:DesignWidth="300" 
@@ -25,18 +26,25 @@
             <RowDefinition Height="Auto" />
             <RowDefinition Height="*" />
           </Grid.RowDefinitions>
-          <WrapPanel x:Name="HeaderPanel" Orientation="Horizontal" Margin="0,0,100,0"
+          <!-- This stack panel serves as the area between the last tab and the widget control on right side of title bar (below).
+               It allows the region in between (if such a region exists) to be grabbed to move the window.
+          -->
+          <StackPanel Grid.Row="0" Margin="0,0,100,0" Background="Transparent" ui:WindowStyle.IsNonClientArea="True">
+            <c:UniformTabGrid x:Name="HeaderPanel" 
                     Grid.Row="0"
                     Panel.ZIndex="1"
-                    IsItemsHost="True"
                     KeyboardNavigation.TabIndex="1"
-                    Background="Transparent" ui:WindowStyle.IsNonClientArea="True">
-          </WrapPanel>
+                    Background="Transparent" 
+                    ui:WindowStyle.IsNonClientArea="True"
+                    >
+            </c:UniformTabGrid>
+          </StackPanel>
           <Border Grid.Row="1" BorderThickness="0" 
                   Padding="{Binding Path=Padding, RelativeSource={RelativeSource AncestorType=Window}}" ui:WindowStyle.IsNonClientArea="False" >
             <ContentPresenter x:Name="PART_SelectedContentHost" ContentSource="SelectedContent" />
           </Border>
-          <Border Grid.Row="0" Width="100" HorizontalAlignment="Right" VerticalAlignment="Stretch" Panel.ZIndex="2" Background="Transparent" ui:WindowStyle.IsNonClientArea="True">
+          <!-- Window widget control area-->
+          <Border Grid.Row="0" MinWidth="100" HorizontalAlignment="Right" Panel.ZIndex="2" Background="Transparent" ui:WindowStyle.IsNonClientArea="True">
             <StackPanel Grid.Row="0" Orientation="Horizontal" HorizontalAlignment="Right" VerticalAlignment="Top" Panel.ZIndex="2" Background="Transparent" Margin="5">
               <Button Style="{StaticResource RoundMinimizeButton}" HorizontalAlignment="Right" Command="{x:Static SystemCommands.MinimizeWindowCommand}" />
               <Button Style="{StaticResource RoundMaximizeButton}" HorizontalAlignment="Right" Command="{x:Static SystemCommands.MaximizeWindowCommand}" />

src/client/spark/view/Shell/TabDocument.xaml

@@ -45,7 +45,7 @@
   </TabItem.RenderTransform>
   <TabItem.Template>
     <ControlTemplate TargetType="{x:Type TabItem}">
-      <Border x:Name="bd" BorderBrush="Black" BorderThickness="0,0,0.5,0" Background="#d6d6d6" Width="{DynamicResource TabWidth}" Height="{DynamicResource TabHeight}" MinWidth="40" Margin="0" SnapsToDevicePixels="True" DockPanel.Dock="Left">
+      <Border x:Name="bd" BorderBrush="Black" BorderThickness="0,0,0.5,0" Background="#d6d6d6" Height="{DynamicResource TabHeight}" MinWidth="40" Margin="0" SnapsToDevicePixels="True" DockPanel.Dock="Left">
         <Border.ToolTip>
           <TextBlock Text="{Binding Title}" />
         </Border.ToolTip>
@@ -90,6 +90,7 @@
   </TabItem.Template>
   <TabItem.Style>
     <Style TargetType="TabItem" BasedOn="{StaticResource {x:Type TabItem}}">
+      <Setter Property="MaxWidth" Value="{DynamicResource TabWidth}" />
       <Setter Property="HeaderTemplate" Value="{StaticResource _template}" />
       <Style.Triggers>
         <DataTrigger Binding="{Binding EditMode}" Value="True">

src/client/spark/view/Spark/ContractPicker/SecurityCriteriaExtensions.cs

@@ -18,10 +18,10 @@ namespace mbts.spark {
          return self == null ? all.Values : all.Values.Where(md => ContractPickerModel.security_in_criteria(self, md.security));
       }
 
-      public static async Task<IEnumerable<SecurityImmutable>> securities(this SecurityCriteria self, IBus bus) {
+      public static async Task<IDictionary<uint,SecurityImmutable>> securities(this SecurityCriteria self, IBus bus) {
          var md_map = await bus.subscribe(SparkTags.SecuritiesTag).FirstAsync();
-         var all_secs = md_map.Values.Select(md => md.security);
-         return self == null ? all_secs : all_secs.Where(s => ContractPickerModel.security_in_criteria(self, s));
+         var all_secs = md_map.Values.Select(md => md.security).ToDictionary(key => key.Id);
+         return self == null ? all_secs : all_secs.Values.Where(s => ContractPickerModel.security_in_criteria(self, s)).ToDictionary(key => key.Id);
       }
 
       public static IObservable<IEnumerable<SecurityImmutable>> subscribe(this SecurityCriteria self, IBus bus) {

src/client/spark/view/Spark/Grid/SecurityGrid.cs

@@ -122,6 +122,21 @@ namespace mbts.spark
          where TRow : class, ISecurityRow =>
          security_grid.selected_securities.ObserveSelf();
 
+      public static IObservable<IReadOnlyList<SecurityImmutable>> sorted_selected_securities_observable<TRow, TRowModel>(this SecurityGridBase<TRow, TRowModel> security_grid)
+         where TRow : class, ISecurityRow =>
+         security_grid.selected_rows
+         .ObserveSelf()
+         .Select(rows =>
+              rows.Select(
+                 row => (security: row.security, row_index: security_grid.rows_view.indexOf(row))
+              )
+              .OrderBy(sec_vs_index => sec_vs_index.row_index)
+              .Select(sec_vs_index => sec_vs_index.security)
+              .ToReadOnlyList()
+         );
+               
+         
+
       public static IObservable<IReadOnlyList<SecurityMD>> selected_security_mds_observable<TRow, TRowModel>(this SecurityMDGrid<TRow, TRowModel> security_md_grid)
          where TRow : class, ISecurityMDRow =>
          security_md_grid.selected_rows.ObserveSelf(true).Select(rows => rows.Select(r => r.md).ToReadOnlyList());

src/client/spark_api/md/Authenticator.cs

@@ -0,0 +1,23 @@
+using System;
+using System.Threading.Tasks;
+using bts.api;
+using bts.api.core;
+
+namespace bts.api
+{
+   public class Authenticator
+   {
+      readonly static Lazy<user.UserService> _auth = new Lazy<user.UserService>(() => Locator.locate<user.UserService>(), true);
+      user.AuthToken _token = null;
+      public Authenticator() { }
+
+      public user.AuthToken last_token => _token;
+      public async Task<user.AuthToken> login(string login, string password)
+      {
+         var res = await _auth.Value.authenticate_login_async(new user.AuthenticateLoginRequest { Login = login, Password = password });
+         if (res.success)
+            _token = res.value.Token;            
+         return _token;
+      }
+   }
+}

src/client/spark_api/md/LazyAsync.cs

@@ -0,0 +1,17 @@
+
+using System;
+using System.Threading.Tasks;
+
+namespace bts
+{
+   public class AsyncLazy<T> : Lazy<Task<T>>
+   {
+      public AsyncLazy(Func<T> valueFactory) :
+          base(() => Task.Factory.StartNew(valueFactory))
+      { }
+
+      public AsyncLazy(Func<Task<T>> taskFactory) :
+          base(() => Task.Factory.StartNew(() => taskFactory()).Unwrap())
+      { }
+   }   
+}

src/client/spark_api/md/Locator.cs

@@ -0,0 +1,16 @@
+using System;
+using System.Threading.Tasks;
+using bts.api;
+using bts.api.core;
+
+namespace bts.api
+{
+   public static class Locator
+   {
+      readonly static Lazy<ServiceContext> _context = new Lazy<ServiceContext>(() => new ServiceContext(), true);      
+      public static T locate<T>(string qualifier="")  where T : class, IApiClient
+      {
+         return _context.Value.locate<T>(qualifier);
+      }      
+   }
+}

src/client/spark_api/md/MarketDataClient.cs

@@ -33,49 +33,48 @@ namespace bts.api.md
       readonly ConcurrentDictionary<security_key, Data> _data = new ConcurrentDictionary<security_key, Data>();
       readonly ConcurrentDictionary<security_key, string> _security_key_to_desc= new ConcurrentDictionary<security_key, string>();
       readonly ConcurrentDictionary<string, security_key> _security_desc_to_key = new ConcurrentDictionary<string, security_key>();      
-      readonly Lazy<MDRequest> _subscription;
+      readonly MDRequest _subscription;
       readonly Venue _venue;
 
-      readonly static Lazy<MarketDataRelay> _md = new Lazy<MarketDataRelay>(() => locate(), true);
+      readonly static Lazy<MarketDataRelay> _md = new Lazy<MarketDataRelay>(() => Locator.locate<MarketDataRelay>(), true);
       readonly static ConcurrentDictionary<api.Venue, MarketDataClient> _venue_cache = new ConcurrentDictionary<api.Venue, api.md.MarketDataClient>();
       static readonly List<Product> _all_products;
       static MarketDataClient()
       {
          _all_products = _md.Value.request_products(new VoidMessage()).Products.ToList();
       }
-      public static bool create_or_get(api.Venue exchange, out MarketDataClient md)
+
+      public static IEnumerable<api.Venue> cached_exchanges => _venue_cache.Keys;
+      public static bool create_or_get(api.Venue exchange, out MarketDataClient md, user.AuthToken token = null)
       {
          if (!_venue_cache.TryGetValue(exchange, out md))
          {
-            _venue_cache[exchange] = md = new MarketDataClient(exchange);
+            _venue_cache[exchange] = md = new MarketDataClient(exchange, token);
             return true;
          }
          return false;
       }      
 
-      public MarketDataClient(bts.api.Venue exchange = Venue.Unset) 
+      public MarketDataClient(bts.api.Venue exchange = Venue.Unset, user.AuthToken token = null) 
       {
          _venue = exchange;
-         _subscription = new Lazy<MDRequest>(() => load_request(exchange));
+         _subscription = load_request(exchange, token);
       }
 
-      MDRequest load_request(bts.api.Venue exchange)
+      public void set_auth_token(user.AuthToken toke)
       {
-         var req = new MDRequest();
+         _subscription.Token = toke?.Token ?? string.Empty;
+      }
+
+      MDRequest load_request(bts.api.Venue exchange, user.AuthToken token)
+      {
+         var req = new MDRequest() { Token = token?.Token ?? string.Empty } ;
          var prods = _all_products.Where(x=>x.Exchange == exchange || exchange == bts.api.Venue.Unset);
          foreach(var prod in prods)
             req.Channels.Add(new SecurityChannel() { Exchange = prod.Exchange, Type = prod.Type, Symbol = prod.Symbol, Flags = 0 });
          return req;
       }
 
-      static MarketDataRelay locate()
-      {
-         using (var ctx = new api.core.ServiceContext())
-         {
-            return ctx.locate<MarketDataRelay>();
-         }
-      }
-
       public Security lookup(security_key key)
       {
          Data value;
@@ -217,7 +216,7 @@ namespace bts.api.md
 
       bool check_subscriptions(uint flags, Func<SecurityChannel, bool> selector)
       {
-         foreach (var sub in _subscription.Value.Channels)
+         foreach (var sub in _subscription.Channels)
          {
             if (selector(sub))
             {
@@ -251,7 +250,7 @@ namespace bts.api.md
          while (!rep.success && count++ < 5);
          while (true)
          {
-            var reply = await _md.Value.requestmd_async(_subscription.Value);
+            var reply = await _md.Value.requestmd_async(_subscription);
             if (reply.success)
             {
                foreach (var channel in reply.value.Channels)

src/client/spark_api/utils/Log.cs

@@ -32,12 +32,14 @@ namespace bts {
       Testing,
       Latency,
       DataFlow,
-      Plugin
+      Plugin,
+      ClientSdk
       // Please keep this list in sync with logging.h/logging.cpp
    }
-   readonly struct Logger {
+   
+   public readonly struct Logger {
       readonly Subsystem _sys;
-      public Logger(Subsystem sys) {
+      internal Logger(Subsystem sys) {
          _sys = sys;
       }
 
@@ -61,19 +63,20 @@ namespace bts {
       static extern void Logger_log (Level level, Subsystem subsystem, string message, string file, int line);
    }
 
-   static class Log {
-      public static Logger Gen => new Logger(Subsystem.gen);
-      public static Logger Config => new Logger(Subsystem.Config);
-      public static Logger QBus => new Logger(Subsystem.QBus);
-      public static Logger Md => new Logger(Subsystem.md);
-      public static Logger Api => new Logger(Subsystem.Api);
-      public static Logger Persist => new Logger(Subsystem.Persist);
-      public static Logger User => new Logger(Subsystem.User);
-      public static Logger OrderMan => new Logger(Subsystem.OrderMan);
-      public static Logger Assert => new Logger(Subsystem.Assert);
-      public static Logger Ui => new Logger(Subsystem.Ui);
-      public static Logger Testing => new Logger(Subsystem.Testing);
-      public static Logger DataFlow => new Logger(Subsystem.DataFlow);
+   public static class Log {
+      internal static Logger Gen => new Logger(Subsystem.gen);
+      internal static Logger Config => new Logger(Subsystem.Config);
+      internal static Logger QBus => new Logger(Subsystem.QBus);
+      internal static Logger Md => new Logger(Subsystem.md);
+      internal static Logger Api => new Logger(Subsystem.Api);
+      internal static Logger Persist => new Logger(Subsystem.Persist);
+      internal static Logger User => new Logger(Subsystem.User);
+      internal static Logger OrderMan => new Logger(Subsystem.OrderMan);
+      internal static Logger Assert => new Logger(Subsystem.Assert);
+      internal static Logger Ui => new Logger(Subsystem.Ui);
+      internal static Logger Testing => new Logger(Subsystem.Testing);
+      internal static Logger DataFlow => new Logger(Subsystem.DataFlow);
+      public static Logger ClientSdk => new Logger(Subsystem.ClientSdk);
 
 
       [Conditional("DEBUG")]
@@ -99,10 +102,10 @@ namespace bts {
          assert(o == null);
       }
       delegate void on_crash_callback([MarshalAs(UnmanagedType.LPStr)]string msg, [MarshalAs(UnmanagedType.LPStr)] string details);
-      public static void system_is_crashing(string message, object details) {
+      internal static void system_is_crashing(string message, object details) {
          OnSystemCrashing?.Invoke(message, details);
       }
-      public static event Action<string, object> OnSystemCrashing;
+      internal static event Action<string, object> OnSystemCrashing;
 
-   }
+   }  
 }

src/client/spark_excel/ExcelAddin.cs

@@ -69,5 +69,21 @@ namespace bts.excel
                Utilities.bts_load_assembly(ass);
          }
       }
+
+      public static void calculate_full_rebuild()
+      {
+         object app = ExcelDnaUtil.Application;
+         var mi = app?.GetType().GetMethod("CalculateFullRebuild", BindingFlags.Instance | BindingFlags.Public);
+         if(mi != null)
+            mi.Invoke(app, null);
+      }
+
+      public static void calculate_full()
+      {
+         object app = ExcelDnaUtil.Application;
+         var mi = app?.GetType().GetMethod("CalculateFull", BindingFlags.Instance | BindingFlags.Public);
+         if (mi != null)
+            mi.Invoke(app, null);
+      }      
    }
 }

src/client/spark_excel/Properties/ExcelDna.Build.props

@@ -38,7 +38,7 @@
     <!--
       Enable/Disable packing of .dna files
     -->
-    <RunExcelDnaPack Condition="'$(RunExcelDnaPack)' == ''">false</RunExcelDnaPack>
+    <RunExcelDnaPack Condition="'$(RunExcelDnaPack)' == ''">true</RunExcelDnaPack>
 
     <!--
       Suffix used for packed .xll files e.g. MyAddIn-packed.xll

src/client/spark_excel/extensions/OrderService_Excel.cs

@@ -1,7 +1,10 @@
 using System;
 using System.Collections.Concurrent;
+using System.Collections.Generic;
 using System.Linq;
 using ExcelDna.Integration;
+using ExcelDna.Integration.Rtd;
+using ExcelDna.Registration.Utils;
 
 using bts.api.admin;
 using bts.excel;
@@ -14,9 +17,30 @@ namespace bts.api.orderman
 {
    public partial class OrderService_Excel 
    {
-      readonly static Lazy<user.UserService> _users = new Lazy<user.UserService>(() => locate<user.UserService>(), true);
-      readonly static Lazy<admin.AdminService> _admin = new Lazy<admin.AdminService>(() => locate<admin.AdminService>(), true);
-      
+      readonly static Lazy<user.UserService> _users = new Lazy<user.UserService>(() => Locator.locate<user.UserService>(), true);
+      readonly static Lazy<admin.AdminService> _admin = new Lazy<admin.AdminService>(() => Locator.locate<admin.AdminService>(), true);
+
+      [ExcelFunction(Category = "BTS", Description = "Get All Accessible Accounts")]
+      async public static void spark_get_accounts(string login_token, ExcelAsyncHandle handle)
+      {
+         try
+         {
+            List<string> accts = new List<string>();
+            var admin = await _admin.Value.get_filtered_admin_data_async(new FilteredAdminDataRequest { Token = login_token });
+            if (admin.success)
+            {
+               foreach (var acc in admin.value.Accounts)
+                  if(acc.IsPresent)
+                     accts.Add(acc.Name);
+            }
+            var res = Utilities.make_column_vector(accts.ToArray());
+            handle.SetResult(res);
+         }
+         catch (Exception ex)
+         {
+            handle.SetException(ex);
+         }
+      }
 
       [ExcelFunction(Category = "BTS", Description = "Cancel an Order by Order ID", IsMacroType = true)]
       public static object spark_orderservice_cancel_order(long order_id)

src/client/spark_excel/rtd/formatting/JsonFormatter.cs

@@ -6,6 +6,7 @@ using System.Reactive.Linq;
 using Google.Protobuf;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
+using ExcelDna.Integration;
 
 namespace bts.excel.rtd
 {
@@ -17,7 +18,7 @@ namespace bts.excel.rtd
          _data = data;
       }
 
-      #region parser
+      #region parser         
       static IDictionary<string, object> parse_dictionary(object[,] range)
       {
          Dictionary<string, object> dict = new Dictionary<string, object>();

src/client/spark_excel/rtd/formatting/TopOfBookMDFormatter.cs

@@ -3,10 +3,10 @@ using System;
 
 namespace bts.excel.rtd
 {
-   class TopMDFormatter : IExcelFormatter
+   class TopOfBookMDFormatter : IExcelFormatter
    {
       readonly co2.ChannelUpdate _update;
-      public TopMDFormatter(co2.ChannelUpdate upd)
+      public TopOfBookMDFormatter(co2.ChannelUpdate upd)
       {
          _update = upd;
       }

src/client/spark_excel/rtd/formatting/TopicMDFormatter.cs

@@ -0,0 +1,90 @@
+
+using System;
+
+namespace bts.excel.rtd
+{
+   class TopicMDFormatter : IExcelFormatter
+   {
+      readonly co2.ChannelUpdate _update;
+      readonly string _topic;
+      public TopicMDFormatter(co2.ChannelUpdate upd, string topic)
+      {
+         _update = upd;
+         _topic = topic;
+      }
+
+      public object[,] format(bool include_headers = false)
+      {
+         var retval = new object[1, 1];
+         switch (_topic.ToLower())
+         {
+            case "bid":
+            case "bidprice":
+               retval[0, 0] = BookMDFormatter.format_md(_update?.Direct?.Bids?[0]?.Mantissa ?? 0, _update?.Direct?.Bids?[0]?.Exponent ?? 0);
+               break;
+            case "ask":
+            case "askprice":
+               retval[0, 0] = BookMDFormatter.format_md(_update?.Direct?.Asks?[0]?.Mantissa ?? 0, _update?.Direct?.Asks?[0]?.Exponent ?? 0);
+               break;
+            case "bidsize":
+            case "bidqty":
+               retval[0, 0] = _update?.Direct?.Bids?[0]?.Size;
+               break;
+            case "asksize":
+            case "askqty":
+               retval[0, 0] = _update?.Direct?.Asks?[0]?.Size;
+               break;
+            case "impliedbid":
+            case "impliedbidprice":
+               retval[0, 0] = BookMDFormatter.format_md(_update?.Implied?.Bids?[0]?.Mantissa ?? 0, _update?.Implied?.Bids?[0]?.Exponent ?? 0);
+               break;
+            case "impliedask":
+            case "impliedaskprice":
+               retval[0, 0] = BookMDFormatter.format_md(_update?.Implied?.Asks?[0]?.Mantissa ?? 0, _update?.Implied?.Asks?[0]?.Exponent ?? 0);
+               break;
+            case "impliedbidsize":
+            case "impliedbidqty":
+               retval[0, 0] = _update?.Implied?.Bids?[0]?.Size;
+               break;
+            case "impliedasksize":
+            case "impliedaskqty":
+               retval[0, 0] = _update?.Implied?.Asks?[0]?.Size;
+               break;
+            case "settlement":
+            case "settlementprice":
+               retval[0, 0] = BookMDFormatter.format_md(_update?.Misc?.Settlement?.Mantissa ?? 0, _update?.Misc?.Settlement?.Exponent ?? 0);
+               break;
+            case "previoussettlement":
+            case "previoussettlementprice":
+               retval[0, 0] = BookMDFormatter.format_md(_update?.Misc?.PreviousSettlementPrice?.Mantissa ?? 0, _update?.Misc?.PreviousSettlementPrice?.Exponent ?? 0);
+               break;
+            case "open":
+            case "openprice":
+               retval[0, 0] = _update?.Misc?.OpenPrice;
+               break;
+            case "close":
+            case "closeprice":
+               retval[0, 0] = _update?.Misc?.ClosePrice;
+               break;
+            case "high":
+            case "highprice":
+               retval[0, 0] = _update?.Misc?.SessionHigh;
+               break;
+            case "low":
+            case "lowprice":
+               retval[0, 0] = _update?.Misc?.SessionLow;
+               break;
+            case "last":
+            case "lastprice":
+               retval[0, 0] = BookMDFormatter.format_md(_update?.LastTrade?.Mantissa ?? 0, _update?.LastTrade?.Exponent ?? 0);
+               break;
+            case "lastsize":
+            case "lastqty":
+               retval[0, 0] = _update?.LastTrade?.Size;
+               break;               
+         }
+         return retval;
+      }
+   }
+
+}

src/client/spark_excel/rtd/subscriptions/MDSubscribe.cs

@@ -4,16 +4,36 @@ using System.Linq;
 
 using ExcelDna.Integration;
 using ExcelDna.Integration.Rtd;
+using ExcelDna.Registration.Utils;
 using System.Reactive.Linq;
 using bts.api;
 using bts.api.md;
 using bts.api.orderman;
-using ExcelDna.Registration.Utils;
 
 namespace bts.excel.rtd
 {
    public class MDSubscribe
    {
+      static Authenticator _auth = new Authenticator();
+      [ExcelFunction(Category = "BTS", Description = "Log in to BTS Spark")]
+      async public static void spark_login(string login, string password, ExcelAsyncHandle handle) 
+      {
+         try
+         {
+            var toke = await _auth.login(login, password);
+            foreach (var ven in MarketDataClient.cached_exchanges)
+            {
+               MarketDataClient.create_or_get(ven, out MarketDataClient md);
+               md?.set_auth_token(toke);
+            }
+            handle.SetResult(toke?.Token ?? "Could not log in check user/pass");
+         }
+         catch(Exception ex)
+         {
+            handle.SetException(ex);
+         }
+      }
+
       public static MarketDataClient market_data(api.Venue exchange)
       {
          var newc = MarketDataClient.create_or_get(exchange, out MarketDataClient md);
@@ -83,18 +103,18 @@ namespace bts.excel.rtd
    
 
       [ExcelFunction(Category = "BTS", Description = "Subscribe to BTS Spark top of book market data")]
-      public static object spark_md_top(
+      public static object spark_md_top_of_book(
                                     [ExcelArgument("Required. The specific contract, GCZ8 or ZNU8-ZNZ8, etc")] string contract,
                                     [ExcelArgument("Required. The symbol, GC or ZN, etc")] string symbol,
                                     [ExcelArgument("Optional. The type, Future or Spread, default = Future")] string security_type = null,
                                     [ExcelArgument("Optional. The exchange, default = GLBX")] string exchange = null)
       {
          var param = new object[] { contract, symbol, security_type, exchange };
-         return ObservableRtdUtil.Observe(nameof(spark_md_top), param,
+         return ObservableRtdUtil.Observe(nameof(spark_md_top_of_book), param,
                                           () => md_observable(contract, symbol, security_type, exchange).Select(x =>
                                           {
                                              if (x is co2.ChannelUpdate upd)
-                                                return new TopMDFormatter(upd).format();
+                                                return new TopOfBookMDFormatter(upd).format();
                                              return x;
                                           }));
       }      
@@ -116,5 +136,23 @@ namespace bts.excel.rtd
                                           }));
 
       }      
+      
+      [ExcelFunction(Category = "BTS", Description = "Subscribe to BTS Spark top of book market data")]
+      public static object spark_md(
+                                    [ExcelArgument("Required. The 'Topic' string, 'bid', 'ask', etc")] string topic,
+                                    [ExcelArgument("Required. The specific contract, GCZ8 or ZNU8-ZNZ8, etc")] string contract,
+                                    [ExcelArgument("Required. The symbol, GC or ZN, etc")] string symbol,
+                                    [ExcelArgument("Optional. The type, Future or Spread, default = Future")] string security_type = null,
+                                    [ExcelArgument("Optional. The exchange, default = GLBX")] string exchange = null)
+      {
+         var param = new object[] { topic, contract, symbol, security_type, exchange };
+         return ObservableRtdUtil.Observe(nameof(spark_md), param,
+                                          () => md_observable(contract, symbol, security_type, exchange).Select(x =>
+                                          {
+                                             if (x is co2.ChannelUpdate upd)
+                                                return new TopicMDFormatter(upd, topic).format();
+                                             return x;
+                                          }));
+      }           
    }
 }

src/client/spark_excel/spark_excel-addin.dna

@@ -2,8 +2,13 @@
   <ExternalLibrary Path="spark_excel.dll" ExplicitExports="false" LoadFromBytes="true" Pack="true" />
   
    <Reference Path="ExcelDna.IntelliSense.dll" Pack="true" />
-   
-   <Reference Path="nunit.framework.dll" Pack="true" ExplicitExports="true" />
+   <Reference Path="ExcelDna.Registration.dll" Pack="true" />
+   <Reference Path="ExcelDna.Integration.dll" Pack="true" />   
+      
+   <Reference Path="Google.Protobuf.dll" Pack="true" ExplicitExports="true" />
+   <Reference Path="System.Linq.Dynamic.dll" Pack="true" ExplicitExports="true" />
+   <Reference Path="Newtonsoft.Json.dll" Pack="true" ExplicitExports="true" />
+   <Reference Path="nunit.framework.dll" Pack="true" ExplicitExports="true" />   
    <Reference Path="System.Reactive.Core.dll" Pack="true" ExplicitExports="true" />
    <Reference Path="System.Reactive.Interfaces.dll" Pack="true" ExplicitExports="true" />
    <Reference Path="System.Reactive.Providers.dll" Pack="true" ExplicitExports="true" />

src/client/spark_excel/spark_excel.csproj

@@ -62,7 +62,7 @@
   </ItemGroup>
   
   <ItemGroup>
-    <None Include="spark_excel_example.xlsm"><CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory></None>  
+    <None Include="spark_excel_example_2018.xlsm"><CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory></None>  
   </ItemGroup>
   
   <!-- <Target Name="CopyProjectReferencesToPackage" DependsOnTargets="ResolveReferences">

src/client/spark_excel/utils/Utilities.cs

@@ -5,12 +5,28 @@ using System.Reflection;
 using System.Text.RegularExpressions;
 using ExcelDna.Integration;
 using bts.api;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
 
 namespace bts.excel
 {
    public class Utilities
    {
 
+      [ExcelFunction]
+      public static object bts_json_lookup(string json, string key1, string key2 = null, string key3 = null)
+      {
+         var jo = JObject.Parse(json);
+         JToken val = json;
+         if (!string.IsNullOrEmpty(key1) && !string.IsNullOrEmpty(key2) && !string.IsNullOrEmpty(key3))
+            val = jo[key1][key2][key3];
+         else if (!string.IsNullOrEmpty(key1) && !string.IsNullOrEmpty(key2))
+            val = jo[key1][key2];
+         else if (!string.IsNullOrEmpty(key1))
+            val = jo[key1];
+         return val.ToString();
+      }      
+
       public static bool parse_venue(string exch, out Venue ven)
       {
          if (!Enum.TryParse(exch.ToUpper(), true, out ven))

src/client/spark_sdk/README.md

@@ -0,0 +1,88 @@
+# BTS Spark Client SDK
+
+#### Getting Started
+
+## Git 
+
+* First clone the spark_client_sdk repo
+
+```
+> git clone https://github.com/BlueTradingSystems/spark_client_sdk.git
+```
+
+
+## Build
+* Use msbuild to target windows / .net 4.71
+   * Install studio here if you don't have it: https://visualstudio.microsoft.com/downloads/
+   * Use VSCode for editing, it's better: https://code.visualstudio.com/
+```
+> cd spark_client_sdk\src\spark_sdk
+spark_client_sdk\src\spark_sdk> msbuild /t:restore
+spark_client_sdk\src\spark_sdk> msbuild
+ 
+ ```
+
+ * spark_sdk binaries will be produced in the bin\Debug directory
+
+## Connecting to Spark Server
+
+* Start your exe using the -ConfigServer parameter to point at a running Spark Server
+ ```
+> spark_sdk.exe -ConfigServer http://location:8000/config
+ ```
+ Or create a file called spark.conf at %APPDATA%\bts\conf\ that contains:
+ ```
+include: "http://location:8000/config/"
+ ```
+ 
+ ## Examples
+* Make sure you can connect to a running server
+* run spark_sdk.exe with no parameters to see help
+```
+spark_sdk\bin\Debug\net471> spark_sdk.exe
+
+BTS Spark SDK
+Usages: spark_sdk [command] [arguments]
+
+Commands:
+   md <exchange> <security_type> <symbol> [<security_desc>]
+   ui <exchange> <security_type> <symbol> <security_desc> <poll_interval_in_ms>
+   order <exchange> <security_type> <symbol> <security_desc> <signed_quantity> [<price>]
+Arguments:
+   --pump=<pump_interval_in_ms=100 default>
+```
+
+* Check out sample code
+   * [Market Data](src/spark_sdk/samples/MarketDataSample.cs)
+   * [Orders](src/spark_sdk/samples/OrderSample.cs)
+   * [Driver](src/spark_sdk/samples/SDKMain.cs) 
+
+
+ ## Excel
+ ```
+ spark_sdk\bin\Debug\net471> start spark_excel-addin64-packed.xll 
+ ```
+ * This will start excel and load the bts addin
+ * Check the [```excel```](excel/) directory for an example spreadsheet
+ 
+ 
+## Notes
+
+* Our client side API is based on Google's protocol buffers; the message types are all protos. https://developers.google.com/protocol-buffers/ 
+
+* We extended the protoc compiler to generate c#, c++, excel and js bindings to the services provided by our server process (https://github.com/BlueTradingSystems/spark_sdk) 
+
+
+* ### Dotnet core
+
+   * Right now we only support c# Windows 10 clients, but the package targets both net471 and .net core 2.1.6, so in theory one could run a .net core or c++ client on linux with a little work.  
+
+   * https://dotnet.microsoft.com/
+   
+
+      ```
+      spark_client_sdk\src\spark_sdk> dotnet build
+      ...> dotnet spark_sdk.dll
+      ```
+
+* Send us an email for help: api_support@bluetradesys.com

src/client/spark_sdk/samples/OrderSample.cs

@@ -26,16 +26,9 @@ namespace bts.sdk
          _md = client;
       }
 
-      readonly static Lazy<OrderService> _orders = new Lazy<OrderService>(() => locate<OrderService>(), true);
-      readonly static Lazy<UserService> _users = new Lazy<UserService>(() => locate<UserService>(), true);
-      readonly static Lazy<AdminService> _admin = new Lazy<AdminService>(() => locate<AdminService>(), true);
-      static T locate<T>() where T : class, IApiClient
-      {
-         using (var ctx = new api.core.ServiceContext())
-         {
-            return ctx.locate<T>();
-         }
-      }      
+      readonly static Lazy<OrderService> _orders = new Lazy<OrderService>(() => Locator.locate<OrderService>(), true);
+      readonly static Lazy<UserService> _users = new Lazy<UserService>(() => Locator.locate<UserService>(), true);
+      readonly static Lazy<AdminService> _admin = new Lazy<AdminService>(() => Locator.locate<AdminService>(), true);
 
       void print_update<T>(T update) 
       {

src/client/spark_sdk/samples/SDKMain.cs

@@ -13,56 +13,73 @@ namespace bts.sdk
    
    public class SDKMain
    {
+      private static void user_test()
+      {
+         var ctx = new ServiceContext();
+
+         var s = ctx.locate<api.user.UserService>();
+         var r = new api.user.GetUsersRequest();
+         var resp = s.get_users(r);
+         Console.WriteLine("{0}", resp);
+      }      
       public static void Main(string[] args)
       {
-         MarketDataClient client = new MarketDataClient();
-         CompositeDisposable disp = new CompositeDisposable(client);               
-         if (args.FirstOrDefault() == "ui")
-         {            
-            if (args.Length >= 6)
-            {
-               using (var ui = new ui.UISample(client))
-               {
-                  ui.simple_price_ui(args[1], args[2], args[3], args[4], args[5]);
-               }
-            }
-            else
-               ui_help();
-         }
-         else if (args.Length > 0 && args.FirstOrDefault().Contains("md"))
+         Log.ClientSdk.info("Welcome to the BTS Client Side SDK example");
+         if (args.FirstOrDefault() == "users")
          {
-            var s = new MarketDataSample(client);
-            if (args.FirstOrDefault() == "md")
+            user_test();
+         }
+         else
+         {
+            MarketDataClient client = new MarketDataClient();
+            CompositeDisposable disp = new CompositeDisposable(client);
+            if (args.FirstOrDefault() == "ui")
             {
-               if (args.Length >= 5)
+               if (args.Length >= 6)
                {
-                  s.print_security(args[1], args[2], args[3], args[4]);
+                  using (var ui = new ui.UISample(client))
+                  {
+                     ui.simple_price_ui(args[1], args[2], args[3], args[4], args[5]);
+                  }
                }
-               else if (args.Length >= 4)
+               else
+                  ui_help();
+            }
+            else if (args.Length > 0 && args.FirstOrDefault().Contains("md"))
+            {
+               var s = new MarketDataSample(client);
+               if (args.FirstOrDefault() == "md")
                {
-                  s.print_channel(args[1], args[2], args[3]);
+                  if (args.Length >= 5)
+                  {
+                     s.print_security(args[1], args[2], args[3], args[4]);
+                  }
+                  else if (args.Length >= 4)
+                  {
+                     s.print_channel(args[1], args[2], args[3]);
+                  }
+                  else
+                     md_help();
                }
                else
                   md_help();
-            }    
+            }
+            else if (args.FirstOrDefault() == "order")
+            {
+               OrderSample os = new OrderSample(client);
+               disp.Add(os);
+               order_sample(os, args);
+            }
             else
-               md_help();               
-         }
-         else if (args.FirstOrDefault() == "order")
-         {
-            OrderSample os = new OrderSample(client);
-            disp.Add(os);
-            order_sample(os, args);            
-         }
-         else
-            help();
+               help();
 
-         int ms = 100;
-         string p = args.FirstOrDefault(x => x.Contains("--pump="));
-         if (p != null)
-            ms = int.Parse(p.Replace("--pump=", string.Empty));
-         client.run_pump(TimeSpan.FromMilliseconds(ms));
-         wait_for_stop(disp);
+            int ms = 100;
+            string p = args.FirstOrDefault(x => x.Contains("--pump="));
+            if (p != null)
+               ms = int.Parse(p.Replace("--pump=", string.Empty));
+            client.run_pump(TimeSpan.FromMilliseconds(ms));
+            wait_for_stop(disp);
+         }
       }
 
       static async void order_sample(OrderSample os, string[] args)

src/client/spark_sdk/spark_sdk.csproj

@@ -1,9 +1,8 @@
 <Project Sdk="Microsoft.NET.Sdk">
-  <PropertyGroup>
-  
-    <TargetFrameworks>netstandard2.0;net471</TargetFrameworks>
+  <PropertyGroup>  
+    <TargetFrameworks>netcoreapp2.1;net471</TargetFrameworks>
     <OutputType>Exe</OutputType>
-    <GeneratePackageOnBuild>true</GeneratePackageOnBuild>
+    <GeneratePackageOnBuild Condition="$(Configuration) == 'Release' or $(Configuration) == 'Release'">true</GeneratePackageOnBuild>
     <PackageOutputPath>$(OutputPath)\net471</PackageOutputPath>
     <IncludeSource>true</IncludeSource>
     <IncludeSymbols>true</IncludeSymbols>
@@ -41,17 +40,63 @@
     <Reference Include="System.Windows.Forms" />
     <Reference Include="System.Xml" />  
   </ItemGroup>
-
   
-  
-  <ItemGroup>
+  <!-- <PropertyGroup Condition="'$(TargetFramework)' == 'netcoreapp2.1'">
+    <CopyLocalLockFileAssemblies>true</CopyLocalLockFileAssemblies>
+  </PropertyGroup>  
+  -->
+  <!-- $(SparkRoot) is something we define on our build at BTS, if it's empty, this project file is in the wild-->
+  <ItemGroup Condition= " '$(SparkRoot)' != '' "> 
     <ProjectReference Include="../spark_api/spark_api.csproj"/>
   </ItemGroup>
+ 
+  <ItemGroup Condition= " '$(SparkRoot)' != '' and '$(TargetFramework)' == 'net471' "> 
+    <!-- This is so we can ship excel in the nupkg -->
+    <ProjectReference Include="../spark_excel/spark_excel.csproj"/>     
+  </ItemGroup>
   
-  <Target Name="CopyProjectReferencesToPackage" DependsOnTargets="ResolveReferences">
+  <ItemGroup Condition= " '$(SparkRoot)' != '' ">
+     <None Include="README.md" pack="true" PackagePath="." />
+     <None Include="$(OutputPath)\net471\spark_excel_example_2018.xlsm" pack="true" PackagePath=".\excel\spark_excel_example_2018.xlsm" />
+     <None Include="$(OutputPath)\net471\spark_excel-addin64-packed.xll" pack="true" PackagePath=".\excel\spark_excel-addin64-packed.xll" />           
+     <None Include="$(OutputPath)\net471\spark_excel-addin64.dna" pack="true" PackagePath=".\excel\spark_excel-addin64.dna" />
+  </ItemGroup>  
+      
+  <Target Name="CopyProjectReferencesToPackage" DependsOnTargets="ResolveReferences" Condition= " '$(SparkRoot)' != '' ">
     <ItemGroup>
       <BuildOutputInPackage Include="@(ReferenceCopyLocalPaths->WithMetadataValue('ReferenceSourceTarget', 'ProjectReference'))" />
       <BuildOutputInPackage Include="$(OutputPath)..\net471\libspark.dll"/>
     </ItemGroup>
   </Target>  
+  
+   <Target Name="copy_scripts" AfterTargets="PostBuildEvent" Condition= " '$(SparkRoot)' != '' " >
+     <Copy SourceFiles="../../sdk/update_github.cmd" DestinationFolder="$(OutputPath)\" SkipUnchangedFiles="true" />
+   </Target>  
+   
+  <!-- deployed sdk build net471 --> 
+  <ItemGroup Condition= " '$(SparkRoot)' == '' and '$(TargetFramework)' == 'net471'">
+    <Reference Include="spark_api">
+      <HintPath>..\..\lib\net471\spark_api.dll</HintPath>
+    </Reference>
+  </ItemGroup>
+  
+  <!-- deployed sdk build netcoreapp2.1 -->
+  <ItemGroup Condition= " '$(SparkRoot)' == '' and '$(TargetFramework)' == 'netcoreapp2.1'">
+    <Reference Include="spark_api">
+      <HintPath>..\..\lib\netcoreapp2.1\spark_api.dll</HintPath>
+    </Reference>
+  </ItemGroup>     
+  
+  
+   <Target Name="copy_lib_deployed" AfterTargets="PostBuildEvent" Condition= " '$(SparkRoot)' == '' " >
+     <Copy SourceFiles="..\..\lib\net471\libspark.dll" DestinationFolder="$(OutputPath)\" SkipUnchangedFiles="true" />
+   </Target>    
+   
+   <Target Name="copy_lib_deployed471" AfterTargets="PostBuildEvent" Condition= " '$(SparkRoot)' == '' and '$(TargetFramework)' == 'net471'" >
+     <Copy SourceFiles="..\..\lib\net471\spark_excel.dll" DestinationFolder="$(OutputPath)\" SkipUnchangedFiles="true" />
+     <Copy SourceFiles="..\..\excel\spark_excel_example_2018.xlsm" DestinationFolder="$(OutputPath)\" SkipUnchangedFiles="true" />
+     <Copy SourceFiles="..\..\excel\spark_excel-addin64-packed.xll" DestinationFolder="$(OutputPath)\" SkipUnchangedFiles="true" />
+     <Copy SourceFiles="..\..\excel\spark_excel-addin64.dna" DestinationFolder="$(OutputPath)\" SkipUnchangedFiles="true" />     
+   </Target>       
+  
 </Project>

src/client/sparkcmd/Cmd/ConsoleCommandEx.cs

@@ -0,0 +1,20 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace ManyConsole {
+   public static class ConsoleCommandEx {
+      public static string get_argument(this ConsoleCommand _, string arg_name) {
+         var arg = Environment.GetCommandLineArgs().FirstOrDefault(str => str.StartsWith("--"+arg_name));
+         if (arg != null) {
+            var parts = arg.Split(new[] { '=' }, 2);
+            if (parts.Length > 1) {
+               return parts[1];
+            }
+         }
+         return "";
+      }
+   }
+}

src/client/sparkcmd/Cmd/ServiceTail.cs

@@ -0,0 +1,78 @@
+using bts.api.core;
+using bts.test;
+using ManyConsole;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace bts {
+   class TailService : ConsoleCommand {
+      static bool _all = false;
+      static bool _timestamp = false;
+      public TailService() {
+         void ignore(string _) { }
+         IsCommand("tailservice", "Tail updates to named service api. Ex: tail --method=SyntheticSecurityService.metadata");
+         HasRequiredOption("method", "Service.method to tail", ignore);
+         HasOption("time", "Include timestamp", _ => _timestamp = true);
+         // The following are not implemented. If you need them get to work.
+         HasOption("path", "Path inside the row to use for display. Ex: 'Id' or 'Name' ", ignore);
+         HasOption("all", "Invlude 'all' rows on every print", _ => _all = true);
+      }
+      public override int Run(string[] remainingArguments) {
+         var arg = this.get_argument("method");
+         var split = arg.Split('.');
+         if (split.Length != 2) {
+            Console.WriteLine("Try --method=[ServiceName].[Method]");
+            return -1;
+         }
+         tail(split[0], split[1]);
+         return 0;
+      }
+      class SubscriptionLogger<T> : IObserver<T> {
+         public void OnCompleted() {
+            ConsoleEx.WriteLine("Completed.", ConsoleColor.Yellow);
+         }
+
+         public void OnError(Exception error) {
+            ConsoleEx.WriteLine($"Error. '{error}'", ConsoleColor.Red);
+         }
+
+         public void OnNext(T value) {
+            if (_timestamp) {
+               Console.Write(DateTime.Now.ToString("HH:mm:ss.fff"));
+               Console.Write(" ");
+            }
+            if (_all) { }
+            Console.WriteLine(value);
+         }
+      }
+      private static void tail(string service_name, string method_name) {
+         var spark_api = typeof(ServiceContext).Assembly;
+         var service_type = spark_api.GetExportedTypes().FirstOrDefault(t => t.Name == service_name);
+         if (service_type == null) {
+            Console.WriteLine($"Could not find service type {service_name} in spark_api.dll");
+            return;
+         }
+         var subscribe_method = service_type.GetMethod("subscribei_" + method_name);
+         using (var ctx = new ServiceContext()) {
+            var locate = typeof(ServiceContext).GetMethod("locate").MakeGenericMethod(service_type);
+            var service = locate.Invoke(ctx, new[] { "" });
+            // Subscribe with null qualifier
+            object observable = subscribe_method.Invoke(service, new object[] { null });
+            Type update_type = observable.GetType().GetGenericArguments().First(arg => arg.Name.StartsWith("Update"));
+            Type observer_type = typeof(SubscriptionLogger<>).MakeGenericType(update_type);
+            object observer = Activator.CreateInstance(observer_type);
+
+            var subscribe = observable.GetType().GetMethod("Subscribe", new[] { observer_type });
+            var dispose = subscribe.Invoke(observable, new[] { observer });
+
+            while (true) {
+               System.Threading.Thread.Sleep(1000);
+            }
+         }
+      }
+
+   }
+}

src/client/sparkcmd/Cmd/ServiceTestCommand.cs

@@ -1,8 +1,10 @@
 using bts.api;
 using bts.api.core;
 using bts.api.synthetic;
+using bts.test;
 using ManyConsole;
 using System;
+using System.Linq;
 
 namespace bts {
    public class ServiceTest : ConsoleCommand {
@@ -28,6 +30,8 @@ namespace bts {
          }
          return 0;
       }
+
+
       private static void synthetic_test() {
          using (var ctx = new ServiceContext()) {
             var security = new Security {

src/data_files/files/cert_secure_logon.txt

@@ -1,12 +1,6 @@
 Session ID           Access ID                      Secret Key                                         Creation Date        Expiration Date      Environment          
-1U7                  i9pLY5mVhWLXs7FWNb9X           8esbwiaHaqmR2rXZMkJRHt00fW6l_-ppCGoheWWuBZQ        2018-04-05           2019-04-05           CERT                 
-1U7                  ozXmORijPtJriTg6Wwny           A1KOwrhH8GojNotjkCfVwHRSi_OS280xihaRZwJLFKo        2018-04-03           2018-05-03           CERT                 
-ANS                  6ogfYoF54g695tqaHsFC           CtDkFnuDuStB1S05nC_pWvANb6FWsV2IMNKkmM3EcP4        2018-04-05           2019-04-05           NEWRELEASE           
-KTK                  IrUp518ekr8SzbfJGdPZ           QmilC-cKgqlIiIBs_BV59fowbPfuBWn0QjeEc1dYj38        2018-04-05           2019-04-05           NEWRELEASE           
-M5Y                  bUFtyvZgPhZFCYa434ap           rfZ2XLLwtHt26szWwMxYRw9uPb7jE4evEWaKuKf2uyU        2018-04-05           2019-04-05           CERT                 
-MBX                  tbna9JLxT4Jlrzj0wtSS           EcYqrLmR4ysSIT_NCBeC2AjDo7IAHsVZ0rOZ0cpAh_o        2018-04-05           2019-04-05           CERT                 
 MC1                  0N10hKkE2B2KUt86ewd8           dJ5I52JNA-6aZZ8hnO0byVKcyqA8Of6kkstfIF89AAM        2018-04-05           2019-04-05           CERT                 
 MC5                  e4uBmhQ1ebaXAeWHa7HK           PNOM86eofsiCwzzKUsB5yAmrNoyZypnYD72pT2A8zgc        2018-04-05           2019-04-05           CERT                 
 MC7                  S6KfCnay5UJ67P55ibHd           5lSs1TbX7N0p6Duu63_JDMghPUqBAWacvkM91MUv1uM        2018-04-05           2019-04-05           CERT                 
 MC8                  AbEEtBn63WIAuItPDz1c           n3ndjYb03Hx6wBO3me35HoONyDgO5TZKWk0tDSaX9KA        2018-04-05           2019-04-05           CERT                 
-T39                  UWggBgHWAe07aqW3ps2I           tbb3it0rUQSSTa0TC9QcRwkrHEMDLWhaJH4VQ0FqGB4        2018-04-05           2019-04-05           NEWRELEASE           
+V06                  WOC3GVA6l8ffrcOKUT9P           Az1TJc2XHJyZ4C0S1bG_p5eWPQlVYUYhfjmdPqSGJ8o        2018-11-26           2019-11-26           NEWRELEASE           

src/data_files/files/perf_trace/main.js

@@ -37,13 +37,17 @@ $( document ).ready(function (){
           $("#output").html(summary_info);
 
           let whole_result = result.requotes.concat(result.hedges);
-          let unique_sessions = whole_result.filter(function(value, index, self){
-            return self.find(function(elm){
-                return elm.session_name === value.session_name;
-            });
+          let unique_sessions = [];
+          whole_result.filter(function(value) {
+            if (unique_sessions.find(function(elm) {
+              return elm.session_name === value.session_name;
+            }) === undefined) {
+              unique_sessions.push(value);
+            }
           });
 
           $(unique_sessions).each(function (index, sess_val){
+            console.log(sess_val.session_name);
             let r = result.requotes.find(function(elm){
                 return elm.session_name === sess_val.session_name;
             });

src/lib/CMakeLists.txt

@@ -54,11 +54,27 @@ if(NOT old_config_proto_str STREQUAL config_proto_str)
   file(WRITE "${config_proto_file}" "${config_proto_str}")
 endif()
 
-# copy defaults to output directory
+# generate static defaults
+set(config_defaults_file ${CMAKE_CURRENT_BINARY_DIR}/config_defaults.h)
+# Defaults cannot be a single string because the compiler complains about a string longer than 65k
+set(config_defaults_str "const char * config_static_defaults[] = { ")
 foreach (cfg_file_path ${Conf_Defaults})
-  get_filename_component(cfg_file_name ${cfg_file_path} NAME)
-  configure_file(${cfg_file_path} ${CMAKE_BINARY_DIR}/src/server/sparkd/defaults/${cfg_file_name})
+  file(READ ${cfg_file_path} cfg_file_content)
+  string(REPLACE "\"" "\\\"" cfg_file_content_escaped ${cfg_file_content})
+  string(REPLACE "\n" ";" cfg_lines ${cfg_file_content_escaped})
+  foreach(line ${cfg_lines})
+    string(APPEND config_defaults_str "\n  \"${line}\",")
+  endforeach(line)
 endforeach(cfg_file_path)
+string(APPEND config_defaults_str "};")
+set(old_defaults_str "")
+if(EXISTS ${config_defaults_file})
+  file(READ ${config_defaults_file} old_defaults_str)
+endif()
+if(NOT config_defaults_str STREQUAL old_defaults_str)
+  file(WRITE "${config_defaults_file}" "${config_defaults_str}")
+endif()
+
 
 # lib 
 if (WIN32)
@@ -125,4 +141,4 @@ if (WIN32)
   include(${CMAKE_SOURCE_DIR}/tools/vs_filters.cmake)
 endif()
 
- 
+ 

src/lib/api/apigateway.proto

@@ -10,6 +10,7 @@ message GatewayLoginResult {
     bool is_valid = 1;
     string reason = 2;
     string token = 3;
+    bool can_create = 4;
 }
 message GatewayCallHeader {
    int32 request_id = 1;

src/lib/api/service_compiler/templates/excel/excel.pb.cs

@@ -25,14 +25,9 @@ namespace {{package}}
 // {{comment}}
 public partial class {{name}}_Excel : IDisposable 
 {
-   readonly static Lazy<{{name}}> _service = new Lazy<{{name}}>(() => locate<{{name}}>(), true);    
-   static T locate<T>() where T : class, IApiClient
-   {
-      using (var ctx = new api.core.ServiceContext())
-      {
-         return ctx.locate<T>();
-      }
-   }
+   readonly static object _lock = new object();
+   readonly static Lazy<{{name}}> _service = new Lazy<{{name}}>(() => bts.api.Locator.locate<{{name}}>(), true);    
+
 
 public void Dispose()
 {
@@ -80,6 +75,54 @@ public void Dispose()
    
       {{#unless skip_excel_generation}}
    
+   [ExcelFunction(Category = "BTS", Description = "Call BTS Spark {{../lname}} API Method {{lname}} with parameter {{input_type_full}}")]
+   async public static void spark_{{../lname}}_{{lname}}_async (object[,] parameters, ExcelAsyncHandle handle, bool show_headers = false, bool display_vertically = false)
+   {
+      try
+      {  
+         Google.Protobuf.IMessage req = null;
+         {{input_type_full}} casted_req = null;
+         try
+         {
+            if(parameters.Rank == 2 && parameters.GetLength(0) == 1 && parameters.GetLength(1) == 1)
+            {
+               if(parameters[0,0] == ExcelMissing.Value)
+                  req = (Google.Protobuf.IMessage) Activator.CreateInstance(typeof({{input_type_full}}));                        
+               else if(parameters[0,0] is string json)
+                  req = APIDispatch.generate_imessage(typeof({{input_type_full}}), json);               
+            }         
+            if(req == null)
+            {
+               string json = JsonFormatter.parse_json(typeof({{input_type_full}}), parameters);
+               req = APIDispatch.generate_imessage(typeof({{input_type_full}}), json);
+            }
+            casted_req = ({{input_type_full}}) req;
+         }
+         catch 
+         {
+            var members = bts.excel.extensions.ReflectionExtensions
+               .get_fields_or_properties(typeof({{input_type_full}}))
+               .Where(x => !DataParser.excluded_member(x));            
+            handle.SetResult(
+               "Error in calling method {{lname}}( {{input_type_full}} ), input members are: " + 
+                  string.Join(",", members.Select(x => x.Name).ToArray()) 
+            );            
+         }
+         
+         var res = await _service.Value.{{name}}_async( casted_req );
+         object res_res = res.value;
+         object[,] scattered = new JsonFormatter(res_res).format(show_headers); 
+         if(display_vertically)
+            scattered = bts.excel.Utilities.transpose_keep_dimension(scattered);
+         handle.SetResult(scattered);
+      }
+      catch(Exception ex)
+      {
+         Trace.TraceError(ex.ToString());
+         handle.SetException(ex);
+      }
+   }
+   
    [ExcelFunction(Category = "BTS", Description = "Call BTS Spark {{../lname}} API Method {{lname}} with parameter {{input_type_full}}")]
    public static object spark_{{../lname}}_{{lname}} (object[,] parameters, bool show_headers = false, bool display_vertically = false)
    {
@@ -113,7 +156,7 @@ public void Dispose()
          .Where(x => !DataParser.excluded_member(x));            
       return "Error in calling method {{lname}}( {{input_type_full}} ), input members are: " + 
          string.Join(",", members.Select(x => x.Name).ToArray());
-   }
+   }   
       {{/unless}}
    
    {{/if}}

src/lib/api/service_helpers/AdminServiceUtils.cpp

@@ -846,12 +846,35 @@ void AdminServiceUtils::find_user_trading_accounts (const bts::api::admin::Admin
    const bts::api::admin::UserAccountPermissions* user_perm_ptr = get_user_permissions (admin_data, user_id);
    if (user_perm_ptr != nullptr)
       for(auto it=user_perm_ptr->account_permissions().begin(); it!=user_perm_ptr->account_permissions().end(); it++)
-         if (it->second.has_trading_permission()){
+         if (it->second.has_trading_permission() && it->second.trading_permission().has_permissions()){
              const api::admin::BTSAccount* account_ptr = find_account(admin_data, it->first);
              if (account_ptr != nullptr)
                 associated_accounts.push_back(account_ptr);
          }
 }
+
+// for a given account, fills in the associated_users ref arg
+// with the users that can trade into the account
+void AdminServiceUtils::find_users_for_trading_account (const AdminData& admin_data,
+                                                        uint32_t account_id,
+                                                        std::vector<const BTSUser*>& associated_users)
+{
+   for(const auto& u : AdminServiceUtils::users(admin_data, false))
+   {
+      const auto user_perm_ptr = get_user_permissions (admin_data, u->id());
+      if (user_perm_ptr)
+      {
+         const auto& permission_unit = user_perm_ptr->account_permissions().find(account_id);
+         if(permission_unit != user_perm_ptr->account_permissions().end() && permission_unit->second.has_trading_permission() &&
+            permission_unit->second.trading_permission().has_permissions())
+         {
+            associated_users.push_back(u);
+         }
+      }            
+   }
+}
+   
+
 void AdminServiceUtils::validate_user_update (const bts::api::admin::AdminData& admin_data,
                                               const bts::api::admin::BTSUser& updated_user,
                                               bts::api::admin::AdminServiceOperationResult& result){
@@ -891,7 +914,7 @@ void AdminServiceUtils::get_md_entitlement_report(const AdminData& admin_data,
                                                                   admin_data);
       if(cme_data!=nullptr){
          for (const auto& cme_mic_data : cme_data->md_permissions()){
-            MDEntitlementReportUnit rv=get_cme_md_entitlement_unit(user.name(),
+            MDEntitlementReportUnit rv=get_cme_md_entitlement_unit(user,
                                                                  cme_mic_data);
             entitlements.push_back(rv);
          }
@@ -899,10 +922,11 @@ void AdminServiceUtils::get_md_entitlement_report(const AdminData& admin_data,
    }   
 }
 MDEntitlementReportUnit AdminServiceUtils::get_cme_md_entitlement_unit(
-                                             const std::string& user,
+                                             const BTSUser& user,
                                              const CMEMicData& cme_mic_data){
    MDEntitlementReportUnit rv;
-   rv.set_user(user);
+   rv.set_user(user.name());
+   rv.set_user_id(user.id());
    rv.set_exchange("GLBX");
    switch(cme_mic_data.mic())
    {
@@ -974,6 +998,8 @@ bts::api::admin::BTSUser* AdminServiceUtils::create_user (bts::api::admin::Admin
                                                           uint32_t parent_group_id,
                                                           const std::string& name,
                                                           const std::string& password,
+                                                          const std::string& full_name,
+                                                          const std::string& company,
                                                           bool is_active,
                                                           const bts::api::admin::LicenseData& license_data,
                                                           const bts::api::admin::LicenseEntitlements& license_entitlements,
@@ -999,6 +1025,8 @@ bts::api::admin::BTSUser* AdminServiceUtils::create_user (bts::api::admin::Admin
    new_user_ptr->set_parent_group_id(parent_group_id);
    new_user_ptr->set_name(name);
    new_user_ptr->set_password (password);
+   new_user_ptr->set_full_name (full_name);
+   new_user_ptr->set_company (company);
    new_user_ptr->set_is_active(is_active);
    new_user_ptr->mutable_license_data()->CopyFrom(license_data);
    new_user_ptr->mutable_license_entitlements()->CopyFrom(license_entitlements);
@@ -1268,10 +1296,10 @@ void AdminServiceUtils::bootstrap_data (bts::api::admin::AdminData& data){
    auto def_group = create_group(data, 0, std::string("default"), result);
    // adding users to the group
    bts::api::admin::LicenseData license_data;
-   bts::api::admin::LicenseEntitlements license_entitlements;
-   auto user_1 = create_user(data, def_group->id(), std::string("bts"), std::string(""), false, license_data, license_entitlements, user_result); user_1->set_is_active(true);
-   auto user_2 = create_user(data, def_group->id(), std::string("bts_2"), std::string("bts_2"), false, license_data, license_entitlements, user_result); user_2->set_is_active(true);
-   auto user_3 = create_user(data, def_group->id(), std::string("bts_3"), std::string("bts_3"), false, license_data, license_entitlements, user_result); user_3->set_is_active(true);
+   bts::api::admin::LicenseEntitlements license_entitlements; std::string empty("");
+   auto user_1 = create_user(data, def_group->id(), std::string("bts"), std::string(""), empty, empty, false, license_data, license_entitlements, user_result); user_1->set_is_active(true);
+   auto user_2 = create_user(data, def_group->id(), std::string("bts_2"), std::string("bts_2"), empty, empty,  false, license_data, license_entitlements, user_result); user_2->set_is_active(true);
+   auto user_3 = create_user(data, def_group->id(), std::string("bts_3"), std::string("bts_3"), empty, empty, false, license_data, license_entitlements, user_result); user_3->set_is_active(true);
    //associate users
    bts::api::admin::UserAccountPermissionUnit permission_unit;
    permission_unit.mutable_trading_permission()->set_submit_market_orders(true);

src/lib/api/service_helpers/AdminServiceUtils.h

@@ -217,6 +217,12 @@ public:
                         const AdminData& admin_data,
                         uint32_t user_id,
                         std::vector<const BTSAccount*>& associated_accounts);
+   // for a given account, fills in the associated_users ref arg
+   // with the users that can trade into the account
+   static void find_users_for_trading_account (
+                        const AdminData& admin_data,
+                        uint32_t account_id,
+                        std::vector<const BTSUser*>& associated_users);
    // validates that changes in user data satisfy all the 
    // constraints
    static void validate_user_update (const AdminData& admin_data,
@@ -230,7 +236,7 @@ public:
                            std::vector<MDEntitlementReportUnit>& entitlements);
    // Gets a reporting unit out of CMEMicData
    static MDEntitlementReportUnit get_cme_md_entitlement_unit(
-                                             const std::string& user,
+                                             const BTSUser& user,
                                              const CMEMicData& cme_mic_data);
    static void get_user_symbol_permissions(
                                  const AdminData& data,
@@ -243,6 +249,8 @@ public:
                                 uint32_t parent_group_id,
                                 const std::string& name,
                                 const std::string& password,
+                                const std::string& full_name,
+                                const std::string& company,                                
                                 bool is_active,
                                 const LicenseData& license_data,
                                 const LicenseEntitlements& license_entitlements,

src/lib/api/services/AdminService.proto

@@ -54,9 +54,12 @@ message BTSUser{
    uint32 parent_group_id = 20;
    string name = 30;
    string password = 40;
+   string full_name = 42;
+   string company = 44;   
    bool is_active = 50;
-   LicenseData license_data = 60;
-   LicenseEntitlements license_entitlements = 70;
+   bool can_create = 55;
+   LicenseData license_data = 60; //unused
+   LicenseEntitlements license_entitlements = 70; //unused
    BTSUserDerivableData user_data = 80;
 }
 
@@ -89,21 +92,23 @@ message LicenseEntitlements{
 }
 
 message AlgoTradingPermissionsUnit{
-   bool submit_iceberg_orders = 10;
-   bool submit_time_box_orders = 20;
-   bool submit_spreader_orders = 30;
+   bool submit_iceberg_orders = 10; //unused
+   bool submit_time_box_orders = 20; //unused
+   bool submit_spreader_orders = 30; //unused
 }
 
 message TradingPermissionUnit{
-   bool submit_market_orders = 10;
-   bool submit_gtc_orders = 20;   
-   AlgoTradingPermissionsUnit algo_trading_permission = 30;
+   bool has_permissions = 5;
+   bool submit_market_orders = 10; //unused
+   bool submit_gtc_orders = 20; //unused  
+   AlgoTradingPermissionsUnit algo_trading_permission = 30; //unused
 }
 
 message AdminPermissionUnit{
    // TODO: needs to be more ellaborate (account writing, 
    // reading etc, routing etc)
    bool has_permissions = 10;
+   bool has_md_permissions = 20;
 }
 
 // this is the functionality permissions
@@ -113,6 +118,7 @@ message UserAccountPermissionUnit{
 }
 
 message UserAccountPermissions{
+   //uint32 = account_id
    map<uint32, UserAccountPermissionUnit> account_permissions = 10;   
 }
 
@@ -141,6 +147,12 @@ message BTSAccount{
    AccountSecurityClipLimits security_clip_limits = 90;
    AccountExchangeConnections connections = 100;
    ExchangeAccountData exchange_account_data = 110;
+   // extra parameters for risk enforcement
+   RiskLimitEnforcement risk_limit_enforcement=120;
+}
+
+message RiskLimitEnforcement{   
+   bool product_limit_in_net_terms = 10;
 }
 
 message AccountProductPositionLimitUnit{
@@ -277,6 +289,18 @@ message SIMAccountData {
    string clearing_account = 40;
 }
 
+enum ProfessionalDesignation
+{
+   PROFESSIONALDESIGNATION_PRO = 0;
+   PROFESSIONALDESIGNATION_NONPRO = 1;
+}
+
+enum DisplayDesignation
+{
+   DISPLAYDESIGNATION_DISPLAY = 0;
+   DISPLAYDESIGNATION_NONDISPLAY = 1;
+}
+
 message CMEUserData{
    string operator_id = 10;
    string ats_operator_id = 20;
@@ -287,7 +311,11 @@ message CMEUserData{
    string smp_instruction = 70;
    string max_order_rate_per_connection = 80;
    string location = 90;
-   // add suff here
+   ProfessionalDesignation pro_nonpro = 100;
+   DisplayDesignation display_nondisplay = 110;
+   
+   enum AdminDisplay { All = 0; MD = 1; Routing = 2; None = 100; } 
+   AdminDisplay admin_display_flag = 120; 
    repeated CMEMicData md_permissions = 1000;
 }
 
@@ -371,6 +399,7 @@ message OrderCancelRequest{
 message MDEntitlementReportUnit{
    string record_id = 1 [(bts.api.key)=true];
    string user = 10;
+   uint32 user_id=11;
    string exchange = 20;
    string mic = 30;
    bool enabled = 40;
@@ -394,6 +423,7 @@ message AccountCreateAPIArgument{
    uint32 parent_account_id = 10;
    string account_name = 20;
    string description = 30;
+   string creator_user = 40;
 }
 message AccountUpdateAPIArgument{
    BTSAccount complete_data=10;    
@@ -402,6 +432,8 @@ message UserCreateAPIArgument{
    uint32 parent_group_id = 20;
    string name = 30;
    string password = 40;
+   string full_name = 42;
+   string company = 44;
    LicenseData license_data = 50;
    LicenseEntitlements license_entitlements = 70;
 }
@@ -427,6 +459,10 @@ message AdminServiceCreateAccountResult {
    uint32 new_id = 30;
 }
 
+message FilteredAdminDataRequest {
+   string token = 1;
+}
+
 service AdminService {
    option (bts.api.service_description) = "Provides a management interface to the user model";
    // account
@@ -440,6 +476,7 @@ service AdminService {
    // subscription
    rpc admin_data(bts.api.VoidMessage) returns (stream AdminData);
    rpc get_admin_data(bts.api.VoidMessage) returns (AdminData);
+   rpc get_filtered_admin_data(FilteredAdminDataRequest) returns (AdminData);
    // orders
    rpc get_order_summary(bts.api.VoidMessage) returns (OrdersSummary);
    rpc cancel_orders(OrderCancelRequest) returns (bts.api.VoidMessage);

src/lib/api/services/UserService.proto

@@ -47,6 +47,7 @@ message AuthenticateLoginResponse {
     string reason = 2;
     AuthToken token = 3;
     uint32 user_id = 4;
+    bool can_create = 5;
 }
 message AuthenticateTokenRequest {AuthToken token = 1;}
 message AuthenticateTokenResponse {bool is_valid = 1; string reason = 2; User user = 3;}

src/lib/config/ConfigurationState.cpp

@@ -38,6 +38,7 @@
 #endif
 
 #include "lib/config_proto_files.h"
+#include "lib/config_defaults.h"
 
 using namespace std;
 
@@ -77,12 +78,6 @@ ConfigurationState::~ConfigurationState(void)
 {
 }
 
-std::string ConfigurationState::defaults_path()
-{
-   std::string appdir = os::get_install_path();
-   return appdir + "/defaults/";
-}
-
 std::string get_command_argument(const char* key)
 {
    std::vector<std::string> cmd_line = os::get_command_line();
@@ -127,18 +122,18 @@ void ConfigurationState::initialize_defaults(IConfigLog& log)
    
    std::shared_ptr<proto::Message> new_message(_prototype->New());
    ErrorsOnlyLog elog(log);
-   load_message(defaults_path(), *new_message.get(), elog, *_deferred_queue.get());
+
+   std::string defaults;
+   for (const char* str : config_static_defaults) {
+      defaults += str;
+      defaults += "\n";
+   }
+
+   proto::io::ArrayInputStream input_stream(defaults.c_str(), (int)defaults.length());
+   parse_stream("static_defaults.conf", input_stream, *new_message.get(), elog, *_deferred_queue.get());
    _defaults = new_message;
 
-   // If defaults has an include process that now
-   // FEATURE: Unlike settings loaded from ~/sparkdata/ these includes
-   // take precedence over the settings in the file which includes them.
-   // While this seems backwards it is always the behavior you want
-   std::string include = get_include(*_defaults.get());
-   if (!include.empty())
-   {
-      load(include, "", log);
-   }
+   // do not process 'include' for static defaults
 
    _is_initialized = true;
 }

src/lib/config/ConfigurationState.h

@@ -81,7 +81,6 @@ namespace internal {
       void generate_all_config_proto(IConfigLog&);
       const proto::Message* try_get_value(const std::string& field_name, const proto::Message& message) const;
 
-      std::string defaults_path();
       std::string settings_path();
 
       bool _is_initialized;

src/lib/config/defaults/Executor.conf

@@ -1,13 +0,0 @@
-ExecutorCfg
-{
-   value
-   {
-   # executor for latency sensitive code
-      executor { affinity: 0 yield_cpu: false }
-
-   # normal executor
-      executor { affinity: -1 yield_cpu: true }
-   }
-}
-
-

src/lib/domain_model/Security.h

@@ -80,6 +80,10 @@ inline Price round_to_quote_price(api::Security const&sec, Price in, api::Side s
    else 
       return Price(raw_round_to_quote_price(p,sec.contract_spec().low_tick(),side));
 }
+
+// Looking to round md::IntPrice? Look in IntPrice.h
+
+
 // computes the difference between 2 prices in ticks.
 // the result is an a signed int so the order of args
 // matter, its second price - first price

src/lib/domain_model/Security.proto

@@ -53,7 +53,11 @@ enum CMEMics {
 }
 
 enum ICEMics{
-   ICE_FUTURES_CANADA=0;
+   ICE_UNSET=0;
+   ICE_US = 1;
+   ICE_EU = 2;
+   ICE_LIFFE = 3;
+   ICE_CA = 4;
 }
 
 enum Side {

src/lib/logging/Logger.h

@@ -140,7 +140,7 @@ public:
 
    ~Logger()
    {
-#ifndef WIN32
+#ifdef false //This breaks .net core and windows
       assert(!_active);
       assert(_wcursor == 0);
 #endif

src/lib/logging/logging.cpp

@@ -193,7 +193,7 @@ LoggingConfigPoller::~LoggingConfigPoller ()
    _poller.stop();
 }
 
-#ifdef _WIN32
+
 namespace interop {
 
 const char* intern(const char* str) {
@@ -205,7 +205,11 @@ const char* intern(const char* str) {
       return *iter;
    }
    else {
+#ifdef _WIN32      
       const char* v = _strdup(str);
+#else      
+      const char* v = strdup(str);
+#endif
       set.insert(v);
       return v;
    }
@@ -218,7 +222,6 @@ extern "C" BTS_EXPORT void Logger_log (int level, int subsystem, const char *mes
    }
 }
 }
-#endif
 
 
 }}

src/lib/logging/logging.h

@@ -42,7 +42,8 @@ constexpr log::Subsystem Mercury    {ExSim.ordinal+1       };
 constexpr log::Subsystem Testing    {Mercury.ordinal+1     };
 constexpr log::Subsystem Latency    {Testing.ordinal+1     };
 constexpr log::Subsystem DataFlow   {Latency.ordinal+1     };
-constexpr log::Subsystem Plugin     {DataFlow.ordinal+1     };
+constexpr log::Subsystem Plugin     {DataFlow.ordinal+1    };
+constexpr log::Subsystem ClientSdk  {Plugin.ordinal+1      };
 
 struct LoggerHandle {
    LoggerHandle(LoggerHandle const&) = delete;

src/lib/md/Diag/ShmDumper.cpp

@@ -116,6 +116,7 @@ string dump_security(const FixedSecurity& security) {
             auto& leg = psec->legs[x];
             if (x > 0) res << ",";
             res << "{ 'key': " << leg.key
+                << ", 'exkey': " << leg.exchange_key
                 << ", 'quantity': " << leg.leg_quantity
                 << ", 'leg_price': " << leg.leg_price
                 << ", 'delta': " << leg.leg_delta

src/lib/md/IntPrice.h

@@ -1,10 +1,13 @@
 #pragma once
 #include "lib/utils/bts_assert.h"
 #include "lib/utils/bts_array_size.h"
+#include "lib/domain_model/Security.pb.h"
 
 namespace bts {
 namespace md {
 
+const extern double NO_PRICE;
+
 static const double powers_10[] = {
   1E-20, 1E-19, 1E-18, 1E-17, 1E-16, 1E-15, 1E-14, 1E-13, 1E-12, 1E-11, 1E-10, 1E-9, 1E-8, 1E-7, 1E-6, 1E-5, 1E-4, 1E-3, 1E-2, 1E-1,
   1E0,
@@ -35,14 +38,12 @@ class IntPrice {
    IntPrice(const IntPrice& other) : _mantissa(other._mantissa), _exponent(other._exponent) {}
 
    bool operator==(const IntPrice& other) const {
-      // this requires both mantissa and exponent to be equal
-      // logically  100*pow10(1) == 10*pow10(2) 
-      // HOWEVER, once converted to double there are subtelties that can make them not equal so
-      // this function is conservative
-      if(_exponent == other._exponent)
+      if(_exponent == other._exponent) {
          return _mantissa == other._mantissa;
-      else
+      }
+      else {
          return price()==other.price();
+      }
    }
    bool operator!=(const IntPrice& other) const {
       return !(*this == other);
@@ -65,16 +66,40 @@ class IntPrice {
          bts_assert_d(o.exponent() <= 0 && _exponent <= 0);
          if (o.exponent() < _exponent) {
             int8_t delta = _exponent - o.exponent();
-            auto m = _mantissa * pow10(delta);
-            return IntPrice(m - o.mantissa(), o.exponent());
+            int64_t m = _mantissa * pow10(delta);
+            return { m - o.mantissa(), o.exponent() };
          }
          else {
             int8_t delta = o.exponent() - _exponent;
-            auto m = o.mantissa() * pow10(delta);
-            return IntPrice(_mantissa - m, _exponent);
+            int64_t m = o.mantissa() * pow10(delta);
+            return { _mantissa - m, exponent() };
          }
       }
    }
+   IntPrice operator + (const IntPrice& o) const {
+      if (_exponent==o._exponent) {
+         return IntPrice { _mantissa+o.mantissa(), exponent() };
+      }
+      else {
+         if (is_nan() || o.is_nan()) return IntPrice();
+         bts_assert_d(o.exponent() <= 0 && _exponent <= 0);
+         if (o.exponent() < _exponent) {
+            int8_t delta = _exponent - o.exponent();
+            int64_t m = _mantissa * pow10(delta);
+            return { m + o.mantissa(), o.exponent() };
+         }
+         else {
+            int8_t delta = o.exponent() - _exponent;
+            int64_t m = o.mantissa() * pow10(delta);
+            return { _mantissa + m, exponent() };
+         }
+      }
+   }
+   IntPrice sign_extend(int8_t new_exponent) const {
+      int8_t delta = _exponent - new_exponent;
+      int64_t m = _mantissa * pow10(delta);
+      return { m, new_exponent };
+   }
    int64_t mantissa() const { return _mantissa; }
    int8_t  exponent() const { return _exponent; }
    double price() const { 
@@ -85,6 +110,7 @@ class IntPrice {
 };
 static_assert(sizeof(IntPrice) == 8, "IntPrice should be this big |--------|");
 extern IntPrice INT_NAN;
+extern IntPrice INT_ZERO;
 
 template <uint8_t PRECISION>
 IntPrice IntPrice_from_double(double price) {
@@ -98,6 +124,52 @@ IntPrice IntPrice_from_double(double price) {
    }
    return {(int64_t)price, 0};
 }
+// inline IntPrice IntPrice_from_double(double price, int8_t precision) {
+//    if (os::is_nan(price)) return INT_NAN;
+//    return { (int64_t)(price * pow10(-precision)), precision };
+// }
+
+inline md::IntPrice round_to_quote_price(const IntPrice& tick, const IntPrice& price, api::Side side) {
+   bts_assert_d(tick.mantissa() != 0);
+   if (price.is_nan()) return price;
+   bts_assert_d(price.exponent() <= 0 && tick.exponent() <= 0);
+   if (price.exponent() == tick.exponent()) {
+      auto mod = price.mantissa() % tick.mantissa();
+      if (mod == 0) return price;
+      else if (side == api::SIDE_BUY) {
+         // buy, round down
+         if (price.mantissa() > 0) {
+            return { price.mantissa() - mod, price.exponent() };
+         }
+         else {
+            return { price.mantissa() - mod - tick.mantissa(), price.exponent() };
+         }
+      }
+      else if (side == api::SIDE_SELL) {
+         // sell, round up
+         if (price.mantissa() > 0) {
+            return { price.mantissa() - mod + tick.mantissa(), price.exponent() };
+         }
+         else {
+            return { price.mantissa() - mod, price.exponent() };
+         }
+      }
+   }
+   else {
+      // Uncommon path
+      // Math is reversed because exponents are negative
+      if (tick.exponent() < price.exponent()) {
+         return round_to_quote_price(tick, price.sign_extend(tick.exponent()), side);
+      }
+      else {
+         // Extend tick to price's exponent
+         return round_to_quote_price(tick.sign_extend(price.exponent()), price, side);
+      }
+   }
+
+   return price;
+}
+
 
 inline std::ostream& operator<<(std::ostream& os, const IntPrice& p) {
    os << p.price();

src/lib/md/MDConsumerHandle.cpp

@@ -85,7 +85,7 @@ TimeStamp copy_depth(
 /////////////////////////////
 struct Level {
    double price;
-   uint16_t size;
+   double size;
 };
 struct DepthTraversal {
    std::array<Level, COCONUT_MAX_DEPTH> levels;
@@ -100,42 +100,92 @@ struct DepthTraversal {
          levels[x].size = depth[x].size();
       }
    }
-   double try_take1() {
-      if (head >= levels.size()) return os::nan();
-      if (levels[head].size >= qty) {
-         levels[head].size -= qty;
-         return levels[head].price * price_mult;
-      }
-      else {
-         double price = 0;
-         size_t count = 0;
-         while(count < qty) {
-            if (levels[head].size >= 1) {
-               price += levels[head].price;
-               levels[head].size -= 1;
-               count += 1;
-            }
-            else {
-               head++;
-               if (head >= levels.size()) return os::nan();
-            }
-         }
-         return (price / qty) * price_mult;
+   #ifdef PRECISE_SYNTHETICS
+   double available_size() const {
+      double available = 0.0;
+      auto cur = head;
+      bool partial = false;
+      while(cur < levels.size() && available < 1) {
+         available += levels[cur].size / qty;
+         if (available < 1.0) partial = true;
+         ++cur;
       }
+      if (partial && available >= 1.0) return 1.0;
+      return available;
    }
+   double consume(double size) {
+      bts_assert_d(size <= available_size());
+      double take = size * qty;
+      double price = 0.0;
+      while(take > 0) {
+         if (levels[head].size >= take) {
+            price += levels[head].price * price_mult * (take/qty);
+            levels[head].size -= take;
+            if (levels[head].size <= 0) ++head;
+            break;
+         }
+         else {
+            double available = levels[head].size;
+            price += levels[head].price * price_mult * (available/qty);
+            levels[head].size = 0;
+            ++head;
+            take -= available;
+         }
+      }
+      return price/size;
+   }
+   #else
+   double available_size() const {
+      double available = 0.0;
+      auto cur = head;
+      while(cur < levels.size() && available < 1) {
+         available += levels[cur].size / qty;
+         ++cur;
+      }
+      return available;
+   }
+   double consume(double size) {
+      bts_assert_d(size <= available_size());
+      double take = size * qty;
+      double price = 0.0;
+      while(take > 0) {
+         if (levels[head].size >= take) {
+            price = levels[head].price * price_mult;
+            levels[head].size -= take;
+            if (levels[head].size <= 0) ++head;
+            break;
+         }
+         else {
+            double available = levels[head].size;
+            price = levels[head].price * price_mult;
+            levels[head].size = 0;
+            ++head;
+            take -= available;
+         }
+      }
+      return price;
+   }
+   #endif
 };
 
-void copy_depth(size_t count, api::Side side, const api::ContractSpecification& spec, small_vector<DepthTraversal>& leg_travs, Level* out_price) {
+void copy_synthetic_side(size_t count, api::Side side, const api::ContractSpecification& spec, small_vector<DepthTraversal>& leg_travs, Level* out_price) {
    size_t out_index = 0;
    bool first = true;
    while(out_index < count) {
-      double price = 0.0;
-      for (auto& dt : leg_travs) {
-         double temp = dt.try_take1();
-         price += temp;
-      }
 
-      if (os::is_nan(price)) {
+      double price = 0.0;
+      // Find the leg with the minimum available size
+      double size = std::numeric_limits<double>().max();
+
+      for (const auto& dt : leg_travs) {
+         auto sz = dt.available_size();
+         // If we can't get 1.0 from any leg we're done
+         if (sz <= size) size = sz;
+         if (size < 1.0) break;
+      }
+      size = (int64_t)size;
+
+      if (size < 1.0) {
          // We've run out of book so we nan out any remaining levels and finish
          size_t start = first ? out_index : out_index+1;
          for (size_t x=start; x < count; ++x) {
@@ -145,6 +195,13 @@ void copy_depth(size_t count, api::Side side, const api::ContractSpecification&
          return;
       }
 
+      // Take that size from each leg
+      for (auto& dt : leg_travs) {
+         double temp = dt.consume(size);
+         price += temp;
+      }
+      bts_assert_d(!os::is_nan(price));
+
       // Round to tick size
       price = securities::checked_round_to_quote_price(spec, price, side);
 
@@ -154,19 +211,19 @@ void copy_depth(size_t count, api::Side side, const api::ContractSpecification&
       }
 
       if (price == out_price[out_index].price) {
-         out_price[out_index].size += 1;
+         out_price[out_index].size += size;
       }
       else {
          out_index += 1;
          if (out_index < count) {
             out_price[out_index].price = price;
-            out_price[out_index].size = 1;
+            out_price[out_index].size = size;
          }
       }
    }
 }
 
-TimeStamp copy_depth(const SyntheticStrategy& synth, size_t count, PriceSize* bid, PriceSize* ask) {
+TimeStamp copy_synthetic_depth(const SyntheticStrategy& synth, size_t count, PriceSize* bid, PriceSize* ask) {
    small_vector<DepthTraversal> bid_leg_travs;
    small_vector<DepthTraversal> ask_leg_travs;
    bid_leg_travs.reserve(synth.legs.size());
@@ -210,12 +267,12 @@ TimeStamp copy_depth(const SyntheticStrategy& synth, size_t count, PriceSize* bi
    }
    std::array<Level, COCONUT_MAX_DEPTH> scratch {};
    const auto& spec = synth.security.contract_spec();
-   copy_depth(count, api::SIDE_BUY, spec, bid_leg_travs, scratch.data());
+   copy_synthetic_side(count, api::SIDE_BUY, spec, bid_leg_travs, scratch.data());
    for (size_t x = 0; x < count; ++x) {
       bid[x] = PriceSize_from_double<7>(scratch[x].price, scratch[x].size);
    }
    memset(scratch.data(), 0, sizeof(scratch));
-   copy_depth(count, api::SIDE_SELL, spec, ask_leg_travs, scratch.data());
+   copy_synthetic_side(count, api::SIDE_SELL, spec, ask_leg_travs, scratch.data());
    for (size_t x = 0; x < count; ++x) {
       ask[x] = PriceSize_from_double<7>(scratch[x].price, scratch[x].size);
    }

src/lib/md/MDConsumerHandle.h

@@ -22,7 +22,7 @@ struct SyntheticStrategy {
 
 TimeStamp copy_bbo(const HotEntry& src, PriceSize& out_bid, PriceSize& out_ask);
 TimeStamp copy_depth(const HotDepthEntry&, size_t count, PriceSize* bid, PriceSize* ask);
-TimeStamp copy_depth(const SyntheticStrategy&, size_t count, PriceSize* bid, PriceSize* ask);
+TimeStamp copy_synthetic_depth(const SyntheticStrategy&, size_t count, PriceSize* bid, PriceSize* ask);
    
 enum class HandleReadResult {
     NO_CHANGE,
@@ -281,7 +281,7 @@ private:
          case HANDLETYPE_SYNTHETIC:
             auto t = last_market_update();
             if (t > time || t == TimeStamp::invalid()) {
-               TimeStamp temp = copy_depth(*synthetic.def, std::min(copydepth, COCONUT_MAX_DEPTH), bids, asks);
+               TimeStamp temp = copy_synthetic_depth(*synthetic.def, std::min(copydepth, COCONUT_MAX_DEPTH), bids, asks);
                if (temp == TimeStamp::invalid()) { return HandleReadResult::BUSY; }
                time = temp;
                return HandleReadResult::SUCCESS;

src/lib/md/MDProducerHandle.cpp

@@ -61,18 +61,6 @@ void MDProducerHandle::set_key(security_key key) {
    }
 }
 
-void MDProducerHandle::set_legs(const std::vector<security_key>& legs) {
-   bts_assert(cdata().security.type == api::SECURITYTYPE_SPREAD);
-   FixedSpreadSecurityL* fss = (FixedSpreadSecurityL*)&cdata().security;
-   bts_assert(fss->leg_count() == legs.size());
-   for (size_t x = 0; x < legs.size(); ++x) {
-      security_key key = legs[x];
-      fss->legs[x].key = key;
-      // attempt to validate that this FixedSpreadLeg is valid
-      bts_assert_d(fss->legs[x].leg_quantity > 0);
-   }
-}
-
 void MDProducerHandle::update_bbo(TimeStamp time, const PriceSize bid, const PriceSize ask, uint16_t seq) {
    if (!has_depth()) {
       auto oldtime = hdata().time();

src/lib/md/MDProducerHandle.h

@@ -32,7 +32,6 @@ public:
       bts_assert_d(key == cdata().security.key);
       return key;
    }
-   void set_legs(const std::vector<security_key>& legs);
 
    void update_bbo(TimeStamp time, const PriceSize bid_price, const PriceSize ask_price, uint16_t seq=0);
    void update_direct(TimeStamp time, const std::array<PriceSize,COCONUT_MAX_DEPTH>& bids, const std::array<PriceSize,COCONUT_MAX_DEPTH>& asks, uint32_t seq=0) {

src/lib/md/PriceSize.cpp

@@ -5,6 +5,7 @@ namespace bts {
 namespace md {
 
 IntPrice INT_NAN = IntPrice();
+IntPrice INT_ZERO = { 0, -7 };
 PriceSize NO_MARKET = PriceSize();
 
 int to_exponent(double value) {

src/lib/md/SecurityDB.cpp

@@ -34,16 +34,17 @@ sqlite3_ptr open_db() {
    }
    
    // Create table if it does not exist
-   auto dbptr = utils::open_db(db_path.c_str());
+   auto dbptr = sqlite::open_db(db_path.c_str());
+   bts_assert(dbptr);
    auto db = dbptr.get();
    sqlite3_busy_timeout(db, 9999);
-   sqlite3_exec_nofail(db, "PRAGMA journal_mode=OFF");
-   sqlite3_exec_nofail(db, "PRAGMA syncronous=0");
+   sqlite::exec_nofail(db, "PRAGMA journal_mode=WAL");
+   sqlite::exec_nofail(db, "PRAGMA syncronous=0");
 
-   sqlite3_exec_nofail(db, 
+   sqlite::exec_nofail(db, 
       "CREATE TABLE IF NOT EXISTS securities (key INTEGER primary key, symbol TEXT NOT NULL, venue INTEGER NOT NULL, type INTEGER NOT NULL, security_blob BLOB NOT NULL, expiration INTEGER NOT NULL);");
 
-   sqlite3_exec_nofail(db, 
+   sqlite::exec_nofail(db, 
       "CREATE TABLE IF NOT EXISTS metadata (key INTEGER PRIMARY KEY, data BLOB NOT NULL);");
 
    return dbptr;
@@ -67,7 +68,7 @@ size_t load_securities(
 
    // Load all existing securities and call the callback
    auto db = dbptr.get();
-   auto select_stmt = prepare_query(db, "select key,symbol,venue,type,security_blob from securities where expiration > ?");
+   auto select_stmt = sqlite::prepare_query(db, "select key,symbol,venue,type,security_blob from securities where expiration > ?");
    auto select = select_stmt.get();
    sqlite3_bind_int(select, 1, sec_expiration_cut_off().to_short());
    auto result = sqlite3_step(select);
@@ -112,8 +113,8 @@ size_t load_securities(
 
 void store_if_new(const std::vector<Security*>& securities) {
    auto dbptr = open_db();
-   auto insert = utils::prepare_query(dbptr, "insert or ignore into securities (key, symbol, venue, type, security_blob, expiration) values (?,?,?,?,?,?)");
-   sqlite3_exec_nofail(dbptr, "BEGIN TRANSACTION");
+   auto insert = sqlite::prepare_query(dbptr, "insert or ignore into securities (key, symbol, venue, type, security_blob, expiration) values (?,?,?,?,?,?)");
+   sqlite::exec_nofail(dbptr, "BEGIN TRANSACTION");
    std::string buffer;
    for (const api::Security* psecurity : securities) {
       if (psecurity && psecurity->SerializeToString(&buffer)) {
@@ -127,7 +128,7 @@ void store_if_new(const std::vector<Security*>& securities) {
          sqlite3_reset(insert.get());
       }
    }
-   sqlite3_exec_nofail(dbptr, "COMMIT TRANSACTION");
+   sqlite::exec_nofail(dbptr, "COMMIT TRANSACTION");
 }
 
 
@@ -135,7 +136,7 @@ std::vector<api::synthetic::Metadata> load_metadata() {
    std::vector<api::synthetic::Metadata> values;
    auto dbptr = open_db();
    auto db = dbptr.get();
-   auto select_stmt = prepare_query(db, "select key,data from metadata");
+   auto select_stmt = sqlite::prepare_query(db, "select key,data from metadata");
    auto select = select_stmt.get();
    auto result = sqlite3_step(select);
    while(result == SQLITE_ROW) {
@@ -156,7 +157,7 @@ std::vector<api::synthetic::Metadata> load_metadata() {
 }
 bool store_metadata(const api::synthetic::Metadata& data) {
    auto dbptr = open_db();
-   auto insert = utils::prepare_query(dbptr, "insert or replace into metadata (key, data) values (?,?)");
+   auto insert = sqlite::prepare_query(dbptr, "insert or replace into metadata (key, data) values (?,?)");
 
    SQLITE_CHK(sqlite3_bind_int(insert.get(), 1, data.security_key()), dbptr);
    std::string buffer = data.SerializeAsString();

src/lib/md/SecurityIDRecorder.cpp

@@ -7,7 +7,7 @@
 #include "lib/logging/logging.h"
 #include "lib/md/SecurityDB.h"
 #include <boost/filesystem.hpp>
-#include <tbb/queuing_mutex.h> // for reservation
+#include <tbb/concurrent_queue.h>
 
 #include "lib/config/protos/MarketData.pb.h"
 #include "lib/config/protos/system.pb.h"
@@ -22,53 +22,68 @@ using std::size_t;
 namespace bts {
 namespace md {
 
+uint64_t exchange_id(const api::Security& security);
+
 struct SecurityIDEntry {
    api::Security security;
    SecurityIDRecorder::Callback callback;
 };
 
 struct SecurityIDRecorderState {
-   SecurityIDRecorderState() :
+   SecurityIDRecorderState(SecurityIDRecorderParams& p) :
       work(),
       pending_handle(),
+      thread_id(),
       db(nullptr, sqlite3_close),
       timer(),
-      id((size_t)this)
+      lookup_key(p.lookup_key),
+      unresolved_legs()
    {}
+
    tbb::concurrent_queue<std::shared_ptr<SecurityIDEntry>> work;
    tbb::concurrent_queue<MDProducerHandle> pending_handle;
+   std::thread::id thread_id;
    sqlite3_ptr db;
    unique_ptr<DispatcherTimer> timer;
-   size_t id;
+
+   KeyLookupOption lookup_key;
+   std::map<ExchangeKey, std::vector<FixedSpreadLeg*>> unresolved_legs;
+
+   // Oh no, I got methods in my state
 
    void run(utils::Dispatcher& dispatcher) {
       timer = std::make_unique<utils::DispatcherTimer>(
-         dispatcher, 500,
+         dispatcher, 200,
          [=]() { do_work(); });
       timer->start();
    }
    void do_work() {
+      thread_id = std::this_thread::get_id();
+      work_spreads();
       work_handles();
       work_db();
    }
+   void work_spreads();
    void work_handles();
    void work_db();
+
+   void check_access() { bts_assert(thread_id == std::this_thread::get_id()); }
    void assign_key(const api::Security& security, const SecurityIDRecorder::Callback& key_assigned);
+   void add_unresolved_spread(MDProducerHandle& handle);
    security_key search(const std::string& descriptor);
    security_key insert(const std::string& descriptor, SecurityType);
 };
 
-SecurityIDRecorder::SecurityIDRecorder():
-   _self(new SecurityIDRecorderState())
+SecurityIDRecorder::SecurityIDRecorder(SecurityIDRecorderParams params):
+   _self(new SecurityIDRecorderState(params))
 {
+   if (params.db) {
+      _self->db = std::move(params.db);
+   }
    init();
-}
-
-SecurityIDRecorder::SecurityIDRecorder(utils::Dispatcher& dispatcher):
-   _self(new SecurityIDRecorderState())
-{
-   init();
-   _self->run(dispatcher);
+   if (params.dispatcher) {
+      _self->run(*params.dispatcher);
+   }
 }
 
 SecurityIDRecorder::~SecurityIDRecorder() { }
@@ -81,13 +96,16 @@ void SecurityIDRecorder::init() {
       boost::filesystem::create_directories(db_dir);
    }
 
-   _self->db = open_db(db_path.c_str());
+   if (!_self->db) {
+      _self->db = sqlite::open_db(db_path.c_str());
+      bts_assert(_self->db);
+   }
 
    // Set a long busy timeout
    sqlite3_busy_timeout(_self->db.get(), 9999);
-   sqlite3_exec_nofail(_self->db, "PRAGMA journal_mode=WAL");
-   sqlite3_exec_nofail(_self->db, "PRAGMA syncronous=0");
-   sqlite3_exec_nofail(_self->db, "CREATE TABLE IF NOT EXISTS security_keys (descriptor STRING NOT NULL PRIMARY KEY, type int);");
+   sqlite::exec_nofail(_self->db, "PRAGMA journal_mode=WAL");
+   sqlite::exec_nofail(_self->db, "PRAGMA syncronous=0");
+   sqlite::exec_nofail(_self->db, "CREATE TABLE IF NOT EXISTS security_keys (descriptor STRING NOT NULL PRIMARY KEY, type int);");
 }
 
 void SecurityIDRecorder::ensure_key(MDProducerHandle handle) {
@@ -95,6 +113,15 @@ void SecurityIDRecorder::ensure_key(MDProducerHandle handle) {
 }
 
 void SecurityIDRecorder::assign_key(const api::Security& security, const SecurityIDRecorder::Callback& callback) {
+   if (security.Contract_case() == Security::ContractCase::kSpread) {
+      for(const auto& leg : security.spread().legs()) {
+         if (leg.security().id() == invalid_security_key) {
+            LOG(md,Error, "Security {} has legs which will never be resolved.", security.has_glbx() ? security.glbx().security_desc() : security.symbol());
+            // No point in trying to add this to the queue
+            return;
+         }
+      }
+   }
    _self->assign_key(security, callback);
 }
 
@@ -111,6 +138,46 @@ void SecurityIDRecorder::do_work() {
    _self->do_work();
 }
 
+void SecurityIDRecorderState::add_unresolved_spread(MDProducerHandle& handle) {
+   bts_assert_d(lookup_key);
+   FixedSpreadSecurityXL* pspread = static_cast<FixedSpreadSecurityXL*>(handle.mutable_security());
+   if (pspread != nullptr) {
+      for (size_t x=0; x < pspread->leg_count(); ++x) {
+         auto& leg = pspread->legs[x];
+         bts_assert_d(leg.exchange_key.value() != 0);
+         if (leg.key == invalid_security_key) {
+            // Because leg is a reference to shared memory we know it will not move
+            for (const auto* existing : unresolved_legs[leg.exchange_key]) {
+               if (existing == &leg) goto next;
+            }
+            unresolved_legs[leg.exchange_key].push_back(&leg);
+         }
+         next:;
+      }
+   }
+}
+
+void SecurityIDRecorderState::work_spreads() {
+   check_access();
+   if (!lookup_key) return;
+   KeyLookup& lookup = *lookup_key;
+   int count = 0;
+   for (auto iter = unresolved_legs.begin(); count < 500 && iter != unresolved_legs.end();) {
+      const auto& pair = *iter;
+      ++count;
+      SecurityKey key = lookup(pair.first);
+      if (key.value() != invalid_security_key) {
+         for (FixedSpreadLeg* leg : pair.second) {
+            leg->key = key.value();
+         }
+         iter = unresolved_legs.erase(iter);
+      }
+      else {
+         ++iter;
+      }
+   }
+}
+
 bool has_all_leg_keys(const MDProducerHandle& handle) {
    if (handle.security().type == SecurityType::SECURITYTYPE_SPREAD) {
       const FixedSpreadSecurityXL* pspread = static_cast<const FixedSpreadSecurityXL*>(&handle.security());
@@ -120,11 +187,12 @@ bool has_all_leg_keys(const MDProducerHandle& handle) {
          }
       }
    }
-   // SyntheticFixedSecurity is guarunteed to have all of its legs
+   // SyntheticFixedSecurity is guaranteed to have all of its legs
    return true;
 }
 
 void SecurityIDRecorderState::work_handles() {
+   check_access();
    Security security;
    MDProducerHandle handle;
    vector<MDProducerHandle> retry;
@@ -133,6 +201,8 @@ void SecurityIDRecorderState::work_handles() {
       security.Clear();
       // if a spread does not have all of its legs we need to wait
       if (!has_all_leg_keys(handle)) {
+         // add 
+         add_unresolved_spread(handle);
          retry.push_back(handle);
       }
       else {
@@ -150,6 +220,7 @@ void SecurityIDRecorderState::work_handles() {
 }
 
 void SecurityIDRecorderState::work_db() {
+   check_access();
    // Drain the work queue
    // Query the database for each security. If it has an ID set it
    // If it does not have an ID add it to a list
@@ -184,7 +255,7 @@ void SecurityIDRecorderState::work_db() {
             new_descriptors.push_back(descriptor);
             ++count;
             // Limit the amount of time we spend writing to sqlite during startup to avoid blocking the heartbeat
-            if (count > 500) {
+            if (count > 200) {
                break;
             }
          }
@@ -196,7 +267,7 @@ void SecurityIDRecorderState::work_db() {
    
 
    if (!new_entries.empty()) {
-      sqlite3_exec_nofail(db.get(), "BEGIN TRANSACTION");
+      sqlite::exec_nofail(db.get(), "BEGIN TRANSACTION");
 
       for (size_t x = 0; x < new_entries.size(); ++x) {
          Security& security = new_entries[x]->security;
@@ -205,7 +276,7 @@ void SecurityIDRecorderState::work_db() {
          security.set_id(key);
       }
 
-      sqlite3_exec_nofail(db.get(), "COMMIT TRANSACTION");
+      sqlite::exec_nofail(db.get(), "COMMIT TRANSACTION");
    }
 
    // record the securities
@@ -231,10 +302,10 @@ void SecurityIDRecorderState::work_db() {
 
 void SecurityIDRecorder::bulk_insert(const std::vector<const api::Security*>& data, const char* prefix) {
    if (data.size() == 0) return;
-   sqlite3_exec_nofail(_self->db.get(), "BEGIN TRANSACTION");
+   sqlite::exec_nofail(_self->db.get(), "BEGIN TRANSACTION");
    std::string descriptor;
    {
-      auto insert_stmt = prepare_query(_self->db, "INSERT INTO security_keys (rowid, descriptor, type) values (?, ?, ?)");
+      auto insert_stmt = sqlite::prepare_query(_self->db, "INSERT INTO security_keys (rowid, descriptor, type) values (?, ?, ?)");
       for (const api::Security* sec : data) {
          descriptor = prefix + security_descriptor(*sec);
          auto type = securitytype_of(*sec);
@@ -246,11 +317,11 @@ void SecurityIDRecorder::bulk_insert(const std::vector<const api::Security*>& da
          sqlite3_reset(insert_stmt.get());
       }
    }
-   sqlite3_exec_nofail(_self->db.get(), "COMMIT TRANSACTION");
+   sqlite::exec_nofail(_self->db.get(), "COMMIT TRANSACTION");
 }
 
 security_key SecurityIDRecorderState::search(const std::string& descriptor) {
-   auto select_stmt = prepare_query(db, "SELECT rowid from security_keys where descriptor=? LIMIT 1;");
+   auto select_stmt = sqlite::prepare_query(db, "SELECT rowid from security_keys where descriptor=? LIMIT 1;");
    sqlite3_bind_text(select_stmt.get(), 1, descriptor.c_str(), descriptor.size(), SQLITE_STATIC);
    int result = sqlite3_step(select_stmt.get());
    if (result == SQLITE_DONE || result == SQLITE_ROW) {
@@ -260,7 +331,7 @@ security_key SecurityIDRecorderState::search(const std::string& descriptor) {
 }
 
 security_key SecurityIDRecorderState::insert(const std::string& descriptor, SecurityType type) {
-   auto insert_stmt = prepare_query(db, "INSERT INTO security_keys (descriptor, type) values (?, ?)");
+   auto insert_stmt = sqlite::prepare_query(db, "INSERT INTO security_keys (descriptor, type) values (?, ?)");
    sqlite3_bind_text(insert_stmt.get(), 1, descriptor.c_str(), descriptor.size(), SQLITE_STATIC);
    sqlite3_bind_int(insert_stmt.get(), 2, (int)type);
    int result = sqlite3_step(insert_stmt.get());

src/lib/md/SecurityIDRecorder.h

@@ -3,11 +3,14 @@
 #include "common.h"
 #include "MDProducerHandle.h"
 
+#include "lib/utils/Sqlite_utils.h"
 #include <functional>
 #include <memory>
 #include <atomic>
 #include <vector>
-#include <tbb/concurrent_queue.h>
+#include <optional>
+
+typedef struct sqlite3 sqlite3;
 
 namespace bts {
 namespace md {
@@ -17,13 +20,24 @@ namespace md {
 * Provides a callback when ids are safely persisted.
 */
 
+typedef const std::function<SecurityKey(ExchangeKey)> KeyLookup;
+typedef std::optional<KeyLookup> KeyLookupOption;
+struct SecurityIDRecorderParams {
+   SecurityIDRecorderParams() : dispatcher(), db(sqlite::null_db()), lookup_key() {}
+   SecurityIDRecorderParams(utils::Dispatcher& d): dispatcher(&d), db(sqlite::null_db()), lookup_key() {}
+   SecurityIDRecorderParams(utils::Dispatcher& d, KeyLookup& get_key): dispatcher(&d), db(sqlite::null_db()), lookup_key(get_key) {}
+   SecurityIDRecorderParams(sqlite3_ptr& db): dispatcher(), db(std::move(db)), lookup_key() {}
+   SecurityIDRecorderParams(sqlite3_ptr& db, KeyLookup& get_key): dispatcher(), db(std::move(db)), lookup_key(get_key) {}
+   utils::Dispatcher* dispatcher;
+   sqlite3_ptr db;
+   KeyLookupOption lookup_key;
+};
 struct SecurityIDRecorderState;
+
 class SecurityIDRecorder
 {
 public:
-   SecurityIDRecorder();
-   // Automatically call do_work on this dispatcher
-   SecurityIDRecorder(utils::Dispatcher&);
+   SecurityIDRecorder(SecurityIDRecorderParams);
    ~SecurityIDRecorder();
    typedef std::function<void(const api::Security&)> Callback;
 
@@ -32,7 +46,7 @@ public:
    // Slow (copies security) - for use by services
    void assign_key(const api::Security& security, const Callback& key_assigned);
 
-   // Someone must call work periodically to do lookups and flush the reserved ids to disk.
+   // If a dispatcher is not provided you must call work periodically to do lookups and flush the reserved ids to disk.
    void do_work();
 
    void bulk_insert(const std::vector<const api::Security*>&, const char* prefix);

src/lib/md/SecurityIndex.h

@@ -4,7 +4,6 @@
 #include "lib/logging/logging.h"
 #include "MDProducerHandle.h"
 #include "common.h"
-#include "SpreadResolver.h"
 #include <tbb/concurrent_hash_map.h>
 #include <tbb/concurrent_queue.h>
 #include <tbb/queuing_rw_mutex.h>
@@ -26,8 +25,7 @@ class SimpleSecurityIndex
 {
 public:
    SimpleSecurityIndex():
-      _map(),
-      _spread_resolver()
+      _map()
       {}
    ~SimpleSecurityIndex() {}
 
@@ -79,35 +77,11 @@ public:
       return add_security(key, std::move(data));
    }
 
-   //
-   // Spread resolve logic
-
-   void add_spread_legs(ExchangeKey key, const std::vector<ExchangeKey>& legs) {
-      _spread_resolver.add(key, legs);
-   }
-
-   // Resolves spreads and returns the number spreads resolved
-   size_t resolve_spreads() {
-      return _spread_resolver.resolve(
-         [&](ExchangeKey key) { return get_key(key); },
-         [&](ExchangeKey& key, std::vector<security_key>& legs) { on_spread_resolved(key, legs); });
-   }
-
-protected:
-   void on_spread_resolved(ExchangeKey& key, std::vector<security_key>& legs) {
-      auto* entry = get_security(key);
-      bts_assert_d(entry);
-      if (entry) {
-         entry->handle.set_legs(legs);
-      }
-   }
-
 private:
    BTS_NO_COPY(SimpleSecurityIndex);
    typedef std::unordered_map<ExchangeKey, std::unique_ptr<SecurityData>> MapType;
    MapType _map;
    mutable tbb::queuing_rw_mutex _mutex;
-   SpreadResolver<ExchangeKey> _spread_resolver;
 };
 
 
@@ -120,8 +94,7 @@ class ConcurrentSecurityIndex
 {
 public:
    ConcurrentSecurityIndex() :
-      _map(),
-      _spread_resolver()
+      _map()
    {}
    ~ConcurrentSecurityIndex() {}
 
@@ -171,33 +144,10 @@ public:
       return add_security(key, std::move(data));
    }
 
-   //
-   // Spread resolve logic
-
-   void add_spread_legs(ExchangeKey key, const std::vector<ExchangeKey>& legs) {
-      _spread_resolver.add(key, legs);
-   }
-
-   // Resolves spreads and returns the number spreads resolved
-   size_t resolve_spreads() {
-      return _spread_resolver.resolve(
-         [&](ExchangeKey key) { return get_key(key); },
-         [&](ExchangeKey& key, std::vector<security_key>& legs) { on_spread_resolved(key, legs); });
-   }
-
-protected:
-   void on_spread_resolved(ExchangeKey& key, std::vector<security_key>& legs) {
-      auto* entry = get_security(key);
-      bts_assert_d(entry);
-      if (entry) {
-         entry->handle.set_legs(legs);
-      }
-   }
 private:
    BTS_NO_COPY(ConcurrentSecurityIndex);
    typedef tbb::concurrent_hash_map<ExchangeKey, SecurityData*> MapType;
    MapType _map;
-   SpreadResolver<ExchangeKey> _spread_resolver;
 };
 
 }}//namespace

src/lib/md/SettlementRecorder.cpp

@@ -94,13 +94,14 @@ SettlementRecorder::SettlementRecorder(Platform platform) :
    }
 
    // initialize database
-   _self->db = open_db(db_path.c_str());
+   _self->db = sqlite::open_db(db_path.c_str());
+   bts_assert_msg(_self->db, db_path.c_str());
    sqlite3* db = _self->db.get();
    sqlite3_busy_timeout(db, 9999);
-   sqlite3_exec_nofail(db, "PRAGMA journal_mode=OFF");
-   sqlite3_exec_nofail(db, "PRAGMA syncronous=0");
+   sqlite::exec_nofail(db, "PRAGMA journal_mode=WAL");
+   sqlite::exec_nofail(db, "PRAGMA syncronous=0");
    // key=(security_key+date)
-   sqlite3_exec_nofail(db, 
+   sqlite::exec_nofail(db, 
       "CREATE TABLE IF NOT EXISTS settlements2 (key INTEGER NOT NULL, date INTEGER NOT NULL, mantissa INTEGER NOT NULL, exponent INTEGER NOT NULL, PRIMARY KEY (key,date));");
 
    // Delete any records > 10 days old
@@ -108,11 +109,11 @@ SettlementRecorder::SettlementRecorder(Platform platform) :
    auto encoded = datetime::ll_to_date(datetime::ptime_to_ll(cutoff)).to_short();
    char query[256];
    sprintf(query, "delete from settlements2 where date < %i", encoded);
-   sqlite3_exec_nofail(db, query);
+   sqlite::exec_nofail(db, query);
 
    // statements
-   _self->select_stmt = prepare_query(db, "select mantissa,exponent,date from settlements2 where key=? order by date desc limit 2");
-   _self->insert_stmt = prepare_query(db, "insert or replace into settlements2(key, date, mantissa,exponent) values(?,?,?,?)");
+   _self->select_stmt = sqlite::prepare_query(db, "select mantissa,exponent,date from settlements2 where key=? order by date desc limit 2");
+   _self->insert_stmt = sqlite::prepare_query(db, "insert or replace into settlements2(key, date, mantissa,exponent) values(?,?,?,?)");
 }
 
 SettlementRecorder::~SettlementRecorder() { }

src/lib/md/Shm/FixedSecurity.cpp

@@ -251,6 +251,20 @@ size_t num_legs(const Security& sec) {
    return 0;
 }
 
+uint64_t exchange_id(const Security& security) {
+   switch(security.ExchangeData_case()) {
+      case Security::ExchangeDataCase::kCboe: return security.cboe().exchange_id();
+      case Security::ExchangeDataCase::kGlbx: return security.glbx().exchange_id();
+      case Security::ExchangeDataCase::kCfe: return security.cfe().exchange_id();
+      case Security::ExchangeDataCase::kIce: bts_assert(false);
+      // Debatable whether this should be an error or not.
+      // If is possible that we might create a spread with no exchange in the future
+      // and if we did this would be a runtime error.
+      case Security::ExchangeDataCase::EXCHANGEDATA_NOT_SET: return 0;
+   }
+   return -1;
+}
+
 void convert_security(const Security& src, FixedSecurity& dest, bool set_security_key) {
    dest.key = set_security_key ? src.id() : invalid_security_key;
    dest.type = securitytype_of_contractcase(src.Contract_case());
@@ -272,7 +286,9 @@ void convert_security(const Security& src, FixedSecurity& dest, bool set_securit
       case Security::ExchangeDataCase::kCfe:
          dest.cboe_class_key = src.cfe().class_key();
          break;
-      default:
+      case Security::ExchangeDataCase::kIce:
+         bts_assert_d(false);
+      case Security::ExchangeDataCase::EXCHANGEDATA_NOT_SET:
          break;
    }
    dest.price_break = src.contract_spec().price_break() / src.contract_spec().price_multiplier();
@@ -307,6 +323,7 @@ void convert_security(const Security& src, FixedSecurity& dest, bool set_securit
          for (int x = 0; x < src.spread().legs_size(); ++x) {
             const auto& src_leg = src.spread().legs(x);
             spread.legs[x].key = set_security_key ? src_leg.security().id() : invalid_security_key;
+            spread.legs[x].exchange_key = ExchangeKey(exchange_id(src_leg.security()));
             spread.legs[x].leg_side = src_leg.side();
             spread.legs[x].leg_quantity = src_leg.quantity();
             spread.legs[x].leg_price = src_leg.price_multiplier();

src/lib/md/Shm/FixedSecurity.h

@@ -6,6 +6,7 @@
 #include "lib/utils/bts_array_size.h"
 #include "lib/utils/bts_assert.h"
 #include "lib/utils/bts_no_copy.h"
+#include "lib/utils/ValueWrapper.h"
 
 #include <string>
 #include <sstream>
@@ -13,6 +14,9 @@
 namespace bts {
 namespace md {
 
+using SecurityKey = utils::ValueWrapper<struct SecurityKeyTag,uint32_t>;
+using ExchangeKey = utils::ValueWrapper<struct ExchangeKeyTag,uint64_t>;
+
 GCC_DIAG_OFF("-Weffc++") // No Virtual Destructors by design
 
 #define STRING_PROPERTY(NAME)  const char* NAME() const { return _##NAME; } \
@@ -95,6 +99,7 @@ struct BTS_EXPORT FixedSpreadLeg
 {
    FixedSpreadLeg();
    security_key key;
+   ExchangeKey exchange_key;
    Side leg_side;
    double leg_quantity;
    double leg_price;
@@ -180,6 +185,8 @@ void convert_security(const Security& src, FixedSecurity& dest, bool set_securit
 SecurityType securitytype_of_contractcase(Security::ContractCase cc);
 SecurityType securitytype_of(const Security&);
 size_t num_legs(const Security&);
+uint64_t exchange_id(const Security& security);
+
 
 // Stack allocates a security of the given type
 template <class C>

src/lib/md/SpreadResolver.h

@@ -1,79 +0,0 @@
-#pragma once
-
-#include "lib/utils/bts_no_copy.h"
-#include <vector>
-#include <tbb/concurrent_queue.h>
-
-namespace bts {
-namespace md {
-
-
-/*
- * SpreadResolved accepts exchange spread definitions and resolves the legs
- */
-
-template <typename ExchangeKey>
-class SpreadResolver {
-   struct ResolverEntry {
-      ResolverEntry(ExchangeKey key) : exchange_key(key), legs() {}
-      ResolverEntry() : exchange_key(), legs() {}
-      ExchangeKey exchange_key;
-      std::vector<ExchangeKey> legs;
-   };
-public:
-   SpreadResolver() : _queue() {}
-   template <class LegContainer>
-   void add(ExchangeKey key, LegContainer legs) {
-      ResolverEntry entry(key);
-      entry.legs.insert(end(entry.legs), begin(legs), end(legs));
-      _queue.push(entry);
-   }
-
-   // callback should accept const ExchangeKey& key, vector<security_key>& legs)
-   // When the callback is called the legs will be completely resolved
-   template <class MapKey, class CallBack>
-   size_t resolve(MapKey keymap, CallBack on_resolved) {
-      // Drain the queue and call keymap to resolve each leg
-      size_t result = 0;
-      std::vector<ResolverEntry> unresolved;
-      std::vector<security_key> legs;
-      while(!_queue.empty()) {
-         legs.clear();
-         ResolverEntry entry;
-         if(_queue.try_pop(entry)) {
-            // Attempt to look up each spread leg
-            for(ExchangeKey key : entry.legs) {
-               security_key result = keymap(key);
-               if (result == invalid_security_key) {
-                  unresolved.push_back(entry);
-                  break;
-               }
-               else {
-                  legs.push_back(result);
-               }
-            }
-            if (legs.size() == entry.legs.size()) {
-               ++result;
-               on_resolved(entry.exchange_key, legs);
-            }
-         }
-      }
-
-      // If any legs remain unresolved push the entry back in the queue for later
-      for(auto& entry : unresolved) {
-         _queue.push(entry);
-      }
-
-      return result;
-   }
-
-   size_t count() const { return _queue.unsafe_size(); }
-private:
-   BTS_NO_COPY(SpreadResolver);
-   tbb::concurrent_queue<ResolverEntry> _queue;
-};
-
-
-
-}}//namespace
-

src/lib/md/SyntheticSecurityService.cpp

@@ -131,7 +131,31 @@ void SyntheticSecurityService::create_synthetic(
       // Notify listeners that a new security is added
       auto iter = _me->synthetics.find(SecurityKey(security.id()));
       if (iter != _me->synthetics.end()) {
-         send_reply(reply, CreateResult::ERROR_SECURITY_ALREADY_EXISTS, security.id());
+         auto metadata = get_metadata(iter->first);
+         if (metadata && metadata->strategy().deleted()) {
+            // If this security is deleted we can just undelete it
+            Metadata newmeta;
+            // Rather than copy from the existing metatdata create a default. 
+            // The existing metadata is from a deleted spread so may not be valid
+            // UI will call update_security_metadata if non-defaults are needed
+            newmeta.set_security_key(security.id());
+            newmeta.mutable_strategy()->set_deleted(false);
+            LOG(Api,Info,"Un-deleting security {}", security.id());
+            auto id = security.id();
+            update_security_metadata(newmeta, [id,reply](const UpdateResult& result) {
+               if (result.error() == UpdateResult::ERROR_OK) {
+                  // undelete was successful. Metadata update will make this visible again
+                  send_reply(reply, CreateResult::ERROR_OK, id);
+               }
+               else {
+                  LOG(Api,Info,"Failed to un-delete security {}. Error {}", id, result.error());
+                  send_reply(reply, CreateResult::ERROR_SECURITY_ALREADY_EXISTS, id);
+               }
+            });
+         }
+         else {
+            send_reply(reply, CreateResult::ERROR_SECURITY_ALREADY_EXISTS, security.id());
+         }
       }
       else {
          _me->synthetics.insert(std::make_pair(security.id(), security));
@@ -192,4 +216,4 @@ void SyntheticSecurityService::query_security_metadata(
 
 }}}//namespace
 
-#endif
+#endif

src/lib/md/common.h

@@ -4,6 +4,7 @@
 #include "lib/utils/bts_export.h"
 #include "lib/utils/datetime.h"
 #include "lib/domain_model/SecurityKey.h"
+#include "IntPrice.h"
 
 #include "lib/md/protos/co2.pb.h"
 #include "lib/domain_model/Security.pb.h"
@@ -12,7 +13,6 @@
 
 namespace bts {
 namespace md {
-   BTS_EXPORT const extern double NO_PRICE;
    const extern char* SYNTHETIC_SYMBOL;
 
    typedef uint32_t security_key;

src/lib/md/relay/QueryServer.cpp

@@ -17,6 +17,8 @@
 #include "lib/api/services/MarketDataRelay.server.pb.h"
 #include "lib/api/public/ServiceContext.h"
 #include "lib/api/services/AdminService.client.pb.h"
+#include "lib/api/service_helpers/AdminServiceUtils.h"
+#include "lib/config/protos/cme_reference_data.pb.h"
 #ifndef _WIN32
 #include "lib/metrics/metrics.h"
 #endif
@@ -357,7 +359,7 @@ public:
          if (groupiter != _groups.end()) {
             auto& group = groupiter->second;
             bool is_entitled = bypass_entitlement || 
-                              ch.exchange() == api::VENUE_SIM || 
+                              /*ch.exchange() == api::VENUE_SIM || */
                               ch.exchange() == api::VENUE_BTS || 
                               (entitlements && entitlements->count(ch.symbol()));
             fill(ch, *group, reply->add_channels(), is_entitled);
@@ -437,7 +439,7 @@ struct MDRelayServer : bts::co2::MarketDataRelayBase {
              }
           }
           // Temporarily bypass all entitlements
-         _db.query(request, preply, user_entitlement, true);
+         _db.query(request, preply, user_entitlement, bypass_entitlement);
           reply(*preply);
 #ifndef _WIN32
           auto elapsed = (TimeStamp::now() - start);
@@ -518,28 +520,39 @@ QueryServerPtr run_query_server(const bts::api::ServiceContext& ctx, std::shared
        db.check_heartbeats(threadpool);
    });
    container->health_timer->start();
-
+   
    // Subscribe to entitlement data
    container->entitlement_sub =
-      container->admin_client.subscribe_md_permissions_data().subscribe([=](const api::admin::Md_permissions_dataUpdate& update) {
-         auto entitlements = std::make_shared<std::unordered_map<uint32_t, Entitlements>>();
-         for (const auto& row : update.all()) {
-            if (row && row->enabled()) {
-               auto iter = entitlements->find(row->user_id());
-               if (iter == entitlements->end()) {
-                  Entitlements ents;
-                  ents.insert(row->symbol());
-                  entitlements->insert(iter, std::make_pair(row->user_id(), std::move(ents)));
-               }
-               else {
-                  iter->second.insert(row->symbol());
+      container->admin_client.subscribe_md_entitlement_data().subscribe([=](const api::admin::Md_entitlement_dataUpdate& update) {         
+         const auto& all_updates = update.all();
+         utils::Dispatcher::background().post([all_updates, container]() 
+         {
+            const auto& cme_symb_mics = config::get<config::CmeSymbolMics>().cme_mic();
+            auto entitlements = std::make_shared<std::unordered_map<uint32_t, Entitlements>>();
+            for (const auto& row : all_updates) {            
+               if (row && row->enabled()) {                                 
+                  for(auto mic=cme_symb_mics.begin();mic!=cme_symb_mics.end();mic++){
+                     if(api::CMEMics_Name(mic->second) == row->mic()) {
+                        const auto& symbol = mic->first;
+                        auto iter = entitlements->find(row->user_id());
+                        if (iter == entitlements->end()) {
+                           Entitlements ents;
+                           ents.insert(symbol);
+                           ents.insert("SIM:" + symbol);
+                           entitlements->insert(iter, std::make_pair(row->user_id(), std::move(ents)));
+                        }
+                        else {
+                           iter->second.insert(symbol);
+                           iter->second.insert("SIM:" + symbol);
+                        }
+                     }
+                  }
                }
             }
-         }
-         container->relay.update_entitlements(entitlements);
+            container->relay.update_entitlements(entitlements);
+         });
       });
 
-
    // Add a few threads to keep md latency low
    for (int x = 1; x < std::min(4, os::num_cores()); ++x) {
        threadpool.add_thread();

src/lib/md/relay/RelayRecorder.cpp

@@ -130,7 +130,7 @@ void replay_recording(istream& in, IRelayWriter& writer, int32_t naggle_ms) {
    CoconutReplayHeader header;
    size_t count = 0;
    #ifndef _WIN32
-   md::SecurityIDRecorder idrec;
+   md::SecurityIDRecorder idrec {{}};
    #endif
    if (readbuf(in, &header) && header.magic() == magic && header.version() == 3) {
       co2::MDReply message;

src/lib/md/test/md2_tests.cpp

@@ -13,6 +13,7 @@
 #include "lib/metrics/metrics.h"
 
 #include "lib/config/protos/MarketData.pb.h"
+#include "lib/utils/Sqlite_utils.h"
 
 #include <vector>
 #include <string>
@@ -27,7 +28,6 @@
 #include <thread>
 #include <iterator>
 
-
 using namespace std;
 
 namespace bts {
@@ -151,6 +151,7 @@ void assign_ids(vector<MDProducerHandle>::iterator begin, vector<MDProducerHandl
    }
 }
 
+
 class Co2Tests 
 {
 public:
@@ -171,12 +172,20 @@ public:
    }
 
 #ifndef WIN32
-   void test_security_id_recorder() {
+   void test_security_id_recorder(const std::string& db_name) {
       // Verify callbacks happen after work complete
-      SecurityIDRecorder idsvc, idsvc2;
+      auto db1 = sqlite::open_db(db_name.c_str());
+      REQUIRE(db1);
+      auto db2 = sqlite::open_db(db_name.c_str());
+      REQUIRE(db2);
+
+      SecurityIDRecorder idsvc{{db1}}, idsvc2{{db2}};
       security_key result = invalid_security_key;
-      std::string desc = "FOOX 1";
       Security s1 = mkoption("FOOX", invalid_security_key, 2018, 1, 20, 1850);
+      Security f1 = mkfuture("FOO", invalid_security_key, 2018, 1, 20);
+      idsvc.assign_key(f1, [](const api::Security& s) {
+         REQUIRE(s.id() != invalid_security_key);
+      });
       idsvc.assign_key(s1, [&result](const api::Security& s) {
             result = s.id();
          });
@@ -188,7 +197,66 @@ public:
             result2 = s.id();
          });
       idsvc2.do_work();
-      REQUIRE(result == result2);
+
+      if (db_name == ":memory:") {
+         // This is a meta-test to make sure we don't accidently generate the same key sequence
+         // when writing to different databases.
+         REQUIRE(result != result2);
+      }
+      else {
+         REQUIRE(result == result2);
+      }
+   }
+
+   void test_security_id_recorder_spreads() {
+      // Test that security id recorder can resolve spread legs
+
+      // Set up "shared memory"
+      auto cold_header = std::make_shared<ColdHeader>();
+      auto hot_header = std::make_shared<HotHeader>();
+
+      // Create a spread with 2 legs
+      auto f1 = mkfuture("FO1", invalid_security_key, 2018, 1, 17);
+      f1.mutable_glbx()->set_exchange_id(100);
+      REQUIRE(f1.id() == invalid_security_key);
+      auto f1handle = std::make_unique<Co2TestInstrument> (f1, cold_header, hot_header);
+
+      auto f2 = mkfuture("FO2", invalid_security_key, 2018, 1, 20);
+      f2.mutable_glbx()->set_exchange_id(200);
+      auto f2handle = std::make_unique<Co2TestInstrument> (f2, cold_header, hot_header);
+
+      auto s1 = mkspread("FOO", invalid_security_key, { f1, f2 });
+      auto s1handle = std::make_unique<Co2TestInstrument> (s1, cold_header, hot_header);
+
+      // Add spread to SecurityIDRecorder and work it
+      auto db = sqlite::open_db(":memory:");
+      REQUIRE(db);
+
+      SecurityIDRecorder idsvc { {db, [&](ExchangeKey k) -> SecurityKey {
+         for (const auto fut : { f1handle->producer(), f2handle->producer() }) {
+            if (k.value() == fut.security().numeric_id) return SecurityKey(fut.key());
+         }
+         return SecurityKey(invalid_security_key);
+      } } };
+
+      auto workwork = [&]() { idsvc.do_work(); idsvc.do_work(); };
+      idsvc.ensure_key(s1handle->producer());
+      workwork();
+
+      // Expect no key
+      REQUIRE(s1handle->consumer().key() == invalid_security_key);
+
+      // Add leg spreads
+      idsvc.ensure_key(f1handle->producer());
+      idsvc.ensure_key(f2handle->producer());
+      workwork();
+
+      // Expect legs to have keys
+      REQUIRE(f1handle->consumer().key() != invalid_security_key);
+      REQUIRE(f2handle->consumer().key() != invalid_security_key);
+
+      // Expect spread to have keys
+      REQUIRE(s1handle->consumer().key() != invalid_security_key);
    }
 #endif
 
@@ -518,50 +586,6 @@ public:
       REQUIRE(bbo_count > 0); 
    }
 
-   void test_spread_resolver() {
-      SpreadResolver<string> resolver;
-      vector<string> legs;
-      legs.push_back("1");
-      legs.push_back("2");
-      legs.push_back("3");
-      legs.push_back("4");
-      resolver.add("1234", legs);
-      legs.clear();
-      legs.push_back("4");
-      legs.push_back("5");
-      resolver.add("45", legs);
-      auto key_map = [](string& key) -> security_key {
-         if (key == "1") return 1;
-         if (key == "2") return 2;
-         if (key == "3") return 3;
-         if (key == "4") return 4;
-         if (key == "5") return invalid_security_key;
-         REQUIRE(false);
-         return 0;
-      };
-      auto key_map2 = [](string& key) -> security_key {
-         if (key == "4") return 4;
-         if (key == "5") return 5;
-         REQUIRE(false);
-         return 0;
-      };
-
-      // 2 unresolved spreads
-      REQUIRE(resolver.count() == 2);
-      resolver.resolve(key_map, [&](const string& key, const vector<security_key>& legs) {
-            REQUIRE(key == "1234");
-            REQUIRE(legs.size() == 4);
-         });
-      // 1 spread remains
-      REQUIRE(resolver.count() == 1);
-      resolver.resolve(key_map2, [&](const string& key, const vector<security_key>& legs) {
-            REQUIRE(key == "45");
-            REQUIRE(legs.size() == 2);
-         });
-      REQUIRE(resolver.count() == 0);
-   }
-
-
    void test_no_double_security_callback() {
       MDProducer producer(TEST_PLATFORM);
       vector<MDProducerHandle> producers = create_test_securities(producer);
@@ -1002,7 +1026,11 @@ TEST_CASE("CO2Tests","[md]") {
      
    #ifndef WIN32
    SECTION("test_security_id_recorder") {
-      co2_tests->test_security_id_recorder();
+      co2_tests->test_security_id_recorder(":memory:");
+      co2_tests->test_security_id_recorder("file::memory:?cache=shared");
+   }
+   SECTION("test_security_id_recorder_spreads") {
+      co2_tests->test_security_id_recorder_spreads();
    }
    #endif
    SECTION("test_price_size") {
@@ -1026,9 +1054,6 @@ TEST_CASE("CO2Tests","[md]") {
    SECTION("test_consumer_defer_handle2") {
       co2_tests->test_consumer_defer_handle2();
    }
-   SECTION("test_spread_resolver") {
-      co2_tests->test_spread_resolver();
-   }
    SECTION("test_no_double_security_callback") {
       co2_tests->test_no_double_security_callback();
    }
@@ -2231,6 +2256,17 @@ TEST_CASE("MergeImpliedTests", "[md]") {
 }
 
 TEST_CASE("IntPrice", "[md]") {
+   SECTION("sign extend") {
+      auto a = IntPrice_from_double<4>(101.111);
+      REQUIRE(std::to_string(a.price()) == "101.111000");
+      REQUIRE(a.mantissa() == 1011110);
+      a = a.sign_extend(-5);
+      REQUIRE(std::to_string(a.price()) == "101.111000");
+      REQUIRE(a.mantissa() == 10111100);
+      a = a.sign_extend(-9);
+      REQUIRE(std::to_string(a.price()) == "101.111000");
+      REQUIRE(a.mantissa() == 101111000000);
+   }
    SECTION("< equal exponent") {
       auto a = IntPrice_from_double<4>(101.111);
       auto b = IntPrice_from_double<4>(100.111);
@@ -2282,7 +2318,45 @@ TEST_CASE("IntPrice", "[md]") {
       IntPrice delta = INT_NAN - a;
       REQUIRE(delta == INT_NAN);
    }
+
+   SECTION("round equal exponents") {
+      auto price = IntPrice_from_double<4>(100.333);
+      auto tick = IntPrice_from_double<4>(0.25);
+      // buy - round down
+      auto buy = round_to_quote_price(tick, price, api::SIDE_BUY);
+      REQUIRE(buy == IntPrice_from_double<4>(100.25));
+      // sell - round up
+      auto sell = round_to_quote_price(tick, price, api::SIDE_SELL);
+      REQUIRE(sell == IntPrice_from_double<4>(100.50));
+   }
+   SECTION("round smaller tick exponent") {
+      auto price = IntPrice_from_double<4>(100.333);
+      auto tick = IntPrice_from_double<3>(0.25);
+      // buy - round down
+      auto buy = round_to_quote_price(tick, price, api::SIDE_BUY);
+      REQUIRE(buy == IntPrice_from_double<4>(100.25));
+      // sell - round up
+      auto sell = round_to_quote_price(tick, price, api::SIDE_SELL);
+      REQUIRE(sell == IntPrice_from_double<4>(100.50));
+   }
+   SECTION("round smaller price exponent") {
+      auto price = IntPrice_from_double<4>(100.333);
+      auto tick = IntPrice_from_double<7>(0.25);
+      // buy - round down
+      auto buy = round_to_quote_price(tick, price, api::SIDE_BUY);
+      REQUIRE(buy == IntPrice_from_double<7>(100.25));
+      // sell - round up
+      auto sell = round_to_quote_price(tick, price, api::SIDE_SELL);
+      REQUIRE(sell == IntPrice_from_double<4>(100.50));
+   }
+   SECTION("round 0.005") {
+      IntPrice price = { -43595000, -7 };
+      IntPrice tick =  { 50000, -7 };
+      auto buy = round_to_quote_price(tick, price, api::SIDE_BUY);
+      REQUIRE(buy == IntPrice_from_double<7>(-4.360));
+   }
 }
 
+
 }}}
 

src/lib/md/test/synthetic_book_merge_tests.cpp

@@ -27,7 +27,10 @@ struct PS {
    double price;
    Size size;
    bool operator == (const PS& other) const {
-    return price == other.price && size == other.size;
+      // Convert to IntPrice to avoid double rounding issues
+      auto p1 = IntPrice_from_double<11>(price);
+      auto p2 = IntPrice_from_double<11>(other.price);
+      return p1 == p2 && size == other.size;
    }
 };
 
@@ -46,6 +49,25 @@ void setup_market(MDProducerHandle producer, const vector<PS>& bids, const vecto
    producer.update_direct(ts, bids_a, asks_a);
 }
 
+void dump_market(const MDConsumerHandle& handle) {
+   array<PriceSize,COCONUT_MAX_DEPTH> raw_bids,raw_asks;
+   TimeStamp t;
+   handle.get_book(t, COCONUT_MAX_DEPTH, raw_bids.data(), raw_asks.data());
+   vector<PS> src_bids, src_asks;
+   sane::map(raw_bids, src_bids, [](const PriceSize& ps) -> PS {return {ps.price(), ps.size()};});
+   sane::map(raw_asks, src_asks, [](const PriceSize& ps) -> PS {return {ps.price(), ps.size()}; });
+
+   std::cout << std::endl << "bid: ";
+   for (const auto& bid : src_bids) {
+      if (bid.size != 0) std::cout << bid;
+   }
+   std::cout << std::endl << "ask: ";
+   for (const auto& ask : src_asks) {
+      if (ask.size != 0) std::cout << ask;
+   }
+   std::cout << std::endl;
+}
+
 void expect_market(const MDConsumerHandle& handle, const vector<PS> bids, const vector<PS>& asks) {
    array<PriceSize,COCONUT_MAX_DEPTH> raw_bids,raw_asks;
    TimeStamp t;
@@ -71,15 +93,15 @@ void expect_market(const MDConsumerHandle& handle, const vector<PS> bids, const
 }
 
 // Crate some securities with given side
-// tuple = (quantity, tick_size, side)
-tuple<shared_ptr<SyntheticStrategy>,vector<Co2TestInstrument>> make_securities(const vector<tuple<double, double,api::Side>>& leg_descr) {
+// (quantity, multiplier, tick_size, side)
+tuple<shared_ptr<SyntheticStrategy>,vector<Co2TestInstrument>> make_securities(const vector<tuple<double,double,double,api::Side>>& leg_descr) {
    vector<Co2TestInstrument> legs;
    auto synthetic = make_shared<SyntheticStrategy>();
    uint32_t security_key = 100;
    for (const auto& ldesc : leg_descr) {
-      double quantity, tick_size;
+      double quantity, mult, tick_size;
       api::Side side;
-      std::tie(quantity, tick_size, side) = ldesc;
+      std::tie(quantity, mult, tick_size, side) = ldesc;
       Co2TestInstrument instr("ES", api::SECURITYTYPE_FUTURE);
       instr.producer().set_key(security_key);
       ++security_key;
@@ -99,8 +121,20 @@ tuple<shared_ptr<SyntheticStrategy>,vector<Co2TestInstrument>> make_securities(c
    }
    return { synthetic, legs };
 }
+// tuple = (quantity, tick_size, side)
+tuple<shared_ptr<SyntheticStrategy>,vector<Co2TestInstrument>> make_securities(const vector<tuple<double, double,api::Side>>& leg_descr) {
+   std::vector<tuple<double,double,double,api::Side>> legs;
+   for (const auto& t : leg_descr) {
+      double quantity, tick_size;
+      api::Side side;
+      std::tie(quantity, tick_size, side) = t;
+      legs.push_back({ quantity, quantity, tick_size, side });
+   }
+   return make_securities(legs);
+}
 
-void set_leg_params(SyntheticStrategy& strat, uint security_key, const spark::LegSpreaderParams& params) {
+
+void set_leg_params(SyntheticStrategy& strat, uint32_t security_key, const spark::LegSpreaderParams& params) {
    auto* map = strat.strategy.mutable_leg_params_map();
    map->insert({ security_key, params });
 }
@@ -162,7 +196,8 @@ TEST_CASE("SyntheticBook+1-2", "[md][synthetic]") {
                   {{100,16}, {110,8}, {120,12}, {130,5}});
       expect_market(synthetic_handle, 
                   {{-10,1}, {-20,7}, {-40,4}, {-70,4}, {-80,2}, {-100,2}},
-                  {{20,2}, {40,1}, {50,4}, {70,1}, {80,4}, {90,1}, {100,3}, {110,1}, {120,1}});
+                  {{20,2}, {50,5}, {70,1}, {80,4}, {100,4}, {110,1}, {130,1}});
+                  //precise {{20,2}, {40,1}, {50,4}, {70,1}, {80,4}, {90,1}, {100,3}, {110,1}, {120,1}});
    }
 }
 TEST_CASE("SyntheticBook+2-2", "[md][synthetic]") {
@@ -181,8 +216,10 @@ TEST_CASE("SyntheticBook+2-2", "[md][synthetic]") {
                      {{2003.25, 12}, {2003.5, 3}, {2003.75,5}});
 
    expect_market(synthetic_handle, 
-               {{-6.5,5}, {-7.0,1}, {-7.5,1}, {-8.0,1}, {-8.5,2}},
-               {{-5.50,2}, {-5.25,1}, {-5.00,4}, {-4.75,1}, {-4.5,2}, {-3.00,5}});
+               {{-6.5,5}, {-7.0,1}, {-7.5,1}, {-8.5,3}},
+               //precise {{-6.5,5}, {-7.0,1}, {-7.5,1}, {-8.0,1}, {-8.5,2}},
+               {{-5.50,2}, {-5.00,5}, {-4.5,3}, {-3.00,5}});
+               //precise {{-5.50,2}, {-5.25,1}, {-5.00,4}, {-4.75,1}, {-4.5,2}, {-3.00,5}});
 }
 
 
@@ -209,6 +246,26 @@ TEST_CASE("Synthetic3Legs", "[md][synthetic]") {
                   {{-790,2}, {-788,1}, {-778,2}, {-777,5}, {-767,3}, {-766,4}},
                   {{-802,8}, {-803,1}, {-813,1}, {-814,4}, {-824,4}});
    }
+}
+
+TEST_CASE("AllBlank", "[md][synthetic]") {
+   shared_ptr<SyntheticStrategy> synthetic;
+   vector<Co2TestInstrument> test_instr;
+
+   tie(synthetic, test_instr) = make_securities({ {1.0, 0.1, api::SIDE_BUY}, {1.0, 0.1, api::SIDE_SELL }, { 1.0, 0.1, api::SIDE_BUY}});
+   auto leg1 = test_instr[0];
+   auto leg2 = test_instr[1];
+   MDConsumerHandle synthetic_handle(synthetic);
+
+   setup_market(leg1.producer(),
+         {},
+         {});
+   setup_market(leg2.producer(),
+         {},
+         {});
+   expect_market(synthetic_handle,
+         {},
+         {});
 
 }
 
@@ -232,10 +289,92 @@ TEST_CASE("SPARK-955", "[md][synthetic]") {
                {{2607.25,  3}, {2607.50, 1}, {2608.25,  9}});
    expect_market(synthetic_handle,
                // If we do strict price this is right. If we're pessimistic we get {-4.25,2}
-               { {-4.00, 1}, {-4.25, 1}, { -5.00, 3}, {-5.25, 3}, {-5.50,1}, {-5.75, 6} }, 
+               { {-4.25, 2}, { -5.00, 3}, {-5.25, 3}, {-5.75, 7} }, 
+               //precise { {-4.00, 1}, {-4.25, 1}, { -5.00, 3}, {-5.25, 3}, {-5.50,1}, {-5.75, 6} }, 
                { {5.00, 3}, {5.25, 1}, {6.00,3}, {6.50,6} });
 }
 
+TEST_CASE("SPARK-954", "[md][synthetic]") {
+   shared_ptr<SyntheticStrategy> synthetic;
+   vector<Co2TestInstrument> test_instr;
+
+   SECTION("price multiplier = 0.9,1.1") {
+      tie(synthetic, test_instr) = make_securities({ {0.9, 0.25, api::SIDE_BUY}, {1.1, 0.25, api::SIDE_SELL } });
+      auto leg1 = test_instr[0];
+      auto leg2 = test_instr[1];
+      MDConsumerHandle synthetic_handle(synthetic);
+
+      setup_market(leg1.producer(),
+            { {2796.75,22}, {2796.50,38}, {2796.25,57}, {2796.00,59}, {2795.75,60}, {2795.50,67}, {2795.25,68}, {2795.00,84}, {2794.75,115}, {2794.50,98}, },
+            { {2797.00,158}, {2797.25,110}, {2797.50,83}, {2797.75,168}, {2798.00,130}, {2798.25,180}, {2798.50,75}, {2798.75,89}, {2799.00,123}, {2799.25,193}, });
+      setup_market(leg2.producer(),
+            { {2800.50,1}, {2800.25,6}, {2800.00,8}, {2799.75,14}, {2799.50,15}, {2799.25,14}, {2799.00,44}, {2798.75,13}, {2798.50,18}, {2798.25,14}, },
+            { {2801.00,1}, {2801.25,4}, {2801.50,16}, {2801.75,15}, {2802.00,16}, {2802.25,15}, {2802.50,45}, {2802.75,13}, {2803.00,14}, {2803.25,14}, });
+      expect_market(synthetic_handle,
+            { {-564.50,4}, {-564.75,15}, {-565.00,5}, {-565.25,8}, {-565.50,15}, {-565.75,13}, {-566.00,6}, {-566.25,35}, {-566.50,12}, {-566.75,13}, },
+            { {-562.75,6}, {-562.50,7}, {-562.25,13}, {-562.00,13}, {-561.75,13}, {-561.50,40}, {-561.25,12}, {-561.00,16}, {-560.75,13}, });
+
+   }
+   
+   SECTION("price multiplier = 1.0,-1.3") {
+      tie(synthetic, test_instr) = make_securities({ {1.0, 0.005, api::SIDE_BUY}, {1.3, 0.005, api::SIDE_SELL } });
+      auto leg1 = test_instr[0];
+      auto leg2 = test_instr[1];
+      MDConsumerHandle synthetic_handle(synthetic);
+      setup_market(leg1.producer(),
+            { { 14.185, 1 }, { 14.180, 7 } },
+            { { 14.205, 1 }, { 14.210, 5 } });
+      setup_market(leg2.producer(),
+            { { 14.255, 10 }, { 14.250, 42 }, { 14.230, 1 }, { 14.220, 1 }, { 14.215, 1 }, { 14.210, 3 }, { 14.205, 1 }, { 14.200, 2 } },
+            { { 14.265, 18 }, { 14.270, 19 } });
+      expect_market(synthetic_handle,
+            { { -4.360, 1 }, { -4.365, 7 } },
+            { { -4.325, 1 }, { -4.320, 5 } });
+   }
+
+}
+
+TEST_CASE("AggregationAcrossLevels", "[md][synthetic]") {
+   shared_ptr<SyntheticStrategy> synthetic;
+   vector<Co2TestInstrument> test_instr;
+   SECTION("buy 1 sell 3") {
+      tie(synthetic, test_instr) = make_securities({ {1.0, 0.005, api::SIDE_BUY}, {3, 0.005, api::SIDE_SELL } });
+      auto leg1 = test_instr[0];
+      auto leg2 = test_instr[1];
+      MDConsumerHandle synthetic_handle(synthetic);
+      setup_market(leg1.producer(),
+         { {16.215, 6} },
+         { {16.220, 5} }
+      );
+      setup_market(leg2.producer(),
+         { {16.500, 2 }, { 16.495, 2 } },
+         { {16.505, 1 }, { 16.510, 1 }, { 16.515, 1 }, {16.520,1}, {16.525,1}, {16.530,1}}
+      );
+      expect_market(synthetic_handle,
+         { {-33.330,1}, {-33.375, 1} },
+         { { -33.265,1} }
+      );
+   }
+   SECTION("buy 1 sell 4") {
+      tie(synthetic, test_instr) = make_securities({ {1.0, 0.005, api::SIDE_BUY}, {4, 0.005, api::SIDE_SELL } });
+      auto leg1 = test_instr[0];
+      auto leg2 = test_instr[1];
+      MDConsumerHandle synthetic_handle(synthetic);
+      setup_market(leg1.producer(),
+         { {16.215, 6} },
+         { {16.220, 5} }
+      );
+      setup_market(leg2.producer(),
+         { {16.500, 2 }, { 16.495, 2 } },
+         { {16.505, 1 }, { 16.510, 1 }, { 16.515, 1 }, {16.520,1}, {16.525,1}, {16.530,1}, {16.535,2}}
+      );
+      expect_market(synthetic_handle,
+         { { -49.865,1}, {-49.925,1} },
+         { { -49.760,1} }
+      );
+   }
+}
+
 TEST_CASE("Synthetic_net_change", "[md][synthetic]") {
    shared_ptr<SyntheticStrategy> synthetic;
    vector<Co2TestInstrument> test_instr;
@@ -265,7 +404,8 @@ TEST_CASE("Synthetic_net_change", "[md][synthetic]") {
       // Validate merge
       expect_market(synthetic_handle, 
                   {{-10,1}, {-20,7}, {-40,4}, {-70,4}, {-80,2}, {-100,2}},
-                  {{20,2}, {40,1}, {50,4}, {70,1}, {80,4}, {90,1}, {100,3}, {110,1}, {120,1}});
+                  {{20,2}, {50,5}, {70,1}, {80,4}, {100,4}, {110,1}, {130,1}});
+                  // precise {{20,2}, {40,1}, {50,4}, {70,1}, {80,4}, {90,1}, {100,3}, {110,1}, {120,1}});
    }
 }
 
@@ -325,4 +465,4 @@ BENCHMARK(synthetic_get_book_speed3);
 #endif
 
 }}}//namespace
-#endif
+#endif

src/lib/orderman/actors/order_manager/OrderManager.cpp

@@ -15,6 +15,7 @@
 
 #include <algorithm>
 #include <unordered_map>
+#include <boost/algorithm/string/trim.hpp>
 
 namespace bts {
 namespace orderman {
@@ -123,10 +124,11 @@ void OrderManager::on_fill_recovery(
              update.events->size());
          me->state = State::WAIT_ORDERS;
 
-         for (auto const &ev : *(update.events))
-            me->services_data.limiter.add(*ev.security, ev.fill.account(),
-                                          ev.fill.side(),
+         for (auto const &ev : *(update.events)) {
+            auto trimmed_account_name = boost::algorithm::trim_copy(ev.fill.account());
+            me->services_data.limiter.add(*ev.security, trimmed_account_name,ev.fill.side(),
                                           Quantity(ev.fill.last_shares()));
+         }
 
          me->stored_order_sub = subscribe<StoredOrderEvent>(
              [this](auto const &orders) { this->on_order_recovery(orders); });

src/lib/orderman/limiter/LimiterUtils.h

@@ -9,10 +9,13 @@ namespace orderman{
 
 class LimiterUtils{
 public:
-   static void can_send_new_order(api::Side side, Quantity quantity, bool is_security_level,
-                                  int long_pos_limit, int short_pos_limit,
-                                  int net_pos,
-                                  Limiter::LimiterQueryResult& out_res){
+   static void can_send_new_order_net(api::Side side,
+                                      Quantity quantity,
+                                      bool is_security_level,
+                                      int long_pos_limit,
+                                      int short_pos_limit,
+                                      int net_pos,
+                                      Limiter::LimiterQueryResult& out_res){
       out_res.ok = true;
       bts_assert(long_pos_limit >= 0);
       bts_assert(short_pos_limit <= 0);
@@ -35,13 +38,15 @@ public:
          else
             out_res.error_msg = fmt::format("Product Short Position (Max {} Current {})", short_pos_limit, net_pos);
       }
-   }
-   static void can_update_order (api::Side side,
-                                 Quantity from, Quantity to,
-                                 bool is_security_level,
-                                 int long_pos_limit, int short_pos_limit,
-                                 int net_pos,
-                                 Limiter::LimiterQueryResult& out_res){
+   }  
+   static void can_update_order_net (api::Side side,
+                                     Quantity from,
+                                     Quantity to,
+                                     bool is_security_level,
+                                     int long_pos_limit,
+                                     int short_pos_limit,
+                                     int net_pos,
+                                     Limiter::LimiterQueryResult& out_res){
       out_res.ok = true;
       if (to.value() > from.value()){
          int net_pos_new = 0;
@@ -65,6 +70,60 @@ public:
                out_res.error_msg = fmt::format("Product Short Position (Max {} Current {})", short_pos_limit, net_pos);
          }
       }
+   }
+    static void can_send_new_order_abs(api::Side side,
+                                      Quantity quantity,
+                                      uint32_t long_pos_limit,
+                                      uint32_t short_pos_limit,
+                                      uint32_t long_pos,
+                                      uint32_t short_pos,
+                                      Limiter::LimiterQueryResult& out_res){
+      out_res.ok = true;
+      if(side == api::Side::SIDE_BUY){
+         if(long_pos + quantity.value() > long_pos_limit){
+            out_res.error_msg = fmt::format("Long Position (Abs) "
+                                            "(Max {} Current {})",
+                                            long_pos_limit,long_pos);
+            out_res.ok = false; 
+         }
+      }
+      else{
+         if(short_pos + quantity.value() > short_pos_limit){
+            out_res.error_msg = fmt::format("Short Position (Abs) "
+                                            "(Max {} Current {})",
+                                            short_pos_limit,short_pos);
+            out_res.ok = false; 
+         }
+      }
+   }
+   static void can_update_order_abs (api::Side side,
+                                     Quantity from,
+                                     Quantity to,
+                                     uint32_t long_pos_limit,
+                                     uint32_t short_pos_limit,
+                                     uint32_t long_pos,
+                                     uint32_t short_pos,
+                                     Limiter::LimiterQueryResult& out_res){
+      out_res.ok = true;
+      if (to.value() > from.value()){
+         uint32_t diff = to.value() - from.value();
+         if(side == api::Side::SIDE_BUY){
+            if(long_pos + diff > long_pos_limit){
+               out_res.error_msg = fmt::format("Long Position (Abs) "
+                                               "(Max {} Current {})",
+                                               long_pos_limit,long_pos);
+               out_res.ok = false;
+            }
+         }
+         else{
+            if(short_pos + diff > short_pos_limit){
+               out_res.error_msg = fmt::format("Short Position (Abs) "
+                                               "(Max {} Current {})",
+                                               short_pos_limit,short_pos);
+               out_res.ok = false;
+            }
+         }
+      }
    }
    static api::Side invert_side(api::Side side){
       return side == api::SIDE_BUY ? api::SIDE_SELL : api::SIDE_BUY;

src/lib/orderman/limiter/ProductLimitNode.cpp

@@ -5,14 +5,17 @@ namespace orderman{
 
 ProductLimitNode::ProductLimitNode(uint32_t account_id,
                                    uint32_t, // parent_account_id,
-                                   const bts::api::admin::AccountProductPositionLimitUnit& limit)
+                                   const bts::api::admin::AccountProductPositionLimitUnit& limit,
+                                   bool enforce_net)
    :
    _account_id(account_id),
    //_parent_account_id(parent_account_id),
    _limit(limit),
+   _enforce_net(enforce_net),
    _long_pos_limit (limit.long_position_limit()),
-   _short_pos_limit (-1 * limit.short_position_limit()),
-   _net_pos(0)
+   _short_pos_limit (limit.short_position_limit()),
+   _long_pos(0),
+   _short_pos(0)
    {}
 }}
 

src/lib/orderman/limiter/ProductLimitNode.h

@@ -20,49 +20,92 @@ class ProductLimitNode{
 private:
    const uint32_t _account_id; // , _parent_account_id;
    const bts::api::admin::AccountProductPositionLimitUnit& _limit;
-   const int _long_pos_limit, _short_pos_limit;
-   int _net_pos;
+   const bool _enforce_net;
+   const uint32_t _long_pos_limit, _short_pos_limit;
+   // this is the only state here
+   uint32_t _long_pos, _short_pos;
 public:
    ProductLimitNode(uint32_t account_id,
                     uint32_t parent_account_id,
-                    const bts::api::admin::AccountProductPositionLimitUnit& limit);
+                    const bts::api::admin::AccountProductPositionLimitUnit& limit,
+                    bool enforce_net);
    // accessors
    ////////////
    uint32_t account_id() const {return _account_id;}   
-   int net_pos() const {return _net_pos;}
+   int net_pos() const {
+      return static_cast<int>(_long_pos)-static_cast<int>(_short_pos);
+   }
    // position updates
    ///////////////////
    void copy_positions_from(const ProductLimitNode& source){
-      _net_pos = source.net_pos();
+      _long_pos=source._long_pos;
+      _short_pos=source._short_pos;
    }
    void add_position(api::Side side, uint32_t qty){
       if (side==api::Side::SIDE_BUY)
-         _net_pos += qty;
+         _long_pos += qty;
       else
-         _net_pos -= qty;
+         _short_pos += qty;
    }
    void remove_position(api::Side side, uint32_t qty){
       if (side==api::Side::SIDE_BUY)
-        _net_pos -= qty;
+         _long_pos = _subtract_unsigned(_long_pos, qty);
       else
-         _net_pos += qty;
+         _short_pos = _subtract_unsigned(_short_pos, qty);
    }
    // risk limit checks
    ////////////////////
    Limiter::LimiterQueryResult can_send_new_order(api::Side side, Quantity quantity) const{
       Limiter::LimiterQueryResult result;
-      LimiterUtils::can_send_new_order (side, quantity, false,
-                                        _long_pos_limit, _short_pos_limit,
-                                        _net_pos, result);
+      if (_enforce_net)
+         LimiterUtils::can_send_new_order_net (side, 
+                                               quantity,
+                                               false,
+                                               static_cast<int>(_long_pos_limit),
+                                               -1 * static_cast<int>(_short_pos_limit),
+                                               net_pos(),
+                                               result);
+      else
+         LimiterUtils::can_send_new_order_abs (side, 
+                                               quantity,
+                                               _long_pos_limit, 
+                                               _short_pos_limit,
+                                               _long_pos,
+                                               _short_pos,
+                                               result);
       return result;
    }
    Limiter::LimiterQueryResult can_update_order(api::Side side, Quantity from, Quantity to) const{
       Limiter::LimiterQueryResult result;
-      LimiterUtils::can_update_order (side, from, to, false,
-                                      _long_pos_limit, _short_pos_limit,
-                                      _net_pos, result);
+      if(_enforce_net)
+         LimiterUtils::can_update_order_net (side, 
+                                             from, 
+                                             to, 
+                                             false,
+                                             static_cast<int>(_long_pos_limit),
+                                             -1 * static_cast<int>(_short_pos_limit),
+                                             net_pos(),
+                                             result);
+      else
+         LimiterUtils::can_update_order_abs (side,
+                                             from,
+                                             to,
+                                             _long_pos_limit,
+                                             _short_pos_limit,
+                                             _long_pos,
+                                             _short_pos,
+                                             result);
       return result;
    }
+   uint32_t _subtract_unsigned(uint32_t n1, uint32_t n2){
+      if(n1 >= n2)
+         return n1-n2;
+      else{
+         LOG(OrderMan, Error, "ProductLimitNode._subtract_unsigned {0}-{1} ",
+                              n1, n2);
+         return 0;
+      }
+   }
    // test only functions
    //////////////////////
    const bts::api::admin::AccountProductPositionLimitUnit& test_only_func_limit() const {return _limit;}

src/lib/orderman/limiter/ProductLimitTree.cpp

@@ -10,8 +10,14 @@ ProductLimitTree::ProductLimitTree(const api::Product& product,
 _product (product)
 {
    for(const auto& account : admin_data.accounts()){
+      bool enforce_net=true;  // default is net
+      if(account.has_risk_limit_enforcement())
+         enforce_net=account.risk_limit_enforcement().product_limit_in_net_terms();
       const bts::api::admin::AccountProductPositionLimitUnit* limit = bts::admin::AdminServiceUtils::find_account_product_position_limit(account, _product);
-      _tree.emplace_back(account.id(), bts::admin::AdminServiceUtils::has_id(account.parent_account_id()) ? account.parent_account_id() : 0, limit != nullptr ? *limit : zero_limit);
+      _tree.emplace_back(account.id(), 
+                         bts::admin::AdminServiceUtils::has_id(account.parent_account_id()) ? account.parent_account_id() : 0, 
+                         limit != nullptr ? *limit : zero_limit,
+                         enforce_net);
    }
 }
 }}

src/lib/orderman/limiter/SecurityLimitNode.h

@@ -71,9 +71,9 @@ public:
          return result;
       }
       if (_apply_sec_position_limits)
-         LimiterUtils::can_send_new_order (side, quantity, true,
-                                           _long_pos_limit, _short_pos_limit,
-                                           _net_pos, result);
+         LimiterUtils::can_send_new_order_net (side, quantity, true,
+                                               _long_pos_limit, _short_pos_limit,
+                                               _net_pos, result);
       if (result.ok)
          result = _product_node.can_send_new_order(side, quantity);
       return result;
@@ -86,9 +86,9 @@ public:
          return result;
       }
       if (_apply_sec_position_limits)
-         LimiterUtils::can_update_order (side, from, to, true, 
-                                         _long_pos_limit, _short_pos_limit,
-                                         _net_pos, result);
+         LimiterUtils::can_update_order_net (side, from, to, true, 
+                                             _long_pos_limit, _short_pos_limit,
+                                             _net_pos, result);
       if (result.ok)
          result = _product_node.can_update_order(side, from, to);
       return result;

src/lib/orderman/orders/spreader_orders/spreader/SpreaderUtils.cpp

@@ -30,12 +30,12 @@ bool SpreaderUtils::resolve_synthetic_spread (const SpreaderPreActivationOrderSe
             const bts::api::ContractLeg& leg = *it;
             if (leg.security().has_future() || leg.security().has_spread()){
                const bts::api::Security* leg_sec_ptr = spreader_pao_services.resolve(bts::SecurityKey (leg.security().id()));
-               if (leg_sec_ptr != nullptr){
+               auto leg_exec_strategy_it = execution_strategy.leg_params_map().find(leg.security().id());
+               if (leg_sec_ptr != nullptr && 
+                  leg_exec_strategy_it != execution_strategy.leg_params_map().end()) {
                   bts_assert(leg_sec_ptr->id() == leg.security().id());
                   LOG(OrderMan, Info, "resolve_synthetic_spread resolved leg key {}", leg_sec_ptr->id());
-                  auto leg_exec_strategy_it = execution_strategy.leg_params_map().find(leg.security().id());
-                  bts_assert(leg_exec_strategy_it != execution_strategy.leg_params_map().end());
-                  const auto& leg_params=leg_exec_strategy_it->second;
+                  const auto& leg_params = leg_exec_strategy_it->second;
                   api::Side leg_order_side = SpreaderUtils::get_leg_side(order_def.side(), leg);
                   // quoter order def
                   api::orderman::OrderDefinition quoter_order_def = order_def;

src/lib/persistence/SqliteStore.cpp

@@ -57,42 +57,43 @@ void SqliteStore::init_db() {
       LOG(persist, Warn, "{} database does not exist yet. Creating it at {}",
           me->db, path.string());
 
-      me->conn = utils::open_db(path.c_str());
+      me->conn = sqlite::open_db(path.c_str());
+      bts_assert_msg(me->conn, path.c_str());
 
       const char *create_schema_sql =
           "create table store ( ns text, key text, value text)";
-      utils::sqlite3_exec_nofail(me->conn, create_schema_sql);
+      sqlite::exec_nofail(me->conn, create_schema_sql);
 
       const char *key_index_sql =
           "create unique index key_index on store(ns,key)";
-      utils::sqlite3_exec_nofail(me->conn, key_index_sql);
+      sqlite::exec_nofail(me->conn, key_index_sql);
 
       const char *ns_index_sql = "create index ns_index on store(ns)";
-      utils::sqlite3_exec_nofail(me->conn, ns_index_sql);
+      sqlite::exec_nofail(me->conn, ns_index_sql);
    } else {
-      me->conn = utils::open_db(path.c_str());
+      me->conn = sqlite::open_db(path.c_str());
    }
 
    // Asynchronous for speed
-   utils::sqlite3_exec_nofail(me->conn, "PRAGMA synchronous = 0;");
-   utils::sqlite3_exec_nofail(me->conn, "PRAGMA journal_mode=WAL");
+   sqlite::exec_nofail(me->conn, "PRAGMA synchronous = 0;");
+   sqlite::exec_nofail(me->conn, "PRAGMA journal_mode=WAL");
 
    // prepare insert statement
    const char *insert_sql = "insert into store (ns, key, value) values(?,?,?)";
-   me->insert = utils::prepare_query(me->conn, insert_sql);
+   me->insert = sqlite::prepare_query(me->conn, insert_sql);
 
    // prepare update statement
    const char *update_sql =
        "update store set value = ?  where ns = ? and key = ?";
-   me->update = utils::prepare_query(me->conn, update_sql);
+   me->update = sqlite::prepare_query(me->conn, update_sql);
 
    // prepare remove statement
    const char *remove_all_sql = "delete from store where ns = ?";
-   me->remove_all = utils::prepare_query(me->conn, remove_all_sql);
+   me->remove_all = sqlite::prepare_query(me->conn, remove_all_sql);
 
    // prepare remove statement
    const char *remove_sql = "delete from store where ns = ? and key = ?";
-   me->remove = utils::prepare_query(me->conn, remove_sql);
+   me->remove = sqlite::prepare_query(me->conn, remove_sql);
 }
 
 SqliteStore::~SqliteStore() {}
@@ -100,7 +101,7 @@ SqliteStore::~SqliteStore() {}
 SqliteStore::Data SqliteStore::retrieve_all() {
    SqliteStore::Data result;
    sqlite3_stmt_ptr query =
-       utils::prepare_query(me->conn, "select ns,key,value from store");
+       sqlite::prepare_query(me->conn, "select ns,key,value from store");
 
    int sqlite_result;
    do {
@@ -125,7 +126,7 @@ std::pair<bool, std::string> SqliteStore::retrieve_one(std::string const &ns,
                                                        std::string const &k) {
    auto query_str = fmt::format(
        "select ns,key,value from store where ns='{}' and key='{}'", ns, k);
-   sqlite3_stmt_ptr query = utils::prepare_query(me->conn, query_str.c_str());
+   sqlite3_stmt_ptr query = sqlite::prepare_query(me->conn, query_str.c_str());
 
    std::pair<bool, std::string> result{false, ""};
 

src/lib/quickbus/network/QBConnection.cpp

@@ -167,7 +167,7 @@ void QBConnection::recv_msg_body()
                               handle_msg(&m);
                         }
                         else
-                           recv_msg_body();
+                           close_connection(error.message());
 
                      });
 }

src/lib/tracing/PerfLog.cpp

@@ -0,0 +1,184 @@
+
+
+#include "lib/tracing/PerfLog.h"
+#include "lib/persistence/persistence_utils.h"
+#include "lib/utils/datetime.h"
+#include "lib/config/config.h"
+#include "lib/config/protos/system.pb.h"
+
+#include <SQLiteCpp/SQLiteCpp.h>
+#include <boost/date_time/posix_time/ptime.hpp>
+#include <boost/filesystem.hpp>
+#include <pthread.h>
+#include <thread>
+
+namespace bts {
+namespace trace {
+
+auto create_session = "create table session ("
+                      "version TEXT, "
+                      "host TEXT, "
+                      "build_type TEXT, "
+                      "session_start TIMESTAMP PRIMARY KEY"
+                      ")";
+
+auto create_trace = "create table trace ("
+                    "session_start TIMESTAMP,"
+                    "ts INTEGER,"
+                    "hts timestamp,"
+                    "channel TEXT,"
+                    "entry_kind TEXT,"
+                    "txt_1 TEXT,"
+                    "sn INTEGER,"
+                    "uint32_1 INTEGER,"
+                    "uint32_2 INTEGER,"
+                    "uint64_1 INTEGER,"
+                    "thread TEXT"
+                    ")";
+
+auto insert_trace = "insert into trace ("
+                    "session_start,"
+                    "ts,"
+                    "hts,"
+                    "channel,"
+                    "entry_kind,"
+                    "txt_1,"
+                    "sn,"
+                    "uint32_1,"
+                    "uint32_2,"
+                    "uint64_1,"
+                    "thread"
+                    ") values ("
+                    ":session_start,"
+                    ":ts,"
+                    ":hts,"
+                    ":channel,"
+                    ":entry_kind,"
+                    ":txt_1,"
+                    ":sn,"
+                    ":uint32_1,"
+                    ":uint32_2,"
+                    ":uint64_1,"
+                    ":thread"
+                    ")";
+
+auto insert_session = "insert into session ("
+                      "version,"
+                      "host,"
+                      "build_type,"
+                      "session_start"
+                      ") values ("
+                      ":version,"
+                      ":host,"
+                      ":build_type,"
+                      ":session_start"
+                      ")";
+
+struct PerfLog::PImpl {
+   PImpl(std::string const &loc)
+       : db(loc, SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE),
+         insert_trace_statement(), insert_session_statement(), session_start() {}
+
+   SQLite::Database db;
+   std::unique_ptr<SQLite::Statement> insert_trace_statement;
+   std::unique_ptr<SQLite::Statement> insert_session_statement;
+   std::string session_start;
+   bool disabled;
+};
+
+PerfLog::PerfLog() : _pending(), _disabled(false), me() {
+   auto loc = init_storage_location();
+
+   if (loc.empty()) {
+      _disabled = true;
+      return;
+   }
+   
+   try {
+      me = std::make_unique<PImpl>(loc);
+      if (!me->db.tableExists("session")) {
+         LOG(persist, Warn, "{} database does not exist yet. Creating it ",
+             std::string(me->db.getFilename()));
+         me->db.exec(create_session);
+         me->db.exec(create_trace);
+      }
+
+      me->insert_trace_statement =
+          std::make_unique<SQLite::Statement>(me->db, insert_trace);
+      me->insert_session_statement =
+          std::make_unique<SQLite::Statement>(me->db, insert_session);
+      me->session_start = datetime::FIX_timestamp(datetime::local_now());
+
+      me->insert_session_statement->bind(":version",
+                                         config::get<config::System>().version());
+      me->insert_session_statement->bind(":host",
+                                         config::get<config::System>().host_name());
+#ifdef DEBUG
+      me->insert_session_statement->bind(":build_type", "debug");
+#else
+      me->insert_session_statement->bind(":build_type", "release");
+#endif
+      me->insert_session_statement->bind(":session_start", me->session_start);
+      me->insert_session_statement->exec();
+      me->insert_session_statement->reset();
+   } catch (SQLite::Exception const &e) {
+      LOG(persist, Error, "Error initializing perf_trace DB {}. Error: {}", loc,
+          std::string(e.getErrorStr()));
+      _disabled = true;
+   }
+}
+
+PerfLog::~PerfLog() {}
+
+std::string PerfLog::init_storage_location() {
+   auto loc = persistence::storage_location("perf_trace2");
+   auto parent = boost::filesystem::path(loc).parent_path();
+
+   if (!boost::filesystem::is_directory(parent)) {
+      if (boost::filesystem::create_directories(parent))
+         return loc;
+      else
+         return "";
+   }
+   else
+      return loc;
+}
+
+size_t PerfLog::store_log_entries() {
+   if (_disabled)
+      return 0;
+
+   PerfLogEntry ev;
+   size_t count = 0;
+   while (_pending.try_pop(ev)) 
+      try {
+         me->insert_trace_statement->bind(":session_start", me->session_start);
+         me->insert_trace_statement->bind(":ts", (long long)ev.ts.to_uint64());
+         me->insert_trace_statement->bind(
+             ":hts",
+             datetime::FIX_timestamp(datetime::timestamp_to_ptime(ev.ts)));
+         me->insert_trace_statement->bind(":channel", ev.channel);
+         me->insert_trace_statement->bind(":entry_kind", ev.entry_kind);
+         me->insert_trace_statement->bind(":txt_1", ev.txt_1);
+         me->insert_trace_statement->bind(":sn", ev.sn);
+         me->insert_trace_statement->bind(":uint32_1", ev.uint32_1);
+         me->insert_trace_statement->bind(":uint32_2", ev.uint32_2);
+         me->insert_trace_statement->bind(":uint64_1", (long long)ev.uint64_1);
+
+         char tn[17];
+         pthread_getname_np(pthread_self(), tn,16);
+         me->insert_trace_statement->bind(":thread", tn);
+         me->insert_trace_statement->exec();
+         me->insert_trace_statement->reset();
+         count ++;
+      } catch (SQLite::Exception const &e) {
+         LOG(persist, Error, "Unable to insert perf trace in {}. Error: {}",
+             std::string(me->db.getFilename()), std::string(e.getErrorStr()));
+
+          _disabled = true;
+      }
+   return count;
+}
+
+} // namespace trace
+} // namespace trace

src/lib/tracing/PerfLog.h

@@ -0,0 +1,60 @@
+
+#pragma once
+
+#include "lib/utils/TimeStamp.h"
+#include <memory>
+#include <tbb/concurrent_queue.h>
+
+namespace bts {
+namespace trace {
+
+struct CorID {
+   const char *channel = "";
+   uint32_t sn = 0;
+};
+
+struct PerfLogEntry {
+   utils::TimeStamp ts;
+   const char *channel;
+   const char *entry_kind;
+   const char *txt_1;
+   uint32_t sn;
+   uint32_t uint32_1;
+   uint32_t uint32_2;
+   uint64_t uint64_1;
+   std::thread::id thread_id;
+};
+
+class PerfLog {
+   tbb::concurrent_queue<PerfLogEntry> _pending;
+   bool _disabled;
+   struct PImpl;
+   std::unique_ptr<PImpl> me;
+
+ public:
+   PerfLog() ;
+   ~PerfLog();
+   size_t store_log_entries();
+   
+   void log(CorID cid, const char *entry_kind, uint32_t uint32_1 = 0,
+            uint32_t uint32_2 = 0, const char *txt_1 = "",
+            uint64_t uint64_1 = 0) {
+      _pending.push({utils::TimeStamp::now(), cid.channel, entry_kind, txt_1,
+                     cid.sn, uint32_1, uint32_2, uint64_1,
+                     std::this_thread::get_id()});
+   }
+
+   void log(CorID cid, utils::TimeStamp ts, const char *entry_kind,
+            uint32_t uint32_1 = 0, uint32_t uint32_2 = 0,
+            const char *txt_1 = nullptr, uint64_t uint64_1 = 0) {
+      _pending.push({ts,cid.channel, entry_kind, txt_1,
+                     cid.sn, uint32_1, uint32_2, uint64_1,
+                     std::this_thread::get_id()});
+   }
+
+ private:
+   std::string init_storage_location();
+};
+
+} // namespace trace
+} // namespace bts