IE notification in Sprak Admin

Client sdk

.vscode/c_cpp_properties.json

@@ -5,7 +5,9 @@
             "includePath": [
                 "/usr/include",
                 "/usr/local/include",
-                "${workspaceRoot}"
+                "${workspaceRoot}",
+                "${workspaceFolder}/src",
+                "${workspaceRoot}/externals/linux/include"
             ],
             "defines": [],
             "intelliSenseMode": "clang-x64",
@@ -13,13 +15,16 @@
                 "path": [
                     "/usr/include",
                     "/usr/local/include",
-                    "${workspaceRoot}"
+                    "${workspaceRoot}",
+                    "${workspaceFolder}/src",
+                    "${workspaceRoot}/externals/linux/include"
                 ],
                 "limitSymbolsToIncludedHeaders": true,
                 "databaseFilename": ""
             },
             "cStandard": "c11",
-            "cppStandard": "c++17"
+            "cppStandard": "c++17",
+            "compileCommands": "${workspaceFolder}/build/ninja_debug/compile_commands.json"
         },
         {
             "name": "Windows",

.vscode/tasks.json

@@ -14,24 +14,21 @@
             "focus": false,
             "panel": "shared"
          },
-         "group": {
-            "kind": "build",
-            "isDefault": true
-         }
+         "group": "build"
       },
       {
-         "label": "server_build",         
+         "label": "server_build",
          "type": "shell",
          "command": ".\\tools\\buildsparkd.cmd",
-         "problemMatcher": {                        
-            "base": "$gcc",       
+         "problemMatcher": {
+            "base": "$gcc",
             "owner": "cpp",
             "fileLocation": [
                "relative",
-               "${workspaceFolder}"               
+               "${workspaceFolder}"
             ],
             "pattern": {
-               "regexp": "(.?)(home)(.?)(bts)(.?)(spark)(.?)(.*):(\\d+):(\\d+):\\s+(warning|error):\\s+(.*)",               
+               "regexp": "(.?)(home)(.?)(bts)(.?)(spark)(.?)(.*):(\\d+):(\\d+):\\s+(warning|error):\\s+(.*)",
                "file": 8,
                "line": 9,
                "column": 10,
@@ -49,6 +46,18 @@
             "kind": "build",
             "isDefault": true
          }
-      }      
+      },
+      {
+         "label": "linux build (Debug)",
+         "type": "shell",
+         "command": "cd build/ninja_debug; time ninja",
+         "group": {
+            "kind": "build",
+            "isDefault": true
+         },
+         "problemMatcher": [
+            "$gcc"
+         ]
+      }
    ]
 } 

doc/api/content/client/_index.md

@@ -1,6 +1,6 @@
 +++
 title="Client API"
-template="client.html"
+redirect_to = "client/getting-started/overview/"
 sort_by="weight"
 +++
 

doc/api/content/client/getting-started/_index.md

@@ -1,6 +1,5 @@
 +++
 title="Getting Started"
-template="client.html"
 sort_by="weight"
 weight = 1
 +++ 

doc/api/content/client/getting-started/install.md

@@ -1,9 +1,25 @@
 +++
 title = "Install Prerequisites"
-template="client.html"
-weight = 1
+weight = 10
 +++
 
-## Coming Soon
+# C#/.NET
+
+We target [.NET Standard 2.0](https://docs.microsoft.com/en-us/dotnet/standard/net-standard) so you can use .NET Framework 4.6.1+ or .NET Core 2.0+.  Simply add a reference to the [Spark Nuget package](https://nuget.org) to get started.
+
+```
+dotnet new console -lang c#
+dotnet add package spark_sdk
+dotnet build
+```
+
+# C++
+
+The easiest way to get started with C++ is to use the Spark Server SDK to develop your application.  Follow the instructions to [download and install the Spark Server SDK](/server/getting-started/install) and open the client_sdk sample project. If you want to run your C++ Client SDK application on Windows [contact us](mailto://api@bluetradesys.com) to get the required headers and binaries.
+
+# JavaScript
+
+We use JavaScript internally with the Spark Client API over a websocket connection but the documentation and packaging isn't ready yet.  If this is something you're interested in get in touch with us at [api@bluetradesys.com](mailto:api@bluetradesys.com).
+
+
 
-* Install .net core

doc/api/content/client/getting-started/overview.md

@@ -0,0 +1,31 @@
++++
+title = "Overview"
+weight = 1
++++
+
+The Spark Client SDK is a high level command and control API which can be used to build scripts or integrate Spark into your existing ecosystem.  The API is divided into several independent **"services"** responsible for different aspects of the system.  A Service can define both [remote procedure call (RPC)](/client/getting-started/services#rpc) and incremental [streaming](/client/getting-started/services#streams) data.
+
+The [Spark Desktop](https://www.bluetradingsystems.com/bts-spark/) is built on top of this API so you can be assured that it is complete and well tested.
+
+Spark uses [Google Protocol Buffers](https://developers.google.com/protocol-buffers/) as an interface definition language similar to [GRPC](https://grpc.io) but backed by our custom high-performance TCP protocol.  We currently support C#/.NET and C++ <!--and Javascript--> but the architecture is designed for easy extension so contact us about your specific business needs.
+
+## Orders
+
+Use the [OrderService](/client/services/orderservice) to send and manage orders including native exchange orders, spreader orders, and custom Algo orders created using the [Spark Server SDK](https://api.bluetradesys.com/server/).
+
+## Spreader
+
+Use the [SyntheticSecurityService](/client/services/syntheticsecurityservice) to create and manage synthetic spreads.  Once created these spreads show up in the security and market data feeds just like exchange listed spreads.
+
+## Position
+
+Use the [PositionService](/client/services/positionservice) to get your position and P&L reports or add fills manually to keep the Spark system up to date with an external system.
+
+## Administration
+
+Use the [AdminService](/client/services/adminservice) to manage users and control limits.
+
+## Market Data
+
+For .NET subscribe to market data though the high level MarketDataClient interface.  From C++ or JavaScript use the MarketDataRelay service directly.
+

doc/api/content/client/getting-started/services.md

@@ -0,0 +1,127 @@
++++
+title = "Services"
+weight = 20
++++
+
+The Spark Client SDK uses [Google Protocol Buffers](https://developers.google.com/protocol-buffers/) to define the interface for each service in a language independent manner.  Services have both functions which return a single result and streams of data which return values continuously while subscribed.  This page describes the mapping from the Protocol Buffer definition to the concrete definition in each supported language.
+
+# ServiceContext
+
+To connect your client to the Spark Server you need to get or create a `ServiceContext` instance.  A `ServiceContext` represents a connection to the Spark Server and a background thread for handling messages on that connection.  Generally there should be a single `ServiceContext` per process.  All of the methods on the `ServiceContext` are safe thread safe.
+
+Construct a ServiceContext by calling the default constructor.  The ServiceContext will find the Spark Server by loading the server configuration over an HTTP connection.  To determine the URL it will search the following locations and use the first one that is present.  
+
+* A command line argument `-configserver <url>`
+* An environment variable `BTSCONFIG`
+* `%APPDATA%/bts/conf/spark.conf` (Windows)
+* /sparkdata/conf/spark.conf (Linux)
+* $HOME/sparkdata/conf/spark.conf (Linux)
+
+If your application is running on a system with the BTS Client installed this will probably work without any further configuration.  If you plan to run your client application on a separate machine we recommend setting the `BTSCONFIG` environment variable.
+
+<div class="info">
+<i class="fa fa-lightbulb-o fa-fw" aria-hidden="true"></i>
+If you are using the Spark Server SDK to run a local copy of the Spark server set <p><strong>BTSCONFIG = http://localhost:9000/config/</strong></p>
+</div>
+
+To get a proxy for a given service call ServiceContext.locate&lt;T>("qualifier") where T is the type of the service and "qualifier" is an optional qualifier used to disambiguate between 2 instances of the same service.  The locate call cannot fail but it may return a disconnected proxy.  When you make a call on the proxy you should check the call result to see if the method call succeeded.
+
+# RPC
+
+Here is the OrderService reduced to a single method definition. A RPC method always takes exactly 1 argument and returns 1 value.  In this example the `create_order` call takes a `Create` instance and returns a `Response`.  
+
+```C
+service OrderService { 
+  rpc create_order(Create) returns (Response);
+  ...
+}
+```
+
+The `Create` and `Response` types are also defined as protocol buffer messages.  Each field can either be a primitive type another message or a list or map.
+
+```C
+message Create {
+  OrderDefinition new_order = 1;
+  double price = 2;
+  uint32 quantity = 3;
+}
+
+message Response {
+    uint32 order_key = 1; 
+    OrderError error_code = 2; 
+    string msg = 3;
+}
+```
+
+Each RPC call generates both synchronous (blocking) and asynchronous variants. In C# the asynchronous calls have the `_async` suffix and return a `Task`.  In C++ the call is overloaded to either return a value synchronously or accept a callback function to be called on the ServiceContext thread when the call completes. <!-- In Javascript only async methods returning a Promise are generated because it never makes sense to make a blocking call in the browser. -->
+
+To call the resulting service locate a proxy using the [ServiceContext](#servicecontext) and then call `cancel_order` or `cancel_order_async`.  If you are calling the Spark API from a GUI application you should always prefer the `_async` version of the calls.
+
+
+### C# example
+```c#
+var ctx = new ServiceContext();
+var orders = ctx.locate<OrderService>();
+var response = 
+  await orders.create_order_async(
+    new Create { price = 2.25, quantity = 42 });
+if (response.success) {
+  Console.WriteLine($"success: {response.value}");
+}
+else {
+  Console.WriteLine($"Error: {response.error_message}");
+}
+```
+### C++ example
+```cpp
+auto ctx = ServiceContext{};
+auto orders = ctx.locate<OrderService>();
+Create c;
+c.set_price(2.25);
+c.set_quantity(42);
+// Instead of providing an _async method C++ uses overloading.
+// To make a syncronous call simply omit the callback
+orders.create_order(c, [=](const Result<Response>& response) {
+  if(response.success()) {
+    std::cout << "success.";
+  }
+  else {
+    std::cout << "error: " << response.error_message();
+  }
+});
+```
+
+# Streams
+
+Streams allow you to subscribe to live updating data. Each stream represents a changing collection of protocol buffer objects.  Similar to a database row each item has an `id` which uniquely identifies it within the collection.  Updates from the Spark server can add, remove or replace entries in the collection.  Since only the changed items are sent over the wire the stream can remain responsive even when it represents a large collection.
+
+Streams are implemented using the language bindings for [ReativeX](http://reactivex.io/) which provide utility functions to drill down to the specific data your application needs. In C# we provide both mutable and immutable versions of the generated data types (C++ users can rely on `const`).
+
+When defined in a proto file streams use the `stream` keyword on the result type.
+```C
+service OrderService { 
+  ...
+  rpc fills (FillsSub) returns (stream Fill);
+}
+```
+
+### C# example
+```c#
+  var sc = new ServiceContext();
+  var orders = sc.locate<OrderService>();
+  var fills_observable = orders.subscribei_fills();
+  var unsub = fills_observable.Subscribe(update => {
+    Console.WriteLine(
+        "Added {0} items, Removed {1} items, Updated {2} items.  Total items {3}", 
+        update.added.Count(), 
+        update.removed.Count(), 
+        update.changes.Count(), 
+        update.all.Count());
+  });
+
+  Thread.Sleep(TimeSpan.FromMinutes(5));
+  unsub.Dispose();
+```
+### C++ Example
+```cpp
+```

doc/api/content/client/samples/_index.md

@@ -0,0 +1,6 @@
++++
+title="Samples"
+sort_by="weight"
+weight = 30
++++ 
+

doc/api/content/client/services/AdminService.md

@@ -0,0 +1,3 @@
++++
+title = "AdminService"
++++

doc/api/content/client/services/OrderService.md

@@ -0,0 +1,3 @@
++++
+title = "OrderService"
++++

doc/api/content/client/services/PositionService.md

@@ -0,0 +1,3 @@
++++
+title = "PositionService"
++++

doc/api/content/client/services/SyntheticSecurityService.md

@@ -0,0 +1,3 @@
++++
+title = "SyntheticService"
++++

doc/api/content/client/services/_index.md

@@ -0,0 +1,6 @@
++++
+title="Services"
+#sort_by="weight"
+weight = 10
++++ 
+

doc/api/content/client/services/marketdata.md

@@ -0,0 +1,3 @@
++++
+title = "Market Data"
++++

doc/api/content/server/_index.md

@@ -1,6 +1,5 @@
 +++
 title="Algo API"
-template="documentation.html"
 redirect_to = "server/getting-started/overview/"
 sort_by="weight"
 +++ 

doc/api/content/server/getting-started/_index.md

@@ -1,6 +1,5 @@
 +++
 title="Getting Started"
-template="documentation.html"
 sort_by="weight"
 weight = 1
 +++ 

doc/api/content/server/getting-started/development.md

@@ -71,12 +71,12 @@ The first step is to start the spark_sdk Docker container with the examples dire
 
 Windows Command Prompt:
 {% clipable() %}
-docker run --rm -it -v %CD%:/examples -u bts -w /examples --shm-size 1024m --name spark_sdk -h spark_sdk --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -p 9000:9000 -p 9001:9001 spark_sdk bash
+docker run --rm -it -v %CD%:/examples -u bts -w /examples --shm-size 2048m --name spark_sdk -h spark_sdk --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -p 9000:9000 -p 9001:9001 spark_sdk bash
 {% end %}
 
 **Powershell** / Linux / Mac:
 {% clipable() %}
-docker run --rm -it -v ${PWD}:/examples -u bts -w /examples --shm-size 1024m --name spark_sdk -h spark_sdk --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -p 9000:9000 -p 9001:9001 spark_sdk bash
+docker run --rm -it -v ${PWD}:/examples -u bts -w /examples --shm-size 2048m --name spark_sdk -h spark_sdk --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -p 9000:9000 -p 9001:9001 spark_sdk bash
 {% end %}
 
 ### Attach

doc/api/content/server/getting-started/download.md

@@ -5,7 +5,7 @@ weight = 2
 
 ## Request Access 🔑
 
-To start using the Server SDK you will need to request access though our sales department.  Please email <a href="mailto:info@bluetradesys.com">info@bluetradesys.com</a> or call <a href="tel:919-913-0850">(919) 913-0850</a> and we will provide download credentials.
+To start using the Server SDK you will need to request access though our sales department.  Please email <a href="mailto:api@bluetradesys.com">api@bluetradesys.com</a> or call <a href="tel:919-913-0850">(919) 913-0850</a> and we will provide download credentials.
 
 ## Download
 

doc/api/content/server/ref/_index.md

@@ -1,6 +1,5 @@
 +++
 title="Reference"
-template="documentation.html"
 sort_by="weight"
 weight = 2
 +++ 

doc/api/content/server/ref/faq.md

@@ -6,3 +6,9 @@ weight = 40
 ## I need to make a call that might take a few seconds. What is the best way to do this?
 
 Create a background thread in [do_before_launch](/server/ref/initialization#before-launch) or [do_launch](/server/ref/initialization#launch-launch-attached) to do the work.  Use [on_timer](/server/ref/subscriptions#timer) to poll until the result is available.
+
+## I don't see the header files in the .zip file. Where are they?
+
+The SDK headers are baked into the docker image along with the build tools. Once you've loaded the image with `docker load < spark_sdk.img` you can run this command to copy them out to your local file system `docker run --rm -it -v $PWD:/out spark_sdk /bin/tar -cvf /out/headers.tar /sdk/include`  This will copy the headers to the current directory in headers.tar.  We still recommend doing the actual build inside the docker environment but having a copy of the headers can be useful for setting up your development environment.
+
+

doc/api/content/server/samples/_index.md

@@ -1,6 +1,5 @@
 +++
 title="Samples"
-template="documentation.html"
 sort_by="weight"
 weight = 3
 +++ 

doc/api/templates/client.html

@@ -1,31 +0,0 @@
-{% extends "index.html" %}
-
-{% block extra_nav_class %}{% endblock extra_nav_class %}
-{% block extra_content_class %}content--reversed{% endblock extra_content_class %}
-
-{% block content %}
-{% set section = get_section(path="client/_index.md") %}
-<div class="documentation">
-    <aside class="documentation__sidebar">
-        <ul>
-        {% for p in section.subsections %}
-            {% set subsection = get_section(path=p) %}
-            <li>
-                <span class="documentation__sidebar__title">{{ subsection.title }}</span>
-                <ul>
-                    {% for page in subsection.pages %}
-                        <li class="{% if current_path == page.path %}active{% endif %}">
-                            <a href="{{page.permalink}}">{{page.title}}</a>
-                        </li>
-                    {% endfor %}
-                </ul>
-            </li>
-        {% endfor %}
-        </ul>
-    </aside>
-    <div class="documentation__content">
-        {% block doc_content %}
-        {% endblock doc_content %}
-    </div>
-</div>
-{% endblock content %}

doc/api/templates/documentation.html

@@ -1,36 +0,0 @@
-{% extends "index.html" %}
-
-{% block extra_nav_class %}{% endblock extra_nav_class %}
-{% block extra_content_class %}content--reversed{% endblock extra_content_class %}
-
-{% block content %}
-{%if page.extra.section %}
-{% set section = get_section(path="{{page.extra.section}}/_index.md") %}
-{% else %}
-{% set section = get_section(path="server/_index.md") %}
-{% endif %}
-<div class="documentation">
-    <aside class="documentation__sidebar">
-        <ul>
-        {% for p in section.subsections %}
-            {% set subsection = get_section(path=p) %}
-            <li>
-                <span class="documentation__sidebar__title">{{ subsection.title }}</span>
-                <ul>
-                    {% for page in subsection.pages %}
-                        <li class="{% if current_path == page.path %}active{% endif %}">
-                            <a href="{{page.permalink}}">{{page.title}}</a>
-                        </li>
-                    {% endfor %}
-                </ul>
-            </li>
-        {% endfor %}
-        </ul>
-    </aside>
-    <div class="documentation__content">
-        {% block doc_content %}
-        {% endblock doc_content %}
-        {% block next_link %} {% endblock next_link %}
-    </div>
-</div>
-{% endblock content %}

doc/api/templates/index.html

@@ -1,5 +1,5 @@
 <!DOCTYPE html>
-<html lang="en-gb">
+<html lang="en-us">
     <head>
         <meta charset="UTF-8">
         <meta http-equiv="X-UA-Compatible" content="IE=edge">
@@ -14,6 +14,7 @@
         <link rel="stylesheet" href="{{ get_url(path="site.css") }}"/>
         <link rel="icon" href="{{ get_url(path="favicon.ico") }}">
         <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/font-awesome/4.7.0/css/font-awesome.min.css">
+        <script> dataLayer = [{ 'visitorType': 'geek' }]; </script>
     </head>
     <body>
 
@@ -36,13 +37,13 @@
             </nav>
         </header>
 
-        <div class="content {% block extra_content_class %}{% endblock extra_content_class %}">
+        <div class="content content--reversed">
             {% block content %}
                 <div class="hero">
                     <h1>Spark SDK</h1>
                     <p class="hero__tagline">
                         Spark consists of a client UI 
-                        (<a class="white" href="https://www.bluetradingsystems.com/bts-spark#ladder-algo-screen">Spark Client</a>), 
+                        (<a href="https://www.bluetradingsystems.com/bts-spark#ladder-algo-screen">Spark Client</a>), 
                         a high-performance professional futures front end, connected via our proprietary protocol over TCP to our Spark Server, which in turn connects to execution venues via their native protocols.
                         Use the Spark SDK to extend Spark Server with your custom Algos, then use the Client SDK to launch and control them.
                     </p>
@@ -64,7 +65,7 @@
                             <a href="{{ get_url(path="@/server/_index.md") }}" class="button">Get started</a>
                         </div>
 
-                        <div class="selling-point disabled">
+                        <div class="selling-point">
                             <h2>
                                 Client SDK
                                 <i class="fa fa-gamepad" aria-hidden="true"></i>
@@ -72,19 +73,9 @@
                             <p>
                                 The Spark client SDK allows you to integrate spark with your existing ecosystem. The Spark Client is built entirely on this simple, powerful API.
                             </p>
-                            <a href="" class="button" disabled>Get started</a>
+                            <a href="" class="button">Get started</a>
                         </div>
                         
-                        <!-- <div class="selling-point disabled">
-                            <h2>
-                                Microsoft Excel
-                                <i class="fa fa-table" aria-hidden="true"></i>
-                            </h2>
-                            <p>
-                                More of your business is built on excel than probably should be. Continue this worrying trend with the Spark Excel API.
-                            </p>
-                            <a href="" class="button" disabled>Get started</a>
-                        </div> -->
                     </div>
                 </div>
 
@@ -94,6 +85,15 @@
             © Blue Trading Systems
         </footer>
 
+        <script>
+            if (location.hostname == "api.bluetradesys.com") {
+                (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':
+                new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
+                j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
+                'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
+                })(window,document,'script','dataLayer','GTM-56FHG8R');
+            }
+        </script>
         <script type="text/javascript" src="{{ get_url(path="elasticlunr.min.js") }}"></script>
         <script type="text/javascript" src="{{ get_url(path="search_index.en.js") }}"></script>
         <script type="text/javascript" src="{{ get_url(path="search.js") }}"></script>

doc/api/templates/page.html

@@ -1,13 +1,39 @@
-{% extends "documentation.html" %}
+{% extends "index.html" %}
+{# this has access to the "page" variable but documentation.html does not #}
 
 {% block title %}{{ page.title }} | {{ super() }} {% endblock title %}
-{% block doc_content %}
-    <h1>{{page.title}}</h1>
-    {{page.content | safe}}
-{% endblock doc_content %}
 
 {% block next_link %}
-<p>
+{% endblock next_link %}
+
+{% block content %}
+{% set section = get_section(path=page.ancestors[1]) %}
+{# 
+{#<div>{{ section.title }} </div>#}
+<div class="documentation">
+    <aside class="documentation__sidebar">
+        <ul>
+        {% for p in section.subsections %}
+            {% set subsection = get_section(path=p) %}
+            <li>
+                <span class="documentation__sidebar__title">{{ subsection.title }}</span>
+                <ul>
+                    {% for page in subsection.pages %}
+                        <li class="{% if current_path == page.path %}active{% endif %}">
+                            <a href="{{page.permalink}}">{{page.title}}</a>
+                        </li>
+                    {% endfor %}
+                </ul>
+            </li>
+        {% endfor %}
+        </ul>
+    </aside>
+    <div class="documentation__content">
+        <h1>{{page.title}}</h1>
+        {{page.content | safe}}
+
+    {# next/pre links #}
+    <p>
     {% if page.lighter %}
         <a href="{{ page.lighter.permalink }}">
             <i class="fa fa-chevron-left fa-fw" aria-hidden="true"></i>
@@ -21,5 +47,7 @@
             <i class="fa fa-chevron-right fa-fw" aria-hidden="true"></i>
         </a>
     {% endif %}
-</p>
-{% endblock next_link %}
+    </p>
+    </div>
+</div>
+{% endblock content %}

doc/user/CMakeDeps.cmake

@@ -1 +1 @@
-set(sources /home/ppinto/Dropbox/bts/spark/doc/user/build.sh;content/_index.md;content/bts_spark/Glossary/_index.md;content/bts_spark/Help/_index.md;content/bts_spark/Help/about_spark.png;content/bts_spark/Help/image39.png;content/bts_spark/Help/image58.png;content/bts_spark/Launcher/_index.md;content/bts_spark/Launcher/image38.png;content/bts_spark/Launcher/image68.png;content/bts_spark/Logging-In-to-the-System/_index.md;content/bts_spark/Logging-In-to-the-System/image3.png;content/bts_spark/Logging-In-to-the-System/image64.png;content/bts_spark/Market/_index.md;content/bts_spark/Market/change_securities.png;content/bts_spark/Market/image11.png;content/bts_spark/Market/image14.png;content/bts_spark/Market/image20.png;content/bts_spark/Market/image21.png;content/bts_spark/Market/image23.png;content/bts_spark/Market/image24.png;content/bts_spark/Market/image34.png;content/bts_spark/Market/image49.png;content/bts_spark/Market/image53.png;content/bts_spark/Market/image66.png;content/bts_spark/Market/image69.png;content/bts_spark/Market/market_view.png;content/bts_spark/Market/settings_icon.png;content/bts_spark/Market/time_and_sales.png;content/bts_spark/My-Windows/_index.md;content/bts_spark/My-Windows/image15.png;content/bts_spark/My-Windows/image16.png;content/bts_spark/My-Windows/image37.png;content/bts_spark/Settings/_index.md;content/bts_spark/Settings/image51.png;content/bts_spark/Settings/image9.png;content/bts_spark/Spreader/_index.md;content/bts_spark/Spreader/image10.png;content/bts_spark/Spreader/image13.png;content/bts_spark/Spreader/image18.png;content/bts_spark/Spreader/image19.png;content/bts_spark/Spreader/image28.png;content/bts_spark/Spreader/image29.png;content/bts_spark/Spreader/image31.png;content/bts_spark/Spreader/image35.png;content/bts_spark/Spreader/image40.png;content/bts_spark/Spreader/image43.png;content/bts_spark/Spreader/image54.png;content/bts_spark/Spreader/image55.png;content/bts_spark/Spreader/image56.png;content/bts_spark/Spreader/image6.png;content/bts_spark/Spreader/image7.png;content/bts_spark/Spreader/image70.png;content/bts_spark/Spreader/image72.png;content/bts_spark/Spreader/image8.png;content/bts_spark/Trading/OrderTypes/index.md;content/bts_spark/Trading/_index.md;content/bts_spark/Trading/algo_studio.png;content/bts_spark/Trading/collapse_all_icon.png;content/bts_spark/Trading/cover_algo_control.png;content/bts_spark/Trading/expand_all_icon.png;content/bts_spark/Trading/image1.png;content/bts_spark/Trading/image17.png;content/bts_spark/Trading/image2.png;content/bts_spark/Trading/image25.png;content/bts_spark/Trading/image27.png;content/bts_spark/Trading/image33.png;content/bts_spark/Trading/image42.png;content/bts_spark/Trading/image44.png;content/bts_spark/Trading/image47.png;content/bts_spark/Trading/image48.png;content/bts_spark/Trading/image5.png;content/bts_spark/Trading/image50.png;content/bts_spark/Trading/image52.png;content/bts_spark/Trading/image57.png;content/bts_spark/Trading/image59.png;content/bts_spark/Trading/image60.png;content/bts_spark/Trading/image61.png;content/bts_spark/Trading/image62.png;content/bts_spark/Trading/image73.png;content/bts_spark/Trading/image74.png;content/bts_spark/Trading/join_algo_control.png;content/bts_spark/Trading/ladder.png;content/bts_spark/Trading/ladder_badges.png;content/bts_spark/Trading/ladder_main_part.png;content/bts_spark/Trading/lean_algo_control.png;content/bts_spark/Trading/mini_orderbook.png;content/bts_spark/Trading/oco_algo_control.png;content/bts_spark/Trading/offset_algo_control.png;content/bts_spark/Trading/pnl_calculation.png;content/bts_spark/Trading/pnl_window.png;content/bts_spark/Trading/recreate_icon.png;content/bts_spark/Trading/settings_icon.png;content/bts_spark/Trading/spark_menu_order_book.png;content/bts_spark/Trading/top_algo_control.png;content/bts_spark/Trading/trigger_algo_control.png;content/bts_spark/_index.md;content/release_notes/_index.md;content/system_requirements/_index.md;static/book.js;static/favicon.ico;static/image.js;templates/index.html;templates/page.html;templates/section.html;templates/shortcodes/back_button.html)
+set(sources /spark/doc/user/build.sh;content/_index.md;content/bts_spark/Glossary/_index.md;content/bts_spark/Help/_index.md;content/bts_spark/Help/about_spark.png;content/bts_spark/Help/image39.png;content/bts_spark/Help/image58.png;content/bts_spark/Launcher/_index.md;content/bts_spark/Launcher/image38.png;content/bts_spark/Launcher/image68.png;content/bts_spark/Logging-In-to-the-System/_index.md;content/bts_spark/Logging-In-to-the-System/image3.png;content/bts_spark/Logging-In-to-the-System/image64.png;content/bts_spark/Market/_index.md;content/bts_spark/Market/change_securities.png;content/bts_spark/Market/image11.png;content/bts_spark/Market/image14.png;content/bts_spark/Market/image20.png;content/bts_spark/Market/image21.png;content/bts_spark/Market/image23.png;content/bts_spark/Market/image24.png;content/bts_spark/Market/image34.png;content/bts_spark/Market/image49.png;content/bts_spark/Market/image53.png;content/bts_spark/Market/image66.png;content/bts_spark/Market/image69.png;content/bts_spark/Market/market_view.png;content/bts_spark/Market/settings_icon.png;content/bts_spark/Market/time_and_sales.png;content/bts_spark/My-Windows/_index.md;content/bts_spark/My-Windows/image15.png;content/bts_spark/My-Windows/image16.png;content/bts_spark/My-Windows/image37.png;content/bts_spark/Settings/_index.md;content/bts_spark/Settings/image51.png;content/bts_spark/Settings/image9.png;content/bts_spark/Spreader/_index.md;content/bts_spark/Spreader/image10.png;content/bts_spark/Spreader/image13.png;content/bts_spark/Spreader/image18.png;content/bts_spark/Spreader/image19.png;content/bts_spark/Spreader/image28.png;content/bts_spark/Spreader/image29.png;content/bts_spark/Spreader/image31.png;content/bts_spark/Spreader/image35.png;content/bts_spark/Spreader/image40.png;content/bts_spark/Spreader/image43.png;content/bts_spark/Spreader/image54.png;content/bts_spark/Spreader/image55.png;content/bts_spark/Spreader/image56.png;content/bts_spark/Spreader/image6.png;content/bts_spark/Spreader/image7.png;content/bts_spark/Spreader/image70.png;content/bts_spark/Spreader/image72.png;content/bts_spark/Spreader/image8.png;content/bts_spark/Trading/OrderTypes/index.md;content/bts_spark/Trading/_index.md;content/bts_spark/Trading/algo_studio.png;content/bts_spark/Trading/collapse_all_icon.png;content/bts_spark/Trading/cover_algo_control.png;content/bts_spark/Trading/expand_all_icon.png;content/bts_spark/Trading/image1.png;content/bts_spark/Trading/image17.png;content/bts_spark/Trading/image2.png;content/bts_spark/Trading/image25.png;content/bts_spark/Trading/image27.png;content/bts_spark/Trading/image33.png;content/bts_spark/Trading/image42.png;content/bts_spark/Trading/image44.png;content/bts_spark/Trading/image47.png;content/bts_spark/Trading/image48.png;content/bts_spark/Trading/image5.png;content/bts_spark/Trading/image50.png;content/bts_spark/Trading/image52.png;content/bts_spark/Trading/image57.png;content/bts_spark/Trading/image59.png;content/bts_spark/Trading/image60.png;content/bts_spark/Trading/image61.png;content/bts_spark/Trading/image62.png;content/bts_spark/Trading/image73.png;content/bts_spark/Trading/image74.png;content/bts_spark/Trading/join_algo_control.png;content/bts_spark/Trading/ladder.png;content/bts_spark/Trading/ladder_badges.png;content/bts_spark/Trading/ladder_main_part.png;content/bts_spark/Trading/lean_algo_control.png;content/bts_spark/Trading/mini_orderbook.png;content/bts_spark/Trading/oco_algo_control.png;content/bts_spark/Trading/offset_algo_control.png;content/bts_spark/Trading/pnl_calculation.png;content/bts_spark/Trading/pnl_window.png;content/bts_spark/Trading/recreate_icon.png;content/bts_spark/Trading/settings_icon.png;content/bts_spark/Trading/spark_menu_order_book.png;content/bts_spark/Trading/top_algo_control.png;content/bts_spark/Trading/trigger_algo_control.png;content/bts_spark/_index.md;content/release_notes/_index.md;content/system_requirements/_index.md;static/book.js;static/favicon.ico;static/image.js;templates/index.html;templates/page.html;templates/section.html;templates/shortcodes/back_button.html)

doc/user/content/release_notes/_index.md

@@ -4,6 +4,15 @@ weight = 3
 sort_by = "weight"
 insert_anchor_links = "right"
 +++
+## 2.15.3
+### Bug Fixes
+  - Fixed a bug where some CME currency spreads exhibited incorrect tick values
+## 2.15.2
+### User Visible Changes  
+  - Excel RTD plugin can now receive ICE market data
+## 2.15.1
+### Bug Fixes
+  - Fixed a FIX log parsing issue that resulted in exchange gateway logon problems
 ## 2.15.0
 ### User Visible Changes  
   - Added ability to hide Ladder status messages via Ladder Settings -> Left Panel tab -> Show/Hide groupbox -> Status Message

src/client/spark/Spark.csproj

@@ -19,7 +19,8 @@
     <LangVersion>7.2</LangVersion>
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
     <WarningsAsErrors />
-    <Version>1.0.0</Version>
+    <Version>$(major).$(minor).$(patch)</Version>
+    <Version Condition="'$(version)'=='..'">1.0.0</Version>
   </PropertyGroup>
   <ItemGroup>
     <ProjectReference Include="../spark_api/spark_api.csproj" />

src/client/spark/view/Spark/Core/TickUtils.cs

@@ -292,7 +292,9 @@ namespace mbts.spark {
             int count = 0;
             double factored_base = log10_diff > 0 ? reduced_target_base * Math.Pow(10, Math.Ceiling(log10_diff)) : reduced_target_base;
             do {
-               if (factored_base % raw_base == 0)
+               // Note that the modulus operator is not used here due to rounding errors.
+               // Such a rounding errror was the cause of the bug https://bluetradesys.atlassian.net/browse/SP-283.
+               if (MathUtils.is_int(factored_base / raw_base))
                   return new TickFactors(factored_base, factored_base / raw_base);
                factored_base *= 10;
             } while (++count < 10);

src/client/spark_api/spark_api.csproj

@@ -3,17 +3,19 @@
     <TargetFrameworks>netstandard2.0</TargetFrameworks>
     <Authors>Blue Trading Systems</Authors>
     <Company>Blue Trading Systems</Company>
-    <Product>Spark</Product>
     <Description />
+    <PackageId>Spark_API</PackageId>
+    <Product>Spark</Product>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <Platforms>x64</Platforms>
     <PlatformTarget>AnyCPU</PlatformTarget>
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
     <WarningsAsErrors />
-    <Version>1.0.0</Version>
+    <Version>$(major).$(minor).$(patch)</Version>
+    <Version Condition="'$(version)'=='..'">0.0.0</Version>
     <LangVersion>7.2</LangVersion>
   </PropertyGroup>
-  
+
   <ItemGroup>
     <PackageReference Include="Google.Protobuf" Version="3.6.1" />
     <PackageReference Include="System.Collections.Immutable" Version="1.5.0" />
@@ -24,12 +26,10 @@
     <Proto3 Include="**/*.proto" />
     <Proto3 Include="../../lib/api/*.proto" />
     <Proto3 Include="../../lib/api/services/*.proto" />
-    <!--Proto3 Remove="../../lib/api/services/MarketDataRelay.proto" /-->
     <Proto3 Include="../../lib/md/protos/co2.proto" />
     <Proto3 Include="../../lib/domain_model/Security.proto" />
     <Proto3 Include="../../lib/sparkproto/SparkProtos.proto" />
     <Proto3 Update="../../lib/api/services/OutOfProcessURLListener.proto">
-      <!-- <CompileOptions>internal_access:</CompileOptions> -->
     </Proto3>
   </ItemGroup>
 </Project>

src/client/spark_api/utils/ProductExtensions.cs

@@ -0,0 +1,10 @@
+using Google.Protobuf;
+namespace bts.api {
+
+   public sealed partial class Product : ICustomDiagnosticMessage {
+      public string ToDiagnosticString() {
+         return $"{this.Symbol}:{this.Type}:{this.Exchange}";
+      }
+   }
+
+}

src/client/spark_core/spark_core.csproj

@@ -7,7 +7,8 @@
     <PlatformTarget>AnyCPU</PlatformTarget>
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
     <WarningsAsErrors />
-    <Version>1.0.0</Version>
+    <Version>$(major).$(minor).$(patch)</Version>
+    <Version Condition="'$(version)'=='..'">0.0.0</Version>
     <LangVersion>7.2</LangVersion>
   </PropertyGroup>
 

src/client/spark_excel/rtd/subscriptions/MDSubscribe.cs

@@ -66,9 +66,23 @@ namespace bts.excel.rtd
          if (string.IsNullOrEmpty(contract))
             return Observable.Return(new []{"Can't Parse Contract: " + contract ?? string.Empty});
 
-         Func<SecurityImmutable, bool> where = (x => x.Exchange == ven && x.Symbol == symbol && x.security_type() == sectype &&
-                                                x.Glbx.SecurityDesc == contract);
+         Func<SecurityImmutable, bool> desc = null;
+         switch(ven)
+         {
+            case Venue.Sim:
+            case Venue.Glbx:
+               desc = (x=> x.Glbx.SecurityDesc == contract);
+            break;
+            case Venue.Ice:
+               desc = (x=> x.Ice.SecurityDesc == contract);
+               break;
+            default:
+               desc = (x => true);
+            break;
+         }
 
+         Func<SecurityImmutable, bool> where = (x => x.Exchange == ven && x.Symbol == symbol && x.security_type() == sectype && desc(x) );
+                                                
          var stream = market_data(ven).subscribe_combined(where);
          return stream;
       }      

src/client/spark_excel/spark_excel.csproj

@@ -17,7 +17,8 @@
     <PlatformTarget>AnyCPU</PlatformTarget>
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
     <WarningsAsErrors />
-    <Version>1.0.0</Version>
+    <Version>$(major).$(minor).$(patch)</Version>
+    <Version Condition="'$(version)'=='..'">1.0.0</Version>
   </PropertyGroup>
   
   <ItemGroup>

src/client/sparkcmd/Cmd/ServiceTestCommand.cs

@@ -19,6 +19,8 @@ namespace bts {
       public bool admin { get; set; }
       [Option(nameof(latency))]
       public bool latency { get; set; }
+      [Option("md")]
+      public bool market_data { get; set; }
    }
 
    public class ServiceTest {
@@ -35,9 +37,24 @@ namespace bts {
          if (o.latency) {
             latency_test();
          }
+         if (o.market_data) {
+            market_data();
+         }
          return 0;
       }
 
+      private static void market_data() {
+         using (var ctx = new ServiceContext()) {
+            var md = ctx.locate<MarketDataService>();
+            md.subscribe_securities().Table().Subscribe(update => {
+               Console.WriteLine($"got {update.state.Count} securities");
+            });
+            while(true) {
+               Thread.Sleep(1000);
+            }
+         }
+      }
+
       static string try_get(string[] args, int index) {
          if (args.Length < index) {
             return args[index];

src/lib/CMakeDeps.cmake

@@ -1 +1 @@
-set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/service_helpers/PositionUtilities.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/BasePositionService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MBOService.proto;api/services/MarginService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/MoneyFileService.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/keycode.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/position_cfg.conf;config/defaults/span.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/keycode.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/position_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/span.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/sod.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MBOProcessor.cpp;md/MBOServiceServer.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityCache.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SecurityMapper.cpp;md/SecurityOracle.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/ice/ICEUtils.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/mbo_tests.cpp;md/test/md2_tests.cpp;md/test/md_test_utils.cpp;md/test/synthetic_book_merge_tests.cpp;md/vendor/OnixsUtils.cpp;mercury/actors/IPollableActor.cpp;mercury/broker/Broker.cpp;mercury/broker/BrokerProxy.cpp;mercury/broker/ILocalBroker.cpp;mercury/broker/IRemoteBroker.cpp;mercury/common/mercury_common.cpp;mercury/events/TimerEvent.cpp;mercury/executor/Executor.cpp;mercury/topics/Topic.cpp;metrics/metrics.cpp;metrics/system_metrics.cpp;orderman/actors/AsyncFactory.cpp;orderman/actors/ExecutionGateway.cpp;orderman/actors/FillStore.cpp;orLine truncated
+set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/service_helpers/PositionUtilities.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/BasePositionService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MBOService.proto;api/services/MarginService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/MarketDataService.proto;api/services/MoneyFileService.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/keycode.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/position_cfg.conf;config/defaults/span.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/keycode.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/position_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/span.proto;config/protos/spreader.proto;config/protos/subprocess.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/sod.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MBOProcessor.cpp;md/MBOServiceServer.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/Product.cpp;md/SecurityCache.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SecurityMapper.cpp;md/SecurityOracle.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/ice/ICEUtils.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/service/MarketDataServiceServer.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/mbo_tests.cpp;md/test/md2_tests.cpp;md/test/md_test_utils.cpp;md/test/synthetic_book_merge_tests.cpp;md/vendor/OnixsUtils.cpp;mercury/actors/IPollableActor.cpp;mercury/broker/Broker.cpp;mercury/broker/BrokerProxy.cpp;mercury/broker/ILocalBroker.cpp;mercury/broker/IRemoteBroker.cpp;mercury/common/mercury_common.cpp;mercury/events/TimerEvent.cpp;mercury/executor/Executor.cpp;mercury/topics/Topic.cpp;metrics/metrics.cpp;metricsLine truncated

src/lib/CMakeDeps_win.cmake

@@ -1 +1 @@
-set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/service_helpers/PositionUtilities.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/BasePositionService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MBOService.proto;api/services/MarginService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/MoneyFileService.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/keycode.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/position_cfg.conf;config/defaults/span.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/keycode.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/position_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/span.proto;config/protos/spreader.proto;config/protos/subprocess.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/sod.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MBOProcessor.cpp;md/MBOServiceServer.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityCache.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SecurityMapper.cpp;md/SecurityOracle.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/ice/ICEUtils.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/mbo_tests.cpp;md/test/md2_tests.cpp;md/test/md_test_utils.cpp;md/test/synthetic_book_merge_tests.cpp;md/vendor/OnixsUtils.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;utils/AuthSessionManager.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;Line truncated
+set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/service_helpers/PositionUtilities.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/BasePositionService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MBOService.proto;api/services/MarginService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/MarketDataService.proto;api/services/MoneyFileService.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/keycode.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/position_cfg.conf;config/defaults/span.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/keycode.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/position_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/span.proto;config/protos/spreader.proto;config/protos/subprocess.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/sod.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MBOProcessor.cpp;md/MBOServiceServer.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/Product.cpp;md/SecurityCache.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SecurityMapper.cpp;md/SecurityOracle.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/ice/ICEUtils.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/service/MarketDataServiceServer.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/mbo_tests.cpp;md/test/md2_tests.cpp;md/test/md_test_utils.cpp;md/test/synthetic_book_merge_tests.cpp;md/vendor/OnixsUtils.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;utils/AuthSessionManager.cpp;utils/AuthToken.cLine truncated

src/lib/api/services/MarketDataService.proto

@@ -0,0 +1,46 @@
+syntax="proto3";
+package bts.api;
+option csharp_namespace="bts.api";
+option cc_enable_arenas=true;
+import "lib/api/apioptions.proto";
+import "lib/api/apicore.proto";
+import "lib/domain_model/Security.proto";
+import "lib/md/protos/co2.proto";
+
+message ProductList {
+   repeated bts.api.Product products = 1;
+}
+
+// Should this cary int prices?
+message PriceEntry {
+   double price = 1;
+   uint32 direct = 10;
+   uint32 implied = 11;
+}
+
+message BookEntry {
+   uint32 key = 1 [(bts.api.key)=true];
+   repeated PriceEntry bid = 10;
+   repeated PriceEntry ask = 20;
+}
+
+message TickerEntry {
+   uint32 key = 1 [(bts.api.key)=true];
+   int64 time = 2;
+   double price = 3;
+   uint32 size = 4; 
+   uint32 total_volume = 5;
+   bts.co2.AggressorSide aggressor_side = 6;
+   bool from_spread = 7;
+   bool gap = 10;
+}
+
+
+service MarketDataService {
+    rpc get_products(bts.api.VoidMessage) returns (ProductList);
+    // stream all securities
+    rpc securities (bts.api.VoidMessage) returns (stream bts.api.Security);
+    rpc book (bts.api.Product) returns (stream BookEntry);
+    rpc trades(bts.api.VoidMessage) returns (stream TickerEntry);
+    // subscribe_misc(Product)
+}

src/lib/jetfix/codec/MessageTypes.h

@@ -49,6 +49,7 @@ enum class MessageTypes : uint32_t
    TestRequest                 = '1',
    TradingSessionStatus        = 'h',
    TradingSessionStatusRequest = 'g',
+   MDIncrementalUpdate         = 'X',
    ICETraderLogin              = ('U' << 16) + ('C' << 8) + 'G',
    ICETraderLoginResponse      = ('U' << 16) + ('C' << 8) + 'I',   
    ICETraderLogout             = ('U' << 16) + ('C' << 8) + 'H',   

src/lib/jetfix/store/FIXFileStore.cpp

@@ -18,8 +18,8 @@
 
 #include <fmt/format.h>
 #include <fstream>
-#include <thread>
 #include <regex>
+#include <thread>
 
 namespace bts {
 namespace jetfix {
@@ -114,16 +114,17 @@ std::string store_path(std::string base_path,
    return result.string();
 }
 
-bool is_valid_store_line(std::string const&line) {
-   std::regex rx("\\d\\d\\d\\d\\d\\d\\d\\d-"
-                 "\\d\\d:\\d\\d:\\d\\d\\."
-                 "\\d\\d\\d . 8=FIX\\.4\\.2.*");
-   return std::regex_match(line, rx);
+static std::regex store_line_rx("\\d\\d\\d\\d\\d\\d\\d\\d-"
+                                "\\d\\d:\\d\\d:\\d\\d\\."
+                                "\\d\\d\\d . 8=FIX\\.4\\.2.*");
+
+bool is_valid_store_line(std::string const &line) {
+   return std::regex_match(line, store_line_rx);
 }
 
 std::map<uint32_t, InMsgBuffer>
 load_replay(config::JetFIX::Connection const &cfg, std::string const &path,
-            uint32_t ) {
+            uint32_t) {
    std::map<uint32_t, InMsgBuffer> result;
    if (boost::filesystem::exists(path)) {
 
@@ -137,7 +138,7 @@ load_replay(config::JetFIX::Connection const &cfg, std::string const &path,
          if (!f.eof()) {
             if (!is_valid_store_line(line_buffer))
                continue;
-            
+
             if (line_buffer[22] == '<') {
                auto ts =
                    datetime::parse_FIX_timestamp(line_buffer.substr(0, 21));

src/lib/md/Product.cpp

@@ -0,0 +1,37 @@
+#ifndef _WIN32
+#include "Product.h"
+
+#include <boost/tokenizer.hpp>
+#include <vector>
+
+namespace bts {
+namespace api {
+std::string product_to_string(const api::Product& p) {
+   std::string buffer = p.symbol();
+   buffer += ":";
+   buffer += SecurityType_Name(p.type());
+   buffer += ":";
+   buffer += Venue_Name(p.exchange());
+   return buffer;
+}
+std::optional<api::Product> string_to_product(const std::string& s) {
+   boost::tokenizer<boost::char_separator<char>> tok { s };
+   std::vector<std::string> parts;
+   for (const auto& t : tok) {
+      parts.push_back(t);
+   }
+   if (parts.size() != 3) return std::nullopt;
+   SecurityType type = SecurityType::SECURITYTYPE_UNSET;
+   if (!SecurityType_Parse(parts[1], &type)) return std::nullopt;
+   Venue v = Venue::VENUE_UNSET;
+   if (!Venue_Parse(parts[2], &v)) return std::nullopt;
+   Product p;
+   p.set_symbol(parts[0]);
+   p.set_type(type);
+   p.set_exchange(v);
+   return {p};
+}
+
+
+}}//ns
+#endif

src/lib/md/Product.h

@@ -0,0 +1,13 @@
+#pragma once
+#include "lib/domain_model/Security.pb.h"
+#include "lib/utils/optional.h"
+#include <string>
+
+namespace bts {
+namespace api {
+
+std::string product_to_string(const api::Product&);
+std::optional<api::Product> string_to_product(const std::string&);
+
+
+} }

src/lib/md/service/MarketDataServiceServer.cpp

@@ -0,0 +1,116 @@
+#ifndef _WIN32
+#include "MarketDataServiceServer.h" 
+#include "lib/metrics/metrics.h"
+#include "lib/md/UserSymbol.h"
+#include "lib/md/get_symbols.h"
+#include "lib/md/Product.h"
+#include "lib/utils/AuthToken.h"
+
+#include <atomic>
+#include <set>
+
+
+namespace bts { 
+namespace md { 
+
+struct MarketDataServiceImpl {
+   MarketDataServiceImpl(api::MarketDataServiceBase::SecuritiesStatefulPublisher);
+   utils::Dispatcher& dispatcher;
+   api::MarketDataServiceBase::SecuritiesStatefulPublisher security_publisher;
+   std::atomic<uint32_t> sec_outstanding;
+   void add_security(const SecurityPtr& sec) {
+      ++sec_outstanding;
+      dispatcher.post([this,sec]() {
+         security_publisher.add_or_update(*sec);
+         auto flush = (--sec_outstanding) == 0;
+         if (flush) {
+            security_publisher.flush();
+         }
+      });
+   }
+};
+
+MarketDataServiceImpl::MarketDataServiceImpl(
+   api::MarketDataServiceBase::SecuritiesStatefulPublisher security_pub) :
+      dispatcher(utils::Dispatcher::get_named("MDService")),
+      security_publisher(security_pub),
+      sec_outstanding(0)
+{
+}
+
+MarketDataService::MarketDataService(
+   MDClient& md,
+   api::ServiceContext ctx,
+   metrics::MetricsContainer&) :
+      api::AuthenticatedMarketDataServiceBase(ctx)
+{
+   _me = std::make_shared<MarketDataServiceImpl>(make_securities_stateful_publisher());
+   md.current_consumer().on_security([me=_me](const SecurityPtr& sec, MDConsumerHandle) {
+      me->add_security(sec);
+   });
+}
+
+// This method duplicates MarketDataRelayServer::request_products()
+void MarketDataService::get_products(
+   const api::AuthToken&,
+   const ::bts::api::VoidMessage&,
+   const api::Response<::bts::api::ProductList>& reply) {
+
+      api::ProductList list;
+      std::set<md::UserSymbol> syms;
+      md::get_system_md_symbols(syms);
+      for(const auto& i : syms) {             
+         auto prod = list.add_products();
+         prod->set_exchange(i.exchange());
+         prod->set_type(i.type());
+         prod->set_symbol(i.symbol());
+      }
+      reply(list);
+}
+
+api::StreamAccessResponse authorized(bool is_authorized) {
+   api::StreamAccessResponse r;
+   r.set_authorized(is_authorized);
+   return r;
+}
+
+void MarketDataService::allow_securities(
+   const bts::api::AuthToken&, 
+   const std::string& qualifier, 
+   const api::Response<bts::api::StreamAccessResponse>& reply) {
+      // Non-empty qualifier is invalid and rejected
+      reply(authorized(qualifier.empty()));
+}
+
+void MarketDataService::allow_book(
+   const bts::api::AuthToken& token, 
+   const std::string& qualifier, 
+   const api::Response<bts::api::StreamAccessResponse>& reply) {
+      api::user::TokenPayload payload;
+      if (api::auth::decode_token(token, &payload)) {
+         auto product = api::string_to_product(qualifier);
+         if (product) {
+            // todo: start streaming
+            reply(authorized(true));
+         }
+      }
+      reply(authorized(false));
+}
+   
+void MarketDataService::allow_trades(
+   const bts::api::AuthToken& token, 
+   const std::string& qualifier, 
+   const api::Response<bts::api::StreamAccessResponse>& reply) {
+      api::user::TokenPayload payload;
+      if (api::auth::decode_token(token, &payload)) {
+         auto product = api::string_to_product(qualifier);
+         if (product) {
+            // todo: start streaming
+            reply(authorized(true));
+         }
+      }
+      reply(authorized(false));
+}
+
+}} //namespace 
+#endif

src/lib/md/service/MarketDataServiceServer.h

@@ -0,0 +1,43 @@
+#pragma once 
+#include "lib/utils/Dispatcher.h"
+#include "lib/api/public/ServiceContext.h"
+#include "lib/api/services/MarketDataService.server.pb.h"
+#include "lib/md/MDClient.h"
+#include <memory>
+
+namespace bts { 
+namespace metrics { class MetricsContainer; }
+namespace md { 
+
+struct MarketDataServiceImpl;
+
+class MarketDataService : public api::AuthenticatedMarketDataServiceBase {
+   std::shared_ptr<MarketDataServiceImpl> _me;
+public:
+   MarketDataService(
+      MDClient& md,
+      api::ServiceContext ctx,
+      metrics::MetricsContainer& stats);
+
+   void get_products(
+      const api::AuthToken&,
+      const ::bts::api::VoidMessage&,
+      const api::Response<::bts::api::ProductList>&) override;
+      
+   void allow_securities(
+      const bts::api::AuthToken&, 
+      const std::string& qualifier, 
+      const api::Response<bts::api::StreamAccessResponse>& reply) override;
+      
+   void allow_book(
+      const bts::api::AuthToken& token, 
+      const std::string& qualifier, 
+      const api::Response<bts::api::StreamAccessResponse>& reply) override;
+      
+   void allow_trades(
+      const bts::api::AuthToken& token, 
+      const std::string& qualifier, 
+      const api::Response<bts::api::StreamAccessResponse>& reply) override;
+};
+
+}} //namespace 

src/lib/orderman/actors/MDSource.h

@@ -31,9 +31,9 @@ class MDSource : public mercury::Actor, public mercury::IPollableActor {
  public:
    MDSource(md::MDConsumer &consumer, metrics::MetricsContainer &stats,
             trace::PerfLog &log)
-       : _handles(), _status_subs(), _md_consumer(consumer), 
+       : _handles(), _pending_handles(), _status_subs(), _md_consumer(consumer), 
          _book_update(), _ticker_update(), _status_update(),
-         _perf_log(log), _stats(stats), _loop_timer() {
+         _perf_log(log), _stats(stats), _loop_timer(), _counter(0) {
 
       _trades_buffer.Reserve(1000);
       auto cme_channels = md::get_configured_cme_channels();
@@ -169,6 +169,47 @@ class MDSource : public mercury::Actor, public mercury::IPollableActor {
       // Publish stats
       auto stop = utils::TimeStamp::now();
       _loop_timer->observe((start - stop).nanoseconds());
+
+      _counter += 1;
+      if (_counter % (2<<20) == 0) {
+         // Invoke periodic work every 2M calls on on_poll
+         // This is approximately 20-30 seconds
+         do_periodic_work();
+      }
+   }
+
+   void do_periodic_work() {
+      if (!_pending_handles.empty()) {
+         for (auto iter = _pending_handles.begin(); iter != _pending_handles.end(); ++iter) {
+            if (iter->handle->is_initialized()) {
+               auto handle = *(iter->handle);
+               LOG(OrderMan, Info, "Found md for {}. Pending handle removed.", handle.key());
+               switch(iter->type) {
+                  case MDRequestType::BOOK:
+                     _handles.push_back({utils::TimeStamp(), handle,
+                        get_external_event_channel(handle.get_usersymbol()),
+                        std::make_unique<std::string>(securities::descriptor(
+                              handle.get_normalized_security()))});
+                     break;
+                  case MDRequestType::STATUS:
+                     for (auto& h : _status_subs) {
+                        // no need to reset status
+                        if (h.handle.key() == handle.key()) break; 
+                     }
+                     _status_subs.push_back(
+                        StatusRecord{ handle, co2::TRADINGSTATUS_UNKNOWN });
+                  case MDRequestType::TICKER:
+                  // We aren't currently monitoring for pending ticker requests
+                  break;
+               }
+            }
+         }
+         _pending_handles.erase(
+            std::remove_if(_pending_handles.begin(),
+                           _pending_handles.end(),
+                           [](const auto& h) { return h.handle->is_initialized(); }),
+            _pending_handles.end());
+      }
    }
 
    void on_book_sub(SecurityKey sk,
@@ -184,14 +225,23 @@ class MDSource : public mercury::Actor, public mercury::IPollableActor {
 
       LOG(OrderMan, Warn,
           "Starting to publish book updates for security key: {}", k);
-      auto handle = _md_consumer.get_handle(k);
 
-      _handles.push_back({utils::TimeStamp(), handle,
-                          get_external_event_channel(handle.get_usersymbol()),
-                          std::make_unique<std::string>(securities::descriptor(
-                              handle.get_normalized_security()))});
-      if (handle.is_synthetic())
-         synthetic_count_metric.increment();
+      auto mhandle = _md_consumer.get_managed_handle(k);
+      if (mhandle->is_initialized()) {
+         auto handle = *mhandle;
+         _handles.push_back({utils::TimeStamp(), handle,
+                           get_external_event_channel(handle.get_usersymbol()),
+                           std::make_unique<std::string>(securities::descriptor(
+                                 handle.get_normalized_security()))});
+         if (handle.is_synthetic())
+            synthetic_count_metric.increment();
+      }
+      else {
+         // A pending handle is a handle for security we are reasonbly sure exists
+         // but we haven't yet found live market data for.
+         LOG(OrderMan, Warn, "No market data for {}. Pending handle added.", k);
+         _pending_handles.push_back({ mhandle, MDRequestType::BOOK });
+      }
    }
 
    const char *get_external_event_channel(const md::UserSymbol &user_symbol) {
@@ -205,7 +255,7 @@ class MDSource : public mercury::Actor, public mercury::IPollableActor {
                       metrics::CounterMetric<int> &channel_count_metric) {
 
       auto user_symbol = _md_consumer.get_handle(sk.value()).get_usersymbol();
-      for (auto &ch : _channels)
+      for (auto &ch : _channels) {
          if (ch.user_symbol == user_symbol) {
             auto it =
                 std::find_if(ch.securities.begin(), ch.securities.end(),
@@ -222,6 +272,7 @@ class MDSource : public mercury::Actor, public mercury::IPollableActor {
             }
             return;
          }
+      }
 
       // this is a channel we are not subscribed to
       channel_count_metric.increment();
@@ -250,8 +301,14 @@ class MDSource : public mercury::Actor, public mercury::IPollableActor {
             return;
          }
       }
-      auto handle = _md_consumer.get_handle(k);
-      _status_subs.push_back(StatusRecord{ handle, co2::TRADINGSTATUS_UNKNOWN });
+      auto handle = _md_consumer.get_managed_handle(k);
+      if (handle->is_initialized()) {
+         _status_subs.push_back(StatusRecord{ *handle, co2::TRADINGSTATUS_UNKNOWN });
+      }
+      else {
+         LOG(OrderMan, Warn, "No market data for {}. Pending handle added.", k);
+         _pending_handles.push_back({ handle, MDRequestType::STATUS });
+      }
    }
 
    struct HandleRecord {
@@ -261,6 +318,11 @@ class MDSource : public mercury::Actor, public mercury::IPollableActor {
       std::unique_ptr<std::string> symbol;
    };
    boost::container::small_vector<HandleRecord, 100> _handles;
+   struct PendingRecord {
+      std::shared_ptr<md::MDConsumerHandle> handle;
+      MDRequestType type;
+   };
+   boost::container::small_vector<PendingRecord, 4> _pending_handles;
 
    struct ChannelRecord {
       int64_t cursor;
@@ -285,6 +347,7 @@ class MDSource : public mercury::Actor, public mercury::IPollableActor {
    metrics::MetricsContainer &_stats;
    std::shared_ptr<metrics::HistogramMetric<30>> _loop_timer;
    std::map<md::UserSymbol, std::string> _symbol_to_channel;
+   uint64_t _counter;
 };
 } // namespace orderman
 } // namespace bts

src/lib/orderman/orders/algo/algo_orders/PyramidAlgo.h

@@ -6,7 +6,7 @@ namespace bts {
 namespace orderman {
 
 class ChildTemplate;
-class LegData;
+struct LegData;
 
 class PyramidAlgo : public IAlgoOrder {
 public:

src/lib/orderman/orders/algo/algo_orders/TASAlgo.h

@@ -6,7 +6,7 @@ namespace bts {
 namespace orderman {
 
 class ChildTemplate;
-class LegData;
+struct LegData;
 
 class TASAlgo : public IAlgoOrder {
  public:

src/sdk/Dockerfile

@@ -5,7 +5,6 @@ from bluetradingsystems/linux_docker
 # This lets us query an image to see when it was built
 ARG SDK_VERSION=0
 ARG SDK_BUILD_DATE=0
-ARG USERNAME=bts
 
 # configure build_examples
 env SDK_ROOT=/sdk \
@@ -13,13 +12,14 @@ env SDK_ROOT=/sdk \
     SDK_BUILD_DATE=$SDK_BUILD_DATE
 
 # add user bts
-RUN groupadd --gid 1000 $USERNAME && \
-    useradd -s /bin/bash --uid 1000 --gid 1000 -m $USERNAME && \
+RUN groupadd --gid 1000 bts && \
+    useradd -s /bin/bash --uid 1000 --gid 1000 -m bts && \
     mkdir -p /sparkbin/plugins && \
     mkdir /sparkdata && \
-    ln -s /sparkbin/ sdk/lib && \
-    echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME && \
-    chmod 0440 /etc/sudoers.d/$USERNAME && \
+    mkdir /sdk && \
+    ln -s /sparkbin/ /sdk/lib && \
+    echo bts ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/bts && \
+    chmod 0440 /etc/sudoers.d/bts && \
     chown -R bts.bts /sparkbin/plugins && \
     chown -R bts.bts /sparkdata 
 
@@ -28,6 +28,6 @@ COPY include/ /sdk/include/
 COPY lib/ /sparkbin/
 COPY externals /sdk/externals/
 COPY spark.conf /sparkdata/conf/
-COPY --chown=$USERNAME futures.coconut /sparkdata/data/replay/futures.coconut
+COPY --chown=bts futures.coconut /sparkdata/data/replay/futures.coconut
 
 

src/sdk/examples/algo_template/.devcontainer.json

@@ -2,7 +2,7 @@
    "name": "Spark sdk container",
    "image": "spark_sdk",
    "runArgs": [
-      "--shm-size", "1024m",
+      "--shm-size", "2048m",
       "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined",
       "-u", "bts"
    ],

src/sdk/examples/client_sdk/.devcontainer.json

@@ -0,0 +1,20 @@
+{
+   "name": "Spark sdk container",
+   "image": "spark_sdk",
+   "runArgs": [
+      "--shm-size", "2048m",
+      "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined",
+      "-u", "bts"
+   ],
+   "settings": {
+      "terminal.integrated.shell.linux": "/bin/bash",
+      "C_Cpp.default.cppStandard": "c++20"
+   },
+   // use 9000 to avoid conflict with typical bts dev setup
+   "appPort": [9000,9001],
+   "postCreateCommand": "",
+   "extensions": [
+      "ms-vscode.cpptools"
+   ]
+}
+

src/sdk/examples/client_sdk/.gitignore

@@ -0,0 +1 @@
+client_sdk_example

src/sdk/examples/client_sdk/.vscode/c_cpp_properties.json

@@ -0,0 +1,21 @@
+{
+    "configurations": [
+        {
+            "name": "Linux",
+            "includePath": [
+                "/usr/include",
+                "/usr/local/include",
+                "${workspaceFolder}/**",
+                "/sdk/include/",
+                "/sdk/include/spark",
+                "/sdk/externals/include"
+            ],
+            "defines": [],
+            "compilerPath": "/usr/bin/clang",
+            "cStandard": "c11",
+            "cppStandard": "c++17",
+            "intelliSenseMode": "clang-x64"
+        }
+    ],
+    "version": 4
+}

src/sdk/examples/client_sdk/.vscode/launch.json

@@ -0,0 +1,27 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "(gdb) Launch",
+            "type": "cppdbg",
+            "request": "launch",
+            "program": "/sparkbin/sparkd",
+            "args": ["run"],
+            "stopAtEntry": false,
+            "cwd": "/sparkbin",
+            "environment": [],
+            "externalConsole": false,
+            "MIMode": "gdb",
+            "setupCommands": [
+                {
+                    "description": "Enable pretty-printing for gdb",
+                    "text": "-enable-pretty-printing",
+                    "ignoreFailures": true
+                }
+            ]
+        }
+    ]
+}

src/sdk/examples/client_sdk/.vscode/tasks.json

@@ -0,0 +1,29 @@
+{
+    // See https://go.microsoft.com/fwlink/?LinkId=733558
+    // for the documentation about the tasks.json format
+    // variables: ${workspaceRoot}
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "build",
+            "type": "shell",
+            "command": "./build.sh",
+            "group": {
+                "kind": "build",
+                "isDefault": true
+            }
+        },
+        {
+            "label": "Start Sparkd",
+            "type": "shell",
+            "command": "cd /sparkbin; ./sparkd run",
+            "isBackground": true,
+            "problemMatcher": []
+        },
+        {
+            "label": "Stop Sparkd",
+            "type": "shell",
+            "command": "pkill -f sparkd"
+        }
+    ]
+}

src/sdk/examples/client_sdk/build.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+set -e
+
+SDK_ROOT=${SDK_ROOT:-../..}
+NAME=client_sdk_example
+
+g++ -isystem $SDK_ROOT/externals/include \
+    -I $SDK_ROOT/include \
+    -I $SDK_ROOT/include/spark \
+    -std=c++1z \
+    -fPIC \
+    -g \
+    -o $NAME \
+    $NAME.cpp \
+    -Wl,-rpath,/sparkbin\
+    -L $SDK_ROOT/lib  \
+    -l spark 
+
+

src/sdk/examples/client_sdk/client_sdk_example.cpp

@@ -0,0 +1,35 @@
+#include "lib/api/public/ServiceContext.h"
+#include "spark/lib/domain_model/Security.h"
+#include "lib/api/services/AdminService.client.pb.h"
+#include "lib/api/services/PositionService.client.pb.h"
+
+#include <string>
+#include <vector>
+#include <thread>
+
+using namespace bts::api;
+
+int main(int argc, char** argv) {
+   ServiceContext ctx;
+   auto admin_service = ctx.locate<admin::AdminService>();
+   auto res = admin_service.get_admin_data({});
+   if (res.success()) {
+      std::cout << "Users:" << std::endl;
+      const auto& data = *res.value();
+      for (const auto& u : data.users()) {
+         std::cout << u.id() << "\t" << u.name() << std::endl;
+      }
+   }
+
+   auto position_service = ctx.locate<position::PositionService>();
+   auto sub = position_service.subscribe_position_data().subscribe([](const auto& update) {
+      std::cout << "Position:" << std::endl;
+      std::cout << "Account\tSecurity\tday long\tday short";
+      for (const auto& a : update.all()) {
+         std::cout << a->account() << "\t" << a->security_id() << "\t" << a->day_long_pos() << "\t" << a->day_short_pos();
+      }
+   });
+
+   std::this_thread::sleep_for(std::chrono::minutes(5));
+   return 0;
+}

src/sdk/examples/dynamic_join_order/.devcontainer.json

@@ -2,7 +2,7 @@
    "name": "Spark sdk container",
    "image": "spark_sdk",
    "runArgs": [
-      "--shm-size", "1024m",
+      "--shm-size", "2048m",
       "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined",
       "-u", "bts"
    ],

src/sdk/sdk_spark.conf

@@ -23,7 +23,6 @@ Ports {
    value {
       config_port: 9000
       qbus_port: 9001
-      exchange_sim_fix_port: 9002
    }
 }
 

src/server/sparkd/CMakeDeps.cmake

@@ -1 +1 @@
-set(sources algo/AlgoDefinitionServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/FIXSim.cpp;cmds/Help.cpp;cmds/KeyCode.cpp;cmds/Md.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/RunSim.cpp;cmds/Span.cpp;cmds/Test.cpp;cmds/TestCeloxica.cpp;cmds/Web.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/MultipartFormDecoder.cpp;http/WebServer.cpp;http/tests/MultiPartFormTests.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/IMDPlatform.cpp;md/platforms/celoxica/CXListener.cpp;md/platforms/celoxica/CXMain.cpp;md/platforms/celoxica/tests/cx_tests.cpp;md/platforms/ice_onixs/ICEListener.cpp;md/platforms/ice_onixs/ICEMain.cpp;md/platforms/ice_onixs/tests/ice_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/MarginServiceServer.cpp;position/PositionAggregator.cpp;position/PositionMap.cpp;position/PositionServicePNLHelper.cpp;position/PositionServiceServer.cpp;position/import/BasePositionServiceServer.cpp;position/import/DownloadCommonStateMachine.cpp;position/import/DownloadWindow.cpp;position/import/MoneyFileServiceServer.cpp;position/import/parsing/gmi_parser.cpp;position/import/parsing/money_parser.cpp;position/import/parsing/parser_common.cpp;position/tests/PositionTests.cpp;position/tests/position_tests_internal.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceServer.cpp;user/AuthServiceServer.cpp;user/tests/admin_service_tests.cpp)
+set(sources algo/AlgoDefinitionServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/FIXSim.cpp;cmds/Help.cpp;cmds/KeyCode.cpp;cmds/Md.cpp;cmds/ParseHistoricalFIX.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/RunSim.cpp;cmds/Span.cpp;cmds/Test.cpp;cmds/TestCeloxica.cpp;cmds/Web.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/MultipartFormDecoder.cpp;http/WebServer.cpp;http/tests/MultiPartFormTests.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/IMDPlatform.cpp;md/platforms/celoxica/CXListener.cpp;md/platforms/celoxica/CXMain.cpp;md/platforms/celoxica/tests/cx_tests.cpp;md/platforms/ice_onixs/ICEListener.cpp;md/platforms/ice_onixs/ICEMain.cpp;md/platforms/ice_onixs/tests/ice_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/MarginServiceServer.cpp;position/PositionAggregator.cpp;position/PositionMap.cpp;position/PositionServicePNLHelper.cpp;position/PositionServiceServer.cpp;position/import/BasePositionServiceServer.cpp;position/import/DownloadCommonStateMachine.cpp;position/import/DownloadWindow.cpp;position/import/MoneyFileServiceServer.cpp;position/import/parsing/gmi_parser.cpp;position/import/parsing/money_parser.cpp;position/import/parsing/parser_common.cpp;position/tests/PositionTests.cpp;position/tests/position_tests_internal.cpp;subprocess/SubProcessRunner.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceServer.cpp;user/AuthServiceServer.cpp;user/tests/admin_service_tests.cpp)

src/server/sparkd/cmds/ParseHistoricalFIX.cpp

@@ -0,0 +1,416 @@
+#include "ParseHistoricalFIX.h"
+#include "lib/jetfix/codec/FIXMsg.h"
+#include "lib/jetfix/codec/FIXParser.h"
+#include "lib/jetfix/codec/Fields.h"
+#include "lib/jetfix/codec/Group.h"
+#include "lib/jetfix/codec/MessageTypes.h"
+#include "lib/utils/bts_chrono.h"
+
+#include <fstream>
+#include <unordered_map>
+
+namespace bts {
+namespace cmd {
+
+using namespace jetfix;
+
+typedef StringField<75> TradeDate;
+typedef IntField<34> SeqNum;
+typedef StringField<52> SendingTime;
+typedef StringField<60> TransactTime;
+typedef StringField<5799> MatchEventIndicator;
+
+typedef IntField<268> NoMDEntries;
+typedef StringField<279> MDUpateAction;
+typedef StringField<269> MDEntryType;
+typedef StringField<48> SecurityID;
+typedef StringField<55> Symbol;
+typedef StringField<83> ReptSeq;
+typedef FloatField<270> MDEntryPx;
+typedef IntField<271> MDEntrySize;
+typedef IntField<346> NumberOfOrders;
+typedef IntField<1023> MDPriceLevel;
+typedef StringField<286> OpenCloseSettlFlag;
+typedef StringField<731> SettlPriceType;
+typedef StringField<5797> AggressorSide;
+typedef StringField<37711> MDTradeEntryID;
+typedef StringField<5796> TradingReferenceDate;
+typedef StringField<1149> HighLimitPrice;
+typedef StringField<1148> LowLimitPrice;
+typedef StringField<1143> MaxPriceVariation;
+typedef StringField<1180> ApplID;
+typedef IntField<9633> ReferenceID;
+typedef IntField<32> LastQty;
+
+typedef IntField<37705> NoOrderIDEntries;
+typedef StringField<37> OrderID;
+typedef IntField<37707> MDOrderPriority;
+typedef IntField<37706> MDDisplayQty;
+typedef StringField<37708> OrderUpdateAction;
+
+typedef Group<NoOrderIDEntries, OrderID, MDDisplayQty, MDOrderPriority,
+              OrderUpdateAction, ReferenceID, LastQty>
+    OrderIDEntries;
+
+typedef Group<NoMDEntries, MDUpateAction, MDEntryType, SecurityID, Symbol,
+              ReptSeq, MDEntryPx, MDEntrySize, NumberOfOrders, MDPriceLevel,
+              OpenCloseSettlFlag, SettlPriceType, AggressorSide, MDTradeEntryID,
+              TradingReferenceDate, HighLimitPrice, LowLimitPrice,
+              MaxPriceVariation, ApplID, ReferenceID, LastQty, OrderID,
+              MDOrderPriority, MDDisplayQty>
+    MDEntries;
+
+typedef FIXMsg<MessageTypes::MDIncrementalUpdate, TradeDate, SeqNum,
+               SendingTime, TransactTime, MatchEventIndicator, MDEntries,
+               OrderIDEntries>
+    MDIncrementalUpdate;
+
+std::string to_str(std::string const &s) { return s; }
+
+template <class V> std::string to_str(V const &v) { return std::to_string(v); }
+
+template <class F, class G> std::string get_if_valid(G const &update) {
+   if (update.template has<F>())
+      return to_str(update.template get<F>());
+   else
+      return "";
+}
+
+void print_msg_headers(std::ostream &out) {
+   out << "MsgID,"
+       << "TradeDate,"
+       << "SeqNum,"
+       << "SendingTime,"
+       << "TransactTime,"
+       << "MatchEventIndicator,"
+       << "NoMDEntries,"
+       << "NoOrderIDEntries\n";
+}
+
+void print_msgs(size_t msg_idx, MDIncrementalUpdate const &line,
+                std::ostream &out) {
+   line.get<SeqNum>();
+   out << msg_idx << "," << get_if_valid<TradeDate>(line) << ","
+       << get_if_valid<SeqNum>(line) << "," << get_if_valid<SendingTime>(line)
+       << "," << get_if_valid<TransactTime>(line) << ","
+       << get_if_valid<MatchEventIndicator>(line) << ","
+       << (line.has<MDEntries>() ? line.get<MDEntries>().size() : 0) << ","
+       << (line.has<OrderIDEntries>() ? line.get<OrderIDEntries>().size() : 0)
+       << "\n";
+}
+
+struct PriceSize {
+   double price = 0;
+   int32_t size = 0;
+};
+
+struct Book {
+   Book() {
+      book[0].resize(10);
+      book[1].resize(10);
+   }
+   std::string symbol;
+   std::string transact_time;
+   std::string send_time;
+   uint32_t seqnum;
+   std::array<std::vector<PriceSize>, 2> book;
+};
+
+size_t n_printed_books = 2;
+
+void print_book_side(std::vector<PriceSize> const &side,
+                     std::ostream &book_stream) {
+   for (size_t i = 0; i < n_printed_books; i++)
+      if (i < side.size())
+         book_stream << "," << side[i].price << "," << side[i].size;
+      else
+         book_stream << "," << 0 << "," << 0;
+}
+
+void print_book_header(std::ostream &book_stream) {
+   book_stream << "Symbol,SendingTime,TransactTime, SeqnNum";
+   for (size_t i = 0; i < n_printed_books; i++)
+      book_stream << ",bid_p_" << i << ",bid_sz_" << i;
+   for (size_t i = 0; i < n_printed_books; i++)
+      book_stream << ",ask_p_" << i << ",ask_sz_" << i;
+   book_stream << ",\n";
+}
+
+void print_book(Book &b, std::ostream &book_stream) {
+   book_stream << b.symbol << "," << b.send_time << "," << b.transact_time
+               << "," << b.seqnum;
+   print_book_side(b.book[0], book_stream);
+   print_book_side(b.book[1], book_stream);
+   book_stream << "\n";
+}
+
+void update_and_maybe_print_book(MDIncrementalUpdate const &update,
+                                 std::unordered_map<std::string, Book> &books,
+                                 std::ostream &book_stream) {
+   std::set<Book *> to_update;
+   if (update.has<MDEntries>()) {
+      auto &md_entries = update.get<MDEntries>();
+      for (size_t i = 0; i < md_entries.size(); i++) {
+         auto &entry = md_entries[i];
+         if (entry.has<MDPriceLevel>()) {
+            if (entry.get<MDEntryType>() == "0" ||
+                entry.get<MDEntryType>() == "1") {
+               auto side = entry.get<MDEntryType>()[0] - '0';
+               auto &sym = entry.get<Symbol>();
+               auto &book = books[sym];
+               to_update.insert(&book);
+               auto &book_side = book.book[side];
+               book.symbol = sym;
+               book.transact_time = update.get<TransactTime>();
+               book.send_time = update.get<SendingTime>();
+               book.transact_time = update.get<TransactTime>();
+               book.seqnum = update.get<SeqNum>();
+               auto level = entry.get<MDPriceLevel>() - 1;
+
+               if (entry.get<MDUpateAction>() == "0") // new
+                  book_side.insert(
+                      book_side.begin() + level,
+                      {entry.get<MDEntryPx>(), entry.get<MDEntrySize>()});
+               else if (entry.get<MDUpateAction>() == "1")
+                  book_side[level] = {entry.get<MDEntryPx>(),
+                                      entry.get<MDEntrySize>()};
+               else
+                  book_side.erase(book_side.begin() + level);
+            }
+         }
+      }
+      auto indicator = update.get<MatchEventIndicator>();
+      if (indicator[5] == '1')
+         for (auto b : to_update)
+            print_book(*b, book_stream);
+   }
+}
+void print_trades_header(std::ostream &trade_stream) {
+   trade_stream << "Symbol,SendingTime,TransactTime,MDEntryPx,MDEntrySize,"
+                   "NumberOfOrders,AggressorSide,MDTradeEntryID,AgressorOID,"
+                   "RestingOID,PartialQty";
+   trade_stream << "\n";
+}
+
+struct TradeSummaryItem {
+   std::string symbol;
+   std::string sending_time;
+   std::string transact_time;
+   double md_entry_px;
+   uint32_t md_entry_size;
+   uint32_t n_orders;
+   std::string agressor_side;
+   std::string md_trade_entry_id;
+};
+
+struct TradeOrder {
+   std::string order_id;
+   uint32_t last_qty;
+};
+
+struct TradeSummaryEvent {
+   std::vector<TradeSummaryItem> md_entries;
+   std::vector<TradeOrder> orderid_entries;
+};
+
+void print_trades(TradeSummaryEvent &event, std::ostream &trade_stream) {
+   size_t current_order = 0;
+   for (auto &trade : event.md_entries) {
+      for (auto i = current_order; i < current_order + trade.n_orders; i++) {
+         trade_stream << trade.symbol << "," << trade.sending_time << ","
+                      << trade.transact_time << "," << trade.md_entry_px << ","
+                      << trade.md_entry_size << "," << trade.n_orders << ","
+                      << trade.agressor_side << "," << trade.md_trade_entry_id
+                      << "," << event.orderid_entries[current_order].order_id
+                      << "," << event.orderid_entries[i].order_id << ","
+                      << event.orderid_entries[i].last_qty << "\n";
+      }
+      current_order += trade.n_orders;
+   }
+}
+
+void update_and_maybe_print_trades(MDIncrementalUpdate const &update,
+                                   TradeSummaryEvent &event,
+                                   std::ostream &trade_stream) {
+   if (update.has<MDEntries>()) {
+      auto &md_entries = update.get<MDEntries>();
+      bool is_trade_summary = false;
+      for (size_t i = 0; i < md_entries.size(); i++) {
+
+         auto &entry = md_entries[i];
+         if (entry.get<MDEntryType>() == "2") {
+            is_trade_summary = true;
+            TradeSummaryItem item;
+            item.symbol = entry.get<Symbol>();
+            item.sending_time = update.get<SendingTime>();
+            item.transact_time = update.get<TransactTime>();
+            item.md_entry_px = entry.get<MDEntryPx>();
+            item.md_entry_size = entry.get<MDEntrySize>();
+            item.n_orders = entry.get<NumberOfOrders>();
+            item.agressor_side = entry.get<AggressorSide>();
+            item.md_trade_entry_id = entry.get<MDTradeEntryID>();
+            event.md_entries.emplace_back(std::move(item));
+         }
+
+         if (is_trade_summary) {
+            if (update.has<OrderIDEntries>()) {
+               auto &oid_entries = update.get<OrderIDEntries>();
+               for (size_t i = 0; i < oid_entries.size(); i++) {
+                  auto &oid = oid_entries[i];
+                  TradeOrder order_id_entry;
+                  order_id_entry.order_id = oid.get<OrderID>();
+                  order_id_entry.last_qty = oid.get<LastQty>();
+                  event.orderid_entries.emplace_back(std::move(order_id_entry));
+               }
+            }
+            auto indicator = update.get<MatchEventIndicator>();
+            if (indicator[7] == '1') {
+               print_trades(event, trade_stream);
+               event.md_entries.clear();
+               event.orderid_entries.clear();
+            }
+         }
+      }
+   }
+}
+
+void print_orders_header(std::ostream &orders_stream) {
+   orders_stream << "Symbol,"
+                 << "SendingTime,"
+                 << "TransactTime,"
+                 << "MDEntryPx,"
+                 << "OrderID,"
+                 << "MDDisplayQty,"
+                 << "MDOrderPriority,"
+                 << "OrderUpdateAction,"
+                 << "Side"
+                 << "\n";
+}
+
+struct Order {
+   std::string sending_time;
+   std::string transact_time;
+   std::string side;
+   
+   std::string order_id;
+   std::string order_update_action;
+   uint32_t md_order_priority;
+   uint32_t display_qty;
+
+   std::string symbol;
+   double md_entry_px;
+};
+
+void print_order(Order const& order, std::ostream &orders_stream) {
+   orders_stream << order.symbol << ","
+                 << order.sending_time << ","
+                 << order.transact_time << ","
+                 << order.md_entry_px << ","
+                 << order.order_id << ","
+                 << order.display_qty << ","
+                 << order.md_order_priority << ","
+                 << order.order_update_action << ","
+                 << order.side << "\n";
+}
+
+
+void print_orders(MDIncrementalUpdate const &update,std::ostream &orders_stream) {
+   if (update.has<MDEntries>()) {
+      auto &md_entries = update.get<MDEntries>();
+      auto & first_entry = md_entries[0];
+
+      if (first_entry.get<MDEntryType> () == "0" ||
+          first_entry.get<MDEntryType> () == "1") {
+
+         Order order;
+         order.sending_time = update.get<SendingTime>();
+         order.transact_time = update.get<TransactTime>();
+         order.side = first_entry.get<MDEntryType> ();
+         
+         if (first_entry.has<MDPriceLevel>()) {
+            // combined template https://www.cmegroup.com/confluence/display/EPICSANDBOX/Incremental+Refresh+SBE+Template+Book+Processing
+            if (update.has<OrderIDEntries>()) {
+               auto &oid_entries = update.get<OrderIDEntries>();
+               for (size_t i = 0; i < oid_entries.size(); i++) {
+
+                  auto &oid = oid_entries[i];
+                  order.order_id = oid.get<OrderID>();
+                  order.order_update_action = oid.get<OrderUpdateAction>();
+                  order.md_order_priority = oid.get<MDOrderPriority>();
+                  order.display_qty = oid.get<MDDisplayQty>();
+                  
+                  auto &md_entry = update.get<MDEntries>()[oid.get<ReferenceID>()-1];
+                  order.symbol = md_entry.get<Symbol>();
+                  order.md_entry_px = md_entry.get<MDEntryPx>();
+                  print_order(order, orders_stream);
+               }
+            }
+         }
+         else {
+            // MBO only template
+            for (size_t i = 0; i < md_entries.size(); i++) {
+               auto &entry = md_entries[i]; 
+               order.order_id = entry.get<OrderID>();
+               order.order_update_action = entry.get<MDUpateAction>();
+               order.md_order_priority = entry.get<MDOrderPriority>();
+               order.display_qty = entry.get<MDDisplayQty>();
+               order.symbol = entry.get<Symbol>();
+               order.md_entry_px = entry.get<MDEntryPx>();
+               print_order(order,orders_stream);
+            }
+         }
+      }
+   }
+}
+
+int ParseHistoricalFIX::execute(std::vector<std::string> const &args) {
+   std::ifstream input(args[0]);
+   std::ofstream books_stream(args[0] + "_books.csv");
+   std::ofstream msgs_stream(args[0] + "_msgs.csv");
+   std::ofstream trades_stream(args[0] + "_trades.csv");
+   std::ofstream orders_stream(args[0] + "_orders.csv");
+
+   std::unordered_map<std::string, Book> books;
+   TradeSummaryEvent current_trade_summary;
+
+   MDIncrementalUpdate update;
+   FIXParser<MDIncrementalUpdate> parser;
+
+   std::string line;
+   size_t c = 0;
+   auto ok = 0;
+   auto start = Timer::now();
+
+   print_book_header(books_stream);
+   print_msg_headers(msgs_stream);
+   print_trades_header(trades_stream);
+   print_orders_header(orders_stream);
+      
+   while (std::getline(input, line)) {
+      c++;
+      auto parsed_ptr = parser.parse(InMsgBuffer(
+          line.data(), line.size(), std::shared_ptr<std::string>{}));
+      if (parsed_ptr) {
+         auto msg =
+             std::dynamic_pointer_cast<const MDIncrementalUpdate>(parsed_ptr);
+         update_and_maybe_print_book(*msg, books, books_stream);
+         print_msgs(c, *msg, msgs_stream);
+         update_and_maybe_print_trades(*msg, current_trade_summary,
+                                       trades_stream);
+         print_orders(*msg, orders_stream);
+      }
+      ok += (bool)(parsed_ptr);
+   }
+   auto sec_duration = sec_diff(Timer::now(), start);
+   std::cout << (int)(c / 1000 / sec_duration) << " Km/sec parsed: " << ok
+             << " out of " << c << " time: " << sec_duration << " secs\n";
+   return 0;
+}
+
+std::string ParseHistoricalFIX::help() {
+   return "ParseHistoricalFIX <input> will produce 2 files, "
+          "<input>_mdentries.csv and <input>order_entries.csv";
+}
+
+} // namespace cmd
+} // namespace bts

src/server/sparkd/cmds/ParseHistoricalFIX.h

@@ -0,0 +1,17 @@
+#pragma once
+
+#include "ICmd.h"
+
+namespace bts {
+namespace cmd {
+
+class ParseHistoricalFIX : public ICmd 
+{
+public:
+   int execute(std::vector<std::string> const &args);
+   std::string help ();
+};
+
+}}
+
+

src/server/sparkd/cmds/Run.cpp

@@ -16,6 +16,7 @@
 #include "lib/md/MDClient.h"
 #include "lib/md/SyntheticSecurityService.h"
 #include "lib/md/relay/QueryServer.h"
+#include "lib/md/service/MarketDataServiceServer.h"
 #include "lib/metrics/metrics.h"
 #include "lib/metrics/system_metrics.h"
 #include "lib/orderman/order_factory/DynamicOrderRegistry.h"
@@ -83,6 +84,7 @@ int Run::execute(std::vector<std::string> const &) {
 
    md::MBOServiceServer mboserver(sc, utils::Dispatcher::background(),
                                   mdclient);
+   md::MarketDataService mdserver(mdclient, sc, stats);
 
    // initialize order plugins
    orderman::DynamicOrderRegistry dyn_order_registry;

src/server/sparkd/main.cpp

@@ -12,6 +12,7 @@
 #include "server/sparkd/cmds/Help.h"
 #include "server/sparkd/cmds/KeyCode.h"
 #include "server/sparkd/cmds/Md.h"
+#include "server/sparkd/cmds/ParseHistoricalFIX.h"
 #include "server/sparkd/cmds/PrintHedgeOrders.h"
 #include "server/sparkd/cmds/QuickTest.h"
 #include "server/sparkd/cmds/Run.h"
@@ -41,6 +42,7 @@ int main(int argc, char **argv) {
    REGISTER(FIXSim);
    REGISTER(Span);
    REGISTER(Web);
+   REGISTER(ParseHistoricalFIX);
    REGISTER(PrintHedgeOrders);
    REGISTER(AdminReport);
    END_CMD;

src/web/CMakeDeps.cmake

@@ -1 +1 @@
-set(sources ../../externals/linux/include/google/protobuf/descriptor.proto;../../externals/linux/include/google/protobuf/timestamp.proto;../lib/api/apicore.proto;../lib/api/apigateway.proto;../lib/api/apioptions.proto;../lib/api/services/AdminService.proto;../lib/api/services/AlgoDefinitionService.proto;../lib/api/services/AlgoService.proto;../lib/api/services/ApiUnitTestService.proto;../lib/api/services/AuthService.proto;../lib/api/services/BasePositionService.proto;../lib/api/services/ErrorReportService.proto;../lib/api/services/HelloService.proto;../lib/api/services/MBOService.proto;../lib/api/services/MarginService.proto;../lib/api/services/MarketDataAuxiliaryService.proto;../lib/api/services/MarketDataRelay.proto;../lib/api/services/MoneyFileService.proto;../lib/api/services/OrderService.proto;../lib/api/services/OutOfProcessURLListener.proto;../lib/api/services/PersistenceService.proto;../lib/api/services/PositionService.proto;../lib/api/services/ShellDiagnosticService.proto;../lib/api/services/ShellLatencyService.proto;../lib/api/services/SyntheticSecurityService.proto;../lib/domain_model/Security.proto;../lib/md/protos/co2.proto;../lib/sparkproto/SparkProtos.proto;api/build.sh;api/node_modules/exit/test/fixtures/create-files.sh;api/node_modules/protobufjs/cli/node_modules/acorn-jsx/node_modules/acorn/bin/update_authors.sh;api/node_modules/watch/release.sh;api/quiet_build.sh;api/src/DynamicService.ts;api/src/ServiceContext.ts;api/src/bts_assert.ts;ui/src/Access.fs;ui/src/Accounts/ClipLimits.fs;ui/src/Accounts/ProductLimits.fs;ui/src/Accounts/SecurityLimits.fs;ui/src/Accounts/State.fs;ui/src/Accounts/Types.fs;ui/src/Accounts/View.fs;ui/src/App.fs;ui/src/Cmd.fs;ui/src/Controls/Button.fs;ui/src/Controls/DataGrid.fs;ui/src/Controls/Fields.fs;ui/src/Controls/Modal.fs;ui/src/Controls/ReactGrid.fs;ui/src/Controls/Table.fs;ui/src/Controls/nbsp.fs;ui/src/Global.fs;ui/src/Imports.fs;ui/src/Lens.fs;ui/src/Login/Login.fs;ui/src/Nav.fs;ui/src/State.fs;ui/src/Types.fs;ui/src/Users/State.fs;ui/src/Users/Types.fs;ui/src/Users/View.fs;ui/src/Util/EnumUtils.fs;ui/src/Util/JSMap.fs;ui/src/Util/StreamUtils.fs;ui/src/obj/Debug/netstandard2.0/ui.AssemblyInfo.fs)
+set(sources ../../externals/linux/include/google/protobuf/descriptor.proto;../../externals/linux/include/google/protobuf/timestamp.proto;../lib/api/apicore.proto;../lib/api/apigateway.proto;../lib/api/apioptions.proto;../lib/api/services/AdminService.proto;../lib/api/services/AlgoDefinitionService.proto;../lib/api/services/AlgoService.proto;../lib/api/services/ApiUnitTestService.proto;../lib/api/services/AuthService.proto;../lib/api/services/BasePositionService.proto;../lib/api/services/ErrorReportService.proto;../lib/api/services/HelloService.proto;../lib/api/services/MBOService.proto;../lib/api/services/MarginService.proto;../lib/api/services/MarketDataAuxiliaryService.proto;../lib/api/services/MarketDataRelay.proto;../lib/api/services/MarketDataService.proto;../lib/api/services/MoneyFileService.proto;../lib/api/services/OrderService.proto;../lib/api/services/OutOfProcessURLListener.proto;../lib/api/services/PersistenceService.proto;../lib/api/services/PositionService.proto;../lib/api/services/ShellDiagnosticService.proto;../lib/api/services/ShellLatencyService.proto;../lib/api/services/SyntheticSecurityService.proto;../lib/domain_model/Security.proto;../lib/md/protos/co2.proto;../lib/sparkproto/SparkProtos.proto;api/build.sh;api/node_modules/exit/test/fixtures/create-files.sh;api/node_modules/object.assign/test.sh;api/node_modules/protobufjs/cli/node_modules/acorn-jsx/node_modules/acorn/bin/update_authors.sh;api/node_modules/watch/release.sh;api/quiet_build.sh;api/src/DynamicService.ts;api/src/ServiceContext.ts;api/src/bts_assert.ts;ui/src/Access.fs;ui/src/Accounts/ClipLimits.fs;ui/src/Accounts/ProductLimits.fs;ui/src/Accounts/SecurityLimits.fs;ui/src/Accounts/State.fs;ui/src/Accounts/Types.fs;ui/src/Accounts/View.fs;ui/src/App.fs;ui/src/Cmd.fs;ui/src/Controls/Button.fs;ui/src/Controls/DataGrid.fs;ui/src/Controls/Fields.fs;ui/src/Controls/Modal.fs;ui/src/Controls/ReactGrid.fs;ui/src/Controls/Table.fs;ui/src/Controls/nbsp.fs;ui/src/Global.fs;ui/src/Imports.fs;ui/src/Lens.fs;ui/src/Login/Login.fs;ui/src/Nav.fs;ui/src/State.fs;ui/src/Types.fs;ui/src/Users/State.fs;ui/src/Users/Types.fs;ui/src/Users/View.fs;ui/src/Util/EnumUtils.fs;ui/src/Util/JSMap.fs;ui/src/Util/StreamUtils.fs;ui/src/obj/Debug/netstandard2.0/ui.AssemblyInfo.fs)

src/web/ui/src/Login/Login.fs

@@ -65,6 +65,12 @@ let root model dispatch =
                               img [ Src sparkImg ]
                               ]
                             h1 [ ClassName "is-size-1" ] [ span [] [str "Spark Admin"]]
+                            (match Fable.Import.Browser.navigator.userAgent.Contains("Trident") with
+                            | false -> span [][]
+                            | true ->
+                                div [ Class "notification is-danger" ]
+                                    [   str "This browser is not supported. Saving changes may have unexpected results." ]
+                            )
                           ]
                           form [ OnSubmit (fun e ->
                             e.preventDefault()

tools/spark_vm_shell.sh

@@ -25,4 +25,6 @@ docker run \
        -t bluetradingsystems/linux_docker \
        /bin/bash -c "(sudo /usr/sbin/sshd) & (cd ~ && $DIR/../externals/linux/bin/unison -socket 8080) & /bin/bash --login"
 
+find /etc/ssh/ssh_host_* |xargs -I{} sudo docker cp {} bts:/etc/ssh/
+docker exec bts sudo service ssh restart