Merge branch 'master' into prod

This was crashing the spreader.

src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesRowVM.cs

@@ -1,6 +1,7 @@
 using bts.api;
 using bts.co2;
 using bts.utils;
+using bts.utils.csv;
 using mbts;
 using mbts.spark;
 using mbts.ui.controls;
@@ -49,6 +50,18 @@ namespace bts.shell.time_and_sales {
       public void raise_invalidated() =>
          invalidated?.Invoke();
 
+      public static System.Collections.Specialized.OrderedDictionary csv_ordered_properties =>
+            CsvHelper.create_ordering_maintainer(
+               typeof(TimeAndSalesRowVM),
+               new string[] {
+                  nameof(security),
+                  nameof(time),
+                  nameof(price),
+                  nameof(size),
+                  nameof(aggressor_side),
+                  nameof(security_key),
+               });
+
       public bool Equals(TimeAndSalesRowVM other) =>
          other != null &&
          row_key == other.row_key;

src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesWindow.cs

@@ -2,6 +2,7 @@
 using bts.shell.fills;
 using bts.spark;
 using bts.view;
+using bts.utils.csv;
 using mbts;
 using mbts.co2;
 using mbts.persist;
@@ -124,6 +125,8 @@ namespace bts.shell.time_and_sales {
             show_in_context_menu = _can_pause_updates
          };
          _commands = new SparkCommand[] {
+            // csv export
+            new SparkCommandCommand("Export to csv", Icons.Export, FileDialogCommand.export_csv(() => _vm.rows.ToCsv(TimeAndSalesRowVM.csv_ordered_properties), "TaS")),
             // ladder
             SparkCommands.ShowLadderAutoLookup(SparkCommands.KeyGestures.ShowLadder)
                .complete(current_sec_key_src, sec_keys => sec_keys.Count > 0),

src/client/spark/Modules/Trading/Fills/FillsVM.cs

@@ -5,6 +5,7 @@ using bts.collections;
 using bts.shell.fills;
 using bts.spark;
 using bts.utils;
+using bts.utils.csv;
 using mbts.collections;
 using mbts.persist;
 using mbts.spark.filters;
@@ -333,6 +334,27 @@ namespace mbts.spark {
          public override bool Equals(object obj) => Equals(obj as Row);
          public override int GetHashCode() => _fill.FillKey.GetHashCode() ^ _order.Key.GetHashCode();
 
+         public static System.Collections.Specialized.OrderedDictionary csv_ordered_properties =>
+            CsvHelper.create_ordering_maintainer(typeof(Row),
+               new string[] {
+                  nameof(security),
+                  nameof(order_type),
+                  nameof(root_algo_display_name),
+                  nameof(side),
+                  nameof(fill_time),
+                  nameof(user),
+                  nameof(account),
+                  nameof(exchange),
+                  nameof(time_in_force),
+                  nameof(size),
+                  nameof(price),
+                  nameof(fill_kind),
+                  nameof(fill_status),
+                  nameof(fill_key),
+                  nameof(order_key),
+                  nameof(security_key)
+               });
+
          public event Action invalidated;
          void IRowVM.raise_invalidated() => invalidated?.Invoke();
       }

src/client/spark/Modules/Trading/Fills/FillsWindow.cs

@@ -4,6 +4,7 @@ using bts.shell;
 using bts.shell.fills;
 using bts.spark;
 using bts.view;
+using bts.utils.csv;
 using mbts.persist;
 using mbts.ui;
 using mbts.ui.settings;
@@ -61,6 +62,7 @@ namespace mbts.spark {
          _view = new FillsView { DataContext = _vm };
 
          _commands = new List<SparkCommand> {
+            new SparkCommandCommand("Export to csv", Icons.Export, FileDialogCommand.export_csv(() => _vm.rows.ToCsv(FillsVM.Row.csv_ordered_properties), "Fills")),
             // wrap the grid settings view factory in a dialogCommand here since the vm is destroyed/recreated when the security criteria changes
             SparkCommands.window_settings_command(new DialogCommand(() => {
                var vm = new ShellSettingsDialogVM<GridSettings, SimpleGridSettingsVM>(

src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs

@@ -4,6 +4,7 @@ using bts.api.orderman;
 using bts.collections;
 using bts.spark;
 using bts.utils;
+using bts.utils.csv;
 using mbts.ui;
 using mbts.ui.controls;
 using System;
@@ -11,6 +12,7 @@ using System.Collections.Generic;
 using System.ComponentModel;
 using System.Reactive.Linq;
 using System.Runtime.CompilerServices;
+using System.Linq;
 using System.Windows.Input;
 using System.Windows.Media;
 using Columns = mbts.spark.OrderBookGridView.ColumnKeys;
@@ -60,6 +62,35 @@ namespace mbts.spark {
       public event Action<(OrderSecurityRootPair oldval, OrderSecurityRootPair newval)> updated;
       public event PropertyChangedEventHandler PropertyChanged;
 
+      public static System.Collections.Specialized.OrderedDictionary csv_ordered_properties =>
+            CsvHelper.create_ordering_maintainer(
+               typeof(OrderBookRowVM), 
+               new string[] {
+                  nameof(security_text),
+                  nameof(symbol),
+                  nameof(time_created),
+                  nameof(last_change),
+                  nameof(user),
+                  nameof(account),
+                  nameof(exchange),
+                  nameof(time_in_force),
+                  nameof(order_type),
+                  nameof(root_algo_display_name),
+                  nameof(order_sub_type),
+                  nameof(side),
+                  nameof(quantity),
+                  nameof(filled_quantity),
+                  nameof(price),
+                  nameof(avg_price),
+                  nameof(last_price),
+                  nameof(piq),
+                  nameof(status),
+                  // last message (could contains commas, so skipping)
+                  nameof(security_key),
+                  nameof(order_key),
+                  nameof(parent_order_key),
+               });
+
       // Core
       public OrderSecurityRootPair order_tuple { get; private set; }
       public OrderRecordImmutable order => order_tuple.order.order;

src/client/spark/Modules/Trading/OrderBook/OrderBookWindow.cs

@@ -6,6 +6,7 @@ using bts.shell.fills;
 using bts.shell.orderbook;
 using bts.spark;
 using bts.view;
+using bts.utils.csv;
 using mbts.persist;
 using mbts.spark.utilities;
 using mbts.ui;
@@ -231,6 +232,7 @@ namespace mbts.spark {
             toggle_collapse_all_command,
             toggle_expand_all_command,
             show_grid_settings_command,
+            new SparkCommandCommand("Export to csv", Icons.Export, FileDialogCommand.export_csv(() => _vm.rows.ToCsv(OrderBookRowVM.csv_ordered_properties), "OrderBook")),
             // hidden cmds (i.e., keyboard shortcuts at window level)
             hidden_cmd(new SparkCommandCommand(OrderBookView.CommandDescriptions.ModifyOrders, null, _vm.modify_selected_orders_cmd, OrderBookView.KeyboardShortcuts.ModifyOrders)),
             hidden_cmd(new SparkCommandCommand(OrderBookView.CommandDescriptions.RevertOrders, null, _vm.revert_selected_orders_cmd, OrderBookView.KeyboardShortcuts.RevertOrders)),

src/client/spark/view/Extensions/Command/FileDialogCommand.cs

@@ -64,6 +64,25 @@ namespace mbts.ui {
             }
          }, can_execute == null ? null : new Func<object, bool>(_ => can_execute()));
 
+      public static IDialogCommand export_csv(
+         Func<string> get_file_content,
+         string source_window_name) =>
+         new AnonymousFileDialogCommand<object>(_ =>
+            _create_save_file_dialog(
+               Environment.GetFolderPath(Environment.SpecialFolder.Desktop),
+               $"BTS_Spark_{source_window_name}_{DateTime.Now.ToString("yyyyMMdd_HHmm")}.csv",
+               new[] { ("CSV (comma delimited)", new string[] { "csv" }) }
+            ),
+            (_, file) => {
+               try {
+                  File.WriteAllText(file, get_file_content.Invoke());
+               } catch (Exception ex) {
+                  BTSMessageBox.Show($"Error exporting data to csv: {ex}.", "CSV Export Error", BTSMessageBoxButtons.OK, System.Windows.MessageBoxImage.Error, true, false, true);
+               }
+            },
+            // we are always able to perform an export, even when no rows (it will just be a header)
+            (_) => true);
+
       static TDialog _create_file_dialog<TDialog>(string initial_dir = null, IReadOnlyList<(string name, string[] extensions)> filter_items = null) where TDialog : FileDialog, new() {
          var rv = new TDialog();
          if (!string.IsNullOrEmpty(initial_dir))

src/client/spark_core/utils/CsvHelper.cs

@@ -0,0 +1,52 @@
+using System;
+using System.IO;
+using System.Collections.Generic;
+using System.Collections.Specialized;
+using System.Linq;
+using System.Reflection;
+
+namespace bts.utils.csv {
+   public static class CsvHelper {
+
+      public static string ToCsv<T>(this IEnumerable<T> items, OrderedDictionary properties = null) where T : class {
+         var output = "";
+         var delimiter = ",";
+         if (properties == null) {
+            properties = new OrderedDictionary();
+            foreach (var pi in typeof(T).GetProperties()) {
+               properties.Add(pi.Name, pi);
+            }
+         }
+
+         using (var sw = new StringWriter()) {
+            var dict_values = properties.Values.OfType<PropertyInfo>();
+            var header = dict_values.Select(n => n.Name).Aggregate((first, second) => first + delimiter + second);
+            sw.WriteLine(header);
+            foreach (var i in items) {
+               var row = dict_values.Select(x => x.GetValue(i, null))
+                  .Select(x => x == null ? "null" : x.ToString())
+                  .Aggregate((first, second) => first + delimiter + second);
+               sw.WriteLine(row);
+            }
+            output = sw.ToString();
+         }
+         return output;
+      }
+
+      public static OrderedDictionary create_ordering_maintainer(Type t, string[] property_names_in_order) {
+         OrderedDictionary od = new OrderedDictionary();
+         var props = t.GetProperties();
+         foreach (var n in property_names_in_order) {
+            var query = props.Where(pi => pi.Name == n);
+            if (query.Any()) {
+               var property_info = query.First();
+               if (!od.Contains(property_info)) {
+                  od.Add(n, property_info);
+               }
+            }
+         }
+         return od;
+      }
+
+   }
+}

src/data_files/CMakeDeps.cmake

@@ -1 +1 @@
-set(sources files/cert_secure_logon.txt;files/futures.coconut.xz;files/onixs_ice/ConnectivityConfiguration.APITest.xml;files/onixs_ice/ConnectivityConfiguration.APITest.xml.orig;files/onixs_ice/ConnectivityConfiguration.PerfTest.xml;files/onixs_ice/ConnectivityConfiguration.Production.xml;files/perf_trace/compiled.css;files/perf_trace/index.html;files/perf_trace/jquery-3.3.1.min.js;files/perf_trace/main.js;files/perf_trace/perf_log.sql;files/perf_trace/perf_trace_analysis.sql;files/perf_trace/plotly-latest.min.js)
+set(sources files/audit_ilink3.py;files/audit_upload.sh;files/audit_uploader.py;files/cert_secure_logon.txt;files/example.conf;files/futures.coconut.xz;files/onixs_ice/ConnectivityConfiguration.APITest.xml;files/onixs_ice/ConnectivityConfiguration.APITest.xml.orig;files/onixs_ice/ConnectivityConfiguration.PerfTest.xml;files/onixs_ice/ConnectivityConfiguration.Production.xml;files/perf_trace/compiled.css;files/perf_trace/index.html;files/perf_trace/jquery-3.3.1.min.js;files/perf_trace/main.js;files/perf_trace/perf_log.sql;files/perf_trace/perf_trace_analysis.sql;files/perf_trace/plotly-latest.min.js)

src/data_files/files/audit_ilink3.py

@@ -0,0 +1,405 @@
+import json
+import csv
+import datetime, time
+import argparse, os
+
+
+def valid_file(f):
+    if not os.path.exists(f):
+        # Argparse uses the ArgumentTypeError to give a rejection message like:
+        # error: argument input: x does not exist
+        raise argparse.ArgumentTypeError("{0} does not exist".format(f))
+    return f
+
+def valid_date(s):
+    try:
+        return datetime.datetime.strptime(s, "%Y%m%d")
+    except ValueError:
+        raise argparse.ArgumentTypeError("Not a valid date: '{0}'.".format(s))
+
+def print_headers():
+    # From https://www.cmegroup.com/confluence/display/EPICSANDBOX/iLink+3+-+Minimum+Acceptable+Audit+Trail+Elements+-+Data+Definitions+and+Validation+Rules
+    line_to_write = "Sending Timestamps,Message Direction,Operator ID,Session ID,Manual Order Indicator,Message Type,Instrument,Order Flow ID,\
+CME Globex Order ID,Client Order ID,Order Request ID,Buy/Sell Indicator,Quantity,Limit Price,Stop Price,Order Type,Order Qualifier,IFM Flag,\
+Display Quantity,Minimum Quantity,Country of Origin,Fill Price,Fill Quantity,Cumulative Quantity,Remaining Quantity,Aggressor Flag,Source of Cancellation,\
+Reject Reason,CME Globex Message ID,Cross ID,Quote Request ID,Message Quote ID,Quote Set ID,Quote Entry ID,Bid Price,Bid Size,Offer Price,Offer Size,\
+Processed Entries,Party Details List Request ID,ListUpdateAction,Self-Matching Prevention ID,Self-Match Prevention Instr,Customer Type Indicator,\
+AvgPxGroupID,Average Px Ind,CmtaGiveUpCD,Origin,Clearing Trade Price Type,Customer Handling Instr,Executing Firm ID,Account Number,Take Up Firm,Take up Account"
+
+    with open (args.output_file, 'w') as the_file:
+                    the_file.write(line_to_write)
+                    the_file.write("\n")
+
+def set_blanks():
+    return ""
+
+def get_value(json_key):
+    prices = ["Price", "LastPx", "StopPx"]
+    if json_key in json_data:
+        if json_key in prices:
+            x = json_data[json_key]
+            return x['mantissa']/100000000000
+        else:
+            return json_data[json_key]
+    else:
+        return ""
+
+
+# Column 1  - Sending Timestamps
+def SendingTimeEpoch(epochtime):   #1
+    #TIME = time.strftime('%Y%m%d-%H:%M:%S.{}'.format(epochtime/1000000000), time.gmtime(epochtime/1000000000))
+    TIME = datetime.datetime.fromtimestamp(int(epochtime) / 1e9).strftime('%Y%m%d-%H:%M:%S.%f')
+    return TIME
+
+# Column 2 - Message Direction
+def direction(whichway):
+    if whichway == '<':    
+        DIRECTION = "FROM CME"
+    else:                       
+        DIRECTION = "TO CME"     
+    return DIRECTION
+
+# Column 3 - Operator ID
+def SenderID(sender):
+    if "SenderID" in json_data:
+        SENDER = json_data["SenderID"] 
+        return SENDER
+    else:
+        return SESSIONID
+
+
+# Column 4 - Session ID
+
+# Column 5 - Manual Order Indicator
+def ManualOrderIndicator(manual):
+    if "ManualOrderIndicator" in json_data:
+        if manual == "Manual":
+            return "1"
+        else:
+            return "0"
+
+# Column 6 - Message Type
+def MessageType(msg_type,orderstatus):
+    if msg_type == "NewOrderSingle514":
+       return "D" 
+    elif msg_type == "ExecutionReportNew522":
+        return "8--0"
+    if "ExecutionReportTrade" in msg_type:
+        if orderstatus == "PartiallyFilled":
+            return "8--1"
+        else:
+            return "8--2"
+    elif msg_type == "ExecutionReportCancel534":
+        return "8--4"
+    elif msg_type == "ExecutionReportModify531":
+        return "8--5"
+    elif msg_type == "ExecutionReportCancel534":
+        return "8--H"
+    elif msg_type == "ExecutionReportReject523":
+        return "8--8"
+    elif msg_type == "OrderCancelReplaceReject536":
+        return "9--2"
+    elif msg_type == "OrderCancelReject535":
+        return "9--1"
+    elif msg_type == "PartyDetailsDefinitionRequest518":
+        return "CX"
+    elif msg_type == "PartyDetailsDefinitionRequestAck519":
+        return "CY"
+    elif msg_type == "OrderCancelRequest516":
+        return "F"
+    elif msg_type == "OrderCancelReplaceRequest515":
+        return "G"
+    
+    
+
+# Column 12 - Buy-Sell Indicator/Side
+def Side(indicator):
+    if "Side" in json_data:
+        if indicator == "Buy":
+            return "1"
+        else:
+            return "2"
+
+# Column 16 - Order Type/OrdType
+def OrdType(ordertype):
+    if "OrdType" in json_data:
+        if ordertype == "MarketWithProtection" or ordertype == "MarketwithProtection":
+            return "1"
+        elif ordertype == "Limit":
+            return "2"
+        elif ordertype == "StopWithProtection" or ordertype == "StopwithProtection":
+            return "3"
+        elif ordertype == "StopLimit":
+            return "4"
+        elif ordertype == "MarketLimit":
+            return "K"
+        else:
+            return ""
+
+# Column 17 - Order Qualifier/TimeInForce
+def TimeInForce(orderqual):
+    if "TimeInForce" in json_data:
+        if orderqual == "Day":
+            return "0"
+        elif orderqual == "GTC":
+            return "1"
+        elif orderqual == "FAK":
+            return "3"
+        elif orderqual == "GTD":
+            return "6"
+        else:
+            return ""
+
+# Column 18 IFM Flag/OFMOverride
+def OFMOverride(imfflag):
+    if "OFMOverride" in json_data:
+        if imfflag == "Enabled":
+            return "1"
+        else:
+            return "0"
+
+# Column 19 Display Quantity/DisplayQty
+def DisplayQty(qty):
+    if "DisplayQty" in json_data:
+        if str(qty) == "0":
+            return ""
+        elif str(qty) == "4294967295":
+            return ""
+        else:
+            return qty
+
+
+# Column 26 - Aggressor Flag/AggressorIndicator
+def AggressorIndicator(aggressor):
+    if "AggressorIndicator" in json_data:
+        if aggressor == "True":
+            return "1"
+        else:
+            return "0"
+
+# Column 27 - Source of Cancellation/ExecRestatementReason
+def ExecRestatementReason(sourcecancel):
+    if "ExecRestatementReason" in json_data:
+        if sourcecancel == "Exchange":
+            return "8"
+        elif sourcecancel == "CancelledNotBest":
+            return "9"
+        elif sourcecancel == "CancelOnDisconnect":
+            return "100"
+        elif sourcecancel == "CancelRestingDueToSelfMatchPrevention":
+            return "103"
+        elif sourcecancel == "CancelFromCMEGlobexCreditControlsViolation":
+            return "104"
+        elif sourcecancel == "CancelFromCMEOne":
+            return "105"
+        elif sourcecancel == "CancelFromRiskManagementAPI":
+            return "106"
+        elif sourcecancel == "CancelAggressingDueToSelfMatchPrevention":
+            return "107"
+        elif sourcecancel == "CancelDueToVolatilityQuoteOptionLessThanMinOrderSize":
+            return "108"
+        else:
+            return ""
+
+# Column 43 - Self-Match Prevention Instr/SelfMatchPreventionInstruction
+def SelfMatchPreventionInstruction(instr):
+    if "SelfMatchPreventionInstruction" in json_data:
+        if instr == "CancelOldest":
+            return "O"
+        elif instr == "CancelNewest":
+            return "N"
+        else:
+            return ""
+
+# Column 44 - Customer Type Indicator/CustOrderCapacity
+def CustOrderCapacity(custord):
+    if "CustOrderCapacity" in json_data:
+        if custord == "NULL_VALUE":
+            return ""
+        elif custord == "Membertradingfortheirownaccount":
+            return "1"
+        elif custord == "ClearingFirmtradingforitsproprietaryaccount":
+            return "2"
+        elif custord == "Membertradingforanothermember":
+            return "3"
+        else:
+            return "4" 
+
+# Column 46 - Average Px Ind/AvgPxInd
+def AvgPxInd(avgpxindicator):
+    if "AvgPxInd" in json_data:
+        if avgpxindicator == "NULL_VALUE":
+            return ""
+        else:
+            return avgpxindicator
+
+# Column 48 - Origin/ClearingAccountType
+def ClearingAccountType(accounttype):
+    if "ClearingAccountType" in json_data:
+        if accounttype == "Customer":
+            return "0"
+        elif accounttype == "Firm":
+            return "1"
+        else:
+            return ""
+
+# Column 49 - Clearing Trade Price Type/ClearingTradePriceType
+def ClearingTradePriceType(clrtrdprice):
+    if "ClearingTradePriceType" in json_data:
+        if clrtrdprice == "NULL_VALUE":
+            return "0"
+        else:
+            return "3"
+
+# Column 50 - Customer Handling Instr/CustOrderHandlingInst
+def CustOrderHandlingInst(custorderinst):
+    if "CustOrderHandlingInst" in json_data:
+        if custorderinst == "ClientElectronic":
+            return "C"
+        elif custorderinst == "TEST":
+            return "D"
+        else:
+            return ""
+
+# Column 51 and 52 - Getting the Firm and Account Number
+def partyid(partydetailid,column):
+    for x in partydetailid:
+        role = x['PartyDetailRole']
+        id = x['PartyDetailID']
+        if column == "51" and role == "ExecutingFirm":
+            return id
+        if column == "52" and role == "CustomerAccount":
+            return id
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument("json_log", help="the json log[.gz] you wish to process", type=valid_file)
+parser.add_argument("output_file", help="the destination csv file you wish to write")
+parser.add_argument("-d", "--date", help="the date you wish to parse from the log, specified as YYYYmmdd",
+                    default=datetime.datetime.today().strftime("%Y%m%d"), type=valid_date)
+args = parser.parse_args()
+
+print_headers()
+
+list_types = ['NewOrderSingle514', 'OrderCancelReplaceRequest515', 'OrderCancelRequest516', 'PartyDetailsDefinitionRequest518', 'PartyDetailsDefinitionRequestAck519', \
+        'ExecutionReportNew522', 'ExecutionReportReject523', 'ExecutionReportTradeOutright525', 'ExecutionReportTradeSpread526', 'ExecutionReportTradeSpreadLeg527', \
+        'ExecutionReportModify531', 'ExecutionReportCancel534', 'OrderCancelReject535', 'OrderCancelReplaceReject536', ]
+
+# Note that the timedeltas specificed in trade_day_start and trade_day_end
+# are based on Chicago times; as such, this script should run on a machine
+# set to a Chicago timezone. A future impromovement would be to determine
+# the current machine's timezone and determine the appropriate offset
+# versus Chicago time, but for now we assume this will run on a Chicago
+# tz machine.
+input_as_datetime = args.date
+trade_day_start = input_as_datetime - datetime.timedelta(days=1) + datetime.timedelta(hours=16, minutes=30)
+trade_day_utc_start = time.gmtime((time.mktime(trade_day_start.timetuple())))
+trade_day_end = input_as_datetime + datetime.timedelta(hours=16)
+trade_day_utc_end = time.gmtime((time.mktime(trade_day_end.timetuple())))
+
+file_opener = None
+if args.json_log.endswith(".gz"):
+    import gzip
+    file_opener = gzip.open
+else:
+    file_opener = open
+
+
+#inputFile = open(fileInput) #open json file
+#outputFile = open(fileOutput, 'w') #load csv file
+# Get Session
+with file_opener(args.json_log) as json_file:
+    for line in json_file:
+        data = json.loads(line)
+        json_data = data['msg_json']
+        if "Session" in json_data:
+            SESSIONID = json_data["Session"]
+
+#inputFile = open(fileInput) #open json file
+#outputFile = open(fileOutput, 'w') #load csv file
+with file_opener(args.json_log) as json_file:
+    for line in json_file:
+        data = json.loads(line)
+        line_timestamp = time.strptime(data['time'], "%Y%m%d-%H:%M:%S.%f")
+        if trade_day_utc_start <= line_timestamp <= trade_day_utc_end:
+            json_data = data['msg_json']
+            NAME = json_data["Name"]
+            if NAME in list_types:
+                EXECUTING_FIRM_ID = ""
+                ACCOUNT_NUMBER = ""
+                DIRECTION = direction(data['direction'])
+                # Set Blanks
+                blank_json = {'QuoteReqID': '', 'QuoteID': '', 'QuoteSetID': '', 'QuoteEntryID': '', 'BidPx': '', 'BidSize': '', 'OfferPx': '', \
+                'OfferSize': '', 'NoProcessedEntries': '', 'ListUpdateAction': '', 'MinQty': '', 'AvgPxGroupID': '', \
+                'CmtaGiveUpCD': '', 'TAKE_UP_FIRM': '', 'TAKE_UP_ACCOUNT': '',}
+
+                # No Data Processing
+                straight_json = {'SecurityID': '', 'ClOrdID': '', 'OrderID': '', 'OrderRequestID': '', 'OrderQty': '', \
+                'Location': '', 'LastQty': '', 'CumQty': '', 'LeavesQty': '', 'Text': '', 'SideTradeID': '', 'CrossID': '', 'SelfMatchPreventionID': '', \
+                'Price': '', 'StopPx': '', 'LastPx': '', 'PartyDetailsListReqID': '', 'OrdStatus': ''}
+                        
+                # Processing Data
+                process_json = {'SendingTimeEpoch': '', 'ManualOrderIndicator': '', 'SenderID': '', 'Side': '', 'OrdType': '', 'TimeInForce': '', \
+                'OFMOverride': '', 'AggressorIndicator': '', 'ExecRestatementReason': '', 'SelfMatchPreventionInstruction': '', 'CustOrderCapacity': '', \
+                'AvgPxInd': '', 'ClearingAccountType': '', 'ClearingTradePriceType': '', 'CustOrderHandlingInst': '', 'DisplayQty': ''}
+                
+                for d in straight_json:
+                    test = get_value(d)
+                    if test != "None" or test != "":
+                        if "," in str(test):
+                            test = test.replace(",",";")
+                        if "4294967295" in str(test):
+                            test = ""
+                        if "us;" in str(test):
+                            test = test.upper()
+                        if d == "SelfMatchPreventionID" and len(str(test)) > 8:
+                            test = ""
+                        straight_json[d] = test
+                    else:
+                        straight_json[d] = ''
+                        
+                for d in process_json:
+                    if d in json_data:
+                        function = locals()[d]
+                        test = function(json_data[d])
+                        if test != "None":
+                            process_json[d] = test
+                        else:
+                            process_json[d] = ''
+                    elif d == "SenderID":
+                        function = locals()[d]
+                        test = function(SenderID)
+                        if test != "None":
+                            process_json[d] = test
+
+                # Party Message Fields
+                # Column 51 - Executing Firm ID
+                if "NoPartyDetails" in json_data:
+                    EXECUTING_FIRM_ID = partyid(json_data['NoPartyDetails'], "51")
+                    # Column 52 - Party Account Number
+                    ACCOUNT_NUMBER = partyid(json_data['NoPartyDetails'], "52")
+                
+                line_to_write = process_json['SendingTimeEpoch'] + "," + DIRECTION + "," + process_json['SenderID'] + "," + SESSIONID + "," + process_json['ManualOrderIndicator'] + "," + \
+                MessageType(NAME,straight_json['OrdStatus']) + "," + str(straight_json['SecurityID']) + "," + str(straight_json['ClOrdID']) + "," + str(straight_json['OrderID']) + "," + str(straight_json['ClOrdID']) + "," + \
+                str(straight_json['OrderRequestID']) + "," + process_json['Side'] + "," + str(straight_json['OrderQty']) + "," + str(straight_json['Price']) + "," + \
+                str(straight_json['StopPx'])  + "," + process_json['OrdType'] + "," + process_json['TimeInForce'] + "," + process_json['OFMOverride'] + "," + str(process_json['DisplayQty']) + "," + \
+                blank_json['MinQty'] + "," + straight_json['Location'] + "," + str(straight_json['LastPx']) + "," + str(straight_json['LastQty']) + "," + str(straight_json['CumQty']) + "," + \
+                str(straight_json['LeavesQty']) + "," + process_json['AggressorIndicator'] + "," + process_json['ExecRestatementReason'] + "," + str(straight_json['Text']) + "," + \
+                str(straight_json['SideTradeID']) + "," + str(straight_json['CrossID']) + "," + blank_json['QuoteReqID'] + "," + blank_json['QuoteID'] + "," + blank_json['QuoteSetID'] + "," + \
+                blank_json['QuoteEntryID'] + "," + blank_json['BidPx'] + "," + blank_json['BidSize'] + "," + blank_json['OfferPx'] + "," + blank_json['OfferSize'] + "," + \
+                blank_json['NoProcessedEntries'] + "," + str(straight_json['PartyDetailsListReqID']) + "," + blank_json['ListUpdateAction'] + "," + str(straight_json['SelfMatchPreventionID']) + "," + \
+                process_json['SelfMatchPreventionInstruction'] + "," + process_json['CustOrderCapacity'] + "," + blank_json['AvgPxGroupID'] + "," + process_json['AvgPxInd'] + "," + \
+                blank_json['CmtaGiveUpCD'] + "," + str(process_json['ClearingAccountType']) + "," + process_json['ClearingTradePriceType'] + "," + process_json['CustOrderHandlingInst'] + "," + \
+                EXECUTING_FIRM_ID + "," + str(ACCOUNT_NUMBER) + "," + blank_json['TAKE_UP_FIRM'] + "," + blank_json['TAKE_UP_ACCOUNT']
+
+                
+                #print(line_to_write)
+
+                with open (args.output_file, 'a') as the_file:
+                    the_file.write(line_to_write)
+                    the_file.write("\n")
+        
+
+
+
+ 

src/data_files/files/audit_upload.sh

@@ -0,0 +1,73 @@
+#/bin/bash
+
+function help_and_exit()
+{
+  echo
+  echo Usage: $0 [option]... config_file
+  echo 
+  echo Can also be used in cron like the following
+  echo 40 16 * * 1-5 /usr/bin/docker exec spark_10000 bash /sparkbin/data_files/audit_upload.sh /sparkdata/data/audit_uploads/example.conf
+  echo example.conf included.  The location of the conf file should be relative to the docker container, not the host
+  echo
+  echo Options are as follows:
+  echo "     -d|--date YYYYMMDD              Example is 20180731"
+  echo "  Help: "
+  echo "     -h|--help                       Display this message."
+  echo
+  exit 1
+}
+
+DATE=$(date +%Y%m%d)
+ORIG_DIR=$(pwd)
+ERROR_LEVEL="0"
+
+while getopts "d:h" option; do
+    case $option in
+        d) DATE=$OPTARG ;;
+        h) help_and_exit;;
+        \?) echo "Invalid option: -$OPTARG" >&2;
+        exit 1;;
+    esac
+done
+
+shift $((OPTIND-1))
+
+CONFIG_FILE=$1
+
+declare -A conf_vars; while read line; do conf_vars[${line%=*}]=${line#*=}; done <$CONFIG_FILE
+
+SESSION_ID=${conf_vars[session_id]}
+FCM=${conf_vars[fcm]}
+CLIENT=${conf_vars[client]}
+
+cd /sparkbin/data_files
+
+for SESSION in $(ls /sparkdata/data/ilink3/); do
+   if [[ $SESSION == *$SESSION_ID* ]]; then
+      echo "Session is $SESSION"
+      SESSION_DAY=$(date +%Y-%m-%d)
+      if [[ $SESSION == "cme_"* ]]; then 
+         SESSION_DAY=$(date -d "$DATE -$(date -d $DATE +%u) days" +"%Y-%m-%d")
+         AUDIT_SCRIPT="audit_ilink3.py"
+      fi
+      echo $SESSION_DAY
+      if [[ ! -d  /sparkdata/data/ilink3/$SESSION/audit_logs ]]; then
+         mkdir -p /sparkdata/data/ilink3/$SESSION/audit_logs
+      fi
+      if [[ -d /sparkdata/data/ilink3/$SESSION/$SESSION_DAY ]]; then
+         echo "Starting the creation of the audit log" 
+         python $AUDIT_SCRIPT -d $DATE /sparkdata/data/ilink3/$SESSION/$SESSION_DAY/log /sparkdata/data/ilink3/$SESSION/audit_logs/"$FCM"_audit_"$SESSION"_"$DATE".csv
+      fi
+      if [[ ! -d /sparkdata/data/md_audit ]]; then
+         mkdir /sparkdata/data/md_audit
+      fi
+      if [[ -d /sparkdata/data/md_audit ]]; then
+         # Running the MD Audit csv generation
+         cd /sparkbin
+         ./sparkd adminreport md > /sparkdata/data/md_audit/"$FCM"_md_"$CLIENT"_"$DATE".csv
+         cd /sparkbin/data_files
+      fi
+      echo "Uploading to the (s)FTP Server"
+      python audit_uploader.py --config_file="$CONFIG_FILE" --datestamp $DATE --session="$SESSION" --client="$CLIENT" --fcm="$FCM" 
+   fi 
+done

src/data_files/files/audit_uploader.py

@@ -0,0 +1,150 @@
+import getopt, sys, datetime
+import paramiko, ftplib
+
+#########################################################
+#                  Commandline Args                     #
+#########################################################
+
+def usage():
+    print "-------------------------------------------------------"
+    print "-Usage:                                               -"
+    print "-------------------------------------------------------"
+    print "-    --config_file= : Path to upload config file      -"
+    print "-                      -if no path is specified look  -"
+    print "-                       in the working directory.     -"    
+    print "-                                                     -"
+    print "-    --help         : Print this message              -"
+    print "-                                                     -"
+    print "-    --datestamp=   : Optional, defaults to today     -"
+    print "-                       -replaces *datestamp in the   -"
+    print "-                        the upload config file       -"
+    print "-                                                     -"
+    print "-    --fcm          : Optional, spark input           -"
+    print "-                                                     -"
+    print "-    --session      : Optional, spark jf session      -"
+    print "-------------------------------------------------------"
+
+fcm = ''
+session = ''
+
+try:
+   options, arguments = getopt.getopt(sys.argv[1:],"h",['config_file=', 'datestamp=', 'help', 'fcm=', 'client=', 'session='])
+except getopt.GetoptError, err:
+    print str(err)
+    usage()
+    sys.exit(2)
+
+for option, value in options:
+    if option in ("-h","--h","-help","--help"):
+        usage()
+        sys.exit()
+    elif option in ("-config_file","--config_file"):
+        config_file = value
+    elif option in ("-datestamp","--datestamp"):
+        datestamp = value
+    elif option in ("-fcm","--fcm"):
+        fcm = value
+    elif option in ("-client","--client"):
+        client = value
+    elif option in ("-session","--session"):
+        session = value
+
+for argument in arguments:
+    if argument in ("h", "help"):
+        usage()
+        sys.exit()
+    else:
+        print "Unrecognized commandline argument. Type help for usage."
+        sys.exit()
+
+if len(options) <= 0:
+    print "Requires a config file to run."
+    usage()
+    sys.exit()
+
+
+#########################################################
+#              Read Config Function                     #
+#########################################################
+
+def read_config_file(config_file, datestamp=datetime.datetime.today().strftime('%Y%m%d'), fcm=fcm, session=session):
+        '''
+        Parse our config file that gets passed in as a commandline parameter.
+        All config keys must be of the form key=value. White space is ignored.
+        All config keys must be seperated by a newline character.
+        Called during initialization.
+        '''
+        upload_config = {}
+        config_file = open(config_file, 'r')        
+        config_file = config_file.read()
+        for config in config_file.split("\n"):
+            if '=' in config:
+                cleaned_key = config.split('=')[0].strip().upper()
+                cleaned_value = config.split('=')[1].strip()
+                if '*datestamp*' in cleaned_value:
+                    cleaned_value = cleaned_value.replace('*datestamp*',datestamp)
+                if '*fcm*' in cleaned_value:
+                    cleaned_value = cleaned_value.replace('*fcm*',fcm)
+                if '*client*' in cleaned_value:
+                    cleaned_value = cleaned_value.replace('*client*',client)
+                if '*session*' in cleaned_value:
+                    cleaned_value = cleaned_value.replace('*session*',session)                    
+                upload_config[cleaned_key] = cleaned_value
+            else:
+                pass
+        return upload_config   
+
+#########################################################
+#             FTP File Transfer Function                #
+#########################################################
+def putfile_ftp(file, put_path, site, user, password, port=21, verbose=True):
+    if verbose: print 'Uploading', file
+    local = open(file, 'rb')    
+    remote = ftplib.FTP(site)   
+    remote.login(user, password)         
+    remote.storbinary(put_path, local, 1024)
+    remote.quit()
+    local.close()
+    if verbose: print 'Upload done.'
+
+#########################################################
+#            SFTP File Transfer Function                #
+#########################################################
+def putfile_sftp(file, put_path, site, username, password, port=22, verbose=True):
+    transport = paramiko.Transport((site, port))
+    transport.connect(username = username, password = password)
+    sftp = paramiko.SFTPClient.from_transport(transport)
+    sftp.put(file, put_path)
+    sftp.close()
+    transport.close()
+
+
+#########################################################
+#                       Main                            #
+#########################################################
+#if datestamp != None:
+#    upload_config = read_config_file(config_file, datestamp)
+#else:
+#    upload_config = read_config_file(config_file)
+
+try:
+    datestamp
+except NameError:
+    upload_config = read_config_file(config_file)
+else:
+    upload_config = read_config_file(config_file, datestamp)
+
+
+
+print upload_config
+
+if upload_config["PROTOCOL_PUT"] in ['FTP','ftp']:
+    print "Protocol = FTP"
+    putfile_ftp(upload_config["GET_ILINK3_FILE_PATH"] + upload_config["GET_ILINK3_FILE_NAME"], upload_config["PUT_ILINK3_FILE_PATH"] + upload_config["PUT_ILINK3_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], upload_config["PORT_PUT"] )
+    putfile_ftp(upload_config["GET_MD_FILE_PATH"] + upload_config["GET_MD_FILE_NAME"], upload_config["PUT_MD_FILE_PATH"] + upload_config["PUT_MD_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], upload_config["PORT_PUT"] )
+elif upload_config["PROTOCOL_PUT"] in ['SFTP','sftp']:
+    print "Protocol = SFTP"
+    putfile_sftp(upload_config["GET_ILINK3_FILE_PATH"] + upload_config["GET_ILINK3_FILE_NAME"], upload_config["PUT_ILINK3_FILE_PATH"] + upload_config["PUT_ILINK3_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], int(upload_config["PORT_PUT"]) )
+    putfile_sftp(upload_config["GET_MD_FILE_PATH"] + upload_config["GET_MD_FILE_NAME"], upload_config["PUT_MD_FILE_PATH"] + upload_config["PUT_MD_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], int(upload_config["PORT_PUT"]) )
+
+

src/data_files/files/example.conf

@@ -0,0 +1,16 @@
+fcm=FCM
+client=NAME
+session_id=ses
+put_host=ftpc.example.com
+port_put=2222
+username_put=user
+password_put=password
+protocol_put=sftp
+get_ilink3_file_path=/sparkdata/data/ilink3/*session*/audit_logs/
+put_ilink3_file_path=/
+get_ilink3_file_name=*fcm*_audit_*session*_*datestamp*.csv
+put_ilink3_file_name=*fcm*_audit_*session*_*datestamp*.csv
+get_md_file_path=/sparkdata/data/md_audit/
+put_md_file_path=/
+get_md_file_name=*fcm*_md_*client*_*datestamp*.csv
+put_md_file_name=*fcm*_md_*client*_*datestamp*.csv

src/lib/logging/logging.cpp

@@ -120,7 +120,7 @@ void configure_logging(bts::config::Logging const&cfg, bts::config::ThreadGroupC
                                 {log::Plugin,"Plugin"},
                                 {log::Sdk,"Sdk"},
                                 {log::Http,"Http"},
-                                {log::iLink3,"ilink3"},
+                                {log::iLink3,"iLink3"},
    };
 
    real_cfg.sinks["console"] =

src/lib/orderman/actors/GLBXiLink3ExecutionGateway.cpp

@@ -243,19 +243,19 @@ void GLBXiLink3ExecutionGateway::send_msgs(std::vector<OrderMsg> const &msgs,
          auto create_ptr = std::get_if<CreateOrderMsg>(&m);
          if (create_ptr) {
             me->gw->send_order(*create_ptr);
-            break;
+            continue;
          }
          // CancelReplace
          auto cancel_replace_ptr = std::get_if<CancelReplaceOrderMsg>(&m);
          if (cancel_replace_ptr) {
             me->gw->send_cancel_replace(*cancel_replace_ptr);
-            break;
+            continue;
          }
          // Cancel
          auto cancel_ptr = std::get_if<CancelOrderMsg>(&m);
          if (cancel_ptr) {
             me->gw->send_cancel(*cancel_ptr);
-            break;
+            continue;
          }
       }
       me->gw->flush();

src/lib/orderman/actors/OrderStore.cpp

@@ -101,6 +101,8 @@ void OrderStore::on_order_state_update(
       for (auto const &ev : *u.events) {
 
          auto &rec_info = me->known_orders[ev.key];
+         LOG(iLink3,Info, "Received order state update for {}",
+             ev.key.value());
          auto &rec = rec_info.rec;
          bts_assert(rec.has_def());
          rec.set_key(ev.key.value());
@@ -123,6 +125,8 @@ void OrderStore::on_order_definition_update(
    std::set<OrderKey> modified_orders;
    if (u.events)
       for (auto const &ev : *u.events) {
+         LOG(iLink3,Info, "Received order definition update for {}",
+             ev.key.value());
          auto &rec_info = me->known_orders[ev.key];
          auto &rec = rec_info.rec;
          rec.set_key(ev.key.value());

src/lib/orderman/actors/ilink3/IL3GW.cpp

@@ -176,12 +176,7 @@ uint64_t IL3GW::send_party_details(const api::orderman::RoutingRecord &route) {
       msg.selfMatchPreventionID(
           PartyDetailsDefinitionRequest518::selfMatchPreventionIDNullValue());
    } else {
-      // CME needs the SMP_ID to be 12 characters or less
-      // after they to_string() the uint64
-      bts_assert_d(route.smp_id().size() == 8);
-      const uint64_t *psmp_id =
-          reinterpret_cast<const uint64_t *>(&route.smp_id());
-      uint64_t smp_id = (*psmp_id) % (2UL << 39);
+      auto smp_id = std::atoi(route.smp_id().c_str());
       msg.selfMatchPreventionID(smp_id);
    }
    msg.cmtaGiveupCD(CmtaGiveUpCD::NULL_VALUE);

src/lib/orderman/actors/ilink3/ILink3Utils.h

@@ -25,7 +25,7 @@ iLinkBinary::SideReq::Value map_side(api::Side side);
 api::Side map_side(iLinkBinary::SideReq::Value side);
 
 inline int64_t to_price9(Price p) {
-   return p.value() * 1E9;
+   return std::round(p.value() * 1E9);
 }
 
 iLinkBinary::OrderTypeReq::Value map_order_type(api::orderman::NativeOrderType ot);

src/lib/orderman/actors/ilink3/TimeKeeper.h

@@ -18,7 +18,7 @@ struct TimeKeeper {
    uint64_t now() const {
       auto now = utils::TimeStamp::now();
       auto delta = now - start;
-      return unix_time + delta.microseconds();
+      return unix_time + delta.nanoseconds();
    }
 };
 

src/lib/orderman/doc/ilink3.org

@@ -0,0 +1,179 @@
+* iLink3 Reference Documentation
+** Overview
+The iLink3 work leverages and extends the iLink2 functionality. Most of the order management logic is unchanged from iLink2 with the substantial changes being confined to session management and encoding/decoding. In an effort to facilitate future addition of new execution protocols a set of protocol independent data structures and methods was added. 
+**  Main classes
+***  GLBXNativeorder
+GLBXNativeOrder implements the common iLink2 and iLink3 order management logic, including status updates, buffering of cancel/updates, recovery from disk, handling of execution reports, special order handling and routing. This is essentially unchanged from iLink2. From the point of view of this class the differences between iLink2 and iLink2 are delegated to the IGLBXOrderEncoder interface. At run time depending on the type of connection used an implementation of this interface for iLink2 or iLink3 is selected.
+*** IGLBXOrderEncoder
+Implementations of the IGLBXOrderEncoder interface below realize wire protocol specific implementations of encoding allowing GLBXNativeOrder to manage both iLink2 and iLink3 orders. The interface provides methods for creating,updating, canceling and dispatching orders. Currently there are two concrete implementations of this interface: iLink2OrderEncoder and iLink3OrderEncoder.
+
+#+BEGIN_SRC c++
+class IGLBXOrderEncoder {
+ public:
+   virtual ~IGLBXOrderEncoder() {}
+   virtual void init(OrderKey key, api::orderman::RoutingRecord const &,
+                     api::Security const &security,
+                     api::orderman::OrderDefinition const &def,
+                     int price_encoding_precision,
+                     bool stop_order_has_been_triggered) = 0;
+
+   virtual void update_exchange_order_id(ExchangeOrderIDVar const&) = 0;
+
+   virtual void update_create_order_encoding(Price p, int price_precision,
+                                             Quantity q, OrderServices &) = 0;
+
+   virtual void update_cancel_replace_stop_px(Price stop_px,
+                                              int price_precision) = 0;
+   virtual void update_cancel_replace_max_show(Quantity max_show) = 0;
+   virtual void update_cancel_replace_encoding(Price p, int price_precision,
+                                               Quantity q, OrderServices &) = 0;
+   virtual void update_cancel_replace_encoding(Quantity q, OrderServices &) = 0;
+   virtual void update_cancel_replace_encoding(Price p,
+                                               int price_precision,
+                                               OrderServices &) = 0;
+   virtual void update_cancel_encoding(OrderServices&) = 0;
+   virtual void send_create(OrderServices&,SessionServices&) = 0;
+   virtual void send_cancel_replace(OrderServices&,SessionServices&) = 0;
+   virtual void send_cancel(OrderServices&,SessionServices&) = 0;
+};
+
+ #+END_SRC
+*** class iLink2OrderEncoder : public IGLBXOrderEncoder 
+iLink2OrderEncoder provides FIX based encoding of GLBX orders. This logic is carried over essentially unchanged from previous versions of Spark with care being taken to re-use as much as possible elements of previous encodings on every call. The generated FIX messages are forwarded, as FIX, to the OrderServices::queue_fix template member function:
+
+#+BEGIN_SRC c++
+void queue_fix(SessionServices &ss, jetfix::MessageTypes mt,
+                  jetfix::FIXFragment const &f) {
+      ss.queue_fix(mt, f, corid());
+}
+ #+END_SRC
+
+*** class iLink3OrderEncoder : public IGLBXOrderEncoder 
+iLink3OrderEncoder provides a thin adapter that accepts order encoding requests from GLBXNativeOrder and maps those requests to the protocol agnostic order management data structures: CreateOrderMsg, CancelReplaceOrderMsg and CancelOrderMsg.  
+
+After this mapping, iLink3OrderEncoder forwards the order operation in question to its OrderServices instance, through the protocol independent queue_order_msg template function:
+
+#+BEGIN_SRC c++
+   template <class OP>
+   void queue_order_msg (SessionServices &ss, OP const&msg) {
+      ss.queue_order_msg(msg,corid());
+   }
+ #+END_SRC
+
+*** Protocol independent  order operation data structures
+The simple data structures in IExecutionGateway.h provide an exchange protocol independent way of representing order operations. They are intended to facilitate the addition of new exchange protocols. They are as follows:
+
+#+BEGIN_SRC c++
+  struct CreateOrderMsg {
+     OrderKey key;
+     Price price;
+     Quantity quantity;
+     api::Security const *security;
+     api::Side side;
+     Price stop_px;
+     Quantity display_quantity;
+     api::orderman::NativeOrderType order_type;
+     api::orderman::TimeInForce time_in_force;
+     api::orderman::ManualOrderIndicator manual_order_indicator;
+     api::orderman::RoutingRecord const *routing_record;
+  };
+
+  struct CancelReplaceOrderMsg {
+     CreateOrderMsg create_fields;
+     ExchangeOrderID64 exchange_order_id;
+  };
+
+  struct CancelOrderMsg {
+     OrderKey key;
+     ExchangeOrderID64 exchange_order_id;
+     api::orderman::ManualOrderIndicator manual_order_indicator;
+     api::Security const *security;
+     api::orderman::RoutingRecord const *routing_record;
+     api::Side side;
+  };
+
+  using OrderMsg = 
+     std::variant<CreateOrderMsg, CancelReplaceOrderMsg, CancelOrderMsg>;
+
+ #+END_SRC
+
+These data structures are used as vocabulary data types understood by all future exchange execution implementation code. They are consumed by implementations of the IExecutionGateway interface.
+
+*** class IExecutionGateway
+IExecutionGateway provides an abstraction for the implementation of exchange gateways. It provides methods for both FIX and non FIX based exchanges. Ordinarily implementations of this interface will either provide FIX or non FIX functionality but not both.
+
+#+BEGIN_SRC c++
+class IExecutionGateway {
+ public:   
+   IExecutionGateway() = default;
+   virtual ~IExecutionGateway() {}
+
+   virtual void start() = 0;
+   virtual void dispatch_incoming_fix_messages() = 0;
+   virtual void on_securities(const mercury::Update<NewSecuritiesEvent> &) = 0;
+   virtual void on_admin_data(const mercury::Update<NewAdminDataEvent> &) = 0;
+   virtual void send_fix(const jetfix::OutFIXPtr&, trace::CorID) = 0;
+   virtual void send_msgs(std::vector<OrderMsg> const&, trace::CorID) = 0;
+   virtual OnExeReport& on_exe_report() = 0;
+   virtual OnGwStatusChange& on_gw_status_change() = 0;
+   virtual OnCancelReject &on_cancel_reject() = 0;
+private:
+   BTS_NO_COPY(IExecutionGateway);
+};
+ #+END_SRC
+
+ IExecutionGateway implements are responsible for establishing execution sessions as well as mapping to and from protocol independent order operations to their gateway specific implementations. Presently there are three implementations of IExecutionGateway: ICE, iLink2 and iLink3. Only the iLink3 implementation is new. 
+ 
+*** class GLBXiLink3ExecutionGateway : public IExecutionGateway 
+This class implements provides a facade that hides, coordinates and interfaces with all the components needed to implement an iLink3 gateway including a TCP connection, a persistent store and the actual iLink3 encoding/decoding logic. GLBXiLink3ExecutionGateway role is identical to those of the GLBXExecutionGateway and ICEExecutionGateway, which existed prior to iLink3. In essence GLBXilink3executiongateway is in charge of managing the life-cycle of an iLink3 exchange connection as well as translating between exchange agnostic order operations and the iLink3 specific interfaces. The actual exchange interaction is delegated to the IL3GW class.
+*** class IL3GW : IL3ConnectionHandler
+The IL3GW class is responsible for iLink3 session management, including login, logout, recovery and heartbeat response. Additionally this class is responsible for mapping order messages to and from their network binary representation. This class is able to handle network events including connection, disconnection and data arrival via its implementation of the IL3ConnectionHandler interface:
+
+
+#+BEGIN_SRC c++
+struct IL3ConnectionHandler {
+   virtual ~IL3ConnectionHandler() ;
+   virtual void on_connect_success() = 0;
+   virtual void on_disconnect() = 0;
+   // Send available data and length. Handler will return what it consumed
+   virtual size_t on_data(const std::string_view&) = 0;
+};
+ #+END_SRC
+
+IL3GW integrates these external events with the instructions it receives from the rest of the system through its public interface:
+
+#+BEGIN_SRC c++
+   void connect();
+   void send_order(const CreateOrderMsg &);
+   void send_cancel_replace(const CancelReplaceOrderMsg &);
+   void send_cancel(const CancelOrderMsg &);
+   void send_terminate(uint16_t code = 0);
+ #+END_SRC
+
+ And provides an integrated high-level view of the connection state via its event callbacks:
+
+#+BEGIN_SRC c++
+   void subscribe_state(OnStateChange o);
+   void subscribe_execution_reports(OnExecutionReport sub);
+   void subscribe_session(OnSessionChange cb); 
+   void subscribe_cancel_reject(OnCancelReject cb); 
+ #+END_SRC
+
+** Code Generation
+To facilitate encoding IL3GW uses code generated by the CME sponsored tools by real logic. These tools can be found here:
+
+https://github.com/real-logic/simple-binary-encoding/wiki/Sbe-Tool-Guide
+
+The CME provides template files that be used as an input to these tools. For more details see:
+
+https://www.cmegroup.com/confluence/display/EPICSANDBOX/iLink+3+-+Simple+Binary+Encoding
+
+The generation was done manually and the generated code integrated into source control. 
+
+** Operational Notes
+- Like iLink2, the iLink3 implementation uses weekly sessions
+- Sessions are stored under sparkdata/ilink3/session_name/date-of-session
+- Within each session directory there are two files:
+  - log: is a json log of all messages sent and received from the exchange
+  - seqnum: contains the current sequence numbers for the session. Additionally the session binary name is also represented for use in trouble-shooting
+    

src/lib/orderman/orders/GLBXNativeOrder.cpp

@@ -861,7 +861,7 @@ void GLBXNativeOrder::on_exe_report(const NewExecutionReportEvent &er) {
    auto external_status = map_external_status(er.data->order_status);
 
    // filled quantity is reset back to 0 when the order is canceled
-   if (external_status != ORDER_STATUS_CANCELED) {
+   if (external_status != ORDER_STATUS_CANCELED ) {
       if (er.data->exec_type == api::orderman::EXEC_TYPE_PARTIAL_FILL ||
           er.data->exec_type == api::orderman::EXEC_TYPE_FILL) {
 
@@ -876,9 +876,9 @@ void GLBXNativeOrder::on_exe_report(const NewExecutionReportEvent &er) {
              (previous_sz + fill_sz);
 
          state.set_avg_price(new_avg);
+         state.set_filled_quantity(er.data->filled_quantity.value());
+         state.set_last_price(er.data->last_price.value());
       }
-      state.set_filled_quantity(er.data->filled_quantity.value());
-      state.set_last_price(er.data->last_price.value());
    }
 
    state.set_last_msg(er.data->msg);

src/lib/orderman/orders/algo/algo_orders/PyramidAlgo.cpp

@@ -16,7 +16,7 @@ namespace orderman {
 
 using namespace api::orderman;
 
-enum class PyramidAlgo::Status { WAITING_FOR_MD, WAITING_FOR_MD_ATTACH, WORKING, DONE };
+enum class PyramidAlgo::Status { WAITING_FOR_MD, WAITING_FOR_MD_ATTACH, WORKING, WAITING_FOR_CXL_ACK, DONE };
 enum class PyramidAlgo::ExitPath { CANCEL, ERR, COMPLETE };
 
 struct LegData{
@@ -192,6 +192,7 @@ void PyramidAlgo::do_update(const api::orderman::Update& update) {
                          "Order cancelled (Cover prices not met)");
          break;
       }
+      case PyramidAlgo::Status::WAITING_FOR_CXL_ACK:
       case PyramidAlgo::Status::DONE:
          break;
    }
@@ -215,6 +216,7 @@ IAlgoOrder::OnCancelResponse PyramidAlgo::on_cancel(){
       case PyramidAlgo::Status::WORKING:
          _cxl_and_end(PyramidAlgo::ExitPath::CANCEL,"Algo cancelled");
          break;
+      case (PyramidAlgo::Status::WAITING_FOR_CXL_ACK):
       case PyramidAlgo::Status::DONE:
          break;
    }
@@ -242,6 +244,7 @@ void PyramidAlgo::on_child_change(OrderKey key, const ChildState &st) {
             _check_for_completion();
             break;
          case PyramidAlgo::Status::WORKING:
+         case (PyramidAlgo::Status::WAITING_FOR_CXL_ACK):
             leg->order_state = st;
             _algo().update_fill_state(st.total_fills, st.avg_price);
             _send_hedges(st.last_fills);
@@ -388,6 +391,7 @@ void PyramidAlgo::_analyze_md_and_act(){
             _cxl_and_end(PyramidAlgo::ExitPath::CANCEL,
                          "Order cancelled (Cover prices not met)");
          break;
+      case PyramidAlgo::Status::WAITING_FOR_CXL_ACK:   
       case PyramidAlgo::Status::DONE:
          break;
    }
@@ -405,14 +409,19 @@ void PyramidAlgo::_switch_state(Status new_state){
          bts_assert(me->state == PyramidAlgo::Status::WAITING_FOR_MD ||
                     me->state == PyramidAlgo::Status::WAITING_FOR_MD_ATTACH);
          break;
+      case (PyramidAlgo::Status::WAITING_FOR_CXL_ACK):
+         break;
       case (PyramidAlgo::Status::DONE):
          bts_assert(
                // when an order is entered and the criteria is not met
                me->state == PyramidAlgo::Status::WAITING_FOR_MD ||
                // when an order is attached and the market conditions are not met
                me->state == PyramidAlgo::Status::WAITING_FOR_MD_ATTACH ||
-               // when the child order gets into an unexpected state
-               me->state == PyramidAlgo::Status::WORKING);
+               // when the child order enters a terminal state, while the algo
+               // is in a working state (external cxl, target reached)
+               me->state == PyramidAlgo::Status::WORKING ||
+               // the usual controlled cxl, ie working->waiting_for_cxl_ack->done
+               me->state == PyramidAlgo::Status::WAITING_FOR_CXL_ACK);
          break;
    }
    LOG(OrderMan, Info,"PyramidAlgo::_switch_state key:{} {}->{} ",
@@ -494,8 +503,8 @@ void PyramidAlgo::_check_for_completion(){
          break;
       case (ChildStatus::CANCELED):
          _cxl_and_end(PyramidAlgo::ExitPath::ERR,
-                      "Top spread unexpectedly cancelled");
-         break;      
+                      "Top spread cancelled");
+         break;
    }
 }
 
@@ -507,8 +516,11 @@ void PyramidAlgo::_cxl_and_end(PyramidAlgo::ExitPath exit_path,
       if(top_ord_state.status == ChildStatus::WORKING){
          if(detach_top)
             _algo().detach_child(me->legs[0].order_key);
-         else
+         else{
             _algo().cancel_child(me->legs[0].order_key);
+            _switch_state(PyramidAlgo::Status::WAITING_FOR_CXL_ACK);
+            return;
+         }
       }
    }
    switch(exit_path){
@@ -524,6 +536,7 @@ void PyramidAlgo::_cxl_and_end(PyramidAlgo::ExitPath exit_path,
    }
    _switch_state(PyramidAlgo::Status::DONE);
 }
+
 // true the out qty is changed it be below
 // the filled qty
 bool PyramidAlgo::_target_reached() const{

src/lib/orderman/orders/algo/algo_orders/TASCounterAlgo.cpp

@@ -26,17 +26,21 @@ enum class TASCounterAlgo::Status { WORKING, DONE };
 // (the vwap of the TAS contracts is not used, but for uniformity is kept in the
 // the same structure)
 struct ContractMarketData{
+private:
    uint32_t tot_vol = 0;
    double running_sum = 0;
+public:
    Book book;
    double estimated_settle = 0.0;
    void on_trade (const TickerUpdateEvent::Trade& t){
       tot_vol += t.quantity.value();
       running_sum += t.price.value()*t.quantity.value();
    }
-   const double vwap(){
+   double vwap() const{
       return tot_vol == 0 ? os::nan() : running_sum/tot_vol;
    }
+   uint32_t traded_qty() const {return tot_vol;}
+   double traded_value() const {return running_sum;}
 };
 // the input for the t1 front, second and third orders, as well as the template
 // and working order key are kept here
@@ -100,6 +104,47 @@ bool trade_within_time_win (const Timestamp&, const Timestamp&,
                             const TickerUpdateEvent::Trade& t);
 bool now_within_time_win (const Timestamp&, const Timestamp&);
 
+double compute_t1_front_settle (const AlgoData& d){
+   double rv = d.t1_front.vwap();
+   rv = round_to_nearest(rv,d.tick_size);
+   return rv;   
+}
+
+double compute_t1_second_settle (const AlgoData& d){
+   double p_t1_front = compute_t1_front_settle(d);
+   double part_1 = d.t1_second.traded_value();
+   double part_2 = d.t2_front.traded_value() - 
+                   (d.t2_front.traded_qty() * p_t1_front);
+   uint32_t traded_qty_sum = d.t2_front.traded_qty() + 
+                             d.t1_second.traded_qty();
+   double rv = traded_qty_sum == 0 ? os::nan() : 
+               (part_1+part_2)/traded_qty_sum;
+   LOG(OrderMan, Diag, "TASCounterAlgo::compute_t1_second_settle "
+      "p_t1_front:{} part_1:{} part_2:{} traded_qty_sum:{} rv:{}",
+      p_t1_front, part_1, part_2, traded_qty_sum, rv);
+   return round_to_nearest(rv,d.tick_size);
+}
+
+double compute_t1_third_settle (const AlgoData& d){
+   double p_t1_front = compute_t1_front_settle(d);
+   double p_t1_second = compute_t1_second_settle(d);
+   double part_1 = d.t1_third.traded_value();
+   double part_2 = d.t2_second.traded_value() - 
+                   (d.t2_second.traded_qty()*p_t1_second);
+   double part_3 = d.t3_front.traded_value() - 
+                   (d.t3_front.traded_qty()*p_t1_front) -
+                   (d.t3_front.traded_qty()*p_t1_second);
+   double traded_qty_sum = d.t3_front.traded_qty() + 
+                           d.t2_second.traded_qty() + 
+                           d.t1_third.traded_qty();
+   double rv = traded_qty_sum == 0 ? os::nan() : 
+               (part_1+part_2+part_3)/traded_qty_sum;
+   LOG(OrderMan, Diag, "TASCounterAlgo::compute_t1_third_settle "
+      "p_t1_front:{} p_t1_second:{} part_1:{} part_2:{} part_3:{} "
+      "traded_qty_sum:{} rv:{}",
+      p_t1_front, p_t1_second, part_1, part_2, part_3, traded_qty_sum, rv);
+   return round_to_nearest(rv,d.tick_size);
+}
 
 TASCounterAlgo::TASCounterAlgo() : me(new PImpl()) {}
 TASCounterAlgo::~TASCounterAlgo() {}
@@ -214,17 +259,9 @@ void TASCounterAlgo::_cxl_and_end (bool is_error, const std::string& err_msg){
 
 void TASCounterAlgo::_compute_settles(){
    AlgoData& d = me->data;
-   d.t1_front.estimated_settle = d.t1_front.vwap();
-   d.t1_second.estimated_settle = d.t2_front.vwap() - d.t1_front.vwap();
-   d.t1_third.estimated_settle = d.t3_front.vwap() - d.t1_front.vwap() - 
-                                 d.t1_second.vwap();
-   // round to closest value
-   d.t1_front.estimated_settle = round_to_nearest(
-                        d.t1_front.estimated_settle,me->data.tick_size);
-   d.t1_second.estimated_settle = round_to_nearest(
-                        d.t1_second.estimated_settle,me->data.tick_size);
-   d.t1_third.estimated_settle = round_to_nearest(
-                        d.t1_third.estimated_settle,me->data.tick_size);
+   d.t1_front.estimated_settle = compute_t1_front_settle(d);
+   d.t1_second.estimated_settle = compute_t1_second_settle(d);
+   d.t1_third.estimated_settle = compute_t1_third_settle(d);
 }
 
 // the decision to send an order can be caused by