Merge branch 'master' of bitbucket.org:bluetradesys/spark

The compute_synthetic_book method (also to be imported
from spartan) depends on a couple of them.

.gitignore

@@ -37,3 +37,4 @@ trace_data/
 *.rej
 /build
 .clangd/
+*CMakeDeps.cmake

doc/user/CMakeDeps.cmake

@@ -1 +1 @@
-set(sources /home/ppinto/Dropbox/bts/spark/doc/user/build.sh;content/_index.md;content/bts_spark/Glossary/_index.md;content/bts_spark/Help/_index.md;content/bts_spark/Help/about_spark.png;content/bts_spark/Help/image39.png;content/bts_spark/Help/image58.png;content/bts_spark/Launcher/_index.md;content/bts_spark/Launcher/image38.png;content/bts_spark/Launcher/image68.png;content/bts_spark/Logging-In-to-the-System/_index.md;content/bts_spark/Logging-In-to-the-System/image3.png;content/bts_spark/Logging-In-to-the-System/image64.png;content/bts_spark/Market/_index.md;content/bts_spark/Market/change_securities.png;content/bts_spark/Market/image11.png;content/bts_spark/Market/image14.png;content/bts_spark/Market/image20.png;content/bts_spark/Market/image21.png;content/bts_spark/Market/image23.png;content/bts_spark/Market/image24.png;content/bts_spark/Market/image34.png;content/bts_spark/Market/image49.png;content/bts_spark/Market/image53.png;content/bts_spark/Market/image66.png;content/bts_spark/Market/image69.png;content/bts_spark/Market/market_view.png;content/bts_spark/Market/settings_icon.png;content/bts_spark/Market/time_and_sales.png;content/bts_spark/My-Windows/_index.md;content/bts_spark/My-Windows/image15.png;content/bts_spark/My-Windows/image16.png;content/bts_spark/My-Windows/image37.png;content/bts_spark/Settings/_index.md;content/bts_spark/Settings/image51.png;content/bts_spark/Settings/image9.png;content/bts_spark/Spreader/_index.md;content/bts_spark/Spreader/image10.png;content/bts_spark/Spreader/image13.png;content/bts_spark/Spreader/image18.png;content/bts_spark/Spreader/image19.png;content/bts_spark/Spreader/image28.png;content/bts_spark/Spreader/image29.png;content/bts_spark/Spreader/image31.png;content/bts_spark/Spreader/image35.png;content/bts_spark/Spreader/image40.png;content/bts_spark/Spreader/image43.png;content/bts_spark/Spreader/image54.png;content/bts_spark/Spreader/image55.png;content/bts_spark/Spreader/image56.png;content/bts_spark/Spreader/image6.png;content/bts_spark/Spreader/image7.png;content/bts_spark/Spreader/image70.png;content/bts_spark/Spreader/image72.png;content/bts_spark/Spreader/image8.png;content/bts_spark/Trading/OrderTypes/index.md;content/bts_spark/Trading/_index.md;content/bts_spark/Trading/algo_studio.png;content/bts_spark/Trading/collapse_all_icon.png;content/bts_spark/Trading/cover_algo_control.png;content/bts_spark/Trading/expand_all_icon.png;content/bts_spark/Trading/image1.png;content/bts_spark/Trading/image17.png;content/bts_spark/Trading/image2.png;content/bts_spark/Trading/image25.png;content/bts_spark/Trading/image27.png;content/bts_spark/Trading/image33.png;content/bts_spark/Trading/image42.png;content/bts_spark/Trading/image44.png;content/bts_spark/Trading/image47.png;content/bts_spark/Trading/image48.png;content/bts_spark/Trading/image5.png;content/bts_spark/Trading/image50.png;content/bts_spark/Trading/image52.png;content/bts_spark/Trading/image57.png;content/bts_spark/Trading/image59.png;content/bts_spark/Trading/image60.png;content/bts_spark/Trading/image61.png;content/bts_spark/Trading/image62.png;content/bts_spark/Trading/image73.png;content/bts_spark/Trading/image74.png;content/bts_spark/Trading/join_algo_control.png;content/bts_spark/Trading/ladder.png;content/bts_spark/Trading/ladder_badges.png;content/bts_spark/Trading/ladder_main_part.png;content/bts_spark/Trading/lean_algo_control.png;content/bts_spark/Trading/mini_orderbook.png;content/bts_spark/Trading/oco_algo_control.png;content/bts_spark/Trading/offset_algo_control.png;content/bts_spark/Trading/pnl_calculation.png;content/bts_spark/Trading/pnl_window.png;content/bts_spark/Trading/recreate_icon.png;content/bts_spark/Trading/settings_icon.png;content/bts_spark/Trading/spark_menu_order_book.png;content/bts_spark/Trading/top_algo_control.png;content/bts_spark/Trading/trigger_algo_control.png;content/bts_spark/_index.md;content/release_notes/_index.md;content/system_requirements/_index.md;static/book.js;static/favicon.ico;static/image.js;templates/index.html;templates/page.html;templates/section.html;templates/shortcodes/back_button.html)
+set(sources /home/pfisher/bts/spark/doc/user/build.sh;content/_index.md;content/bts_spark/Glossary/_index.md;content/bts_spark/Help/_index.md;content/bts_spark/Help/about_spark.png;content/bts_spark/Help/image39.png;content/bts_spark/Help/image58.png;content/bts_spark/Launcher/_index.md;content/bts_spark/Launcher/image38.png;content/bts_spark/Launcher/image68.png;content/bts_spark/Logging-In-to-the-System/_index.md;content/bts_spark/Logging-In-to-the-System/image3.png;content/bts_spark/Logging-In-to-the-System/image64.png;content/bts_spark/Market/_index.md;content/bts_spark/Market/change_securities.png;content/bts_spark/Market/image11.png;content/bts_spark/Market/image14.png;content/bts_spark/Market/image20.png;content/bts_spark/Market/image21.png;content/bts_spark/Market/image23.png;content/bts_spark/Market/image24.png;content/bts_spark/Market/image34.png;content/bts_spark/Market/image49.png;content/bts_spark/Market/image53.png;content/bts_spark/Market/image66.png;content/bts_spark/Market/image69.png;content/bts_spark/Market/market_view.png;content/bts_spark/Market/settings_icon.png;content/bts_spark/Market/time_and_sales.png;content/bts_spark/My-Windows/_index.md;content/bts_spark/My-Windows/image15.png;content/bts_spark/My-Windows/image16.png;content/bts_spark/My-Windows/image37.png;content/bts_spark/Settings/_index.md;content/bts_spark/Settings/image51.png;content/bts_spark/Settings/image9.png;content/bts_spark/Spreader/_index.md;content/bts_spark/Spreader/image10.png;content/bts_spark/Spreader/image13.png;content/bts_spark/Spreader/image18.png;content/bts_spark/Spreader/image19.png;content/bts_spark/Spreader/image28.png;content/bts_spark/Spreader/image29.png;content/bts_spark/Spreader/image31.png;content/bts_spark/Spreader/image35.png;content/bts_spark/Spreader/image40.png;content/bts_spark/Spreader/image43.png;content/bts_spark/Spreader/image54.png;content/bts_spark/Spreader/image55.png;content/bts_spark/Spreader/image56.png;content/bts_spark/Spreader/image6.png;content/bts_spark/Spreader/image7.png;content/bts_spark/Spreader/image70.png;content/bts_spark/Spreader/image72.png;content/bts_spark/Spreader/image8.png;content/bts_spark/Trading/OrderTypes/index.md;content/bts_spark/Trading/_index.md;content/bts_spark/Trading/algo_studio.png;content/bts_spark/Trading/collapse_all_icon.png;content/bts_spark/Trading/cover_algo_control.png;content/bts_spark/Trading/expand_all_icon.png;content/bts_spark/Trading/image1.png;content/bts_spark/Trading/image17.png;content/bts_spark/Trading/image2.png;content/bts_spark/Trading/image25.png;content/bts_spark/Trading/image27.png;content/bts_spark/Trading/image33.png;content/bts_spark/Trading/image42.png;content/bts_spark/Trading/image44.png;content/bts_spark/Trading/image47.png;content/bts_spark/Trading/image48.png;content/bts_spark/Trading/image5.png;content/bts_spark/Trading/image50.png;content/bts_spark/Trading/image52.png;content/bts_spark/Trading/image57.png;content/bts_spark/Trading/image59.png;content/bts_spark/Trading/image60.png;content/bts_spark/Trading/image61.png;content/bts_spark/Trading/image62.png;content/bts_spark/Trading/image73.png;content/bts_spark/Trading/image74.png;content/bts_spark/Trading/join_algo_control.png;content/bts_spark/Trading/ladder.png;content/bts_spark/Trading/ladder_badges.png;content/bts_spark/Trading/ladder_main_part.png;content/bts_spark/Trading/lean_algo_control.png;content/bts_spark/Trading/mini_orderbook.png;content/bts_spark/Trading/oco_algo_control.png;content/bts_spark/Trading/offset_algo_control.png;content/bts_spark/Trading/pnl_calculation.png;content/bts_spark/Trading/pnl_window.png;content/bts_spark/Trading/recreate_icon.png;content/bts_spark/Trading/settings_icon.png;content/bts_spark/Trading/spark_menu_order_book.png;content/bts_spark/Trading/top_algo_control.png;content/bts_spark/Trading/trigger_algo_control.png;content/bts_spark/_index.md;content/release_notes/_index.md;content/system_requirements/_index.md;static/book.js;static/favicon.ico;static/image.js;templates/index.html;templates/page.html;templates/section.html;templates/shortcodes/back_button.html)

src/data_files/CMakeDeps.cmake

@@ -1 +1 @@
-set(sources files/cert_secure_logon.txt;files/futures.coconut.xz;files/onixs_ice/ConnectivityConfiguration.APITest.xml;files/onixs_ice/ConnectivityConfiguration.APITest.xml.orig;files/onixs_ice/ConnectivityConfiguration.PerfTest.xml;files/onixs_ice/ConnectivityConfiguration.Production.xml;files/perf_trace/compiled.css;files/perf_trace/index.html;files/perf_trace/jquery-3.3.1.min.js;files/perf_trace/main.js;files/perf_trace/perf_log.sql;files/perf_trace/perf_trace_analysis.sql;files/perf_trace/plotly-latest.min.js)
+set(sources files/audit_ilink3.py;files/audit_upload.sh;files/audit_uploader.py;files/cert_secure_logon.txt;files/example.conf;files/futures.coconut.xz;files/onixs_ice/ConnectivityConfiguration.APITest.xml;files/onixs_ice/ConnectivityConfiguration.APITest.xml.orig;files/onixs_ice/ConnectivityConfiguration.PerfTest.xml;files/onixs_ice/ConnectivityConfiguration.Production.xml;files/perf_trace/compiled.css;files/perf_trace/index.html;files/perf_trace/jquery-3.3.1.min.js;files/perf_trace/main.js;files/perf_trace/perf_log.sql;files/perf_trace/perf_trace_analysis.sql;files/perf_trace/plotly-latest.min.js)

src/data_files/files/audit_ilink3.py

@@ -0,0 +1,405 @@
+import json
+import csv
+import datetime, time
+import argparse, os
+
+
+def valid_file(f):
+    if not os.path.exists(f):
+        # Argparse uses the ArgumentTypeError to give a rejection message like:
+        # error: argument input: x does not exist
+        raise argparse.ArgumentTypeError("{0} does not exist".format(f))
+    return f
+
+def valid_date(s):
+    try:
+        return datetime.datetime.strptime(s, "%Y%m%d")
+    except ValueError:
+        raise argparse.ArgumentTypeError("Not a valid date: '{0}'.".format(s))
+
+def print_headers():
+    # From https://www.cmegroup.com/confluence/display/EPICSANDBOX/iLink+3+-+Minimum+Acceptable+Audit+Trail+Elements+-+Data+Definitions+and+Validation+Rules
+    line_to_write = "Sending Timestamps,Message Direction,Operator ID,Session ID,Manual Order Indicator,Message Type,Instrument,Order Flow ID,\
+CME Globex Order ID,Client Order ID,Order Request ID,Buy/Sell Indicator,Quantity,Limit Price,Stop Price,Order Type,Order Qualifier,IFM Flag,\
+Display Quantity,Minimum Quantity,Country of Origin,Fill Price,Fill Quantity,Cumulative Quantity,Remaining Quantity,Aggressor Flag,Source of Cancellation,\
+Reject Reason,CME Globex Message ID,Cross ID,Quote Request ID,Message Quote ID,Quote Set ID,Quote Entry ID,Bid Price,Bid Size,Offer Price,Offer Size,\
+Processed Entries,Party Details List Request ID,ListUpdateAction,Self-Matching Prevention ID,Self-Match Prevention Instr,Customer Type Indicator,\
+AvgPxGroupID,Average Px Ind,CmtaGiveUpCD,Origin,Clearing Trade Price Type,Customer Handling Instr,Executing Firm ID,Account Number,Take Up Firm,Take up Account"
+
+    with open (args.output_file, 'w') as the_file:
+                    the_file.write(line_to_write)
+                    the_file.write("\n")
+
+def set_blanks():
+    return ""
+
+def get_value(json_key):
+    prices = ["Price", "LastPx", "StopPx"]
+    if json_key in json_data:
+        if json_key in prices:
+            x = json_data[json_key]
+            return x['mantissa']/100000000000
+        else:
+            return json_data[json_key]
+    else:
+        return ""
+
+
+# Column 1  - Sending Timestamps
+def SendingTimeEpoch(epochtime):   #1
+    #TIME = time.strftime('%Y%m%d-%H:%M:%S.{}'.format(epochtime/1000000000), time.gmtime(epochtime/1000000000))
+    TIME = datetime.datetime.fromtimestamp(int(epochtime) / 1e9).strftime('%Y%m%d-%H:%M:%S.%f')
+    return TIME
+
+# Column 2 - Message Direction
+def direction(whichway):
+    if whichway == '<':    
+        DIRECTION = "FROM CME"
+    else:                       
+        DIRECTION = "TO CME"     
+    return DIRECTION
+
+# Column 3 - Operator ID
+def SenderID(sender):
+    if "SenderID" in json_data:
+        SENDER = json_data["SenderID"] 
+        return SENDER
+    else:
+        return SESSIONID
+
+
+# Column 4 - Session ID
+
+# Column 5 - Manual Order Indicator
+def ManualOrderIndicator(manual):
+    if "ManualOrderIndicator" in json_data:
+        if manual == "Manual":
+            return "1"
+        else:
+            return "0"
+
+# Column 6 - Message Type
+def MessageType(msg_type,orderstatus):
+    if msg_type == "NewOrderSingle514":
+       return "D" 
+    elif msg_type == "ExecutionReportNew522":
+        return "8--0"
+    if "ExecutionReportTrade" in msg_type:
+        if orderstatus == "PartiallyFilled":
+            return "8--1"
+        else:
+            return "8--2"
+    elif msg_type == "ExecutionReportCancel534":
+        return "8--4"
+    elif msg_type == "ExecutionReportModify531":
+        return "8--5"
+    elif msg_type == "ExecutionReportCancel534":
+        return "8--H"
+    elif msg_type == "ExecutionReportReject523":
+        return "8--8"
+    elif msg_type == "OrderCancelReplaceReject536":
+        return "9--2"
+    elif msg_type == "OrderCancelReject535":
+        return "9--1"
+    elif msg_type == "PartyDetailsDefinitionRequest518":
+        return "CX"
+    elif msg_type == "PartyDetailsDefinitionRequestAck519":
+        return "CY"
+    elif msg_type == "OrderCancelRequest516":
+        return "F"
+    elif msg_type == "OrderCancelReplaceRequest515":
+        return "G"
+    
+    
+
+# Column 12 - Buy-Sell Indicator/Side
+def Side(indicator):
+    if "Side" in json_data:
+        if indicator == "Buy":
+            return "1"
+        else:
+            return "2"
+
+# Column 16 - Order Type/OrdType
+def OrdType(ordertype):
+    if "OrdType" in json_data:
+        if ordertype == "MarketWithProtection" or ordertype == "MarketwithProtection":
+            return "1"
+        elif ordertype == "Limit":
+            return "2"
+        elif ordertype == "StopWithProtection" or ordertype == "StopwithProtection":
+            return "3"
+        elif ordertype == "StopLimit":
+            return "4"
+        elif ordertype == "MarketLimit":
+            return "K"
+        else:
+            return ""
+
+# Column 17 - Order Qualifier/TimeInForce
+def TimeInForce(orderqual):
+    if "TimeInForce" in json_data:
+        if orderqual == "Day":
+            return "0"
+        elif orderqual == "GTC":
+            return "1"
+        elif orderqual == "FAK":
+            return "3"
+        elif orderqual == "GTD":
+            return "6"
+        else:
+            return ""
+
+# Column 18 IFM Flag/OFMOverride
+def OFMOverride(imfflag):
+    if "OFMOverride" in json_data:
+        if imfflag == "Enabled":
+            return "1"
+        else:
+            return "0"
+
+# Column 19 Display Quantity/DisplayQty
+def DisplayQty(qty):
+    if "DisplayQty" in json_data:
+        if str(qty) == "0":
+            return ""
+        elif str(qty) == "4294967295":
+            return ""
+        else:
+            return qty
+
+
+# Column 26 - Aggressor Flag/AggressorIndicator
+def AggressorIndicator(aggressor):
+    if "AggressorIndicator" in json_data:
+        if aggressor == "True":
+            return "1"
+        else:
+            return "0"
+
+# Column 27 - Source of Cancellation/ExecRestatementReason
+def ExecRestatementReason(sourcecancel):
+    if "ExecRestatementReason" in json_data:
+        if sourcecancel == "Exchange":
+            return "8"
+        elif sourcecancel == "CancelledNotBest":
+            return "9"
+        elif sourcecancel == "CancelOnDisconnect":
+            return "100"
+        elif sourcecancel == "CancelRestingDueToSelfMatchPrevention":
+            return "103"
+        elif sourcecancel == "CancelFromCMEGlobexCreditControlsViolation":
+            return "104"
+        elif sourcecancel == "CancelFromCMEOne":
+            return "105"
+        elif sourcecancel == "CancelFromRiskManagementAPI":
+            return "106"
+        elif sourcecancel == "CancelAggressingDueToSelfMatchPrevention":
+            return "107"
+        elif sourcecancel == "CancelDueToVolatilityQuoteOptionLessThanMinOrderSize":
+            return "108"
+        else:
+            return ""
+
+# Column 43 - Self-Match Prevention Instr/SelfMatchPreventionInstruction
+def SelfMatchPreventionInstruction(instr):
+    if "SelfMatchPreventionInstruction" in json_data:
+        if instr == "CancelOldest":
+            return "O"
+        elif instr == "CancelNewest":
+            return "N"
+        else:
+            return ""
+
+# Column 44 - Customer Type Indicator/CustOrderCapacity
+def CustOrderCapacity(custord):
+    if "CustOrderCapacity" in json_data:
+        if custord == "NULL_VALUE":
+            return ""
+        elif custord == "Membertradingfortheirownaccount":
+            return "1"
+        elif custord == "ClearingFirmtradingforitsproprietaryaccount":
+            return "2"
+        elif custord == "Membertradingforanothermember":
+            return "3"
+        else:
+            return "4" 
+
+# Column 46 - Average Px Ind/AvgPxInd
+def AvgPxInd(avgpxindicator):
+    if "AvgPxInd" in json_data:
+        if avgpxindicator == "NULL_VALUE":
+            return ""
+        else:
+            return avgpxindicator
+
+# Column 48 - Origin/ClearingAccountType
+def ClearingAccountType(accounttype):
+    if "ClearingAccountType" in json_data:
+        if accounttype == "Customer":
+            return "0"
+        elif accounttype == "Firm":
+            return "1"
+        else:
+            return ""
+
+# Column 49 - Clearing Trade Price Type/ClearingTradePriceType
+def ClearingTradePriceType(clrtrdprice):
+    if "ClearingTradePriceType" in json_data:
+        if clrtrdprice == "NULL_VALUE":
+            return "0"
+        else:
+            return "3"
+
+# Column 50 - Customer Handling Instr/CustOrderHandlingInst
+def CustOrderHandlingInst(custorderinst):
+    if "CustOrderHandlingInst" in json_data:
+        if custorderinst == "ClientElectronic":
+            return "C"
+        elif custorderinst == "TEST":
+            return "D"
+        else:
+            return ""
+
+# Column 51 and 52 - Getting the Firm and Account Number
+def partyid(partydetailid,column):
+    for x in partydetailid:
+        role = x['PartyDetailRole']
+        id = x['PartyDetailID']
+        if column == "51" and role == "ExecutingFirm":
+            return id
+        if column == "52" and role == "CustomerAccount":
+            return id
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument("json_log", help="the json log[.gz] you wish to process", type=valid_file)
+parser.add_argument("output_file", help="the destination csv file you wish to write")
+parser.add_argument("-d", "--date", help="the date you wish to parse from the log, specified as YYYYmmdd",
+                    default=datetime.datetime.today().strftime("%Y%m%d"), type=valid_date)
+args = parser.parse_args()
+
+print_headers()
+
+list_types = ['NewOrderSingle514', 'OrderCancelReplaceRequest515', 'OrderCancelRequest516', 'PartyDetailsDefinitionRequest518', 'PartyDetailsDefinitionRequestAck519', \
+        'ExecutionReportNew522', 'ExecutionReportReject523', 'ExecutionReportTradeOutright525', 'ExecutionReportTradeSpread526', 'ExecutionReportTradeSpreadLeg527', \
+        'ExecutionReportModify531', 'ExecutionReportCancel534', 'OrderCancelReject535', 'OrderCancelReplaceReject536', ]
+
+# Note that the timedeltas specificed in trade_day_start and trade_day_end
+# are based on Chicago times; as such, this script should run on a machine
+# set to a Chicago timezone. A future impromovement would be to determine
+# the current machine's timezone and determine the appropriate offset
+# versus Chicago time, but for now we assume this will run on a Chicago
+# tz machine.
+input_as_datetime = args.date
+trade_day_start = input_as_datetime - datetime.timedelta(days=1) + datetime.timedelta(hours=16, minutes=30)
+trade_day_utc_start = time.gmtime((time.mktime(trade_day_start.timetuple())))
+trade_day_end = input_as_datetime + datetime.timedelta(hours=16)
+trade_day_utc_end = time.gmtime((time.mktime(trade_day_end.timetuple())))
+
+file_opener = None
+if args.json_log.endswith(".gz"):
+    import gzip
+    file_opener = gzip.open
+else:
+    file_opener = open
+
+
+#inputFile = open(fileInput) #open json file
+#outputFile = open(fileOutput, 'w') #load csv file
+# Get Session
+with file_opener(args.json_log) as json_file:
+    for line in json_file:
+        data = json.loads(line)
+        json_data = data['msg_json']
+        if "Session" in json_data:
+            SESSIONID = json_data["Session"]
+
+#inputFile = open(fileInput) #open json file
+#outputFile = open(fileOutput, 'w') #load csv file
+with file_opener(args.json_log) as json_file:
+    for line in json_file:
+        data = json.loads(line)
+        line_timestamp = time.strptime(data['time'], "%Y%m%d-%H:%M:%S.%f")
+        if trade_day_utc_start <= line_timestamp <= trade_day_utc_end:
+            json_data = data['msg_json']
+            NAME = json_data["Name"]
+            if NAME in list_types:
+                EXECUTING_FIRM_ID = ""
+                ACCOUNT_NUMBER = ""
+                DIRECTION = direction(data['direction'])
+                # Set Blanks
+                blank_json = {'QuoteReqID': '', 'QuoteID': '', 'QuoteSetID': '', 'QuoteEntryID': '', 'BidPx': '', 'BidSize': '', 'OfferPx': '', \
+                'OfferSize': '', 'NoProcessedEntries': '', 'ListUpdateAction': '', 'MinQty': '', 'AvgPxGroupID': '', \
+                'CmtaGiveUpCD': '', 'TAKE_UP_FIRM': '', 'TAKE_UP_ACCOUNT': '',}
+
+                # No Data Processing
+                straight_json = {'SecurityID': '', 'ClOrdID': '', 'OrderID': '', 'OrderRequestID': '', 'OrderQty': '', \
+                'Location': '', 'LastQty': '', 'CumQty': '', 'LeavesQty': '', 'Text': '', 'SideTradeID': '', 'CrossID': '', 'SelfMatchPreventionID': '', \
+                'Price': '', 'StopPx': '', 'LastPx': '', 'PartyDetailsListReqID': '', 'OrdStatus': ''}
+                        
+                # Processing Data
+                process_json = {'SendingTimeEpoch': '', 'ManualOrderIndicator': '', 'SenderID': '', 'Side': '', 'OrdType': '', 'TimeInForce': '', \
+                'OFMOverride': '', 'AggressorIndicator': '', 'ExecRestatementReason': '', 'SelfMatchPreventionInstruction': '', 'CustOrderCapacity': '', \
+                'AvgPxInd': '', 'ClearingAccountType': '', 'ClearingTradePriceType': '', 'CustOrderHandlingInst': '', 'DisplayQty': ''}
+                
+                for d in straight_json:
+                    test = get_value(d)
+                    if test != "None" or test != "":
+                        if "," in str(test):
+                            test = test.replace(",",";")
+                        if "4294967295" in str(test):
+                            test = ""
+                        if "us;" in str(test):
+                            test = test.upper()
+                        if d == "SelfMatchPreventionID" and len(str(test)) > 8:
+                            test = ""
+                        straight_json[d] = test
+                    else:
+                        straight_json[d] = ''
+                        
+                for d in process_json:
+                    if d in json_data:
+                        function = locals()[d]
+                        test = function(json_data[d])
+                        if test != "None":
+                            process_json[d] = test
+                        else:
+                            process_json[d] = ''
+                    elif d == "SenderID":
+                        function = locals()[d]
+                        test = function(SenderID)
+                        if test != "None":
+                            process_json[d] = test
+
+                # Party Message Fields
+                # Column 51 - Executing Firm ID
+                if "NoPartyDetails" in json_data:
+                    EXECUTING_FIRM_ID = partyid(json_data['NoPartyDetails'], "51")
+                    # Column 52 - Party Account Number
+                    ACCOUNT_NUMBER = partyid(json_data['NoPartyDetails'], "52")
+                
+                line_to_write = process_json['SendingTimeEpoch'] + "," + DIRECTION + "," + process_json['SenderID'] + "," + SESSIONID + "," + process_json['ManualOrderIndicator'] + "," + \
+                MessageType(NAME,straight_json['OrdStatus']) + "," + str(straight_json['SecurityID']) + "," + str(straight_json['ClOrdID']) + "," + str(straight_json['OrderID']) + "," + str(straight_json['ClOrdID']) + "," + \
+                str(straight_json['OrderRequestID']) + "," + process_json['Side'] + "," + str(straight_json['OrderQty']) + "," + str(straight_json['Price']) + "," + \
+                str(straight_json['StopPx'])  + "," + process_json['OrdType'] + "," + process_json['TimeInForce'] + "," + process_json['OFMOverride'] + "," + str(process_json['DisplayQty']) + "," + \
+                blank_json['MinQty'] + "," + straight_json['Location'] + "," + str(straight_json['LastPx']) + "," + str(straight_json['LastQty']) + "," + str(straight_json['CumQty']) + "," + \
+                str(straight_json['LeavesQty']) + "," + process_json['AggressorIndicator'] + "," + process_json['ExecRestatementReason'] + "," + str(straight_json['Text']) + "," + \
+                str(straight_json['SideTradeID']) + "," + str(straight_json['CrossID']) + "," + blank_json['QuoteReqID'] + "," + blank_json['QuoteID'] + "," + blank_json['QuoteSetID'] + "," + \
+                blank_json['QuoteEntryID'] + "," + blank_json['BidPx'] + "," + blank_json['BidSize'] + "," + blank_json['OfferPx'] + "," + blank_json['OfferSize'] + "," + \
+                blank_json['NoProcessedEntries'] + "," + str(straight_json['PartyDetailsListReqID']) + "," + blank_json['ListUpdateAction'] + "," + str(straight_json['SelfMatchPreventionID']) + "," + \
+                process_json['SelfMatchPreventionInstruction'] + "," + process_json['CustOrderCapacity'] + "," + blank_json['AvgPxGroupID'] + "," + process_json['AvgPxInd'] + "," + \
+                blank_json['CmtaGiveUpCD'] + "," + str(process_json['ClearingAccountType']) + "," + process_json['ClearingTradePriceType'] + "," + process_json['CustOrderHandlingInst'] + "," + \
+                EXECUTING_FIRM_ID + "," + str(ACCOUNT_NUMBER) + "," + blank_json['TAKE_UP_FIRM'] + "," + blank_json['TAKE_UP_ACCOUNT']
+
+                
+                #print(line_to_write)
+
+                with open (args.output_file, 'a') as the_file:
+                    the_file.write(line_to_write)
+                    the_file.write("\n")
+        
+
+
+
+ 

src/data_files/files/audit_upload.sh

@@ -0,0 +1,73 @@
+#/bin/bash
+
+function help_and_exit()
+{
+  echo
+  echo Usage: $0 [option]... config_file
+  echo 
+  echo Can also be used in cron like the following
+  echo 40 16 * * 1-5 /usr/bin/docker exec spark_10000 bash /sparkbin/data_files/audit_upload.sh /sparkdata/data/audit_uploads/example.conf
+  echo example.conf included.  The location of the conf file should be relative to the docker container, not the host
+  echo
+  echo Options are as follows:
+  echo "     -d|--date YYYYMMDD              Example is 20180731"
+  echo "  Help: "
+  echo "     -h|--help                       Display this message."
+  echo
+  exit 1
+}
+
+DATE=$(date +%Y%m%d)
+ORIG_DIR=$(pwd)
+ERROR_LEVEL="0"
+
+while getopts "d:h" option; do
+    case $option in
+        d) DATE=$OPTARG ;;
+        h) help_and_exit;;
+        \?) echo "Invalid option: -$OPTARG" >&2;
+        exit 1;;
+    esac
+done
+
+shift $((OPTIND-1))
+
+CONFIG_FILE=$1
+
+declare -A conf_vars; while read line; do conf_vars[${line%=*}]=${line#*=}; done <$CONFIG_FILE
+
+SESSION_ID=${conf_vars[session_id]}
+FCM=${conf_vars[fcm]}
+CLIENT=${conf_vars[client]}
+
+cd /sparkbin/data_files
+
+for SESSION in $(ls /sparkdata/data/ilink3/); do
+   if [[ $SESSION == *$SESSION_ID* ]]; then
+      echo "Session is $SESSION"
+      SESSION_DAY=$(date +%Y-%m-%d)
+      if [[ $SESSION == "cme_"* ]]; then 
+         SESSION_DAY=$(date -d "$DATE -$(date -d $DATE +%u) days" +"%Y-%m-%d")
+         AUDIT_SCRIPT="audit_ilink3.py"
+      fi
+      echo $SESSION_DAY
+      if [[ ! -d  /sparkdata/data/ilink3/$SESSION/audit_logs ]]; then
+         mkdir -p /sparkdata/data/ilink3/$SESSION/audit_logs
+      fi
+      if [[ -d /sparkdata/data/ilink3/$SESSION/$SESSION_DAY ]]; then
+         echo "Starting the creation of the audit log" 
+         python $AUDIT_SCRIPT -d $DATE /sparkdata/data/ilink3/$SESSION/$SESSION_DAY/log /sparkdata/data/ilink3/$SESSION/audit_logs/"$FCM"_audit_"$SESSION"_"$DATE".csv
+      fi
+      if [[ ! -d /sparkdata/data/md_audit ]]; then
+         mkdir /sparkdata/data/md_audit
+      fi
+      if [[ -d /sparkdata/data/md_audit ]]; then
+         # Running the MD Audit csv generation
+         cd /sparkbin
+         ./sparkd adminreport md > /sparkdata/data/md_audit/"$FCM"_md_"$CLIENT"_"$DATE".csv
+         cd /sparkbin/data_files
+      fi
+      echo "Uploading to the (s)FTP Server"
+      python audit_uploader.py --config_file="$CONFIG_FILE" --datestamp $DATE --session="$SESSION" --client="$CLIENT" --fcm="$FCM" 
+   fi 
+done

src/data_files/files/audit_uploader.py

@@ -0,0 +1,150 @@
+import getopt, sys, datetime
+import paramiko, ftplib
+
+#########################################################
+#                  Commandline Args                     #
+#########################################################
+
+def usage():
+    print "-------------------------------------------------------"
+    print "-Usage:                                               -"
+    print "-------------------------------------------------------"
+    print "-    --config_file= : Path to upload config file      -"
+    print "-                      -if no path is specified look  -"
+    print "-                       in the working directory.     -"    
+    print "-                                                     -"
+    print "-    --help         : Print this message              -"
+    print "-                                                     -"
+    print "-    --datestamp=   : Optional, defaults to today     -"
+    print "-                       -replaces *datestamp in the   -"
+    print "-                        the upload config file       -"
+    print "-                                                     -"
+    print "-    --fcm          : Optional, spark input           -"
+    print "-                                                     -"
+    print "-    --session      : Optional, spark jf session      -"
+    print "-------------------------------------------------------"
+
+fcm = ''
+session = ''
+
+try:
+   options, arguments = getopt.getopt(sys.argv[1:],"h",['config_file=', 'datestamp=', 'help', 'fcm=', 'client=', 'session='])
+except getopt.GetoptError, err:
+    print str(err)
+    usage()
+    sys.exit(2)
+
+for option, value in options:
+    if option in ("-h","--h","-help","--help"):
+        usage()
+        sys.exit()
+    elif option in ("-config_file","--config_file"):
+        config_file = value
+    elif option in ("-datestamp","--datestamp"):
+        datestamp = value
+    elif option in ("-fcm","--fcm"):
+        fcm = value
+    elif option in ("-client","--client"):
+        client = value
+    elif option in ("-session","--session"):
+        session = value
+
+for argument in arguments:
+    if argument in ("h", "help"):
+        usage()
+        sys.exit()
+    else:
+        print "Unrecognized commandline argument. Type help for usage."
+        sys.exit()
+
+if len(options) <= 0:
+    print "Requires a config file to run."
+    usage()
+    sys.exit()
+
+
+#########################################################
+#              Read Config Function                     #
+#########################################################
+
+def read_config_file(config_file, datestamp=datetime.datetime.today().strftime('%Y%m%d'), fcm=fcm, session=session):
+        '''
+        Parse our config file that gets passed in as a commandline parameter.
+        All config keys must be of the form key=value. White space is ignored.
+        All config keys must be seperated by a newline character.
+        Called during initialization.
+        '''
+        upload_config = {}
+        config_file = open(config_file, 'r')        
+        config_file = config_file.read()
+        for config in config_file.split("\n"):
+            if '=' in config:
+                cleaned_key = config.split('=')[0].strip().upper()
+                cleaned_value = config.split('=')[1].strip()
+                if '*datestamp*' in cleaned_value:
+                    cleaned_value = cleaned_value.replace('*datestamp*',datestamp)
+                if '*fcm*' in cleaned_value:
+                    cleaned_value = cleaned_value.replace('*fcm*',fcm)
+                if '*client*' in cleaned_value:
+                    cleaned_value = cleaned_value.replace('*client*',client)
+                if '*session*' in cleaned_value:
+                    cleaned_value = cleaned_value.replace('*session*',session)                    
+                upload_config[cleaned_key] = cleaned_value
+            else:
+                pass
+        return upload_config   
+
+#########################################################
+#             FTP File Transfer Function                #
+#########################################################
+def putfile_ftp(file, put_path, site, user, password, port=21, verbose=True):
+    if verbose: print 'Uploading', file
+    local = open(file, 'rb')    
+    remote = ftplib.FTP(site)   
+    remote.login(user, password)         
+    remote.storbinary(put_path, local, 1024)
+    remote.quit()
+    local.close()
+    if verbose: print 'Upload done.'
+
+#########################################################
+#            SFTP File Transfer Function                #
+#########################################################
+def putfile_sftp(file, put_path, site, username, password, port=22, verbose=True):
+    transport = paramiko.Transport((site, port))
+    transport.connect(username = username, password = password)
+    sftp = paramiko.SFTPClient.from_transport(transport)
+    sftp.put(file, put_path)
+    sftp.close()
+    transport.close()
+
+
+#########################################################
+#                       Main                            #
+#########################################################
+#if datestamp != None:
+#    upload_config = read_config_file(config_file, datestamp)
+#else:
+#    upload_config = read_config_file(config_file)
+
+try:
+    datestamp
+except NameError:
+    upload_config = read_config_file(config_file)
+else:
+    upload_config = read_config_file(config_file, datestamp)
+
+
+
+print upload_config
+
+if upload_config["PROTOCOL_PUT"] in ['FTP','ftp']:
+    print "Protocol = FTP"
+    putfile_ftp(upload_config["GET_ILINK3_FILE_PATH"] + upload_config["GET_ILINK3_FILE_NAME"], upload_config["PUT_ILINK3_FILE_PATH"] + upload_config["PUT_ILINK3_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], upload_config["PORT_PUT"] )
+    putfile_ftp(upload_config["GET_MD_FILE_PATH"] + upload_config["GET_MD_FILE_NAME"], upload_config["PUT_MD_FILE_PATH"] + upload_config["PUT_MD_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], upload_config["PORT_PUT"] )
+elif upload_config["PROTOCOL_PUT"] in ['SFTP','sftp']:
+    print "Protocol = SFTP"
+    putfile_sftp(upload_config["GET_ILINK3_FILE_PATH"] + upload_config["GET_ILINK3_FILE_NAME"], upload_config["PUT_ILINK3_FILE_PATH"] + upload_config["PUT_ILINK3_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], int(upload_config["PORT_PUT"]) )
+    putfile_sftp(upload_config["GET_MD_FILE_PATH"] + upload_config["GET_MD_FILE_NAME"], upload_config["PUT_MD_FILE_PATH"] + upload_config["PUT_MD_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], int(upload_config["PORT_PUT"]) )
+
+

src/data_files/files/example.conf

@@ -0,0 +1,16 @@
+fcm=FCM
+client=NAME
+session_id=ses
+put_host=ftpc.example.com
+port_put=2222
+username_put=user
+password_put=password
+protocol_put=sftp
+get_ilink3_file_path=/sparkdata/data/ilink3/*session*/audit_logs/
+put_ilink3_file_path=/
+get_ilink3_file_name=*fcm*_audit_*session*_*datestamp*.csv
+put_ilink3_file_name=*fcm*_audit_*session*_*datestamp*.csv
+get_md_file_path=/sparkdata/data/md_audit/
+put_md_file_path=/
+get_md_file_name=*fcm*_md_*client*_*datestamp*.csv
+put_md_file_name=*fcm*_md_*client*_*datestamp*.csv

src/lib/api/service_compiler/plugin.py

@@ -319,7 +319,7 @@ if __name__ == "__main__":
 
     if DEBUG: 
         for f in response.file:
-            print f.content
+            print (f.content)
     else:
         output = response.SerializeToString()
         sys.stdout.write(output)

src/lib/logging/logging.cpp

@@ -120,7 +120,7 @@ void configure_logging(bts::config::Logging const&cfg, bts::config::ThreadGroupC
                                 {log::Plugin,"Plugin"},
                                 {log::Sdk,"Sdk"},
                                 {log::Http,"Http"},
-                                {log::iLink3,"ilink3"},
+                                {log::iLink3,"iLink3"},
    };
 
    real_cfg.sinks["console"] =

src/lib/md/IntPrice.h

@@ -60,10 +60,13 @@ class IntPrice {
          return delta.mantissa() < 0;
       }
    }
-   bool operator <=(const IntPrice& other) {
+   bool operator <=(const IntPrice& other) const {
       return *this == other || *this < other;
    }
-   bool operator >=(const IntPrice& other) {
+   bool operator >(const IntPrice& other) const {
+      return other < *this;
+   }
+   bool operator >=(const IntPrice& other) const {
       return !(*this < other);
    }
    IntPrice operator - (const IntPrice& o) const {
@@ -72,9 +75,31 @@ class IntPrice {
    IntPrice operator + (const IntPrice& o) const {
       return op(o, [](int64_t m1, int64_t m2) { return m1 + m2; });
    }
+   IntPrice& operator +=(const IntPrice& o) {
+      auto temp = *this + o;
+      _mantissa = temp._mantissa;
+      _exponent = temp._exponent;
+      return *this;
+   }
+   IntPrice& operator -=(const IntPrice& o) {
+      auto temp = *this - o;
+      _mantissa = temp._mantissa;
+      _exponent = temp._exponent;
+      return *this;
+   }
+   IntPrice operator -() {
+      return IntPrice(-_mantissa, _exponent);
+   }
    IntPrice operator * (const IntPrice& other) const {
       return op(other, [](int64_t m1, int64_t m2) { return m1 * m2; });
    }
+   IntPrice operator * (const int& other) const {
+      return IntPrice(_mantissa * other, _exponent);
+   }
+   IntPrice& operator *=(const int& other) {
+      _mantissa *= other;
+      return *this;
+   }
    IntPrice operator / (double v) const {
       return { (int64_t)(_mantissa / v), exponent() };
    }
@@ -183,4 +208,4 @@ inline std::ostream& operator<<(std::ostream& os, const IntPrice& p) {
    return os;
 }
 
-}}
+}}

src/lib/orderman/actors/GLBXiLink3ExecutionGateway.cpp

@@ -235,7 +235,7 @@ void GLBXiLink3ExecutionGateway::on_admin_data(
     const mercury::Update<NewAdminDataEvent> &) {}
 
 void GLBXiLink3ExecutionGateway::send_msgs(std::vector<OrderMsg> const &msgs,
-                                           trace::CorID) {
+                                           trace::CorID corid) {
 
    if (me->state == State::READY) {
       for (auto &m : msgs) {
@@ -243,19 +243,24 @@ void GLBXiLink3ExecutionGateway::send_msgs(std::vector<OrderMsg> const &msgs,
          auto create_ptr = std::get_if<CreateOrderMsg>(&m);
          if (create_ptr) {
             me->gw->send_order(*create_ptr);
-            break;
+            continue;
          }
          // CancelReplace
          auto cancel_replace_ptr = std::get_if<CancelReplaceOrderMsg>(&m);
          if (cancel_replace_ptr) {
+            LOG(iLink3,Info,"Cancel Replace. CorID:{}.{} Order:{}",
+                log::CharPtrWrapper{corid.channel},
+                corid.sn,
+                cancel_replace_ptr->exchange_order_id);
+                
             me->gw->send_cancel_replace(*cancel_replace_ptr);
-            break;
+            continue;
          }
          // Cancel
          auto cancel_ptr = std::get_if<CancelOrderMsg>(&m);
          if (cancel_ptr) {
             me->gw->send_cancel(*cancel_ptr);
-            break;
+            continue;
          }
       }
       me->gw->flush();

src/lib/orderman/actors/OrderStore.cpp

@@ -101,6 +101,8 @@ void OrderStore::on_order_state_update(
       for (auto const &ev : *u.events) {
 
          auto &rec_info = me->known_orders[ev.key];
+         LOG(iLink3,Info, "Received order state update for {}",
+             ev.key.value());
          auto &rec = rec_info.rec;
          bts_assert(rec.has_def());
          rec.set_key(ev.key.value());
@@ -123,6 +125,8 @@ void OrderStore::on_order_definition_update(
    std::set<OrderKey> modified_orders;
    if (u.events)
       for (auto const &ev : *u.events) {
+         LOG(iLink3,Info, "Received order definition update for {}",
+             ev.key.value());
          auto &rec_info = me->known_orders[ev.key];
          auto &rec = rec_info.rec;
          rec.set_key(ev.key.value());

src/lib/orderman/actors/ilink3/IL3GW.cpp

@@ -176,12 +176,7 @@ uint64_t IL3GW::send_party_details(const api::orderman::RoutingRecord &route) {
       msg.selfMatchPreventionID(
           PartyDetailsDefinitionRequest518::selfMatchPreventionIDNullValue());
    } else {
-      // CME needs the SMP_ID to be 12 characters or less
-      // after they to_string() the uint64
-      bts_assert_d(route.smp_id().size() == 8);
-      const uint64_t *psmp_id =
-          reinterpret_cast<const uint64_t *>(&route.smp_id());
-      uint64_t smp_id = (*psmp_id) % (2UL << 39);
+      auto smp_id = std::atoi(route.smp_id().c_str());
       msg.selfMatchPreventionID(smp_id);
    }
    msg.cmtaGiveupCD(CmtaGiveUpCD::NULL_VALUE);

src/lib/orderman/actors/ilink3/ILink3Utils.h

@@ -25,7 +25,7 @@ iLinkBinary::SideReq::Value map_side(api::Side side);
 api::Side map_side(iLinkBinary::SideReq::Value side);
 
 inline int64_t to_price9(Price p) {
-   return p.value() * 1E9;
+   return std::round(p.value() * 1E9);
 }
 
 iLinkBinary::OrderTypeReq::Value map_order_type(api::orderman::NativeOrderType ot);

src/lib/orderman/actors/ilink3/TimeKeeper.h

@@ -18,7 +18,7 @@ struct TimeKeeper {
    uint64_t now() const {
       auto now = utils::TimeStamp::now();
       auto delta = now - start;
-      return unix_time + delta.microseconds();
+      return unix_time + delta.nanoseconds();
    }
 };
 

src/lib/orderman/doc/ilink3.org

@@ -0,0 +1,179 @@
+* iLink3 Reference Documentation
+** Overview
+The iLink3 work leverages and extends the iLink2 functionality. Most of the order management logic is unchanged from iLink2 with the substantial changes being confined to session management and encoding/decoding. In an effort to facilitate future addition of new execution protocols a set of protocol independent data structures and methods was added. 
+**  Main classes
+***  GLBXNativeorder
+GLBXNativeOrder implements the common iLink2 and iLink3 order management logic, including status updates, buffering of cancel/updates, recovery from disk, handling of execution reports, special order handling and routing. This is essentially unchanged from iLink2. From the point of view of this class the differences between iLink2 and iLink2 are delegated to the IGLBXOrderEncoder interface. At run time depending on the type of connection used an implementation of this interface for iLink2 or iLink3 is selected.
+*** IGLBXOrderEncoder
+Implementations of the IGLBXOrderEncoder interface below realize wire protocol specific implementations of encoding allowing GLBXNativeOrder to manage both iLink2 and iLink3 orders. The interface provides methods for creating,updating, canceling and dispatching orders. Currently there are two concrete implementations of this interface: iLink2OrderEncoder and iLink3OrderEncoder.
+
+#+BEGIN_SRC c++
+class IGLBXOrderEncoder {
+ public:
+   virtual ~IGLBXOrderEncoder() {}
+   virtual void init(OrderKey key, api::orderman::RoutingRecord const &,
+                     api::Security const &security,
+                     api::orderman::OrderDefinition const &def,
+                     int price_encoding_precision,
+                     bool stop_order_has_been_triggered) = 0;
+
+   virtual void update_exchange_order_id(ExchangeOrderIDVar const&) = 0;
+
+   virtual void update_create_order_encoding(Price p, int price_precision,
+                                             Quantity q, OrderServices &) = 0;
+
+   virtual void update_cancel_replace_stop_px(Price stop_px,
+                                              int price_precision) = 0;
+   virtual void update_cancel_replace_max_show(Quantity max_show) = 0;
+   virtual void update_cancel_replace_encoding(Price p, int price_precision,
+                                               Quantity q, OrderServices &) = 0;
+   virtual void update_cancel_replace_encoding(Quantity q, OrderServices &) = 0;
+   virtual void update_cancel_replace_encoding(Price p,
+                                               int price_precision,
+                                               OrderServices &) = 0;
+   virtual void update_cancel_encoding(OrderServices&) = 0;
+   virtual void send_create(OrderServices&,SessionServices&) = 0;
+   virtual void send_cancel_replace(OrderServices&,SessionServices&) = 0;
+   virtual void send_cancel(OrderServices&,SessionServices&) = 0;
+};
+
+ #+END_SRC
+*** class iLink2OrderEncoder : public IGLBXOrderEncoder 
+iLink2OrderEncoder provides FIX based encoding of GLBX orders. This logic is carried over essentially unchanged from previous versions of Spark with care being taken to re-use as much as possible elements of previous encodings on every call. The generated FIX messages are forwarded, as FIX, to the OrderServices::queue_fix template member function:
+
+#+BEGIN_SRC c++
+void queue_fix(SessionServices &ss, jetfix::MessageTypes mt,
+                  jetfix::FIXFragment const &f) {
+      ss.queue_fix(mt, f, corid());
+}
+ #+END_SRC
+
+*** class iLink3OrderEncoder : public IGLBXOrderEncoder 
+iLink3OrderEncoder provides a thin adapter that accepts order encoding requests from GLBXNativeOrder and maps those requests to the protocol agnostic order management data structures: CreateOrderMsg, CancelReplaceOrderMsg and CancelOrderMsg.  
+
+After this mapping, iLink3OrderEncoder forwards the order operation in question to its OrderServices instance, through the protocol independent queue_order_msg template function:
+
+#+BEGIN_SRC c++
+   template <class OP>
+   void queue_order_msg (SessionServices &ss, OP const&msg) {
+      ss.queue_order_msg(msg,corid());
+   }
+ #+END_SRC
+
+*** Protocol independent  order operation data structures
+The simple data structures in IExecutionGateway.h provide an exchange protocol independent way of representing order operations. They are intended to facilitate the addition of new exchange protocols. They are as follows:
+
+#+BEGIN_SRC c++
+  struct CreateOrderMsg {
+     OrderKey key;
+     Price price;
+     Quantity quantity;
+     api::Security const *security;
+     api::Side side;
+     Price stop_px;
+     Quantity display_quantity;
+     api::orderman::NativeOrderType order_type;
+     api::orderman::TimeInForce time_in_force;
+     api::orderman::ManualOrderIndicator manual_order_indicator;
+     api::orderman::RoutingRecord const *routing_record;
+  };
+
+  struct CancelReplaceOrderMsg {
+     CreateOrderMsg create_fields;
+     ExchangeOrderID64 exchange_order_id;
+  };
+
+  struct CancelOrderMsg {
+     OrderKey key;
+     ExchangeOrderID64 exchange_order_id;
+     api::orderman::ManualOrderIndicator manual_order_indicator;
+     api::Security const *security;
+     api::orderman::RoutingRecord const *routing_record;
+     api::Side side;
+  };
+
+  using OrderMsg = 
+     std::variant<CreateOrderMsg, CancelReplaceOrderMsg, CancelOrderMsg>;
+
+ #+END_SRC
+
+These data structures are used as vocabulary data types understood by all future exchange execution implementation code. They are consumed by implementations of the IExecutionGateway interface.
+
+*** class IExecutionGateway
+IExecutionGateway provides an abstraction for the implementation of exchange gateways. It provides methods for both FIX and non FIX based exchanges. Ordinarily implementations of this interface will either provide FIX or non FIX functionality but not both.
+
+#+BEGIN_SRC c++
+class IExecutionGateway {
+ public:   
+   IExecutionGateway() = default;
+   virtual ~IExecutionGateway() {}
+
+   virtual void start() = 0;
+   virtual void dispatch_incoming_fix_messages() = 0;
+   virtual void on_securities(const mercury::Update<NewSecuritiesEvent> &) = 0;
+   virtual void on_admin_data(const mercury::Update<NewAdminDataEvent> &) = 0;
+   virtual void send_fix(const jetfix::OutFIXPtr&, trace::CorID) = 0;
+   virtual void send_msgs(std::vector<OrderMsg> const&, trace::CorID) = 0;
+   virtual OnExeReport& on_exe_report() = 0;
+   virtual OnGwStatusChange& on_gw_status_change() = 0;
+   virtual OnCancelReject &on_cancel_reject() = 0;
+private:
+   BTS_NO_COPY(IExecutionGateway);
+};
+ #+END_SRC
+
+ IExecutionGateway implements are responsible for establishing execution sessions as well as mapping to and from protocol independent order operations to their gateway specific implementations. Presently there are three implementations of IExecutionGateway: ICE, iLink2 and iLink3. Only the iLink3 implementation is new. 
+ 
+*** class GLBXiLink3ExecutionGateway : public IExecutionGateway 
+This class implements provides a facade that hides, coordinates and interfaces with all the components needed to implement an iLink3 gateway including a TCP connection, a persistent store and the actual iLink3 encoding/decoding logic. GLBXiLink3ExecutionGateway role is identical to those of the GLBXExecutionGateway and ICEExecutionGateway, which existed prior to iLink3. In essence GLBXilink3executiongateway is in charge of managing the life-cycle of an iLink3 exchange connection as well as translating between exchange agnostic order operations and the iLink3 specific interfaces. The actual exchange interaction is delegated to the IL3GW class.
+*** class IL3GW : IL3ConnectionHandler
+The IL3GW class is responsible for iLink3 session management, including login, logout, recovery and heartbeat response. Additionally this class is responsible for mapping order messages to and from their network binary representation. This class is able to handle network events including connection, disconnection and data arrival via its implementation of the IL3ConnectionHandler interface:
+
+
+#+BEGIN_SRC c++
+struct IL3ConnectionHandler {
+   virtual ~IL3ConnectionHandler() ;
+   virtual void on_connect_success() = 0;
+   virtual void on_disconnect() = 0;
+   // Send available data and length. Handler will return what it consumed
+   virtual size_t on_data(const std::string_view&) = 0;
+};
+ #+END_SRC
+
+IL3GW integrates these external events with the instructions it receives from the rest of the system through its public interface:
+
+#+BEGIN_SRC c++
+   void connect();
+   void send_order(const CreateOrderMsg &);
+   void send_cancel_replace(const CancelReplaceOrderMsg &);
+   void send_cancel(const CancelOrderMsg &);
+   void send_terminate(uint16_t code = 0);
+ #+END_SRC
+
+ And provides an integrated high-level view of the connection state via its event callbacks:
+
+#+BEGIN_SRC c++
+   void subscribe_state(OnStateChange o);
+   void subscribe_execution_reports(OnExecutionReport sub);
+   void subscribe_session(OnSessionChange cb); 
+   void subscribe_cancel_reject(OnCancelReject cb); 
+ #+END_SRC
+
+** Code Generation
+To facilitate encoding IL3GW uses code generated by the CME sponsored tools by real logic. These tools can be found here:
+
+https://github.com/real-logic/simple-binary-encoding/wiki/Sbe-Tool-Guide
+
+The CME provides template files that be used as an input to these tools. For more details see:
+
+https://www.cmegroup.com/confluence/display/EPICSANDBOX/iLink+3+-+Simple+Binary+Encoding
+
+The generation was done manually and the generated code integrated into source control. 
+
+** Operational Notes
+- Like iLink2, the iLink3 implementation uses weekly sessions
+- Sessions are stored under sparkdata/ilink3/session_name/date-of-session
+- Within each session directory there are two files:
+  - log: is a json log of all messages sent and received from the exchange
+  - seqnum: contains the current sequence numbers for the session. Additionally the session binary name is also represented for use in trouble-shooting
+    

src/lib/orderman/orderman_utils/BookUtils.cpp

@@ -1,9 +1,9 @@
 #include "BookUtils.h"
 
-namespace bts {
-namespace orderman {
-using Size = bts::md::Size;
-using Side = bts::api::Side;
+namespace bts::orderman {
+using Size = md::Size;
+using Side = api::Side;
+using PriceSize = md::PriceSize;
 
 Size BookUtils::volume_at_or_better(const Book& book, Side side, Price price) {
    return volume_at_or_better(book, side, price.value());
@@ -22,5 +22,154 @@ Size BookUtils::volume_at_or_better(const Book& book, Side side, double price) {
 SidePriceComp BookUtils::price_comp(Side side) {
    return SidePriceComp(side);
 }
-} // namespace orderman
-} // namespace bts
+
+inline bool is_empty(const PriceSize& ps) {
+   return ps.size() == 0 || ps.int_price().is_nan();
+}
+
+void BookUtils::change_unit_qty(
+   const BookSide& book_in, std::vector<md::PriceSize>& book_out, int unit_qty) {
+   //
+   bts_assert(unit_qty > 0);
+   int mlev = 0, mlev_qty_used = 0;
+   std::vector<md::PriceSize> unit_lev_span;
+   for (uint32_t i = 0; i < book_in.size();) {
+      int mlev_qty_rem = book_in[i].size() - mlev_qty_used;
+      if (mlev_qty_rem == 0) {
+         mlev_qty_used = 0;
+         mlev++;
+         i++;
+         continue;
+      }
+      if (unit_lev_span.size() == 0) {
+         md::Size mlev_units_rem = mlev_qty_rem / unit_qty;
+         if (mlev_units_rem > 0) {
+            book_out.emplace_back(book_in[i].int_price(), mlev_units_rem);
+            mlev_qty_used += mlev_units_rem * unit_qty;
+         } else {
+            unit_lev_span.emplace_back(book_in[i].int_price(), mlev_qty_rem);
+            mlev_qty_used = 0;
+            mlev++;
+            i++;
+         }
+      } else {
+         // the unit spans across multiple [input]price levels: compute its
+         // [output]price. It will be in its own [output]price level.
+         auto unit_rem = unit_qty;
+         for (auto lev : unit_lev_span)
+            unit_rem -= lev.size();
+         if (mlev_qty_rem >= unit_rem) {
+            mlev_qty_used += unit_rem;
+            auto unit_price = md::INT_NAN;
+            for (uint32_t adj_lev = 0; adj_lev < unit_lev_span.size(); adj_lev++) {
+               auto leg_price = unit_lev_span[adj_lev].int_price() *
+                                static_cast<int>(unit_lev_span[adj_lev].size());
+               if (unit_price.is_nan())
+                  unit_price = leg_price;
+               else
+                  unit_price += leg_price;
+            }
+            unit_lev_span.clear();
+            book_out.emplace_back(unit_price, 1);
+         }
+      }
+   }
+}
+
+void BookUtils::compute_synthetic_book(
+   std::vector<PriceSize>& dest,
+   const std::vector<std::tuple<Size, md::IntPrice, BookSide>>&
+      leg_mkts,
+   uint32_t depth) {
+   //
+   if (leg_mkts.size() == 0)
+      return;
+
+   struct Leg {
+      std::vector<PriceSize> book;
+      uint32_t curr_level;
+      Size curr_level_qty_used;
+      int leg_qty;
+
+      Leg() : book(), curr_level(0), curr_level_qty_used(0), leg_qty(0) {}
+
+      md::IntPrice curr_price() const {
+         // Assert that the book isn't 'empty' yet.
+         // Note: one could argue that nan be returned in this event,
+         // but since the loop is designed to return "greedily,"
+         // 'curr_price' should not be accessed after it has been emptied.
+         // This assertion enforces that the behavior is only changed
+         // as a result of deliberate action, rather than an inadvertant
+         // side-effect.
+         bts_assert(!is_empty());
+         return book[curr_level].int_price();
+      }
+      Size curr_level_qty_rem() const {
+         return book[curr_level].size() - curr_level_qty_used;
+      }
+      // The implied market's volume is limited by its thinnest market.
+      // When one leg 'is_empty,' the full volume of the implied market
+      // has been realized.
+      bool is_empty() const { return curr_level >= book.size(); }
+   };
+
+   std::vector<Leg> legs;
+   for (const auto& [qty, price_mult, book] : leg_mkts) {
+      // todo: implement price mult logic (for now just assert nan or 1)
+      bts_assert(price_mult.is_nan() || price_mult.price() == 1);
+      auto& l = legs.emplace_back();
+      bts_assert(qty != 0);
+      l.leg_qty = qty;
+      if (book.empty() || is_empty(book.front()))
+         return;
+      change_unit_qty(
+         book, l.book, std::abs(l.leg_qty));
+      if (l.book.empty())
+         return;
+   }
+   
+   auto any_legs_empty = [&]() {
+      for (auto& l : legs)
+         if (l.is_empty())
+            return true;
+      return false;
+   };
+
+   while (!any_legs_empty() && depth-- > 0) {
+      // legs can't be empty, because it is a 1:1 mapping of leg_mkts,
+      // and it has already been established that the latter isn't empty.
+      auto limiting_factor = legs[0].curr_level_qty_rem();
+      for (uint32_t i = 0; i < legs.size(); i++) {
+         if (legs[i].curr_level_qty_rem() == 0) {
+            legs[i].curr_level++;
+            if (legs[i].is_empty()) {
+               // book is complete
+               return;
+            }
+            legs[i].curr_level_qty_used = Size(0);
+         }
+         limiting_factor =
+            std::min(legs[i].curr_level_qty_rem(), limiting_factor);
+         if (limiting_factor == 0)
+            return;
+      }
+      bts_assert(limiting_factor > 0);
+      for (uint32_t i = 0; i < legs.size(); i++)
+         legs[i].curr_level_qty_used += limiting_factor;
+      auto marginal_cost = md::INT_NAN;
+      for (auto& l : legs) {
+         // The book's prices have already been multiplied by abs(leg_qty);
+         // Subtract the leg's cost if qty is negative, otherwise add it
+         // to the synthetic's marginal cost.
+         auto lcost = l.curr_price();
+         if (l.leg_qty < 0)
+            lcost = -lcost;
+         if (marginal_cost.is_nan())
+            marginal_cost = lcost;
+         else
+            marginal_cost += lcost;
+      }
+      dest.emplace_back(marginal_cost, limiting_factor);
+   }
+}
+} // namespace bts::orderman

src/lib/orderman/orderman_utils/BookUtils.h

@@ -6,13 +6,18 @@
 #include "lib/orderman/orderman_utils/Book.h"
 #include "lib/orderman/orderman_utils/PriceComp.h"
 
-namespace bts {
-namespace orderman {
-
+namespace bts::orderman {
 struct BookUtils {
    static md::Size volume_at_or_better(const Book& book, api::Side side, Price price);
    static md::Size volume_at_or_better(const Book& book, api::Side side, double price);
    static SidePriceComp price_comp(api::Side side);
+
+   static void change_unit_qty(
+      const BookSide& book_in, std::vector<md::PriceSize>& book_out, int unit_qty);
+
+   static void compute_synthetic_book(
+      std::vector<md::PriceSize>& dest,
+      const std::vector<std::tuple<md::Size, md::IntPrice, BookSide>>& leg_mkts,
+      uint32_t depth);
 };
-} // namespace orderman
-} // namespace bts
+} // namespace bts::orderman

src/lib/orderman/orders/GLBXNativeOrder.cpp

@@ -861,7 +861,7 @@ void GLBXNativeOrder::on_exe_report(const NewExecutionReportEvent &er) {
    auto external_status = map_external_status(er.data->order_status);
 
    // filled quantity is reset back to 0 when the order is canceled
-   if (external_status != ORDER_STATUS_CANCELED) {
+   if (external_status != ORDER_STATUS_CANCELED ) {
       if (er.data->exec_type == api::orderman::EXEC_TYPE_PARTIAL_FILL ||
           er.data->exec_type == api::orderman::EXEC_TYPE_FILL) {
 
@@ -876,9 +876,9 @@ void GLBXNativeOrder::on_exe_report(const NewExecutionReportEvent &er) {
              (previous_sz + fill_sz);
 
          state.set_avg_price(new_avg);
+         state.set_filled_quantity(er.data->filled_quantity.value());
+         state.set_last_price(er.data->last_price.value());
       }
-      state.set_filled_quantity(er.data->filled_quantity.value());
-      state.set_last_price(er.data->last_price.value());
    }
 
    state.set_last_msg(er.data->msg);