Compare commits
8
Commits
2.17.0_rc3
...
2.17.0_rc8
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
167c8c23a0 | ||
|
|
d235f4be41 | ||
|
|
ec43dbd82d | ||
|
|
f07998d6a6 | ||
|
|
b3dc2cdcd4 | ||
|
|
0963aee137 | ||
|
|
c2f8e92b10 | ||
|
|
7ae0142dff |
No files matched your search
@@ -1 +1 @@
|
||||
set(sources files/cert_secure_logon.txt;files/futures.coconut.xz;files/onixs_ice/ConnectivityConfiguration.APITest.xml;files/onixs_ice/ConnectivityConfiguration.PerfTest.xml;files/onixs_ice/ConnectivityConfiguration.Production.xml;files/perf_trace/compiled.css;files/perf_trace/index.html;files/perf_trace/jquery-3.3.1.min.js;files/perf_trace/main.js;files/perf_trace/perf_log.sql;files/perf_trace/perf_trace_analysis.sql;files/perf_trace/plotly-latest.min.js)
|
||||
set(sources files/audit_ilink3.py;files/audit_upload.sh;files/audit_uploader.py;files/cert_secure_logon.txt;files/example.conf;files/futures.coconut.xz;files/onixs_ice/ConnectivityConfiguration.APITest.xml;files/onixs_ice/ConnectivityConfiguration.APITest.xml.orig;files/onixs_ice/ConnectivityConfiguration.PerfTest.xml;files/onixs_ice/ConnectivityConfiguration.Production.xml;files/perf_trace/compiled.css;files/perf_trace/index.html;files/perf_trace/jquery-3.3.1.min.js;files/perf_trace/main.js;files/perf_trace/perf_log.sql;files/perf_trace/perf_trace_analysis.sql;files/perf_trace/plotly-latest.min.js)
|
||||
@@ -0,0 +1,405 @@
|
||||
import json
|
||||
import csv
|
||||
import datetime, time
|
||||
import argparse, os
|
||||
|
||||
|
||||
def valid_file(f):
|
||||
if not os.path.exists(f):
|
||||
# Argparse uses the ArgumentTypeError to give a rejection message like:
|
||||
# error: argument input: x does not exist
|
||||
raise argparse.ArgumentTypeError("{0} does not exist".format(f))
|
||||
return f
|
||||
|
||||
def valid_date(s):
|
||||
try:
|
||||
return datetime.datetime.strptime(s, "%Y%m%d")
|
||||
except ValueError:
|
||||
raise argparse.ArgumentTypeError("Not a valid date: '{0}'.".format(s))
|
||||
|
||||
def print_headers():
|
||||
# From https://www.cmegroup.com/confluence/display/EPICSANDBOX/iLink+3+-+Minimum+Acceptable+Audit+Trail+Elements+-+Data+Definitions+and+Validation+Rules
|
||||
line_to_write = "Sending Timestamps,Message Direction,Operator ID,Session ID,Manual Order Indicator,Message Type,Instrument,Order Flow ID,\
|
||||
CME Globex Order ID,Client Order ID,Order Request ID,Buy/Sell Indicator,Quantity,Limit Price,Stop Price,Order Type,Order Qualifier,IFM Flag,\
|
||||
Display Quantity,Minimum Quantity,Country of Origin,Fill Price,Fill Quantity,Cumulative Quantity,Remaining Quantity,Aggressor Flag,Source of Cancellation,\
|
||||
Reject Reason,CME Globex Message ID,Cross ID,Quote Request ID,Message Quote ID,Quote Set ID,Quote Entry ID,Bid Price,Bid Size,Offer Price,Offer Size,\
|
||||
Processed Entries,Party Details List Request ID,ListUpdateAction,Self-Matching Prevention ID,Self-Match Prevention Instr,Customer Type Indicator,\
|
||||
AvgPxGroupID,Average Px Ind,CmtaGiveUpCD,Origin,Clearing Trade Price Type,Customer Handling Instr,Executing Firm ID,Account Number,Take Up Firm,Take up Account"
|
||||
|
||||
with open (args.output_file, 'w') as the_file:
|
||||
the_file.write(line_to_write)
|
||||
the_file.write("\n")
|
||||
|
||||
def set_blanks():
|
||||
return ""
|
||||
|
||||
def get_value(json_key):
|
||||
prices = ["Price", "LastPx", "StopPx"]
|
||||
if json_key in json_data:
|
||||
if json_key in prices:
|
||||
x = json_data[json_key]
|
||||
return x['mantissa']/100000000000
|
||||
else:
|
||||
return json_data[json_key]
|
||||
else:
|
||||
return ""
|
||||
|
||||
|
||||
# Column 1 - Sending Timestamps
|
||||
def SendingTimeEpoch(epochtime): #1
|
||||
#TIME = time.strftime('%Y%m%d-%H:%M:%S.{}'.format(epochtime/1000000000), time.gmtime(epochtime/1000000000))
|
||||
TIME = datetime.datetime.fromtimestamp(epochtime/1000000000).strftime('%Y%m%d-%H:%M:%S.%f')
|
||||
return TIME
|
||||
|
||||
# Column 2 - Message Direction
|
||||
def direction(whichway):
|
||||
if whichway == '<':
|
||||
DIRECTION = "FROM CME"
|
||||
else:
|
||||
DIRECTION = "TO CME"
|
||||
return DIRECTION
|
||||
|
||||
# Column 3 - Operator ID
|
||||
def SenderID(sender):
|
||||
if "SenderID" in json_data:
|
||||
SENDER = json_data["SenderID"]
|
||||
return SENDER
|
||||
else:
|
||||
return SESSIONID
|
||||
|
||||
|
||||
# Column 4 - Session ID
|
||||
|
||||
# Column 5 - Manual Order Indicator
|
||||
def ManualOrderIndicator(manual):
|
||||
if "ManualOrderIndicator" in json_data:
|
||||
if manual == "Manual":
|
||||
return "1"
|
||||
else:
|
||||
return "0"
|
||||
|
||||
# Column 6 - Message Type
|
||||
def MessageType(msg_type,orderstatus):
|
||||
if msg_type == "NewOrderSingle514":
|
||||
return "D"
|
||||
elif msg_type == "ExecutionReportNew522":
|
||||
return "8--0"
|
||||
if "ExecutionReportTrade" in msg_type:
|
||||
if orderstatus == "PartiallyFilled":
|
||||
return "8--1"
|
||||
else:
|
||||
return "8--2"
|
||||
elif msg_type == "ExecutionReportCancel534":
|
||||
return "8--4"
|
||||
elif msg_type == "ExecutionReportModify531":
|
||||
return "8--5"
|
||||
elif msg_type == "ExecutionReportCancel534":
|
||||
return "8--H"
|
||||
elif msg_type == "ExecutionReportReject523":
|
||||
return "8--8"
|
||||
elif msg_type == "OrderCancelReplaceReject536":
|
||||
return "9--2"
|
||||
elif msg_type == "OrderCancelReject535":
|
||||
return "9--1"
|
||||
elif msg_type == "PartyDetailsDefinitionRequest518":
|
||||
return "CX"
|
||||
elif msg_type == "PartyDetailsDefinitionRequestAck519":
|
||||
return "CY"
|
||||
elif msg_type == "OrderCancelRequest516":
|
||||
return "F"
|
||||
elif msg_type == "OrderCancelReplaceRequest515":
|
||||
return "G"
|
||||
|
||||
|
||||
|
||||
# Column 12 - Buy-Sell Indicator/Side
|
||||
def Side(indicator):
|
||||
if "Side" in json_data:
|
||||
if indicator == "Buy":
|
||||
return "1"
|
||||
else:
|
||||
return "2"
|
||||
|
||||
# Column 16 - Order Type/OrdType
|
||||
def OrdType(ordertype):
|
||||
if "OrdType" in json_data:
|
||||
if ordertype == "MarketWithProtection" or ordertype == "MarketwithProtection":
|
||||
return "1"
|
||||
elif ordertype == "Limit":
|
||||
return "2"
|
||||
elif ordertype == "StopWithProtection" or ordertype == "StopwithProtection":
|
||||
return "3"
|
||||
elif ordertype == "StopLimit":
|
||||
return "4"
|
||||
elif ordertype == "MarketLimit":
|
||||
return "K"
|
||||
else:
|
||||
return ""
|
||||
|
||||
# Column 17 - Order Qualifier/TimeInForce
|
||||
def TimeInForce(orderqual):
|
||||
if "TimeInForce" in json_data:
|
||||
if orderqual == "Day":
|
||||
return "0"
|
||||
elif orderqual == "GTC":
|
||||
return "1"
|
||||
elif orderqual == "FAK":
|
||||
return "3"
|
||||
elif orderqual == "GTD":
|
||||
return "6"
|
||||
else:
|
||||
return ""
|
||||
|
||||
# Column 18 IFM Flag/OFMOverride
|
||||
def OFMOverride(imfflag):
|
||||
if "OFMOverride" in json_data:
|
||||
if imfflag == "Enabled":
|
||||
return "1"
|
||||
else:
|
||||
return "0"
|
||||
|
||||
# Column 19 Display Quantity/DisplayQty
|
||||
def DisplayQty(qty):
|
||||
if "DisplayQty" in json_data:
|
||||
if str(qty) == "0":
|
||||
return ""
|
||||
elif str(qty) == "4294967295":
|
||||
return ""
|
||||
else:
|
||||
return qty
|
||||
|
||||
|
||||
# Column 26 - Aggressor Flag/AggressorIndicator
|
||||
def AggressorIndicator(aggressor):
|
||||
if "AggressorIndicator" in json_data:
|
||||
if aggressor == "True":
|
||||
return "1"
|
||||
else:
|
||||
return "0"
|
||||
|
||||
# Column 27 - Source of Cancellation/ExecRestatementReason
|
||||
def ExecRestatementReason(sourcecancel):
|
||||
if "ExecRestatementReason" in json_data:
|
||||
if sourcecancel == "Exchange":
|
||||
return "8"
|
||||
elif sourcecancel == "CancelledNotBest":
|
||||
return "9"
|
||||
elif sourcecancel == "CancelOnDisconnect":
|
||||
return "100"
|
||||
elif sourcecancel == "CancelRestingDueToSelfMatchPrevention":
|
||||
return "103"
|
||||
elif sourcecancel == "CancelFromCMEGlobexCreditControlsViolation":
|
||||
return "104"
|
||||
elif sourcecancel == "CancelFromCMEOne":
|
||||
return "105"
|
||||
elif sourcecancel == "CancelFromRiskManagementAPI":
|
||||
return "106"
|
||||
elif sourcecancel == "CancelAggressingDueToSelfMatchPrevention":
|
||||
return "107"
|
||||
elif sourcecancel == "CancelDueToVolatilityQuoteOptionLessThanMinOrderSize":
|
||||
return "108"
|
||||
else:
|
||||
return ""
|
||||
|
||||
# Column 43 - Self-Match Prevention Instr/SelfMatchPreventionInstruction
|
||||
def SelfMatchPreventionInstruction(instr):
|
||||
if "SelfMatchPreventionInstruction" in json_data:
|
||||
if instr == "CancelOldest":
|
||||
return "O"
|
||||
elif instr == "CancelNewest":
|
||||
return "N"
|
||||
else:
|
||||
return ""
|
||||
|
||||
# Column 44 - Customer Type Indicator/CustOrderCapacity
|
||||
def CustOrderCapacity(custord):
|
||||
if "CustOrderCapacity" in json_data:
|
||||
if custord == "NULL_VALUE":
|
||||
return ""
|
||||
elif custord == "Membertradingfortheirownaccount":
|
||||
return "1"
|
||||
elif custord == "ClearingFirmtradingforitsproprietaryaccount":
|
||||
return "2"
|
||||
elif custord == "Membertradingforanothermember":
|
||||
return "3"
|
||||
else:
|
||||
return "4"
|
||||
|
||||
# Column 46 - Average Px Ind/AvgPxInd
|
||||
def AvgPxInd(avgpxindicator):
|
||||
if "AvgPxInd" in json_data:
|
||||
if avgpxindicator == "NULL_VALUE":
|
||||
return ""
|
||||
else:
|
||||
return avgpxindicator
|
||||
|
||||
# Column 48 - Origin/ClearingAccountType
|
||||
def ClearingAccountType(accounttype):
|
||||
if "ClearingAccountType" in json_data:
|
||||
if accounttype == "Customer":
|
||||
return "0"
|
||||
elif accounttype == "Firm":
|
||||
return "1"
|
||||
else:
|
||||
return ""
|
||||
|
||||
# Column 49 - Clearing Trade Price Type/ClearingTradePriceType
|
||||
def ClearingTradePriceType(clrtrdprice):
|
||||
if "ClearingTradePriceType" in json_data:
|
||||
if clrtrdprice == "NULL_VALUE":
|
||||
return "0"
|
||||
else:
|
||||
return "3"
|
||||
|
||||
# Column 50 - Customer Handling Instr/CustOrderHandlingInst
|
||||
def CustOrderHandlingInst(custorderinst):
|
||||
if "CustOrderHandlingInst" in json_data:
|
||||
if custorderinst == "ClientElectronic":
|
||||
return "C"
|
||||
elif custorderinst == "TEST":
|
||||
return "D"
|
||||
else:
|
||||
return ""
|
||||
|
||||
# Column 51 and 52 - Getting the Firm and Account Number
|
||||
def partyid(partydetailid,column):
|
||||
for x in partydetailid:
|
||||
role = x['PartyDetailRole']
|
||||
id = x['PartyDetailID']
|
||||
if column == "51" and role == "ExecutingFirm":
|
||||
return id
|
||||
if column == "52" and role == "CustomerAccount":
|
||||
return id
|
||||
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("json_log", help="the json log[.gz] you wish to process", type=valid_file)
|
||||
parser.add_argument("output_file", help="the destination csv file you wish to write")
|
||||
parser.add_argument("-d", "--date", help="the date you wish to parse from the log, specified as YYYYmmdd",
|
||||
default=datetime.datetime.today().strftime("%Y%m%d"), type=valid_date)
|
||||
args = parser.parse_args()
|
||||
|
||||
print_headers()
|
||||
|
||||
list_types = ['NewOrderSingle514', 'OrderCancelReplaceRequest515', 'OrderCancelRequest516', 'PartyDetailsDefinitionRequest518', 'PartyDetailsDefinitionRequestAck519', \
|
||||
'ExecutionReportNew522', 'ExecutionReportReject523', 'ExecutionReportTradeOutright525', 'ExecutionReportTradeSpread526', 'ExecutionReportTradeSpreadLeg527', \
|
||||
'ExecutionReportModify531', 'ExecutionReportCancel534', 'OrderCancelReject535', 'OrderCancelReplaceReject536', ]
|
||||
|
||||
# Note that the timedeltas specificed in trade_day_start and trade_day_end
|
||||
# are based on Chicago times; as such, this script should run on a machine
|
||||
# set to a Chicago timezone. A future impromovement would be to determine
|
||||
# the current machine's timezone and determine the appropriate offset
|
||||
# versus Chicago time, but for now we assume this will run on a Chicago
|
||||
# tz machine.
|
||||
input_as_datetime = args.date
|
||||
trade_day_start = input_as_datetime - datetime.timedelta(days=1) + datetime.timedelta(hours=16, minutes=30)
|
||||
trade_day_utc_start = time.gmtime((time.mktime(trade_day_start.timetuple())))
|
||||
trade_day_end = input_as_datetime + datetime.timedelta(hours=16)
|
||||
trade_day_utc_end = time.gmtime((time.mktime(trade_day_end.timetuple())))
|
||||
|
||||
file_opener = None
|
||||
if args.json_log.endswith(".gz"):
|
||||
import gzip
|
||||
file_opener = gzip.open
|
||||
else:
|
||||
file_opener = open
|
||||
|
||||
|
||||
#inputFile = open(fileInput) #open json file
|
||||
#outputFile = open(fileOutput, 'w') #load csv file
|
||||
# Get Session
|
||||
with file_opener(args.json_log) as json_file:
|
||||
for line in json_file:
|
||||
data = json.loads(line)
|
||||
json_data = data['msg_json']
|
||||
if "Session" in json_data:
|
||||
SESSIONID = json_data["Session"]
|
||||
|
||||
#inputFile = open(fileInput) #open json file
|
||||
#outputFile = open(fileOutput, 'w') #load csv file
|
||||
with file_opener(args.json_log) as json_file:
|
||||
for line in json_file:
|
||||
data = json.loads(line)
|
||||
line_timestamp = time.strptime(data['time'], "%Y%m%d-%H:%M:%S.%f")
|
||||
if trade_day_utc_start <= line_timestamp <= trade_day_utc_end:
|
||||
json_data = data['msg_json']
|
||||
NAME = json_data["Name"]
|
||||
if NAME in list_types:
|
||||
EXECUTING_FIRM_ID = ""
|
||||
ACCOUNT_NUMBER = ""
|
||||
DIRECTION = direction(data['direction'])
|
||||
# Set Blanks
|
||||
blank_json = {'QuoteReqID': '', 'QuoteID': '', 'QuoteSetID': '', 'QuoteEntryID': '', 'BidPx': '', 'BidSize': '', 'OfferPx': '', \
|
||||
'OfferSize': '', 'NoProcessedEntries': '', 'ListUpdateAction': '', 'MinQty': '', 'AvgPxGroupID': '', \
|
||||
'CmtaGiveUpCD': '', 'TAKE_UP_FIRM': '', 'TAKE_UP_ACCOUNT': '',}
|
||||
|
||||
# No Data Processing
|
||||
straight_json = {'SecurityID': '', 'ClOrdID': '', 'OrderID': '', 'OrderRequestID': '', 'OrderQty': '', \
|
||||
'Location': '', 'LastQty': '', 'CumQty': '', 'LeavesQty': '', 'Text': '', 'SideTradeID': '', 'CrossID': '', 'SelfMatchPreventionID': '', \
|
||||
'Price': '', 'StopPx': '', 'LastPx': '', 'PartyDetailsListReqID': '', 'OrdStatus': ''}
|
||||
|
||||
# Processing Data
|
||||
process_json = {'SendingTimeEpoch': '', 'ManualOrderIndicator': '', 'SenderID': '', 'Side': '', 'OrdType': '', 'TimeInForce': '', \
|
||||
'OFMOverride': '', 'AggressorIndicator': '', 'ExecRestatementReason': '', 'SelfMatchPreventionInstruction': '', 'CustOrderCapacity': '', \
|
||||
'AvgPxInd': '', 'ClearingAccountType': '', 'ClearingTradePriceType': '', 'CustOrderHandlingInst': '', 'DisplayQty': ''}
|
||||
|
||||
for d in straight_json:
|
||||
test = get_value(d)
|
||||
if test != "None" or test != "":
|
||||
if "," in str(test):
|
||||
test = test.replace(",",";")
|
||||
if "4294967295" in str(test):
|
||||
test = ""
|
||||
if "us;" in str(test):
|
||||
test = test.upper()
|
||||
if d == "SelfMatchPreventionID" and len(str(test)) > 8:
|
||||
test = ""
|
||||
straight_json[d] = test
|
||||
else:
|
||||
straight_json[d] = ''
|
||||
|
||||
for d in process_json:
|
||||
if d in json_data:
|
||||
function = locals()[d]
|
||||
test = function(json_data[d])
|
||||
if test != "None":
|
||||
process_json[d] = test
|
||||
else:
|
||||
process_json[d] = ''
|
||||
elif d == "SenderID":
|
||||
function = locals()[d]
|
||||
test = function(SenderID)
|
||||
if test != "None":
|
||||
process_json[d] = test
|
||||
|
||||
# Party Message Fields
|
||||
# Column 51 - Executing Firm ID
|
||||
if "NoPartyDetails" in json_data:
|
||||
EXECUTING_FIRM_ID = partyid(json_data['NoPartyDetails'], "51")
|
||||
# Column 52 - Party Account Number
|
||||
ACCOUNT_NUMBER = partyid(json_data['NoPartyDetails'], "52")
|
||||
|
||||
line_to_write = process_json['SendingTimeEpoch'] + "," + DIRECTION + "," + process_json['SenderID'] + "," + SESSIONID + "," + process_json['ManualOrderIndicator'] + "," + \
|
||||
MessageType(NAME,straight_json['OrdStatus']) + "," + str(straight_json['SecurityID']) + "," + str(straight_json['ClOrdID']) + "," + str(straight_json['OrderID']) + "," + str(straight_json['ClOrdID']) + "," + \
|
||||
str(straight_json['OrderRequestID']) + "," + process_json['Side'] + "," + str(straight_json['OrderQty']) + "," + str(straight_json['Price']) + "," + \
|
||||
str(straight_json['StopPx']) + "," + process_json['OrdType'] + "," + process_json['TimeInForce'] + "," + process_json['OFMOverride'] + "," + str(process_json['DisplayQty']) + "," + \
|
||||
blank_json['MinQty'] + "," + straight_json['Location'] + "," + str(straight_json['LastPx']) + "," + str(straight_json['LastQty']) + "," + str(straight_json['CumQty']) + "," + \
|
||||
str(straight_json['LeavesQty']) + "," + process_json['AggressorIndicator'] + "," + process_json['ExecRestatementReason'] + "," + str(straight_json['Text']) + "," + \
|
||||
str(straight_json['SideTradeID']) + "," + str(straight_json['CrossID']) + "," + blank_json['QuoteReqID'] + "," + blank_json['QuoteID'] + "," + blank_json['QuoteSetID'] + "," + \
|
||||
blank_json['QuoteEntryID'] + "," + blank_json['BidPx'] + "," + blank_json['BidSize'] + "," + blank_json['OfferPx'] + "," + blank_json['OfferSize'] + "," + \
|
||||
blank_json['NoProcessedEntries'] + "," + str(straight_json['PartyDetailsListReqID']) + "," + blank_json['ListUpdateAction'] + "," + str(straight_json['SelfMatchPreventionID']) + "," + \
|
||||
process_json['SelfMatchPreventionInstruction'] + "," + process_json['CustOrderCapacity'] + "," + blank_json['AvgPxGroupID'] + "," + process_json['AvgPxInd'] + "," + \
|
||||
blank_json['CmtaGiveUpCD'] + "," + str(process_json['ClearingAccountType']) + "," + process_json['ClearingTradePriceType'] + "," + process_json['CustOrderHandlingInst'] + "," + \
|
||||
EXECUTING_FIRM_ID + "," + str(ACCOUNT_NUMBER) + "," + blank_json['TAKE_UP_FIRM'] + "," + blank_json['TAKE_UP_ACCOUNT']
|
||||
|
||||
|
||||
#print(line_to_write)
|
||||
|
||||
with open (args.output_file, 'a') as the_file:
|
||||
the_file.write(line_to_write)
|
||||
the_file.write("\n")
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Executable
+73
@@ -0,0 +1,73 @@
|
||||
#/bin/bash
|
||||
|
||||
function help_and_exit()
|
||||
{
|
||||
echo
|
||||
echo Usage: $0 [option]... config_file
|
||||
echo
|
||||
echo Can also be used in cron like the following
|
||||
echo 40 16 * * 1-5 /usr/bin/docker exec spark_10000 bash /sparkbin/data_files/audit_upload.sh /sparkdata/data/audit_uploads/example.conf
|
||||
echo example.conf included. The location of the conf file should be relative to the docker container, not the host
|
||||
echo
|
||||
echo Options are as follows:
|
||||
echo " -d|--date YYYYMMDD Example is 20180731"
|
||||
echo " Help: "
|
||||
echo " -h|--help Display this message."
|
||||
echo
|
||||
exit 1
|
||||
}
|
||||
|
||||
DATE=$(date +%Y%m%d)
|
||||
ORIG_DIR=$(pwd)
|
||||
ERROR_LEVEL="0"
|
||||
|
||||
while getopts "d:h" option; do
|
||||
case $option in
|
||||
d) DATE=$OPTARG ;;
|
||||
h) help_and_exit;;
|
||||
\?) echo "Invalid option: -$OPTARG" >&2;
|
||||
exit 1;;
|
||||
esac
|
||||
done
|
||||
|
||||
shift $((OPTIND-1))
|
||||
|
||||
CONFIG_FILE=$1
|
||||
|
||||
declare -A conf_vars; while read line; do conf_vars[${line%=*}]=${line#*=}; done <$CONFIG_FILE
|
||||
|
||||
SESSION_ID=${conf_vars[session_id]}
|
||||
FCM=${conf_vars[fcm]}
|
||||
CLIENT=${conf_vars[client]}
|
||||
|
||||
cd /sparkbin/data_files
|
||||
|
||||
for SESSION in $(ls /sparkdata/data/ilink3/); do
|
||||
if [[ $SESSION == *$SESSION_ID* ]]; then
|
||||
echo "Session is $SESSION"
|
||||
SESSION_DAY=$(date +%Y-%m-%d)
|
||||
if [[ $SESSION == "cme_"* ]]; then
|
||||
SESSION_DAY=$(date -d "$DATE -$(date -d $DATE +%u) days" +"%Y-%m-%d")
|
||||
AUDIT_SCRIPT="audit_ilink3.py"
|
||||
fi
|
||||
echo $SESSION_DAY
|
||||
if [[ ! -d /sparkdata/data/ilink3/$SESSION/audit_logs ]]; then
|
||||
mkdir -p /sparkdata/data/ilink3/$SESSION/audit_logs
|
||||
fi
|
||||
if [[ -d /sparkdata/data/ilink3/$SESSION/$SESSION_DAY ]]; then
|
||||
echo "Starting the creation of the audit log"
|
||||
python $AUDIT_SCRIPT -d $DATE /sparkdata/data/ilink3/$SESSION/$SESSION_DAY/log /sparkdata/data/ilink3/$SESSION/audit_logs/"$FCM"_audit_"$SESSION"_"$DATE".csv
|
||||
fi
|
||||
if [[ ! -d /sparkdata/data/md_audit ]]; then
|
||||
mkdir /sparkdata/data/md_audit
|
||||
fi
|
||||
if [[ -d /sparkdata/data/md_audit ]]; then
|
||||
# Running the MD Audit csv generation
|
||||
cd /sparkbin
|
||||
./sparkd adminreport md > /sparkdata/data/md_audit/"$FCM"_md_"$CLIENT"_"$DATE".csv
|
||||
cd /sparkbin/data_files
|
||||
fi
|
||||
echo "Uploading to the (s)FTP Server"
|
||||
python audit_uploader.py --config_file="$CONFIG_FILE" --datestamp $DATE --session="$SESSION" --client="$CLIENT" --fcm="$FCM"
|
||||
fi
|
||||
done
|
||||
Executable
+150
@@ -0,0 +1,150 @@
|
||||
import getopt, sys, datetime
|
||||
import paramiko, ftplib
|
||||
|
||||
#########################################################
|
||||
# Commandline Args #
|
||||
#########################################################
|
||||
|
||||
def usage():
|
||||
print "-------------------------------------------------------"
|
||||
print "-Usage: -"
|
||||
print "-------------------------------------------------------"
|
||||
print "- --config_file= : Path to upload config file -"
|
||||
print "- -if no path is specified look -"
|
||||
print "- in the working directory. -"
|
||||
print "- -"
|
||||
print "- --help : Print this message -"
|
||||
print "- -"
|
||||
print "- --datestamp= : Optional, defaults to today -"
|
||||
print "- -replaces *datestamp in the -"
|
||||
print "- the upload config file -"
|
||||
print "- -"
|
||||
print "- --fcm : Optional, spark input -"
|
||||
print "- -"
|
||||
print "- --session : Optional, spark jf session -"
|
||||
print "-------------------------------------------------------"
|
||||
|
||||
fcm = ''
|
||||
session = ''
|
||||
|
||||
try:
|
||||
options, arguments = getopt.getopt(sys.argv[1:],"h",['config_file=', 'datestamp=', 'help', 'fcm=', 'client=', 'session='])
|
||||
except getopt.GetoptError, err:
|
||||
print str(err)
|
||||
usage()
|
||||
sys.exit(2)
|
||||
|
||||
for option, value in options:
|
||||
if option in ("-h","--h","-help","--help"):
|
||||
usage()
|
||||
sys.exit()
|
||||
elif option in ("-config_file","--config_file"):
|
||||
config_file = value
|
||||
elif option in ("-datestamp","--datestamp"):
|
||||
datestamp = value
|
||||
elif option in ("-fcm","--fcm"):
|
||||
fcm = value
|
||||
elif option in ("-client","--client"):
|
||||
client = value
|
||||
elif option in ("-session","--session"):
|
||||
session = value
|
||||
|
||||
for argument in arguments:
|
||||
if argument in ("h", "help"):
|
||||
usage()
|
||||
sys.exit()
|
||||
else:
|
||||
print "Unrecognized commandline argument. Type help for usage."
|
||||
sys.exit()
|
||||
|
||||
if len(options) <= 0:
|
||||
print "Requires a config file to run."
|
||||
usage()
|
||||
sys.exit()
|
||||
|
||||
|
||||
#########################################################
|
||||
# Read Config Function #
|
||||
#########################################################
|
||||
|
||||
def read_config_file(config_file, datestamp=datetime.datetime.today().strftime('%Y%m%d'), fcm=fcm, session=session):
|
||||
'''
|
||||
Parse our config file that gets passed in as a commandline parameter.
|
||||
All config keys must be of the form key=value. White space is ignored.
|
||||
All config keys must be seperated by a newline character.
|
||||
Called during initialization.
|
||||
'''
|
||||
upload_config = {}
|
||||
config_file = open(config_file, 'r')
|
||||
config_file = config_file.read()
|
||||
for config in config_file.split("\n"):
|
||||
if '=' in config:
|
||||
cleaned_key = config.split('=')[0].strip().upper()
|
||||
cleaned_value = config.split('=')[1].strip()
|
||||
if '*datestamp*' in cleaned_value:
|
||||
cleaned_value = cleaned_value.replace('*datestamp*',datestamp)
|
||||
if '*fcm*' in cleaned_value:
|
||||
cleaned_value = cleaned_value.replace('*fcm*',fcm)
|
||||
if '*client*' in cleaned_value:
|
||||
cleaned_value = cleaned_value.replace('*client*',client)
|
||||
if '*session*' in cleaned_value:
|
||||
cleaned_value = cleaned_value.replace('*session*',session)
|
||||
upload_config[cleaned_key] = cleaned_value
|
||||
else:
|
||||
pass
|
||||
return upload_config
|
||||
|
||||
#########################################################
|
||||
# FTP File Transfer Function #
|
||||
#########################################################
|
||||
def putfile_ftp(file, put_path, site, user, password, port=21, verbose=True):
|
||||
if verbose: print 'Uploading', file
|
||||
local = open(file, 'rb')
|
||||
remote = ftplib.FTP(site)
|
||||
remote.login(user, password)
|
||||
remote.storbinary(put_path, local, 1024)
|
||||
remote.quit()
|
||||
local.close()
|
||||
if verbose: print 'Upload done.'
|
||||
|
||||
#########################################################
|
||||
# SFTP File Transfer Function #
|
||||
#########################################################
|
||||
def putfile_sftp(file, put_path, site, username, password, port=22, verbose=True):
|
||||
transport = paramiko.Transport((site, port))
|
||||
transport.connect(username = username, password = password)
|
||||
sftp = paramiko.SFTPClient.from_transport(transport)
|
||||
sftp.put(file, put_path)
|
||||
sftp.close()
|
||||
transport.close()
|
||||
|
||||
|
||||
#########################################################
|
||||
# Main #
|
||||
#########################################################
|
||||
#if datestamp != None:
|
||||
# upload_config = read_config_file(config_file, datestamp)
|
||||
#else:
|
||||
# upload_config = read_config_file(config_file)
|
||||
|
||||
try:
|
||||
datestamp
|
||||
except NameError:
|
||||
upload_config = read_config_file(config_file)
|
||||
else:
|
||||
upload_config = read_config_file(config_file, datestamp)
|
||||
|
||||
|
||||
|
||||
print upload_config
|
||||
|
||||
if upload_config["PROTOCOL_PUT"] in ['FTP','ftp']:
|
||||
print "Protocol = FTP"
|
||||
putfile_ftp(upload_config["GET_ILINK3_FILE_PATH"] + upload_config["GET_ILINK3_FILE_NAME"], upload_config["PUT_ILINK3_FILE_PATH"] + upload_config["PUT_ILINK3_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], upload_config["PORT_PUT"] )
|
||||
putfile_ftp(upload_config["GET_MD_FILE_PATH"] + upload_config["GET_MD_FILE_NAME"], upload_config["PUT_MD_FILE_PATH"] + upload_config["PUT_MD_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], upload_config["PORT_PUT"] )
|
||||
elif upload_config["PROTOCOL_PUT"] in ['SFTP','sftp']:
|
||||
print "Protocol = SFTP"
|
||||
putfile_sftp(upload_config["GET_ILINK3_FILE_PATH"] + upload_config["GET_ILINK3_FILE_NAME"], upload_config["PUT_ILINK3_FILE_PATH"] + upload_config["PUT_ILINK3_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], int(upload_config["PORT_PUT"]) )
|
||||
putfile_sftp(upload_config["GET_MD_FILE_PATH"] + upload_config["GET_MD_FILE_NAME"], upload_config["PUT_MD_FILE_PATH"] + upload_config["PUT_MD_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], int(upload_config["PORT_PUT"]) )
|
||||
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
fcm=FCM
|
||||
client=NAME
|
||||
session_id=ses
|
||||
put_host=ftpc.example.com
|
||||
port_put=2222
|
||||
username_put=user
|
||||
password_put=password
|
||||
protocol_put=sftp
|
||||
get_ilink3_file_path=/sparkdata/data/ilink3/*session*/audit_logs/
|
||||
put_ilink3_file_path=/
|
||||
get_ilink3_file_name=*fcm*_audit_*session*_*datestamp*.csv
|
||||
put_ilink3_file_name=*fcm*_audit_*session*_*datestamp*.csv
|
||||
get_md_file_path=/sparkdata/data/md_audit/
|
||||
put_md_file_path=/
|
||||
get_md_file_name=*fcm*_md_*client*_*datestamp*.csv
|
||||
put_md_file_name=*fcm*_md_*client*_*datestamp*.csv
|
||||
+1006
-10
File diff suppressed because it is too large.
Load diff
@@ -120,7 +120,7 @@ void configure_logging(bts::config::Logging const&cfg, bts::config::ThreadGroupC
|
||||
{log::Plugin,"Plugin"},
|
||||
{log::Sdk,"Sdk"},
|
||||
{log::Http,"Http"},
|
||||
{log::iLink3,"ilink3"},
|
||||
{log::iLink3,"iLink3"},
|
||||
};
|
||||
|
||||
real_cfg.sinks["console"] =
|
||||
|
||||
@@ -243,19 +243,19 @@ void GLBXiLink3ExecutionGateway::send_msgs(std::vector<OrderMsg> const &msgs,
|
||||
auto create_ptr = std::get_if<CreateOrderMsg>(&m);
|
||||
if (create_ptr) {
|
||||
me->gw->send_order(*create_ptr);
|
||||
break;
|
||||
continue;
|
||||
}
|
||||
// CancelReplace
|
||||
auto cancel_replace_ptr = std::get_if<CancelReplaceOrderMsg>(&m);
|
||||
if (cancel_replace_ptr) {
|
||||
me->gw->send_cancel_replace(*cancel_replace_ptr);
|
||||
break;
|
||||
continue;
|
||||
}
|
||||
// Cancel
|
||||
auto cancel_ptr = std::get_if<CancelOrderMsg>(&m);
|
||||
if (cancel_ptr) {
|
||||
me->gw->send_cancel(*cancel_ptr);
|
||||
break;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
me->gw->flush();
|
||||
|
||||
@@ -101,6 +101,8 @@ void OrderStore::on_order_state_update(
|
||||
for (auto const &ev : *u.events) {
|
||||
|
||||
auto &rec_info = me->known_orders[ev.key];
|
||||
LOG(iLink3,Info, "Received order state update for {}",
|
||||
ev.key.value());
|
||||
auto &rec = rec_info.rec;
|
||||
bts_assert(rec.has_def());
|
||||
rec.set_key(ev.key.value());
|
||||
@@ -123,6 +125,8 @@ void OrderStore::on_order_definition_update(
|
||||
std::set<OrderKey> modified_orders;
|
||||
if (u.events)
|
||||
for (auto const &ev : *u.events) {
|
||||
LOG(iLink3,Info, "Received order definition update for {}",
|
||||
ev.key.value());
|
||||
auto &rec_info = me->known_orders[ev.key];
|
||||
auto &rec = rec_info.rec;
|
||||
rec.set_key(ev.key.value());
|
||||
|
||||
@@ -176,12 +176,7 @@ uint64_t IL3GW::send_party_details(const api::orderman::RoutingRecord &route) {
|
||||
msg.selfMatchPreventionID(
|
||||
PartyDetailsDefinitionRequest518::selfMatchPreventionIDNullValue());
|
||||
} else {
|
||||
// CME needs the SMP_ID to be 12 characters or less
|
||||
// after they to_string() the uint64
|
||||
bts_assert_d(route.smp_id().size() == 8);
|
||||
const uint64_t *psmp_id =
|
||||
reinterpret_cast<const uint64_t *>(&route.smp_id());
|
||||
uint64_t smp_id = (*psmp_id) % (2UL << 39);
|
||||
auto smp_id = std::atoi(route.smp_id().c_str());
|
||||
msg.selfMatchPreventionID(smp_id);
|
||||
}
|
||||
msg.cmtaGiveupCD(CmtaGiveUpCD::NULL_VALUE);
|
||||
|
||||
@@ -33,7 +33,7 @@ struct Session {
|
||||
// cme->bts
|
||||
uint32_t from_cme;
|
||||
|
||||
operator bool const() { return from_cme; }
|
||||
operator bool const() { return to_cme || from_cme; }
|
||||
};
|
||||
|
||||
struct Config {
|
||||
|
||||
@@ -25,7 +25,7 @@ iLinkBinary::SideReq::Value map_side(api::Side side);
|
||||
api::Side map_side(iLinkBinary::SideReq::Value side);
|
||||
|
||||
inline int64_t to_price9(Price p) {
|
||||
return p.value() * 1E9;
|
||||
return std::round(p.value() * 1E9);
|
||||
}
|
||||
|
||||
iLinkBinary::OrderTypeReq::Value map_order_type(api::orderman::NativeOrderType ot);
|
||||
|
||||
@@ -108,7 +108,7 @@ void save_session(std::string const &path, Session const &session) {
|
||||
}
|
||||
|
||||
Session load_session(jetfix::SessionName const &sn, std::string const &path) {
|
||||
auto result = Session{sn.as_uint64_t, 1, 0};
|
||||
auto result = Session{sn.as_uint64_t, 0, 0};
|
||||
if (boost::filesystem::exists(path)) {
|
||||
std::ifstream f(path);
|
||||
uint64_t dummy;
|
||||
|
||||
@@ -861,7 +861,7 @@ void GLBXNativeOrder::on_exe_report(const NewExecutionReportEvent &er) {
|
||||
auto external_status = map_external_status(er.data->order_status);
|
||||
|
||||
// filled quantity is reset back to 0 when the order is canceled
|
||||
if (external_status != ORDER_STATUS_CANCELED) {
|
||||
if (external_status != ORDER_STATUS_CANCELED ) {
|
||||
if (er.data->exec_type == api::orderman::EXEC_TYPE_PARTIAL_FILL ||
|
||||
er.data->exec_type == api::orderman::EXEC_TYPE_FILL) {
|
||||
|
||||
@@ -876,9 +876,9 @@ void GLBXNativeOrder::on_exe_report(const NewExecutionReportEvent &er) {
|
||||
(previous_sz + fill_sz);
|
||||
|
||||
state.set_avg_price(new_avg);
|
||||
state.set_filled_quantity(er.data->filled_quantity.value());
|
||||
state.set_last_price(er.data->last_price.value());
|
||||
}
|
||||
state.set_filled_quantity(er.data->filled_quantity.value());
|
||||
state.set_last_price(er.data->last_price.value());
|
||||
}
|
||||
|
||||
state.set_last_msg(er.data->msg);
|
||||
|
||||
Reference in new issue
Block a user