ice audit scripts

This commit is contained in:
Pete Fisher committed 2021-03-18 11:05:34 -04:00
1 parent 0cb11b7f8d
commit 07591f8e28
4 files changed
+267 -36

No files matched your search

+206
View File
@@ -0,0 +1,206 @@
#!/usr/bin/python
import os
import re
import fileinput
import sys
import collections
import csv
import time, datetime
import copy
import itertools
import argparse
# region fix record processing
valid_message_types = {
'D': 'NEW ORDER',
'G': 'MODIFY',
'F': 'CANCEL',
'8': 'EXECUTION',
'i': 'MASS QUOTE',
'R': 'QUOTE REQUEST',
'b': 'QUOTE ACK',
'Z': 'MASS QUOTE CANCEL',
# 'c': 'SECURITY DEFINITION REQUEST',
'3': 'SESSION LEVEL REJECT',
'j': 'BUSINESS LEVEL REJECT',
'9': 'CANCEL REJECT',
'CA': 'ORDER MASS ACTION REQUEST',
'BZ': 'ORDER MASS ACTION REPORT'
}
def print_headers():
# From https://www.theice.com/publicdocs/futures_us/exchange_notices/ICE_Advisory_0113_009.pdf
line_to_write = "Active Date,Cl Ord ID,OrderID,Transact Time,Symbol,Symbol,SecurityType,Stk Prc,P/C,Side,OrderQty,Max Show,OrdType,\
Price,StopPx,Clearing Account,Clearing Firm,Order State,AccountCode,CTI Code,Originator User ID,OnBehalfOfCompID,\
OnBehalfOfCompID,On Behalf Of Sub ID,On Behalf Of Sub ID,On Behalf Of Location ID,Time In Force"
with open (args.output_file, 'w') as the_file:
the_file.write(line_to_write)
the_file.write("\n")
line_to_write = "Activity Date,Client Order ID,Original Order ID,Order Entry/Modification Time,Commodity,Contract Month,Future/Option Indicator,\
Strike Price,,Put or Call,Buy/Sell Indicator,Quantity,Reserve Quantity,Order Type,Price,Stop Price,Account #,CM#,\
Order Status,Clearing Account Code,CTI,ETS Login ID,Routing Member ID,Authorized Trader ID,Authorized Trdr ID,\
Routing Trdr,Routing Grp,Time In Force"
with open (args.output_file, 'a') as the_file:
the_file.write(line_to_write)
the_file.write("\n")
def getter(field_name):
def result(parsed_fix_record):
if field_name in parsed_fix_record:
return parsed_fix_record[field_name]
else:
return None
return result
def parse_fix_line(line):
ts = line[:21]
fix_line = line[24:].split('\1')
result = {}
result['timestamp'] = ts
for fix_field in fix_line:
rx = r"(?P<tag>\d+)?=(?P<value>.*)"
match = re.match(rx,fix_field)
if match:
result[match.group('tag')] = match.group('value')
return result
def tagged_field(line,number):
if number in line:
return line[number]
else:
return ""
def formatted_tagged_field(line,number):
if number in line:
return line[number].lstrip("0")
else:
return ""
def lr_tagged_field(line,number):
if number in line:
return line[number].rsplit('|',1)
else:
return "",""
def lr_contract_month(line,number):
try:
ice_line="ICE_SECURITY_ID_TO_SYMBOL_MAPPING: " + line[number]
with open('/sparkdata/data/md_dump.txt') as f:
datafile = f.readlines()
for data in datafile:
if data.find(str(ice_line)) != -1:
placeholder=data.find('> ')+1
get_ending=data[placeholder:]
get_ending=get_ending.strip()
return get_ending.split(" ")
except KeyError:
print("Can not find key 55")
return "",""
def valid_file(f):
if not os.path.exists(f):
# Argparse uses the ArgumentTypeError to give a rejection message like:
# error: argument input: x does not exist
raise argparse.ArgumentTypeError("{0} does not exist".format(f))
return f
def valid_date(s):
try:
return datetime.datetime.strptime(s, "%Y%m%d")
except ValueError:
raise argparse.ArgumentTypeError("Not a valid date: '{0}'.".format(s))
parser = argparse.ArgumentParser()
parser.add_argument("quickfix_log", help="the quickfix log[.gz] you wish to process", type=valid_file)
parser.add_argument("output_file", help="the destination csv file you wish to write")
parser.add_argument("-d", "--date", help="the date you wish to parse from the log, specified as YYYYmmdd",
default=datetime.datetime.today().strftime("%Y%m%d"), type=valid_date)
#quickfix_log="/home/pfisher/temp/audit/fix"
#output_file="/home/pfisher/temp/audit/fix.csv"
args = parser.parse_args()
# Note that the timedeltas specificed in trade_day_start and trade_day_end
# are based on Chicago times; as such, this script should run on a machine
# set to a Chicago timezone. A future impromovement would be to determine
# the current machine's timezone and determine the appropriate offset
# versus Chicago time, but for now we assume this will run on a Chicago
# tz machine.
input_as_datetime = args.date
trade_day_start = input_as_datetime - datetime.timedelta(days=1) + datetime.timedelta(hours=17, minutes=30)
trade_day_utc_start = time.gmtime((time.mktime(trade_day_start.timetuple())))
trade_day_end = input_as_datetime + datetime.timedelta(hours=17)
trade_day_utc_end = time.gmtime((time.mktime(trade_day_end.timetuple())))
#file_opener = open
file_opener = None
if args.quickfix_log.endswith(".gz"):
import gzip
file_opener = gzip.open
else:
file_opener = open
audit_entries = []
line_number = 0
print_headers()
with file_opener(args.quickfix_log) as input_file:
#with file_opener(quickfix_log) as input_file:
for line in input_file:
try:
line_timestamp = time.strptime(line[:21], "%Y%m%d-%H:%M:%S.%f")
except:
continue
if trade_day_utc_start <= line_timestamp <= trade_day_utc_end:
parsed_fix_record = parse_fix_line(line)
if parsed_fix_record['35'] in valid_message_types:
line_number = line_number + 1
# Static fields
tt = datetime.datetime.strptime(line[:21], "%Y%m%d-%H:%M:%S.%f")
transacttime = tt.strftime('%Y%m%d-%H:%M:%S.%f')
activedate = time.strftime('%Y%m%d',time.strptime(line[:8], "%Y%m%d"))
securitytype = "FUT"
stkprc = ""
putcall = ""
#symbol = "HNG"
#contractmonth = "U20"
# No formatting fields
ciorderid = tagged_field(parsed_fix_record, '11')
orderid = tagged_field(parsed_fix_record,'37')
side = tagged_field(parsed_fix_record,'54')
orderquanity = tagged_field(parsed_fix_record,'38')
maxshow = tagged_field(parsed_fix_record,'201')
ordertype = tagged_field(parsed_fix_record,'40')
stoppx = tagged_field(parsed_fix_record,'99')
clearingaccount = tagged_field(parsed_fix_record,'440')
clearingfirm = tagged_field(parsed_fix_record,'439')
orderstate = tagged_field(parsed_fix_record,'9175')
accountcode = tagged_field(parsed_fix_record,'9195')
cticode = tagged_field(parsed_fix_record,'9208')
originatoruserid = tagged_field(parsed_fix_record,'9139')
timeinforce = tagged_field(parsed_fix_record,'59')
# Needs Formatting
price = formatted_tagged_field(parsed_fix_record,'44')
# Left Right
OnBehalfOfCompID_Left,OnBehalfOfCompID_Right = lr_tagged_field(parsed_fix_record,'115')
OnBehalfOfSubID_Left,OnBehalfOfSubID_Right = lr_tagged_field(parsed_fix_record,'116')
OnBehalfOfLocationID_Left,OnBehalfOfLocationID_Right = lr_tagged_field(parsed_fix_record,'144')
if parsed_fix_record['35'] == "9":
symbol = ""
contractmonth = ""
else:
symbol,contractmonth = lr_contract_month(parsed_fix_record,'55')
line_to_write = activedate + "," + ciorderid + "," + orderid + "," + transacttime + "," + symbol + "," + contractmonth + "," + securitytype + "," + \
stkprc + "," + putcall + "," + side + "," + orderquanity + "," + maxshow + "," + ordertype + "," + price + "," + stoppx + "," + \
clearingaccount + "," + clearingfirm + "," + orderstate + "," + accountcode + "," + cticode + "," + originatoruserid + "," + OnBehalfOfCompID_Left + "," + \
OnBehalfOfCompID_Right + "," + OnBehalfOfSubID_Left + "," + OnBehalfOfSubID_Right + "," + OnBehalfOfLocationID_Left + "," + timeinforce
with open (args.output_file, 'a') as the_file:
the_file.write(line_to_write)
the_file.write("\n")
# except Exception:
# pass
+18 -7
View File
@@ -40,23 +40,34 @@ SESSION_ID=${conf_vars[session_id]}
FCM=${conf_vars[fcm]}
CLIENT=${conf_vars[client]}
# Create MD dump
cd /sparkbin
./sparkd md dumpsecurity all > /sparkdata/data/md_dump.txt
cd /sparkbin/data_files
for SESSION in $(ls /sparkdata/data/ilink3/); do
for SESSION in $(ls /sparkdata/data/{ilink3,jetfix}/); do
if [[ $SESSION == *$SESSION_ID* ]]; then
echo "Session is $SESSION"
SESSION_DAY=$(date +%Y-%m-%d)
SESSION_DAY=$(date -d $DATE +%Y-%m-%d)
if [[ $SESSION == "cme_"* ]]; then
SESSION_DAY=$(date -d "$DATE -$(date -d $DATE +%u) days" +"%Y-%m-%d")
AUDIT_SCRIPT="audit_ilink3.py"
ILINK_TYPE="ilink3"
LOG_NAME="log"
elif [[ $SESSION == "ice_"* ]]; then
SESSION_DAY=$(date -d "$DATE 2 days ago" +%Y-%m-%d)
AUDIT_SCRIPT="audit_ice.py"
ILINK_TYPE="jetfix"
LOG_NAME="fix"
fi
echo $SESSION_DAY
if [[ ! -d /sparkdata/data/ilink3/$SESSION/audit_logs ]]; then
mkdir -p /sparkdata/data/ilink3/$SESSION/audit_logs
if [[ ! -d /sparkdata/data/$ILINK_TYPE/$SESSION/audit_logs ]]; then
mkdir -p /sparkdata/data/$ILINK_TYPE/$SESSION/audit_logs
fi
if [[ -d /sparkdata/data/ilink3/$SESSION/$SESSION_DAY ]]; then
if [[ -d /sparkdata/data/$ILINK_TYPE/$SESSION/$SESSION_DAY ]]; then
echo "Starting the creation of the audit log"
python $AUDIT_SCRIPT -d $DATE /sparkdata/data/ilink3/$SESSION/$SESSION_DAY/log /sparkdata/data/ilink3/$SESSION/audit_logs/"$FCM"_audit_"$SESSION"_"$DATE".csv
python3 $AUDIT_SCRIPT -d $DATE /sparkdata/data/$ILINK_TYPE/$SESSION/$SESSION_DAY/$LOG_NAME /sparkdata/data/$ILINK_TYPE/$SESSION/audit_logs/"$FCM"_audit_"$SESSION"_"$DATE".csv
fi
if [[ ! -d /sparkdata/data/md_audit ]]; then
mkdir /sparkdata/data/md_audit
@@ -68,6 +79,6 @@ for SESSION in $(ls /sparkdata/data/ilink3/); do
cd /sparkbin/data_files
fi
echo "Uploading to the (s)FTP Server"
python audit_uploader.py --config_file="$CONFIG_FILE" --datestamp $DATE --session="$SESSION" --client="$CLIENT" --fcm="$FCM"
python3 audit_uploader.py --config_file="$CONFIG_FILE" --datestamp $DATE --session="$SESSION" --client="$CLIENT" --fcm="$FCM"
fi
done
+42 -28
View File
@@ -6,31 +6,31 @@ import paramiko, ftplib
#########################################################
def usage():
print "-------------------------------------------------------"
print "-Usage: -"
print "-------------------------------------------------------"
print "- --config_file= : Path to upload config file -"
print "- -if no path is specified look -"
print "- in the working directory. -"
print "- -"
print "- --help : Print this message -"
print "- -"
print "- --datestamp= : Optional, defaults to today -"
print "- -replaces *datestamp in the -"
print "- the upload config file -"
print "- -"
print "- --fcm : Optional, spark input -"
print "- -"
print "- --session : Optional, spark jf session -"
print "-------------------------------------------------------"
print ("-------------------------------------------------------")
print ("-Usage: -")
print ("-------------------------------------------------------")
print ("- --config_file= : Path to upload config file -")
print ("- -if no path is specified look -")
print ("- in the working directory. -")
print ("- -")
print ("- --help : Print this message -")
print ("- -")
print ("- --datestamp= : Optional, defaults to today -")
print ("- -replaces *datestamp in the -")
print ("- the upload config file -")
print ("- -")
print ("- --fcm : Optional, spark input -")
print ("- -")
print ("- --session : Optional, spark jf session -")
print ("-------------------------------------------------------")
fcm = ''
session = ''
try:
options, arguments = getopt.getopt(sys.argv[1:],"h",['config_file=', 'datestamp=', 'help', 'fcm=', 'client=', 'session='])
except getopt.GetoptError, err:
print str(err)
except getopt.GetoptError as err:
print(err)
usage()
sys.exit(2)
@@ -54,11 +54,11 @@ for argument in arguments:
usage()
sys.exit()
else:
print "Unrecognized commandline argument. Type help for usage."
print ("Unrecognized commandline argument. Type help for usage.")
sys.exit()
if len(options) <= 0:
print "Requires a config file to run."
print("Requires a config file to run.")
usage()
sys.exit()
@@ -98,14 +98,14 @@ def read_config_file(config_file, datestamp=datetime.datetime.today().strftime('
# FTP File Transfer Function #
#########################################################
def putfile_ftp(file, put_path, site, user, password, port=21, verbose=True):
if verbose: print 'Uploading', file
if verbose: print("Uploading", file)
local = open(file, 'rb')
remote = ftplib.FTP(site)
remote.login(user, password)
remote.storbinary(put_path, local, 1024)
remote.quit()
local.close()
if verbose: print 'Upload done.'
if verbose: print("Upload done.")
#########################################################
# SFTP File Transfer Function #
@@ -136,15 +136,29 @@ else:
print upload_config
print(upload_config)
if upload_config["PROTOCOL_PUT"] in ['FTP','ftp']:
print "Protocol = FTP"
putfile_ftp(upload_config["GET_ILINK3_FILE_PATH"] + upload_config["GET_ILINK3_FILE_NAME"], upload_config["PUT_ILINK3_FILE_PATH"] + upload_config["PUT_ILINK3_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], upload_config["PORT_PUT"] )
print("Protocol = FTP")
try:
putfile_ftp(upload_config["GET_ILINK3_FILE_PATH"] + upload_config["GET_ILINK3_FILE_NAME"], upload_config["PUT_ILINK3_FILE_PATH"] + upload_config["PUT_ILINK3_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], upload_config["PORT_PUT"] )
except KeyError:
pass
try:
putfile_ftp(upload_config["GET_JETFIX_FILE_PATH"] + upload_config["GET_JETFIX_FILE_NAME"], upload_config["PUT_JETFIX_FILE_PATH"] + upload_config["PUT_JETFIX_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], upload_config["PORT_PUT"] )
except KeyError:
pass
putfile_ftp(upload_config["GET_MD_FILE_PATH"] + upload_config["GET_MD_FILE_NAME"], upload_config["PUT_MD_FILE_PATH"] + upload_config["PUT_MD_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], upload_config["PORT_PUT"] )
elif upload_config["PROTOCOL_PUT"] in ['SFTP','sftp']:
print "Protocol = SFTP"
putfile_sftp(upload_config["GET_ILINK3_FILE_PATH"] + upload_config["GET_ILINK3_FILE_NAME"], upload_config["PUT_ILINK3_FILE_PATH"] + upload_config["PUT_ILINK3_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], int(upload_config["PORT_PUT"]) )
print("Protocol = SFTP")
try:
putfile_sftp(upload_config["GET_ILINK3_FILE_PATH"] + upload_config["GET_ILINK3_FILE_NAME"], upload_config["PUT_ILINK3_FILE_PATH"] + upload_config["PUT_ILINK3_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], int(upload_config["PORT_PUT"]) )
except KeyError:
pass
try:
putfile_sftp(upload_config["GET_JETFIX_FILE_PATH"] + upload_config["GET_JETFIX_FILE_NAME"], upload_config["PUT_JETFIX_FILE_PATH"] + upload_config["PUT_JETFIX_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], int(upload_config["PORT_PUT"]) )
except KeyError:
pass
putfile_sftp(upload_config["GET_MD_FILE_PATH"] + upload_config["GET_MD_FILE_NAME"], upload_config["PUT_MD_FILE_PATH"] + upload_config["PUT_MD_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], int(upload_config["PORT_PUT"]) )