ice audit scripts
This commit is contained in:
1 parent
0cb11b7f8d
commit
07591f8e28
4 files changed
+267
-36
No files matched your search
+1
-1
Submodule docker/linux updated: 6f0dd9324b...3d98b7fa19.
Executable
+206
@@ -0,0 +1,206 @@
|
||||
#!/usr/bin/python
|
||||
|
||||
import os
|
||||
import re
|
||||
import fileinput
|
||||
import sys
|
||||
import collections
|
||||
import csv
|
||||
import time, datetime
|
||||
import copy
|
||||
import itertools
|
||||
import argparse
|
||||
|
||||
# region fix record processing
|
||||
|
||||
valid_message_types = {
|
||||
'D': 'NEW ORDER',
|
||||
'G': 'MODIFY',
|
||||
'F': 'CANCEL',
|
||||
'8': 'EXECUTION',
|
||||
'i': 'MASS QUOTE',
|
||||
'R': 'QUOTE REQUEST',
|
||||
'b': 'QUOTE ACK',
|
||||
'Z': 'MASS QUOTE CANCEL',
|
||||
# 'c': 'SECURITY DEFINITION REQUEST',
|
||||
'3': 'SESSION LEVEL REJECT',
|
||||
'j': 'BUSINESS LEVEL REJECT',
|
||||
'9': 'CANCEL REJECT',
|
||||
'CA': 'ORDER MASS ACTION REQUEST',
|
||||
'BZ': 'ORDER MASS ACTION REPORT'
|
||||
}
|
||||
|
||||
def print_headers():
|
||||
# From https://www.theice.com/publicdocs/futures_us/exchange_notices/ICE_Advisory_0113_009.pdf
|
||||
line_to_write = "Active Date,Cl Ord ID,OrderID,Transact Time,Symbol,Symbol,SecurityType,Stk Prc,P/C,Side,OrderQty,Max Show,OrdType,\
|
||||
Price,StopPx,Clearing Account,Clearing Firm,Order State,AccountCode,CTI Code,Originator User ID,OnBehalfOfCompID,\
|
||||
OnBehalfOfCompID,On Behalf Of Sub ID,On Behalf Of Sub ID,On Behalf Of Location ID,Time In Force"
|
||||
|
||||
with open (args.output_file, 'w') as the_file:
|
||||
the_file.write(line_to_write)
|
||||
the_file.write("\n")
|
||||
|
||||
line_to_write = "Activity Date,Client Order ID,Original Order ID,Order Entry/Modification Time,Commodity,Contract Month,Future/Option Indicator,\
|
||||
Strike Price,,Put or Call,Buy/Sell Indicator,Quantity,Reserve Quantity,Order Type,Price,Stop Price,Account #,CM#,\
|
||||
Order Status,Clearing Account Code,CTI,ETS Login ID,Routing Member ID,Authorized Trader ID,Authorized Trdr ID,\
|
||||
Routing Trdr,Routing Grp,Time In Force"
|
||||
|
||||
with open (args.output_file, 'a') as the_file:
|
||||
the_file.write(line_to_write)
|
||||
the_file.write("\n")
|
||||
|
||||
|
||||
def getter(field_name):
|
||||
def result(parsed_fix_record):
|
||||
if field_name in parsed_fix_record:
|
||||
return parsed_fix_record[field_name]
|
||||
else:
|
||||
return None
|
||||
return result
|
||||
|
||||
def parse_fix_line(line):
|
||||
ts = line[:21]
|
||||
fix_line = line[24:].split('\1')
|
||||
result = {}
|
||||
result['timestamp'] = ts
|
||||
for fix_field in fix_line:
|
||||
rx = r"(?P<tag>\d+)?=(?P<value>.*)"
|
||||
match = re.match(rx,fix_field)
|
||||
if match:
|
||||
result[match.group('tag')] = match.group('value')
|
||||
return result
|
||||
|
||||
def tagged_field(line,number):
|
||||
if number in line:
|
||||
return line[number]
|
||||
else:
|
||||
return ""
|
||||
|
||||
def formatted_tagged_field(line,number):
|
||||
if number in line:
|
||||
return line[number].lstrip("0")
|
||||
else:
|
||||
return ""
|
||||
|
||||
def lr_tagged_field(line,number):
|
||||
if number in line:
|
||||
return line[number].rsplit('|',1)
|
||||
else:
|
||||
return "",""
|
||||
|
||||
def lr_contract_month(line,number):
|
||||
try:
|
||||
ice_line="ICE_SECURITY_ID_TO_SYMBOL_MAPPING: " + line[number]
|
||||
with open('/sparkdata/data/md_dump.txt') as f:
|
||||
datafile = f.readlines()
|
||||
for data in datafile:
|
||||
if data.find(str(ice_line)) != -1:
|
||||
placeholder=data.find('> ')+1
|
||||
get_ending=data[placeholder:]
|
||||
get_ending=get_ending.strip()
|
||||
return get_ending.split(" ")
|
||||
except KeyError:
|
||||
print("Can not find key 55")
|
||||
return "",""
|
||||
|
||||
|
||||
def valid_file(f):
|
||||
if not os.path.exists(f):
|
||||
# Argparse uses the ArgumentTypeError to give a rejection message like:
|
||||
# error: argument input: x does not exist
|
||||
raise argparse.ArgumentTypeError("{0} does not exist".format(f))
|
||||
return f
|
||||
|
||||
def valid_date(s):
|
||||
try:
|
||||
return datetime.datetime.strptime(s, "%Y%m%d")
|
||||
except ValueError:
|
||||
raise argparse.ArgumentTypeError("Not a valid date: '{0}'.".format(s))
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("quickfix_log", help="the quickfix log[.gz] you wish to process", type=valid_file)
|
||||
parser.add_argument("output_file", help="the destination csv file you wish to write")
|
||||
parser.add_argument("-d", "--date", help="the date you wish to parse from the log, specified as YYYYmmdd",
|
||||
default=datetime.datetime.today().strftime("%Y%m%d"), type=valid_date)
|
||||
#quickfix_log="/home/pfisher/temp/audit/fix"
|
||||
#output_file="/home/pfisher/temp/audit/fix.csv"
|
||||
args = parser.parse_args()
|
||||
|
||||
# Note that the timedeltas specificed in trade_day_start and trade_day_end
|
||||
# are based on Chicago times; as such, this script should run on a machine
|
||||
# set to a Chicago timezone. A future impromovement would be to determine
|
||||
# the current machine's timezone and determine the appropriate offset
|
||||
# versus Chicago time, but for now we assume this will run on a Chicago
|
||||
# tz machine.
|
||||
input_as_datetime = args.date
|
||||
trade_day_start = input_as_datetime - datetime.timedelta(days=1) + datetime.timedelta(hours=17, minutes=30)
|
||||
trade_day_utc_start = time.gmtime((time.mktime(trade_day_start.timetuple())))
|
||||
trade_day_end = input_as_datetime + datetime.timedelta(hours=17)
|
||||
trade_day_utc_end = time.gmtime((time.mktime(trade_day_end.timetuple())))
|
||||
|
||||
#file_opener = open
|
||||
file_opener = None
|
||||
if args.quickfix_log.endswith(".gz"):
|
||||
import gzip
|
||||
file_opener = gzip.open
|
||||
else:
|
||||
file_opener = open
|
||||
|
||||
audit_entries = []
|
||||
line_number = 0
|
||||
print_headers()
|
||||
with file_opener(args.quickfix_log) as input_file:
|
||||
#with file_opener(quickfix_log) as input_file:
|
||||
for line in input_file:
|
||||
try:
|
||||
line_timestamp = time.strptime(line[:21], "%Y%m%d-%H:%M:%S.%f")
|
||||
except:
|
||||
continue
|
||||
if trade_day_utc_start <= line_timestamp <= trade_day_utc_end:
|
||||
parsed_fix_record = parse_fix_line(line)
|
||||
if parsed_fix_record['35'] in valid_message_types:
|
||||
line_number = line_number + 1
|
||||
# Static fields
|
||||
tt = datetime.datetime.strptime(line[:21], "%Y%m%d-%H:%M:%S.%f")
|
||||
transacttime = tt.strftime('%Y%m%d-%H:%M:%S.%f')
|
||||
activedate = time.strftime('%Y%m%d',time.strptime(line[:8], "%Y%m%d"))
|
||||
securitytype = "FUT"
|
||||
stkprc = ""
|
||||
putcall = ""
|
||||
#symbol = "HNG"
|
||||
#contractmonth = "U20"
|
||||
# No formatting fields
|
||||
ciorderid = tagged_field(parsed_fix_record, '11')
|
||||
orderid = tagged_field(parsed_fix_record,'37')
|
||||
side = tagged_field(parsed_fix_record,'54')
|
||||
orderquanity = tagged_field(parsed_fix_record,'38')
|
||||
maxshow = tagged_field(parsed_fix_record,'201')
|
||||
ordertype = tagged_field(parsed_fix_record,'40')
|
||||
stoppx = tagged_field(parsed_fix_record,'99')
|
||||
clearingaccount = tagged_field(parsed_fix_record,'440')
|
||||
clearingfirm = tagged_field(parsed_fix_record,'439')
|
||||
orderstate = tagged_field(parsed_fix_record,'9175')
|
||||
accountcode = tagged_field(parsed_fix_record,'9195')
|
||||
cticode = tagged_field(parsed_fix_record,'9208')
|
||||
originatoruserid = tagged_field(parsed_fix_record,'9139')
|
||||
timeinforce = tagged_field(parsed_fix_record,'59')
|
||||
# Needs Formatting
|
||||
price = formatted_tagged_field(parsed_fix_record,'44')
|
||||
# Left Right
|
||||
OnBehalfOfCompID_Left,OnBehalfOfCompID_Right = lr_tagged_field(parsed_fix_record,'115')
|
||||
OnBehalfOfSubID_Left,OnBehalfOfSubID_Right = lr_tagged_field(parsed_fix_record,'116')
|
||||
OnBehalfOfLocationID_Left,OnBehalfOfLocationID_Right = lr_tagged_field(parsed_fix_record,'144')
|
||||
if parsed_fix_record['35'] == "9":
|
||||
symbol = ""
|
||||
contractmonth = ""
|
||||
else:
|
||||
symbol,contractmonth = lr_contract_month(parsed_fix_record,'55')
|
||||
line_to_write = activedate + "," + ciorderid + "," + orderid + "," + transacttime + "," + symbol + "," + contractmonth + "," + securitytype + "," + \
|
||||
stkprc + "," + putcall + "," + side + "," + orderquanity + "," + maxshow + "," + ordertype + "," + price + "," + stoppx + "," + \
|
||||
clearingaccount + "," + clearingfirm + "," + orderstate + "," + accountcode + "," + cticode + "," + originatoruserid + "," + OnBehalfOfCompID_Left + "," + \
|
||||
OnBehalfOfCompID_Right + "," + OnBehalfOfSubID_Left + "," + OnBehalfOfSubID_Right + "," + OnBehalfOfLocationID_Left + "," + timeinforce
|
||||
with open (args.output_file, 'a') as the_file:
|
||||
the_file.write(line_to_write)
|
||||
the_file.write("\n")
|
||||
# except Exception:
|
||||
# pass
|
||||
@@ -40,23 +40,34 @@ SESSION_ID=${conf_vars[session_id]}
|
||||
FCM=${conf_vars[fcm]}
|
||||
CLIENT=${conf_vars[client]}
|
||||
|
||||
# Create MD dump
|
||||
cd /sparkbin
|
||||
./sparkd md dumpsecurity all > /sparkdata/data/md_dump.txt
|
||||
|
||||
cd /sparkbin/data_files
|
||||
|
||||
for SESSION in $(ls /sparkdata/data/ilink3/); do
|
||||
for SESSION in $(ls /sparkdata/data/{ilink3,jetfix}/); do
|
||||
if [[ $SESSION == *$SESSION_ID* ]]; then
|
||||
echo "Session is $SESSION"
|
||||
SESSION_DAY=$(date +%Y-%m-%d)
|
||||
SESSION_DAY=$(date -d $DATE +%Y-%m-%d)
|
||||
if [[ $SESSION == "cme_"* ]]; then
|
||||
SESSION_DAY=$(date -d "$DATE -$(date -d $DATE +%u) days" +"%Y-%m-%d")
|
||||
AUDIT_SCRIPT="audit_ilink3.py"
|
||||
ILINK_TYPE="ilink3"
|
||||
LOG_NAME="log"
|
||||
elif [[ $SESSION == "ice_"* ]]; then
|
||||
SESSION_DAY=$(date -d "$DATE 2 days ago" +%Y-%m-%d)
|
||||
AUDIT_SCRIPT="audit_ice.py"
|
||||
ILINK_TYPE="jetfix"
|
||||
LOG_NAME="fix"
|
||||
fi
|
||||
echo $SESSION_DAY
|
||||
if [[ ! -d /sparkdata/data/ilink3/$SESSION/audit_logs ]]; then
|
||||
mkdir -p /sparkdata/data/ilink3/$SESSION/audit_logs
|
||||
if [[ ! -d /sparkdata/data/$ILINK_TYPE/$SESSION/audit_logs ]]; then
|
||||
mkdir -p /sparkdata/data/$ILINK_TYPE/$SESSION/audit_logs
|
||||
fi
|
||||
if [[ -d /sparkdata/data/ilink3/$SESSION/$SESSION_DAY ]]; then
|
||||
if [[ -d /sparkdata/data/$ILINK_TYPE/$SESSION/$SESSION_DAY ]]; then
|
||||
echo "Starting the creation of the audit log"
|
||||
python $AUDIT_SCRIPT -d $DATE /sparkdata/data/ilink3/$SESSION/$SESSION_DAY/log /sparkdata/data/ilink3/$SESSION/audit_logs/"$FCM"_audit_"$SESSION"_"$DATE".csv
|
||||
python3 $AUDIT_SCRIPT -d $DATE /sparkdata/data/$ILINK_TYPE/$SESSION/$SESSION_DAY/$LOG_NAME /sparkdata/data/$ILINK_TYPE/$SESSION/audit_logs/"$FCM"_audit_"$SESSION"_"$DATE".csv
|
||||
fi
|
||||
if [[ ! -d /sparkdata/data/md_audit ]]; then
|
||||
mkdir /sparkdata/data/md_audit
|
||||
@@ -68,6 +79,6 @@ for SESSION in $(ls /sparkdata/data/ilink3/); do
|
||||
cd /sparkbin/data_files
|
||||
fi
|
||||
echo "Uploading to the (s)FTP Server"
|
||||
python audit_uploader.py --config_file="$CONFIG_FILE" --datestamp $DATE --session="$SESSION" --client="$CLIENT" --fcm="$FCM"
|
||||
python3 audit_uploader.py --config_file="$CONFIG_FILE" --datestamp $DATE --session="$SESSION" --client="$CLIENT" --fcm="$FCM"
|
||||
fi
|
||||
done
|
||||
@@ -6,31 +6,31 @@ import paramiko, ftplib
|
||||
#########################################################
|
||||
|
||||
def usage():
|
||||
print "-------------------------------------------------------"
|
||||
print "-Usage: -"
|
||||
print "-------------------------------------------------------"
|
||||
print "- --config_file= : Path to upload config file -"
|
||||
print "- -if no path is specified look -"
|
||||
print "- in the working directory. -"
|
||||
print "- -"
|
||||
print "- --help : Print this message -"
|
||||
print "- -"
|
||||
print "- --datestamp= : Optional, defaults to today -"
|
||||
print "- -replaces *datestamp in the -"
|
||||
print "- the upload config file -"
|
||||
print "- -"
|
||||
print "- --fcm : Optional, spark input -"
|
||||
print "- -"
|
||||
print "- --session : Optional, spark jf session -"
|
||||
print "-------------------------------------------------------"
|
||||
print ("-------------------------------------------------------")
|
||||
print ("-Usage: -")
|
||||
print ("-------------------------------------------------------")
|
||||
print ("- --config_file= : Path to upload config file -")
|
||||
print ("- -if no path is specified look -")
|
||||
print ("- in the working directory. -")
|
||||
print ("- -")
|
||||
print ("- --help : Print this message -")
|
||||
print ("- -")
|
||||
print ("- --datestamp= : Optional, defaults to today -")
|
||||
print ("- -replaces *datestamp in the -")
|
||||
print ("- the upload config file -")
|
||||
print ("- -")
|
||||
print ("- --fcm : Optional, spark input -")
|
||||
print ("- -")
|
||||
print ("- --session : Optional, spark jf session -")
|
||||
print ("-------------------------------------------------------")
|
||||
|
||||
fcm = ''
|
||||
session = ''
|
||||
|
||||
try:
|
||||
options, arguments = getopt.getopt(sys.argv[1:],"h",['config_file=', 'datestamp=', 'help', 'fcm=', 'client=', 'session='])
|
||||
except getopt.GetoptError, err:
|
||||
print str(err)
|
||||
except getopt.GetoptError as err:
|
||||
print(err)
|
||||
usage()
|
||||
sys.exit(2)
|
||||
|
||||
@@ -54,11 +54,11 @@ for argument in arguments:
|
||||
usage()
|
||||
sys.exit()
|
||||
else:
|
||||
print "Unrecognized commandline argument. Type help for usage."
|
||||
print ("Unrecognized commandline argument. Type help for usage.")
|
||||
sys.exit()
|
||||
|
||||
if len(options) <= 0:
|
||||
print "Requires a config file to run."
|
||||
print("Requires a config file to run.")
|
||||
usage()
|
||||
sys.exit()
|
||||
|
||||
@@ -98,14 +98,14 @@ def read_config_file(config_file, datestamp=datetime.datetime.today().strftime('
|
||||
# FTP File Transfer Function #
|
||||
#########################################################
|
||||
def putfile_ftp(file, put_path, site, user, password, port=21, verbose=True):
|
||||
if verbose: print 'Uploading', file
|
||||
if verbose: print("Uploading", file)
|
||||
local = open(file, 'rb')
|
||||
remote = ftplib.FTP(site)
|
||||
remote.login(user, password)
|
||||
remote.storbinary(put_path, local, 1024)
|
||||
remote.quit()
|
||||
local.close()
|
||||
if verbose: print 'Upload done.'
|
||||
if verbose: print("Upload done.")
|
||||
|
||||
#########################################################
|
||||
# SFTP File Transfer Function #
|
||||
@@ -136,15 +136,29 @@ else:
|
||||
|
||||
|
||||
|
||||
print upload_config
|
||||
print(upload_config)
|
||||
|
||||
if upload_config["PROTOCOL_PUT"] in ['FTP','ftp']:
|
||||
print "Protocol = FTP"
|
||||
putfile_ftp(upload_config["GET_ILINK3_FILE_PATH"] + upload_config["GET_ILINK3_FILE_NAME"], upload_config["PUT_ILINK3_FILE_PATH"] + upload_config["PUT_ILINK3_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], upload_config["PORT_PUT"] )
|
||||
print("Protocol = FTP")
|
||||
try:
|
||||
putfile_ftp(upload_config["GET_ILINK3_FILE_PATH"] + upload_config["GET_ILINK3_FILE_NAME"], upload_config["PUT_ILINK3_FILE_PATH"] + upload_config["PUT_ILINK3_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], upload_config["PORT_PUT"] )
|
||||
except KeyError:
|
||||
pass
|
||||
try:
|
||||
putfile_ftp(upload_config["GET_JETFIX_FILE_PATH"] + upload_config["GET_JETFIX_FILE_NAME"], upload_config["PUT_JETFIX_FILE_PATH"] + upload_config["PUT_JETFIX_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], upload_config["PORT_PUT"] )
|
||||
except KeyError:
|
||||
pass
|
||||
putfile_ftp(upload_config["GET_MD_FILE_PATH"] + upload_config["GET_MD_FILE_NAME"], upload_config["PUT_MD_FILE_PATH"] + upload_config["PUT_MD_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], upload_config["PORT_PUT"] )
|
||||
elif upload_config["PROTOCOL_PUT"] in ['SFTP','sftp']:
|
||||
print "Protocol = SFTP"
|
||||
putfile_sftp(upload_config["GET_ILINK3_FILE_PATH"] + upload_config["GET_ILINK3_FILE_NAME"], upload_config["PUT_ILINK3_FILE_PATH"] + upload_config["PUT_ILINK3_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], int(upload_config["PORT_PUT"]) )
|
||||
print("Protocol = SFTP")
|
||||
try:
|
||||
putfile_sftp(upload_config["GET_ILINK3_FILE_PATH"] + upload_config["GET_ILINK3_FILE_NAME"], upload_config["PUT_ILINK3_FILE_PATH"] + upload_config["PUT_ILINK3_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], int(upload_config["PORT_PUT"]) )
|
||||
except KeyError:
|
||||
pass
|
||||
try:
|
||||
putfile_sftp(upload_config["GET_JETFIX_FILE_PATH"] + upload_config["GET_JETFIX_FILE_NAME"], upload_config["PUT_JETFIX_FILE_PATH"] + upload_config["PUT_JETFIX_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], int(upload_config["PORT_PUT"]) )
|
||||
except KeyError:
|
||||
pass
|
||||
putfile_sftp(upload_config["GET_MD_FILE_PATH"] + upload_config["GET_MD_FILE_NAME"], upload_config["PUT_MD_FILE_PATH"] + upload_config["PUT_MD_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], int(upload_config["PORT_PUT"]) )
|
||||
|
||||
|
||||
Reference in new issue
Block a user