diff --git a/docker/linux b/docker/linux index 6f0dd9324..3d98b7fa1 160000 --- a/docker/linux +++ b/docker/linux @@ -1 +1 @@ -Subproject commit 6f0dd9324bd0cc8785eed987402a9d91f816efa9 +Subproject commit 3d98b7fa19e39ef80b95a45a31d7bdf51346a38c diff --git a/src/data_files/files/audit_ice.py b/src/data_files/files/audit_ice.py new file mode 100755 index 000000000..a50095907 --- /dev/null +++ b/src/data_files/files/audit_ice.py @@ -0,0 +1,206 @@ +#!/usr/bin/python + +import os +import re +import fileinput +import sys +import collections +import csv +import time, datetime +import copy +import itertools +import argparse + +# region fix record processing + +valid_message_types = { + 'D': 'NEW ORDER', + 'G': 'MODIFY', + 'F': 'CANCEL', + '8': 'EXECUTION', + 'i': 'MASS QUOTE', + 'R': 'QUOTE REQUEST', + 'b': 'QUOTE ACK', + 'Z': 'MASS QUOTE CANCEL', + # 'c': 'SECURITY DEFINITION REQUEST', + '3': 'SESSION LEVEL REJECT', + 'j': 'BUSINESS LEVEL REJECT', + '9': 'CANCEL REJECT', + 'CA': 'ORDER MASS ACTION REQUEST', + 'BZ': 'ORDER MASS ACTION REPORT' +} + +def print_headers(): + # From https://www.theice.com/publicdocs/futures_us/exchange_notices/ICE_Advisory_0113_009.pdf + line_to_write = "Active Date,Cl Ord ID,OrderID,Transact Time,Symbol,Symbol,SecurityType,Stk Prc,P/C,Side,OrderQty,Max Show,OrdType,\ +Price,StopPx,Clearing Account,Clearing Firm,Order State,AccountCode,CTI Code,Originator User ID,OnBehalfOfCompID,\ +OnBehalfOfCompID,On Behalf Of Sub ID,On Behalf Of Sub ID,On Behalf Of Location ID,Time In Force" + + with open (args.output_file, 'w') as the_file: + the_file.write(line_to_write) + the_file.write("\n") + + line_to_write = "Activity Date,Client Order ID,Original Order ID,Order Entry/Modification Time,Commodity,Contract Month,Future/Option Indicator,\ +Strike Price,,Put or Call,Buy/Sell Indicator,Quantity,Reserve Quantity,Order Type,Price,Stop Price,Account #,CM#,\ +Order Status,Clearing Account Code,CTI,ETS Login ID,Routing Member ID,Authorized Trader ID,Authorized Trdr ID,\ +Routing Trdr,Routing Grp,Time In Force" + + with open (args.output_file, 'a') as the_file: + the_file.write(line_to_write) + the_file.write("\n") + + +def getter(field_name): + def result(parsed_fix_record): + if field_name in parsed_fix_record: + return parsed_fix_record[field_name] + else: + return None + return result + +def parse_fix_line(line): + ts = line[:21] + fix_line = line[24:].split('\1') + result = {} + result['timestamp'] = ts + for fix_field in fix_line: + rx = r"(?P\d+)?=(?P.*)" + match = re.match(rx,fix_field) + if match: + result[match.group('tag')] = match.group('value') + return result + +def tagged_field(line,number): + if number in line: + return line[number] + else: + return "" + +def formatted_tagged_field(line,number): + if number in line: + return line[number].lstrip("0") + else: + return "" + +def lr_tagged_field(line,number): + if number in line: + return line[number].rsplit('|',1) + else: + return "","" + +def lr_contract_month(line,number): + try: + ice_line="ICE_SECURITY_ID_TO_SYMBOL_MAPPING: " + line[number] + with open('/sparkdata/data/md_dump.txt') as f: + datafile = f.readlines() + for data in datafile: + if data.find(str(ice_line)) != -1: + placeholder=data.find('> ')+1 + get_ending=data[placeholder:] + get_ending=get_ending.strip() + return get_ending.split(" ") + except KeyError: + print("Can not find key 55") + return "","" + + +def valid_file(f): + if not os.path.exists(f): + # Argparse uses the ArgumentTypeError to give a rejection message like: + # error: argument input: x does not exist + raise argparse.ArgumentTypeError("{0} does not exist".format(f)) + return f + +def valid_date(s): + try: + return datetime.datetime.strptime(s, "%Y%m%d") + except ValueError: + raise argparse.ArgumentTypeError("Not a valid date: '{0}'.".format(s)) + +parser = argparse.ArgumentParser() +parser.add_argument("quickfix_log", help="the quickfix log[.gz] you wish to process", type=valid_file) +parser.add_argument("output_file", help="the destination csv file you wish to write") +parser.add_argument("-d", "--date", help="the date you wish to parse from the log, specified as YYYYmmdd", + default=datetime.datetime.today().strftime("%Y%m%d"), type=valid_date) +#quickfix_log="/home/pfisher/temp/audit/fix" +#output_file="/home/pfisher/temp/audit/fix.csv" +args = parser.parse_args() + +# Note that the timedeltas specificed in trade_day_start and trade_day_end +# are based on Chicago times; as such, this script should run on a machine +# set to a Chicago timezone. A future impromovement would be to determine +# the current machine's timezone and determine the appropriate offset +# versus Chicago time, but for now we assume this will run on a Chicago +# tz machine. +input_as_datetime = args.date +trade_day_start = input_as_datetime - datetime.timedelta(days=1) + datetime.timedelta(hours=17, minutes=30) +trade_day_utc_start = time.gmtime((time.mktime(trade_day_start.timetuple()))) +trade_day_end = input_as_datetime + datetime.timedelta(hours=17) +trade_day_utc_end = time.gmtime((time.mktime(trade_day_end.timetuple()))) + +#file_opener = open +file_opener = None +if args.quickfix_log.endswith(".gz"): + import gzip + file_opener = gzip.open +else: + file_opener = open + +audit_entries = [] +line_number = 0 +print_headers() +with file_opener(args.quickfix_log) as input_file: +#with file_opener(quickfix_log) as input_file: + for line in input_file: + try: + line_timestamp = time.strptime(line[:21], "%Y%m%d-%H:%M:%S.%f") + except: + continue + if trade_day_utc_start <= line_timestamp <= trade_day_utc_end: + parsed_fix_record = parse_fix_line(line) + if parsed_fix_record['35'] in valid_message_types: + line_number = line_number + 1 + # Static fields + tt = datetime.datetime.strptime(line[:21], "%Y%m%d-%H:%M:%S.%f") + transacttime = tt.strftime('%Y%m%d-%H:%M:%S.%f') + activedate = time.strftime('%Y%m%d',time.strptime(line[:8], "%Y%m%d")) + securitytype = "FUT" + stkprc = "" + putcall = "" + #symbol = "HNG" + #contractmonth = "U20" + # No formatting fields + ciorderid = tagged_field(parsed_fix_record, '11') + orderid = tagged_field(parsed_fix_record,'37') + side = tagged_field(parsed_fix_record,'54') + orderquanity = tagged_field(parsed_fix_record,'38') + maxshow = tagged_field(parsed_fix_record,'201') + ordertype = tagged_field(parsed_fix_record,'40') + stoppx = tagged_field(parsed_fix_record,'99') + clearingaccount = tagged_field(parsed_fix_record,'440') + clearingfirm = tagged_field(parsed_fix_record,'439') + orderstate = tagged_field(parsed_fix_record,'9175') + accountcode = tagged_field(parsed_fix_record,'9195') + cticode = tagged_field(parsed_fix_record,'9208') + originatoruserid = tagged_field(parsed_fix_record,'9139') + timeinforce = tagged_field(parsed_fix_record,'59') + # Needs Formatting + price = formatted_tagged_field(parsed_fix_record,'44') + # Left Right + OnBehalfOfCompID_Left,OnBehalfOfCompID_Right = lr_tagged_field(parsed_fix_record,'115') + OnBehalfOfSubID_Left,OnBehalfOfSubID_Right = lr_tagged_field(parsed_fix_record,'116') + OnBehalfOfLocationID_Left,OnBehalfOfLocationID_Right = lr_tagged_field(parsed_fix_record,'144') + if parsed_fix_record['35'] == "9": + symbol = "" + contractmonth = "" + else: + symbol,contractmonth = lr_contract_month(parsed_fix_record,'55') + line_to_write = activedate + "," + ciorderid + "," + orderid + "," + transacttime + "," + symbol + "," + contractmonth + "," + securitytype + "," + \ + stkprc + "," + putcall + "," + side + "," + orderquanity + "," + maxshow + "," + ordertype + "," + price + "," + stoppx + "," + \ + clearingaccount + "," + clearingfirm + "," + orderstate + "," + accountcode + "," + cticode + "," + originatoruserid + "," + OnBehalfOfCompID_Left + "," + \ + OnBehalfOfCompID_Right + "," + OnBehalfOfSubID_Left + "," + OnBehalfOfSubID_Right + "," + OnBehalfOfLocationID_Left + "," + timeinforce + with open (args.output_file, 'a') as the_file: + the_file.write(line_to_write) + the_file.write("\n") +# except Exception: +# pass diff --git a/src/data_files/files/audit_upload.sh b/src/data_files/files/audit_upload.sh index 2aad9604d..a077a187b 100755 --- a/src/data_files/files/audit_upload.sh +++ b/src/data_files/files/audit_upload.sh @@ -40,23 +40,34 @@ SESSION_ID=${conf_vars[session_id]} FCM=${conf_vars[fcm]} CLIENT=${conf_vars[client]} +# Create MD dump +cd /sparkbin +./sparkd md dumpsecurity all > /sparkdata/data/md_dump.txt + cd /sparkbin/data_files -for SESSION in $(ls /sparkdata/data/ilink3/); do +for SESSION in $(ls /sparkdata/data/{ilink3,jetfix}/); do if [[ $SESSION == *$SESSION_ID* ]]; then echo "Session is $SESSION" - SESSION_DAY=$(date +%Y-%m-%d) + SESSION_DAY=$(date -d $DATE +%Y-%m-%d) if [[ $SESSION == "cme_"* ]]; then SESSION_DAY=$(date -d "$DATE -$(date -d $DATE +%u) days" +"%Y-%m-%d") AUDIT_SCRIPT="audit_ilink3.py" + ILINK_TYPE="ilink3" + LOG_NAME="log" + elif [[ $SESSION == "ice_"* ]]; then + SESSION_DAY=$(date -d "$DATE 2 days ago" +%Y-%m-%d) + AUDIT_SCRIPT="audit_ice.py" + ILINK_TYPE="jetfix" + LOG_NAME="fix" fi echo $SESSION_DAY - if [[ ! -d /sparkdata/data/ilink3/$SESSION/audit_logs ]]; then - mkdir -p /sparkdata/data/ilink3/$SESSION/audit_logs + if [[ ! -d /sparkdata/data/$ILINK_TYPE/$SESSION/audit_logs ]]; then + mkdir -p /sparkdata/data/$ILINK_TYPE/$SESSION/audit_logs fi - if [[ -d /sparkdata/data/ilink3/$SESSION/$SESSION_DAY ]]; then + if [[ -d /sparkdata/data/$ILINK_TYPE/$SESSION/$SESSION_DAY ]]; then echo "Starting the creation of the audit log" - python $AUDIT_SCRIPT -d $DATE /sparkdata/data/ilink3/$SESSION/$SESSION_DAY/log /sparkdata/data/ilink3/$SESSION/audit_logs/"$FCM"_audit_"$SESSION"_"$DATE".csv + python3 $AUDIT_SCRIPT -d $DATE /sparkdata/data/$ILINK_TYPE/$SESSION/$SESSION_DAY/$LOG_NAME /sparkdata/data/$ILINK_TYPE/$SESSION/audit_logs/"$FCM"_audit_"$SESSION"_"$DATE".csv fi if [[ ! -d /sparkdata/data/md_audit ]]; then mkdir /sparkdata/data/md_audit @@ -68,6 +79,6 @@ for SESSION in $(ls /sparkdata/data/ilink3/); do cd /sparkbin/data_files fi echo "Uploading to the (s)FTP Server" - python audit_uploader.py --config_file="$CONFIG_FILE" --datestamp $DATE --session="$SESSION" --client="$CLIENT" --fcm="$FCM" + python3 audit_uploader.py --config_file="$CONFIG_FILE" --datestamp $DATE --session="$SESSION" --client="$CLIENT" --fcm="$FCM" fi done diff --git a/src/data_files/files/audit_uploader.py b/src/data_files/files/audit_uploader.py index 51dcbc5ca..f10587bf2 100755 --- a/src/data_files/files/audit_uploader.py +++ b/src/data_files/files/audit_uploader.py @@ -6,31 +6,31 @@ import paramiko, ftplib ######################################################### def usage(): - print "-------------------------------------------------------" - print "-Usage: -" - print "-------------------------------------------------------" - print "- --config_file= : Path to upload config file -" - print "- -if no path is specified look -" - print "- in the working directory. -" - print "- -" - print "- --help : Print this message -" - print "- -" - print "- --datestamp= : Optional, defaults to today -" - print "- -replaces *datestamp in the -" - print "- the upload config file -" - print "- -" - print "- --fcm : Optional, spark input -" - print "- -" - print "- --session : Optional, spark jf session -" - print "-------------------------------------------------------" + print ("-------------------------------------------------------") + print ("-Usage: -") + print ("-------------------------------------------------------") + print ("- --config_file= : Path to upload config file -") + print ("- -if no path is specified look -") + print ("- in the working directory. -") + print ("- -") + print ("- --help : Print this message -") + print ("- -") + print ("- --datestamp= : Optional, defaults to today -") + print ("- -replaces *datestamp in the -") + print ("- the upload config file -") + print ("- -") + print ("- --fcm : Optional, spark input -") + print ("- -") + print ("- --session : Optional, spark jf session -") + print ("-------------------------------------------------------") fcm = '' session = '' try: options, arguments = getopt.getopt(sys.argv[1:],"h",['config_file=', 'datestamp=', 'help', 'fcm=', 'client=', 'session=']) -except getopt.GetoptError, err: - print str(err) +except getopt.GetoptError as err: + print(err) usage() sys.exit(2) @@ -54,11 +54,11 @@ for argument in arguments: usage() sys.exit() else: - print "Unrecognized commandline argument. Type help for usage." + print ("Unrecognized commandline argument. Type help for usage.") sys.exit() if len(options) <= 0: - print "Requires a config file to run." + print("Requires a config file to run.") usage() sys.exit() @@ -98,14 +98,14 @@ def read_config_file(config_file, datestamp=datetime.datetime.today().strftime(' # FTP File Transfer Function # ######################################################### def putfile_ftp(file, put_path, site, user, password, port=21, verbose=True): - if verbose: print 'Uploading', file + if verbose: print("Uploading", file) local = open(file, 'rb') remote = ftplib.FTP(site) remote.login(user, password) remote.storbinary(put_path, local, 1024) remote.quit() local.close() - if verbose: print 'Upload done.' + if verbose: print("Upload done.") ######################################################### # SFTP File Transfer Function # @@ -136,15 +136,29 @@ else: -print upload_config +print(upload_config) if upload_config["PROTOCOL_PUT"] in ['FTP','ftp']: - print "Protocol = FTP" - putfile_ftp(upload_config["GET_ILINK3_FILE_PATH"] + upload_config["GET_ILINK3_FILE_NAME"], upload_config["PUT_ILINK3_FILE_PATH"] + upload_config["PUT_ILINK3_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], upload_config["PORT_PUT"] ) + print("Protocol = FTP") + try: + putfile_ftp(upload_config["GET_ILINK3_FILE_PATH"] + upload_config["GET_ILINK3_FILE_NAME"], upload_config["PUT_ILINK3_FILE_PATH"] + upload_config["PUT_ILINK3_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], upload_config["PORT_PUT"] ) + except KeyError: + pass + try: + putfile_ftp(upload_config["GET_JETFIX_FILE_PATH"] + upload_config["GET_JETFIX_FILE_NAME"], upload_config["PUT_JETFIX_FILE_PATH"] + upload_config["PUT_JETFIX_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], upload_config["PORT_PUT"] ) + except KeyError: + pass putfile_ftp(upload_config["GET_MD_FILE_PATH"] + upload_config["GET_MD_FILE_NAME"], upload_config["PUT_MD_FILE_PATH"] + upload_config["PUT_MD_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], upload_config["PORT_PUT"] ) elif upload_config["PROTOCOL_PUT"] in ['SFTP','sftp']: - print "Protocol = SFTP" - putfile_sftp(upload_config["GET_ILINK3_FILE_PATH"] + upload_config["GET_ILINK3_FILE_NAME"], upload_config["PUT_ILINK3_FILE_PATH"] + upload_config["PUT_ILINK3_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], int(upload_config["PORT_PUT"]) ) + print("Protocol = SFTP") + try: + putfile_sftp(upload_config["GET_ILINK3_FILE_PATH"] + upload_config["GET_ILINK3_FILE_NAME"], upload_config["PUT_ILINK3_FILE_PATH"] + upload_config["PUT_ILINK3_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], int(upload_config["PORT_PUT"]) ) + except KeyError: + pass + try: + putfile_sftp(upload_config["GET_JETFIX_FILE_PATH"] + upload_config["GET_JETFIX_FILE_NAME"], upload_config["PUT_JETFIX_FILE_PATH"] + upload_config["PUT_JETFIX_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], int(upload_config["PORT_PUT"]) ) + except KeyError: + pass putfile_sftp(upload_config["GET_MD_FILE_PATH"] + upload_config["GET_MD_FILE_NAME"], upload_config["PUT_MD_FILE_PATH"] + upload_config["PUT_MD_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], int(upload_config["PORT_PUT"]) )