Compare commits

..
Author SHA1 Message Date
Pete Fisher 602929b945 updating submodule to latest 2021-03-18 12:13:11 -04:00
Pete Fisher 07591f8e28 ice audit scripts 2021-03-18 11:05:34 -04:00
Pedro Pinto 0cb11b7f8d Made parsing of dump security easier to facilitate locating ice symbols. 2021-03-16 11:32:59 -04:00
Pete Fisher 1921f02d30 Merged in DJ_ICE_ManualOrderIndicator (pull request #417)
added ManualOrderIndicator to ICENativeOrder.cpp after merge with Prod
2021-03-02 15:47:34 +00:00
Pete Fisher 2199efc95d added ManualOrderIndicator to ICENativeOrder.cpp after merge with Prod 2021-02-26 15:43:41 -05:00
Pete Fisher 54ff55f22f merge with prod 2021-02-26 14:59:38 -05:00
Pedro Pinto e7806d2fec Merge branch 'master' into prod 2020-09-16 16:28:36 -04:00
Taha Afzal c30bab899f Changing TASCounter to use the new formula 2020-07-29 09:43:00 -04:00
Taha Afzal 46544f36f1 Pyramid Algo now sends hedges for fills while a cancel is in flight 2020-07-23 13:13:52 -04:00
John Colvin c7df3ac43c Merge remote-tracking branch 'origin/csv_export' into prod 2020-07-22 10:44:58 -05:00
Jake Norbut 9d1b03d1e2 fixing merge 2020-03-04 11:51:05 -06:00
Jake Norbut 9a677cf980 merging master into csv_export; taking master 2020-03-04 11:50:56 -06:00
Jake Norbut 25445d58d0 refactoring TaS CSV export 2019-12-18 13:48:14 -06:00
Jake Norbut 0b313a8aa8 refactoring Fills CSV export 2019-12-18 13:47:36 -06:00
Jake Norbut 54f954fd4b adding CSV export to order book 2019-12-18 13:47:03 -06:00
Jake Norbut 980f38ba03 using OrderedDictionary to CSVs are output with a predictable column order 2019-12-18 13:46:22 -06:00
Jake Norbut 9eada2724b Having the Command be on the Window rather than the VM; nice thinking John 2019-12-18 10:11:10 -06:00
Jake Norbut b5f53ee918 adding csv_properties to TimeAndSalesVM.Row 2019-12-18 10:10:04 -06:00
Jake Norbut efc175f654 adding export to csv command to Fills window 2019-12-17 10:32:58 -06:00
Jake Norbut 6074afd999 adding csv export command to fills vm 2019-12-17 10:32:15 -06:00
Jake Norbut 219dff4055 adding new export_csv IDialogCommand which just creates a new file 2019-12-17 10:28:57 -06:00
Jake Norbut 9a3b07b622 adding ToCsv IEnumerable ext method in a new util namespace 2019-12-17 10:21:38 -06:00
24 changed files with 439 additions and 245 deletions

No files matched your search

-1
View File
@@ -37,4 +37,3 @@ trace_data/
*.rej
/build
.clangd/
*CMakeDeps.cmake
+1 -1
View File
@@ -1 +1 @@
set(sources /home/ppinto/Dropbox/bts/spark/doc/user/build.sh;content/_index.md;content/bts_spark/Glossary/_index.md;content/bts_spark/Help/_index.md;content/bts_spark/Help/about_spark.png;content/bts_spark/Help/image39.png;content/bts_spark/Help/image58.png;content/bts_spark/Launcher/_index.md;content/bts_spark/Launcher/image38.png;content/bts_spark/Launcher/image68.png;content/bts_spark/Logging-In-to-the-System/_index.md;content/bts_spark/Logging-In-to-the-System/image3.png;content/bts_spark/Logging-In-to-the-System/image64.png;content/bts_spark/Market/_index.md;content/bts_spark/Market/change_securities.png;content/bts_spark/Market/image11.png;content/bts_spark/Market/image14.png;content/bts_spark/Market/image20.png;content/bts_spark/Market/image21.png;content/bts_spark/Market/image23.png;content/bts_spark/Market/image24.png;content/bts_spark/Market/image34.png;content/bts_spark/Market/image49.png;content/bts_spark/Market/image53.png;content/bts_spark/Market/image66.png;content/bts_spark/Market/image69.png;content/bts_spark/Market/market_view.png;content/bts_spark/Market/settings_icon.png;content/bts_spark/Market/time_and_sales.png;content/bts_spark/My-Windows/_index.md;content/bts_spark/My-Windows/image15.png;content/bts_spark/My-Windows/image16.png;content/bts_spark/My-Windows/image37.png;content/bts_spark/Settings/_index.md;content/bts_spark/Settings/image51.png;content/bts_spark/Settings/image9.png;content/bts_spark/Spreader/_index.md;content/bts_spark/Spreader/image10.png;content/bts_spark/Spreader/image13.png;content/bts_spark/Spreader/image18.png;content/bts_spark/Spreader/image19.png;content/bts_spark/Spreader/image28.png;content/bts_spark/Spreader/image29.png;content/bts_spark/Spreader/image31.png;content/bts_spark/Spreader/image35.png;content/bts_spark/Spreader/image40.png;content/bts_spark/Spreader/image43.png;content/bts_spark/Spreader/image54.png;content/bts_spark/Spreader/image55.png;content/bts_spark/Spreader/image56.png;content/bts_spark/Spreader/image6.png;content/bts_spark/Spreader/image7.png;content/bts_spark/Spreader/image70.png;content/bts_spark/Spreader/image72.png;content/bts_spark/Spreader/image8.png;content/bts_spark/Trading/OrderTypes/index.md;content/bts_spark/Trading/_index.md;content/bts_spark/Trading/algo_studio.png;content/bts_spark/Trading/collapse_all_icon.png;content/bts_spark/Trading/cover_algo_control.png;content/bts_spark/Trading/expand_all_icon.png;content/bts_spark/Trading/image1.png;content/bts_spark/Trading/image17.png;content/bts_spark/Trading/image2.png;content/bts_spark/Trading/image25.png;content/bts_spark/Trading/image27.png;content/bts_spark/Trading/image33.png;content/bts_spark/Trading/image42.png;content/bts_spark/Trading/image44.png;content/bts_spark/Trading/image47.png;content/bts_spark/Trading/image48.png;content/bts_spark/Trading/image5.png;content/bts_spark/Trading/image50.png;content/bts_spark/Trading/image52.png;content/bts_spark/Trading/image57.png;content/bts_spark/Trading/image59.png;content/bts_spark/Trading/image60.png;content/bts_spark/Trading/image61.png;content/bts_spark/Trading/image62.png;content/bts_spark/Trading/image73.png;content/bts_spark/Trading/image74.png;content/bts_spark/Trading/join_algo_control.png;content/bts_spark/Trading/ladder.png;content/bts_spark/Trading/ladder_badges.png;content/bts_spark/Trading/ladder_main_part.png;content/bts_spark/Trading/lean_algo_control.png;content/bts_spark/Trading/mini_orderbook.png;content/bts_spark/Trading/oco_algo_control.png;content/bts_spark/Trading/offset_algo_control.png;content/bts_spark/Trading/pnl_calculation.png;content/bts_spark/Trading/pnl_window.png;content/bts_spark/Trading/recreate_icon.png;content/bts_spark/Trading/settings_icon.png;content/bts_spark/Trading/spark_menu_order_book.png;content/bts_spark/Trading/top_algo_control.png;content/bts_spark/Trading/trigger_algo_control.png;content/bts_spark/_index.md;content/release_notes/_index.md;content/system_requirements/_index.md;static/book.js;static/favicon.ico;static/image.js;templates/index.html;templates/page.html;templates/section.html;templates/shortcodes/back_button.html)
set(sources /home/pfisher/bts/DJ_ICE/doc/user/build.sh;content/_index.md;content/bts_spark/Glossary/_index.md;content/bts_spark/Help/_index.md;content/bts_spark/Help/about_spark.png;content/bts_spark/Help/image39.png;content/bts_spark/Help/image58.png;content/bts_spark/Launcher/_index.md;content/bts_spark/Launcher/image38.png;content/bts_spark/Launcher/image68.png;content/bts_spark/Logging-In-to-the-System/_index.md;content/bts_spark/Logging-In-to-the-System/image3.png;content/bts_spark/Logging-In-to-the-System/image64.png;content/bts_spark/Market/_index.md;content/bts_spark/Market/change_securities.png;content/bts_spark/Market/image11.png;content/bts_spark/Market/image14.png;content/bts_spark/Market/image20.png;content/bts_spark/Market/image21.png;content/bts_spark/Market/image23.png;content/bts_spark/Market/image24.png;content/bts_spark/Market/image34.png;content/bts_spark/Market/image49.png;content/bts_spark/Market/image53.png;content/bts_spark/Market/image66.png;content/bts_spark/Market/image69.png;content/bts_spark/Market/market_view.png;content/bts_spark/Market/settings_icon.png;content/bts_spark/Market/time_and_sales.png;content/bts_spark/My-Windows/_index.md;content/bts_spark/My-Windows/image15.png;content/bts_spark/My-Windows/image16.png;content/bts_spark/My-Windows/image37.png;content/bts_spark/Settings/_index.md;content/bts_spark/Settings/image51.png;content/bts_spark/Settings/image9.png;content/bts_spark/Spreader/_index.md;content/bts_spark/Spreader/image10.png;content/bts_spark/Spreader/image13.png;content/bts_spark/Spreader/image18.png;content/bts_spark/Spreader/image19.png;content/bts_spark/Spreader/image28.png;content/bts_spark/Spreader/image29.png;content/bts_spark/Spreader/image31.png;content/bts_spark/Spreader/image35.png;content/bts_spark/Spreader/image40.png;content/bts_spark/Spreader/image43.png;content/bts_spark/Spreader/image54.png;content/bts_spark/Spreader/image55.png;content/bts_spark/Spreader/image56.png;content/bts_spark/Spreader/image6.png;content/bts_spark/Spreader/image7.png;content/bts_spark/Spreader/image70.png;content/bts_spark/Spreader/image72.png;content/bts_spark/Spreader/image8.png;content/bts_spark/Trading/OrderTypes/index.md;content/bts_spark/Trading/_index.md;content/bts_spark/Trading/algo_studio.png;content/bts_spark/Trading/collapse_all_icon.png;content/bts_spark/Trading/cover_algo_control.png;content/bts_spark/Trading/expand_all_icon.png;content/bts_spark/Trading/image1.png;content/bts_spark/Trading/image17.png;content/bts_spark/Trading/image2.png;content/bts_spark/Trading/image25.png;content/bts_spark/Trading/image27.png;content/bts_spark/Trading/image33.png;content/bts_spark/Trading/image42.png;content/bts_spark/Trading/image44.png;content/bts_spark/Trading/image47.png;content/bts_spark/Trading/image48.png;content/bts_spark/Trading/image5.png;content/bts_spark/Trading/image50.png;content/bts_spark/Trading/image52.png;content/bts_spark/Trading/image57.png;content/bts_spark/Trading/image59.png;content/bts_spark/Trading/image60.png;content/bts_spark/Trading/image61.png;content/bts_spark/Trading/image62.png;content/bts_spark/Trading/image73.png;content/bts_spark/Trading/image74.png;content/bts_spark/Trading/join_algo_control.png;content/bts_spark/Trading/ladder.png;content/bts_spark/Trading/ladder_badges.png;content/bts_spark/Trading/ladder_main_part.png;content/bts_spark/Trading/lean_algo_control.png;content/bts_spark/Trading/mini_orderbook.png;content/bts_spark/Trading/oco_algo_control.png;content/bts_spark/Trading/offset_algo_control.png;content/bts_spark/Trading/pnl_calculation.png;content/bts_spark/Trading/pnl_window.png;content/bts_spark/Trading/recreate_icon.png;content/bts_spark/Trading/settings_icon.png;content/bts_spark/Trading/spark_menu_order_book.png;content/bts_spark/Trading/top_algo_control.png;content/bts_spark/Trading/trigger_algo_control.png;content/bts_spark/_index.md;content/release_notes/_index.md;content/system_requirements/_index.md;static/book.js;static/favicon.ico;static/image.js;templates/index.html;templates/page.html;templates/section.html;templates/shortcodes/back_button.html)
@@ -1,6 +1,7 @@
using bts.api;
using bts.co2;
using bts.utils;
using bts.utils.csv;
using mbts;
using mbts.spark;
using mbts.ui.controls;
@@ -49,6 +50,18 @@ namespace bts.shell.time_and_sales {
public void raise_invalidated() =>
invalidated?.Invoke();
public static System.Collections.Specialized.OrderedDictionary csv_ordered_properties =>
CsvHelper.create_ordering_maintainer(
typeof(TimeAndSalesRowVM),
new string[] {
nameof(security),
nameof(time),
nameof(price),
nameof(size),
nameof(aggressor_side),
nameof(security_key),
});
public bool Equals(TimeAndSalesRowVM other) =>
other != null &&
row_key == other.row_key;
@@ -2,6 +2,7 @@
using bts.shell.fills;
using bts.spark;
using bts.view;
using bts.utils.csv;
using mbts;
using mbts.co2;
using mbts.persist;
@@ -124,6 +125,8 @@ namespace bts.shell.time_and_sales {
show_in_context_menu = _can_pause_updates
};
_commands = new SparkCommand[] {
// csv export
new SparkCommandCommand("Export to csv", Icons.Export, FileDialogCommand.export_csv(() => _vm.rows.ToCsv(TimeAndSalesRowVM.csv_ordered_properties), "TaS")),
// ladder
SparkCommands.ShowLadderAutoLookup(SparkCommands.KeyGestures.ShowLadder)
.complete(current_sec_key_src, sec_keys => sec_keys.Count > 0),
@@ -5,6 +5,7 @@ using bts.collections;
using bts.shell.fills;
using bts.spark;
using bts.utils;
using bts.utils.csv;
using mbts.collections;
using mbts.persist;
using mbts.spark.filters;
@@ -333,6 +334,27 @@ namespace mbts.spark {
public override bool Equals(object obj) => Equals(obj as Row);
public override int GetHashCode() => _fill.FillKey.GetHashCode() ^ _order.Key.GetHashCode();
public static System.Collections.Specialized.OrderedDictionary csv_ordered_properties =>
CsvHelper.create_ordering_maintainer(typeof(Row),
new string[] {
nameof(security),
nameof(order_type),
nameof(root_algo_display_name),
nameof(side),
nameof(fill_time),
nameof(user),
nameof(account),
nameof(exchange),
nameof(time_in_force),
nameof(size),
nameof(price),
nameof(fill_kind),
nameof(fill_status),
nameof(fill_key),
nameof(order_key),
nameof(security_key)
});
public event Action invalidated;
void IRowVM.raise_invalidated() => invalidated?.Invoke();
}
@@ -4,6 +4,7 @@ using bts.shell;
using bts.shell.fills;
using bts.spark;
using bts.view;
using bts.utils.csv;
using mbts.persist;
using mbts.ui;
using mbts.ui.settings;
@@ -61,6 +62,7 @@ namespace mbts.spark {
_view = new FillsView { DataContext = _vm };
_commands = new List<SparkCommand> {
new SparkCommandCommand("Export to csv", Icons.Export, FileDialogCommand.export_csv(() => _vm.rows.ToCsv(FillsVM.Row.csv_ordered_properties), "Fills")),
// wrap the grid settings view factory in a dialogCommand here since the vm is destroyed/recreated when the security criteria changes
SparkCommands.window_settings_command(new DialogCommand(() => {
var vm = new ShellSettingsDialogVM<GridSettings, SimpleGridSettingsVM>(
@@ -4,6 +4,7 @@ using bts.api.orderman;
using bts.collections;
using bts.spark;
using bts.utils;
using bts.utils.csv;
using mbts.ui;
using mbts.ui.controls;
using System;
@@ -11,6 +12,7 @@ using System.Collections.Generic;
using System.ComponentModel;
using System.Reactive.Linq;
using System.Runtime.CompilerServices;
using System.Linq;
using System.Windows.Input;
using System.Windows.Media;
using Columns = mbts.spark.OrderBookGridView.ColumnKeys;
@@ -60,6 +62,35 @@ namespace mbts.spark {
public event Action<(OrderSecurityRootPair oldval, OrderSecurityRootPair newval)> updated;
public event PropertyChangedEventHandler PropertyChanged;
public static System.Collections.Specialized.OrderedDictionary csv_ordered_properties =>
CsvHelper.create_ordering_maintainer(
typeof(OrderBookRowVM),
new string[] {
nameof(security_text),
nameof(symbol),
nameof(time_created),
nameof(last_change),
nameof(user),
nameof(account),
nameof(exchange),
nameof(time_in_force),
nameof(order_type),
nameof(root_algo_display_name),
nameof(order_sub_type),
nameof(side),
nameof(quantity),
nameof(filled_quantity),
nameof(price),
nameof(avg_price),
nameof(last_price),
nameof(piq),
nameof(status),
// last message (could contains commas, so skipping)
nameof(security_key),
nameof(order_key),
nameof(parent_order_key),
});
// Core
public OrderSecurityRootPair order_tuple { get; private set; }
public OrderRecordImmutable order => order_tuple.order.order;
@@ -6,6 +6,7 @@ using bts.shell.fills;
using bts.shell.orderbook;
using bts.spark;
using bts.view;
using bts.utils.csv;
using mbts.persist;
using mbts.spark.utilities;
using mbts.ui;
@@ -231,6 +232,7 @@ namespace mbts.spark {
toggle_collapse_all_command,
toggle_expand_all_command,
show_grid_settings_command,
new SparkCommandCommand("Export to csv", Icons.Export, FileDialogCommand.export_csv(() => _vm.rows.ToCsv(OrderBookRowVM.csv_ordered_properties), "OrderBook")),
// hidden cmds (i.e., keyboard shortcuts at window level)
hidden_cmd(new SparkCommandCommand(OrderBookView.CommandDescriptions.ModifyOrders, null, _vm.modify_selected_orders_cmd, OrderBookView.KeyboardShortcuts.ModifyOrders)),
hidden_cmd(new SparkCommandCommand(OrderBookView.CommandDescriptions.RevertOrders, null, _vm.revert_selected_orders_cmd, OrderBookView.KeyboardShortcuts.RevertOrders)),
@@ -64,6 +64,25 @@ namespace mbts.ui {
}
}, can_execute == null ? null : new Func<object, bool>(_ => can_execute()));
public static IDialogCommand export_csv(
Func<string> get_file_content,
string source_window_name) =>
new AnonymousFileDialogCommand<object>(_ =>
_create_save_file_dialog(
Environment.GetFolderPath(Environment.SpecialFolder.Desktop),
$"BTS_Spark_{source_window_name}_{DateTime.Now.ToString("yyyyMMdd_HHmm")}.csv",
new[] { ("CSV (comma delimited)", new string[] { "csv" }) }
),
(_, file) => {
try {
File.WriteAllText(file, get_file_content.Invoke());
} catch (Exception ex) {
BTSMessageBox.Show($"Error exporting data to csv: {ex}.", "CSV Export Error", BTSMessageBoxButtons.OK, System.Windows.MessageBoxImage.Error, true, false, true);
}
},
// we are always able to perform an export, even when no rows (it will just be a header)
(_) => true);
static TDialog _create_file_dialog<TDialog>(string initial_dir = null, IReadOnlyList<(string name, string[] extensions)> filter_items = null) where TDialog : FileDialog, new() {
var rv = new TDialog();
if (!string.IsNullOrEmpty(initial_dir))
+52
View File
@@ -0,0 +1,52 @@
using System;
using System.IO;
using System.Collections.Generic;
using System.Collections.Specialized;
using System.Linq;
using System.Reflection;
namespace bts.utils.csv {
public static class CsvHelper {
public static string ToCsv<T>(this IEnumerable<T> items, OrderedDictionary properties = null) where T : class {
var output = "";
var delimiter = ",";
if (properties == null) {
properties = new OrderedDictionary();
foreach (var pi in typeof(T).GetProperties()) {
properties.Add(pi.Name, pi);
}
}
using (var sw = new StringWriter()) {
var dict_values = properties.Values.OfType<PropertyInfo>();
var header = dict_values.Select(n => n.Name).Aggregate((first, second) => first + delimiter + second);
sw.WriteLine(header);
foreach (var i in items) {
var row = dict_values.Select(x => x.GetValue(i, null))
.Select(x => x == null ? "null" : x.ToString())
.Aggregate((first, second) => first + delimiter + second);
sw.WriteLine(row);
}
output = sw.ToString();
}
return output;
}
public static OrderedDictionary create_ordering_maintainer(Type t, string[] property_names_in_order) {
OrderedDictionary od = new OrderedDictionary();
var props = t.GetProperties();
foreach (var n in property_names_in_order) {
var query = props.Where(pi => pi.Name == n);
if (query.Any()) {
var property_info = query.First();
if (!od.Contains(property_info)) {
od.Add(n, property_info);
}
}
}
return od;
}
}
}
+1 -5
View File
@@ -1,5 +1 @@
<<<<<<< HEAD
set(sources files/cert_secure_logon.txt;files/futures.coconut.xz;files/onixs_ice/ConnectivityConfiguration.APITest.xml;files/onixs_ice/ConnectivityConfiguration.APITest.xml.orig;files/onixs_ice/ConnectivityConfiguration.PerfTest.xml;files/onixs_ice/ConnectivityConfiguration.Production.xml;files/perf_trace/compiled.css;files/perf_trace/index.html;files/perf_trace/jquery-3.3.1.min.js;files/perf_trace/main.js;files/perf_trace/perf_log.sql;files/perf_trace/perf_trace_analysis.sql;files/perf_trace/plotly-latest.min.js)
=======
set(sources files/audit_ilink3.py;files/audit_upload.sh;files/audit_uploader.py;files/cert_secure_logon.txt;files/example.conf;files/futures.coconut.xz;files/onixs_ice/ConnectivityConfiguration.APITest.xml;files/onixs_ice/ConnectivityConfiguration.APITest.xml.orig;files/onixs_ice/ConnectivityConfiguration.PerfTest.xml;files/onixs_ice/ConnectivityConfiguration.Production.xml;files/perf_trace/compiled.css;files/perf_trace/index.html;files/perf_trace/jquery-3.3.1.min.js;files/perf_trace/main.js;files/perf_trace/perf_log.sql;files/perf_trace/perf_trace_analysis.sql;files/perf_trace/plotly-latest.min.js)
>>>>>>> master
set(sources files/audit_ilink3.py;files/audit_upload.sh;files/audit_uploader.py;files/cert_secure_logon.txt;files/example.conf;files/futures.coconut.xz;files/onixs_ice/ConnectivityConfiguration.APITest.xml;files/onixs_ice/ConnectivityConfiguration.PerfTest.xml;files/onixs_ice/ConnectivityConfiguration.Production.xml;files/perf_trace/compiled.css;files/perf_trace/index.html;files/perf_trace/jquery-3.3.1.min.js;files/perf_trace/main.js;files/perf_trace/perf_log.sql;files/perf_trace/perf_trace_analysis.sql;files/perf_trace/plotly-latest.min.js)
+206
View File
@@ -0,0 +1,206 @@
#!/usr/bin/python
import os
import re
import fileinput
import sys
import collections
import csv
import time, datetime
import copy
import itertools
import argparse
# region fix record processing
valid_message_types = {
'D': 'NEW ORDER',
'G': 'MODIFY',
'F': 'CANCEL',
'8': 'EXECUTION',
'i': 'MASS QUOTE',
'R': 'QUOTE REQUEST',
'b': 'QUOTE ACK',
'Z': 'MASS QUOTE CANCEL',
# 'c': 'SECURITY DEFINITION REQUEST',
'3': 'SESSION LEVEL REJECT',
'j': 'BUSINESS LEVEL REJECT',
'9': 'CANCEL REJECT',
'CA': 'ORDER MASS ACTION REQUEST',
'BZ': 'ORDER MASS ACTION REPORT'
}
def print_headers():
# From https://www.theice.com/publicdocs/futures_us/exchange_notices/ICE_Advisory_0113_009.pdf
line_to_write = "Active Date,Cl Ord ID,OrderID,Transact Time,Symbol,Symbol,SecurityType,Stk Prc,P/C,Side,OrderQty,Max Show,OrdType,\
Price,StopPx,Clearing Account,Clearing Firm,Order State,AccountCode,CTI Code,Originator User ID,OnBehalfOfCompID,\
OnBehalfOfCompID,On Behalf Of Sub ID,On Behalf Of Sub ID,On Behalf Of Location ID,Time In Force"
with open (args.output_file, 'w') as the_file:
the_file.write(line_to_write)
the_file.write("\n")
line_to_write = "Activity Date,Client Order ID,Original Order ID,Order Entry/Modification Time,Commodity,Contract Month,Future/Option Indicator,\
Strike Price,,Put or Call,Buy/Sell Indicator,Quantity,Reserve Quantity,Order Type,Price,Stop Price,Account #,CM#,\
Order Status,Clearing Account Code,CTI,ETS Login ID,Routing Member ID,Authorized Trader ID,Authorized Trdr ID,\
Routing Trdr,Routing Grp,Time In Force"
with open (args.output_file, 'a') as the_file:
the_file.write(line_to_write)
the_file.write("\n")
def getter(field_name):
def result(parsed_fix_record):
if field_name in parsed_fix_record:
return parsed_fix_record[field_name]
else:
return None
return result
def parse_fix_line(line):
ts = line[:21]
fix_line = line[24:].split('\1')
result = {}
result['timestamp'] = ts
for fix_field in fix_line:
rx = r"(?P<tag>\d+)?=(?P<value>.*)"
match = re.match(rx,fix_field)
if match:
result[match.group('tag')] = match.group('value')
return result
def tagged_field(line,number):
if number in line:
return line[number]
else:
return ""
def formatted_tagged_field(line,number):
if number in line:
return line[number].lstrip("0")
else:
return ""
def lr_tagged_field(line,number):
if number in line:
return line[number].rsplit('|',1)
else:
return "",""
def lr_contract_month(line,number):
try:
ice_line="ICE_SECURITY_ID_TO_SYMBOL_MAPPING: " + line[number]
with open('/sparkdata/data/md_dump.txt') as f:
datafile = f.readlines()
for data in datafile:
if data.find(str(ice_line)) != -1:
placeholder=data.find('> ')+1
get_ending=data[placeholder:]
get_ending=get_ending.strip()
return get_ending.split(" ")
except KeyError:
print("Can not find key 55")
return "",""
def valid_file(f):
if not os.path.exists(f):
# Argparse uses the ArgumentTypeError to give a rejection message like:
# error: argument input: x does not exist
raise argparse.ArgumentTypeError("{0} does not exist".format(f))
return f
def valid_date(s):
try:
return datetime.datetime.strptime(s, "%Y%m%d")
except ValueError:
raise argparse.ArgumentTypeError("Not a valid date: '{0}'.".format(s))
parser = argparse.ArgumentParser()
parser.add_argument("quickfix_log", help="the quickfix log[.gz] you wish to process", type=valid_file)
parser.add_argument("output_file", help="the destination csv file you wish to write")
parser.add_argument("-d", "--date", help="the date you wish to parse from the log, specified as YYYYmmdd",
default=datetime.datetime.today().strftime("%Y%m%d"), type=valid_date)
#quickfix_log="/home/pfisher/temp/audit/fix"
#output_file="/home/pfisher/temp/audit/fix.csv"
args = parser.parse_args()
# Note that the timedeltas specificed in trade_day_start and trade_day_end
# are based on Chicago times; as such, this script should run on a machine
# set to a Chicago timezone. A future impromovement would be to determine
# the current machine's timezone and determine the appropriate offset
# versus Chicago time, but for now we assume this will run on a Chicago
# tz machine.
input_as_datetime = args.date
trade_day_start = input_as_datetime - datetime.timedelta(days=1) + datetime.timedelta(hours=17, minutes=30)
trade_day_utc_start = time.gmtime((time.mktime(trade_day_start.timetuple())))
trade_day_end = input_as_datetime + datetime.timedelta(hours=17)
trade_day_utc_end = time.gmtime((time.mktime(trade_day_end.timetuple())))
#file_opener = open
file_opener = None
if args.quickfix_log.endswith(".gz"):
import gzip
file_opener = gzip.open
else:
file_opener = open
audit_entries = []
line_number = 0
print_headers()
with file_opener(args.quickfix_log) as input_file:
#with file_opener(quickfix_log) as input_file:
for line in input_file:
try:
line_timestamp = time.strptime(line[:21], "%Y%m%d-%H:%M:%S.%f")
except:
continue
if trade_day_utc_start <= line_timestamp <= trade_day_utc_end:
parsed_fix_record = parse_fix_line(line)
if parsed_fix_record['35'] in valid_message_types:
line_number = line_number + 1
# Static fields
tt = datetime.datetime.strptime(line[:21], "%Y%m%d-%H:%M:%S.%f")
transacttime = tt.strftime('%Y%m%d-%H:%M:%S.%f')
activedate = time.strftime('%Y%m%d',time.strptime(line[:8], "%Y%m%d"))
securitytype = "FUT"
stkprc = ""
putcall = ""
#symbol = "HNG"
#contractmonth = "U20"
# No formatting fields
ciorderid = tagged_field(parsed_fix_record, '11')
orderid = tagged_field(parsed_fix_record,'37')
side = tagged_field(parsed_fix_record,'54')
orderquanity = tagged_field(parsed_fix_record,'38')
maxshow = tagged_field(parsed_fix_record,'201')
ordertype = tagged_field(parsed_fix_record,'40')
stoppx = tagged_field(parsed_fix_record,'99')
clearingaccount = tagged_field(parsed_fix_record,'440')
clearingfirm = tagged_field(parsed_fix_record,'439')
orderstate = tagged_field(parsed_fix_record,'9175')
accountcode = tagged_field(parsed_fix_record,'9195')
cticode = tagged_field(parsed_fix_record,'9208')
originatoruserid = tagged_field(parsed_fix_record,'9139')
timeinforce = tagged_field(parsed_fix_record,'59')
# Needs Formatting
price = formatted_tagged_field(parsed_fix_record,'44')
# Left Right
OnBehalfOfCompID_Left,OnBehalfOfCompID_Right = lr_tagged_field(parsed_fix_record,'115')
OnBehalfOfSubID_Left,OnBehalfOfSubID_Right = lr_tagged_field(parsed_fix_record,'116')
OnBehalfOfLocationID_Left,OnBehalfOfLocationID_Right = lr_tagged_field(parsed_fix_record,'144')
if parsed_fix_record['35'] == "9":
symbol = ""
contractmonth = ""
else:
symbol,contractmonth = lr_contract_month(parsed_fix_record,'55')
line_to_write = activedate + "," + ciorderid + "," + orderid + "," + transacttime + "," + symbol + "," + contractmonth + "," + securitytype + "," + \
stkprc + "," + putcall + "," + side + "," + orderquanity + "," + maxshow + "," + ordertype + "," + price + "," + stoppx + "," + \
clearingaccount + "," + clearingfirm + "," + orderstate + "," + accountcode + "," + cticode + "," + originatoruserid + "," + OnBehalfOfCompID_Left + "," + \
OnBehalfOfCompID_Right + "," + OnBehalfOfSubID_Left + "," + OnBehalfOfSubID_Right + "," + OnBehalfOfLocationID_Left + "," + timeinforce
with open (args.output_file, 'a') as the_file:
the_file.write(line_to_write)
the_file.write("\n")
# except Exception:
# pass
+18 -7
View File
@@ -40,23 +40,34 @@ SESSION_ID=${conf_vars[session_id]}
FCM=${conf_vars[fcm]}
CLIENT=${conf_vars[client]}
# Create MD dump
cd /sparkbin
./sparkd md dumpsecurity all > /sparkdata/data/md_dump.txt
cd /sparkbin/data_files
for SESSION in $(ls /sparkdata/data/ilink3/); do
for SESSION in $(ls /sparkdata/data/{ilink3,jetfix}/); do
if [[ $SESSION == *$SESSION_ID* ]]; then
echo "Session is $SESSION"
SESSION_DAY=$(date +%Y-%m-%d)
SESSION_DAY=$(date -d $DATE +%Y-%m-%d)
if [[ $SESSION == "cme_"* ]]; then
SESSION_DAY=$(date -d "$DATE -$(date -d $DATE +%u) days" +"%Y-%m-%d")
AUDIT_SCRIPT="audit_ilink3.py"
ILINK_TYPE="ilink3"
LOG_NAME="log"
elif [[ $SESSION == "ice_"* ]]; then
SESSION_DAY=$(date -d "$DATE 2 days ago" +%Y-%m-%d)
AUDIT_SCRIPT="audit_ice.py"
ILINK_TYPE="jetfix"
LOG_NAME="fix"
fi
echo $SESSION_DAY
if [[ ! -d /sparkdata/data/ilink3/$SESSION/audit_logs ]]; then
mkdir -p /sparkdata/data/ilink3/$SESSION/audit_logs
if [[ ! -d /sparkdata/data/$ILINK_TYPE/$SESSION/audit_logs ]]; then
mkdir -p /sparkdata/data/$ILINK_TYPE/$SESSION/audit_logs
fi
if [[ -d /sparkdata/data/ilink3/$SESSION/$SESSION_DAY ]]; then
if [[ -d /sparkdata/data/$ILINK_TYPE/$SESSION/$SESSION_DAY ]]; then
echo "Starting the creation of the audit log"
python $AUDIT_SCRIPT -d $DATE /sparkdata/data/ilink3/$SESSION/$SESSION_DAY/log /sparkdata/data/ilink3/$SESSION/audit_logs/"$FCM"_audit_"$SESSION"_"$DATE".csv
python3 $AUDIT_SCRIPT -d $DATE /sparkdata/data/$ILINK_TYPE/$SESSION/$SESSION_DAY/$LOG_NAME /sparkdata/data/$ILINK_TYPE/$SESSION/audit_logs/"$FCM"_audit_"$SESSION"_"$DATE".csv
fi
if [[ ! -d /sparkdata/data/md_audit ]]; then
mkdir /sparkdata/data/md_audit
@@ -68,6 +79,6 @@ for SESSION in $(ls /sparkdata/data/ilink3/); do
cd /sparkbin/data_files
fi
echo "Uploading to the (s)FTP Server"
python audit_uploader.py --config_file="$CONFIG_FILE" --datestamp $DATE --session="$SESSION" --client="$CLIENT" --fcm="$FCM"
python3 audit_uploader.py --config_file="$CONFIG_FILE" --datestamp $DATE --session="$SESSION" --client="$CLIENT" --fcm="$FCM"
fi
done
+42 -28
View File
@@ -6,31 +6,31 @@ import paramiko, ftplib
#########################################################
def usage():
print "-------------------------------------------------------"
print "-Usage: -"
print "-------------------------------------------------------"
print "- --config_file= : Path to upload config file -"
print "- -if no path is specified look -"
print "- in the working directory. -"
print "- -"
print "- --help : Print this message -"
print "- -"
print "- --datestamp= : Optional, defaults to today -"
print "- -replaces *datestamp in the -"
print "- the upload config file -"
print "- -"
print "- --fcm : Optional, spark input -"
print "- -"
print "- --session : Optional, spark jf session -"
print "-------------------------------------------------------"
print ("-------------------------------------------------------")
print ("-Usage: -")
print ("-------------------------------------------------------")
print ("- --config_file= : Path to upload config file -")
print ("- -if no path is specified look -")
print ("- in the working directory. -")
print ("- -")
print ("- --help : Print this message -")
print ("- -")
print ("- --datestamp= : Optional, defaults to today -")
print ("- -replaces *datestamp in the -")
print ("- the upload config file -")
print ("- -")
print ("- --fcm : Optional, spark input -")
print ("- -")
print ("- --session : Optional, spark jf session -")
print ("-------------------------------------------------------")
fcm = ''
session = ''
try:
options, arguments = getopt.getopt(sys.argv[1:],"h",['config_file=', 'datestamp=', 'help', 'fcm=', 'client=', 'session='])
except getopt.GetoptError, err:
print str(err)
except getopt.GetoptError as err:
print(err)
usage()
sys.exit(2)
@@ -54,11 +54,11 @@ for argument in arguments:
usage()
sys.exit()
else:
print "Unrecognized commandline argument. Type help for usage."
print ("Unrecognized commandline argument. Type help for usage.")
sys.exit()
if len(options) <= 0:
print "Requires a config file to run."
print("Requires a config file to run.")
usage()
sys.exit()
@@ -98,14 +98,14 @@ def read_config_file(config_file, datestamp=datetime.datetime.today().strftime('
# FTP File Transfer Function #
#########################################################
def putfile_ftp(file, put_path, site, user, password, port=21, verbose=True):
if verbose: print 'Uploading', file
if verbose: print("Uploading", file)
local = open(file, 'rb')
remote = ftplib.FTP(site)
remote.login(user, password)
remote.storbinary(put_path, local, 1024)
remote.quit()
local.close()
if verbose: print 'Upload done.'
if verbose: print("Upload done.")
#########################################################
# SFTP File Transfer Function #
@@ -136,15 +136,29 @@ else:
print upload_config
print(upload_config)
if upload_config["PROTOCOL_PUT"] in ['FTP','ftp']:
print "Protocol = FTP"
putfile_ftp(upload_config["GET_ILINK3_FILE_PATH"] + upload_config["GET_ILINK3_FILE_NAME"], upload_config["PUT_ILINK3_FILE_PATH"] + upload_config["PUT_ILINK3_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], upload_config["PORT_PUT"] )
print("Protocol = FTP")
try:
putfile_ftp(upload_config["GET_ILINK3_FILE_PATH"] + upload_config["GET_ILINK3_FILE_NAME"], upload_config["PUT_ILINK3_FILE_PATH"] + upload_config["PUT_ILINK3_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], upload_config["PORT_PUT"] )
except KeyError:
pass
try:
putfile_ftp(upload_config["GET_JETFIX_FILE_PATH"] + upload_config["GET_JETFIX_FILE_NAME"], upload_config["PUT_JETFIX_FILE_PATH"] + upload_config["PUT_JETFIX_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], upload_config["PORT_PUT"] )
except KeyError:
pass
putfile_ftp(upload_config["GET_MD_FILE_PATH"] + upload_config["GET_MD_FILE_NAME"], upload_config["PUT_MD_FILE_PATH"] + upload_config["PUT_MD_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], upload_config["PORT_PUT"] )
elif upload_config["PROTOCOL_PUT"] in ['SFTP','sftp']:
print "Protocol = SFTP"
putfile_sftp(upload_config["GET_ILINK3_FILE_PATH"] + upload_config["GET_ILINK3_FILE_NAME"], upload_config["PUT_ILINK3_FILE_PATH"] + upload_config["PUT_ILINK3_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], int(upload_config["PORT_PUT"]) )
print("Protocol = SFTP")
try:
putfile_sftp(upload_config["GET_ILINK3_FILE_PATH"] + upload_config["GET_ILINK3_FILE_NAME"], upload_config["PUT_ILINK3_FILE_PATH"] + upload_config["PUT_ILINK3_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], int(upload_config["PORT_PUT"]) )
except KeyError:
pass
try:
putfile_sftp(upload_config["GET_JETFIX_FILE_PATH"] + upload_config["GET_JETFIX_FILE_NAME"], upload_config["PUT_JETFIX_FILE_PATH"] + upload_config["PUT_JETFIX_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], int(upload_config["PORT_PUT"]) )
except KeyError:
pass
putfile_sftp(upload_config["GET_MD_FILE_PATH"] + upload_config["GET_MD_FILE_NAME"], upload_config["PUT_MD_FILE_PATH"] + upload_config["PUT_MD_FILE_NAME"], upload_config["PUT_HOST"], upload_config["USERNAME_PUT"], upload_config["PASSWORD_PUT"], int(upload_config["PORT_PUT"]) )
+1 -1
View File
@@ -1 +1 @@
set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/service_helpers/PositionUtilities.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/BasePositionService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MBOService.proto;api/services/MarginService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/MarketDataService.proto;api/services/MoneyFileService.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/keycode.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/position_cfg.conf;config/defaults/span.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/AlgoTypes.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/keycode.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/position_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/span.proto;config/protos/spreader.proto;config/protos/subprocess.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/sod.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MBOProcessor.cpp;md/MBOServiceServer.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/Product.cpp;md/SecurityCache.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SecurityMapper.cpp;md/SecurityOracle.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/ice/ICEUtils.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/service/MarketDataServiceServer.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/mbo_tests.cpp;md/test/md2_tests.cpp;md/test/md_test_utils.cpp;md/test/synthetic_book_merge_tests.cpp;md/vendor/OnixsUtils.cpp;mercury/actors/IPollableActor.cpp;mercury/broker/Broker.cpp;mercury/broker/BrokerProxy.cpp;mercury/broker/ILocalBroker.cpp;mercury/broker/IRemoteBroker.cpp;mercury/common/mercury_common.cpp;mercury/events/TimerEvent.cpp;mercury/executor/Executor.cpp;mercury/topics/Topic.cLine truncated
set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/service_helpers/AlgoServiceUtils.cpp;api/service_helpers/PositionUtilities.cpp;api/services/AdminService.proto;api/services/AlgoDefinitionService.proto;api/services/AlgoService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/BasePositionService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MBOService.proto;api/services/MarginService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/MarketDataService.proto;api/services/MoneyFileService.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/keycode.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/position_cfg.conf;config/defaults/span.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/AlgoTypes.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/keycode.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/position_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/span.proto;config/protos/spreader.proto;config/protos/subprocess.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/sod.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MBOProcessor.cpp;md/MBOServiceServer.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/Product.cpp;md/SecurityCache.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SecurityMapper.cpp;md/SecurityOracle.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/ice/ICEUtils.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/service/MarketDataServiceServer.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/mbo_tests.cpp;md/test/md2_tests.cpp;md/test/md_test_utils.cpp;md/test/synthetic_book_merge_tests.cpp;md/vendor/OnixsUtils.cpp;mercury/actors/IPollableActor.cpp;mercury/broker/Broker.cpp;mercury/broker/BrokerProxy.cpp;mercury/broker/ILocalBroker.cpp;mercury/broker/IRemoteBroker.cpp;mercury/common/mercury_common.cpp;mercury/events/TimerEvent.cpp;mercury/executor/Executor.cpp;mercury/topics/Topic.cLine truncated
+1 -1
View File
@@ -319,7 +319,7 @@ if __name__ == "__main__":
if DEBUG:
for f in response.file:
print (f.content)
print f.content
else:
output = response.SerializeToString()
sys.stdout.write(output)
+3 -28
View File
@@ -60,13 +60,10 @@ class IntPrice {
return delta.mantissa() < 0;
}
}
bool operator <=(const IntPrice& other) const {
bool operator <=(const IntPrice& other) {
return *this == other || *this < other;
}
bool operator >(const IntPrice& other) const {
return other < *this;
}
bool operator >=(const IntPrice& other) const {
bool operator >=(const IntPrice& other) {
return !(*this < other);
}
IntPrice operator - (const IntPrice& o) const {
@@ -75,31 +72,9 @@ class IntPrice {
IntPrice operator + (const IntPrice& o) const {
return op(o, [](int64_t m1, int64_t m2) { return m1 + m2; });
}
IntPrice& operator +=(const IntPrice& o) {
auto temp = *this + o;
_mantissa = temp._mantissa;
_exponent = temp._exponent;
return *this;
}
IntPrice& operator -=(const IntPrice& o) {
auto temp = *this - o;
_mantissa = temp._mantissa;
_exponent = temp._exponent;
return *this;
}
IntPrice operator -() {
return IntPrice(-_mantissa, _exponent);
}
IntPrice operator * (const IntPrice& other) const {
return op(other, [](int64_t m1, int64_t m2) { return m1 * m2; });
}
IntPrice operator * (const int& other) const {
return IntPrice(_mantissa * other, _exponent);
}
IntPrice& operator *=(const int& other) {
_mantissa *= other;
return *this;
}
IntPrice operator / (double v) const {
return { (int64_t)(_mantissa / v), exponent() };
}
@@ -208,4 +183,4 @@ inline std::ostream& operator<<(std::ostream& os, const IntPrice& p) {
return os;
}
}}
}}
@@ -235,7 +235,7 @@ void GLBXiLink3ExecutionGateway::on_admin_data(
const mercury::Update<NewAdminDataEvent> &) {}
void GLBXiLink3ExecutionGateway::send_msgs(std::vector<OrderMsg> const &msgs,
trace::CorID corid) {
trace::CorID) {
if (me->state == State::READY) {
for (auto &m : msgs) {
@@ -248,11 +248,6 @@ void GLBXiLink3ExecutionGateway::send_msgs(std::vector<OrderMsg> const &msgs,
// CancelReplace
auto cancel_replace_ptr = std::get_if<CancelReplaceOrderMsg>(&m);
if (cancel_replace_ptr) {
LOG(iLink3,Info,"Cancel Replace. CorID:{}.{} Order:{}",
log::CharPtrWrapper{corid.channel},
corid.sn,
cancel_replace_ptr->exchange_order_id);
me->gw->send_cancel_replace(*cancel_replace_ptr);
continue;
}
+6 -155
View File
@@ -1,9 +1,9 @@
#include "BookUtils.h"
namespace bts::orderman {
using Size = md::Size;
using Side = api::Side;
using PriceSize = md::PriceSize;
namespace bts {
namespace orderman {
using Size = bts::md::Size;
using Side = bts::api::Side;
Size BookUtils::volume_at_or_better(const Book& book, Side side, Price price) {
return volume_at_or_better(book, side, price.value());
@@ -22,154 +22,5 @@ Size BookUtils::volume_at_or_better(const Book& book, Side side, double price) {
SidePriceComp BookUtils::price_comp(Side side) {
return SidePriceComp(side);
}
inline bool is_empty(const PriceSize& ps) {
return ps.size() == 0 || ps.int_price().is_nan();
}
void BookUtils::change_unit_qty(
const BookSide& book_in, std::vector<md::PriceSize>& book_out, int unit_qty) {
//
bts_assert(unit_qty > 0);
int mlev = 0, mlev_qty_used = 0;
std::vector<md::PriceSize> unit_lev_span;
for (uint32_t i = 0; i < book_in.size();) {
int mlev_qty_rem = book_in[i].size() - mlev_qty_used;
if (mlev_qty_rem == 0) {
mlev_qty_used = 0;
mlev++;
i++;
continue;
}
if (unit_lev_span.size() == 0) {
md::Size mlev_units_rem = mlev_qty_rem / unit_qty;
if (mlev_units_rem > 0) {
book_out.emplace_back(book_in[i].int_price(), mlev_units_rem);
mlev_qty_used += mlev_units_rem * unit_qty;
} else {
unit_lev_span.emplace_back(book_in[i].int_price(), mlev_qty_rem);
mlev_qty_used = 0;
mlev++;
i++;
}
} else {
// the unit spans across multiple [input]price levels: compute its
// [output]price. It will be in its own [output]price level.
auto unit_rem = unit_qty;
for (auto lev : unit_lev_span)
unit_rem -= lev.size();
if (mlev_qty_rem >= unit_rem) {
mlev_qty_used += unit_rem;
auto unit_price = md::INT_NAN;
for (uint32_t adj_lev = 0; adj_lev < unit_lev_span.size(); adj_lev++) {
auto leg_price = unit_lev_span[adj_lev].int_price() *
static_cast<int>(unit_lev_span[adj_lev].size());
if (unit_price.is_nan())
unit_price = leg_price;
else
unit_price += leg_price;
}
unit_lev_span.clear();
book_out.emplace_back(unit_price, 1);
}
}
}
}
void BookUtils::compute_synthetic_book(
std::vector<PriceSize>& dest,
const std::vector<std::tuple<Size, md::IntPrice, BookSide>>&
leg_mkts,
uint32_t depth) {
//
if (leg_mkts.size() == 0)
return;
struct Leg {
std::vector<PriceSize> book;
uint32_t curr_level;
Size curr_level_qty_used;
int leg_qty;
Leg() : book(), curr_level(0), curr_level_qty_used(0), leg_qty(0) {}
md::IntPrice curr_price() const {
// Assert that the book isn't 'empty' yet.
// Note: one could argue that nan be returned in this event,
// but since the loop is designed to return "greedily,"
// 'curr_price' should not be accessed after it has been emptied.
// This assertion enforces that the behavior is only changed
// as a result of deliberate action, rather than an inadvertant
// side-effect.
bts_assert(!is_empty());
return book[curr_level].int_price();
}
Size curr_level_qty_rem() const {
return book[curr_level].size() - curr_level_qty_used;
}
// The implied market's volume is limited by its thinnest market.
// When one leg 'is_empty,' the full volume of the implied market
// has been realized.
bool is_empty() const { return curr_level >= book.size(); }
};
std::vector<Leg> legs;
for (const auto& [qty, price_mult, book] : leg_mkts) {
// todo: implement price mult logic (for now just assert nan or 1)
bts_assert(price_mult.is_nan() || price_mult.price() == 1);
auto& l = legs.emplace_back();
bts_assert(qty != 0);
l.leg_qty = qty;
if (book.empty() || is_empty(book.front()))
return;
change_unit_qty(
book, l.book, std::abs(l.leg_qty));
if (l.book.empty())
return;
}
auto any_legs_empty = [&]() {
for (auto& l : legs)
if (l.is_empty())
return true;
return false;
};
while (!any_legs_empty() && depth-- > 0) {
// legs can't be empty, because it is a 1:1 mapping of leg_mkts,
// and it has already been established that the latter isn't empty.
auto limiting_factor = legs[0].curr_level_qty_rem();
for (uint32_t i = 0; i < legs.size(); i++) {
if (legs[i].curr_level_qty_rem() == 0) {
legs[i].curr_level++;
if (legs[i].is_empty()) {
// book is complete
return;
}
legs[i].curr_level_qty_used = Size(0);
}
limiting_factor =
std::min(legs[i].curr_level_qty_rem(), limiting_factor);
if (limiting_factor == 0)
return;
}
bts_assert(limiting_factor > 0);
for (uint32_t i = 0; i < legs.size(); i++)
legs[i].curr_level_qty_used += limiting_factor;
auto marginal_cost = md::INT_NAN;
for (auto& l : legs) {
// The book's prices have already been multiplied by abs(leg_qty);
// Subtract the leg's cost if qty is negative, otherwise add it
// to the synthetic's marginal cost.
auto lcost = l.curr_price();
if (l.leg_qty < 0)
lcost = -lcost;
if (marginal_cost.is_nan())
marginal_cost = lcost;
else
marginal_cost += lcost;
}
dest.emplace_back(marginal_cost, limiting_factor);
}
}
} // namespace bts::orderman
} // namespace orderman
} // namespace bts
+5 -10
View File
@@ -6,18 +6,13 @@
#include "lib/orderman/orderman_utils/Book.h"
#include "lib/orderman/orderman_utils/PriceComp.h"
namespace bts::orderman {
namespace bts {
namespace orderman {
struct BookUtils {
static md::Size volume_at_or_better(const Book& book, api::Side side, Price price);
static md::Size volume_at_or_better(const Book& book, api::Side side, double price);
static SidePriceComp price_comp(api::Side side);
static void change_unit_qty(
const BookSide& book_in, std::vector<md::PriceSize>& book_out, int unit_qty);
static void compute_synthetic_book(
std::vector<md::PriceSize>& dest,
const std::vector<std::tuple<md::Size, md::IntPrice, BookSide>>& leg_mkts,
uint32_t depth);
};
} // namespace bts::orderman
} // namespace orderman
} // namespace bts
@@ -1136,6 +1136,7 @@ void ICENativeOrder::init_fix()
create_encoding.add<fix::HandlInst>('1');
create_encoding.add<fix::Int_Symbol>(sec.ice().exchange_id());
create_encoding.add<fix::Side>(def.side());
create_encoding.add<fix::ManualOrderIndicator>(def.manual_order_indicator());
me->create_transact_time_bm = create_encoding.add<fix::TransactTimeStr>(datetime::FIX_timestamp(datetime::now()));
me->create_quantity_bm = create_encoding.add<fix::OrderQty>(0, quantity_sz());
create_encoding.add<fix::TimeInForce>(def.time_in_force());
@@ -1165,6 +1166,7 @@ void ICENativeOrder::init_fix()
update_encoding.add<fix::HandlInst>('1');
update_encoding.add<fix::Int_Symbol>(sec.ice().exchange_id());
update_encoding.add<fix::Side>(def.side());
update_encoding.add<fix::ManualOrderIndicator>(def.manual_order_indicator());
me->update_transact_time_bm = update_encoding.add<fix::TransactTimeStr>(datetime::FIX_timestamp(datetime::now()));
me->update_quantity_bm = update_encoding.add<fix::OrderQty>(0, quantity_sz());
me->update_clordid_bm = update_encoding.add<fix::Int_ClOrdID>(get_clordid(0), ice_orderid_sz());
@@ -1189,6 +1191,7 @@ void ICENativeOrder::init_fix()
cancel_encoding.add<fix::OriginatorUserID>(rr.authorized_trader());
cancel_encoding.add<fix::Int_Symbol>(sec.ice().exchange_id());
cancel_encoding.add<fix::Side>(def.side());
cancel_encoding.add<fix::ManualOrderIndicator>(def.manual_order_indicator());
me->cancel_clordid_bm = cancel_encoding.add<fix::Int_ClOrdID>(get_clordid(0), ice_orderid_sz());
// special tags
+5
View File
@@ -300,6 +300,11 @@ void dump_db(const vector<string>& args) {
md::get_system_md_symbols(symbols);
md::db::load_securities(symbols, [&](const api::Security& security) {
if (all || keys.find(security.id()) != keys.end()) {
if (security.has_ice()) {
std::cout << "#ICE_SECURITY_ID_TO_SYMBOL_MAPPING: "
<< security.ice().exchange_id() << " -> "
<< security.ice().security_desc() << "\n";
}
security.PrintDebugString();
if (md::securitytype_of(security) == api::SECURITYTYPE_SYNTHETIC) {
auto metadata = md::db::load_metadata();
+1 -1
View File
@@ -1 +1 @@
set(sources ../../externals/linux/include/google/protobuf/descriptor.proto;../../externals/linux/include/google/protobuf/timestamp.proto;../lib/api/apicore.proto;../lib/api/apigateway.proto;../lib/api/apioptions.proto;../lib/api/services/AdminService.proto;../lib/api/services/AlgoDefinitionService.proto;../lib/api/services/AlgoService.proto;../lib/api/services/ApiUnitTestService.proto;../lib/api/services/AuthService.proto;../lib/api/services/BasePositionService.proto;../lib/api/services/ErrorReportService.proto;../lib/api/services/HelloService.proto;../lib/api/services/MBOService.proto;../lib/api/services/MarginService.proto;../lib/api/services/MarketDataAuxiliaryService.proto;../lib/api/services/MarketDataRelay.proto;../lib/api/services/MarketDataService.proto;../lib/api/services/MoneyFileService.proto;../lib/api/services/OrderService.proto;../lib/api/services/OutOfProcessURLListener.proto;../lib/api/services/PersistenceService.proto;../lib/api/services/PositionService.proto;../lib/api/services/ShellDiagnosticService.proto;../lib/api/services/ShellLatencyService.proto;../lib/api/services/SyntheticSecurityService.proto;../lib/domain_model/Security.proto;../lib/md/protos/co2.proto;../lib/sparkproto/SparkProtos.proto;api/build.sh;api/quiet_build.sh;api/src/DynamicService.ts;api/src/ServiceContext.ts;api/src/bts_assert.ts;ui/src/Access.fs;ui/src/Accounts/ClipLimits.fs;ui/src/Accounts/ProductLimits.fs;ui/src/Accounts/SecurityLimits.fs;ui/src/Accounts/State.fs;ui/src/Accounts/Types.fs;ui/src/Accounts/View.fs;ui/src/App.fs;ui/src/Cmd.fs;ui/src/Controls/Button.fs;ui/src/Controls/DataGrid.fs;ui/src/Controls/Fields.fs;ui/src/Controls/Modal.fs;ui/src/Controls/ReactGrid.fs;ui/src/Controls/Table.fs;ui/src/Controls/nbsp.fs;ui/src/Global.fs;ui/src/Imports.fs;ui/src/Lens.fs;ui/src/Login/Login.fs;ui/src/Nav.fs;ui/src/State.fs;ui/src/Types.fs;ui/src/Users/State.fs;ui/src/Users/Types.fs;ui/src/Users/View.fs;ui/src/Util/EnumUtils.fs;ui/src/Util/JSMap.fs;ui/src/Util/StreamUtils.fs;ui/src/obj/Debug/netstandard2.0/ui.AssemblyInfo.fs)
set(sources ../../externals/linux/include/google/protobuf/descriptor.proto;../../externals/linux/include/google/protobuf/timestamp.proto;../lib/api/apicore.proto;../lib/api/apigateway.proto;../lib/api/apioptions.proto;../lib/api/services/AdminService.proto;../lib/api/services/AlgoDefinitionService.proto;../lib/api/services/AlgoService.proto;../lib/api/services/ApiUnitTestService.proto;../lib/api/services/AuthService.proto;../lib/api/services/BasePositionService.proto;../lib/api/services/ErrorReportService.proto;../lib/api/services/HelloService.proto;../lib/api/services/MBOService.proto;../lib/api/services/MarginService.proto;../lib/api/services/MarketDataAuxiliaryService.proto;../lib/api/services/MarketDataRelay.proto;../lib/api/services/MarketDataService.proto;../lib/api/services/MoneyFileService.proto;../lib/api/services/OrderService.proto;../lib/api/services/OutOfProcessURLListener.proto;../lib/api/services/PersistenceService.proto;../lib/api/services/PositionService.proto;../lib/api/services/ShellDiagnosticService.proto;../lib/api/services/ShellLatencyService.proto;../lib/api/services/SyntheticSecurityService.proto;../lib/domain_model/Security.proto;../lib/md/protos/co2.proto;../lib/sparkproto/SparkProtos.proto;api/build.sh;api/quiet_build.sh;api/src/DynamicService.ts;api/src/ServiceContext.ts;api/src/bts_assert.ts;ui/src/Access.fs;ui/src/Accounts/ClipLimits.fs;ui/src/Accounts/ProductLimits.fs;ui/src/Accounts/SecurityLimits.fs;ui/src/Accounts/State.fs;ui/src/Accounts/Types.fs;ui/src/Accounts/View.fs;ui/src/App.fs;ui/src/Cmd.fs;ui/src/Controls/Button.fs;ui/src/Controls/DataGrid.fs;ui/src/Controls/Fields.fs;ui/src/Controls/Modal.fs;ui/src/Controls/ReactGrid.fs;ui/src/Controls/Table.fs;ui/src/Controls/nbsp.fs;ui/src/Global.fs;ui/src/Imports.fs;ui/src/Lens.fs;ui/src/Login/Login.fs;ui/src/Nav.fs;ui/src/State.fs;ui/src/Types.fs;ui/src/Users/State.fs;ui/src/Users/Types.fs;ui/src/Users/View.fs;ui/src/Util/EnumUtils.fs;ui/src/Util/JSMap.fs;ui/src/Util/StreamUtils.fs)