324 lines
14 KiB
C#
324 lines
14 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using System.Linq;
|
|
using bts.spark;
|
|
using bts.api;
|
|
using bts.api.orderman;
|
|
using bts.co2;
|
|
using mbts.config;
|
|
using bts.config;
|
|
using System.Threading;
|
|
using System.Threading.Tasks;
|
|
using System.Collections.Concurrent;
|
|
using System.IO;
|
|
using mbts.co2;
|
|
using Google.Protobuf.WellKnownTypes;
|
|
|
|
namespace bts.history{
|
|
// secondary level helper functions (parsing, reading files etc)
|
|
class DataIOHelper{
|
|
// ../trade_analysis/date/
|
|
public static string get_data_path (DateTime date){
|
|
return String.Format(@"{0}{1}/{2}",
|
|
Configuration.get<bts.config.System>().data_path,
|
|
"trade_analysis",
|
|
date.ToShortDateString().Replace("/","_"));
|
|
}
|
|
public static string get_data_path (string relative_path){
|
|
return String.Format(@"{0}/{1}",
|
|
Configuration.get<bts.config.System>().data_path,
|
|
relative_path);
|
|
}
|
|
public static string get_rawbook_data_path (){
|
|
return String.Format(@"{0}{1}/{2}",
|
|
Configuration.get<bts.config.System>().data_path,
|
|
"trade_analysis",
|
|
"raw_books");
|
|
}
|
|
public static string get_book_path (string symbol,
|
|
DateTime date){
|
|
return String.Format(@"{0}{1}/{2}/{3}/{4}/{5}/",
|
|
Configuration.get<bts.config.System>().data_path,
|
|
"trade_analysis",
|
|
"market_data",
|
|
date.ToShortDateString().Replace("/","_"),
|
|
"books",
|
|
symbol);
|
|
}
|
|
|
|
// load books from the books csv
|
|
public static IDictionary<string,BookSeries> load_books(string symbol,
|
|
DateTime dt,
|
|
bool top_only){
|
|
DateTime t1 = DateTime.Now;
|
|
string file_name = String.Format("{0}/{1}", get_data_path(dt),
|
|
String.Format("{0}_{1}", symbol,"books.csv"));
|
|
return load_books(file_name,symbol,top_only);
|
|
}
|
|
|
|
public static IDictionary<string,BookSeries> load_books(string file_name,
|
|
string symbol,
|
|
bool top_only){
|
|
DateTime t1 = DateTime.Now;
|
|
var rv = new Dictionary<string,List<ProcessedBook>>();
|
|
int ln_no = 0;
|
|
using (StreamReader sr = new StreamReader(file_name)) {
|
|
string line;
|
|
while ((line = sr.ReadLine()) != null){
|
|
if(ln_no++>0){
|
|
var pbook = _csv_to_processedbook(line,true,top_only);
|
|
if(pbook != null){
|
|
MarketQueriesHelper.fill_exponent(symbol,pbook);
|
|
List<ProcessedBook> contract_books = null;
|
|
if(!rv.TryGetValue(pbook.Security,out contract_books)){
|
|
contract_books = new List<ProcessedBook>();
|
|
rv.Add(pbook.Security,contract_books);
|
|
}
|
|
if(MarketQueriesHelper.is_sparse_book(pbook.Book))
|
|
Log.assert(contract_books.Count==0,
|
|
String.Format("Illegal sparse book {0}",
|
|
pbook.BookSeqNum));
|
|
else
|
|
contract_books.Add(pbook);
|
|
if(ln_no % 100000 == 0)
|
|
Console.WriteLine("{0}: Read {1} lines", symbol, ln_no);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
Console.WriteLine ("{0} secs to load {1} lines",
|
|
(DateTime.Now - t1).TotalSeconds,ln_no);
|
|
t1 = DateTime.Now;
|
|
foreach (var books in rv.Values)
|
|
MarketQueriesHelper.validate_book_series(books);
|
|
var ret = rv.ToDictionary(kv => kv.Key, kv => new BookSeries(kv.Value,top_only));
|
|
//var ret = rv.ToDictionary(kv => kv.Key, kv => new BookSeries(kv.Value,false));
|
|
Console.WriteLine ("{0} secs to load into book ser",
|
|
(DateTime.Now - t1).TotalSeconds);
|
|
return ret;
|
|
}
|
|
|
|
public static IEnumerable<TradeRow> load_trades(string symbol,DateTime dt){
|
|
return null;
|
|
}
|
|
|
|
public static IEnumerable<OrderRow> load_orders(string symbol,DateTime dt){
|
|
return null;
|
|
}
|
|
|
|
static Timestamp _parse_time_string(string ts){
|
|
int yy = Int32.Parse(ts.Substring(0,4)),
|
|
mm = Int32.Parse(ts.Substring(4,2)),
|
|
dd = Int32.Parse(ts.Substring(6,2)),
|
|
hh = Int32.Parse(ts.Substring(8,2)),
|
|
mins = Int32.Parse(ts.Substring(10,2));
|
|
// now we are left with the remainder seconds
|
|
// expressed in ns
|
|
string secs_in_ns_str = ts.Substring(12,ts.Count()-12);
|
|
Int64 secs_in_ns = Int64.Parse(secs_in_ns_str);
|
|
Int64 fractional_ns,
|
|
quotient_secs = Math.DivRem(secs_in_ns,
|
|
1000000000,
|
|
out fractional_ns);
|
|
Log.assert(fractional_ns < Int32.MaxValue);
|
|
Log.assert(quotient_secs < Int32.MaxValue);
|
|
Int64 tot_seconds_epoch = Convert.ToInt64(
|
|
(new DateTime(yy,mm,dd,hh,mins,Convert.ToInt32(quotient_secs)) -
|
|
new DateTime(1970,1,1)).TotalSeconds);
|
|
return new Timestamp(){
|
|
Seconds = tot_seconds_epoch,
|
|
Nanos = Convert.ToInt32(fractional_ns)
|
|
};
|
|
}
|
|
|
|
private static ProcessedBook _csv_to_processedbook(
|
|
string csv_row,
|
|
bool only_cal_sprs,
|
|
bool top_only){
|
|
uint n_levels = 10;
|
|
uint i = 0;
|
|
string[] toks = csv_row.Split(',');
|
|
uint sec_idx = 4+(n_levels*2);
|
|
if (only_cal_sprs)
|
|
if(!MarketQueriesHelper.is_single_prod_cal_spr(toks[sec_idx]))
|
|
return null;
|
|
ProcessedBook rv = new ProcessedBook(){Book = new BookMessage()};
|
|
Log.assert(toks.Length == 5+4*n_levels);
|
|
rv.MsgSeqNum = UInt32.Parse(toks[i++]);
|
|
rv.SendingTime = _parse_time_string(toks[i++]);
|
|
rv.TransactTime = _parse_time_string(toks[i++]);
|
|
rv.BookSeqNum = UInt32.Parse(toks[i++]);
|
|
uint iter_levels = top_only ? 1 : n_levels;
|
|
for (uint n=0; n<iter_levels; n++){
|
|
double b = Double.Parse(toks[i++]);
|
|
Int32 bqty = Int32.Parse(toks[i++]);
|
|
if(bqty < 0)
|
|
break;
|
|
rv.Book.Bids.Add(MarketQueriesHelper.to_pricesize(
|
|
b,Convert.ToUInt32(bqty)));
|
|
}
|
|
i = sec_idx;
|
|
rv.Security = toks[i++];
|
|
for (uint n=0; n<iter_levels; n++){
|
|
double a = Double.Parse(toks[i++]);
|
|
Int32 aqty = Int32.Parse(toks[i++]);
|
|
if(aqty < 0)
|
|
break;
|
|
rv.Book.Asks.Add(MarketQueriesHelper.to_pricesize(
|
|
a,Convert.ToUInt32(aqty)));
|
|
}
|
|
if(rv.Book.Bids.Count == 0)
|
|
rv.Book.Bids.Add(MarketQueriesHelper.to_pricesize(0,0));
|
|
if(rv.Book.Asks.Count == 0)
|
|
rv.Book.Asks.Add(MarketQueriesHelper.to_pricesize(0,0));
|
|
return rv;
|
|
}
|
|
|
|
private static TradeRow _csv_to_traderow(string csv_row,
|
|
bool only_cal_sprs=true){
|
|
UInt32 i = 0;
|
|
string[] toks = csv_row.Split(',');
|
|
TradeRow rv = new TradeRow();
|
|
rv.Contract = toks[i++];
|
|
rv.SendingTime = _parse_time_string(toks[i++]);
|
|
rv.TransactTime = _parse_time_string(toks[i++]);
|
|
rv.MdEntryPx = Double.Parse(toks[i++]);
|
|
rv.MdEntrySz = UInt32.Parse(toks[i++]);
|
|
rv.NoOfOrders = UInt32.Parse(toks[i++]);
|
|
rv.AggressorSide = UInt32.Parse(toks[i++]);
|
|
rv.MdTradeEntryId = UInt64.Parse(toks[i++]);
|
|
rv.AggressorOid = UInt64.Parse(toks[i++]);
|
|
rv.RestingOid = UInt64.Parse(toks[i++]);
|
|
rv.PartialQty = UInt32.Parse(toks[i++]);
|
|
return rv;
|
|
}
|
|
|
|
private static OrderRow _csv_to_orderrow(string csv_row,
|
|
bool only_cal_sprs=true){
|
|
UInt32 i = 0;
|
|
string[] toks = csv_row.Split(',');
|
|
OrderRow rv = new OrderRow();
|
|
rv.Contract = toks[i++];
|
|
rv.SendingTime = _parse_time_string(toks[i++]);
|
|
rv.TransactTime = _parse_time_string(toks[i++]);
|
|
rv.MdEntryPx = Double.Parse(toks[i++]);
|
|
rv.OrderId = UInt64.Parse(toks[i++]);
|
|
rv.MdEntryQty = UInt32.Parse(toks[i++]);
|
|
rv.MdOrderPriority = UInt64.Parse(toks[i++]);
|
|
rv.OrderUpdateAction = UInt32.Parse(toks[i++]);
|
|
rv.Side = UInt32.Parse(toks[i++]);
|
|
return rv;
|
|
}
|
|
|
|
#region Writing out
|
|
|
|
private static string _format_timestamp(Timestamp t, bool show_date = true){
|
|
// string[] tmtoks = t.ToString().Split('T');
|
|
// string dt = tmtoks[0];
|
|
// string tm = tmtoks[1];
|
|
// string[] toks = tm.Split('.');
|
|
// string time_secs = toks[0],
|
|
// nsecs = toks.Length == 2 ? toks[1] : "0";
|
|
// nsecs = new string(nsecs.Where(c => char.IsDigit(c)).ToArray());
|
|
// nsecs = String.Format("{0:n0}", Int32.Parse(nsecs));
|
|
// //nsecs = nsecs.Replace(',','`');
|
|
// return String.Format("{0}{1}:{2}{3}", (show_date ? dt : String.Empty),time_secs,nsecs,'"');
|
|
return t.ToString().Replace("T"," ");
|
|
}
|
|
|
|
private static string _array_to_string<T> (IEnumerable<T> entries){
|
|
string rv = String.Empty;
|
|
foreach(var o in entries)
|
|
rv += String.Format("{0};",o);
|
|
return rv;
|
|
}
|
|
|
|
public static string leg_book_header(){
|
|
return string.Format("{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11}",
|
|
"Contract",
|
|
"MsgID",
|
|
"SeqNum",
|
|
"ProcTime",
|
|
"SameTopBooks",
|
|
"TransTime",
|
|
"SendTime",
|
|
"ActivePeriod",
|
|
"b_0",
|
|
"bq_0",
|
|
"a_0",
|
|
"aq_0");
|
|
}
|
|
public static string leg_book_entry_to_csv(BookEntry b){
|
|
return string.Format("{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11}",
|
|
b.PBook.Security,
|
|
b.PBook.MsgSeqNum,
|
|
b.PBook.BookSeqNum,
|
|
MarketQueriesHelper.book_processing_time(b.PBook),
|
|
(b.PBook.AbsorbedBooks == null || b.PBook.AbsorbedBooks.Count == 0) ? String.Empty :
|
|
String.Format("{0}:{1}-{2}",
|
|
b.PBook.AbsorbedBooks.Count,
|
|
b.PBook.AbsorbedBooks.First(),
|
|
b.PBook.AbsorbedBooks.Last()),
|
|
_format_timestamp(b.TimeRange.T1),
|
|
_format_timestamp(b.TimeRange.T2),
|
|
MarketQueriesHelper.period_ns(b.TimeRange),
|
|
MarketQueriesHelper.book_price(b,true,0),
|
|
MarketQueriesHelper.book_qty(b,true,0),
|
|
MarketQueriesHelper.book_price(b,false,0),
|
|
MarketQueriesHelper.book_qty(b,false,0));
|
|
}
|
|
|
|
public static string synthetic_book_header(){
|
|
return string.Format("{0},{1},{2},{3},{4},{5},{6},{7}",
|
|
"Synth",
|
|
"idx",
|
|
"T1",
|
|
"T2",
|
|
"En:Ex",
|
|
"ActivePeriod",
|
|
"p",
|
|
"q");
|
|
}
|
|
|
|
public static string synthetic_book_entry_to_csv(MarketQueries.SyntheticPrice p){
|
|
var b = p.synthetic_book;
|
|
var min_max = MarketQueriesHelper.intersection_range_idx(
|
|
p.leg_books.Select(lb=>lb.TimeRange).ToArray());
|
|
return string.Format("{0},{1},{2},{3},{4},{5},{6},{7}",
|
|
b.PBook.Security,
|
|
p.idx,
|
|
_format_timestamp(b.TimeRange.T1),
|
|
_format_timestamp(b.TimeRange.T2),
|
|
String.Format("{0}:{1}",min_max.Item1,min_max.Item2),
|
|
MarketQueriesHelper.period_ns(b.TimeRange),
|
|
p.viable_side == Side.Buy ? MarketQueriesHelper.book_price(b,true,0) :
|
|
MarketQueriesHelper.book_price(b,false,0),
|
|
p.viable_side == Side.Buy ? MarketQueriesHelper.book_qty(b,true,0) :
|
|
MarketQueriesHelper.book_qty(b,false,0));
|
|
}
|
|
|
|
public static string book_entry_to_csv(BookEntry b){
|
|
return string.Format("{0},{1},{2},{3},{4},{5},{6},{7},{8},{9}",
|
|
b.PBook.Security,
|
|
b.PBook.MsgSeqNum,
|
|
b.PBook.BookSeqNum,
|
|
_format_timestamp(b.TimeRange.T1),
|
|
_format_timestamp(b.TimeRange.T2),
|
|
MarketQueriesHelper.period_ns(b.TimeRange),
|
|
MarketQueriesHelper.book_price(b,true,0),
|
|
MarketQueriesHelper.book_qty(b,true,0),
|
|
MarketQueriesHelper.book_price(b,false,0),
|
|
MarketQueriesHelper.book_qty(b,false,0));
|
|
}
|
|
|
|
public static string synthetic_price_to_csv(MarketQueries.SyntheticPrice p){
|
|
string rv = String.Empty;
|
|
foreach (var l in p.leg_books)
|
|
rv += String.Format("{0},",leg_book_entry_to_csv(l));
|
|
rv += synthetic_book_entry_to_csv(p);
|
|
return rv;
|
|
}
|
|
|
|
#endregion
|
|
}
|
|
} |