Files
spark/src/client/spark_historical/StaticAnalysis/DataIOHelper.cs
T

324 lines
14 KiB
C#

using System;
using System.Collections.Generic;
using System.Linq;
using bts.spark;
using bts.api;
using bts.api.orderman;
using bts.co2;
using mbts.config;
using bts.config;
using System.Threading;
using System.Threading.Tasks;
using System.Collections.Concurrent;
using System.IO;
using mbts.co2;
using Google.Protobuf.WellKnownTypes;
namespace bts.history{
// secondary level helper functions (parsing, reading files etc)
class DataIOHelper{
// ../trade_analysis/date/
public static string get_data_path (DateTime date){
return String.Format(@"{0}{1}/{2}",
Configuration.get<bts.config.System>().data_path,
"trade_analysis",
date.ToShortDateString().Replace("/","_"));
}
public static string get_data_path (string relative_path){
return String.Format(@"{0}/{1}",
Configuration.get<bts.config.System>().data_path,
relative_path);
}
public static string get_rawbook_data_path (){
return String.Format(@"{0}{1}/{2}",
Configuration.get<bts.config.System>().data_path,
"trade_analysis",
"raw_books");
}
public static string get_book_path (string symbol,
DateTime date){
return String.Format(@"{0}{1}/{2}/{3}/{4}/{5}/",
Configuration.get<bts.config.System>().data_path,
"trade_analysis",
"market_data",
date.ToShortDateString().Replace("/","_"),
"books",
symbol);
}
// load books from the books csv
public static IDictionary<string,BookSeries> load_books(string symbol,
DateTime dt,
bool top_only){
DateTime t1 = DateTime.Now;
string file_name = String.Format("{0}/{1}", get_data_path(dt),
String.Format("{0}_{1}", symbol,"books.csv"));
return load_books(file_name,symbol,top_only);
}
public static IDictionary<string,BookSeries> load_books(string file_name,
string symbol,
bool top_only){
DateTime t1 = DateTime.Now;
var rv = new Dictionary<string,List<ProcessedBook>>();
int ln_no = 0;
using (StreamReader sr = new StreamReader(file_name)) {
string line;
while ((line = sr.ReadLine()) != null){
if(ln_no++>0){
var pbook = _csv_to_processedbook(line,true,top_only);
if(pbook != null){
MarketQueriesHelper.fill_exponent(symbol,pbook);
List<ProcessedBook> contract_books = null;
if(!rv.TryGetValue(pbook.Security,out contract_books)){
contract_books = new List<ProcessedBook>();
rv.Add(pbook.Security,contract_books);
}
if(MarketQueriesHelper.is_sparse_book(pbook.Book))
Log.assert(contract_books.Count==0,
String.Format("Illegal sparse book {0}",
pbook.BookSeqNum));
else
contract_books.Add(pbook);
if(ln_no % 100000 == 0)
Console.WriteLine("{0}: Read {1} lines", symbol, ln_no);
}
}
}
}
Console.WriteLine ("{0} secs to load {1} lines",
(DateTime.Now - t1).TotalSeconds,ln_no);
t1 = DateTime.Now;
foreach (var books in rv.Values)
MarketQueriesHelper.validate_book_series(books);
var ret = rv.ToDictionary(kv => kv.Key, kv => new BookSeries(kv.Value,top_only));
//var ret = rv.ToDictionary(kv => kv.Key, kv => new BookSeries(kv.Value,false));
Console.WriteLine ("{0} secs to load into book ser",
(DateTime.Now - t1).TotalSeconds);
return ret;
}
public static IEnumerable<TradeRow> load_trades(string symbol,DateTime dt){
return null;
}
public static IEnumerable<OrderRow> load_orders(string symbol,DateTime dt){
return null;
}
static Timestamp _parse_time_string(string ts){
int yy = Int32.Parse(ts.Substring(0,4)),
mm = Int32.Parse(ts.Substring(4,2)),
dd = Int32.Parse(ts.Substring(6,2)),
hh = Int32.Parse(ts.Substring(8,2)),
mins = Int32.Parse(ts.Substring(10,2));
// now we are left with the remainder seconds
// expressed in ns
string secs_in_ns_str = ts.Substring(12,ts.Count()-12);
Int64 secs_in_ns = Int64.Parse(secs_in_ns_str);
Int64 fractional_ns,
quotient_secs = Math.DivRem(secs_in_ns,
1000000000,
out fractional_ns);
Log.assert(fractional_ns < Int32.MaxValue);
Log.assert(quotient_secs < Int32.MaxValue);
Int64 tot_seconds_epoch = Convert.ToInt64(
(new DateTime(yy,mm,dd,hh,mins,Convert.ToInt32(quotient_secs)) -
new DateTime(1970,1,1)).TotalSeconds);
return new Timestamp(){
Seconds = tot_seconds_epoch,
Nanos = Convert.ToInt32(fractional_ns)
};
}
private static ProcessedBook _csv_to_processedbook(
string csv_row,
bool only_cal_sprs,
bool top_only){
uint n_levels = 10;
uint i = 0;
string[] toks = csv_row.Split(',');
uint sec_idx = 4+(n_levels*2);
if (only_cal_sprs)
if(!MarketQueriesHelper.is_single_prod_cal_spr(toks[sec_idx]))
return null;
ProcessedBook rv = new ProcessedBook(){Book = new BookMessage()};
Log.assert(toks.Length == 5+4*n_levels);
rv.MsgSeqNum = UInt32.Parse(toks[i++]);
rv.SendingTime = _parse_time_string(toks[i++]);
rv.TransactTime = _parse_time_string(toks[i++]);
rv.BookSeqNum = UInt32.Parse(toks[i++]);
uint iter_levels = top_only ? 1 : n_levels;
for (uint n=0; n<iter_levels; n++){
double b = Double.Parse(toks[i++]);
Int32 bqty = Int32.Parse(toks[i++]);
if(bqty < 0)
break;
rv.Book.Bids.Add(MarketQueriesHelper.to_pricesize(
b,Convert.ToUInt32(bqty)));
}
i = sec_idx;
rv.Security = toks[i++];
for (uint n=0; n<iter_levels; n++){
double a = Double.Parse(toks[i++]);
Int32 aqty = Int32.Parse(toks[i++]);
if(aqty < 0)
break;
rv.Book.Asks.Add(MarketQueriesHelper.to_pricesize(
a,Convert.ToUInt32(aqty)));
}
if(rv.Book.Bids.Count == 0)
rv.Book.Bids.Add(MarketQueriesHelper.to_pricesize(0,0));
if(rv.Book.Asks.Count == 0)
rv.Book.Asks.Add(MarketQueriesHelper.to_pricesize(0,0));
return rv;
}
private static TradeRow _csv_to_traderow(string csv_row,
bool only_cal_sprs=true){
UInt32 i = 0;
string[] toks = csv_row.Split(',');
TradeRow rv = new TradeRow();
rv.Contract = toks[i++];
rv.SendingTime = _parse_time_string(toks[i++]);
rv.TransactTime = _parse_time_string(toks[i++]);
rv.MdEntryPx = Double.Parse(toks[i++]);
rv.MdEntrySz = UInt32.Parse(toks[i++]);
rv.NoOfOrders = UInt32.Parse(toks[i++]);
rv.AggressorSide = UInt32.Parse(toks[i++]);
rv.MdTradeEntryId = UInt64.Parse(toks[i++]);
rv.AggressorOid = UInt64.Parse(toks[i++]);
rv.RestingOid = UInt64.Parse(toks[i++]);
rv.PartialQty = UInt32.Parse(toks[i++]);
return rv;
}
private static OrderRow _csv_to_orderrow(string csv_row,
bool only_cal_sprs=true){
UInt32 i = 0;
string[] toks = csv_row.Split(',');
OrderRow rv = new OrderRow();
rv.Contract = toks[i++];
rv.SendingTime = _parse_time_string(toks[i++]);
rv.TransactTime = _parse_time_string(toks[i++]);
rv.MdEntryPx = Double.Parse(toks[i++]);
rv.OrderId = UInt64.Parse(toks[i++]);
rv.MdEntryQty = UInt32.Parse(toks[i++]);
rv.MdOrderPriority = UInt64.Parse(toks[i++]);
rv.OrderUpdateAction = UInt32.Parse(toks[i++]);
rv.Side = UInt32.Parse(toks[i++]);
return rv;
}
#region Writing out
private static string _format_timestamp(Timestamp t, bool show_date = true){
// string[] tmtoks = t.ToString().Split('T');
// string dt = tmtoks[0];
// string tm = tmtoks[1];
// string[] toks = tm.Split('.');
// string time_secs = toks[0],
// nsecs = toks.Length == 2 ? toks[1] : "0";
// nsecs = new string(nsecs.Where(c => char.IsDigit(c)).ToArray());
// nsecs = String.Format("{0:n0}", Int32.Parse(nsecs));
// //nsecs = nsecs.Replace(',','`');
// return String.Format("{0}{1}:{2}{3}", (show_date ? dt : String.Empty),time_secs,nsecs,'"');
return t.ToString().Replace("T"," ");
}
private static string _array_to_string<T> (IEnumerable<T> entries){
string rv = String.Empty;
foreach(var o in entries)
rv += String.Format("{0};",o);
return rv;
}
public static string leg_book_header(){
return string.Format("{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11}",
"Contract",
"MsgID",
"SeqNum",
"ProcTime",
"SameTopBooks",
"TransTime",
"SendTime",
"ActivePeriod",
"b_0",
"bq_0",
"a_0",
"aq_0");
}
public static string leg_book_entry_to_csv(BookEntry b){
return string.Format("{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11}",
b.PBook.Security,
b.PBook.MsgSeqNum,
b.PBook.BookSeqNum,
MarketQueriesHelper.book_processing_time(b.PBook),
(b.PBook.AbsorbedBooks == null || b.PBook.AbsorbedBooks.Count == 0) ? String.Empty :
String.Format("{0}:{1}-{2}",
b.PBook.AbsorbedBooks.Count,
b.PBook.AbsorbedBooks.First(),
b.PBook.AbsorbedBooks.Last()),
_format_timestamp(b.TimeRange.T1),
_format_timestamp(b.TimeRange.T2),
MarketQueriesHelper.period_ns(b.TimeRange),
MarketQueriesHelper.book_price(b,true,0),
MarketQueriesHelper.book_qty(b,true,0),
MarketQueriesHelper.book_price(b,false,0),
MarketQueriesHelper.book_qty(b,false,0));
}
public static string synthetic_book_header(){
return string.Format("{0},{1},{2},{3},{4},{5},{6},{7}",
"Synth",
"idx",
"T1",
"T2",
"En:Ex",
"ActivePeriod",
"p",
"q");
}
public static string synthetic_book_entry_to_csv(MarketQueries.SyntheticPrice p){
var b = p.synthetic_book;
var min_max = MarketQueriesHelper.intersection_range_idx(
p.leg_books.Select(lb=>lb.TimeRange).ToArray());
return string.Format("{0},{1},{2},{3},{4},{5},{6},{7}",
b.PBook.Security,
p.idx,
_format_timestamp(b.TimeRange.T1),
_format_timestamp(b.TimeRange.T2),
String.Format("{0}:{1}",min_max.Item1,min_max.Item2),
MarketQueriesHelper.period_ns(b.TimeRange),
p.viable_side == Side.Buy ? MarketQueriesHelper.book_price(b,true,0) :
MarketQueriesHelper.book_price(b,false,0),
p.viable_side == Side.Buy ? MarketQueriesHelper.book_qty(b,true,0) :
MarketQueriesHelper.book_qty(b,false,0));
}
public static string book_entry_to_csv(BookEntry b){
return string.Format("{0},{1},{2},{3},{4},{5},{6},{7},{8},{9}",
b.PBook.Security,
b.PBook.MsgSeqNum,
b.PBook.BookSeqNum,
_format_timestamp(b.TimeRange.T1),
_format_timestamp(b.TimeRange.T2),
MarketQueriesHelper.period_ns(b.TimeRange),
MarketQueriesHelper.book_price(b,true,0),
MarketQueriesHelper.book_qty(b,true,0),
MarketQueriesHelper.book_price(b,false,0),
MarketQueriesHelper.book_qty(b,false,0));
}
public static string synthetic_price_to_csv(MarketQueries.SyntheticPrice p){
string rv = String.Empty;
foreach (var l in p.leg_books)
rv += String.Format("{0},",leg_book_entry_to_csv(l));
rv += synthetic_book_entry_to_csv(p);
return rv;
}
#endregion
}
}