cleaned up some wrappers to simplify the code

This commit is contained in:
Taha Afzal committed 2020-01-17 18:43:08 -05:00
1 parent 491d0c0b7a
commit dea2358c51
8 files changed
+302 -348

No files matched your search

@@ -37,9 +37,7 @@ namespace bts.history
public BookAnalyzer(string symbol,
HistoricalDataOptions options) {
_book_path = DataIOHelper.get_book_path(
symbol,
DateTime.Parse(options.replaystart));
_book_path = DataIOHelper.get_data_path("/books");
System.IO.Directory.CreateDirectory(_book_path);
_order_str = new FileStream(
String.Format ("{0}orders_{1}",_book_path,symbol),
@@ -9,37 +9,28 @@ namespace bts.history{
class BookSeries{
// the prices we are getting from BookAnalzyer dont have
// the exponent populated
private readonly List<BookEntry> _books = new List<BookEntry>();
private readonly IEnumerable<ProcessedBook> _books;
// expects raw pbooks series as read from IO
// this will change the ProcessedBook
public BookSeries(IEnumerable<ProcessedBook> pbooks,
bool merge_duplicates_tops=false){
_books = _to_book_entries(pbooks, merge_duplicates_tops);
}
static List<BookEntry> _to_book_entries(IEnumerable<ProcessedBook> pbooks,
bool merge_duplicates_tops){
static IEnumerable<ProcessedBook> _to_book_entries(
IEnumerable<ProcessedBook> pbooks,
bool merge_duplicates_tops){
IEnumerable<ProcessedBook> rv = pbooks;
if(merge_duplicates_tops)
pbooks = MarketQueriesHelper.merge_duplicates_tops(pbooks);
var rv = new List<BookEntry>();
for (int i=0; i<pbooks.Count(); i++){
ProcessedBook pb_i = pbooks.ElementAt(i),
pb_i_next = i+1 >= pbooks.Count() ? null :
pbooks.ElementAt(i+1);
Timerange r = new Timerange{
T1 = pb_i.SendingTime,
T2 = (pb_i_next == null ? pb_i.SendingTime :
pb_i_next.TransactTime)
};
rv.Add(new BookEntry{PBook = pb_i, TimeRange = r});
}
return rv;
rv = MarketQueriesHelper.merge_duplicates_tops(pbooks);
MarketQueriesHelper.fill_time_range(rv);
return rv;
}
public int count(){
return _books.Count;
return _books.Count();
}
public IEnumerable<BookEntry> book_entries(){
public IEnumerable<ProcessedBook> book_entries(){
return _books;
}
}
@@ -17,45 +17,11 @@ using Google.Protobuf.WellKnownTypes;
namespace bts.history{
// secondary level helper functions (parsing, reading files etc)
class DataIOHelper{
// ../trade_analysis/date/
public static string get_data_path (DateTime date){
return String.Format(@"{0}{1}/{2}",
Configuration.get<bts.config.System>().data_path,
"trade_analysis",
date.ToShortDateString().Replace("/","_"));
}
public static string get_data_path (string relative_path){
return String.Format(@"{0}/{1}",
Configuration.get<bts.config.System>().data_path,
relative_path);
}
public static string get_rawbook_data_path (){
return String.Format(@"{0}{1}/{2}",
Configuration.get<bts.config.System>().data_path,
"trade_analysis",
"raw_books");
}
public static string get_book_path (string symbol,
DateTime date){
return String.Format(@"{0}{1}/{2}/{3}/{4}/{5}/",
Configuration.get<bts.config.System>().data_path,
"trade_analysis",
"market_data",
date.ToShortDateString().Replace("/","_"),
"books",
symbol);
}
// load books from the books csv
public static IDictionary<string,BookSeries> load_books(string symbol,
DateTime dt,
bool top_only){
DateTime t1 = DateTime.Now;
string file_name = String.Format("{0}/{1}", get_data_path(dt),
String.Format("{0}_{1}", symbol,"books.csv"));
return load_books(file_name,symbol,top_only);
}
}
public static IDictionary<string,BookSeries> load_books(string file_name,
string symbol,
bool top_only){
@@ -212,17 +178,7 @@ namespace bts.history{
#region Writing out
private static string _format_timestamp(Timestamp t, bool show_date = true){
// string[] tmtoks = t.ToString().Split('T');
// string dt = tmtoks[0];
// string tm = tmtoks[1];
// string[] toks = tm.Split('.');
// string time_secs = toks[0],
// nsecs = toks.Length == 2 ? toks[1] : "0";
// nsecs = new string(nsecs.Where(c => char.IsDigit(c)).ToArray());
// nsecs = String.Format("{0:n0}", Int32.Parse(nsecs));
// //nsecs = nsecs.Replace(',','`');
// return String.Format("{0}{1}:{2}{3}", (show_date ? dt : String.Empty),time_secs,nsecs,'"');
return t.ToString().Replace("T"," ");
return t.ToString().Replace("T"," ");
}
private static string _array_to_string<T> (IEnumerable<T> entries){
@@ -247,17 +203,17 @@ namespace bts.history{
"a_0",
"aq_0");
}
public static string leg_book_entry_to_csv(BookEntry b){
public static string leg_book_entry_to_csv(ProcessedBook b){
return string.Format("{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11}",
b.PBook.Security,
b.PBook.MsgSeqNum,
b.PBook.BookSeqNum,
MarketQueriesHelper.book_processing_time(b.PBook),
(b.PBook.AbsorbedBooks == null || b.PBook.AbsorbedBooks.Count == 0) ? String.Empty :
b.Security,
b.MsgSeqNum,
b.BookSeqNum,
MarketQueriesHelper.book_processing_time(b),
(b.AbsorbedBooks == null || b.AbsorbedBooks.Count == 0) ? String.Empty :
String.Format("{0}:{1}-{2}",
b.PBook.AbsorbedBooks.Count,
b.PBook.AbsorbedBooks.First(),
b.PBook.AbsorbedBooks.Last()),
b.AbsorbedBooks.Count,
b.AbsorbedBooks.First(),
b.AbsorbedBooks.Last()),
_format_timestamp(b.TimeRange.T1),
_format_timestamp(b.TimeRange.T2),
MarketQueriesHelper.period_ns(b.TimeRange),
@@ -285,7 +241,7 @@ namespace bts.history{
var min_max = MarketQueriesHelper.intersection_range_idx(
p.leg_books.Select(lb=>lb.TimeRange).ToArray());
return string.Format("{0},{1},{2},{3},{4},{5},{6},{7},{8}",
b.PBook.Security,
b.Security,
p.idx,
_format_timestamp(b.TimeRange.T1),
_format_timestamp(b.TimeRange.T2),
@@ -298,11 +254,11 @@ namespace bts.history{
MarketQueriesHelper.book_qty(b,false,0));
}
public static string book_entry_to_csv(BookEntry b){
public static string book_entry_to_csv(ProcessedBook b){
return string.Format("{0},{1},{2},{3},{4},{5},{6},{7},{8},{9}",
b.PBook.Security,
b.PBook.MsgSeqNum,
b.PBook.BookSeqNum,
b.Security,
b.MsgSeqNum,
b.BookSeqNum,
_format_timestamp(b.TimeRange.T1),
_format_timestamp(b.TimeRange.T2),
MarketQueriesHelper.period_ns(b.TimeRange),
@@ -9,15 +9,27 @@ using mbts.co2;
using Google.Protobuf.WellKnownTypes;
namespace bts.history{
class BookInstance{
public uint BookSeqNum;
public uint MsgSeqNum;
public Timestamp SendingTime;
public Timestamp TransactTime;
public string Security;
public List<Tuple<double,uint>> bids, asks;
public List<uint> AbsorbedBooks;
public Timerange TimeRange;
}
class MarketQueries{
// works only for glbx
#region Synthetic Book Calc
public class SyntheticPrice{
public readonly IEnumerable<BookEntry> leg_books;
public readonly BookEntry synthetic_book;
public SyntheticPrice(IEnumerable<BookEntry> leg_book_a,
BookEntry synthetic_book_a){
public readonly IEnumerable<ProcessedBook> leg_books;
public readonly ProcessedBook synthetic_book;
public SyntheticPrice(IEnumerable<ProcessedBook> leg_book_a,
ProcessedBook synthetic_book_a){
leg_books = leg_book_a.ToArray();
synthetic_book = synthetic_book_a;
}
@@ -25,8 +37,8 @@ namespace bts.history{
public UInt16 idx = 0;
public Int64 max_processing_time(){
var rv = leg_books.MaxElement(b =>
MarketQueriesHelper.book_processing_time(b.PBook));
return MarketQueriesHelper.book_processing_time(rv.PBook);
MarketQueriesHelper.book_processing_time(b));
return MarketQueriesHelper.book_processing_time(rv);
}
}
public static void get_synthetic_prices(
@@ -42,13 +54,13 @@ namespace bts.history{
}
// stop gap, till the above function gets faster
private static BookEntry _compute_synthetic_top_book(
private static ProcessedBook _compute_synthetic_top_book(
SpreaderOrderDefinition order_def,
params BookEntry[] leg_books){
params ProcessedBook[] leg_books){
double top_bid=0, top_ask=0;
UInt32 top_bid_q=UInt32.MaxValue, top_ask_q=UInt32.MaxValue;
for (int i=0; i<leg_books.Count();i++){
BookEntry lb = leg_books[i];
ProcessedBook lb = leg_books[i];
double b = MarketQueriesHelper.book_price(lb,true,0),
a = MarketQueriesHelper.book_price(lb,false,0);
UInt32 bq = MarketQueriesHelper.book_qty(lb,true,0),
@@ -59,51 +71,40 @@ namespace bts.history{
top_bid_q = Math.Min(top_bid_q, l.Side == Side.Buy ? bq : aq);
top_ask_q = Math.Min(top_ask_q, l.Side == Side.Buy ? aq : bq);
}
// this is just the conversion into PriceSizeMessage
IntPrice bid_ip = mbts.co2.IntPrice.from_double(top_bid),
ask_ip = mbts.co2.IntPrice.from_double(top_ask);
PriceSizeMessage bid_psm = new PriceSizeMessage();
bid_psm.Mantissa = bid_ip.mantissa;
bid_psm.Exponent = bid_ip.exponent;
bid_psm.Size = top_bid_q;
PriceSizeMessage ask_psm = new PriceSizeMessage();
ask_psm.Mantissa = ask_ip.mantissa;
ask_psm.Exponent = ask_ip.exponent;
ask_psm.Size = top_ask_q;
BookMessage sbm = new BookMessage();
sbm.Bids.Add(bid_psm);
sbm.Asks.Add(ask_psm);
sbm.Bids.Add(MarketQueriesHelper.to_pricesize(top_bid, top_bid_q));
sbm.Asks.Add(MarketQueriesHelper.to_pricesize(top_ask, top_ask_q));
return _synth_book_to_entry (order_def,sbm,0,leg_books);
}
private static BookEntry _synth_book_to_entry(
SpreaderOrderDefinition order_def,
private static ProcessedBook _synth_book_to_entry(
SpreaderOrderDefinition order_def,
BookMessage bm,
UInt32 seq_no,
params BookEntry[] leg_books){
params ProcessedBook[] leg_books){
Timerange r = MarketQueriesHelper.intersection_range(
leg_books.Select(l=>l.TimeRange).ToArray());
ProcessedBook pb = new ProcessedBook();
pb.Security = order_def.Security.Synthetic.Name;
pb.BookSeqNum = pb.MsgSeqNum = seq_no;
pb.Book = bm;
return new BookEntry {PBook = pb,TimeRange = r};
leg_books.Select(l=>l.TimeRange).ToArray());
return new ProcessedBook {
Security = order_def.Security.Synthetic.Name,
Book = bm,
TimeRange = r
};
}
#endregion
#region Alligning Books
public static IEnumerable<IEnumerable<BookEntry>> allign_bookseries(
public static IEnumerable<IEnumerable<ProcessedBook>> allign_bookseries(
params BookSeries[] leg_book_serieses){
var rv = new List<IEnumerable<BookEntry>>();
align_bookseries(s => rv.Add(new List<BookEntry>(s)), leg_book_serieses);
var rv = new List<IEnumerable<ProcessedBook>>();
align_bookseries(s => rv.Add(new List<ProcessedBook>(s)), leg_book_serieses);
return rv;
}
public static void align_bookseries(
Action<IEnumerable<BookEntry>> on_book,
Action<IEnumerable<ProcessedBook>> on_book,
params BookSeries[] leg_book_serieses){
// book series with all leg books
var all = new List<Tuple<uint,BookEntry>>();
BookEntry[] curr_vals = leg_book_serieses.Select(i=>(BookEntry)null).ToArray();
var all = new List<Tuple<uint,ProcessedBook>>();
ProcessedBook[] curr_vals = leg_book_serieses.Select(i=>(ProcessedBook)null).ToArray();
Func<bool> is_ready = () => {
foreach(var l in curr_vals)
if(l==null)
@@ -132,7 +133,6 @@ namespace bts.history{
#region Orders and Trades
static IEnumerable<Tuple<double,UInt32>> compute_fifo_fills (
Timerange time_range,
double order_price,
@@ -304,6 +304,19 @@ namespace bts.history{
#region book accessors
public static void fill_time_range (IEnumerable<ProcessedBook> pbooks){
for (int i=0; i<pbooks.Count(); i++){
ProcessedBook pb_i = pbooks.ElementAt(i),
pb_i_next = i+1 >= pbooks.Count() ? null :
pbooks.ElementAt(i+1);
pb_i.TimeRange = new Timerange{
T1 = pb_i.SendingTime,
T2 = (pb_i_next == null ? pb_i.SendingTime :
pb_i_next.TransactTime)
};
}
}
public static IEnumerable<ProcessedBook> merge_duplicates_tops(
IEnumerable<ProcessedBook> bser){
List<ProcessedBook> rv = new List<ProcessedBook>();
@@ -374,19 +387,19 @@ namespace bts.history{
return rv;
}
public static bool reactable (BookEntry be){
public static bool reactable (ProcessedBook be){
return diff_ns(be.TimeRange.T2,be.TimeRange.T1) > 0;
}
public static double book_price(BookEntry b, bool is_bid, UInt16 level){
var vals = is_bid ? b.PBook.Book.Bids : b.PBook.Book.Asks;
public static double book_price(ProcessedBook b, bool is_bid, UInt16 level){
var vals = is_bid ? b.Book.Bids : b.Book.Asks;
var i = vals[level];
var iprice = new mbts.co2.IntPrice(i.Mantissa, Convert.ToSByte(i.Exponent));
return iprice.price();
}
public static UInt32 book_qty(BookEntry b, bool is_bid, UInt16 level){
var vals = is_bid ? b.PBook.Book.Bids : b.PBook.Book.Asks;
public static UInt32 book_qty(ProcessedBook b, bool is_bid, UInt16 level){
var vals = is_bid ? b.Book.Bids : b.Book.Asks;
var i = vals[level];
return i.Size;
}
@@ -1,227 +1,227 @@
using System;
using System.Collections.Generic;
using System.Linq;
using bts.spark;
using bts.api;
using bts.api.orderman;
using bts.co2;
using Google.Protobuf.WellKnownTypes;
// using System;
// using System.Collections.Generic;
// using System.Linq;
// using bts.spark;
// using bts.api;
// using bts.api.orderman;
// using bts.co2;
// using Google.Protobuf.WellKnownTypes;
namespace bts.history{
class SanityTest{
public static void run_tests(){
Console.Write ("Tests started...");
test_books_in_range();
test_random_intersection();
Console.Write ("Tests completed");
}
public static void test_books_in_range(){
// DateTime base_time = new DateTime(2020,1,1,9,30,45,0,DateTimeKind.Utc);
// List<ProcessedBook> bseries_list = new List<ProcessedBook>();
// namespace bts.history{
// class SanityTest{
// public static void run_tests(){
// Console.Write ("Tests started...");
// test_books_in_range();
// test_random_intersection();
// Console.Write ("Tests completed");
// }
// public static void test_books_in_range(){
// // DateTime base_time = new DateTime(2020,1,1,9,30,45,0,DateTimeKind.Utc);
// // List<ProcessedBook> bseries_list = new List<ProcessedBook>();
// Action<UInt32> add_b = (UInt32 usecs) => {
// ProcessedBook b = new ProcessedBook();
// b.MsgSeqNum = b.BookSeqNum = Convert.ToUInt32(bseries_list.Count());
// b.SendingTime = b.TransactTime = from_datetime(base_time, usecs);
// bseries_list.Add(b);
// };
// // Action<UInt32> add_b = (UInt32 usecs) => {
// // ProcessedBook b = new ProcessedBook();
// // b.MsgSeqNum = b.BookSeqNum = Convert.ToUInt32(bseries_list.Count());
// // b.SendingTime = b.TransactTime = from_datetime(base_time, usecs);
// // bseries_list.Add(b);
// // };
// add_b(1000);
// add_b(1020);
// add_b(1025);
// add_b(1035);
// add_b(1050);
// add_b(1055);
// add_b(1060);
// add_b(1065);
// add_b(1080);
// add_b(1110);
// // add_b(1000);
// // add_b(1020);
// // add_b(1025);
// // add_b(1035);
// // add_b(1050);
// // add_b(1055);
// // add_b(1060);
// // add_b(1065);
// // add_b(1080);
// // add_b(1110);
// BookSeries bser_1 = new BookSeries (bseries_list);
// var calc_bes = bser_1.book_entries();
// Log.assert(calc_bes.Count()==10);
// Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(0).TimeRange) == 20000);
// Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(1).TimeRange) == 5000);
// Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(2).TimeRange) == 10000);
// Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(3).TimeRange) == 15000);
// Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(4).TimeRange) == 5000);
// Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(5).TimeRange) == 5000);
// Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(6).TimeRange) == 5000);
// Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(7).TimeRange) == 15000);
// Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(8).TimeRange) == 30000);
// Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(9).TimeRange) > 1000000000);
// // BookSeries bser_1 = new BookSeries (bseries_list);
// // var calc_bes = bser_1.book_entries();
// // Log.assert(calc_bes.Count()==10);
// // Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(0).TimeRange) == 20000);
// // Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(1).TimeRange) == 5000);
// // Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(2).TimeRange) == 10000);
// // Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(3).TimeRange) == 15000);
// // Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(4).TimeRange) == 5000);
// // Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(5).TimeRange) == 5000);
// // Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(6).TimeRange) == 5000);
// // Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(7).TimeRange) == 15000);
// // Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(8).TimeRange) == 30000);
// // Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(9).TimeRange) > 1000000000);
// // find in range /////////////////////////////////////////////////////
// // test find_books_in_time_range
// // 1005-1053
// var bitems = bser_1.find_books_in_time_range(
// new Timerange{
// T1=from_datetime(base_time, 1005),
// T2=from_datetime(base_time, 1053)});
// Log.assert(bitems.Count()==5);
// for (int i=0; i<bitems.Count();i++){
// BookEntry be = bitems.ElementAt(i);
// Log.assert(be.PBook.MsgSeqNum==Convert.ToUInt32(i));
// }
// // test find_books_in_time_range
// // 1030-2000
// bitems = bser_1.find_books_in_time_range(
// new Timerange{
// T1=from_datetime(base_time, 1030),
// T2=from_datetime(base_time, 2000)});
// Log.assert(bitems.Count()==8);
// for (int i=0; i<bitems.Count();i++){
// BookEntry be = bitems.ElementAt(i);
// Log.assert(be.PBook.MsgSeqNum==Convert.ToUInt32(i+2));
// }
// // // find in range /////////////////////////////////////////////////////
// // // test find_books_in_time_range
// // // 1005-1053
// // var bitems = bser_1.find_books_in_time_range(
// // new Timerange{
// // T1=from_datetime(base_time, 1005),
// // T2=from_datetime(base_time, 1053)});
// // Log.assert(bitems.Count()==5);
// // for (int i=0; i<bitems.Count();i++){
// // BookEntry be = bitems.ElementAt(i);
// // Log.assert(be.PBook.MsgSeqNum==Convert.ToUInt32(i));
// // }
// // // test find_books_in_time_range
// // // 1030-2000
// // bitems = bser_1.find_books_in_time_range(
// // new Timerange{
// // T1=from_datetime(base_time, 1030),
// // T2=from_datetime(base_time, 2000)});
// // Log.assert(bitems.Count()==8);
// // for (int i=0; i<bitems.Count();i++){
// // BookEntry be = bitems.ElementAt(i);
// // Log.assert(be.PBook.MsgSeqNum==Convert.ToUInt32(i+2));
// // }
// find coexisting 1 ////////////////////////////////////////////////////
// bseries_list.Clear();
// add_b(900);
// add_b(990);
// add_b(1010);
// add_b(1015); //3
// add_b(1030);
// BookSeries bser_2 = new BookSeries (bseries_list);
// var coex = MarketQueries.find_coexisting_books(
// bser_1,
// bser_2,
// new Timerange{T1=from_datetime(base_time, 999),
// T2=from_datetime(base_time, 1051)});
// _compare_seq_no (coex.ElementAt(0),0,1); //1000
// _compare_seq_no (coex.ElementAt(1),0,2);
// _compare_seq_no (coex.ElementAt(2),0,3);
// _compare_seq_no (coex.ElementAt(3),1,3); //1020
// _compare_seq_no (coex.ElementAt(4),2,3); //1025
// _compare_seq_no (coex.ElementAt(5),2,4); //1025
// _compare_seq_no (coex.ElementAt(6),3,4); //1035
// _compare_seq_no (coex.ElementAt(7),4,4); //1050
// // find coexisting 1 ////////////////////////////////////////////////////
// // bseries_list.Clear();
// // add_b(900);
// // add_b(990);
// // add_b(1010);
// // add_b(1015); //3
// // add_b(1030);
// // BookSeries bser_2 = new BookSeries (bseries_list);
// // var coex = MarketQueries.find_coexisting_books(
// // bser_1,
// // bser_2,
// // new Timerange{T1=from_datetime(base_time, 999),
// // T2=from_datetime(base_time, 1051)});
// // _compare_seq_no (coex.ElementAt(0),0,1); //1000
// // _compare_seq_no (coex.ElementAt(1),0,2);
// // _compare_seq_no (coex.ElementAt(2),0,3);
// // _compare_seq_no (coex.ElementAt(3),1,3); //1020
// // _compare_seq_no (coex.ElementAt(4),2,3); //1025
// // _compare_seq_no (coex.ElementAt(5),2,4); //1025
// // _compare_seq_no (coex.ElementAt(6),3,4); //1035
// // _compare_seq_no (coex.ElementAt(7),4,4); //1050
// MarketQueries.align_bookseries(
// l => {
// var b1 = l.ElementAt(0);
// var b2 = l.ElementAt(1);
// var r = MarketQueriesHelper.intersection_type(b1.TimeRange, b2.TimeRange);
// Log.assert(r == TimerangeIntersection.INTERSECTION);
// },
// bser_1,
// bser_2);
}
// // MarketQueries.align_bookseries(
// // l => {
// // var b1 = l.ElementAt(0);
// // var b2 = l.ElementAt(1);
// // var r = MarketQueriesHelper.intersection_type(b1.TimeRange, b2.TimeRange);
// // Log.assert(r == TimerangeIntersection.INTERSECTION);
// // },
// // bser_1,
// // bser_2);
// }
public static void test_random_intersection(){
// 3 lists
Func<int,int,int,int?,BookSeries> cons_series = (n_items,min_i,max_i,r) => {
List<ProcessedBook> book_ser_list = new List<ProcessedBook>();
Random rnd = new Random(DateTime.Now.Millisecond);
int moving_usecs = 0;
for(int i = 0; i < n_items; i++){
moving_usecs += r.HasValue ? r.Value : rnd.Next(min_i,max_i);
_add_to_ser(ref book_ser_list,
Convert.ToUInt32(moving_usecs));
}
return new BookSeries(book_ser_list);
};
// non ranodm data
BookSeries bser_1 = cons_series(100,1,20,20),
bser_2 = cons_series(200,2,15,10),
bser_3 = cons_series(400,2,15,5);
var res = new List<Tuple<BookEntry,BookEntry,BookEntry>>();
MarketQueries.align_bookseries(
l => res.Add(Tuple.Create(l.ElementAt(0),
l.ElementAt(1),
l.ElementAt(2))),
bser_1,
bser_2,
bser_3);
_verify_intersection(bser_1,bser_2,bser_3,res);
// random data
int n_rnd_tests = 10;
for (int i=0;i<n_rnd_tests;i++){
bser_1 = cons_series(100,1,19,null);
bser_2 = cons_series(100,3,23,null);
bser_3 = cons_series(100,5,31,null);
res = new List<Tuple<BookEntry,BookEntry,BookEntry>>();
MarketQueries.align_bookseries(
l => res.Add(Tuple.Create(l.ElementAt(0),
l.ElementAt(1),
l.ElementAt(2))),
bser_1,
bser_2,
bser_3);
_verify_intersection(bser_1,bser_2,bser_3,res);
}
}
// public static void test_random_intersection(){
// // 3 lists
// Func<int,int,int,int?,BookSeries> cons_series = (n_items,min_i,max_i,r) => {
// List<ProcessedBook> book_ser_list = new List<ProcessedBook>();
// Random rnd = new Random(DateTime.Now.Millisecond);
// int moving_usecs = 0;
// for(int i = 0; i < n_items; i++){
// moving_usecs += r.HasValue ? r.Value : rnd.Next(min_i,max_i);
// _add_to_ser(ref book_ser_list,
// Convert.ToUInt32(moving_usecs));
// }
// return new BookSeries(book_ser_list);
// };
// // non ranodm data
// BookSeries bser_1 = cons_series(100,1,20,20),
// bser_2 = cons_series(200,2,15,10),
// bser_3 = cons_series(400,2,15,5);
// var res = new List<Tuple<BookEntry,BookEntry,BookEntry>>();
// MarketQueries.align_bookseries(
// l => res.Add(Tuple.Create(l.ElementAt(0),
// l.ElementAt(1),
// l.ElementAt(2))),
// bser_1,
// bser_2,
// bser_3);
// _verify_intersection(bser_1,bser_2,bser_3,res);
// // random data
// int n_rnd_tests = 10;
// for (int i=0;i<n_rnd_tests;i++){
// bser_1 = cons_series(100,1,19,null);
// bser_2 = cons_series(100,3,23,null);
// bser_3 = cons_series(100,5,31,null);
// res = new List<Tuple<BookEntry,BookEntry,BookEntry>>();
// MarketQueries.align_bookseries(
// l => res.Add(Tuple.Create(l.ElementAt(0),
// l.ElementAt(1),
// l.ElementAt(2))),
// bser_1,
// bser_2,
// bser_3);
// _verify_intersection(bser_1,bser_2,bser_3,res);
// }
// }
public static Timestamp from_datetime(DateTime dt, UInt32 usecs){
if (dt.Millisecond != 0){
usecs += Convert.ToUInt32(dt.Millisecond) * 1000;
dt = dt.AddMilliseconds(dt.Millisecond*-1.0);
}
Timestamp rv = new Timestamp();
rv.Seconds = Convert.ToInt64((dt - new DateTime(1970,1,1)).TotalSeconds);
rv.Nanos = Convert.ToInt32(usecs) * 1000;
return rv;
}
// public static Timestamp from_datetime(DateTime dt, UInt32 usecs){
// if (dt.Millisecond != 0){
// usecs += Convert.ToUInt32(dt.Millisecond) * 1000;
// dt = dt.AddMilliseconds(dt.Millisecond*-1.0);
// }
// Timestamp rv = new Timestamp();
// rv.Seconds = Convert.ToInt64((dt - new DateTime(1970,1,1)).TotalSeconds);
// rv.Nanos = Convert.ToInt32(usecs) * 1000;
// return rv;
// }
private static void _compare_seq_no (Tuple<BookEntry,BookEntry> b,
UInt32 s1, UInt32 s2){
Log.assert(b.Item1.PBook.BookSeqNum == s1);
Log.assert(b.Item2.PBook.BookSeqNum == s2);
}
// private static void _compare_seq_no (Tuple<BookEntry,BookEntry> b,
// UInt32 s1, UInt32 s2){
// Log.assert(b.Item1.PBook.BookSeqNum == s1);
// Log.assert(b.Item2.PBook.BookSeqNum == s2);
// }
private static void _add_to_ser(ref List<ProcessedBook> bseries_list,
UInt32 usecs){
DateTime base_time = new DateTime(2020,1,1,9,30,45,0,DateTimeKind.Utc);
ProcessedBook b = new ProcessedBook();
b.MsgSeqNum = b.BookSeqNum = Convert.ToUInt32(bseries_list.Count());
b.SendingTime = b.TransactTime = from_datetime(base_time, usecs);
bseries_list.Add(b);
}
// private static void _add_to_ser(ref List<ProcessedBook> bseries_list,
// UInt32 usecs){
// DateTime base_time = new DateTime(2020,1,1,9,30,45,0,DateTimeKind.Utc);
// ProcessedBook b = new ProcessedBook();
// b.MsgSeqNum = b.BookSeqNum = Convert.ToUInt32(bseries_list.Count());
// b.SendingTime = b.TransactTime = from_datetime(base_time, usecs);
// bseries_list.Add(b);
// }
private static void _verify_intersection (
BookSeries bser_1,
BookSeries bser_2,
BookSeries bser_3,
IEnumerable<Tuple<BookEntry,BookEntry,BookEntry>> i_set){
Func<Tuple<BookEntry,BookEntry,BookEntry>, int> count = (t_a) => {
int rv = 0;
foreach (var t in i_set)
if(t.Item1.PBook.BookSeqNum == t_a.Item1.PBook.BookSeqNum &&
t.Item2.PBook.BookSeqNum == t_a.Item2.PBook.BookSeqNum &&
t.Item3.PBook.BookSeqNum == t_a.Item3.PBook.BookSeqNum)
rv++;
return rv;
};
// 1. uniqueness of data
// each tuple should be present only once
foreach (var t in i_set)
Log.assert(count(t)==1);
// 2. make sure each tuple in I has overlap
foreach (var t in i_set){
Timerange r = MarketQueriesHelper.intersection_range(
t.Item1.TimeRange,
t.Item2.TimeRange,
t.Item3.TimeRange);
Log.assert(MarketQueriesHelper.period_ns(r)>0);
}
// 3. find overlaps throgh exhaustive search and verify
// they are found in I
foreach (BookEntry b1 in bser_1.book_entries())
foreach (BookEntry b2 in bser_2.book_entries())
foreach (BookEntry b3 in bser_3.book_entries()){
var intr = MarketQueriesHelper.intersection_range(
b1.TimeRange,
b2.TimeRange,
b3.TimeRange);
if(MarketQueriesHelper.period_ns(intr)>0)
Log.assert(count(Tuple.Create(b1,b2,b3))==1);
}
}
// private static void _verify_intersection (
// BookSeries bser_1,
// BookSeries bser_2,
// BookSeries bser_3,
// IEnumerable<Tuple<BookEntry,BookEntry,BookEntry>> i_set){
// Func<Tuple<BookEntry,BookEntry,BookEntry>, int> count = (t_a) => {
// int rv = 0;
// foreach (var t in i_set)
// if(t.Item1.PBook.BookSeqNum == t_a.Item1.PBook.BookSeqNum &&
// t.Item2.PBook.BookSeqNum == t_a.Item2.PBook.BookSeqNum &&
// t.Item3.PBook.BookSeqNum == t_a.Item3.PBook.BookSeqNum)
// rv++;
// return rv;
// };
// // 1. uniqueness of data
// // each tuple should be present only once
// foreach (var t in i_set)
// Log.assert(count(t)==1);
// // 2. make sure each tuple in I has overlap
// foreach (var t in i_set){
// Timerange r = MarketQueriesHelper.intersection_range(
// t.Item1.TimeRange,
// t.Item2.TimeRange,
// t.Item3.TimeRange);
// Log.assert(MarketQueriesHelper.period_ns(r)>0);
// }
// // 3. find overlaps throgh exhaustive search and verify
// // they are found in I
// foreach (BookEntry b1 in bser_1.book_entries())
// foreach (BookEntry b2 in bser_2.book_entries())
// foreach (BookEntry b3 in bser_3.book_entries()){
// var intr = MarketQueriesHelper.intersection_range(
// b1.TimeRange,
// b2.TimeRange,
// b3.TimeRange);
// if(MarketQueriesHelper.period_ns(intr)>0)
// Log.assert(count(Tuple.Create(b1,b2,b3))==1);
// }
// }
// test find_books_in_time_range
// find_coexisting_books 1
// find_coexisting_books 2
// align_2_bookseries
// align_3_bookseries
}
}
// // test find_books_in_time_range
// // find_coexisting_books 1
// // find_coexisting_books 2
// // align_2_bookseries
// // align_3_bookseries
// }
// }
@@ -117,7 +117,7 @@ namespace bts.history{
private static bool _detect_opp (
string n,
MarketQueries.SyntheticPrice spr){
BookEntry p = spr.synthetic_book;
ProcessedBook p = spr.synthetic_book;
double b = MarketQueriesHelper.book_price(p,true,0),
a = MarketQueriesHelper.book_price(p,false,0);
UInt32 bq = MarketQueriesHelper.book_qty(p,true,0),
@@ -153,6 +153,9 @@ message ProcessedBook
string security = 40;
co2.BookMessage book = 50;
repeated uint32 absorbed_books = 60;
// this is the book's sending time (T1) till
// the next book's transaction time
Timerange time_range = 70;
}
message Timerange{
@@ -160,13 +163,6 @@ message Timerange{
google.protobuf.Timestamp t2 = 20;
}
message BookEntry{
ProcessedBook p_book = 10;
// this is the book's sending time (T1) till
// the next book's transaction time
Timerange time_range = 20;
}
message TradeEntry{
string contract = 1;
double price = 10;