From dea2358c511bb6f760e02b2ec50f54eb8415644a Mon Sep 17 00:00:00 2001 From: Taha Afzal Date: Fri, 17 Jan 2020 18:43:08 -0500 Subject: [PATCH] cleaned up some wrappers to simplify the code --- .../Analyzers/BookAnalyzer.cs | 4 +- .../StaticAnalysis/BookSeries.cs | 31 +- .../StaticAnalysis/DataIOHelper.cs | 76 +--- .../StaticAnalysis/MarketQueries.cs | 76 ++-- .../StaticAnalysis/MarketQueriesHelper.cs | 23 +- .../StaticAnalysis/SanityTests.cs | 428 +++++++++--------- .../SpreaderTickSniperAnalyzer.cs | 2 +- .../api/services/HistoricalDataService.proto | 10 +- 8 files changed, 302 insertions(+), 348 deletions(-) diff --git a/src/client/spark_historical/Analyzers/BookAnalyzer.cs b/src/client/spark_historical/Analyzers/BookAnalyzer.cs index 81c21e961..43ad2bb82 100644 --- a/src/client/spark_historical/Analyzers/BookAnalyzer.cs +++ b/src/client/spark_historical/Analyzers/BookAnalyzer.cs @@ -37,9 +37,7 @@ namespace bts.history public BookAnalyzer(string symbol, HistoricalDataOptions options) { - _book_path = DataIOHelper.get_book_path( - symbol, - DateTime.Parse(options.replaystart)); + _book_path = DataIOHelper.get_data_path("/books"); System.IO.Directory.CreateDirectory(_book_path); _order_str = new FileStream( String.Format ("{0}orders_{1}",_book_path,symbol), diff --git a/src/client/spark_historical/StaticAnalysis/BookSeries.cs b/src/client/spark_historical/StaticAnalysis/BookSeries.cs index 313e202a2..247461d99 100644 --- a/src/client/spark_historical/StaticAnalysis/BookSeries.cs +++ b/src/client/spark_historical/StaticAnalysis/BookSeries.cs @@ -9,37 +9,28 @@ namespace bts.history{ class BookSeries{ // the prices we are getting from BookAnalzyer dont have // the exponent populated - private readonly List _books = new List(); + private readonly IEnumerable _books; - // expects raw pbooks series as read from IO + // this will change the ProcessedBook public BookSeries(IEnumerable pbooks, bool merge_duplicates_tops=false){ _books = _to_book_entries(pbooks, merge_duplicates_tops); } - static List _to_book_entries(IEnumerable pbooks, - bool merge_duplicates_tops){ + static IEnumerable _to_book_entries( + IEnumerable pbooks, + bool merge_duplicates_tops){ + IEnumerable rv = pbooks; if(merge_duplicates_tops) - pbooks = MarketQueriesHelper.merge_duplicates_tops(pbooks); - var rv = new List(); - for (int i=0; i= pbooks.Count() ? null : - pbooks.ElementAt(i+1); - Timerange r = new Timerange{ - T1 = pb_i.SendingTime, - T2 = (pb_i_next == null ? pb_i.SendingTime : - pb_i_next.TransactTime) - }; - rv.Add(new BookEntry{PBook = pb_i, TimeRange = r}); - } - return rv; + rv = MarketQueriesHelper.merge_duplicates_tops(pbooks); + MarketQueriesHelper.fill_time_range(rv); + return rv; } public int count(){ - return _books.Count; + return _books.Count(); } - public IEnumerable book_entries(){ + public IEnumerable book_entries(){ return _books; } } diff --git a/src/client/spark_historical/StaticAnalysis/DataIOHelper.cs b/src/client/spark_historical/StaticAnalysis/DataIOHelper.cs index d6d640ee1..b6809cf91 100644 --- a/src/client/spark_historical/StaticAnalysis/DataIOHelper.cs +++ b/src/client/spark_historical/StaticAnalysis/DataIOHelper.cs @@ -17,45 +17,11 @@ using Google.Protobuf.WellKnownTypes; namespace bts.history{ // secondary level helper functions (parsing, reading files etc) class DataIOHelper{ - // ../trade_analysis/date/ - public static string get_data_path (DateTime date){ - return String.Format(@"{0}{1}/{2}", - Configuration.get().data_path, - "trade_analysis", - date.ToShortDateString().Replace("/","_")); - } public static string get_data_path (string relative_path){ return String.Format(@"{0}/{1}", Configuration.get().data_path, relative_path); - } - public static string get_rawbook_data_path (){ - return String.Format(@"{0}{1}/{2}", - Configuration.get().data_path, - "trade_analysis", - "raw_books"); - } - public static string get_book_path (string symbol, - DateTime date){ - return String.Format(@"{0}{1}/{2}/{3}/{4}/{5}/", - Configuration.get().data_path, - "trade_analysis", - "market_data", - date.ToShortDateString().Replace("/","_"), - "books", - symbol); - } - - // load books from the books csv - public static IDictionary load_books(string symbol, - DateTime dt, - bool top_only){ - DateTime t1 = DateTime.Now; - string file_name = String.Format("{0}/{1}", get_data_path(dt), - String.Format("{0}_{1}", symbol,"books.csv")); - return load_books(file_name,symbol,top_only); - } - + } public static IDictionary load_books(string file_name, string symbol, bool top_only){ @@ -212,17 +178,7 @@ namespace bts.history{ #region Writing out private static string _format_timestamp(Timestamp t, bool show_date = true){ - // string[] tmtoks = t.ToString().Split('T'); - // string dt = tmtoks[0]; - // string tm = tmtoks[1]; - // string[] toks = tm.Split('.'); - // string time_secs = toks[0], - // nsecs = toks.Length == 2 ? toks[1] : "0"; - // nsecs = new string(nsecs.Where(c => char.IsDigit(c)).ToArray()); - // nsecs = String.Format("{0:n0}", Int32.Parse(nsecs)); - // //nsecs = nsecs.Replace(',','`'); - // return String.Format("{0}{1}:{2}{3}", (show_date ? dt : String.Empty),time_secs,nsecs,'"'); - return t.ToString().Replace("T"," "); + return t.ToString().Replace("T"," "); } private static string _array_to_string (IEnumerable entries){ @@ -247,17 +203,17 @@ namespace bts.history{ "a_0", "aq_0"); } - public static string leg_book_entry_to_csv(BookEntry b){ + public static string leg_book_entry_to_csv(ProcessedBook b){ return string.Format("{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11}", - b.PBook.Security, - b.PBook.MsgSeqNum, - b.PBook.BookSeqNum, - MarketQueriesHelper.book_processing_time(b.PBook), - (b.PBook.AbsorbedBooks == null || b.PBook.AbsorbedBooks.Count == 0) ? String.Empty : + b.Security, + b.MsgSeqNum, + b.BookSeqNum, + MarketQueriesHelper.book_processing_time(b), + (b.AbsorbedBooks == null || b.AbsorbedBooks.Count == 0) ? String.Empty : String.Format("{0}:{1}-{2}", - b.PBook.AbsorbedBooks.Count, - b.PBook.AbsorbedBooks.First(), - b.PBook.AbsorbedBooks.Last()), + b.AbsorbedBooks.Count, + b.AbsorbedBooks.First(), + b.AbsorbedBooks.Last()), _format_timestamp(b.TimeRange.T1), _format_timestamp(b.TimeRange.T2), MarketQueriesHelper.period_ns(b.TimeRange), @@ -285,7 +241,7 @@ namespace bts.history{ var min_max = MarketQueriesHelper.intersection_range_idx( p.leg_books.Select(lb=>lb.TimeRange).ToArray()); return string.Format("{0},{1},{2},{3},{4},{5},{6},{7},{8}", - b.PBook.Security, + b.Security, p.idx, _format_timestamp(b.TimeRange.T1), _format_timestamp(b.TimeRange.T2), @@ -298,11 +254,11 @@ namespace bts.history{ MarketQueriesHelper.book_qty(b,false,0)); } - public static string book_entry_to_csv(BookEntry b){ + public static string book_entry_to_csv(ProcessedBook b){ return string.Format("{0},{1},{2},{3},{4},{5},{6},{7},{8},{9}", - b.PBook.Security, - b.PBook.MsgSeqNum, - b.PBook.BookSeqNum, + b.Security, + b.MsgSeqNum, + b.BookSeqNum, _format_timestamp(b.TimeRange.T1), _format_timestamp(b.TimeRange.T2), MarketQueriesHelper.period_ns(b.TimeRange), diff --git a/src/client/spark_historical/StaticAnalysis/MarketQueries.cs b/src/client/spark_historical/StaticAnalysis/MarketQueries.cs index 6f3836479..bf1e0beb3 100644 --- a/src/client/spark_historical/StaticAnalysis/MarketQueries.cs +++ b/src/client/spark_historical/StaticAnalysis/MarketQueries.cs @@ -9,15 +9,27 @@ using mbts.co2; using Google.Protobuf.WellKnownTypes; namespace bts.history{ + + class BookInstance{ + public uint BookSeqNum; + public uint MsgSeqNum; + public Timestamp SendingTime; + public Timestamp TransactTime; + public string Security; + public List> bids, asks; + public List AbsorbedBooks; + public Timerange TimeRange; + } + class MarketQueries{ // works only for glbx #region Synthetic Book Calc public class SyntheticPrice{ - public readonly IEnumerable leg_books; - public readonly BookEntry synthetic_book; - public SyntheticPrice(IEnumerable leg_book_a, - BookEntry synthetic_book_a){ + public readonly IEnumerable leg_books; + public readonly ProcessedBook synthetic_book; + public SyntheticPrice(IEnumerable leg_book_a, + ProcessedBook synthetic_book_a){ leg_books = leg_book_a.ToArray(); synthetic_book = synthetic_book_a; } @@ -25,8 +37,8 @@ namespace bts.history{ public UInt16 idx = 0; public Int64 max_processing_time(){ var rv = leg_books.MaxElement(b => - MarketQueriesHelper.book_processing_time(b.PBook)); - return MarketQueriesHelper.book_processing_time(rv.PBook); + MarketQueriesHelper.book_processing_time(b)); + return MarketQueriesHelper.book_processing_time(rv); } } public static void get_synthetic_prices( @@ -42,13 +54,13 @@ namespace bts.history{ } // stop gap, till the above function gets faster - private static BookEntry _compute_synthetic_top_book( + private static ProcessedBook _compute_synthetic_top_book( SpreaderOrderDefinition order_def, - params BookEntry[] leg_books){ + params ProcessedBook[] leg_books){ double top_bid=0, top_ask=0; UInt32 top_bid_q=UInt32.MaxValue, top_ask_q=UInt32.MaxValue; for (int i=0; il.TimeRange).ToArray()); - ProcessedBook pb = new ProcessedBook(); - pb.Security = order_def.Security.Synthetic.Name; - pb.BookSeqNum = pb.MsgSeqNum = seq_no; - pb.Book = bm; - return new BookEntry {PBook = pb,TimeRange = r}; + leg_books.Select(l=>l.TimeRange).ToArray()); + return new ProcessedBook { + Security = order_def.Security.Synthetic.Name, + Book = bm, + TimeRange = r + }; } #endregion #region Alligning Books - public static IEnumerable> allign_bookseries( + public static IEnumerable> allign_bookseries( params BookSeries[] leg_book_serieses){ - var rv = new List>(); - align_bookseries(s => rv.Add(new List(s)), leg_book_serieses); + var rv = new List>(); + align_bookseries(s => rv.Add(new List(s)), leg_book_serieses); return rv; } public static void align_bookseries( - Action> on_book, + Action> on_book, params BookSeries[] leg_book_serieses){ // book series with all leg books - var all = new List>(); - BookEntry[] curr_vals = leg_book_serieses.Select(i=>(BookEntry)null).ToArray(); + var all = new List>(); + ProcessedBook[] curr_vals = leg_book_serieses.Select(i=>(ProcessedBook)null).ToArray(); Func is_ready = () => { foreach(var l in curr_vals) if(l==null) @@ -132,7 +133,6 @@ namespace bts.history{ #region Orders and Trades - static IEnumerable> compute_fifo_fills ( Timerange time_range, double order_price, diff --git a/src/client/spark_historical/StaticAnalysis/MarketQueriesHelper.cs b/src/client/spark_historical/StaticAnalysis/MarketQueriesHelper.cs index 09d34ce01..82eeb06b6 100644 --- a/src/client/spark_historical/StaticAnalysis/MarketQueriesHelper.cs +++ b/src/client/spark_historical/StaticAnalysis/MarketQueriesHelper.cs @@ -304,6 +304,19 @@ namespace bts.history{ #region book accessors + public static void fill_time_range (IEnumerable pbooks){ + for (int i=0; i= pbooks.Count() ? null : + pbooks.ElementAt(i+1); + pb_i.TimeRange = new Timerange{ + T1 = pb_i.SendingTime, + T2 = (pb_i_next == null ? pb_i.SendingTime : + pb_i_next.TransactTime) + }; + } + } + public static IEnumerable merge_duplicates_tops( IEnumerable bser){ List rv = new List(); @@ -374,19 +387,19 @@ namespace bts.history{ return rv; } - public static bool reactable (BookEntry be){ + public static bool reactable (ProcessedBook be){ return diff_ns(be.TimeRange.T2,be.TimeRange.T1) > 0; } - public static double book_price(BookEntry b, bool is_bid, UInt16 level){ - var vals = is_bid ? b.PBook.Book.Bids : b.PBook.Book.Asks; + public static double book_price(ProcessedBook b, bool is_bid, UInt16 level){ + var vals = is_bid ? b.Book.Bids : b.Book.Asks; var i = vals[level]; var iprice = new mbts.co2.IntPrice(i.Mantissa, Convert.ToSByte(i.Exponent)); return iprice.price(); } - public static UInt32 book_qty(BookEntry b, bool is_bid, UInt16 level){ - var vals = is_bid ? b.PBook.Book.Bids : b.PBook.Book.Asks; + public static UInt32 book_qty(ProcessedBook b, bool is_bid, UInt16 level){ + var vals = is_bid ? b.Book.Bids : b.Book.Asks; var i = vals[level]; return i.Size; } diff --git a/src/client/spark_historical/StaticAnalysis/SanityTests.cs b/src/client/spark_historical/StaticAnalysis/SanityTests.cs index 837a39f5a..4eb1bd061 100644 --- a/src/client/spark_historical/StaticAnalysis/SanityTests.cs +++ b/src/client/spark_historical/StaticAnalysis/SanityTests.cs @@ -1,227 +1,227 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using bts.spark; -using bts.api; -using bts.api.orderman; -using bts.co2; -using Google.Protobuf.WellKnownTypes; +// using System; +// using System.Collections.Generic; +// using System.Linq; +// using bts.spark; +// using bts.api; +// using bts.api.orderman; +// using bts.co2; +// using Google.Protobuf.WellKnownTypes; -namespace bts.history{ - class SanityTest{ - public static void run_tests(){ - Console.Write ("Tests started..."); - test_books_in_range(); - test_random_intersection(); - Console.Write ("Tests completed"); - } - public static void test_books_in_range(){ - // DateTime base_time = new DateTime(2020,1,1,9,30,45,0,DateTimeKind.Utc); - // List bseries_list = new List(); +// namespace bts.history{ +// class SanityTest{ +// public static void run_tests(){ +// Console.Write ("Tests started..."); +// test_books_in_range(); +// test_random_intersection(); +// Console.Write ("Tests completed"); +// } +// public static void test_books_in_range(){ +// // DateTime base_time = new DateTime(2020,1,1,9,30,45,0,DateTimeKind.Utc); +// // List bseries_list = new List(); - // Action add_b = (UInt32 usecs) => { - // ProcessedBook b = new ProcessedBook(); - // b.MsgSeqNum = b.BookSeqNum = Convert.ToUInt32(bseries_list.Count()); - // b.SendingTime = b.TransactTime = from_datetime(base_time, usecs); - // bseries_list.Add(b); - // }; +// // Action add_b = (UInt32 usecs) => { +// // ProcessedBook b = new ProcessedBook(); +// // b.MsgSeqNum = b.BookSeqNum = Convert.ToUInt32(bseries_list.Count()); +// // b.SendingTime = b.TransactTime = from_datetime(base_time, usecs); +// // bseries_list.Add(b); +// // }; - // add_b(1000); - // add_b(1020); - // add_b(1025); - // add_b(1035); - // add_b(1050); - // add_b(1055); - // add_b(1060); - // add_b(1065); - // add_b(1080); - // add_b(1110); +// // add_b(1000); +// // add_b(1020); +// // add_b(1025); +// // add_b(1035); +// // add_b(1050); +// // add_b(1055); +// // add_b(1060); +// // add_b(1065); +// // add_b(1080); +// // add_b(1110); - // BookSeries bser_1 = new BookSeries (bseries_list); - // var calc_bes = bser_1.book_entries(); - // Log.assert(calc_bes.Count()==10); - // Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(0).TimeRange) == 20000); - // Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(1).TimeRange) == 5000); - // Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(2).TimeRange) == 10000); - // Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(3).TimeRange) == 15000); - // Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(4).TimeRange) == 5000); - // Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(5).TimeRange) == 5000); - // Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(6).TimeRange) == 5000); - // Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(7).TimeRange) == 15000); - // Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(8).TimeRange) == 30000); - // Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(9).TimeRange) > 1000000000); +// // BookSeries bser_1 = new BookSeries (bseries_list); +// // var calc_bes = bser_1.book_entries(); +// // Log.assert(calc_bes.Count()==10); +// // Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(0).TimeRange) == 20000); +// // Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(1).TimeRange) == 5000); +// // Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(2).TimeRange) == 10000); +// // Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(3).TimeRange) == 15000); +// // Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(4).TimeRange) == 5000); +// // Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(5).TimeRange) == 5000); +// // Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(6).TimeRange) == 5000); +// // Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(7).TimeRange) == 15000); +// // Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(8).TimeRange) == 30000); +// // Log.assert(MarketQueriesHelper.period_ns(calc_bes.ElementAt(9).TimeRange) > 1000000000); - // // find in range ///////////////////////////////////////////////////// - // // test find_books_in_time_range - // // 1005-1053 - // var bitems = bser_1.find_books_in_time_range( - // new Timerange{ - // T1=from_datetime(base_time, 1005), - // T2=from_datetime(base_time, 1053)}); - // Log.assert(bitems.Count()==5); - // for (int i=0; i { - // var b1 = l.ElementAt(0); - // var b2 = l.ElementAt(1); - // var r = MarketQueriesHelper.intersection_type(b1.TimeRange, b2.TimeRange); - // Log.assert(r == TimerangeIntersection.INTERSECTION); - // }, - // bser_1, - // bser_2); - } +// // MarketQueries.align_bookseries( +// // l => { +// // var b1 = l.ElementAt(0); +// // var b2 = l.ElementAt(1); +// // var r = MarketQueriesHelper.intersection_type(b1.TimeRange, b2.TimeRange); +// // Log.assert(r == TimerangeIntersection.INTERSECTION); +// // }, +// // bser_1, +// // bser_2); +// } - public static void test_random_intersection(){ - // 3 lists - Func cons_series = (n_items,min_i,max_i,r) => { - List book_ser_list = new List(); - Random rnd = new Random(DateTime.Now.Millisecond); - int moving_usecs = 0; - for(int i = 0; i < n_items; i++){ - moving_usecs += r.HasValue ? r.Value : rnd.Next(min_i,max_i); - _add_to_ser(ref book_ser_list, - Convert.ToUInt32(moving_usecs)); - } - return new BookSeries(book_ser_list); - }; - // non ranodm data - BookSeries bser_1 = cons_series(100,1,20,20), - bser_2 = cons_series(200,2,15,10), - bser_3 = cons_series(400,2,15,5); - var res = new List>(); - MarketQueries.align_bookseries( - l => res.Add(Tuple.Create(l.ElementAt(0), - l.ElementAt(1), - l.ElementAt(2))), - bser_1, - bser_2, - bser_3); - _verify_intersection(bser_1,bser_2,bser_3,res); - // random data - int n_rnd_tests = 10; - for (int i=0;i>(); - MarketQueries.align_bookseries( - l => res.Add(Tuple.Create(l.ElementAt(0), - l.ElementAt(1), - l.ElementAt(2))), - bser_1, - bser_2, - bser_3); - _verify_intersection(bser_1,bser_2,bser_3,res); - } - } +// public static void test_random_intersection(){ +// // 3 lists +// Func cons_series = (n_items,min_i,max_i,r) => { +// List book_ser_list = new List(); +// Random rnd = new Random(DateTime.Now.Millisecond); +// int moving_usecs = 0; +// for(int i = 0; i < n_items; i++){ +// moving_usecs += r.HasValue ? r.Value : rnd.Next(min_i,max_i); +// _add_to_ser(ref book_ser_list, +// Convert.ToUInt32(moving_usecs)); +// } +// return new BookSeries(book_ser_list); +// }; +// // non ranodm data +// BookSeries bser_1 = cons_series(100,1,20,20), +// bser_2 = cons_series(200,2,15,10), +// bser_3 = cons_series(400,2,15,5); +// var res = new List>(); +// MarketQueries.align_bookseries( +// l => res.Add(Tuple.Create(l.ElementAt(0), +// l.ElementAt(1), +// l.ElementAt(2))), +// bser_1, +// bser_2, +// bser_3); +// _verify_intersection(bser_1,bser_2,bser_3,res); +// // random data +// int n_rnd_tests = 10; +// for (int i=0;i>(); +// MarketQueries.align_bookseries( +// l => res.Add(Tuple.Create(l.ElementAt(0), +// l.ElementAt(1), +// l.ElementAt(2))), +// bser_1, +// bser_2, +// bser_3); +// _verify_intersection(bser_1,bser_2,bser_3,res); +// } +// } - public static Timestamp from_datetime(DateTime dt, UInt32 usecs){ - if (dt.Millisecond != 0){ - usecs += Convert.ToUInt32(dt.Millisecond) * 1000; - dt = dt.AddMilliseconds(dt.Millisecond*-1.0); - } - Timestamp rv = new Timestamp(); - rv.Seconds = Convert.ToInt64((dt - new DateTime(1970,1,1)).TotalSeconds); - rv.Nanos = Convert.ToInt32(usecs) * 1000; - return rv; - } +// public static Timestamp from_datetime(DateTime dt, UInt32 usecs){ +// if (dt.Millisecond != 0){ +// usecs += Convert.ToUInt32(dt.Millisecond) * 1000; +// dt = dt.AddMilliseconds(dt.Millisecond*-1.0); +// } +// Timestamp rv = new Timestamp(); +// rv.Seconds = Convert.ToInt64((dt - new DateTime(1970,1,1)).TotalSeconds); +// rv.Nanos = Convert.ToInt32(usecs) * 1000; +// return rv; +// } - private static void _compare_seq_no (Tuple b, - UInt32 s1, UInt32 s2){ - Log.assert(b.Item1.PBook.BookSeqNum == s1); - Log.assert(b.Item2.PBook.BookSeqNum == s2); - } +// private static void _compare_seq_no (Tuple b, +// UInt32 s1, UInt32 s2){ +// Log.assert(b.Item1.PBook.BookSeqNum == s1); +// Log.assert(b.Item2.PBook.BookSeqNum == s2); +// } - private static void _add_to_ser(ref List bseries_list, - UInt32 usecs){ - DateTime base_time = new DateTime(2020,1,1,9,30,45,0,DateTimeKind.Utc); - ProcessedBook b = new ProcessedBook(); - b.MsgSeqNum = b.BookSeqNum = Convert.ToUInt32(bseries_list.Count()); - b.SendingTime = b.TransactTime = from_datetime(base_time, usecs); - bseries_list.Add(b); - } +// private static void _add_to_ser(ref List bseries_list, +// UInt32 usecs){ +// DateTime base_time = new DateTime(2020,1,1,9,30,45,0,DateTimeKind.Utc); +// ProcessedBook b = new ProcessedBook(); +// b.MsgSeqNum = b.BookSeqNum = Convert.ToUInt32(bseries_list.Count()); +// b.SendingTime = b.TransactTime = from_datetime(base_time, usecs); +// bseries_list.Add(b); +// } - private static void _verify_intersection ( - BookSeries bser_1, - BookSeries bser_2, - BookSeries bser_3, - IEnumerable> i_set){ - Func, int> count = (t_a) => { - int rv = 0; - foreach (var t in i_set) - if(t.Item1.PBook.BookSeqNum == t_a.Item1.PBook.BookSeqNum && - t.Item2.PBook.BookSeqNum == t_a.Item2.PBook.BookSeqNum && - t.Item3.PBook.BookSeqNum == t_a.Item3.PBook.BookSeqNum) - rv++; - return rv; - }; - // 1. uniqueness of data - // each tuple should be present only once - foreach (var t in i_set) - Log.assert(count(t)==1); - // 2. make sure each tuple in I has overlap - foreach (var t in i_set){ - Timerange r = MarketQueriesHelper.intersection_range( - t.Item1.TimeRange, - t.Item2.TimeRange, - t.Item3.TimeRange); - Log.assert(MarketQueriesHelper.period_ns(r)>0); - } - // 3. find overlaps throgh exhaustive search and verify - // they are found in I - foreach (BookEntry b1 in bser_1.book_entries()) - foreach (BookEntry b2 in bser_2.book_entries()) - foreach (BookEntry b3 in bser_3.book_entries()){ - var intr = MarketQueriesHelper.intersection_range( - b1.TimeRange, - b2.TimeRange, - b3.TimeRange); - if(MarketQueriesHelper.period_ns(intr)>0) - Log.assert(count(Tuple.Create(b1,b2,b3))==1); - } - } +// private static void _verify_intersection ( +// BookSeries bser_1, +// BookSeries bser_2, +// BookSeries bser_3, +// IEnumerable> i_set){ +// Func, int> count = (t_a) => { +// int rv = 0; +// foreach (var t in i_set) +// if(t.Item1.PBook.BookSeqNum == t_a.Item1.PBook.BookSeqNum && +// t.Item2.PBook.BookSeqNum == t_a.Item2.PBook.BookSeqNum && +// t.Item3.PBook.BookSeqNum == t_a.Item3.PBook.BookSeqNum) +// rv++; +// return rv; +// }; +// // 1. uniqueness of data +// // each tuple should be present only once +// foreach (var t in i_set) +// Log.assert(count(t)==1); +// // 2. make sure each tuple in I has overlap +// foreach (var t in i_set){ +// Timerange r = MarketQueriesHelper.intersection_range( +// t.Item1.TimeRange, +// t.Item2.TimeRange, +// t.Item3.TimeRange); +// Log.assert(MarketQueriesHelper.period_ns(r)>0); +// } +// // 3. find overlaps throgh exhaustive search and verify +// // they are found in I +// foreach (BookEntry b1 in bser_1.book_entries()) +// foreach (BookEntry b2 in bser_2.book_entries()) +// foreach (BookEntry b3 in bser_3.book_entries()){ +// var intr = MarketQueriesHelper.intersection_range( +// b1.TimeRange, +// b2.TimeRange, +// b3.TimeRange); +// if(MarketQueriesHelper.period_ns(intr)>0) +// Log.assert(count(Tuple.Create(b1,b2,b3))==1); +// } +// } - // test find_books_in_time_range - // find_coexisting_books 1 - // find_coexisting_books 2 - // align_2_bookseries - // align_3_bookseries - } -} \ No newline at end of file +// // test find_books_in_time_range +// // find_coexisting_books 1 +// // find_coexisting_books 2 +// // align_2_bookseries +// // align_3_bookseries +// } +// } \ No newline at end of file diff --git a/src/client/spark_historical/StaticAnalysis/SpreaderTickSniperAnalyzer.cs b/src/client/spark_historical/StaticAnalysis/SpreaderTickSniperAnalyzer.cs index 6e7b2654b..384d81fe4 100644 --- a/src/client/spark_historical/StaticAnalysis/SpreaderTickSniperAnalyzer.cs +++ b/src/client/spark_historical/StaticAnalysis/SpreaderTickSniperAnalyzer.cs @@ -117,7 +117,7 @@ namespace bts.history{ private static bool _detect_opp ( string n, MarketQueries.SyntheticPrice spr){ - BookEntry p = spr.synthetic_book; + ProcessedBook p = spr.synthetic_book; double b = MarketQueriesHelper.book_price(p,true,0), a = MarketQueriesHelper.book_price(p,false,0); UInt32 bq = MarketQueriesHelper.book_qty(p,true,0), diff --git a/src/lib/api/services/HistoricalDataService.proto b/src/lib/api/services/HistoricalDataService.proto index e0f95890b..caba50fa2 100644 --- a/src/lib/api/services/HistoricalDataService.proto +++ b/src/lib/api/services/HistoricalDataService.proto @@ -153,6 +153,9 @@ message ProcessedBook string security = 40; co2.BookMessage book = 50; repeated uint32 absorbed_books = 60; + // this is the book's sending time (T1) till + // the next book's transaction time + Timerange time_range = 70; } message Timerange{ @@ -160,13 +163,6 @@ message Timerange{ google.protobuf.Timestamp t2 = 20; } -message BookEntry{ - ProcessedBook p_book = 10; - // this is the book's sending time (T1) till - // the next book's transaction time - Timerange time_range = 20; -} - message TradeEntry{ string contract = 1; double price = 10;