diff --git a/.vscode/launch.json b/.vscode/launch.json index bbb0d3a44..4e1141172 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -104,13 +104,13 @@ "request": "launch", "program": "${workspaceRoot}/build/Debug/bin/net471/sparkcmd.exe", "args": [ - "historical", - "--rstart", "11/25/2019", - "--rend", "11/25/2019", - "--rsymbol", "ZN", - "--rsecuritytype", "Synthetic", - "--post_process_file_type", "Trade", - "--upload_to_aws", + "historical", "--server" + // "--rstart", "11/25/2019", + // "--rend", "11/29/2019", + // "--rsymbol", "ZN", + // "--rsecuritytype", "Synthetic", + // "--post_process_file_type", "Trade", + // "--upload_to_aws", ], "cwd": "${workspaceRoot}", "stopAtEntry": false, diff --git a/src/client/spark_excel/historical_query.xlsm b/src/client/spark_excel/historical_query.xlsm index 78e1edb49..d962aa936 100644 Binary files a/src/client/spark_excel/historical_query.xlsm and b/src/client/spark_excel/historical_query.xlsm differ diff --git a/src/client/spark_historical/HistoricalDataServiceServer.cs b/src/client/spark_historical/HistoricalDataServiceServer.cs index 1c1555a55..e3cd5fe17 100644 --- a/src/client/spark_historical/HistoricalDataServiceServer.cs +++ b/src/client/spark_historical/HistoricalDataServiceServer.cs @@ -42,34 +42,55 @@ namespace bts.history request.End = request.End.ToDateTime().Date.AddDays(1.0).ToTimestamp(); //inclusive var tslices = await get_trading_slices(req); + var nslices = tslices.SliceVec.Count; var tradingdays = tslices.SliceVec.Select(x => x.End.ToDateTime().Date).Distinct(); var ntradingdays = tradingdays.Count(); - Timestamp sd = new Timestamp(); - SortedList slices = new SortedList(); - while (sd.less_or_equal(request.Begin)) + Timestamp sd = request.Begin.Clone(); + var slices = new SortedList>(HistoricalExtensions.timestamp_comparer); + //here we create a simulated set of slices on only the first day + var simend = request.Begin.ToDateTime().Date.AddDays(1.0).ToTimestamp(); + while (sd.less(simend)) { var next = sd.add_seconds(request.DT.Seconds); - slices.Add(sd, new SliceDistribution() { Begin = sd, End = next }); + slices.Add(sd, Tuple.Create(new SliceDistribution() { Begin = sd, End = next }, new Moments(), new Moments())); sd = next; } TradingDistribution dist = new TradingDistribution(); + Moments pricem = new Moments(), sizem = new Moments(); + const double seconds_per_day_normalize = 86_400.0; + double slices_per_day = seconds_per_day_normalize / request.DT.ToTimeSpan().TotalSeconds; foreach(var tslice in tslices.SliceVec) { - //yes, you can average variances, they're cumulants - dist.TotalVolume += tslice.TotalVolume; - dist.VolumeMean += tslice.AvgSize / ntradingdays; - dist.VolumeVariance += tslice.SizeVariance / ntradingdays; - dist.PriceMean += tslice.AvgPrice / ntradingdays; - dist.PriceVariance += tslice.PriceVariance / ntradingdays; + pricem.add(tslice.PriceMean); + sizem.add(tslice.TotalVolume); - if(!slices.TryGetValue(tslice.Begin, out SliceDistribution sliced)) + var key = request.Begin.ToDateTime().Date + tslice.Begin.ToDateTime().TimeOfDay; + if(!slices.TryGetValue(key.ToTimestamp(), out Tuple sliced)) throw new ArgumentException($"Can't find slice {tslice.Begin.ToString()}"); - sliced.SliceVolume += tslice.TotalVolume; - sliced.VolumeMean += tslice.AvgSize / ntradingdays; - sliced.VolumeVariance += tslice.SizeVariance / ntradingdays; - sliced.PriceMean += tslice.AvgPrice / ntradingdays; - sliced.PriceVariance += tslice.PriceVariance / ntradingdays; + sliced.Item2.add(tslice.PriceMean); + sliced.Item3.add(tslice.TotalVolume); } + dist.NormalizedDays = TimeSpan.FromSeconds(seconds_per_day_normalize).TotalDays; + dist.NumDays = ntradingdays; + dist.PriceMean = pricem.mean(); + dist.PriceVariance = pricem.variance(); + dist.PriceSkewness = pricem.skewness(); + dist.SliceDt = request.DT; + dist.VolumeMean = sizem.mean() * slices_per_day; + dist.VolumeVariance = sizem.variance() * slices_per_day; + dist.VolumeSkewness = 0.0; + dist.VolumeTotal = sizem.total(); + + foreach (var tup in slices) + { + tup.Value.Item1.PriceMean = tup.Value.Item2.mean(); + tup.Value.Item1.PriceVariance = tup.Value.Item2.variance(); + tup.Value.Item1.VolumeMean = tup.Value.Item3.mean(); + tup.Value.Item1.VolumeVariance = tup.Value.Item3.variance(); + tup.Value.Item1.VolumeTotal = tup.Value.Item3.total(); + } + dist.Xslices.AddRange(slices.Values.Select(x => x.Item1)); + return dist; } @@ -96,7 +117,6 @@ namespace bts.history QueryType = MDQuery.Types.MDQueryType.Tick, }; - Timestamp slicetime = new Timestamp(); List> slices = new List>(); Slice curr = null; int idx = 0; uint sliceidx = 0; Moments pricem = new Moments(), sizem = new Moments(); @@ -105,18 +125,16 @@ namespace bts.history if (data.TransactTime.greater_or_equal(req.Begin) && data.TransactTime.less_or_equal(req.End) && data.security_desc().ToUpper() == request.SecurityDesc.ToUpper()) { - if ((data.TransactTime - slicetime).greater(request.DT)) + if ((data.TransactTime - (curr?.Begin?? new Timestamp()) ).greater(request.DT)) { if (curr != null) { - curr.End = request.Begin.add_seconds((idx + 1) * request.DT.Seconds); + curr.End = curr.Begin.add_seconds(request.DT.Seconds); slices.Add(Tuple.Create(curr, pricem, sizem)); } - curr = new Slice(); + curr = new Slice() { Begin = curr?.End?.Clone() ?? request.Begin.Clone() }; pricem = new Moments(); - sizem = new Moments(); - curr.Begin = request.Begin.add_seconds(idx * request.DT.Seconds); - slicetime = data.TransactTime; + sizem = new Moments(); sliceidx = 0; } if(data.Trade.AggressorSide == co2.AggressorSide.Buy) @@ -134,16 +152,16 @@ namespace bts.history pricem.add(data.Trade.price); sizem.add(data.Trade.Size); sliceidx++; - } - idx++; + idx++; + } }); foreach(var tup in slices) { tup.Item1.Nsamples = (uint)tup.Item2.num_samples; - tup.Item1.AvgPrice = tup.Item2.mean(); + tup.Item1.PriceMean = tup.Item2.mean(); tup.Item1.PriceVariance = tup.Item2.variance(); tup.Item1.TotalVolume = (uint)tup.Item3.total(); - tup.Item1.AvgSize = tup.Item3.mean(); + tup.Item1.SizeMean = tup.Item3.mean(); tup.Item1.SizeVariance = tup.Item3.variance(); } return slices.Select(x => x.Item1).ToList(); diff --git a/src/lib/api/services/HistoricalDataService.proto b/src/lib/api/services/HistoricalDataService.proto index 0156f08d9..f020cbdd8 100644 --- a/src/lib/api/services/HistoricalDataService.proto +++ b/src/lib/api/services/HistoricalDataService.proto @@ -3,7 +3,7 @@ package bts.history; option csharp_namespace="bts.history"; option cc_enable_arenas=true; import "lib/api/Apioptions.proto"; -import "google/protobuf/wrappers.proto"; + import "google/protobuf/timestamp.proto"; import "google/protobuf/duration.proto"; import "lib/domain_model/Security.proto"; @@ -78,6 +78,7 @@ message MarketDataIncrementalRefresh repeated OrderIDEntry order_id_entries = 37705; } +//Price / Size stats about a period of trading message Slice { google.protobuf.Timestamp begin = 10; @@ -88,11 +89,10 @@ message Slice double avg_sell_price = 60; uint32 total_volume = 70; - double avg_price = 80; + double price_mean = 80; double price_variance = 90; - double avg_size = 100; + double size_mean = 100; double size_variance = 110; - uint32 nsamples = 150; } message Slices @@ -100,27 +100,33 @@ message Slices repeated Slice slice_vec = 10; google.protobuf.Duration d_t = 60; } + +//Aggregated from Slices, these are average period [begin, end) means and average variances over N days message SliceDistribution { google.protobuf.Timestamp begin = 10; google.protobuf.Timestamp end = 20; - double slice_volume = 25; + double volume_total = 25; //aggregate volume over N days for this slice double volume_mean = 30; double volume_variance = 40; double price_mean = 50; double price_variance = 60; } + +//Aggregated from Slices, these are 1 day average means and average variances over N days message TradingDistribution { - double total_volume = 5; - double volume_mean = 10; //mean / var normalized to 1 day + double volume_total = 5; //aggregate volume over N days + double volume_mean = 10; double volume_variance = 20; double volume_skewness = 30; double price_mean = 40; double price_variance = 50; double price_skewness = 60; - google.protobuf.Duration d_t = 70; - repeated SliceDistribution slices = 100; + google.protobuf.Duration slice_dt = 70; + double normalized_days = 75; + double num_days = 80; + repeated SliceDistribution xslices = 100; } message SlicesQuery