price / volume distribution looking good

This commit is contained in:
Kevin Darby committed 2020-02-12 17:30:19 -05:00
1 parent f025eed845
commit f07fd7044f
4 files changed
+67 -43

No files matched your search

+7 -7
View File
@@ -104,13 +104,13 @@
"request": "launch",
"program": "${workspaceRoot}/build/Debug/bin/net471/sparkcmd.exe",
"args": [
"historical",
"--rstart", "11/25/2019",
"--rend", "11/25/2019",
"--rsymbol", "ZN",
"--rsecuritytype", "Synthetic",
"--post_process_file_type", "Trade",
"--upload_to_aws",
"historical", "--server"
// "--rstart", "11/25/2019",
// "--rend", "11/29/2019",
// "--rsymbol", "ZN",
// "--rsecuritytype", "Synthetic",
// "--post_process_file_type", "Trade",
// "--upload_to_aws",
],
"cwd": "${workspaceRoot}",
"stopAtEntry": false,
Binary file not shown.
@@ -42,34 +42,55 @@ namespace bts.history
request.End = request.End.ToDateTime().Date.AddDays(1.0).ToTimestamp(); //inclusive
var tslices = await get_trading_slices(req);
var nslices = tslices.SliceVec.Count;
var tradingdays = tslices.SliceVec.Select(x => x.End.ToDateTime().Date).Distinct();
var ntradingdays = tradingdays.Count();
Timestamp sd = new Timestamp();
SortedList<Timestamp, SliceDistribution> slices = new SortedList<Timestamp, SliceDistribution>();
while (sd.less_or_equal(request.Begin))
Timestamp sd = request.Begin.Clone();
var slices = new SortedList<Timestamp, Tuple<SliceDistribution, Moments, Moments>>(HistoricalExtensions.timestamp_comparer);
//here we create a simulated set of slices on only the first day
var simend = request.Begin.ToDateTime().Date.AddDays(1.0).ToTimestamp();
while (sd.less(simend))
{
var next = sd.add_seconds(request.DT.Seconds);
slices.Add(sd, new SliceDistribution() { Begin = sd, End = next });
slices.Add(sd, Tuple.Create(new SliceDistribution() { Begin = sd, End = next }, new Moments(), new Moments()));
sd = next;
}
TradingDistribution dist = new TradingDistribution();
Moments pricem = new Moments(), sizem = new Moments();
const double seconds_per_day_normalize = 86_400.0;
double slices_per_day = seconds_per_day_normalize / request.DT.ToTimeSpan().TotalSeconds;
foreach(var tslice in tslices.SliceVec)
{
//yes, you can average variances, they're cumulants
dist.TotalVolume += tslice.TotalVolume;
dist.VolumeMean += tslice.AvgSize / ntradingdays;
dist.VolumeVariance += tslice.SizeVariance / ntradingdays;
dist.PriceMean += tslice.AvgPrice / ntradingdays;
dist.PriceVariance += tslice.PriceVariance / ntradingdays;
pricem.add(tslice.PriceMean);
sizem.add(tslice.TotalVolume);
if(!slices.TryGetValue(tslice.Begin, out SliceDistribution sliced))
var key = request.Begin.ToDateTime().Date + tslice.Begin.ToDateTime().TimeOfDay;
if(!slices.TryGetValue(key.ToTimestamp(), out Tuple<SliceDistribution, Moments, Moments> sliced))
throw new ArgumentException($"Can't find slice {tslice.Begin.ToString()}");
sliced.SliceVolume += tslice.TotalVolume;
sliced.VolumeMean += tslice.AvgSize / ntradingdays;
sliced.VolumeVariance += tslice.SizeVariance / ntradingdays;
sliced.PriceMean += tslice.AvgPrice / ntradingdays;
sliced.PriceVariance += tslice.PriceVariance / ntradingdays;
sliced.Item2.add(tslice.PriceMean);
sliced.Item3.add(tslice.TotalVolume);
}
dist.NormalizedDays = TimeSpan.FromSeconds(seconds_per_day_normalize).TotalDays;
dist.NumDays = ntradingdays;
dist.PriceMean = pricem.mean();
dist.PriceVariance = pricem.variance();
dist.PriceSkewness = pricem.skewness();
dist.SliceDt = request.DT;
dist.VolumeMean = sizem.mean() * slices_per_day;
dist.VolumeVariance = sizem.variance() * slices_per_day;
dist.VolumeSkewness = 0.0;
dist.VolumeTotal = sizem.total();
foreach (var tup in slices)
{
tup.Value.Item1.PriceMean = tup.Value.Item2.mean();
tup.Value.Item1.PriceVariance = tup.Value.Item2.variance();
tup.Value.Item1.VolumeMean = tup.Value.Item3.mean();
tup.Value.Item1.VolumeVariance = tup.Value.Item3.variance();
tup.Value.Item1.VolumeTotal = tup.Value.Item3.total();
}
dist.Xslices.AddRange(slices.Values.Select(x => x.Item1));
return dist;
}
@@ -96,7 +117,6 @@ namespace bts.history
QueryType = MDQuery.Types.MDQueryType.Tick,
};
Timestamp slicetime = new Timestamp();
List<Tuple<Slice, Moments, Moments>> slices = new List<Tuple<Slice, Moments, Moments>>();
Slice curr = null; int idx = 0; uint sliceidx = 0;
Moments pricem = new Moments(), sizem = new Moments();
@@ -105,18 +125,16 @@ namespace bts.history
if (data.TransactTime.greater_or_equal(req.Begin) && data.TransactTime.less_or_equal(req.End) &&
data.security_desc().ToUpper() == request.SecurityDesc.ToUpper())
{
if ((data.TransactTime - slicetime).greater(request.DT))
if ((data.TransactTime - (curr?.Begin?? new Timestamp()) ).greater(request.DT))
{
if (curr != null)
{
curr.End = request.Begin.add_seconds((idx + 1) * request.DT.Seconds);
curr.End = curr.Begin.add_seconds(request.DT.Seconds);
slices.Add(Tuple.Create(curr, pricem, sizem));
}
curr = new Slice();
curr = new Slice() { Begin = curr?.End?.Clone() ?? request.Begin.Clone() };
pricem = new Moments();
sizem = new Moments();
curr.Begin = request.Begin.add_seconds(idx * request.DT.Seconds);
slicetime = data.TransactTime;
sizem = new Moments();
sliceidx = 0;
}
if(data.Trade.AggressorSide == co2.AggressorSide.Buy)
@@ -134,16 +152,16 @@ namespace bts.history
pricem.add(data.Trade.price);
sizem.add(data.Trade.Size);
sliceidx++;
}
idx++;
idx++;
}
});
foreach(var tup in slices)
{
tup.Item1.Nsamples = (uint)tup.Item2.num_samples;
tup.Item1.AvgPrice = tup.Item2.mean();
tup.Item1.PriceMean = tup.Item2.mean();
tup.Item1.PriceVariance = tup.Item2.variance();
tup.Item1.TotalVolume = (uint)tup.Item3.total();
tup.Item1.AvgSize = tup.Item3.mean();
tup.Item1.SizeMean = tup.Item3.mean();
tup.Item1.SizeVariance = tup.Item3.variance();
}
return slices.Select(x => x.Item1).ToList();
@@ -3,7 +3,7 @@ package bts.history;
option csharp_namespace="bts.history";
option cc_enable_arenas=true;
import "lib/api/Apioptions.proto";
import "google/protobuf/wrappers.proto";
import "google/protobuf/timestamp.proto";
import "google/protobuf/duration.proto";
import "lib/domain_model/Security.proto";
@@ -78,6 +78,7 @@ message MarketDataIncrementalRefresh
repeated OrderIDEntry order_id_entries = 37705;
}
//Price / Size stats about a period of trading
message Slice
{
google.protobuf.Timestamp begin = 10;
@@ -88,11 +89,10 @@ message Slice
double avg_sell_price = 60;
uint32 total_volume = 70;
double avg_price = 80;
double price_mean = 80;
double price_variance = 90;
double avg_size = 100;
double size_mean = 100;
double size_variance = 110;
uint32 nsamples = 150;
}
message Slices
@@ -100,27 +100,33 @@ message Slices
repeated Slice slice_vec = 10;
google.protobuf.Duration d_t = 60;
}
//Aggregated from Slices, these are average period [begin, end) means and average variances over N days
message SliceDistribution
{
google.protobuf.Timestamp begin = 10;
google.protobuf.Timestamp end = 20;
double slice_volume = 25;
double volume_total = 25; //aggregate volume over N days for this slice
double volume_mean = 30;
double volume_variance = 40;
double price_mean = 50;
double price_variance = 60;
}
//Aggregated from Slices, these are 1 day average means and average variances over N days
message TradingDistribution
{
double total_volume = 5;
double volume_mean = 10; //mean / var normalized to 1 day
double volume_total = 5; //aggregate volume over N days
double volume_mean = 10;
double volume_variance = 20;
double volume_skewness = 30;
double price_mean = 40;
double price_variance = 50;
double price_skewness = 60;
google.protobuf.Duration d_t = 70;
repeated SliceDistribution slices = 100;
google.protobuf.Duration slice_dt = 70;
double normalized_days = 75;
double num_days = 80;
repeated SliceDistribution xslices = 100;
}
message SlicesQuery