From f07fd7044fd4b30787285fdff0b13d7d13393a18 Mon Sep 17 00:00:00 2001 From: Kevin Darby Date: Wed, 12 Feb 2020 17:30:19 -0500 Subject: [PATCH] price / volume distribution looking good --- .vscode/launch.json | 14 ++-- src/client/spark_excel/historical_query.xlsm | Bin 10311 -> 10907 bytes .../HistoricalDataServiceServer.cs | 72 +++++++++++------- .../api/services/HistoricalDataService.proto | 24 +++--- 4 files changed, 67 insertions(+), 43 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index bbb0d3a44..4e1141172 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -104,13 +104,13 @@ "request": "launch", "program": "${workspaceRoot}/build/Debug/bin/net471/sparkcmd.exe", "args": [ - "historical", - "--rstart", "11/25/2019", - "--rend", "11/25/2019", - "--rsymbol", "ZN", - "--rsecuritytype", "Synthetic", - "--post_process_file_type", "Trade", - "--upload_to_aws", + "historical", "--server" + // "--rstart", "11/25/2019", + // "--rend", "11/29/2019", + // "--rsymbol", "ZN", + // "--rsecuritytype", "Synthetic", + // "--post_process_file_type", "Trade", + // "--upload_to_aws", ], "cwd": "${workspaceRoot}", "stopAtEntry": false, diff --git a/src/client/spark_excel/historical_query.xlsm b/src/client/spark_excel/historical_query.xlsm index 78e1edb49f4d5e332159f461a9fb75790971aedb..d962aa936da310fb0f839ba71404488e2db62bc1 100644 GIT binary patch delta 3619 zcmZ8kXE+-S+l{@7&=^H+F>16GY6jKUs=d`HwTjr45+gP>DiWjiRyCtGwTXsS&8kte z5vx?oW2fKK_xrx<9rvI6$2q^wxv%R!XT&(q0$a&I|7|&zkA(~X*d)VX8Nf(w3nBKGf7JtPN))06UX8 zpybqHy|UC4Sv#cH$|c|=1hg%mv$~VcXvZ$@l-hD7L^PM5(4`Mz-pZHpR`Mu-$h}^U ztgRx$+K&-Uo{cEKK~YukP9VXseNP(m6$;7UeS29Iciuo+x1FvB=HV`DN%*yX_c$+4 zkc~Iam@{M5_FCIql0&@+N4)%uCTG;yL-3CPzHUD-zD+us-kEX54s?^Es ztTt8hXkdCZR4w!2)l1j{a^((6ea5pJCUbbWcB;J60q%TCshwtSrH|*laOc%Bcav}r z;NECBc_TYmb@#z$>@8jAcdyxcbfCzHM2RNiv+lAP{z#NX%5z)>!(83ln{4GuW+ItC zDvTciDXhAKYD~@0R)3|h`mBbSSpmWR30UB~mpbJnbI`VnUd9lXapS|Dtz^oB*7jw+ z3Vb}pEUpN3O^#$vL&IFR^Wu8{-8u=2SF(U03lnS#l%VvSD2=BUjtu8R^C3GOCa4M@ z#Gxzi=Tuij9Z__ObO)n$67;`|*6h`ytIl8W($S=*eH9)BS#jOaksn&{6lQ5|Y8TG; zhBUcRv7{U*2RX8Nha;5x`E4z?AMsw9^b6t6(k&e6A_ZUYuf==j3{V~ayy~GmVMUb1 zczk!jE_=Q`iSaw?^S%k5)Ix6aApGBtXkVTNzWu9 z65b2>lgo01n3y55vhw+fX3zqY!wT2y;@i_Q3Flf9Ru#T|mkzfy50DTk>t8&>zd>-@ zTUFy#REAuu)_N}PqPRABcb-bD49&t!3OBe&9O}6Fal)|hyPGqI(c)8W8B|t2d->)j z>GTdR<}z-xmK|G+s8Rl_N}qvcL!+nB{=le?dEr$ay%w9HxflD!W2-TvU$Yim3bMY6 zfnXx!+2&dRAr>=6@bSzX@9yimR9%Stjs3mg&B(ZK#Lku=_2$Ac^yjVy$Hrtu*~+-` z1aDvwjoCrqR;VFj|7_ev&utur;8pV2qN&+-31O>TOAI|7Pu=b4L4IjZ<&(a4JwrC6 zYPRIQvlNJ~9!WQ#YCdq2M2}I!q%*M5Go7T*jZgysGu#*g2OlzL*LP7K*w@Rk^}E>P z@svQiE*^2CF}c|^3*c=m3~$>^));H9`vEx#P7{WX$(eP zWf^12lV1im{!rZ+S_YT1jAaNz_^<65S&@`Jw!}Gb1PSQw%kN&7VVWy`vPlCC1?jnp zA5mwIi5b3^j3h?%g|8DjhZAIG5_+IBW8YHEJX)u9BK-F+>VBk!*8xUHb! zW$)Zw^`firNA3wEqo+ELJwCl~hpz64e(=Xo!Qr9zpHES;xo-oD)U+^Y2e)3oR87ts z1>nPOO8@Gpi_fvzs=UomH5G;IAazb^ohTf#8F&NqyFVPsP$b+@1f!aU&%} zT5CFi-`GvEfP}_P&cxinZKaoWD}9$bjVuz$fd=1Vm3pIs`j)moJ5*p;@3YhxCW+s6 zR!F8;=7*a2V)`gK)#~GZq+Y-S!WBsn(oxBhyXdeJQPuq!-7n4=$HfPtw1WqnB~ho{ z1Z1@b1X&#d-K?Vp9~^|z&=I@W3Ry_YUBI?epu)2g0ot~6zcrfvNt@r+0Y}dG*^|P2 zD1U7J_85mL#t0d9vgv--E@=k$AzU!|_@HHY9DGz)&yyyNG{hM8{r7rno!7>FhS70X z^bd0nZa!^~x<zBfFSq@htel@g_G9F;Nam>9OCtIK?imZ}>%vMDnZ_Is^UcxK0bAqr(J@MVLj4+)+f) zPpyO6SqgzWOgN+Bgx163p(`~c079u<_?d@0@NU9=#}UogNPFfe{V0j?7aLo`0P4va(npz4)>J}~;SV3p3SyzO}V1r2E3OxI$$Fzv6 zSr*XYIB#4-@X(#x!9qYGiaC9b%hB&b<-(3 z%~3BgJ4Ek3OnVXx<7$^M7jCvttICig3a+`rQa(5( z2W^yHQ%ftYhZ2a6tNJ1O4YD(gc%1* zN@VF<4XddGUsUW~q`4n}_n&_hwav^nYYH^xFw^M#*6Gcoj+E*WsBHg?&R;&zTz&)4 zoo$`XLYXFyx^O&%SLil|!L)Nkr0xvMW!HKr0;`9$CS5OoYNe;Cy}UuUv-G0fFOndi}#^aBZ${KATiHG`&t68gD53 zg$9Rnb@G19Dpn0_vADp*rPptZEiD&M}Z{X<))^_ zyYXI3cs|N4m4^_esaRldsw*X0$ug}E%`#US#_h}{`>X5oI>{cU`alx1Aipeh2dg~? zcy<@L-Ucz;b8)k2cgYr+PwmT<+zv&nUnysjx1-jw`Sh{OBJtJfU@oD z@73y7l~M-S$V?prq}E>%_*Qpfce_hqO<9gz@ekSj2jw#6{K>gnJ~_rlsqu~C2|URc zc~{{}Tg-p{8x{9p9OwO}W*%8Gq=5;f+1_|QS(?@o|=W`9T1z?gI9ZR-r)`zec zdQV3m)7yb`aCauzpYDc_-G)D5$n6qV&!sgo$hU2OJdSrr6Jk z*vk*mo|#?tAP_e!Q!}@HMF5!{AbsoAo<@SuUIzZKcIt7%=6J(E$sGN+uWK$!9ONFf z&~@yXf6UlM+M8ob1A`GF^4f5z~i`7(A7%?aTg<0bs0xOQI z2gve7k2Yt=`b(pop5y(Q0-_Rxcry=<22w(V2lqYu8H}iwfz6C3{s)xY1#}&x;A=I6Tqa1s*u|XVn#*H$e##f_{0RM J1w{W4{s)iozd8T_ delta 3019 zcmZ9OXE+;-8ipf6QCp0nM$DjU*Qmy(Hm$v@Mi6_|NGh}xH5yfW#NLz=REyNAO{-L` z(i*X9Z>L|+IoEg2_x^c*yw9)udamoeGfjF7Ym2W^Fxjzqst^GHia<>5Ren@2rqjAn zT&-whq>90Ta>m#v6lxKIh(s9sjn`d-DioJ)EFODOFV8Y~`3m~Jz7qG*HJAF7SX)U4 zBba?6^?kR3O?!kqJLa3~Hk?XJpyN0_dPQsT^UC-@^G!%f8$BNoy~Im+ApIs#Y4nS<9$k-Z0v zgMoU+#Ut0O4x1T!&Rw22xS2KoY}za?5t8U^3$AI0TU$H`%%bjw`E=aor|Q> z)D)$0k;14vydbKnPd-bdLg3v(=CW9h=55O-8v->85)_L)U3e?^&I@VrpN`OV%`hRY zyU7BbZ8?-{a=`X)-!eD}PA8IydKv3`}Z`wB{D z%};Lx2mqXs0svG103g`orl0>K4<|p-V1!4Z3BoH=jP~LV{IX*w=%Ws|4kXVq17fmP zuDwwh9?;Hfdfpko+FN{nm>mJa>J$3keK_zzZZ_K+2WbKRm!!WUS1k? zvuF&>BD+>)WtQ;*+xQ9;L_D0?j23$q8dcP5M+}KUdI$-aVIfZuEH2J-4i3CKKlV+9 z^kmpe8NJg59Kj$g$vOn)E4yiOJ9+pK}I9I%+1)R(O_hwUh*x%+gm2Ub==u9B&pv zs@{{2E`BT2&z{OAoD|lfbi>{MMxsO60{!AyuTU+hJ*k@umv8Cpt%SdDuu~2uZyUw? zBX0(KNl%?BOEu325{(C=3jB%MWY>Jeio{Ebh_D%mpg`$z^xSg^Qti}k5*{l)@}{yq zt{$pF%)Oy4#CxQ$LG3hspT@&ZYi9DNLroC&#-!fbjbA5`UAfr(Nrp+<4RmEa}T6yf27h7GvJb`0>a7ZyVy>D|9M``3$G7nvx2IqEvXw83Wn?cBzyut zVs zDpx1SSokBqTn{xaj?~)5y|25zHta$9H6xQXhq1DBRHWm5wP`LBp+dy_=d9ai)UuQ> z)2Q2q+QJXFwjm~W+j*Od9Nmh`G?ST!Pn`GgPJ<5ERqm;XvvXIS?WK|Nty%t&9A7(- z_QIRyrFYN!sy&*&N>6e7!EtO32mpYCF-^>zs5FGvv&L*PalGN8DaL z<+Qpb)atRcAbqpdZg&#j#|g+YA$0-jr^rJNs73O1g%{C^{$0SH)PY5#v&)3NU)^4} zjAAyS2EEWa7Zi0;Z>vG(2t7@TzWe0&LC!dcZezp3mmnqG2 z>kE9iEBae~u z#+YV8OgF@@6TU(geBsZ?yEPv$Vo zHWu)$$}3=ZARLU#DjHzE;a%cFa}qZ6)N^GX1(!bnrbVP&_gOP&*d6MHgzM!dyj02s zo+PDKCwnv9k}9mu4h+A7f2A_8LGPQWUm zNEuvLJYGjr-L6Ad1iDWW0#5NnIG-O_cmxlmp70d+c}%8n-N3DWE(&FqB!h*-vz-#5#i$7O*u& zL^T*pd>+UcuJ#nMa_71c9?v9_LbaCA;!eHb3FH5G)ExqAeb)e+&V{9(E}7CFG%lk> z9X_Mk9#DESFa%GrBz_{8wiMSL8rQq@n^NlRyK^VeZ)M-tShdm8N^m7(UqiW*b!$rR zMT*^4*pf+_a_6cr6O@~?j~_fHn5`rg!_Id$Z&evJLgQL@!>Q6~reNf@o>fW6<$OD| zp^S((*lyD?`2lCwiznftbrh#Vq9zsJG`(PvFQpcN9L3F_50d`!nHrOg)7e$D}OZdDAf2+ zn21&9QK;z^M8v9fMN&G6rfHiGB{}8Yx)lG0(2eKk9t<4rZiDn4qcV|jzIcjna=<5{ zwb2R${nv`El43ar8`+?gc4*CocIpJ<2odh?w$B>q5;qxlB$m*n8GtT&eBXQSfuVPF z=)B(*gQZ-`z&?E6GHA!%M%mW*7WpG0#fGUbl7vgZioNU(`(WKO-g}HdIT}V_Hxd%Kq()r$W_cp3OylhQYHF&!jA0CF#CR@_lRPqbY~IVa|Mg?PkJ5 z*PCZqUuNeBqI-^6LDF}Av^%OetXO!jV0IE6(*meZLk;nFE9 z0O}j%{c&ivk5*TQ8>Q{rK+GgB0#h>S5pB1?jy+9zX*y`tD?T%pU17mrsbu0&uc_wa zliVv&?UEh0@rq(H40P9%(FR((PMJhpvQJ6Du3O$2g-1Pv4xvLAeBMBo_DLkU!u7?W zDRkoZrC9~JYpvBfEnzWqJ0(Zq`+E|VtU5DuvBc#R9nhxsF!u`0$-6f57DexE@mY3H zRBQNRTJe4UqM$L_un!S2Dqwk17}V?dT(wP1%|t_j9wWm4D7#3i6??p)2|F}HaUw)Y z7xz%K;|)W;+Me*7;TQoWlll5B!|*@+P#($tYVUr#jqYz_r2V(!I68PZs=7MBJ^#`i z>m)0RPBB{bOOiv{pLXLh6Ib-&j?5!Oxe6VvdJA}!{SP(Z#0M_+Lj%^E+1L=cdd^Ci z3)Pjzfya5tjhnjo1!Svepi&rZG1EuZrELuw6S-VQW5Gvm>MmOM#d1{5*;G(@pZb~J zZ0k%HJi|l>SVm2@JUyAsFAa}lX~0&9^p_<74VhlZQZsiy_8$e&?pOKG_v}ArnABCi zLoyAG5l4>Qb%c?KXk6OeMRpfRvQ_a$OP%MR{joSWq=)2xeOO-57)VKlNf0yuQd48* z1o_zhsd&E${Ab26oKP;dzh5E%!1CYIpFN1NgbK6&so!sOR=+io?*H!j?3jEg5Bop) g{+(gt#LPo)1FvvlIE3y2sUa9YAqeRp?_Ys`0Rhv7R{#J2 diff --git a/src/client/spark_historical/HistoricalDataServiceServer.cs b/src/client/spark_historical/HistoricalDataServiceServer.cs index 1c1555a55..e3cd5fe17 100644 --- a/src/client/spark_historical/HistoricalDataServiceServer.cs +++ b/src/client/spark_historical/HistoricalDataServiceServer.cs @@ -42,34 +42,55 @@ namespace bts.history request.End = request.End.ToDateTime().Date.AddDays(1.0).ToTimestamp(); //inclusive var tslices = await get_trading_slices(req); + var nslices = tslices.SliceVec.Count; var tradingdays = tslices.SliceVec.Select(x => x.End.ToDateTime().Date).Distinct(); var ntradingdays = tradingdays.Count(); - Timestamp sd = new Timestamp(); - SortedList slices = new SortedList(); - while (sd.less_or_equal(request.Begin)) + Timestamp sd = request.Begin.Clone(); + var slices = new SortedList>(HistoricalExtensions.timestamp_comparer); + //here we create a simulated set of slices on only the first day + var simend = request.Begin.ToDateTime().Date.AddDays(1.0).ToTimestamp(); + while (sd.less(simend)) { var next = sd.add_seconds(request.DT.Seconds); - slices.Add(sd, new SliceDistribution() { Begin = sd, End = next }); + slices.Add(sd, Tuple.Create(new SliceDistribution() { Begin = sd, End = next }, new Moments(), new Moments())); sd = next; } TradingDistribution dist = new TradingDistribution(); + Moments pricem = new Moments(), sizem = new Moments(); + const double seconds_per_day_normalize = 86_400.0; + double slices_per_day = seconds_per_day_normalize / request.DT.ToTimeSpan().TotalSeconds; foreach(var tslice in tslices.SliceVec) { - //yes, you can average variances, they're cumulants - dist.TotalVolume += tslice.TotalVolume; - dist.VolumeMean += tslice.AvgSize / ntradingdays; - dist.VolumeVariance += tslice.SizeVariance / ntradingdays; - dist.PriceMean += tslice.AvgPrice / ntradingdays; - dist.PriceVariance += tslice.PriceVariance / ntradingdays; + pricem.add(tslice.PriceMean); + sizem.add(tslice.TotalVolume); - if(!slices.TryGetValue(tslice.Begin, out SliceDistribution sliced)) + var key = request.Begin.ToDateTime().Date + tslice.Begin.ToDateTime().TimeOfDay; + if(!slices.TryGetValue(key.ToTimestamp(), out Tuple sliced)) throw new ArgumentException($"Can't find slice {tslice.Begin.ToString()}"); - sliced.SliceVolume += tslice.TotalVolume; - sliced.VolumeMean += tslice.AvgSize / ntradingdays; - sliced.VolumeVariance += tslice.SizeVariance / ntradingdays; - sliced.PriceMean += tslice.AvgPrice / ntradingdays; - sliced.PriceVariance += tslice.PriceVariance / ntradingdays; + sliced.Item2.add(tslice.PriceMean); + sliced.Item3.add(tslice.TotalVolume); } + dist.NormalizedDays = TimeSpan.FromSeconds(seconds_per_day_normalize).TotalDays; + dist.NumDays = ntradingdays; + dist.PriceMean = pricem.mean(); + dist.PriceVariance = pricem.variance(); + dist.PriceSkewness = pricem.skewness(); + dist.SliceDt = request.DT; + dist.VolumeMean = sizem.mean() * slices_per_day; + dist.VolumeVariance = sizem.variance() * slices_per_day; + dist.VolumeSkewness = 0.0; + dist.VolumeTotal = sizem.total(); + + foreach (var tup in slices) + { + tup.Value.Item1.PriceMean = tup.Value.Item2.mean(); + tup.Value.Item1.PriceVariance = tup.Value.Item2.variance(); + tup.Value.Item1.VolumeMean = tup.Value.Item3.mean(); + tup.Value.Item1.VolumeVariance = tup.Value.Item3.variance(); + tup.Value.Item1.VolumeTotal = tup.Value.Item3.total(); + } + dist.Xslices.AddRange(slices.Values.Select(x => x.Item1)); + return dist; } @@ -96,7 +117,6 @@ namespace bts.history QueryType = MDQuery.Types.MDQueryType.Tick, }; - Timestamp slicetime = new Timestamp(); List> slices = new List>(); Slice curr = null; int idx = 0; uint sliceidx = 0; Moments pricem = new Moments(), sizem = new Moments(); @@ -105,18 +125,16 @@ namespace bts.history if (data.TransactTime.greater_or_equal(req.Begin) && data.TransactTime.less_or_equal(req.End) && data.security_desc().ToUpper() == request.SecurityDesc.ToUpper()) { - if ((data.TransactTime - slicetime).greater(request.DT)) + if ((data.TransactTime - (curr?.Begin?? new Timestamp()) ).greater(request.DT)) { if (curr != null) { - curr.End = request.Begin.add_seconds((idx + 1) * request.DT.Seconds); + curr.End = curr.Begin.add_seconds(request.DT.Seconds); slices.Add(Tuple.Create(curr, pricem, sizem)); } - curr = new Slice(); + curr = new Slice() { Begin = curr?.End?.Clone() ?? request.Begin.Clone() }; pricem = new Moments(); - sizem = new Moments(); - curr.Begin = request.Begin.add_seconds(idx * request.DT.Seconds); - slicetime = data.TransactTime; + sizem = new Moments(); sliceidx = 0; } if(data.Trade.AggressorSide == co2.AggressorSide.Buy) @@ -134,16 +152,16 @@ namespace bts.history pricem.add(data.Trade.price); sizem.add(data.Trade.Size); sliceidx++; - } - idx++; + idx++; + } }); foreach(var tup in slices) { tup.Item1.Nsamples = (uint)tup.Item2.num_samples; - tup.Item1.AvgPrice = tup.Item2.mean(); + tup.Item1.PriceMean = tup.Item2.mean(); tup.Item1.PriceVariance = tup.Item2.variance(); tup.Item1.TotalVolume = (uint)tup.Item3.total(); - tup.Item1.AvgSize = tup.Item3.mean(); + tup.Item1.SizeMean = tup.Item3.mean(); tup.Item1.SizeVariance = tup.Item3.variance(); } return slices.Select(x => x.Item1).ToList(); diff --git a/src/lib/api/services/HistoricalDataService.proto b/src/lib/api/services/HistoricalDataService.proto index 0156f08d9..f020cbdd8 100644 --- a/src/lib/api/services/HistoricalDataService.proto +++ b/src/lib/api/services/HistoricalDataService.proto @@ -3,7 +3,7 @@ package bts.history; option csharp_namespace="bts.history"; option cc_enable_arenas=true; import "lib/api/Apioptions.proto"; -import "google/protobuf/wrappers.proto"; + import "google/protobuf/timestamp.proto"; import "google/protobuf/duration.proto"; import "lib/domain_model/Security.proto"; @@ -78,6 +78,7 @@ message MarketDataIncrementalRefresh repeated OrderIDEntry order_id_entries = 37705; } +//Price / Size stats about a period of trading message Slice { google.protobuf.Timestamp begin = 10; @@ -88,11 +89,10 @@ message Slice double avg_sell_price = 60; uint32 total_volume = 70; - double avg_price = 80; + double price_mean = 80; double price_variance = 90; - double avg_size = 100; + double size_mean = 100; double size_variance = 110; - uint32 nsamples = 150; } message Slices @@ -100,27 +100,33 @@ message Slices repeated Slice slice_vec = 10; google.protobuf.Duration d_t = 60; } + +//Aggregated from Slices, these are average period [begin, end) means and average variances over N days message SliceDistribution { google.protobuf.Timestamp begin = 10; google.protobuf.Timestamp end = 20; - double slice_volume = 25; + double volume_total = 25; //aggregate volume over N days for this slice double volume_mean = 30; double volume_variance = 40; double price_mean = 50; double price_variance = 60; } + +//Aggregated from Slices, these are 1 day average means and average variances over N days message TradingDistribution { - double total_volume = 5; - double volume_mean = 10; //mean / var normalized to 1 day + double volume_total = 5; //aggregate volume over N days + double volume_mean = 10; double volume_variance = 20; double volume_skewness = 30; double price_mean = 40; double price_variance = 50; double price_skewness = 60; - google.protobuf.Duration d_t = 70; - repeated SliceDistribution slices = 100; + google.protobuf.Duration slice_dt = 70; + double normalized_days = 75; + double num_days = 80; + repeated SliceDistribution xslices = 100; } message SlicesQuery