From f575b82ce0baffe9b08cf7232eeaf456fb466c06 Mon Sep 17 00:00:00 2001 From: Kevin Darby Date: Fri, 30 Aug 2019 17:54:58 -0400 Subject: [PATCH] almost got the replay going --- .../sparkcmd/Cmd/Historical/DoubleWrapper.cs | 97 ++++++++++++ .../Cmd/Historical/HistoricalDataReplayer.cs | 139 ++++++++++++++++-- src/lib/api/services/MarketDataRelay.proto | 5 + src/server/sparkd/cmds/Run.cpp | 2 + .../md/MarketDataReceiverServiceServer.cpp | 59 ++++++++ .../md/MarketDataReceiverServiceServer.h | 26 ++++ 6 files changed, 319 insertions(+), 9 deletions(-) create mode 100644 src/client/sparkcmd/Cmd/Historical/DoubleWrapper.cs create mode 100644 src/server/sparkd/md/MarketDataReceiverServiceServer.cpp create mode 100644 src/server/sparkd/md/MarketDataReceiverServiceServer.h diff --git a/src/client/sparkcmd/Cmd/Historical/DoubleWrapper.cs b/src/client/sparkcmd/Cmd/Historical/DoubleWrapper.cs new file mode 100644 index 000000000..a780f8313 --- /dev/null +++ b/src/client/sparkcmd/Cmd/Historical/DoubleWrapper.cs @@ -0,0 +1,97 @@ +using System; +using System.Collections.Generic; +using System.Text; + +struct DoubleWrapper +{ + private ulong bits; + public DoubleWrapper(double d) + { + this.bits = (ulong)BitConverter.DoubleToInt64Bits(d); + } + + public int Sign + { + get + { + return this.bits.Bit(63); + } + } + + public int Exponent + { + get + { + return (int)this.bits.Bits(62, 52); + } + } + + public IEnumerable ExponentBits + { + get + { + return this.bits.BitSeq(62, 52); + } + } + + public ulong Mantissa + { + get + { + return this.bits.Bits(51, 0); + } + } + + public IEnumerable MantissaBits + { + get + { + return this.bits.BitSeq(51, 0); + } + } + + public static implicit operator DoubleWrapper(double d) + { + return new DoubleWrapper(d); + } +} + +static class Extensions +{ + public static int Bit(this ulong x, int bit) + { + return (int)((x >> bit) & 0x01); + } + + public static ulong Bits(this ulong x, int high, int low) + { + x <<= (63 - high); + x >>= (low + 63 - high); + return x; + } + + public static IEnumerable BitSeq(this ulong x, int high, int low) + { + for (int bit = high; bit >= low; --bit) + yield return x.Bit(bit); + } + + // public static Rational Exp(this Rational x, int y) + // { + // Rational result; + // Rational.Power(x, y, out result); + // return result; + // } + + // public static string ToDecimalString(this Rational x) + // { + // var sb = new StringBuilder(); + // x.AppendDecimalString(sb, 50000); + // return sb.ToString(); + // } + + public static string Join(this IEnumerable seq) + { + return string.Concat(seq); + } +} \ No newline at end of file diff --git a/src/client/sparkcmd/Cmd/Historical/HistoricalDataReplayer.cs b/src/client/sparkcmd/Cmd/Historical/HistoricalDataReplayer.cs index 66b765311..7651a2522 100644 --- a/src/client/sparkcmd/Cmd/Historical/HistoricalDataReplayer.cs +++ b/src/client/sparkcmd/Cmd/Historical/HistoricalDataReplayer.cs @@ -8,37 +8,158 @@ using System.IO; using System.Threading.Tasks; using bts.api; using Google.Protobuf.WellKnownTypes; +using bts.co2; namespace bts.history { class HistoricalDataReplayer { readonly HistoricalDataServiceServer _server; - readonly bts.api.orderman.OrderService _order_service; + readonly bts.co2.MarketDataReceiverService _mdservice; + + Google.Protobuf.WellKnownTypes.Timestamp _time; + Dictionary _channels = new Dictionary(); + public HistoricalDataReplayer(ServiceContext context) { _server = new HistoricalDataServiceServer(context); - context.locate(); + _mdservice = context.locate(); + } + + bts.api.ResultMessage send() + { + co2.MDReply md = new co2.MDReply(); + md.Channels.AddRange(_channels.Values); + var x = _mdservice.push_md(md); + _channels.Clear(); + return x; + } + + co2.ChannelUpdate create(Product prod, MDEntry md) + { + var chanup = new co2.ChannelUpdate() + { + Key = md.SecurityId, + Security = new Security() + { + Id = md.SecurityId, + Symbol = prod.Symbol, + Exchange = Venue.Glbx, + ContractSpec = new ContractSpecification() + { + MainFraction = 32, + SubFraction = 2, + PriceBreak = 0, + LowTick = 0.015625, + HighTick = 0.015625, + CashPerPoint = 1000.0, + PriceDisplayFormat = 3, + PriceMultiplier = 1.0, + }, + Glbx = new GLBXExchangeData() + { + SecurityDesc = md.Symbol, + ExchangeId = md.SecurityId, + CmeSymbol = prod.Symbol, + } + } + }; + if (prod.Type == SecurityType.Future) + { + chanup.Security.Future = new FutureContract() + { + ExpirationDay = 1, + ExpirationMonth = 9, + ExpirationYear = 2019 + }; + } + else if (prod.Type == SecurityType.Spread) + { + chanup.Security.Spread = new SpreadContract() + { + Normalized = true + }; + } + else + bts.Log.assert(false, "unknown contract type"); + return chanup; + } + + void process(Product prod, MarketDataIncrementalRefresh mdr) + { + if(!_channels.TryGetValue(prod, out co2.ChannelReply chan)) + { + chan = new co2.ChannelReply() + { + Platform = co2.Platform.Mdp3Onixs, + Channel = new co2.SecurityChannel() + { + Entitled = true, + Exchange = prod.Exchange, + Type = prod.Type, + Symbol = prod.Symbol, + Flags = (uint)co2.MDRequest.Types.Flags.Direct + }, + }; + } + foreach(var md in mdr.MdEntries) + { + var upd = chan.Updates.FirstOrDefault(x => x.Key == md.SecurityId); + if(upd == null) + { + upd = create(prod, md); + chan.Updates.Add(upd); + } + keep_book(upd, md); + } + } + + private void keep_book(ChannelUpdate upd, MDEntry md) + { + for(int i = 0; i < 10; i++) + { + upd.Direct.Bids.Add(new PriceSizeMessage() { Exponent = 0, Mantissa = 0, Size = 0 }); + upd.Direct.Asks.Add(new PriceSizeMessage() { Exponent = 0, Mantissa = 0, Size = 0}); + } } async public Task replay(DateTime start, DateTime end, string symbol, SecurityType st, int us) { - if(us < 20) - us = 20; - var str = await _server.get_md_stream(new MDQuery { Exchange = Venue.Glbx, Symbol = symbol, SecurityType = st, - Begin = Timestamp.FromDateTime(start), End = Timestamp.FromDateTime(end) } ); + var query = new MDQuery + { + Exchange = Venue.Glbx, + Symbol = symbol, + SecurityType = st, + Begin = Timestamp.FromDateTime(start), + End = Timestamp.FromDateTime(end) + }; + var str = await _server.get_md_stream(query); + api.Product prod = new Product() { Exchange = query.Exchange, Type = query.SecurityType, Symbol = query.Symbol }; foreach(var md in str.Md) { if(md.TransactTime.ToDateTime() < start) { - continue; //do we want a starting book? + //we want a starting book + process(prod, md); } else if(md.TransactTime.ToDateTime() > end) { - break; + break; } else - { + { + if (_time == null) + { + send(); //send the initial books + _time = md.TransactTime; + } + process(prod, md); + var diff = (md.TransactTime - _time).ToTimeSpan().Ticks / (TimeSpan.TicksPerMillisecond / 1000); + if( diff > us ) + { + send(); //send the next book + _time = md.TransactTime; + } } } } diff --git a/src/lib/api/services/MarketDataRelay.proto b/src/lib/api/services/MarketDataRelay.proto index 197eecae0..a49cad7d2 100644 --- a/src/lib/api/services/MarketDataRelay.proto +++ b/src/lib/api/services/MarketDataRelay.proto @@ -29,3 +29,8 @@ service MarketDataRelay { rpc request_names(bts.api.VoidMessage) returns (ExpirationNameListReply); rpc get_securities(SecurityKeyList) returns (bts.api.SecurityList); } + +service MarketDataReceiverService { + rpc push_md(bts.co2.MDReply) returns (bts.api.ResultMessage); +} + diff --git a/src/server/sparkd/cmds/Run.cpp b/src/server/sparkd/cmds/Run.cpp index c75503975..da7d64de2 100644 --- a/src/server/sparkd/cmds/Run.cpp +++ b/src/server/sparkd/cmds/Run.cpp @@ -27,6 +27,7 @@ #include "lib/utils/thread_manager.h" #include "server/sparkd/http/WebServer.h" #include "server/sparkd/md/MarketDataAuxiliaryServiceServer.h" +#include "server/sparkd/md/MarketDataReceiverServiceServer.h" #include "server/sparkd/md/run_md.h" #include "server/sparkd/position/PositionServiceServer.h" #include "server/sparkd/position/MarginServiceServer.h" @@ -87,6 +88,7 @@ int Run::execute(std::vector const &) { support::ErrorReportServer error_report_service(sc); admin::AdminServiceServer admin_service(sc); md::auxiliary::MarketDataAuxiliaryServiceServer md_aux_server(sc); + md::MarketDataReceiverServiceServer md_rec_server(sc); //orderman::AlgoServiceServer algo_server(sc); bts::api::orderman::algo::AlgoDefinitionServiceServer algo_definition_server(sc); diff --git a/src/server/sparkd/md/MarketDataReceiverServiceServer.cpp b/src/server/sparkd/md/MarketDataReceiverServiceServer.cpp new file mode 100644 index 000000000..94479ccab --- /dev/null +++ b/src/server/sparkd/md/MarketDataReceiverServiceServer.cpp @@ -0,0 +1,59 @@ +#include "MarketDataReceiverServiceServer.h" +#include "lib/config/config.h" +#include "lib/utils/TimeStamp.h" +#include "lib/utils/helper_macros.h" +#include "lib/utils/AuthToken.h" +#include "lib/domain_model/Security.h" +#include "lib/logging/logging.h" + +#include "lib/config/protos/system.pb.h" +#include "lib/md/protos/co2.pb.h" +#include "lib/md/relay/RelayClient.h" +#include "lib/md/MDClient.h" +#include "lib/md/MDProducer.h" +#include "lib/api/apicore.pb.h" +#include "lib/api/services/MarketDataRelay.server.pb.h" +#include "lib/api/services/MarketDataRelay.client.pb.h" +#include "lib/api/public/ServiceContext.h" + +using namespace bts::co2; +using std::unique_ptr; +using std::shared_ptr; + +namespace bts { +namespace md { + +shared_ptr mkrelayproducer(bool shared); + +struct MarketDataReceiverServiceServer::PImpl +{ + shared_ptr producer; + PImpl(bts::api::ServiceContext&) : producer(mkrelayproducer(true)) + { + } +}; + +MarketDataReceiverServiceServer::MarketDataReceiverServiceServer(bts::api::ServiceContext& ctx) : + bts::co2::AuthenticatedMarketDataReceiverServiceBase(ctx), me(new PImpl(ctx)) +{ +} + +MarketDataReceiverServiceServer::~MarketDataReceiverServiceServer() {} + +void MarketDataReceiverServiceServer::push_md(const api::AuthToken& token, const bts::co2::MDReply& md, + const std::function& reply) +{ + bts::api::ResultMessage res; + res.set_success(false); + + api::user::TokenPayload payload; + if (api::auth::decode_token(token, &payload)) + { + bool has_new_security = false; + me->producer->write(md, &has_new_security); + res.set_success(true); + } + reply(res); +} + +}} diff --git a/src/server/sparkd/md/MarketDataReceiverServiceServer.h b/src/server/sparkd/md/MarketDataReceiverServiceServer.h new file mode 100644 index 000000000..201c71597 --- /dev/null +++ b/src/server/sparkd/md/MarketDataReceiverServiceServer.h @@ -0,0 +1,26 @@ +#pragma once + +#include "lib/api/services/MarketDataRelay.server.pb.h" +#include "lib/utils/bts_no_copy.h" +#include + + +namespace bts +{ +namespace api {class ServiceContext;} +namespace co2 {class MDReply;} +namespace md{ + +class MarketDataReceiverServiceServer : public bts::co2::AuthenticatedMarketDataReceiverServiceBase +{ +public: + MarketDataReceiverServiceServer(bts::api::ServiceContext& context); + ~MarketDataReceiverServiceServer(); + virtual void push_md(const api::AuthToken& token, const bts::co2::MDReply& md, + const std::function& reply) override; + +private: + struct PImpl; + std::unique_ptr me; +}; +}}