diff --git a/.vscode/launch.json b/.vscode/launch.json index 1c6e2055a..149035fef 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -105,13 +105,14 @@ "request": "launch", "program": "${workspaceRoot}/build/Debug/bin/net471/sparkcmd.exe", "args": [ - "historical", "--server" - // "--rstart", "11/25/2019", - // "--rend", "11/29/2019", - // "--rsymbol", "ZN", - // "--rsecuritytype", "Synthetic", - // "--post_process_file_type", "Trade", - // "--upload_to_aws", + "historical", + //"--server", + "--rstart", "11/22/2019", + "--rend", "11/25/2019", + "--rsymbol", "ES", + "--rsecuritytype", "Synthetic", + "--post_process_file_type", "Trade", + "--upload_to_aws", ], "cwd": "${workspaceRoot}", "stopAtEntry": false, diff --git a/src/client/spark_excel/utils/Utilities.cs b/src/client/spark_excel/utils/Utilities.cs index 4e109d006..0ce2e3fe0 100644 --- a/src/client/spark_excel/utils/Utilities.cs +++ b/src/client/spark_excel/utils/Utilities.cs @@ -79,7 +79,7 @@ namespace bts.excel else { var dd = DateTime.FromOADate((double)dt); - var utc = dd.ToUniversalTime(); + var utc = DateTime.SpecifyKind(dd, DateTimeKind.Utc); var ts = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTime(utc); s = JsonConvert.SerializeObject(ts, Formatting.None); s = _formatter.Format(ts); diff --git a/src/client/spark_historical/BackTester.cs b/src/client/spark_historical/BackTester.cs index 28ed6d121..5c903e90e 100644 --- a/src/client/spark_historical/BackTester.cs +++ b/src/client/spark_historical/BackTester.cs @@ -22,8 +22,8 @@ namespace bts.history async public Task replay(DateTime start, DateTime end, string symbol, SecurityType st, HistoricalDataOptions o) { - start = start.ToUniversalTime(); - end = end.ToUniversalTime(); + start = DateTime.SpecifyKind(start, DateTimeKind.Utc); + end = DateTime.SpecifyKind(end, DateTimeKind.Utc); var query = new MDQuery { Exchange = Venue.Glbx, @@ -33,7 +33,7 @@ namespace bts.history End = Timestamp.FromDateTime(end), }; if(DateTime.TryParse(o.replayskip, out DateTime skip) && skip != DateTime.MinValue) - query.Skip = Timestamp.FromDateTime(skip.ToUniversalTime()); + query.Skip = Timestamp.FromDateTime(DateTime.SpecifyKind(skip, DateTimeKind.Utc)); List anal = new List(); string algo_security = o.algo_security ?? string.Empty; diff --git a/src/client/spark_historical/DataStreamer.cs b/src/client/spark_historical/DataStreamer.cs index 2d3172734..18fb91690 100644 --- a/src/client/spark_historical/DataStreamer.cs +++ b/src/client/spark_historical/DataStreamer.cs @@ -39,7 +39,7 @@ namespace bts.history where P : IParser, new() { DateTime d = request.Begin.ToDateTime(); - while (d < request.End.ToDateTime()) + while (d <= request.End.ToDateTime()) { try { @@ -107,7 +107,7 @@ namespace bts.history Action action, int peek_field = 0) { DateTime d = request.Begin.ToDateTime(); - while (d < request.End.ToDateTime()) + while (d <= request.End.ToDateTime()) { try { diff --git a/src/client/spark_historical/Query/FileGenerator.cs b/src/client/spark_historical/Query/FileGenerator.cs index d8412e825..01487b730 100644 --- a/src/client/spark_historical/Query/FileGenerator.cs +++ b/src/client/spark_historical/Query/FileGenerator.cs @@ -37,7 +37,7 @@ namespace bts.history protected override void send(DateTime dt, Symbol symbol, SecurityType sectype, MarketDataIncrementalRefresh md) { - if( (md.MdEntries.Count > 0 && md.MdEntries[0].MdEntryType == _eventtype) || _eventtype == MDEntryType.Unset) + if( md.MdEntries[0].MdEntryType == _eventtype || _eventtype == MDEntryType.Unset ) { for (int i = 0; i < md.MdEntries.Count; i++) { diff --git a/src/client/spark_historical/Query/Generator.cs b/src/client/spark_historical/Query/Generator.cs index f8003e8db..54807fbd5 100644 --- a/src/client/spark_historical/Query/Generator.cs +++ b/src/client/spark_historical/Query/Generator.cs @@ -46,8 +46,8 @@ namespace bts.history async virtual public Task generate(DateTime start, DateTime end, string symbol, SecurityType st, HistoricalDataOptions o) { - start = start.ToUniversalTime(); - end = end.ToUniversalTime(); + start = DateTime.SpecifyKind(start, DateTimeKind.Utc); + end = DateTime.SpecifyKind(end, DateTimeKind.Utc); var query = new MDQuery { Exchange = Venue.Glbx, @@ -57,7 +57,7 @@ namespace bts.history End = Timestamp.FromDateTime(end), }; if (DateTime.TryParse(o.replayskip, out DateTime skip) && skip != DateTime.MinValue) - query.Skip = Timestamp.FromDateTime(skip.ToUniversalTime()); + query.Skip = Timestamp.FromDateTime(DateTime.SpecifyKind(skip, DateTimeKind.Utc)); int count = 0; Stopwatch watch = new Stopwatch(); watch.Start(); @@ -69,7 +69,7 @@ namespace bts.history if(!line_validator(line)) { if (++count % 10_000 == 1) - Console.WriteLine($"preprocessed {count.ToString("N0") } lines"); + Console.WriteLine($"preprocessed {dt} {count.ToString("N0") } lines"); return; } var md = ProtoFix.parse_fix(line); @@ -82,7 +82,7 @@ namespace bts.history if (++count % 10_000 == 1) { var qlen = get_qlen(); - Console.WriteLine($"preprocessed {count.ToString("N0") } lines, { qlen.ToString("N0") } write queue length"); + Console.WriteLine($"preprocessed {dt} {count.ToString("N0") } lines, { qlen.ToString("N0") } write queue length"); if (qlen > 1_000_000) { var reps = 0; diff --git a/src/client/spark_historical/Query/PostGresGenerator.cs b/src/client/spark_historical/Query/PostGresGenerator.cs index 856db0dc3..666d9e35c 100644 --- a/src/client/spark_historical/Query/PostGresGenerator.cs +++ b/src/client/spark_historical/Query/PostGresGenerator.cs @@ -97,8 +97,8 @@ namespace bts.history async public Task generate(DateTime start, DateTime end, string symbol, SecurityType st, HistoricalDataOptions o) { - start = start.ToUniversalTime(); - end = end.ToUniversalTime(); + start = DateTime.SpecifyKind(start, DateTimeKind.Utc); + end = DateTime.SpecifyKind(end, DateTimeKind.Utc); var query = new MDQuery { Exchange = Venue.Glbx, @@ -108,7 +108,7 @@ namespace bts.history End = Timestamp.FromDateTime(end), }; if (DateTime.TryParse(o.replayskip, out DateTime skip) && skip != DateTime.MinValue) - query.Skip = Timestamp.FromDateTime(skip.ToUniversalTime()); + query.Skip = Timestamp.FromDateTime(DateTime.SpecifyKind(skip, DateTimeKind.Utc)); int count = 0; var comparer = Comparer.Create((x, y) diff --git a/src/client/spark_historical/Query/Query.cs b/src/client/spark_historical/Query/Query.cs index 467faaed8..8646bd3b2 100644 --- a/src/client/spark_historical/Query/Query.cs +++ b/src/client/spark_historical/Query/Query.cs @@ -47,8 +47,8 @@ namespace bts.history { IMongoQueryable res = queryable(collection); Expression> timeexpr = null; - begin = begin.ToUniversalTime(); - end = end.ToUniversalTime(); + begin = DateTime.SpecifyKind(begin, DateTimeKind.Utc); + end = DateTime.SpecifyKind(end, DateTimeKind.Utc); if(use_sending_time) timeexpr = (H h) => h.sending_time >= begin && h.sending_time <= end; else @@ -61,8 +61,8 @@ namespace bts.history where H : IHasDateTimeStamp { Expression> timeexpr = null; - begin = begin.ToUniversalTime(); - end = end.ToUniversalTime(); + begin = DateTime.SpecifyKind(begin, DateTimeKind.Utc); + end = DateTime.SpecifyKind(end, DateTimeKind.Utc); if (use_sending_time) timeexpr = (H h) => h.sending_time >= begin && h.sending_time <= end; else diff --git a/src/client/sparkcmd/Cmd/HistoricalDataCommand.cs b/src/client/sparkcmd/Cmd/HistoricalDataCommand.cs index 84a76cce4..97db469f0 100644 --- a/src/client/sparkcmd/Cmd/HistoricalDataCommand.cs +++ b/src/client/sparkcmd/Cmd/HistoricalDataCommand.cs @@ -32,7 +32,7 @@ namespace bts.history !string.IsNullOrEmpty(o.replaysymbol) ) { var start = DateTime.Parse(o.replaystart); - var end = DateTime.Parse(o.replayend).AddDays(1.0).Date; //incl + var end = DateTime.Parse(o.replayend); //incl SecurityType st = SecurityType.Future; if(!string.IsNullOrEmpty(o.replaysecuritytype)) st = (SecurityType)Enum.Parse(typeof(SecurityType), o.replaysecuritytype, true); diff --git a/src/lib/orderman/orders/algo/algo_orders/bluex/ArrivalPriceAlgo.cpp b/src/lib/orderman/orders/algo/algo_orders/bluex/ArrivalPriceAlgo.cpp index b25809732..3a5cb9c54 100644 --- a/src/lib/orderman/orders/algo/algo_orders/bluex/ArrivalPriceAlgo.cpp +++ b/src/lib/orderman/orders/algo/algo_orders/bluex/ArrivalPriceAlgo.cpp @@ -54,17 +54,14 @@ OrderOpResult ArrivalPriceAlgo::do_before_launch(IBeforeLaunchAlgoServices &serv auto sectype = admin::AdminServiceUtils::security_type(sec); const std::string& desc = bts::securities::descriptor(sec); google::protobuf::Timestamp end = google::protobuf::util::TimeUtil::GetCurrentTime(); + end.set_seconds(end.seconds() - 86400); auto days = algo.distribution_params().time_horizon_days(); if(days <= 0) return OrderOpResult{ ORDER_ERROR_INVALID_ORDER_DEFINITON, "time_horizon_days > 0" }; google::protobuf::Timestamp begin; - begin.set_seconds(end.seconds() - 86400*days); + begin.set_seconds(end.seconds() - 86400*(days+1)); - //TODO - bool b = google::protobuf::util::TimeUtil::FromString("2019-11-25T00:00:00.00Z", &begin); - b = google::protobuf::util::TimeUtil::FromString("2019-11-29T00:00:00.00Z", &end); - b = b && b; query_trading_distribution(serv, symbol, sectype, desc, begin, end, algo.duration().interval_in_seconds()); if (!serv.is_attaching()) diff --git a/src/lib/orderman/orders/algo/algo_orders/bluex/Slicer.cpp b/src/lib/orderman/orders/algo/algo_orders/bluex/Slicer.cpp index b966f1c58..5fecdabfc 100644 --- a/src/lib/orderman/orders/algo/algo_orders/bluex/Slicer.cpp +++ b/src/lib/orderman/orders/algo/algo_orders/bluex/Slicer.cpp @@ -74,7 +74,7 @@ void Slicer::query_trading_distribution(IBeforeLaunchAlgoServices& serv, const s { if(res.success()) { - auto* copy = new TradingDistribution(*res.value()); + auto copy = new TradingDistribution(*res.value()); this->trading_distribution.reset(copy); me->trading_distribution_status = TradingDistributionStatus::Received; } @@ -91,13 +91,28 @@ void Slicer::on_timer() } else if(me->trading_distribution_status == TradingDistributionStatus::Received) { - volume_estimator().update_distribution(*this->trading_distribution); - price_estimator().update_distribution(*this->trading_distribution); - update_params(); - me->trading_distribution_status = TradingDistributionStatus::Initialized; + const auto& td = *this->trading_distribution; + if(td.price_variance() == 0.0 || + os::is_nan(td.price_variance())) + { + algo().terminate_error("Zero or Nan Trading Distribution"); + } + else + { + volume_estimator().update_distribution(td); + price_estimator().update_distribution(td); + update_params(); + me->trading_distribution_status = TradingDistributionStatus::Initialized; + } } } +bool Slicer::setup_schedule() +{ + return me->trading_distribution_status == TradingDistributionStatus::NotRequired || + me->trading_distribution_status == TradingDistributionStatus::Initialized; +} + OrderAggression& Slicer::aggression() { return me->aggression; diff --git a/src/lib/orderman/orders/algo/algo_orders/bluex/Slicer.h b/src/lib/orderman/orders/algo/algo_orders/bluex/Slicer.h index 175a2ec1a..84739233f 100644 --- a/src/lib/orderman/orders/algo/algo_orders/bluex/Slicer.h +++ b/src/lib/orderman/orders/algo/algo_orders/bluex/Slicer.h @@ -35,7 +35,7 @@ protected: enum class Channel { Tactical, Aggressive, Market }; enum class TradingDistributionStatus { NotRequired, Waiting, Error, Received, Initialized}; - IAlgoServices& algo() const; + IAlgoServices& algo() const; void report_status(); bool all_children_complete(); void get_done(); @@ -52,6 +52,7 @@ protected: protected: virtual void update_params() = 0; + virtual bool setup_schedule(); virtual void compute_first_generation(Price& price, Quantity& size) = 0; virtual void compute_next_generation(Price& price, Quantity& size) = 0; @@ -83,7 +84,6 @@ protected: TradeVector& last_trades(); VolumeEstimator& volume_estimator(); PriceEstimator& price_estimator(); - bool trading_distribution_error(); void query_trading_distribution(IBeforeLaunchAlgoServices&, const std::string& symbol, api::SecurityType sectype, const std::string& secdesc, const google::protobuf::Timestamp& begin, const google::protobuf::Timestamp& end, diff --git a/src/lib/orderman/orders/algo/algo_orders/bluex/TimeSlicer.cpp b/src/lib/orderman/orders/algo/algo_orders/bluex/TimeSlicer.cpp index d8a92feb0..e59d845fd 100644 --- a/src/lib/orderman/orders/algo/algo_orders/bluex/TimeSlicer.cpp +++ b/src/lib/orderman/orders/algo/algo_orders/bluex/TimeSlicer.cpp @@ -57,7 +57,7 @@ void TimeSlicer::on_timer() return; } setup_schedule(); - return; + break; } case Slicer::Status::WORKING: { @@ -116,17 +116,20 @@ google::protobuf::Timestamp TimeSlicer::compute_end_time() } } -void TimeSlicer::setup_schedule() +bool TimeSlicer::setup_schedule() { - bts_assert(status() == Slicer::Status::WAITING_FOR_START); - status() = Status::WORKING; - me->start_time = utils::pb::pb_now(); - me->end_time = compute_end_time(); + bool ready = Slicer::setup_schedule(); + if(ready) { + bts_assert(status() == Slicer::Status::WAITING_FOR_START); + status() = Status::WORKING; + me->start_time = utils::pb::pb_now(); + me->end_time = compute_end_time(); expected_generations() = ( me->end_time.seconds() - me->start_time.seconds() ) / me->duration.interval_in_seconds(); if(me->start_time.seconds() > me->end_time.seconds() || expected_generations() <= 0 || expected_generations() > 86400) algo().terminate_error("Invalid scheduling data"); } + return ready; } diff --git a/src/lib/orderman/orders/algo/algo_orders/bluex/TimeSlicer.h b/src/lib/orderman/orders/algo/algo_orders/bluex/TimeSlicer.h index 60e8a2b7b..4a80f1fbf 100644 --- a/src/lib/orderman/orders/algo/algo_orders/bluex/TimeSlicer.h +++ b/src/lib/orderman/orders/algo/algo_orders/bluex/TimeSlicer.h @@ -14,7 +14,7 @@ public: void on_timer() override; protected: google::protobuf::Timestamp compute_end_time(); - void setup_schedule(); + bool setup_schedule() override; virtual bool check_generation_advance(); protected: diff --git a/src/lib/orderman/orders/algo/algo_orders/bluex/VolumeSlicer.cpp b/src/lib/orderman/orders/algo/algo_orders/bluex/VolumeSlicer.cpp index 19e0718cd..fdb4be7ff 100644 --- a/src/lib/orderman/orders/algo/algo_orders/bluex/VolumeSlicer.cpp +++ b/src/lib/orderman/orders/algo/algo_orders/bluex/VolumeSlicer.cpp @@ -59,7 +59,7 @@ void VolumeSlicer::on_ticker(SecurityKey k, const TradeVector& vec) case Slicer::Status::WAITING_FOR_START: { setup_schedule(); - return; + break; } case Slicer::Status::WORKING: { @@ -97,14 +97,19 @@ bool VolumeSlicer::check_generation_advance() return false; } -void VolumeSlicer::setup_schedule() +bool VolumeSlicer::setup_schedule() { - bts_assert(status() == Slicer::Status::WAITING_FOR_START); - status() = Status::WORKING; + bool ready = Slicer::setup_schedule(); + if(ready) + { + bts_assert(status() == Slicer::Status::WAITING_FOR_START); + status() = Status::WORKING; - expected_generations() = (uint32_t) duration().order_duration_in_lots() / duration().interval_in_lots(); - if(expected_generations() <= 0 || expected_generations() > 2000000) - algo().terminate_error("Invalid scheduling data"); + expected_generations() = (uint32_t) duration().order_duration_in_lots() / duration().interval_in_lots(); + if(expected_generations() <= 0 || expected_generations() > 2000000) + algo().terminate_error("Invalid scheduling data"); + } + return ready; } diff --git a/src/lib/orderman/orders/algo/algo_orders/bluex/VolumeSlicer.h b/src/lib/orderman/orders/algo/algo_orders/bluex/VolumeSlicer.h index a452f120a..672015313 100644 --- a/src/lib/orderman/orders/algo/algo_orders/bluex/VolumeSlicer.h +++ b/src/lib/orderman/orders/algo/algo_orders/bluex/VolumeSlicer.h @@ -16,7 +16,7 @@ public: void on_ticker(SecurityKey, const TradeVector&) override; protected: virtual bool check_generation_advance(); - virtual void setup_schedule(); + bool setup_schedule() override; protected: struct PImpl;