From 5a5de65292e472a66fae233a90b9682342033a0c Mon Sep 17 00:00:00 2001 From: Kevin Darby Date: Mon, 17 Dec 2018 13:56:24 -0500 Subject: [PATCH 01/85] OCO and Pricer Algos --- .../Orders/OrderModificationControl.xaml.cs | 4 + .../Orders/OrderTypeVMs/OrderTypeVMFactory.cs | 4 + .../view/Spark/Utilities/OrderDataHelper.cs | 18 ++ src/lib/api/services/OrderService.proto | 16 ++ .../orderman/events/OrderStateUpdateEvent.h | 6 + .../orderman/order_factory/OrderFactory.cpp | 10 +- .../orderman/orders/book_algos/BookAlgo.cpp | 93 ++++++++-- src/lib/orderman/orders/book_algos/BookAlgo.h | 10 +- .../orderman/orders/book_algos/CoverOrder.cpp | 2 +- .../orderman/orders/book_algos/CoverOrder.h | 2 +- .../orderman/orders/book_algos/OCOOrder.cpp | 166 ++++++++++++++++++ src/lib/orderman/orders/book_algos/OCOOrder.h | 35 ++++ .../orders/book_algos/OffsetOrder.cpp | 2 +- .../orders/book_algos/PricerOrder.cpp | 40 +++++ .../orderman/orders/book_algos/PricerOrder.h | 28 +++ 15 files changed, 415 insertions(+), 21 deletions(-) create mode 100644 src/lib/orderman/orders/book_algos/OCOOrder.cpp create mode 100644 src/lib/orderman/orders/book_algos/OCOOrder.h create mode 100644 src/lib/orderman/orders/book_algos/PricerOrder.cpp create mode 100644 src/lib/orderman/orders/book_algos/PricerOrder.h diff --git a/src/client/spark/Orders/OrderModificationControl.xaml.cs b/src/client/spark/Orders/OrderModificationControl.xaml.cs index 6a0c9b18e..721fcaad3 100644 --- a/src/client/spark/Orders/OrderModificationControl.xaml.cs +++ b/src/client/spark/Orders/OrderModificationControl.xaml.cs @@ -25,6 +25,8 @@ namespace mbts.ui.spark { public DataTemplate JoinOrderTemplate { get; set; } public DataTemplate TopOrderTemplate { get; set; } public DataTemplate CoverOrderTemplate { get; set; } + public DataTemplate OcoOrderTemplate { get; set; } + public DataTemplate PricerOrderTemplate { get; set; } public DataTemplate OffsetOrderTemplate { get; set; } public DataTemplate TriggerOrderTemplate { get; set; } public DataTemplate IcebergOrderTemplate { get; set; } @@ -43,6 +45,8 @@ namespace mbts.ui.spark { case CustomOrderDefinition.CustomOneofCase.Join: return JoinOrderTemplate; case CustomOrderDefinition.CustomOneofCase.Top: return TopOrderTemplate; case CustomOrderDefinition.CustomOneofCase.Cover: return CoverOrderTemplate; + case CustomOrderDefinition.CustomOneofCase.Oco : return OcoOrderTemplate; + case CustomOrderDefinition.CustomOneofCase.Pricer: return PricerOrderTemplate; case CustomOrderDefinition.CustomOneofCase.Offset: return OffsetOrderTemplate; case CustomOrderDefinition.CustomOneofCase.Trigger: return TriggerOrderTemplate; case CustomOrderDefinition.CustomOneofCase.Iceberg: return IcebergOrderTemplate; diff --git a/src/client/spark/Orders/OrderTypeVMs/OrderTypeVMFactory.cs b/src/client/spark/Orders/OrderTypeVMs/OrderTypeVMFactory.cs index 495aa46b9..40f4d857f 100644 --- a/src/client/spark/Orders/OrderTypeVMs/OrderTypeVMFactory.cs +++ b/src/client/spark/Orders/OrderTypeVMs/OrderTypeVMFactory.cs @@ -20,6 +20,8 @@ namespace mbts.spark { public static IOrderTypeVM join(OrdersModel model, IObservable security_md_source, string id, string display_name) => new JoinOrderVM(model, security_md_source, id, display_name); public static IOrderTypeVM top(OrdersModel model, IObservable security_md_source, string id, string display_name) => new TopOrderVM(model, security_md_source, id, display_name); public static IOrderTypeVM cover(OrdersModel model, IObservable security_md_source, string id, string display_name) => new CoverOrderVM(model, security_md_source, id, display_name); + public static IOrderTypeVM oco(OrdersModel model, IObservable security_md_source, string id, string display_name) => new OCOOrderVM(model, security_md_source, id, display_name); + public static IOrderTypeVM pricer(OrdersModel model, IObservable security_md_source, string id, string display_name) => new PricerOrderVM(model, security_md_source, id, display_name); public static IOrderTypeVM offset(OrdersModel model, IObservable security_md_source, string id, string display_name) => new OffsetOrderVM(model, security_md_source, id, display_name); public static IOrderTypeVM trigger(OrdersModel model, IObservable security_md_source, string id, string display_name) => new TriggerOrderVM(model, security_md_source, id, display_name); public static IOrderTypeVM iceberg(OrdersModel model, IObservable security_md_source, string id, string display_name) => new IcebergOrderVM(model, security_md_source, id, display_name); @@ -39,6 +41,8 @@ namespace mbts.spark { case CustomOneofCase.Join: return join(model, security_md_source, id, display_name); case CustomOneofCase.Top: return top(model, security_md_source, id, display_name); case CustomOneofCase.Cover: return cover(model, security_md_source, id, display_name); + case CustomOneofCase.Oco : return oco(model, security_md_source, id, display_name); + case CustomOneofCase.Pricer: return pricer(model, security_md_source, id, display_name); case CustomOneofCase.Offset: return offset(model, security_md_source, id, display_name); case CustomOneofCase.Trigger: return trigger(model, security_md_source, id, display_name); case CustomOneofCase.Iceberg: return iceberg(model, security_md_source, id, display_name); diff --git a/src/client/spark/view/Spark/Utilities/OrderDataHelper.cs b/src/client/spark/view/Spark/Utilities/OrderDataHelper.cs index e9ef09d5b..75357ad10 100644 --- a/src/client/spark/view/Spark/Utilities/OrderDataHelper.cs +++ b/src/client/spark/view/Spark/Utilities/OrderDataHelper.cs @@ -37,6 +37,7 @@ namespace mbts.spark.utilities { CustomOrderDefinition.CustomOneofCase.Lean, CustomOrderDefinition.CustomOneofCase.Join, CustomOrderDefinition.CustomOneofCase.Cover, + CustomOrderDefinition.CustomOneofCase.Oco, CustomOrderDefinition.CustomOneofCase.Offset, CustomOrderDefinition.CustomOneofCase.Top, CustomOrderDefinition.CustomOneofCase.Trigger, @@ -227,6 +228,15 @@ namespace mbts.spark.utilities { Child = custom_order(), Hedge = custom_order(), }; + public OCOOrderDefinition oco_order() => new OCOOrderDefinition + { + Child = custom_order(), + Other = custom_order(), + }; + public PricerOrderDefinition pricer_order() => new PricerOrderDefinition + { + Child = custom_order(), + }; public OffsetOrderDefinition offset_order() => new OffsetOrderDefinition { Child = custom_order(), @@ -261,6 +271,8 @@ namespace mbts.spark.utilities { case CustomOrderDefinition.CustomOneofCase.Join: rv.Join = join_order(); break; case CustomOrderDefinition.CustomOneofCase.Top: rv.Top = top_order(); break; case CustomOrderDefinition.CustomOneofCase.Cover: rv.Cover = cover_order(); break; + case CustomOrderDefinition.CustomOneofCase.Oco: rv.Oco = oco_order(); break; + case CustomOrderDefinition.CustomOneofCase.Pricer: rv.Oco = pricer_order(); break; case CustomOrderDefinition.CustomOneofCase.Offset: rv.Offset = offset_order(); break; case CustomOrderDefinition.CustomOneofCase.Trigger: rv.Trigger = trigger_order(); break; case CustomOrderDefinition.CustomOneofCase.Spreader: rv.Spreader = spreader_order(); break; @@ -284,6 +296,8 @@ namespace mbts.spark.utilities { case CustomOrderDefinition.CustomOneofCase.Join: return new OrderMetadata { KeyBinding = "J", Color = BTSColors.Orders.join_order }; case CustomOrderDefinition.CustomOneofCase.Top: return new OrderMetadata { KeyBinding = "D", Color = BTSColors.Orders.top_order }; case CustomOrderDefinition.CustomOneofCase.Cover: return new OrderMetadata { KeyBinding = "C", Color = BTSColors.Orders.cover_order }; + case CustomOrderDefinition.CustomOneofCase.Oco: return new OrderMetadata { KeyBinding = "C", Color = BTSColors.Orders.cover_order }; + case CustomOrderDefinition.CustomOneofCase.Pricer: return new OrderMetadata { KeyBinding = "C", Color = BTSColors.Orders.cover_order }; case CustomOrderDefinition.CustomOneofCase.Offset: return new OrderMetadata { KeyBinding = "O", Color = BTSColors.Orders.tick_order }; case CustomOrderDefinition.CustomOneofCase.Trigger: return new OrderMetadata { KeyBinding = "T", Color = BTSColors.Orders.trigger_order }; case CustomOrderDefinition.CustomOneofCase.Spreader: goto default; @@ -305,6 +319,8 @@ namespace mbts.spark.utilities { case OrderState.CustomOneofCase.Join: return new OrderMetadata { KeyBinding = "J", Color = BTSColors.Orders.join_order }; case OrderState.CustomOneofCase.Top: return new OrderMetadata { KeyBinding = "D", Color = BTSColors.Orders.top_order }; case OrderState.CustomOneofCase.Cover: return new OrderMetadata { KeyBinding = "C", Color = BTSColors.Orders.cover_order }; + case OrderState.CustomOneofCase.Oco: return new OrderMetadata { KeyBinding = "C", Color = BTSColors.Orders.cover_order }; + case OrderState.CustomOneofCase.Pricer: return new OrderMetadata { KeyBinding = "C", Color = BTSColors.Orders.cover_order }; case OrderState.CustomOneofCase.Offset: return new OrderMetadata { KeyBinding = "O", Color = BTSColors.Orders.tick_order }; case OrderState.CustomOneofCase.Trigger: return new OrderMetadata { KeyBinding = "T", Color = BTSColors.Orders.trigger_order }; case OrderState.CustomOneofCase.Spreader: goto default; @@ -344,6 +360,8 @@ namespace mbts.spark.utilities { case nameof(JoinOrderDefinition): return CustomOrderDefinition.CustomOneofCase.Join; case nameof(TopOrderDefinition): return CustomOrderDefinition.CustomOneofCase.Top; case nameof(CoverOrderDefinition): return CustomOrderDefinition.CustomOneofCase.Cover; + case nameof(OCOOrderDefinition): return CustomOrderDefinition.CustomOneofCase.Oco; + case nameof(PricerOrderDefinition): return CustomOrderDefinition.CustomOneofCase.Pricer; case nameof(OffsetOrderDefinition): return CustomOrderDefinition.CustomOneofCase.Offset; case nameof(TriggerOrderDefinition): return CustomOrderDefinition.CustomOneofCase.Trigger; case nameof(DynamicOrderDefinition): return CustomOrderDefinition.CustomOneofCase.Dynamic; diff --git a/src/lib/api/services/OrderService.proto b/src/lib/api/services/OrderService.proto index 8c9a00de1..87711b97d 100644 --- a/src/lib/api/services/OrderService.proto +++ b/src/lib/api/services/OrderService.proto @@ -73,6 +73,8 @@ message CustomOrderDefinition TriggerOrderDefinition trigger = 22; CoverOrderDefinition cover = 23; OffsetOrderDefinition offset = 24; + OCOOrderDefinition oco = 25; + PricerOrderDefinition pricer = 26; DynamicOrderDefinition dynamic = 1000; } @@ -154,6 +156,16 @@ message OffsetOrderDefinition { uint32 tick_offset_on_deploy = 1; CustomOrderDefinition child = 6; } +message OCOOrderDefinition { + CustomOrderDefinition other = 1; + int32 other_ticks = 2; + CustomOrderDefinition child = 3; +} +message PricerOrderDefinition { + int32 price_offset_ticks = 1; + double price_factor = 2; + CustomOrderDefinition child = 3; +} message DynamicOrderDefinition { string dynamic_order_type = 1; map order_params = 2; @@ -203,6 +215,8 @@ message OrderState { TriggerOrderState trigger = 21; CoverOrderState cover = 22; OffsetOrderState offset = 24; + OCOOrderState oco = 25; + PricerOrderState pricer = 26; DynamicOrderState dynamic = 255; // do not use anything bigger than 255, or things will break } @@ -276,6 +290,8 @@ message TopOrderState {} message TriggerOrderState {} message CoverOrderState {} message OffsetOrderState {} +message OCOOrderState {} +message PricerOrderState {} message DynamicOrderState {} enum OrderStatus { diff --git a/src/lib/orderman/events/OrderStateUpdateEvent.h b/src/lib/orderman/events/OrderStateUpdateEvent.h index 5e888040c..fbddf3b9b 100644 --- a/src/lib/orderman/events/OrderStateUpdateEvent.h +++ b/src/lib/orderman/events/OrderStateUpdateEvent.h @@ -118,6 +118,12 @@ class OrderStateUpdateEvent : public mercury::Event { case api::orderman::OrderState::kOffset: result.mutable_offset(); break; + case api::orderman::OrderState::kOco: + result.mutable_oco(); + break; + case api::orderman::OrderState::kPricer: + result.mutable_pricer(); + break; case api::orderman::OrderState::kDynamic: result.mutable_dynamic(); break; diff --git a/src/lib/orderman/order_factory/OrderFactory.cpp b/src/lib/orderman/order_factory/OrderFactory.cpp index 1207b7764..36ad621dd 100644 --- a/src/lib/orderman/order_factory/OrderFactory.cpp +++ b/src/lib/orderman/order_factory/OrderFactory.cpp @@ -19,6 +19,8 @@ #include "lib/orderman/orders/book_algos/LeanOrder.h" #include "lib/orderman/orders/book_algos/TickOrder.h" #include "lib/orderman/orders/book_algos/OffsetOrder.h" +#include "lib/orderman/orders/book_algos/OCOOrder.h" +#include "lib/orderman/orders/book_algos/PricerOrder.h" #include "lib/orderman/orders/book_algos/TopOrder.h" #include "lib/orderman/orders/spreader_orders/SpreaderOrder.h" @@ -153,7 +155,13 @@ Order &OrderFactory::allocate(OrderKey k, OrderKey pk, break; case CustomOrderDefinition::kOffset: order_ptr.reset(new orderman::OffsetOrder()); - break; + break; + case CustomOrderDefinition::kOco: + order_ptr.reset(new orderman::OCOOrder()); + break; + case CustomOrderDefinition::kPricer: + order_ptr.reset(new orderman::PricerOrder()); + break; case CustomOrderDefinition::kDynamic: { order_ptr = me->dynamic_order_registry.build( def.custom_def().dynamic().dynamic_order_type()); diff --git a/src/lib/orderman/orders/book_algos/BookAlgo.cpp b/src/lib/orderman/orders/book_algos/BookAlgo.cpp index 5e3ea18ae..1ff8c8bdf 100644 --- a/src/lib/orderman/orders/book_algos/BookAlgo.cpp +++ b/src/lib/orderman/orders/book_algos/BookAlgo.cpp @@ -28,9 +28,9 @@ struct BookAlgo::PImpl bool supports_hedge; OrderBuilder hedge_builder; - std::vector hedges; - using HedgeStateMap = boost::container::flat_map; - HedgeStateMap hedges_state; + + BookAlgo::HedgeMap hedges; + BookAlgo::HedgeStateMap hedges_state; double tick_size; bool has_activated_child; @@ -102,8 +102,9 @@ void BookAlgo::cancel_child() void BookAlgo::build_and_send_hedge(Price p, Quantity q) { auto hedge = me->hedge_builder.build(); - me->hedges.push_back(std::move(hedge)); - me->hedges.back().activate(p, q); + auto key = hedge.key(); + me->hedges[key] = std::move(hedge); + me->hedges[key].activate(p, q); } double BookAlgo::tick_size() const @@ -111,6 +112,17 @@ double BookAlgo::tick_size() const return me->tick_size; } +BookAlgo::HedgeStateMap& BookAlgo::hedges_state() +{ + return me->hedges_state; +} + +BookAlgo::HedgeMap& BookAlgo::hedges() +{ + return me->hedges; +} + + void BookAlgo::on_init_new(const api::orderman::OrderDefinition& def, const PreActivationOrderServices &services) { @@ -291,7 +303,7 @@ void BookAlgo::on_cancel(PerfTrace const&) if(!me->fake_detached_child) me->child.cancel(); for(auto& i : me->hedges) - i.crack(); + i.second.crack(); transition(ORDER_STATUS_CANCELED); break; case ORDER_STATUS_PENDING_WORKING: @@ -342,9 +354,59 @@ void BookAlgo::on_child_order_update(const OrderStateUpdateEvent& ev) { if(me->fake_detached_child) return; - if(ev.key != me->child.key()) + if(ev.key != me->child.key()) //hedge state change + { + api::orderman::OrderState orig_state = api::orderman::OrderState::default_instance(); //TODO this isn't quite right, should be hedgetype state + auto it = me->hedges_state.find(ev.key); + if(it != me->hedges_state.end()) + orig_state = it->second; me->hedges_state[ev.key] = ev.rebuild_state(); - else + switch (this_state().status()) + { + case ORDER_STATUS_WORKING: + switch (ev.status) + { + case ORDER_STATUS_PENDING_WORKING: + break; + case ORDER_STATUS_WORKING: + process_hedge_state_change(orig_state, ev); + break; + case ORDER_STATUS_COMPLETED: + transition(ORDER_STATUS_COMPLETED); + process_hedge_state_change(orig_state, ev); + break; + + case ORDER_STATUS_DISCONNECTED: + case ORDER_STATUS_REJECTED_BY_SPARK: + case ORDER_STATUS_CANCELED: + case ORDER_STATUS_ERROR: + transition(ORDER_STATUS_ERROR, "Unexpected child order state"); + break; + + case ORDER_STATUS_INACTIVE: + case OrderStatus_INT_MIN_SENTINEL_DO_NOT_USE_: + case OrderStatus_INT_MAX_SENTINEL_DO_NOT_USE_: + bts_assert(false); + break; + } + break; + + case ORDER_STATUS_CANCELED: + case ORDER_STATUS_ERROR: + case ORDER_STATUS_COMPLETED: + break; + + case ORDER_STATUS_DISCONNECTED: + case ORDER_STATUS_PENDING_WORKING: + case ORDER_STATUS_REJECTED_BY_SPARK: + case ORDER_STATUS_INACTIVE: + case OrderStatus_INT_MIN_SENTINEL_DO_NOT_USE_: + case OrderStatus_INT_MAX_SENTINEL_DO_NOT_USE_: + bts_assert(false); + break; + } + } + else //child state change { api::orderman::OrderState orig_state = child_state(); me->child_state = ev.rebuild_state(); @@ -356,13 +418,11 @@ void BookAlgo::on_child_order_update(const OrderStateUpdateEvent& ev) case ORDER_STATUS_PENDING_WORKING: break; case ORDER_STATUS_WORKING: - if(me->supports_hedge) - send_hedge_if_new_fills(orig_state, ev); + process_child_state_change(orig_state, ev); break; case ORDER_STATUS_COMPLETED: transition(ORDER_STATUS_COMPLETED); - if(me->supports_hedge) - send_hedge_if_new_fills(orig_state, ev); + process_child_state_change(orig_state, ev); break; case ORDER_STATUS_DISCONNECTED: @@ -437,11 +497,14 @@ bool BookAlgo::setup_hedge(const api::orderman::OrderDefinition&, const PreActiv return false; } -void BookAlgo::send_hedge_if_new_fills(const api::orderman::OrderState& orig_state, const OrderStateUpdateEvent& new_state) +void BookAlgo::process_child_state_change(const api::orderman::OrderState& orig_state, const OrderStateUpdateEvent& new_state) +{ + UNUSED(orig_state);UNUSED(new_state); +} + +void BookAlgo::process_hedge_state_change(const api::orderman::OrderState& orig_state, const OrderStateUpdateEvent& new_state) { UNUSED(orig_state);UNUSED(new_state); - // uint32_t qty = new_state.state.filled_quantity() - orig_state.filled_quantity(); - // UNUSED(qty); } } diff --git a/src/lib/orderman/orders/book_algos/BookAlgo.h b/src/lib/orderman/orders/book_algos/BookAlgo.h index 7c01878a0..a2f7913c8 100644 --- a/src/lib/orderman/orders/book_algos/BookAlgo.h +++ b/src/lib/orderman/orders/book_algos/BookAlgo.h @@ -34,7 +34,8 @@ protected: api::orderman::OrderState&, api::orderman::OrderDefinition*) = 0; virtual bool setup_hedge(const api::orderman::OrderDefinition& parent, const PreActivationOrderServices&, api::orderman::OrderDefinition*); - virtual void send_hedge_if_new_fills(const api::orderman::OrderState&, const OrderStateUpdateEvent&); + virtual void process_child_state_change(const api::orderman::OrderState&, const OrderStateUpdateEvent&); + virtual void process_hedge_state_change(const api::orderman::OrderState&, const OrderStateUpdateEvent&); virtual void on_child_order_update(const OrderStateUpdateEvent&); virtual void on_book_update(const bts::orderman::BookUpdateEvent&); @@ -50,6 +51,7 @@ private: std::unique_ptr me; protected: + double tick_size() const; const api::orderman::OrderState& this_state() const; void set_price(double); void set_quantity(uint32_t); @@ -62,7 +64,11 @@ protected: void cancel_child(); void build_and_send_hedge(Price, Quantity); - double tick_size() const; + typedef boost::container::flat_map HedgeMap; + typedef boost::container::flat_map HedgeStateMap; + HedgeStateMap& hedges_state(); + HedgeMap& hedges(); + }; }} diff --git a/src/lib/orderman/orders/book_algos/CoverOrder.cpp b/src/lib/orderman/orders/book_algos/CoverOrder.cpp index de1f773be..fcb5d76af 100644 --- a/src/lib/orderman/orders/book_algos/CoverOrder.cpp +++ b/src/lib/orderman/orders/book_algos/CoverOrder.cpp @@ -50,7 +50,7 @@ bool CoverOrder::setup_hedge(const api::orderman::OrderDefinition& this_def, con return false; } -void CoverOrder::send_hedge_if_new_fills(const api::orderman::OrderState& orig_state, const OrderStateUpdateEvent& new_state) +void CoverOrder::process_child_state_change(const api::orderman::OrderState& orig_state, const OrderStateUpdateEvent& new_state) { uint32_t qty = new_state.filled_quantity - orig_state.filled_quantity(); if(qty > 0) diff --git a/src/lib/orderman/orders/book_algos/CoverOrder.h b/src/lib/orderman/orders/book_algos/CoverOrder.h index 9b831f6dd..9bf81494e 100644 --- a/src/lib/orderman/orders/book_algos/CoverOrder.h +++ b/src/lib/orderman/orders/book_algos/CoverOrder.h @@ -22,7 +22,7 @@ protected: api::orderman::OrderState&, api::orderman::OrderDefinition*) override; bool setup_hedge(const api::orderman::OrderDefinition& parent, const PreActivationOrderServices&, api::orderman::OrderDefinition*) override; - void send_hedge_if_new_fills(const api::orderman::OrderState&, const OrderStateUpdateEvent&) override; + void process_child_state_change(const api::orderman::OrderState&, const OrderStateUpdateEvent&) override; void on_book_update(const bts::orderman::BookUpdateEvent&) override; private: diff --git a/src/lib/orderman/orders/book_algos/OCOOrder.cpp b/src/lib/orderman/orders/book_algos/OCOOrder.cpp new file mode 100644 index 000000000..9eb8576f2 --- /dev/null +++ b/src/lib/orderman/orders/book_algos/OCOOrder.cpp @@ -0,0 +1,166 @@ +#include "OCOOrder.h" + +#include "lib/orderman/actors/order_manager/OrderBuilder.h" +#include "lib/orderman/orders/spreader_orders/spreader/SpreaderUtils.h" +#include "lib/utils/PBUtils.h" +#include + +namespace bts { +namespace orderman { + +using namespace api::orderman; + +struct OCOOrder::PImpl +{ + PImpl() {} + + int other_ticks; + OrderDefinition other_def; +}; + + +OCOOrder::OCOOrder() : me(new PImpl()) {} + +OCOOrder::~OCOOrder() {} + +void OCOOrder::setup_child(const api::orderman::OrderDefinition& this_def, const PreActivationOrderServices&, + api::orderman::OrderState& initial_state, api::orderman::OrderDefinition* out_child) +{ + initial_state.mutable_oco(); + auto const &oco_def = this_def.custom_def().oco(); + *out_child->mutable_custom_def() = oco_def.child(); + + me->other_ticks = oco_def.other_ticks(); +} + +//hedge here means 'other' +bool OCOOrder::setup_hedge(const api::orderman::OrderDefinition& this_def, const PreActivationOrderServices&, + api::orderman::OrderDefinition* out_hedge) +{ + auto const &oco_def = this_def.custom_def().oco(); + if(oco_def.has_other()) + { + *out_hedge->mutable_custom_def() = oco_def.other(); + if(definition().side() == api::SIDE_BUY) + out_hedge->set_side(api::SIDE_SELL); + else + out_hedge->set_side(api::SIDE_BUY); + return true; + } + else + return false; +} + +void OCOOrder::process_child_state_change(const api::orderman::OrderState& orig_state, const OrderStateUpdateEvent& new_state) +{ + uint32_t fill_qty = new_state.filled_quantity - orig_state.filled_quantity(); + if(fill_qty > 0) //reduce hedge + { + const auto& the_state = hedges_state(); + auto& the_hedges = hedges(); + for(const auto& i : the_state) + { + const auto& hedge_state = i.second; + + uint32_t orig_qty = hedge_state.quantity(); + uint32_t newqty = orig_qty - fill_qty; + bts_assert(newqty < orig_qty); + + auto it = the_hedges.find(i.first); + if(it != the_hedges.end()) + { + if(newqty > 0) + { + LOG(OrderMan, Info, "OCO reducing hedge qty from {} to {}", hedge_state.quantity(), newqty); + it->second.update(Price(hedge_state.price()), Quantity(newqty)); + } + else + it->second.cancel(); + } + else + { + LOG(OrderMan, Error, "OCO Can't find hedge order {}", i.first); + } + } + } +} + +void OCOOrder::process_hedge_state_change(const api::orderman::OrderState& orig_state, const OrderStateUpdateEvent& new_state) +{ + uint32_t fill_qty = new_state.filled_quantity - orig_state.filled_quantity(); + if(fill_qty > 0) //reduce child + { + uint32_t orig_qty = child_state().quantity(); + uint32_t newqty = orig_qty - fill_qty; + bts_assert(newqty < orig_qty); + if(newqty > 0) + { + LOG(OrderMan, Info, "OCO reducing child qty from {} to {}", child_state().quantity(), newqty); + update_child(Price(child_state().price()), Quantity(newqty)); + } + else + cancel_child(); + } +} + +void OCOOrder::on_book_update(const bts::orderman::BookUpdateEvent& ) +{ + switch (this_state().status()) + { + case ORDER_STATUS_WORKING: + switch (child_state().status()) + { + case ORDER_STATUS_INACTIVE: + { + activate_child(Price(this_state().price()), Quantity(this_state().quantity())); + + double prx = this_state().price(); + if(definition().side() == api::SIDE_BUY) + prx = prx + tick_size() * me->other_ticks; + else + prx = prx - tick_size() * me->other_ticks; + build_and_send_hedge( Price(prx), Quantity(this_state().quantity()) ); + transition(ORDER_STATUS_WORKING); + report_state(); + } + break; + case ORDER_STATUS_WORKING: + { + } + break; + + case ORDER_STATUS_PENDING_WORKING: + break; + + // child cannot be in any of these states if we are in the working state + case ORDER_STATUS_DISCONNECTED: + case ORDER_STATUS_ERROR: + case ORDER_STATUS_REJECTED_BY_SPARK: + case ORDER_STATUS_COMPLETED: + case ORDER_STATUS_CANCELED: + case OrderStatus_INT_MIN_SENTINEL_DO_NOT_USE_: + case OrderStatus_INT_MAX_SENTINEL_DO_NOT_USE_: + bts_assert(false); + break; + } + break; + + case ORDER_STATUS_COMPLETED: + case ORDER_STATUS_CANCELED: + case ORDER_STATUS_ERROR: + break; + + case ORDER_STATUS_INACTIVE: + case ORDER_STATUS_REJECTED_BY_SPARK: + case ORDER_STATUS_PENDING_WORKING: + case ORDER_STATUS_DISCONNECTED: + case OrderStatus_INT_MIN_SENTINEL_DO_NOT_USE_: + case OrderStatus_INT_MAX_SENTINEL_DO_NOT_USE_: + bts_assert(false); + break; + } +} + +} +} + diff --git a/src/lib/orderman/orders/book_algos/OCOOrder.h b/src/lib/orderman/orders/book_algos/OCOOrder.h new file mode 100644 index 000000000..31a1ed29f --- /dev/null +++ b/src/lib/orderman/orders/book_algos/OCOOrder.h @@ -0,0 +1,35 @@ +#pragma once + +#include "lib/orderman/orders/book_algos/BookAlgo.h" + +namespace bts { +namespace orderman { + +class OrderStateUpdateEvent; +class BookUpdateEvent; + +//OCOOrder initializes the child order as normal +//it also initializes a 'hedge' or 'other' at the same time, the child and the hedge are managed with OCO semantics: +//when one fills the other's quantity is reduced (or cancelled on a full fill) + +class OCOOrder : public BookAlgo +{ +public: + OCOOrder(); + virtual ~OCOOrder(); + +protected: + void setup_child(const api::orderman::OrderDefinition&, const PreActivationOrderServices&, + api::orderman::OrderState&, api::orderman::OrderDefinition*) override; + bool setup_hedge(const api::orderman::OrderDefinition& parent, const PreActivationOrderServices&, + api::orderman::OrderDefinition*) override; + void process_child_state_change(const api::orderman::OrderState&, const OrderStateUpdateEvent&) override; + void process_hedge_state_change(const api::orderman::OrderState&, const OrderStateUpdateEvent&) override; + void on_book_update(const bts::orderman::BookUpdateEvent&) override; + +private: + struct PImpl; + std::unique_ptr me; +}; + +}} diff --git a/src/lib/orderman/orders/book_algos/OffsetOrder.cpp b/src/lib/orderman/orders/book_algos/OffsetOrder.cpp index fad23cc55..6a1a6972d 100644 --- a/src/lib/orderman/orders/book_algos/OffsetOrder.cpp +++ b/src/lib/orderman/orders/book_algos/OffsetOrder.cpp @@ -25,7 +25,7 @@ OffsetOrder::~OffsetOrder() {} void OffsetOrder::setup_child(const api::orderman::OrderDefinition& this_def, const PreActivationOrderServices&, api::orderman::OrderState& initial_state, api::orderman::OrderDefinition* out_child) { - initial_state.mutable_tick(); + initial_state.mutable_offset(); auto const &offset_def = this_def.custom_def().offset(); *out_child->mutable_custom_def() = offset_def.child(); diff --git a/src/lib/orderman/orders/book_algos/PricerOrder.cpp b/src/lib/orderman/orders/book_algos/PricerOrder.cpp new file mode 100644 index 000000000..63823af48 --- /dev/null +++ b/src/lib/orderman/orders/book_algos/PricerOrder.cpp @@ -0,0 +1,40 @@ +#include "PricerOrder.h" + +#include "lib/orderman/actors/order_manager/OrderBuilder.h" +#include "lib/orderman/orders/spreader_orders/spreader/SpreaderUtils.h" +#include "lib/utils/PBUtils.h" +#include + +namespace bts { +namespace orderman { + +using namespace api::orderman; + +struct PricerOrder::PImpl +{ + PImpl() {} + + int32_t price_offset_ticks; + double price_factor; +}; + + +PricerOrder::PricerOrder() : me(new PImpl()) {} + +PricerOrder::~PricerOrder() {} + +void PricerOrder::setup_child(const api::orderman::OrderDefinition& this_def, const PreActivationOrderServices&, + api::orderman::OrderState& initial_state, api::orderman::OrderDefinition* out_child) +{ + initial_state.mutable_pricer(); + auto const &pricer_def = this_def.custom_def().pricer(); + *out_child->mutable_custom_def() = pricer_def.child(); + + me->price_offset_ticks = pricer_def.price_offset_ticks(); + me->price_factor = pricer_def.price_factor(); +} + + +} +} + diff --git a/src/lib/orderman/orders/book_algos/PricerOrder.h b/src/lib/orderman/orders/book_algos/PricerOrder.h new file mode 100644 index 000000000..e61986be4 --- /dev/null +++ b/src/lib/orderman/orders/book_algos/PricerOrder.h @@ -0,0 +1,28 @@ +#pragma once + +#include "lib/orderman/orders/book_algos/BookAlgo.h" + +namespace bts { +namespace orderman { + +class OrderStateUpdateEvent; +class BookUpdateEvent; + +//PricerOrder matches the bid (for buy) or offer (for sell) price of it's target security times a factor plus a constant + +class PricerOrder : public BookAlgo +{ +public: + PricerOrder(); + virtual ~PricerOrder(); + +protected: + void setup_child(const api::orderman::OrderDefinition& this_def, const PreActivationOrderServices&, + api::orderman::OrderState&, api::orderman::OrderDefinition*) override; + +private: + struct PImpl; + std::unique_ptr me; +}; + +}} From aae927d7735d6acdb9bd68946218f878cb76f7f5 Mon Sep 17 00:00:00 2001 From: Kevin Darby Date: Mon, 17 Dec 2018 17:38:22 -0500 Subject: [PATCH 02/85] gui for new algos --- .../Orders/OrderModificationControl.xaml.cs | 4 +- .../spark/Orders/OrderTypeVMs/OCOOrderVM.cs | 246 ++++++++++++++++++ .../Orders/OrderTypeVMs/PricerOrderVM.cs | 173 ++++++++++++ .../Order/OrderDetailsPopupControl.xaml | 58 +++++ .../view/Spark/Utilities/OrderDataHelper.cs | 3 +- 5 files changed, 480 insertions(+), 4 deletions(-) create mode 100644 src/client/spark/Orders/OrderTypeVMs/OCOOrderVM.cs create mode 100644 src/client/spark/Orders/OrderTypeVMs/PricerOrderVM.cs diff --git a/src/client/spark/Orders/OrderModificationControl.xaml.cs b/src/client/spark/Orders/OrderModificationControl.xaml.cs index 721fcaad3..2d2f43207 100644 --- a/src/client/spark/Orders/OrderModificationControl.xaml.cs +++ b/src/client/spark/Orders/OrderModificationControl.xaml.cs @@ -25,7 +25,7 @@ namespace mbts.ui.spark { public DataTemplate JoinOrderTemplate { get; set; } public DataTemplate TopOrderTemplate { get; set; } public DataTemplate CoverOrderTemplate { get; set; } - public DataTemplate OcoOrderTemplate { get; set; } + public DataTemplate OCOOrderTemplate { get; set; } public DataTemplate PricerOrderTemplate { get; set; } public DataTemplate OffsetOrderTemplate { get; set; } public DataTemplate TriggerOrderTemplate { get; set; } @@ -45,7 +45,7 @@ namespace mbts.ui.spark { case CustomOrderDefinition.CustomOneofCase.Join: return JoinOrderTemplate; case CustomOrderDefinition.CustomOneofCase.Top: return TopOrderTemplate; case CustomOrderDefinition.CustomOneofCase.Cover: return CoverOrderTemplate; - case CustomOrderDefinition.CustomOneofCase.Oco : return OcoOrderTemplate; + case CustomOrderDefinition.CustomOneofCase.Oco : return OCOOrderTemplate; case CustomOrderDefinition.CustomOneofCase.Pricer: return PricerOrderTemplate; case CustomOrderDefinition.CustomOneofCase.Offset: return OffsetOrderTemplate; case CustomOrderDefinition.CustomOneofCase.Trigger: return TriggerOrderTemplate; diff --git a/src/client/spark/Orders/OrderTypeVMs/OCOOrderVM.cs b/src/client/spark/Orders/OrderTypeVMs/OCOOrderVM.cs new file mode 100644 index 000000000..3349fa9cd --- /dev/null +++ b/src/client/spark/Orders/OrderTypeVMs/OCOOrderVM.cs @@ -0,0 +1,246 @@ +using bts.api; +using bts.api.orderman; +using mbts.ui; +using mbts.ui.utility; +using mbts.utils; +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.ComponentModel; +using System.Linq; +using System.Reactive.Linq; +using static bts.api.orderman.CustomOrderDefinition; + +namespace mbts.spark +{ + static partial class OrderTypeVMFactory + { + class InlineOCOOrderProperties + { + public InlineOCOOrderProperties(OrderTemplateHeader selected_child_order_type, IReadOnlyCollection child_order_types) + { + ExceptionAssert.InvalidOperation.That(child_order_types.Contains(selected_child_order_type), $"Child order type {selected_child_order_type} is not in child_order_types"); + this.selected_child_order_type = new InlineUserProperty(selected_child_order_type, true); + this.child_order_types = new InlineUserProperty>(child_order_types, true); + + var othertype = CustomOneofCase.Limit; + this.selected_other_order_type = new InlineUserProperty(new OrderTemplateHeader(othertype), true); + this.other_order_types = new InlineUserProperty>(child_order_types, true); + + other_ticks = new InlineUserProperty(1, false); + } + public readonly InlineUserProperty selected_child_order_type; + public readonly InlineUserProperty> child_order_types; + public readonly InlineUserProperty selected_other_order_type; + public readonly InlineUserProperty> other_order_types; + + public readonly InlineUserProperty other_ticks; + + } + + class OCOOrderVM : OrderTypeVMBase, IHierarchicalOrderTypeVM, IDisposable + { + readonly IObservable _security_md_source; + SecurityImmutable? _security; + ObservableCollection _order_types = new ObservableCollection(); + ObservableAutoWrapper _child_order_choices; + InlineOCOOrderProperties _inline_props; + IOrderTypeVM _child_order; + IDisposable _dispose, _child_order_errors_sub; + + ObservableAutoWrapper _other_order_choices; + IOrderTypeVM _other_order; + IDisposable _other_order_errors_sub; + + public int other_ticks { get; set; } = 1; + + public OCOOrderVM(OrdersModel model, IObservable security_md_source, string id, string display_name) + : this(model, security_md_source, security_md_source.Select(smd => + new InlineOCOOrderProperties(new OrderTemplateHeader(model.factory.default_order_type(smd.security)), model.factory.default_order_types(smd.security).Select(t => new OrderTemplateHeader(t)).ToList())), id, display_name) + { } + + public OCOOrderVM(OrdersModel model, IObservable security_md_source, + IObservable inline_properties, string id, string display_name) : base(model, id, display_name) + { + _security_md_source = security_md_source; + _dispose = Disposable.Create( + security_md_source.OnDispatcher() + .Subscribe(smd => _security = smd.security), + inline_properties.OnDispatcher() + .Subscribe(p => + { + if (_order_types.Count == 0 || p.child_order_types.force_override) + { + _order_types.SetItems(p.child_order_types.value); + } + if (_child_order_choices != null && (_child_order == null || p.selected_child_order_type.force_override)) + { + _select_child_order(p.selected_child_order_type.value.key.type); + } + if (_other_order_choices != null && (_other_order == null || p.selected_other_order_type.force_override)) + { + _select_other_order(p.selected_other_order_type.value.key.type); + } + if (p.other_ticks.force_override) + { + other_ticks = p.other_ticks.value; + RaisePropertyChanged(nameof(other_ticks)); + } + _inline_props = p; + _validate(); + }) + ); + _validate(); + } + + public override void assign_to(CustomOrderDefinition value) + { + if (_child_order == null) + _select_child_order(model.factory.default_order_type(_security.Value, true)); + if (_other_order == null ) + _select_other_order(model.factory.default_order_type(_security.Value, true)); + _validate(); + if (!HasErrors) + { + var i = factory.oco_order(); + i.OtherTicks = other_ticks; + child_order.assign_to(i.Child); + if (other_order != null) + other_order.assign_to(i.Other); + value.Oco = i; + } + } + + public ICollection child_order_types + { + get + { + if (_child_order_choices == null) + _child_order_choices = new ObservableAutoWrapper(_order_types, ot => create(ot.key.type, ot.key.name, ot.display_name, model, _security_md_source)); + return _child_order_choices; + } + } + + public ICollection other_order_types + { + get + { + if (_other_order_choices == null) + _other_order_choices = new ObservableAutoWrapper(_order_types, ot => create(ot.key.type, ot.key.name, ot.display_name, model, _security_md_source)); + return _other_order_choices; + } + } + + public IOrderTypeVM child_order + { + get + { + init(); + return _child_order; + } + set + { + Disposable.dispose(ref _child_order_errors_sub); + _child_order = value; + _child_order?.init(); + if (_child_order is INotifyDataErrorInfo ndei) + { + ndei.ErrorsChanged += on_errors_changed; + void on_errors_changed(object s, DataErrorsChangedEventArgs e) => + _validate(); + _child_order_errors_sub = Disposable.Create(() => ndei.ErrorsChanged -= on_errors_changed); + } + _validate(); + RaisePropertyChanged(nameof(child_order)); + } + } + + public IOrderTypeVM other_order + { + get + { + init(); + return _other_order; + } + set + { + Disposable.dispose(ref _other_order_errors_sub); + _other_order = value; + _other_order?.init(); + if (_other_order is INotifyDataErrorInfo ndei) + { + ndei.ErrorsChanged += on_errors_changed; + void on_errors_changed(object s, DataErrorsChangedEventArgs e) => + _validate(); + _other_order_errors_sub = Disposable.Create(() => ndei.ErrorsChanged -= on_errors_changed); + } + _validate(); + RaisePropertyChanged(nameof(other_order)); + } + } + + public void Dispose() + { + Disposable.dispose(ref _dispose); + if (_child_order_choices != null) + _child_order_choices.OfType().DisposeAll(); + if (_other_order_choices != null) + _other_order_choices.OfType().DisposeAll(); + } + + public override void init() + { + if (_child_order == null && _security.HasValue) + _select_child_order(); + if (_other_order == null && _security.HasValue) + _select_other_order(); + _validate(); + } + + void _select_child_order() + { + CustomOneofCase ot = _inline_props != null && _inline_props.selected_child_order_type.has_value ? + _inline_props.selected_child_order_type.value.key.type : + model.factory.default_order_type(_security.Value, true); + _select_child_order(ot); + } + + void _select_child_order(CustomOneofCase ot) + { + _child_order = child_order_types.FirstOrDefault(vm => vm.order_type == ot); + if (_child_order == null) + _child_order = child_order_types.FirstOrDefault(); + RaisePropertyChanged(nameof(child_order)); + } + + void _select_other_order() + { + CustomOneofCase ot = _inline_props != null && _inline_props.selected_other_order_type.has_value ? + _inline_props.selected_other_order_type.value.key.type : + model.factory.default_order_type(_security.Value, true); + _select_other_order(ot); + } + + void _select_other_order(CustomOneofCase ot) + { + _other_order = other_order_types.FirstOrDefault(vm => vm.order_type == ot); + if (_other_order == null) + _other_order = other_order_types.FirstOrDefault(); + RaisePropertyChanged(nameof(other_order)); + } + + + void _validate() + { + if (_child_order == null || (_child_order is INotifyDataErrorInfo ndei && ndei.HasErrors)) + set_error(nameof(child_order), string.Empty, true); + else clear_error(nameof(child_order)); + + if ((_other_order == null ) || (_other_order is INotifyDataErrorInfo hdei && hdei.HasErrors)) + set_error(nameof(other_order), string.Empty, true); + else clear_error(nameof(other_order)); + + } + } + } +} \ No newline at end of file diff --git a/src/client/spark/Orders/OrderTypeVMs/PricerOrderVM.cs b/src/client/spark/Orders/OrderTypeVMs/PricerOrderVM.cs new file mode 100644 index 000000000..09d883386 --- /dev/null +++ b/src/client/spark/Orders/OrderTypeVMs/PricerOrderVM.cs @@ -0,0 +1,173 @@ +using bts.api; +using bts.api.orderman; +using mbts.ui; +using mbts.ui.utility; +using mbts.utils; +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.ComponentModel; +using System.Linq; +using System.Reactive.Linq; +using static bts.api.orderman.CustomOrderDefinition; + +namespace mbts.spark +{ + static partial class OrderTypeVMFactory + { + class InlinePricerOrderProperties + { + public InlinePricerOrderProperties (OrderTemplateHeader selected_child_order_type, IReadOnlyCollection child_order_types) + { + ExceptionAssert.InvalidOperation.That(child_order_types.Contains(selected_child_order_type), $"Child order type {selected_child_order_type} is not in child_order_types"); + this.selected_child_order_type = new InlineUserProperty(selected_child_order_type, true); + this.child_order_types = new InlineUserProperty>(child_order_types, true); + price_offset_ticks = new InlineUserProperty(0, false); + price_factor = new InlineUserProperty(0, false); + } + public readonly InlineUserProperty selected_child_order_type; + public readonly InlineUserProperty> child_order_types; + + public readonly InlineUserProperty price_offset_ticks; + public readonly InlineUserProperty price_factor; + } + + class PricerOrderVM : OrderTypeVMBase, IHierarchicalOrderTypeVM, IDisposable + { + readonly IObservable _security_md_source; + SecurityImmutable? _security; + ObservableCollection _order_types = new ObservableCollection(); + ObservableAutoWrapper _child_order_choices; + InlinePricerOrderProperties _inline_props; + IOrderTypeVM _child_order; + IDisposable _dispose, _child_order_errors_sub; + + public int price_offset_ticks { get; set; } = 0; + public double price_factor { get; set; } = 1.0; + + public PricerOrderVM(OrdersModel model, IObservable security_md_source, string id, string display_name) + : this(model, security_md_source, security_md_source.Select(smd => + new InlinePricerOrderProperties(new OrderTemplateHeader(model.factory.default_order_type(smd.security)), model.factory.default_order_types(smd.security).Select(t => new OrderTemplateHeader(t)).ToList())), id, display_name) { } + + public PricerOrderVM(OrdersModel model, IObservable security_md_source, + IObservable inline_properties, string id, string display_name) : base(model, id, display_name) + { + _security_md_source = security_md_source; + _dispose = Disposable.Create( + security_md_source.OnDispatcher() + .Subscribe(smd => _security = smd.security), + inline_properties.OnDispatcher() + .Subscribe(p => { + if (_order_types.Count == 0 || p.child_order_types.force_override) + { + _order_types.SetItems(p.child_order_types.value); + } + if (_child_order_choices != null && (_child_order == null || p.selected_child_order_type.force_override)) + { + _select_child_order(p.selected_child_order_type.value.key.type); + } + if (p.price_offset_ticks.force_override) + { + price_offset_ticks = p.price_offset_ticks.value; + RaisePropertyChanged(nameof(price_offset_ticks)); + } + if (p.price_factor.force_override) + { + price_factor = p.price_factor.value; + RaisePropertyChanged(nameof(price_factor)); + } + _inline_props = p; + _validate(); + }) + ); + _validate(); + } + + public override void assign_to(CustomOrderDefinition value) + { + if (_child_order == null) + _select_child_order(model.factory.default_order_type(_security.Value, true)); + _validate(); + if (!HasErrors) + { + var i = factory.pricer_order(); + i.PriceOffsetTicks = price_offset_ticks; + i.PriceFactor = price_factor; + child_order.assign_to(i.Child); + value.Pricer = i; + } + } + + public ICollection child_order_types + { + get + { + if (_child_order_choices == null) + _child_order_choices = new ObservableAutoWrapper(_order_types, ot => create(ot.key.type, ot.key.name, ot.display_name, model, _security_md_source)); + return _child_order_choices; + } + } + + public IOrderTypeVM child_order + { + get + { + init(); + return _child_order; + } + set + { + Disposable.dispose(ref _child_order_errors_sub); + _child_order = value; + _child_order?.init(); + if (_child_order is INotifyDataErrorInfo ndei) + { + ndei.ErrorsChanged += on_errors_changed; + void on_errors_changed(object s, DataErrorsChangedEventArgs e) => + _validate(); + _child_order_errors_sub = Disposable.Create(() => ndei.ErrorsChanged -= on_errors_changed); + } + _validate(); + RaisePropertyChanged(nameof(child_order)); + } + } + + public void Dispose() + { + Disposable.dispose(ref _dispose); + if (_child_order_choices != null) + _child_order_choices.OfType().DisposeAll(); + } + + public override void init() + { + if (_child_order == null && _security.HasValue) + _select_child_order(); + _validate(); + } + + void _select_child_order() + { + CustomOneofCase ot = _inline_props != null && _inline_props.selected_child_order_type.has_value ? + _inline_props.selected_child_order_type.value.key.type : + model.factory.default_order_type(_security.Value, true); + _select_child_order(ot); + } + + void _select_child_order(CustomOneofCase ot) + { + _child_order = child_order_types.FirstOrDefault(vm => vm.order_type == ot); + if (_child_order == null) + _child_order = child_order_types.FirstOrDefault(); + RaisePropertyChanged(nameof(child_order)); + } + + void _validate() + { + if (_child_order == null || (_child_order is INotifyDataErrorInfo ndei && ndei.HasErrors)) + set_error(nameof(child_order), string.Empty, true); + else clear_error(nameof(child_order)); + } + } + } +} \ No newline at end of file diff --git a/src/client/spark/view/Controls/Order/OrderDetailsPopupControl.xaml b/src/client/spark/view/Controls/Order/OrderDetailsPopupControl.xaml index 09759a533..2a76739e1 100644 --- a/src/client/spark/view/Controls/Order/OrderDetailsPopupControl.xaml +++ b/src/client/spark/view/Controls/Order/OrderDetailsPopupControl.xaml @@ -415,6 +415,64 @@ + + + + + + + + + + + + + + + + + + + + Other Ticks + + + + + Other + + + + + Child + + + + + + + diff --git a/src/client/spark/view/Spark/Utilities/OrderDataHelper.cs b/src/client/spark/view/Spark/Utilities/OrderDataHelper.cs index 75357ad10..3885973f9 100644 --- a/src/client/spark/view/Spark/Utilities/OrderDataHelper.cs +++ b/src/client/spark/view/Spark/Utilities/OrderDataHelper.cs @@ -25,7 +25,6 @@ namespace mbts.spark.utilities { static readonly OrderMetadata _default_order_metadata = new OrderMetadata { KeyBinding = "", Color = BTSColors.Orders.spreader_order }; static readonly CustomOrderDefinition.CustomOneofCase[] // The following standard types are not yet ready for production use: - // CustomOrderDefinition.CustomOneofCase.Tick, // CustomOrderDefinition.CustomOneofCase.Dynamic, _standard_types = new[] { CustomOrderDefinition.CustomOneofCase.Market, @@ -272,7 +271,7 @@ namespace mbts.spark.utilities { case CustomOrderDefinition.CustomOneofCase.Top: rv.Top = top_order(); break; case CustomOrderDefinition.CustomOneofCase.Cover: rv.Cover = cover_order(); break; case CustomOrderDefinition.CustomOneofCase.Oco: rv.Oco = oco_order(); break; - case CustomOrderDefinition.CustomOneofCase.Pricer: rv.Oco = pricer_order(); break; + case CustomOrderDefinition.CustomOneofCase.Pricer: rv.Pricer = pricer_order(); break; case CustomOrderDefinition.CustomOneofCase.Offset: rv.Offset = offset_order(); break; case CustomOrderDefinition.CustomOneofCase.Trigger: rv.Trigger = trigger_order(); break; case CustomOrderDefinition.CustomOneofCase.Spreader: rv.Spreader = spreader_order(); break; From f3adf780d0926bc3b1e219d48737c3ee73ea5b4f Mon Sep 17 00:00:00 2001 From: Kevin Darby Date: Wed, 19 Dec 2018 18:15:04 -0500 Subject: [PATCH 03/85] OCOOrder will be the basis of a StopLoss template Added the beginnings of pricer order but need something from John --- .vscode/launch.json | 6 +- .../spark/Orders/OrderTypeVMs/OCOOrderVM.cs | 11 ++ .../spark/core/FontsAndColors/BTSColors.cs | 2 + .../Order/OrderDetailsPopupControl.xaml | 65 ++------ .../view/Spark/Utilities/OrderDataHelper.cs | 8 +- src/lib/api/services/OrderService.proto | 1 + .../orderman/orders/book_algos/BookAlgo.cpp | 27 ++-- src/lib/orderman/orders/book_algos/BookAlgo.h | 1 + .../orderman/orders/book_algos/OCOOrder.cpp | 143 ++++++++++++++---- src/lib/orderman/orders/book_algos/OCOOrder.h | 1 + 10 files changed, 164 insertions(+), 101 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index d3f66bbaf..7b1fe24a8 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -6,7 +6,7 @@ "name": "(gdb) Launch", "type": "cppdbg", "request": "launch", - "program": "${workspaceRoot}/build/ninja_debug/src/server/sparkd/sparkd", + "program": "${workspaceRoot}/build/ninja_debug/server/sparkd/sparkd", "args": ["run"], "stopAtEntry": false, "cwd": "${workspaceRoot}/build/ninja_debug/src/server/sparkd", @@ -75,10 +75,10 @@ "name": "SparkVM Launch", "type": "cppdbg", "request": "launch", - "program": "/home/bts/spark/build/ninja_debug/src/server/sparkd/sparkd", + "program": "/home/bts/spark/build/ninja_debug/server/sparkd", "args": ["run"], "stopAtEntry": false, - "cwd": "/home/bts/spark/build/ninja_debug/src/server/sparkd/", + "cwd": "/home/bts/spark/build/ninja_debug/server/", "environment": [], "externalConsole": true, "MIMode": "gdb", diff --git a/src/client/spark/Orders/OrderTypeVMs/OCOOrderVM.cs b/src/client/spark/Orders/OrderTypeVMs/OCOOrderVM.cs index 3349fa9cd..393b0a25f 100644 --- a/src/client/spark/Orders/OrderTypeVMs/OCOOrderVM.cs +++ b/src/client/spark/Orders/OrderTypeVMs/OCOOrderVM.cs @@ -28,6 +28,7 @@ namespace mbts.spark this.other_order_types = new InlineUserProperty>(child_order_types, true); other_ticks = new InlineUserProperty(1, false); + other_side = new InlineUserProperty(bts.api.Side.Undefined, false); } public readonly InlineUserProperty selected_child_order_type; public readonly InlineUserProperty> child_order_types; @@ -35,6 +36,7 @@ namespace mbts.spark public readonly InlineUserProperty> other_order_types; public readonly InlineUserProperty other_ticks; + public readonly InlineUserProperty other_side; } @@ -53,6 +55,7 @@ namespace mbts.spark IDisposable _other_order_errors_sub; public int other_ticks { get; set; } = 1; + public bts.api.Side other_side { get; set; } = bts.api.Side.Undefined; public OCOOrderVM(OrdersModel model, IObservable security_md_source, string id, string display_name) : this(model, security_md_source, security_md_source.Select(smd => @@ -86,6 +89,11 @@ namespace mbts.spark other_ticks = p.other_ticks.value; RaisePropertyChanged(nameof(other_ticks)); } + if (p.other_side.force_override) + { + other_side = p.other_side.value; + RaisePropertyChanged(nameof(other_side)); + } _inline_props = p; _validate(); }) @@ -104,6 +112,7 @@ namespace mbts.spark { var i = factory.oco_order(); i.OtherTicks = other_ticks; + i.OtherSide = other_side; child_order.assign_to(i.Child); if (other_order != null) other_order.assign_to(i.Other); @@ -111,6 +120,8 @@ namespace mbts.spark } } + public IEnumerable other_side_types => EnumHelper.get_values(); + public ICollection child_order_types { get diff --git a/src/client/spark/core/FontsAndColors/BTSColors.cs b/src/client/spark/core/FontsAndColors/BTSColors.cs index cb52e8c2e..0290af41f 100644 --- a/src/client/spark/core/FontsAndColors/BTSColors.cs +++ b/src/client/spark/core/FontsAndColors/BTSColors.cs @@ -211,6 +211,8 @@ namespace mbts.ui { public static BTSColor lean_order { get { return new BTSColor(255, 170, 210); } } public static BTSColor join_order { get { return new BTSColor(200, 171, 232); } } public static BTSColor cover_order { get { return new BTSColor(134, 232, 224); } } + public static BTSColor oco_order { get { return new BTSColor(110, 94, 11); } } + public static BTSColor pricer_order { get { return new BTSColor(170, 132, 150); } } public static BTSColor trigger_order { get { return new BTSColor(172, 255, 179); } } public static BTSColor off_order { get { return new BTSColor(0x8a, 0x00, 0x5f); } }//8a005f public static BTSColor top_order { get { return new BTSColor(0xdd, 0x7a, 0x27); } }//dd7a27 diff --git a/src/client/spark/view/Controls/Order/OrderDetailsPopupControl.xaml b/src/client/spark/view/Controls/Order/OrderDetailsPopupControl.xaml index 2a76739e1..3e611e39a 100644 --- a/src/client/spark/view/Controls/Order/OrderDetailsPopupControl.xaml +++ b/src/client/spark/view/Controls/Order/OrderDetailsPopupControl.xaml @@ -166,51 +166,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - Cross Thresh - - - Cross Ticks - - Max Chase Ticks - - Detach on Toward - Initial Tick Offset - - Child - - - - @@ -426,6 +381,8 @@ + + @@ -433,23 +390,25 @@ - + Other Ticks + + Other Side + + + Other Child hedges; } +bool BookAlgo::is_detached() const +{ + return me->is_detached; +} + void BookAlgo::on_init_new(const api::orderman::OrderDefinition& def, const PreActivationOrderServices &services) @@ -243,7 +248,7 @@ void BookAlgo::on_update(Price p, Quantity q,PerfTrace const&) case ORDER_STATUS_WORKING: set_price(p.value()); set_quantity(q.value()); - if(!me->fake_detached_child) + if(!me->is_detached) me->child.update(p,q); break; case ORDER_STATUS_PENDING_WORKING: @@ -300,7 +305,7 @@ void BookAlgo::on_cancel(PerfTrace const&) transition(ORDER_STATUS_CANCELED); break; case ORDER_STATUS_WORKING: - if(!me->fake_detached_child) + if(!me->is_detached) me->child.cancel(); for(auto& i : me->hedges) i.second.crack(); @@ -352,7 +357,7 @@ void BookAlgo::transition(api::orderman::OrderStatus new_status, std::string msg void BookAlgo::on_child_order_update(const OrderStateUpdateEvent& ev) { - if(me->fake_detached_child) + if(me->is_detached) return; if(ev.key != me->child.key()) //hedge state change { @@ -372,14 +377,15 @@ void BookAlgo::on_child_order_update(const OrderStateUpdateEvent& ev) process_hedge_state_change(orig_state, ev); break; case ORDER_STATUS_COMPLETED: - transition(ORDER_STATUS_COMPLETED); process_hedge_state_change(orig_state, ev); + transition(ORDER_STATUS_COMPLETED); break; case ORDER_STATUS_DISCONNECTED: case ORDER_STATUS_REJECTED_BY_SPARK: case ORDER_STATUS_CANCELED: case ORDER_STATUS_ERROR: + process_hedge_state_change(orig_state, ev); transition(ORDER_STATUS_ERROR, "Unexpected child order state"); break; @@ -420,15 +426,16 @@ void BookAlgo::on_child_order_update(const OrderStateUpdateEvent& ev) case ORDER_STATUS_WORKING: process_child_state_change(orig_state, ev); break; - case ORDER_STATUS_COMPLETED: - transition(ORDER_STATUS_COMPLETED); + case ORDER_STATUS_COMPLETED: process_child_state_change(orig_state, ev); + transition(ORDER_STATUS_COMPLETED); break; case ORDER_STATUS_DISCONNECTED: case ORDER_STATUS_REJECTED_BY_SPARK: case ORDER_STATUS_CANCELED: case ORDER_STATUS_ERROR: + process_child_state_change(orig_state, ev); transition(ORDER_STATUS_ERROR, "Unexpected child order state"); break; @@ -460,7 +467,7 @@ void BookAlgo::on_child_order_update(const OrderStateUpdateEvent& ev) void BookAlgo::report_state() { - auto merged_state = me->state; + OrderState merged_state = me->state; merged_state.set_filled_quantity(child_state().filled_quantity()); merged_state.set_last_price(child_state().last_price()); merged_state.set_avg_price(child_state().avg_price()); @@ -484,7 +491,7 @@ void BookAlgo::paydown(const md::PriceSize& opposite, const bts::orderman::BookU void BookAlgo::detach(bts::api::orderman::OrderStatus status_after_detach) { - me->fake_detached_child = true; + me->is_detached = true; me->child.crack(); transition(status_after_detach); report_state(); diff --git a/src/lib/orderman/orders/book_algos/BookAlgo.h b/src/lib/orderman/orders/book_algos/BookAlgo.h index a2f7913c8..0806def1d 100644 --- a/src/lib/orderman/orders/book_algos/BookAlgo.h +++ b/src/lib/orderman/orders/book_algos/BookAlgo.h @@ -62,6 +62,7 @@ protected: void activate_child(Price p, Quantity q); void update_child(Price, Quantity); void cancel_child(); + bool is_detached() const; void build_and_send_hedge(Price, Quantity); typedef boost::container::flat_map HedgeMap; diff --git a/src/lib/orderman/orders/book_algos/OCOOrder.cpp b/src/lib/orderman/orders/book_algos/OCOOrder.cpp index 9eb8576f2..f2213d221 100644 --- a/src/lib/orderman/orders/book_algos/OCOOrder.cpp +++ b/src/lib/orderman/orders/book_algos/OCOOrder.cpp @@ -15,7 +15,9 @@ struct OCOOrder::PImpl PImpl() {} int other_ticks; + bts::api::Side other_side; OrderDefinition other_def; + bool hedge_sent; }; @@ -31,6 +33,8 @@ void OCOOrder::setup_child(const api::orderman::OrderDefinition& this_def, const *out_child->mutable_custom_def() = oco_def.child(); me->other_ticks = oco_def.other_ticks(); + me->other_side = oco_def.other_side(); + me->hedge_sent = false; } //hedge here means 'other' @@ -41,10 +45,15 @@ bool OCOOrder::setup_hedge(const api::orderman::OrderDefinition& this_def, const if(oco_def.has_other()) { *out_hedge->mutable_custom_def() = oco_def.other(); - if(definition().side() == api::SIDE_BUY) - out_hedge->set_side(api::SIDE_SELL); + if(me->other_side == api::SIDE_UNDEFINED) + { + if(definition().side() == api::SIDE_BUY) + out_hedge->set_side(api::SIDE_SELL); + else + out_hedge->set_side(api::SIDE_BUY); + } else - out_hedge->set_side(api::SIDE_BUY); + out_hedge->set_side(me->other_side); return true; } else @@ -54,20 +63,20 @@ bool OCOOrder::setup_hedge(const api::orderman::OrderDefinition& this_def, const void OCOOrder::process_child_state_change(const api::orderman::OrderState& orig_state, const OrderStateUpdateEvent& new_state) { uint32_t fill_qty = new_state.filled_quantity - orig_state.filled_quantity(); - if(fill_qty > 0) //reduce hedge + const auto& the_state = hedges_state(); + auto& the_hedges = hedges(); + for(const auto& i : the_state) { - const auto& the_state = hedges_state(); - auto& the_hedges = hedges(); - for(const auto& i : the_state) - { - const auto& hedge_state = i.second; + const auto& hedge_state = i.second; + + uint32_t orig_qty = hedge_state.quantity(); + uint32_t newqty = orig_qty - std::min(fill_qty, orig_qty); + bts_assert(newqty <= orig_qty); - uint32_t orig_qty = hedge_state.quantity(); - uint32_t newqty = orig_qty - fill_qty; - bts_assert(newqty < orig_qty); - - auto it = the_hedges.find(i.first); - if(it != the_hedges.end()) + auto it = the_hedges.find(i.first); + if(it != the_hedges.end()) + { + if(newqty != orig_qty) { if(newqty > 0) { @@ -77,30 +86,41 @@ void OCOOrder::process_child_state_change(const api::orderman::OrderState& orig_ else it->second.cancel(); } - else + else if(new_state.status != api::orderman::OrderStatus::ORDER_STATUS_PENDING_WORKING && + new_state.status != api::orderman::OrderStatus::ORDER_STATUS_WORKING) { - LOG(OrderMan, Error, "OCO Can't find hedge order {}", i.first); - } + it->second.cancel(); + } + } + else + { + LOG(OrderMan, Error, "OCO Can't find hedge order {}", i.first); + cancel_child(); } } } void OCOOrder::process_hedge_state_change(const api::orderman::OrderState& orig_state, const OrderStateUpdateEvent& new_state) -{ +{ uint32_t fill_qty = new_state.filled_quantity - orig_state.filled_quantity(); - if(fill_qty > 0) //reduce child + uint32_t orig_qty = child_state().quantity(); + uint32_t newqty = orig_qty - std::min(fill_qty, orig_qty); + bts_assert(newqty <= orig_qty); + if(newqty != orig_qty) { - uint32_t orig_qty = child_state().quantity(); - uint32_t newqty = orig_qty - fill_qty; - bts_assert(newqty < orig_qty); if(newqty > 0) { LOG(OrderMan, Info, "OCO reducing child qty from {} to {}", child_state().quantity(), newqty); update_child(Price(child_state().price()), Quantity(newqty)); } - else + else cancel_child(); } + else if(new_state.status != api::orderman::OrderStatus::ORDER_STATUS_PENDING_WORKING && + new_state.status != api::orderman::OrderStatus::ORDER_STATUS_WORKING) + { + cancel_child(); + } } void OCOOrder::on_book_update(const bts::orderman::BookUpdateEvent& ) @@ -113,13 +133,17 @@ void OCOOrder::on_book_update(const bts::orderman::BookUpdateEvent& ) case ORDER_STATUS_INACTIVE: { activate_child(Price(this_state().price()), Quantity(this_state().quantity())); - - double prx = this_state().price(); - if(definition().side() == api::SIDE_BUY) - prx = prx + tick_size() * me->other_ticks; - else - prx = prx - tick_size() * me->other_ticks; - build_and_send_hedge( Price(prx), Quantity(this_state().quantity()) ); + + if(!me->hedge_sent) + { + double prx = this_state().price(); + if(definition().side() == api::SIDE_BUY) + prx = prx + tick_size() * me->other_ticks; + else + prx = prx - tick_size() * me->other_ticks; + build_and_send_hedge( Price(prx), Quantity(this_state().quantity()) ); + me->hedge_sent = true; + } transition(ORDER_STATUS_WORKING); report_state(); } @@ -161,6 +185,63 @@ void OCOOrder::on_book_update(const bts::orderman::BookUpdateEvent& ) } } +void OCOOrder::on_cancel(PerfTrace const&) +{ + LOG(OrderMan, Info, "Canceling OCOOrder Order {}", key().value()); + + switch (this_state().status()) + { + case ORDER_STATUS_WORKING: + switch (child_state().status()) + { + case ORDER_STATUS_INACTIVE: + transition(ORDER_STATUS_CANCELED); + break; + case ORDER_STATUS_WORKING: + if(!is_detached()) + { + cancel_child(); + for(auto& i : hedges()) + i.second.cancel(); + } + transition(ORDER_STATUS_CANCELED); + break; + case ORDER_STATUS_PENDING_WORKING: + // child is running but it cannot be canceled at this point + transition(this_state().status(), "Cannot cancel order"); + break; + + // child cannot be in any of these if we are in the working state + case ORDER_STATUS_DISCONNECTED: + case ORDER_STATUS_ERROR: + case ORDER_STATUS_REJECTED_BY_SPARK: + case ORDER_STATUS_COMPLETED: + case ORDER_STATUS_CANCELED: + case OrderStatus_INT_MIN_SENTINEL_DO_NOT_USE_: + case OrderStatus_INT_MAX_SENTINEL_DO_NOT_USE_: + bts_assert(false); + break; + } + break; + case ORDER_STATUS_REJECTED_BY_SPARK: + case ORDER_STATUS_COMPLETED: + case ORDER_STATUS_CANCELED: + case ORDER_STATUS_ERROR: + transition(this_state().status(), "Cannot cancel order"); + break; + + // these are unreacheable for this order type + case ORDER_STATUS_PENDING_WORKING: + case ORDER_STATUS_DISCONNECTED: + case ORDER_STATUS_INACTIVE: + case OrderStatus_INT_MIN_SENTINEL_DO_NOT_USE_: + case OrderStatus_INT_MAX_SENTINEL_DO_NOT_USE_: + bts_assert(false); + break; + } + report_state(); +} + } } diff --git a/src/lib/orderman/orders/book_algos/OCOOrder.h b/src/lib/orderman/orders/book_algos/OCOOrder.h index 31a1ed29f..49a4eda7b 100644 --- a/src/lib/orderman/orders/book_algos/OCOOrder.h +++ b/src/lib/orderman/orders/book_algos/OCOOrder.h @@ -18,6 +18,7 @@ public: OCOOrder(); virtual ~OCOOrder(); + void on_cancel(PerfTrace const&) override ; protected: void setup_child(const api::orderman::OrderDefinition&, const PreActivationOrderServices&, api::orderman::OrderState&, api::orderman::OrderDefinition*) override; From f04af5af9d204d3d28c520c4b4b95c6f983ae2c3 Mon Sep 17 00:00:00 2001 From: Kevin Darby Date: Thu, 20 Dec 2018 09:39:05 -0500 Subject: [PATCH 04/85] fix build --- src/lib/orderman/order_factory/OrderFactory.cpp | 2 +- src/lib/orderman/orders/book_algos/BookAlgo.cpp | 12 ++++++------ src/lib/orderman/orders/book_algos/OCOOrder.cpp | 2 +- src/lib/orderman/orders/book_algos/OCOOrder.h | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/lib/orderman/order_factory/OrderFactory.cpp b/src/lib/orderman/order_factory/OrderFactory.cpp index 4728c483a..25b44deea 100644 --- a/src/lib/orderman/order_factory/OrderFactory.cpp +++ b/src/lib/orderman/order_factory/OrderFactory.cpp @@ -135,7 +135,7 @@ Order &OrderFactory::allocate(OrderKey k, OrderKey pk, order_ptr.reset(new orderman::TimeBoxOrder()); break; case CustomOrderDefinition::kTick: - order_ptr.reset(new orderman::TickOrder()); + bts_assert(false); break; case CustomOrderDefinition::kLean: order_ptr.reset(new orderman::LeanOrder()); diff --git a/src/lib/orderman/orders/book_algos/BookAlgo.cpp b/src/lib/orderman/orders/book_algos/BookAlgo.cpp index 393266487..4c8874cda 100644 --- a/src/lib/orderman/orders/book_algos/BookAlgo.cpp +++ b/src/lib/orderman/orders/book_algos/BookAlgo.cpp @@ -191,7 +191,7 @@ void BookAlgo::on_init_recovered(const api::orderman::OrderRecord& rec, } } -void BookAlgo::on_activate(Price p, Quantity q, OrderServices& services,PerfTrace const&) +void BookAlgo::on_activate(Price p, Quantity q, OrderServices& services, trace::CorID const&) { bts_assert(!me->services); me->services = &services; @@ -205,9 +205,9 @@ void BookAlgo::on_activate(Price p, Quantity q, OrderServices& services,PerfTrac me->child = me->child_builder.build(); set_price(p.value()); set_quantity(q.value()); - services.child_updates.sub(key(), + services.subscribe_to_child_updates( [this](auto const &os) { this->on_child_order_update(os); }); - services.book_updates.sub(SecurityKey(me->def.security_key()), + services.subscribe_to_book(SecurityKey(me->def.security_key()), [this](auto const &b) { this->on_book_update(b); }); transition(ORDER_STATUS_WORKING); break; @@ -233,7 +233,7 @@ void BookAlgo::on_activate(Price p, Quantity q, OrderServices& services,PerfTrac report_state(); } -void BookAlgo::on_update(Price p, Quantity q,PerfTrace const&) +void BookAlgo::on_update(Price p, Quantity q, trace::CorID const&) { LOG(OrderMan, Info, "Updating BookAlgo {}", key().value()); switch (this_state().status()) @@ -290,9 +290,9 @@ void BookAlgo::on_update(Price p, Quantity q,PerfTrace const&) report_state(); } -void BookAlgo::on_crack(PerfTrace const&) {} +void BookAlgo::on_crack(trace::CorID const&) {} -void BookAlgo::on_cancel(PerfTrace const&) +void BookAlgo::on_cancel(trace::CorID const&) { LOG(OrderMan, Info, "Canceling BookAlgo Order {}", key().value()); diff --git a/src/lib/orderman/orders/book_algos/OCOOrder.cpp b/src/lib/orderman/orders/book_algos/OCOOrder.cpp index f2213d221..0486b2cc8 100644 --- a/src/lib/orderman/orders/book_algos/OCOOrder.cpp +++ b/src/lib/orderman/orders/book_algos/OCOOrder.cpp @@ -185,7 +185,7 @@ void OCOOrder::on_book_update(const bts::orderman::BookUpdateEvent& ) } } -void OCOOrder::on_cancel(PerfTrace const&) +void OCOOrder::on_cancel(trace::CorID const&) { LOG(OrderMan, Info, "Canceling OCOOrder Order {}", key().value()); diff --git a/src/lib/orderman/orders/book_algos/OCOOrder.h b/src/lib/orderman/orders/book_algos/OCOOrder.h index 49a4eda7b..5fd0e8789 100644 --- a/src/lib/orderman/orders/book_algos/OCOOrder.h +++ b/src/lib/orderman/orders/book_algos/OCOOrder.h @@ -18,7 +18,7 @@ public: OCOOrder(); virtual ~OCOOrder(); - void on_cancel(PerfTrace const&) override ; + void on_cancel(trace::CorID const&) override ; protected: void setup_child(const api::orderman::OrderDefinition&, const PreActivationOrderServices&, api::orderman::OrderState&, api::orderman::OrderDefinition*) override; From 196c518ac659e23395bfe093b438a5ca84a38771 Mon Sep 17 00:00:00 2001 From: Kevin Darby Date: Thu, 20 Dec 2018 09:43:44 -0500 Subject: [PATCH 05/85] remove tick order --- .../Orders/OrderTypeVMs/OrderTypeVMFactory.cs | 2 - .../spark/Orders/OrderTypeVMs/TickOrderVM.cs | 201 ------------------ .../orderman/orders/book_algos/TickOrder.cpp | 172 --------------- .../orderman/orders/book_algos/TickOrder.h | 35 --- 4 files changed, 410 deletions(-) delete mode 100644 src/client/spark/Orders/OrderTypeVMs/TickOrderVM.cs delete mode 100644 src/lib/orderman/orders/book_algos/TickOrder.cpp delete mode 100644 src/lib/orderman/orders/book_algos/TickOrder.h diff --git a/src/client/spark/Orders/OrderTypeVMs/OrderTypeVMFactory.cs b/src/client/spark/Orders/OrderTypeVMs/OrderTypeVMFactory.cs index 40f4d857f..6624bf2f5 100644 --- a/src/client/spark/Orders/OrderTypeVMs/OrderTypeVMFactory.cs +++ b/src/client/spark/Orders/OrderTypeVMs/OrderTypeVMFactory.cs @@ -15,7 +15,6 @@ namespace mbts.spark { public static IOrderTypeVM stop(OrdersModel model, string id, string display_name) => new StopOrderVM(model, id, display_name); public static IOrderTypeVM stop_limit(OrdersModel model, IObservable security_md_source, string id, string display_name) => new StopLimitOrderVM(model, security_md_source, id, display_name); public static IOrderTypeVM time_box(OrdersModel model, IObservable security_md_source, string id, string display_name) => new TimeBoxOrderVM(model, security_md_source, id, display_name); - public static IOrderTypeVM tick(OrdersModel model, IObservable security_md_source, string id, string display_name) => new TickOrderVM(model, security_md_source, id, display_name); public static IOrderTypeVM lean(OrdersModel model, IObservable security_md_source, string id, string display_name) => new LeanOrderVM(model, security_md_source, id, display_name); public static IOrderTypeVM join(OrdersModel model, IObservable security_md_source, string id, string display_name) => new JoinOrderVM(model, security_md_source, id, display_name); public static IOrderTypeVM top(OrdersModel model, IObservable security_md_source, string id, string display_name) => new TopOrderVM(model, security_md_source, id, display_name); @@ -36,7 +35,6 @@ namespace mbts.spark { case CustomOneofCase.Stop: return stop(model, id, display_name); case CustomOneofCase.StopLimit: return stop_limit(model, security_md_source, id, display_name); case CustomOneofCase.TimeBox: return time_box(model, security_md_source, id, display_name); - case CustomOneofCase.Tick: return tick(model, security_md_source, id, display_name); case CustomOneofCase.Lean: return lean(model, security_md_source, id, display_name); case CustomOneofCase.Join: return join(model, security_md_source, id, display_name); case CustomOneofCase.Top: return top(model, security_md_source, id, display_name); diff --git a/src/client/spark/Orders/OrderTypeVMs/TickOrderVM.cs b/src/client/spark/Orders/OrderTypeVMs/TickOrderVM.cs deleted file mode 100644 index ee868ef2b..000000000 --- a/src/client/spark/Orders/OrderTypeVMs/TickOrderVM.cs +++ /dev/null @@ -1,201 +0,0 @@ -using bts.api; -using bts.api.orderman; -using mbts.ui; -using mbts.ui.utility; -using mbts.utils; -using System; -using System.Collections.Generic; -using System.Collections.ObjectModel; -using System.ComponentModel; -using System.Linq; -using System.Reactive.Linq; -using static bts.api.orderman.CustomOrderDefinition; - -namespace mbts.spark -{ - static partial class OrderTypeVMFactory - { - class InlineTickOrderProperties - { - public InlineTickOrderProperties (OrderTemplateHeader selected_child_order_type, IReadOnlyCollection child_order_types) - { - ExceptionAssert.InvalidOperation.That(child_order_types.Contains(selected_child_order_type), $"Child order type {selected_child_order_type} is not in child_order_types"); - this.selected_child_order_type = new InlineUserProperty(selected_child_order_type, true); - this.child_order_types = new InlineUserProperty>(child_order_types, true); - tick_offset_on_deploy = new InlineUserProperty(0, false); - cross_threshold = new InlineUserProperty( - new bts.spark.QuantityMeasure() { Ratio = 1.0, Absolute = 1}, false); - cross_ticks = new InlineUserProperty(0, false); - max_chase_ticks = new InlineUserProperty(10, false); - detach_on_toward = new InlineUserProperty(true, false); - } - public readonly InlineUserProperty selected_child_order_type; - public readonly InlineUserProperty> child_order_types; - - public readonly InlineUserProperty tick_offset_on_deploy; - public readonly InlineUserProperty cross_threshold; - public readonly InlineUserProperty cross_ticks; - public readonly InlineUserProperty max_chase_ticks; - public readonly InlineUserProperty detach_on_toward; - } - - class TickOrderVM : OrderTypeVMBase, IHierarchicalOrderTypeVM, IDisposable - { - readonly IObservable _security_md_source; - SecurityImmutable? _security; - ObservableCollection _order_types = new ObservableCollection(); - ObservableAutoWrapper _child_order_choices; - InlineTickOrderProperties _inline_props; - IOrderTypeVM _child_order; - IDisposable _dispose, _child_order_errors_sub; - - public uint tick_offset_on_deploy { get; set; } = 0; - public bts.spark.QuantityMeasure cross_threshold { get; set; } = new bts.spark.QuantityMeasure() { Ratio = 1.0, Absolute = 1 }; - public uint cross_ticks { get; set; } = 0; - public uint max_chase_ticks { get; set; } = 10; - public bool detach_on_toward { get; set; } = true; - - public TickOrderVM(OrdersModel model, IObservable security_md_source, string id, string display_name) - : this(model, security_md_source, security_md_source.Select(smd => - new InlineTickOrderProperties(new OrderTemplateHeader(model.factory.default_order_type(smd.security)), model.factory.default_order_types(smd.security).Select(t => new OrderTemplateHeader(t)).ToList())), id, display_name) { } - - public TickOrderVM(OrdersModel model, IObservable security_md_source, - IObservable inline_properties, string id, string display_name) : base(model, id, display_name) - { - _security_md_source = security_md_source; - _dispose = Disposable.Create( - security_md_source.OnDispatcher() - .Subscribe(smd => _security = smd.security), - inline_properties.OnDispatcher() - .Subscribe(p => { - if (_order_types.Count == 0 || p.child_order_types.force_override) - { - _order_types.SetItems(p.child_order_types.value); - } - if (_child_order_choices != null && (_child_order == null || p.selected_child_order_type.force_override)) - { - _select_child_order(p.selected_child_order_type.value.key.type); - } - if (p.tick_offset_on_deploy.force_override) - { - tick_offset_on_deploy = p.tick_offset_on_deploy.value; - RaisePropertyChanged(nameof(tick_offset_on_deploy)); - } - if (p.cross_threshold.force_override) - { - cross_threshold = p.cross_threshold.value; - RaisePropertyChanged(nameof(cross_threshold)); - } - if (p.cross_ticks.force_override) - { - cross_ticks = p.cross_ticks.value; - RaisePropertyChanged(nameof(cross_ticks)); - } - if (p.max_chase_ticks.force_override) - { - max_chase_ticks = p.max_chase_ticks.value; - RaisePropertyChanged(nameof(max_chase_ticks)); - } - if (p.detach_on_toward.force_override) - { - detach_on_toward = p.detach_on_toward.value; - RaisePropertyChanged(nameof(detach_on_toward)); - } - _inline_props = p; - _validate(); - }) - ); - _validate(); - } - - public override void assign_to(CustomOrderDefinition value) - { - if (_child_order == null) - _select_child_order(model.factory.default_order_type(_security.Value, true)); - _validate(); - if (!HasErrors) - { - var i = factory.tick_order(); - i.TickOffsetOnDeploy = tick_offset_on_deploy; - i.CrossThreshold = cross_threshold; - i.CrossTicks = cross_ticks; - i.MaxChaseTicks = max_chase_ticks; - i.DetachOnToward = detach_on_toward; - child_order.assign_to(i.Child); - value.Tick = i; - } - } - - public ICollection child_order_types - { - get - { - if (_child_order_choices == null) - _child_order_choices = new ObservableAutoWrapper(_order_types, ot => create(ot.key.type, ot.key.name, ot.display_name, model, _security_md_source)); - return _child_order_choices; - } - } - - public IOrderTypeVM child_order - { - get - { - init(); - return _child_order; - } - set - { - Disposable.dispose(ref _child_order_errors_sub); - _child_order = value; - _child_order?.init(); - if (_child_order is INotifyDataErrorInfo ndei) - { - ndei.ErrorsChanged += on_errors_changed; - void on_errors_changed(object s, DataErrorsChangedEventArgs e) => - _validate(); - _child_order_errors_sub = Disposable.Create(() => ndei.ErrorsChanged -= on_errors_changed); - } - _validate(); - RaisePropertyChanged(nameof(child_order)); - } - } - - public void Dispose() - { - Disposable.dispose(ref _dispose); - if (_child_order_choices != null) - _child_order_choices.OfType().DisposeAll(); - } - - public override void init() - { - if (_child_order == null && _security.HasValue) - _select_child_order(); - _validate(); - } - - void _select_child_order() - { - CustomOneofCase ot = _inline_props != null && _inline_props.selected_child_order_type.has_value ? - _inline_props.selected_child_order_type.value.key.type : - model.factory.default_order_type(_security.Value, true); - _select_child_order(ot); - } - - void _select_child_order(CustomOneofCase ot) - { - _child_order = child_order_types.FirstOrDefault(vm => vm.order_type == ot); - if (_child_order == null) - _child_order = child_order_types.FirstOrDefault(); - RaisePropertyChanged(nameof(child_order)); - } - - void _validate() - { - if (_child_order == null || (_child_order is INotifyDataErrorInfo ndei && ndei.HasErrors)) - set_error(nameof(child_order), string.Empty, true); - else clear_error(nameof(child_order)); - } - } - } -} \ No newline at end of file diff --git a/src/lib/orderman/orders/book_algos/TickOrder.cpp b/src/lib/orderman/orders/book_algos/TickOrder.cpp deleted file mode 100644 index d05404458..000000000 --- a/src/lib/orderman/orders/book_algos/TickOrder.cpp +++ /dev/null @@ -1,172 +0,0 @@ -#include "TickOrder.h" - -#include "lib/orderman/actors/order_manager/OrderBuilder.h" -#include "lib/orderman/orders/spreader_orders/spreader/SpreaderUtils.h" -#include "lib/utils/PBUtils.h" -#include - -namespace bts { -namespace orderman { - -using namespace api::orderman; - -struct TickOrder::PImpl { - PImpl() {} - - uint32_t tick_offset_on_deploy; - spark::QuantityMeasure cross_threshold; - uint32_t cross_ticks; - uint32_t max_chase_ticks; - bool detach_on_toward; - - md::PriceSize opposite_at_inception; -}; - -TickOrder::TickOrder() : me(new PImpl()) {} - -TickOrder::~TickOrder() {} - -void TickOrder::setup_child(const api::orderman::OrderDefinition &this_def, - const PreActivationOrderServices &, - api::orderman::OrderState &initial_state, - api::orderman::OrderDefinition *out_child) { - initial_state.mutable_tick(); - auto const &tick_def = this_def.custom_def().tick(); - *out_child->mutable_custom_def() = tick_def.child(); - - me->tick_offset_on_deploy = tick_def.tick_offset_on_deploy(); - me->cross_threshold = tick_def.cross_threshold(); - me->cross_ticks = tick_def.cross_ticks(); - me->max_chase_ticks = tick_def.max_chase_ticks(); - me->cross_threshold = tick_def.cross_threshold(); - me->detach_on_toward = tick_def.detach_on_toward(); -} - -void TickOrder::payup(const md::PriceSize &opposite, - const bts::orderman::BookUpdateEvent &, - uint32_t cross_ticks) { - double new_price = opposite.price() + tick_size() * cross_ticks; - double limit = - me->opposite_at_inception.price() + tick_size() * me->max_chase_ticks; - if (new_price <= limit) - update_child(Price(new_price), Quantity(this_state().quantity())); - else { - update_child(Price(limit), Quantity(this_state().quantity())); - detach(); - } - report_state(); -} - -void TickOrder::paydown(const md::PriceSize &opposite, - const bts::orderman::BookUpdateEvent &, - uint32_t cross_ticks) { - double new_price = opposite.price() - tick_size() * cross_ticks; - double limit = - me->opposite_at_inception.price() - tick_size() * me->max_chase_ticks; - if (new_price >= limit) - update_child(Price(new_price), Quantity(this_state().quantity())); - else { - update_child(Price(limit), Quantity(this_state().quantity())); - detach(); - } - report_state(); -} - -void TickOrder::on_book_update(const bts::orderman::BookUpdateEvent &book) { - switch (this_state().status()) { - case ORDER_STATUS_WORKING: - switch (child_state().status()) { - case ORDER_STATUS_INACTIVE: { - if (definition().side() == api::SIDE_BUY) { - me->opposite_at_inception = book.book.side(api::Side::SIDE_SELL)[0]; - if (me->tick_offset_on_deploy > 0) { - double price = me->opposite_at_inception.price() - - tick_size() * me->tick_offset_on_deploy; - activate_child(Price(price), Quantity(this_state().quantity())); - transition(ORDER_STATUS_WORKING); - } else { - activate_child(Price(this_state().price()), - Quantity(this_state().quantity())); - transition(ORDER_STATUS_WORKING); - } - } else { - me->opposite_at_inception = book.book.side(api::Side::SIDE_BUY)[0]; - if (me->tick_offset_on_deploy > 0) { - double price = me->opposite_at_inception.price() + - tick_size() * me->tick_offset_on_deploy; - activate_child(Price(price), Quantity(this_state().quantity())); - transition(ORDER_STATUS_WORKING); - } else { - activate_child(Price(this_state().price()), - Quantity(this_state().quantity())); - transition(ORDER_STATUS_WORKING); - } - } - report_state(); - } break; - case ORDER_STATUS_WORKING: { - Quantity thresh = SpreaderUtils::_quantity_threshold( - me->cross_threshold, Quantity(this_state().quantity())); - if (definition().side() == api::SIDE_BUY) { - const auto &opposite = book.book.side(api::Side::SIDE_SELL)[0]; - if (opposite.price() < me->opposite_at_inception.price()) { - if (me->detach_on_toward) - detach(); - else - me->opposite_at_inception = opposite; - } else if (opposite.price() > me->opposite_at_inception.price()) - payup(opposite, book, me->cross_ticks); - else { - if (opposite.size() <= thresh.value()) - payup(opposite, book, me->cross_ticks); - } - } else { - const auto &opposite = book.book.side(api::Side::SIDE_BUY)[0]; - if (opposite.price() > me->opposite_at_inception.price()) { - if (me->detach_on_toward) - detach(); - else - me->opposite_at_inception = opposite; - } else if (opposite.price() < me->opposite_at_inception.price()) - paydown(opposite, book, me->cross_ticks); - else { - if (opposite.size() <= thresh.value()) - paydown(opposite, book, me->cross_ticks); - } - } - } break; - - case ORDER_STATUS_PENDING_WORKING: - break; - - // child cannot be in any of these states if we are in the working state - case ORDER_STATUS_DISCONNECTED: - case ORDER_STATUS_ERROR: - case ORDER_STATUS_REJECTED_BY_SPARK: - case ORDER_STATUS_COMPLETED: - case ORDER_STATUS_CANCELED: - case OrderStatus_INT_MIN_SENTINEL_DO_NOT_USE_: - case OrderStatus_INT_MAX_SENTINEL_DO_NOT_USE_: - bts_assert(false); - break; - } - break; - - case ORDER_STATUS_COMPLETED: - case ORDER_STATUS_CANCELED: - case ORDER_STATUS_ERROR: - break; - - case ORDER_STATUS_INACTIVE: - case ORDER_STATUS_REJECTED_BY_SPARK: - case ORDER_STATUS_PENDING_WORKING: - case ORDER_STATUS_DISCONNECTED: - case OrderStatus_INT_MIN_SENTINEL_DO_NOT_USE_: - case OrderStatus_INT_MAX_SENTINEL_DO_NOT_USE_: - bts_assert(false); - break; - } -} - -} // namespace orderman -} // namespace bts diff --git a/src/lib/orderman/orders/book_algos/TickOrder.h b/src/lib/orderman/orders/book_algos/TickOrder.h deleted file mode 100644 index 82679d93c..000000000 --- a/src/lib/orderman/orders/book_algos/TickOrder.h +++ /dev/null @@ -1,35 +0,0 @@ -#pragma once - -#include "lib/orderman/orders/book_algos/BookAlgo.h" - -namespace bts { -namespace orderman { - -class OrderStateUpdateEvent; -class BookUpdateEvent; - -//TickOrder deploys N ticks away from the initial opposite inside market and watches for it to trade below a certain size. -//If that size goes below a threshold or the market moves away, TickOrder Pays up once. -//The process repeats until the algo has paid up a max of N ticks from the original opposite inside order. -//If the market moves toward, the algo either starts watching the new inside market and keeps going or detaches depending on a parameter - -class TickOrder : public BookAlgo -{ -public: - TickOrder(); - virtual ~TickOrder(); - -protected: - void setup_child(const api::orderman::OrderDefinition& this_def, const PreActivationOrderServices&, - api::orderman::OrderState&, api::orderman::OrderDefinition*) override; - void on_book_update(const bts::orderman::BookUpdateEvent&) override; -private: - void payup(const md::PriceSize& opposite, const bts::orderman::BookUpdateEvent&, uint32_t cross_ticks) override; - void paydown(const md::PriceSize& opposite, const bts::orderman::BookUpdateEvent&, uint32_t cross_ticks) override; - -private: - struct PImpl; - std::unique_ptr me; -}; - -}} From 4af4af86778839a9dbbb70c67b86d6f21aab049d Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 20 Dec 2018 15:49:41 -0500 Subject: [PATCH 06/85] SPARK-1731, SPARK-1947 - Don't load security db in md producer --- .../sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp index 425a3e8d3..26d24e5fb 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp @@ -3,7 +3,6 @@ #include "lib/md/cme/MDP3Utils.h" #include "lib/config/config.h" #include "lib/md/coconut2.h" -#include "lib/md/SecurityDB.h" #include "lib/os/os.h" #include "lib/utils/Dispatcher.h" #include "lib/utils/proto_helper.h" @@ -343,16 +342,6 @@ unique_ptr MDP3Main::prepare_listener(onixs::Handler& handler, con listener->add_filter_symbol(symbol.symbol()); } - if (options.load_security_db()) { - // Load the list of known securities and add them to the index for fast startup - db::load_securities(channel.symbols, [&](const api::Security& security) { - with_security_like(security, [&](FixedSecurity& fs) { - convert_security(security, fs, true); - listener->add_security(fs); - }); - }); - } - handler.registerListener(*listener); handler.registerMarketDataListener(*listener); handler.registerSecurityListener(*listener); From 8a7174378a645bfa9dde8cf6915c918f654a57bd Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 1 Jan 2019 13:32:40 -0500 Subject: [PATCH 07/85] SPARK-1948 Implement SecurityCache --- src/lib/md/SecurityCache.cpp | 104 +++++++++++++++++++++++++++++++++++ src/lib/md/SecurityCache.h | 34 ++++++++++++ src/lib/md/SecurityDB.h | 6 +- src/lib/md/SecurityPtr.h | 12 ++++ 4 files changed, 155 insertions(+), 1 deletion(-) create mode 100644 src/lib/md/SecurityCache.cpp create mode 100644 src/lib/md/SecurityCache.h create mode 100644 src/lib/md/SecurityPtr.h diff --git a/src/lib/md/SecurityCache.cpp b/src/lib/md/SecurityCache.cpp new file mode 100644 index 000000000..4e5475515 --- /dev/null +++ b/src/lib/md/SecurityCache.cpp @@ -0,0 +1,104 @@ +#include "SecurityCache.h" +#include "SecurityDB.h" +#include "lib/utils/Dispatcher.h" + +#include +#include +#include + +/* +todo: + 0 - All new code uses SecurityKey + 1 - Implement SecurityCache. It is a simple structure + 2 - MDConsumer has a SecurityCache + 3 - scan_securities inserts securities into SecurityCache + + +*/ + +namespace bts { +namespace md { + +struct SecurityCacheImpl { + std::mutex cache_mutex; + std::unordered_map cache; + tbb::concurrent_queue update_queue; + void update_db(const SecurityPtr&); +}; + +SecurityPtr load_from_security_db(SecurityKey key) { + return db::find_security(key); +} + +void SecurityCacheImpl::update_db(const SecurityPtr& security) { + // Batch updates to the database for performance + // algo: + // push item into queue + // post work to dispatcher with delay + // When dispatcher services request drain update_queue. + // This will cause one update per security but most will be no-op + // This is a little simpler than running a timer and trying to stop it after security discovery is done + update_queue.push(security); + utils::Dispatcher::background().post([this]() { + if (!update_queue.empty()) { + std::vector work; + SecurityPtr temp; + while(update_queue.try_pop(temp)) { work.push_back(temp); } + if (!work.empty()) { + db::insert_or_replace(work); + } + } + }, 1000); +} + +////////////////// + +SecurityCache::SecurityCache() : _me(new SecurityCacheImpl()) {} +SecurityCache::~SecurityCache() {} + +SecurityPtr SecurityCache::try_get_security(SecurityKey key) const { + std::lock_guard lock(_me->cache_mutex); + auto iter = _me->cache.find(key); + if (iter != _me->cache.end()) { + return iter->second; + } + return nullptr; +} + +SecurityPtr SecurityCache::get_security(SecurityKey key) { + std::unique_lock lock(_me->cache_mutex); + auto iter = _me->cache.find(key); + if (iter != _me->cache.end()) { + return iter->second; + } + else { + // unlock for db read + lock.unlock(); + auto security = load_from_security_db(key); + if (security) { + return add_db_security(security); + } + return security; + } +} + +SecurityPtr SecurityCache::add_db_security(const SecurityPtr& security) { + bts_assert(security); + std::lock_guard lock(_me->cache_mutex); + SecurityKey key(security->id()); + auto result = _me->cache.insert(std::make_pair(SecurityKey(security->id()), security)); + // If result was inserted result.first->second is equal to security + // If the value was already present it must be a recently inserted 'live' security and we should use it + return result.first->second; +} +bool SecurityCache::add_live_security(const SecurityPtr& security) { + bts_assert(security); + std::lock_guard lock(_me->cache_mutex); + SecurityKey key(security->id()); + // Insert but don't overwrite existing + auto result = _me->cache.insert(std::make_pair(SecurityKey(security->id()), security)); + _me->update_db(security); + return result.second; +} + +}}//ns \ No newline at end of file diff --git a/src/lib/md/SecurityCache.h b/src/lib/md/SecurityCache.h new file mode 100644 index 000000000..388d66fd8 --- /dev/null +++ b/src/lib/md/SecurityCache.h @@ -0,0 +1,34 @@ +#pragma once + +#include "lib/domain_model/Security.pb.h" +#include "MDConsumerHandle.h" +#include "lib/domain_model/SecurityKey.h" +#include "SecurityPtr.h" +#include + +namespace bts { +namespace md { + +// Get security on demand +// Store security+handle +struct SecurityCacheImpl; +class SecurityCache { + std::unique_ptr _me; + +public: + SecurityCache(); + ~SecurityCache(); + + //Load security from cache with fallback to database + SecurityPtr get_security(SecurityKey key); + + // Get the security if it's in the cache otherwise return nullptr + SecurityPtr try_get_security(SecurityKey key) const; + + // returns true if the cache was updated + bool add_live_security(const SecurityPtr& security); + SecurityPtr add_db_security(const SecurityPtr& security); + +}; + +}}//ns \ No newline at end of file diff --git a/src/lib/md/SecurityDB.h b/src/lib/md/SecurityDB.h index 3e7b9e2ec..77948dcce 100644 --- a/src/lib/md/SecurityDB.h +++ b/src/lib/md/SecurityDB.h @@ -7,6 +7,8 @@ #include "lib/api/services/SyntheticSecurityService.pb.h" #include "common.h" #include "UserSymbol.h" +#include "lib/domain_model/SecurityKey.h" +#include "SecurityPtr.h" #include #include @@ -25,7 +27,9 @@ size_t load_securities( const std::set& symbols, const std::function& callback); -void store_if_new(const std::vector& handles); +SecurityPtr find_security(SecurityKey); + +void insert_or_replace(const std::vector&); // metadata std::vector load_metadata(); diff --git a/src/lib/md/SecurityPtr.h b/src/lib/md/SecurityPtr.h new file mode 100644 index 000000000..34c93965d --- /dev/null +++ b/src/lib/md/SecurityPtr.h @@ -0,0 +1,12 @@ +#pragma once + +#include "lib/domain_model/SecurityKey.h" +#include + +namespace bts { +namespace api {class Security;} +namespace md { + + typedef std::shared_ptr const SecurityPtr; + +}} \ No newline at end of file From 0af2b03ad7a9c91a3e4315a21dbc6d0f1fc427ca Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 1 Jan 2019 13:41:40 -0500 Subject: [PATCH 08/85] SPARK-1953 Remove SecurityDB from SecurityIDRecorder --- src/lib/md/SecurityIDRecorder.cpp | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/lib/md/SecurityIDRecorder.cpp b/src/lib/md/SecurityIDRecorder.cpp index f8cc32613..afeea6f3b 100644 --- a/src/lib/md/SecurityIDRecorder.cpp +++ b/src/lib/md/SecurityIDRecorder.cpp @@ -5,7 +5,6 @@ #include "lib/config/config.h" #include "lib/utils/Sqlite_utils.h" #include "lib/logging/logging.h" -#include "lib/md/SecurityDB.h" #include #include @@ -279,19 +278,6 @@ void SecurityIDRecorderState::work_db() { sqlite::exec_nofail(db.get(), "COMMIT TRANSACTION"); } - // record the securities - if (!new_entries.empty() || !existing_securities.empty()) { - std::vector save_these; - save_these.reserve(new_entries.size() + existing_securities.size()); - for (const auto& e : new_entries) { - save_these.push_back(&e->security); - } - for (const auto& e : existing_securities) { - save_these.push_back(&e->security); - } - db::store_if_new(save_these); - } - if (!new_entries.empty()) { for (size_t x = 0; x < new_entries.size(); ++x) { Security& security = new_entries[x]->security; From 6953f8b351ac4327ea86be25be96c9f6916d0af3 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 1 Jan 2019 13:43:06 -0500 Subject: [PATCH 09/85] Remove obsolete comment --- src/lib/md/SecurityCache.cpp | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/src/lib/md/SecurityCache.cpp b/src/lib/md/SecurityCache.cpp index 4e5475515..2675128bb 100644 --- a/src/lib/md/SecurityCache.cpp +++ b/src/lib/md/SecurityCache.cpp @@ -6,16 +6,6 @@ #include #include -/* -todo: - 0 - All new code uses SecurityKey - 1 - Implement SecurityCache. It is a simple structure - 2 - MDConsumer has a SecurityCache - 3 - scan_securities inserts securities into SecurityCache - - -*/ - namespace bts { namespace md { @@ -26,10 +16,6 @@ struct SecurityCacheImpl { void update_db(const SecurityPtr&); }; -SecurityPtr load_from_security_db(SecurityKey key) { - return db::find_security(key); -} - void SecurityCacheImpl::update_db(const SecurityPtr& security) { // Batch updates to the database for performance // algo: @@ -74,7 +60,7 @@ SecurityPtr SecurityCache::get_security(SecurityKey key) { else { // unlock for db read lock.unlock(); - auto security = load_from_security_db(key); + auto security = db::find_security(key); if (security) { return add_db_security(security); } From ca21ac5f5e5150142a796993254e091e15e504e6 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 2 Jan 2019 15:29:31 -0500 Subject: [PATCH 10/85] SPARK-1949 - Convert SecuritydB to sqlitecpp --- src/lib/md/SecurityDB.cpp | 129 +++++++++++++++++--------------------- src/lib/md/SecurityPtr.h | 2 +- 2 files changed, 60 insertions(+), 71 deletions(-) diff --git a/src/lib/md/SecurityDB.cpp b/src/lib/md/SecurityDB.cpp index 03ac7d67e..d9a94c3e1 100644 --- a/src/lib/md/SecurityDB.cpp +++ b/src/lib/md/SecurityDB.cpp @@ -1,7 +1,9 @@ #include "SecurityDB.h" #ifndef WIN32 -#include "lib/utils/Sqlite_utils.h" +#include +#include + #include "lib/config/config.h" #include "lib/config/protos/system.pb.h" #include "lib/md/Shm/FixedSecurity.h" @@ -15,6 +17,7 @@ using std::string; using namespace bts::utils; +using namespace SQLite; namespace bts { namespace md { @@ -25,7 +28,7 @@ namespace db { * key, created_date, expiration_date */ -sqlite3_ptr open_db() { +std::unique_ptr open_db() { // Directory location on disk auto db_dir = config::get().data_path() + "databases/md/"; auto db_path = db_dir + "/securities.sqlite"; @@ -34,20 +37,14 @@ sqlite3_ptr open_db() { } // Create table if it does not exist - auto dbptr = sqlite::open_db(db_path.c_str()); - bts_assert(dbptr); - auto db = dbptr.get(); - sqlite3_busy_timeout(db, 9999); - sqlite::exec_nofail(db, "PRAGMA journal_mode=WAL"); - sqlite::exec_nofail(db, "PRAGMA syncronous=0"); + auto db = std::make_unique(db_path, OPEN_READWRITE | OPEN_CREATE); + db->setBusyTimeout(9999); + db->exec("PRAGMA journal_mode=WAL"); + db->exec("PRAGMA syncronous=0"); + db->exec("CREATE TABLE IF NOT EXISTS securities (key INTEGER primary key, symbol TEXT NOT NULL, venue INTEGER NOT NULL, type INTEGER NOT NULL, security_blob BLOB NOT NULL, expiration INTEGER NOT NULL);"); + db->exec("CREATE TABLE IF NOT EXISTS metadata (key INTEGER PRIMARY KEY, data BLOB NOT NULL);"); - sqlite::exec_nofail(db, - "CREATE TABLE IF NOT EXISTS securities (key INTEGER primary key, symbol TEXT NOT NULL, venue INTEGER NOT NULL, type INTEGER NOT NULL, security_blob BLOB NOT NULL, expiration INTEGER NOT NULL);"); - - sqlite::exec_nofail(db, - "CREATE TABLE IF NOT EXISTS metadata (key INTEGER PRIMARY KEY, data BLOB NOT NULL);"); - - return dbptr; + return db; } datetime::Date sec_expiration_cut_off() @@ -68,24 +65,22 @@ size_t load_securities( // Load all existing securities and call the callback auto db = dbptr.get(); - auto select_stmt = sqlite::prepare_query(db, "select key,symbol,venue,type,security_blob from securities where expiration > ?"); - auto select = select_stmt.get(); - sqlite3_bind_int(select, 1, sec_expiration_cut_off().to_short()); - auto result = sqlite3_step(select); + SQLite::Statement select_stmt(*db, "select key,symbol,venue,type,security_blob from securities where expiration > :exp"); + select_stmt.bind(":exp", sec_expiration_cut_off().to_short()); + Security security; - while (result == SQLITE_ROW) { - security_key security_key = sqlite3_column_int(select, 0); - auto symbol = std::string( - (char*)sqlite3_column_text(select, 1), - sqlite3_column_bytes(select, 1)); - auto venue = (api::Venue)sqlite3_column_int(select, 2); - auto type = (api::SecurityType)sqlite3_column_int(select, 3); + while (select_stmt.tryExecuteStep() == SQLITE_ROW) { + security_key security_key = select_stmt.getColumn("key"); + std::string symbol = select_stmt.getColumn("symbol"); + api::Venue venue = (api::Venue)(int) select_stmt.getColumn("venue"); + auto type = (api::SecurityType)(int) select_stmt.getColumn("type"); UserSymbol us(symbol, venue, type); if (symbols.find(us) != symbols.end()) { - auto data = sqlite3_column_blob(select, 4); - auto len = sqlite3_column_bytes(select, 4); + auto blob = select_stmt.getColumn("security_blob"); + // auto data = blob.getBlob(); + // auto len = blob.getBytes(); // Migrate from old tick stuff - if (security.ParseFromArray(data, len)) { + if (security.ParseFromArray(blob, blob.getBytes())) { if (security.has_glbx()) { if (security.contract_spec().main_fraction() == 0) { security.mutable_contract_spec()->set_main_fraction(security.glbx().do_not_use_main_fraction()); @@ -105,64 +100,64 @@ size_t load_securities( LOG(md, Warn, "Failed to parse security from database. id = {}", security_key); } } - result = sqlite3_step(select); } return count; } -void store_if_new(const std::vector& securities) { - auto dbptr = open_db(); - auto insert = sqlite::prepare_query(dbptr, "insert or ignore into securities (key, symbol, venue, type, security_blob, expiration) values (?,?,?,?,?,?)"); - sqlite::exec_nofail(dbptr, "BEGIN TRANSACTION"); +SecurityPtr find_security(SecurityKey key) { + //todo: + (void)key; + return nullptr; +} + +void insert_or_replace(const std::vector& securities) { + auto db = open_db(); + SQLite::Statement insert(*db,"insert or replace into securities (key, symbol, venue, type, security_blob, expiration) values (?,?,?,?,?,?)"); + SQLite::Transaction transaction(*db); std::string buffer; - for (const api::Security* psecurity : securities) { + for (const auto& psecurity : securities) { if (psecurity && psecurity->SerializeToString(&buffer)) { - SQLITE_CHK(sqlite3_bind_int(insert.get(), 1, psecurity->id()), dbptr); - SQLITE_CHK(sqlite3_bind_text(insert.get(), 2, psecurity->symbol().c_str(), psecurity->symbol().size(), SQLITE_STATIC), dbptr); - SQLITE_CHK(sqlite3_bind_int(insert.get(), 3, psecurity->exchange()), dbptr); - SQLITE_CHK(sqlite3_bind_int(insert.get(), 4, securitytype_of(*psecurity)), dbptr); - SQLITE_CHK(sqlite3_bind_blob(insert.get(), 5, buffer.c_str(), buffer.size(), SQLITE_STATIC), dbptr); - SQLITE_CHK(sqlite3_bind_int(insert.get(), 6, expiration(*psecurity).to_short()), dbptr); - sqlite3_step(insert.get()); - sqlite3_reset(insert.get()); + insert.bind(1, psecurity->id()); + insert.bind(2, psecurity->symbol()); + insert.bind(3, psecurity->exchange()); + insert.bind(4, securitytype_of(*psecurity)); + insert.bind(5, buffer); + insert.bind(6, expiration(*psecurity).to_short()); + insert.exec(); + insert.reset(); } } - sqlite::exec_nofail(dbptr, "COMMIT TRANSACTION"); + transaction.commit(); } std::vector load_metadata() { std::vector values; - auto dbptr = open_db(); - auto db = dbptr.get(); - auto select_stmt = sqlite::prepare_query(db, "select key,data from metadata"); - auto select = select_stmt.get(); - auto result = sqlite3_step(select); - while(result == SQLITE_ROW) { - uint32_t key = sqlite3_column_int(select, 0); - auto blob = sqlite3_column_blob(select, 1); - auto len = sqlite3_column_bytes(select, 1); - api::synthetic::Metadata data; - if (data.ParseFromArray(blob, len)) { - bts_assert(key == data.security_key()); - values.push_back(data); + auto db = open_db(); + SQLite::Statement select(*db, "select key,data from metadata"); + while(select.tryExecuteStep() == SQLITE_ROW) { + uint32_t key = select.getColumn(0); + auto blob = select.getColumn(1); + api::synthetic::Metadata meta; + if (meta.ParseFromArray(blob, blob.getBytes())) { + bts_assert(key == meta.security_key()); + values.push_back(meta); } else { LOG(md, Warn, "Failed to parse security metadata for security_key={}", key); } - result = sqlite3_step(select); } return values; } bool store_metadata(const api::synthetic::Metadata& data) { - auto dbptr = open_db(); - auto insert = sqlite::prepare_query(dbptr, "insert or replace into metadata (key, data) values (?,?)"); + auto db = open_db(); + SQLite::Statement insert(*db, "insert or replace into metadata (key, data) values (?,?)"); - SQLITE_CHK(sqlite3_bind_int(insert.get(), 1, data.security_key()), dbptr); + insert.bind(1, data.security_key()); std::string buffer = data.SerializeAsString(); - SQLITE_CHK(sqlite3_bind_blob(insert.get(), 2, buffer.c_str(), buffer.size(), SQLITE_STATIC), dbptr); - auto result = sqlite3_step(insert.get()); + insert.bind(2, buffer); + auto result = insert.tryExecuteStep(); if (result != SQLITE_DONE) { LOG(md, Warn, "Failed to insert metadata. sqlite result {}", result); return false; @@ -180,13 +175,7 @@ size_t load_securities( const std::function& callback) { return 0; } -void store_if_new(const std::vector& handles) {} -// std::vector load_metadata() { -// std::vector result; -// return result; -// } -// void store_metadata(const api::synthetic::Metadata&) { -// } +void insert_or_replace(const std::vector& handles) {} #endif diff --git a/src/lib/md/SecurityPtr.h b/src/lib/md/SecurityPtr.h index 34c93965d..228c9a198 100644 --- a/src/lib/md/SecurityPtr.h +++ b/src/lib/md/SecurityPtr.h @@ -7,6 +7,6 @@ namespace bts { namespace api {class Security;} namespace md { - typedef std::shared_ptr const SecurityPtr; + typedef std::shared_ptr SecurityPtr; }} \ No newline at end of file From 1126b772dd8eb046ec1564cfcaf0ae69e2486607 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 2 Jan 2019 17:26:39 -0500 Subject: [PATCH 11/85] SPARK-1952 Update MDConsumer to use SecurityPtr --- src/lib/md/MDConsumer.cpp | 42 +++++++++++++++++++++------------------ src/lib/md/MDConsumer.h | 3 +++ src/lib/md/SecurityDB.cpp | 1 + 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/src/lib/md/MDConsumer.cpp b/src/lib/md/MDConsumer.cpp index dba9eda76..9ab52a906 100644 --- a/src/lib/md/MDConsumer.cpp +++ b/src/lib/md/MDConsumer.cpp @@ -39,15 +39,15 @@ bool match_symbol(const string& val, const string& b) { return b.empty() || (val == b); } -typedef shared_ptr PSecurity; +typedef shared_ptr MutSecurityPtr; struct SecurityLess { - bool operator()(const PSecurity& left, const PSecurity& right) { + bool operator()(const SecurityPtr& left, const SecurityPtr& right) { return left->id() < right->id(); } }; -PSecurity convert_psecurity(const FixedSecurity& security) { - PSecurity psec = make_shared(); +MutSecurityPtr convert_psecurity(const FixedSecurity& security) { + auto psec = make_shared(); convert_security(security, psec.get()); return psec; } @@ -562,10 +562,10 @@ public: // This is a terrible name size_t walk_sec_list(SecurityCallback cb, const string& symbol, SecurityType type) { size_t result = 0; - vector securities; + vector securities; for (auto& inst : _instances) { if (match_symbol(inst->symbol(), symbol) && match_type(inst->type(), type)) { - result += inst->walk_sec_list([&](PSecurity security) { securities.push_back(security); }, symbol, type); + result += inst->walk_sec_list([&](MutSecurityPtr security) { securities.push_back(security); }, symbol, type); } } auto complete_securities = fill_leg_details(securities); @@ -602,7 +602,8 @@ public: return empty_handle; } - map get_handles(const vector& securities) { + template //maybe const + map get_handles(const vector& securities) { set keys; for (const auto& sec : securities) { keys.insert(sec->id()); @@ -641,8 +642,8 @@ public: return handle; } - vector fill_leg_details(const vector& securities, int depth=0) { - vector result; + vector fill_leg_details(const vector& securities, int depth=0) { + vector result; if (securities.empty() || depth > 64) return result; // // Gather ids for legs @@ -655,14 +656,14 @@ public: } } // If there are no spreads everything is (by definition) resolved already - if (keys.empty()) return securities; + if (keys.empty()) return vector{securities.begin(), securities.end()}; // // gather handles for direct legs auto handles = get_handles(keys); // // Some of the legs will themselves be spreads - vector legs; + vector legs; legs.reserve(handles.size()); for (const auto& pair : handles) { if (pair.second.is_initialized()) { @@ -670,9 +671,8 @@ public: } } // recurse to find details - legs = fill_leg_details(legs, depth+1); - map leg_map; - for (auto& leg : legs) { + map leg_map; + for (auto& leg : fill_leg_details(legs, depth+1)) { leg_map.insert(make_pair(leg->id(), move(leg))); } @@ -705,7 +705,7 @@ public: if (_callbacks.empty() && _deferred_handles.empty()) return 0; // Call each _instance and pass it a callback for (auto& inst : _instances) { - vector found_securities; + vector found_securities; inst->scan_new_securities([&](const ColdEntry& entry) { // Check if any deferred handles need this security (could be sped up with get_handles() but it isn't hot auto iter = _deferred_handles.find(entry.security.key); @@ -719,10 +719,11 @@ public: if (resolve_legs && !found_securities.empty()) { sort(found_securities.begin(), found_securities.end(), SecurityLess()); - vector resolved_securities = fill_leg_details(found_securities); + vector resolved_securities = fill_leg_details(found_securities); sort(resolved_securities.begin(), resolved_securities.end(), SecurityLess()); + // Extract the list of unresolved securities - vector unresolved_securities; + vector unresolved_securities; set_difference(found_securities.begin(), found_securities.end(), resolved_securities.begin(), resolved_securities.end(), inserter(unresolved_securities, unresolved_securities.begin()), SecurityLess()); @@ -803,11 +804,14 @@ public: // Note: We have to lock over the initial walk because if scan_securities() finds a new security concurrently with this walk // it might mark it seen without this callback getting called. for (auto& inst : _instances) { - vector securities; + vector securities; inst->walk_old_securities([&](const ColdEntry& entry) { securities.push_back(convert_psecurity(entry.security)); }); - auto filled_securities = resolve_legs ? fill_leg_details(securities) : securities; + auto filled_securities = + resolve_legs ? + fill_leg_details(securities) : + vector{securities.begin(), securities.end()}; auto handles = get_handles(filled_securities); for (const auto& security : filled_securities) { auto find_result = handles.find(security->id()); diff --git a/src/lib/md/MDConsumer.h b/src/lib/md/MDConsumer.h index 643ef2635..8a7d548c1 100644 --- a/src/lib/md/MDConsumer.h +++ b/src/lib/md/MDConsumer.h @@ -7,6 +7,7 @@ #include "MDConsumerHandle.h" #include "UserSymbol.h" #include "ISyntheticSecurityService.h" +#include "SecurityPtr.h" #include #include @@ -51,6 +52,8 @@ public: size_t scan_securities(bool resolve_legs = true); void on_security(SecurityCallback cb, bool resolve_legs = true); + // SecurityPtr get_security(SecurityKey key); + MDConsumerChannel get_channel(const std::string& symbol, SecurityType type, Venue exchange) const; MDConsumerChannel get_channel(const UserSymbol&) const; diff --git a/src/lib/md/SecurityDB.cpp b/src/lib/md/SecurityDB.cpp index d9a94c3e1..c374765a9 100644 --- a/src/lib/md/SecurityDB.cpp +++ b/src/lib/md/SecurityDB.cpp @@ -176,6 +176,7 @@ size_t load_securities( return 0; } void insert_or_replace(const std::vector& handles) {} +SecurityPtr find_security(SecurityKey) { return nullptr; } #endif From 5481a3e9d64966b19439d629e82ef20c8dda991d Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 2 Jan 2019 17:39:41 -0500 Subject: [PATCH 12/85] Update CMakeDeps --- src/lib/CMakeDeps.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/CMakeDeps.cmake b/src/lib/CMakeDeps.cmake index ec7dd860c..dced7a640 100644 --- a/src/lib/CMakeDeps.cmake +++ b/src/lib/CMakeDeps.cmake @@ -1 +1 @@ -set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;mercury/actors/IPollableActor.cpp;mercury/broker/Broker.cpp;mercury/broker/BrokerProxy.cpp;mercury/broker/ILocalBroker.cpp;mercury/broker/IRemoteBroker.cpp;mercury/common/mercury_common.cpp;mercury/events/TimerEvent.cpp;mercury/executor/Executor.cpp;mercury/topics/Topic.cpp;metrics/metrics.cpp;metrics/system_metrics.cpp;orderman/actors/AsyncFactory.cpp;orderman/actors/ExecutionGateway.cpp;orderman/actors/FillStore.cpp;orderman/actors/MDSupplementarySource.cpp;orderman/actors/OrderStore.cpp;orderman/actors/SODPosition.cpp;orderman/actors/SecuritySrc.cpp;orderman/actors/ServiceBridge.cpp;orderman/actors/order_manager/OrderBuilder.cpp;orderman/actors/order_manager/OrderManager.cpp;orderman/actors/order_manager/OrderServicesData.cpp;orderman/events/CancelRejectEvent.cpp;orderman/events/NewExecutionReportEvent.cpp;orderman/limiter/LimitStore.cpp;orderman/limiter/Limiter.cpp;orderman/limiter/ProductLimitNode.cpp;orderman/limiter/ProductLimitTree.cpp;orderman/limiter/SecurityLimitNode.cpp;orderman/limiter/SecurityLimitTree.cpp;orderman/limiter/tests/limit_tests.cpp;orderman/order_factory/DynamicOrderRegistry.cpp;orderman/order_factory/OrderFactory.cpp;orderman/order_router/OrderRouter.cpp;orderman/orderman_utils/fix_order_codec.cpp;orderman/orders/DynamicOrder.cpp;orderman/orders/IcebergOrder.cpp;orderman/orders/NativeOrder.cpp;orderman/orders/Order.cpp;orderman/orders/TimeBoxOrder.cpp;orderman/orders/TriggerOrder.cpp;orderman/orders/book_algos/BookAlgo.cpp;orderman/orders/book_algos/CoverOrder.cpp;orderman/orders/book_algos/JoinOrder.cpp;orderman/orders/book_algos/LeanOrder.cpp;orderman/orders/book_algos/OffsetOrder.cpp;orderman/orders/book_algos/TickOrder.cpp;orderman/orders/book_algos/TopOrder.cpp;orderman/orders/spreader_orders/SpreaderOrder.cpp;orderman/orders/spreader_orders/spreader/LegPropertyData.cpp;orderman/orders/spreader_orders/spreader/QuoteLegPriceQuantityState.cpp;orderman/orders/spreader_orders/spreader/SingleFillHedger.cpp;orderman/orders/spreader_orders/spreader/SingleLegQuoter.cpp;orderman/orders/spreader_orders/spreader/SpreaderBookkeepingData.cpp;orderman/orders/spreader_orders/spreader/SpreaderChildOrderState.cpp;orderman/orders/spreader_orders/spreader/SpreaderLegData.cpp;orderman/orders/spreader_orders/spreader/SpreaderUtils.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderBuilder.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderServices.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderPreActivationOrderServices.cpp;orderman/orders/spreader_orders/spreader/hedger/UnhedgedDelta.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckCancelledStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWorkingStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSet.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSetHistory.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteLegDriver.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckCancelledWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForIOCWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForSizeStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWorkingStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderEndStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderStartStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWaitingForCancelAllStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWaitingForInitialDataStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWorkingStateWorker.cpp;orderman/orders/spreader_orders/tests/spreader_tests.cpp;orderman/server/OrderServiceServer.cpp;orderman/tests/fat_wrapper_tests.cpp;orderman2/orders/NativeOrder.cpp;orderman2/orders/Order.cpp;os/os.cpp;os/tests/os_tests.cpp;persistence/IStore.cpp;persistence/PersistenceServiceServer.cpp;persistence/SqliteStore.cpp;persistence/persistence_utils.cpp;persistence/tests/PersistenceServiceTests.cpp;plugin/SparkServices.cpp;plugin/plugin.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;tracing/PerfLog.cpp;tracing/bts_trace.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp;utils/stats.cpp;utils/tests/AuthTokenTests.cpp;utils/thread_manager.cpp) \ No newline at end of file +set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityCache.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp;utils/stats.cpp;utils/tests/AuthTokenTests.cpp;utils/thread_manager.cpp) \ No newline at end of file From c4bbf5e219926b3e6880ed7704b49305a734eeee Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 3 Jan 2019 13:50:26 -0500 Subject: [PATCH 13/85] SPARK-1952 - convert md::SecurityCallback to SecurityPtr to avoid copying --- src/lib/md/Diag/MDPeriodicDumper.cpp | 6 +-- src/lib/md/MDConsumer.cpp | 12 ++--- src/lib/md/MDConsumer.h | 2 +- src/lib/md/MDConsumerInterop.cpp | 12 ++--- src/lib/md/MDRecoverableConsumer.cpp | 8 +-- src/lib/md/relay/QueryServer.cpp | 2 +- src/lib/md/test/md2_tests.cpp | 70 ++++++++++++------------- src/lib/orderman/actors/SecuritySrc.cpp | 10 ++-- 8 files changed, 60 insertions(+), 62 deletions(-) diff --git a/src/lib/md/Diag/MDPeriodicDumper.cpp b/src/lib/md/Diag/MDPeriodicDumper.cpp index 9fbfdc56c..1704b9719 100644 --- a/src/lib/md/Diag/MDPeriodicDumper.cpp +++ b/src/lib/md/Diag/MDPeriodicDumper.cpp @@ -23,7 +23,7 @@ namespace md { struct Entry { uint32_t seq; MDConsumerHandle handle; - std::unique_ptr security; + SecurityPtr security; }; std::string print_book( @@ -156,8 +156,8 @@ void run_periodic_dumper(const std::set& symbols, uint32_t seq_mod, // read all securities into a vector std::vector entries; - consumer.walk_security_list([&](const api::Security& sec, MDConsumerHandle handle) { - entries.push_back({ 0, handle, std::make_unique(sec) }); + consumer.walk_security_list([&](const SecurityPtr& sec, MDConsumerHandle handle) { + entries.push_back({ 0, handle, sec }); }); entries = filter_entries(entries); diff --git a/src/lib/md/MDConsumer.cpp b/src/lib/md/MDConsumer.cpp index 9ab52a906..fcc6b4762 100644 --- a/src/lib/md/MDConsumer.cpp +++ b/src/lib/md/MDConsumer.cpp @@ -573,7 +573,7 @@ public: for (const auto& security : complete_securities) { auto find_result = handles.find(security->id()); if (find_result != handles.end()) { - cb(*security, find_result->second); + cb(security, find_result->second); } } return result; @@ -749,7 +749,7 @@ public: if (find_result != handles.end()) { // Pass new securities to each callback for (auto& cb : _callbacks) { - cb(*security, find_result->second); + cb(security, find_result->second); } } else { @@ -765,7 +765,7 @@ public: auto find_result = handles.find(security->id()); if (find_result != handles.end()) { for (auto& cb : _callbacks) { - cb(*security, find_result->second); + cb(security, find_result->second); } } else { @@ -789,7 +789,7 @@ public: } // call all subscribers for (auto& cb : _callbacks) { - cb(security, handle); + cb(std::make_shared(security), handle); } } } @@ -817,7 +817,7 @@ public: auto find_result = handles.find(security->id()); bts_assert_d(find_result != handles.end()); if (find_result != handles.end()) { - cb(*security, find_result->second); + cb(security, find_result->second); } } } @@ -826,7 +826,7 @@ public: for (const auto& item : _sci->get_old_synthetics()) { const auto& security = item->security; auto handle = MDConsumerHandle(item); - cb(security, handle); + cb(std::make_shared(security), handle); } } // store the callback diff --git a/src/lib/md/MDConsumer.h b/src/lib/md/MDConsumer.h index 8a7d548c1..85e74f4b9 100644 --- a/src/lib/md/MDConsumer.h +++ b/src/lib/md/MDConsumer.h @@ -20,7 +20,7 @@ namespace bts { namespace md { /* typedef void (security_callback)(const Security& security); */ -typedef std::function SecurityCallback; +typedef std::function SecurityCallback; using std::size_t; class MDConsumerImpl; diff --git a/src/lib/md/MDConsumerInterop.cpp b/src/lib/md/MDConsumerInterop.cpp index 01b0616bb..7048354c5 100644 --- a/src/lib/md/MDConsumerInterop.cpp +++ b/src/lib/md/MDConsumerInterop.cpp @@ -64,10 +64,10 @@ typedef void(*security_cb)(security_key, const char*, size_t, MDConsumerHandle& CAPI size_t md_consumer_walk_security_list(MDConsumer* consumer, security_cb cb) { bts_assert(consumer); std::string buffer; - return consumer->walk_security_list([&](const Security& security, MDConsumerHandle handle) { + return consumer->walk_security_list([&](const SecurityPtr& security, MDConsumerHandle handle) { buffer.clear(); - security.SerializePartialToString(&buffer); - cb(security.id(), buffer.c_str(), buffer.size(), handle); + security->SerializePartialToString(&buffer); + cb(security->id(), buffer.c_str(), buffer.size(), handle); }); } @@ -94,10 +94,10 @@ void safe_call_clr_callback(security_cb cb, security_key key, const char* buffer CAPI void md_consumer_set_on_security(MDConsumer* consumer, security_cb cb) { bts_assert(consumer); - consumer->on_security([=](const Security& security, MDConsumerHandle handle) { + consumer->on_security([=](const SecurityPtr& security, MDConsumerHandle handle) { std::string buffer; - security.SerializePartialToString(&buffer); - safe_call_clr_callback(cb, security.id(), buffer.c_str(), buffer.size(), handle); + security->SerializePartialToString(&buffer); + safe_call_clr_callback(cb, security->id(), buffer.c_str(), buffer.size(), handle); }); } #endif diff --git a/src/lib/md/MDRecoverableConsumer.cpp b/src/lib/md/MDRecoverableConsumer.cpp index 31a554ee5..9e6250388 100644 --- a/src/lib/md/MDRecoverableConsumer.cpp +++ b/src/lib/md/MDRecoverableConsumer.cpp @@ -54,8 +54,8 @@ size_t MDRecoverableConsumer::walk_security_list(SecurityCallback callback) { void MDRecoverableConsumer::on_security(SecurityCallback cb) { // Add a filter to the callback to avoid calling the on_security callback with duplicates - auto filtered_cb = [this,cb](const Security& security, MDConsumerHandle handle) { - auto key = security.id(); + auto filtered_cb = [this,cb](const SecurityPtr& security, MDConsumerHandle handle) { + auto key = security->id(); if (_seen.find(key) == _seen.end()) { _seen.insert(key); cb(security, handle); @@ -101,11 +101,11 @@ bool MDRecoverableConsumer::try_recover() { _current->consumer.on_security(cb); } - _current->consumer.on_security([&](const Security& security, MDConsumerHandle newhandle) { + _current->consumer.on_security([&](const SecurityPtr& security, MDConsumerHandle newhandle) { Mutex::scoped_lock l (_lock); for (auto& group : _old) { for (auto& handle : group->handles) { - if (handle->key() == security.id()) { + if (handle->key() == security->id()) { // update the handle! (if this is called guarunteed success) *handle = newhandle; bts_assert(handle->is_initialized()); diff --git a/src/lib/md/relay/QueryServer.cpp b/src/lib/md/relay/QueryServer.cpp index dea02db98..57ab06e89 100644 --- a/src/lib/md/relay/QueryServer.cpp +++ b/src/lib/md/relay/QueryServer.cpp @@ -184,7 +184,7 @@ public: void subscribe_securities() { auto id = _consumer.id(); - _consumer.on_security([this,id](const Security&, MDConsumerHandle handle) { + _consumer.on_security([this,id](const SecurityPtr&, MDConsumerHandle handle) { if (_consumer.id() == id) { auto iter = _groups.find(handle.get_usersymbol()); bts_assert(iter != _groups.end()); diff --git a/src/lib/md/test/md2_tests.cpp b/src/lib/md/test/md2_tests.cpp index 0ec5f24a2..9d6ef027e 100644 --- a/src/lib/md/test/md2_tests.cpp +++ b/src/lib/md/test/md2_tests.cpp @@ -372,14 +372,14 @@ public: vector consumer_securities; // Poll for security list - none visible until they have keys - size_t count = consumer.walk_security_list([&](const Security& security, MDConsumerHandle) {consumer_securities.push_back(security);}); + size_t count = consumer.walk_security_list([&](const SecurityPtr& security, MDConsumerHandle) {consumer_securities.push_back(*security);}); // Since no security has an id none are "public" REQUIRE(consumer_securities.size() == count); REQUIRE(count == 0); // Assign a unique ID to each security (from the producer) assign_ids(producers.begin(), producers.end()); // Now securities can be enumerated by the client - count = consumer.walk_security_list([&](const Security& security, MDConsumerHandle) {consumer_securities.push_back(security);}); + count = consumer.walk_security_list([&](const SecurityPtr& security, MDConsumerHandle) {consumer_securities.push_back(*security);}); REQUIRE(consumer_securities.size() == count); REQUIRE(count == 24); @@ -442,10 +442,10 @@ public: // Add the on_security callback vector securities; - consumer.on_security([&](const Security& security, MDConsumerHandle handle) { - REQUIRE(security.id() != invalid_security_key); - REQUIRE(security.id() == handle.key()); - securities.push_back(security); + consumer.on_security([&](const SecurityPtr& security, MDConsumerHandle handle) { + REQUIRE(security->id() != invalid_security_key); + REQUIRE(security->id() == handle.key()); + securities.push_back(*security); }); // No securities have ids yet REQUIRE(securities.size() == 0); @@ -598,10 +598,10 @@ public: // Add the on_security callback - callback fires for existing securities vector securities; - consumer.on_security([&](const Security& security, MDConsumerHandle handle) { - REQUIRE(security.id() != invalid_security_key); - REQUIRE(security.id() == handle.key()); - securities.push_back(security); + consumer.on_security([&](const SecurityPtr& security, MDConsumerHandle handle) { + REQUIRE(security->id() != invalid_security_key); + REQUIRE(security->id() == handle.key()); + securities.push_back(*security); }); // Found all securities REQUIRE(securities.size() == producers.size()); @@ -622,9 +622,9 @@ public: // Symbol list for the consumer MDRecoverableConsumer consumer(get_test_symbols(), nullptr); map> products; - consumer.on_security([&](const Security& security, MDConsumerHandle handle) { - REQUIRE(security.id() == handle.key()); - products.insert(make_pair(security.id(), consumer.get_managed_handle(security.id()))); + consumer.on_security([&](const SecurityPtr& security, MDConsumerHandle handle) { + REQUIRE(security->id() == handle.key()); + products.insert(make_pair(security->id(), consumer.get_managed_handle(security->id()))); }); consumer.scan_securities(); REQUIRE(products.size() == producers.size()); @@ -740,7 +740,7 @@ public: MDConsumer consumer(symbols, nullptr); set keys; - consumer.walk_security_list([&](const Security& security, MDConsumerHandle) { keys.insert(security.id()); }); + consumer.walk_security_list([&](const SecurityPtr& security, MDConsumerHandle) { keys.insert(security->id()); }); REQUIRE(keys.size() == producers.size()); auto map = consumer.get_handles(keys); REQUIRE(map.size() == keys.size()); @@ -783,16 +783,16 @@ public: csymbols.insert(UserSymbol("S4", VENUE_BTS, SECURITYTYPE_SPREAD)); MDConsumer consumer(csymbols, nullptr); - auto check_sec = [&](const Security& security, MDConsumerHandle handle) { - REQUIRE(security.id() == handle.key()); - auto iter = find_if(std::begin(all_secs_array), std::end(all_secs_array), [&](const Security& sec) { return sec.id() == security.id(); }); + auto check_sec = [&](const SecurityPtr& security, MDConsumerHandle handle) { + REQUIRE(security->id() == handle.key()); + auto iter = find_if(std::begin(all_secs_array), std::end(all_secs_array), [&](const Security& sec) { return sec.id() == security->id(); }); REQUIRE(iter != end(all_secs_array)); - REQUIRE(iter->symbol() == (security.symbol())); - if (security.Contract_case() == Security::kSpread) { - REQUIRE(security.spread().normalized() == false); - REQUIRE(security.spread().legs_size() == iter->spread().legs_size()); - for (int x = 0; x < security.spread().legs_size(); ++x) { - REQUIRE((security.spread().legs(x).security().id()) == (iter->spread().legs(x).security().id())); + REQUIRE(iter->symbol() == (security->symbol())); + if (security->Contract_case() == Security::kSpread) { + REQUIRE(security->spread().normalized() == false); + REQUIRE(security->spread().legs_size() == iter->spread().legs_size()); + for (int x = 0; x < security->spread().legs_size(); ++x) { + REQUIRE((security->spread().legs(x).security().id()) == (iter->spread().legs(x).security().id())); } } }; @@ -829,16 +829,16 @@ public: csymbols.insert(UserSymbol("SS", VENUE_BTS, SECURITYTYPE_SPREAD)); MDConsumer consumer(csymbols, nullptr); - auto check_sec = [&](const Security& security, MDConsumerHandle handle) { - REQUIRE(security.id() == handle.key()); - auto iter = find_if(begin(all_secs_array), end(all_secs_array), [&](const Security& sec) { return sec.id() == security.id(); }); + auto check_sec = [&](const SecurityPtr& security, MDConsumerHandle handle) { + REQUIRE(security->id() == handle.key()); + auto iter = find_if(begin(all_secs_array), end(all_secs_array), [&](const Security& sec) { return sec.id() == security->id(); }); REQUIRE(iter != end(all_secs_array)); - REQUIRE((iter->id()) == (security.id())); - if (security.Contract_case() == Security::kSpread) { - REQUIRE(security.spread().normalized() == false); - REQUIRE(security.spread().legs_size() == iter->spread().legs_size()); - for (int x = 0; x < security.spread().legs_size(); ++x) { - REQUIRE((security.spread().legs(x).security().id()) == (iter->spread().legs(x).security().id())); + REQUIRE((iter->id()) == (security->id())); + if (security->Contract_case() == Security::kSpread) { + REQUIRE(security->spread().normalized() == false); + REQUIRE(security->spread().legs_size() == iter->spread().legs_size()); + for (int x = 0; x < security->spread().legs_size(); ++x) { + REQUIRE((security->spread().legs(x).security().id()) == (iter->spread().legs(x).security().id())); } } }; @@ -886,7 +886,7 @@ public: check_func = [&](const Security&) { REQUIRE(false); }; - consumer.on_security([&](const Security& sec, MDConsumerHandle) { ++security_count; check_func(sec); }); + consumer.on_security([&](const SecurityPtr& sec, MDConsumerHandle) { ++security_count; check_func(*sec); }); consumer.scan_securities(); REQUIRE(security_count == 0); @@ -1590,13 +1590,13 @@ namespace ImpliedTests { MDConsumer consumer(symbols, nullptr); vector consumer_securities; // Poll for security list - none visible until they have keys - size_t count = consumer.walk_security_list([&](const Security& security, MDConsumerHandle) {consumer_securities.push_back(security);}); + size_t count = consumer.walk_security_list([&](const SecurityPtr& security, MDConsumerHandle) {consumer_securities.push_back(*security);}); // Since no security has an id none are "public" REQUIRE(consumer_securities.size() == count); REQUIRE(count == 0); assign_ids(producers.begin(), producers.end()); // Now securities can be enumerated by the client - count = consumer.walk_security_list([&](const Security& security, MDConsumerHandle) {consumer_securities.push_back(security);}); + count = consumer.walk_security_list([&](const SecurityPtr& security, MDConsumerHandle) {consumer_securities.push_back(*security);}); REQUIRE(consumer_securities.size() == count); REQUIRE(count == 24); vector consumers; diff --git a/src/lib/orderman/actors/SecuritySrc.cpp b/src/lib/orderman/actors/SecuritySrc.cpp index 8067ef85f..eacf58745 100644 --- a/src/lib/orderman/actors/SecuritySrc.cpp +++ b/src/lib/orderman/actors/SecuritySrc.cpp @@ -15,7 +15,7 @@ struct SecuritySrc::PImpl { mercury::Stream new_securities_stream; std::vector to_publish; md::MDConsumer md_consumer; - std::vector> securities_keep_alive; + std::vector securities_keep_alive; }; SecuritySrc::SecuritySrc(md::MDConsumer &md) : me(new PImpl(md)) {} @@ -24,11 +24,9 @@ SecuritySrc::~SecuritySrc() {} void SecuritySrc::init() { me->new_securities_stream = make_stream(); me->md_consumer.on_security( - [this](api::Security const &s, md::MDConsumerHandle) { - auto sec = std::make_unique(s); - auto sec_ptr = sec.get(); - me->securities_keep_alive.push_back(std::move(sec)); - me->to_publish.push_back(sec_ptr); + [this](md::SecurityPtr const &sec, md::MDConsumerHandle) { + me->securities_keep_alive.push_back(sec); + me->to_publish.push_back(sec.get()); }); subscribe([this](auto) { this->on_scan_securities(); }); From fe1b9f906ee37bd78609e922e4fe0ce7a49b463f Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 4 Jan 2019 16:07:08 -0500 Subject: [PATCH 14/85] SPARK-1961 SyntheticStrategy is not properly syncronized --- src/lib/md/MDConsumer.cpp | 13 ++++++++++--- src/lib/md/MDConsumerHandle.cpp | 7 ++++--- src/lib/md/MDConsumerHandle.h | 6 +++--- src/lib/md/test/synthetic_book_merge_tests.cpp | 2 +- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/lib/md/MDConsumer.cpp b/src/lib/md/MDConsumer.cpp index fcc6b4762..e2eb29b0b 100644 --- a/src/lib/md/MDConsumer.cpp +++ b/src/lib/md/MDConsumer.cpp @@ -314,6 +314,7 @@ class SyntheticConsumerInst { bool _initialized; int _security_sub; int _meta_sub; + std::vector> _old_strategies; public: SyntheticConsumerInst(std::shared_ptr& ss, const IGetHandle& gh) : @@ -325,7 +326,8 @@ class SyntheticConsumerInst { _get_handle(gh), _initialized(false), _security_sub(0), - _meta_sub(0) + _meta_sub(0), + _old_strategies() { } @@ -418,8 +420,13 @@ private: // update security metadata auto iter = _data.find(m.security_key()); if (iter != _data.end()) { - iter->second->strategy.CopyFrom(m.strategy()); + auto updated = std::make_unique(); + updated->CopyFrom(m.strategy()); + iter->second->strategy.swap(updated); iter->second->updated = TimeStamp::now(); + // Keep the old strategy alive because it could be in use. + // metadata is changed so seldomly this is not a significant use of memory + if (updated != nullptr) _old_strategies.push_back(updated); } } @@ -475,7 +482,7 @@ private: if (legs.size() == (size_t)strategy.security->synthetic().legs_size()) { auto s = make_shared(); s->security.CopyFrom(*strategy.security); - s->strategy.CopyFrom(strategy.strategy); + s->strategy->CopyFrom(strategy.strategy); s->legs = legs; s->updated = TimeStamp::now(); result.insert( diff --git a/src/lib/md/MDConsumerHandle.cpp b/src/lib/md/MDConsumerHandle.cpp index 4062f1353..890a599c4 100644 --- a/src/lib/md/MDConsumerHandle.cpp +++ b/src/lib/md/MDConsumerHandle.cpp @@ -16,7 +16,7 @@ MDConsumerHandle::MDConsumerHandle() { _type = HANDLETYPE_VOID; static_assert(sizeof(MDConsumerHandle) <= 32, ""); } -MDConsumerHandle::MDConsumerHandle(const std::shared_ptr& def) { +MDConsumerHandle::MDConsumerHandle(const std::shared_ptr& def) { memset(this, 0, sizeof(MDConsumerHandle)); synthetic.def = def; _type = HANDLETYPE_SYNTHETIC; @@ -233,9 +233,10 @@ TimeStamp copy_synthetic_depth(const SyntheticStrategy& synth, size_t count, Pri std::array bids, asks; TimeStamp t; auto key = synth.legs[x].key(); - auto iter = synth.strategy.leg_params_map().find(key); + auto strategy = synth.strategy.get(); + auto iter = strategy->leg_params_map().find(key); const spark::LegSpreaderParams& leg_params = - iter == synth.strategy.leg_params_map().end() ? + iter == strategy->leg_params_map().end() ? spark::LegSpreaderParams::default_instance() : iter->second; diff --git a/src/lib/md/MDConsumerHandle.h b/src/lib/md/MDConsumerHandle.h index 2000ab7cd..770b8ee59 100644 --- a/src/lib/md/MDConsumerHandle.h +++ b/src/lib/md/MDConsumerHandle.h @@ -14,7 +14,7 @@ namespace md { class MDConsumerHandle; struct SyntheticStrategy { - spark::ExecutionStrategy strategy; + std::unique_ptr strategy; api::Security security; std::vector legs; TimeStamp updated; @@ -147,7 +147,7 @@ class BTS_EXPORT MDConsumerHandle { public: MDConsumerHandle(); - explicit MDConsumerHandle(const std::shared_ptr&); + explicit MDConsumerHandle(const std::shared_ptr&); ~MDConsumerHandle(); MDConsumerHandle(const MDConsumerHandle& other); @@ -335,7 +335,7 @@ private: const ColdHeader* cheader; }; struct SyntheticHandle { - std::shared_ptr def; + std::shared_ptr def; }; HandleType _type; union { diff --git a/src/lib/md/test/synthetic_book_merge_tests.cpp b/src/lib/md/test/synthetic_book_merge_tests.cpp index 4d174207a..822ebb907 100644 --- a/src/lib/md/test/synthetic_book_merge_tests.cpp +++ b/src/lib/md/test/synthetic_book_merge_tests.cpp @@ -135,7 +135,7 @@ tuple,vector> make_securities(c void set_leg_params(SyntheticStrategy& strat, uint32_t security_key, const spark::LegSpreaderParams& params) { - auto* map = strat.strategy.mutable_leg_params_map(); + auto* map = strat.strategy->mutable_leg_params_map(); map->insert({ security_key, params }); } From 7a6739c156d9ea05ff97a35721d647511ca207a7 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 4 Jan 2019 16:35:52 -0500 Subject: [PATCH 15/85] Fix build issue with uniqueptr --- src/lib/md/MDConsumer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/md/MDConsumer.cpp b/src/lib/md/MDConsumer.cpp index e2eb29b0b..5cc1d2abf 100644 --- a/src/lib/md/MDConsumer.cpp +++ b/src/lib/md/MDConsumer.cpp @@ -426,7 +426,7 @@ private: iter->second->updated = TimeStamp::now(); // Keep the old strategy alive because it could be in use. // metadata is changed so seldomly this is not a significant use of memory - if (updated != nullptr) _old_strategies.push_back(updated); + if (updated != nullptr) _old_strategies.push_back(std::move(updated)); } } From 501cd93fd67a096a17d6e7d8d0cb58857e6ed466 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 7 Jan 2019 15:52:37 -0500 Subject: [PATCH 16/85] SPARK-1950 Integrate SecurityCache into MDConsumer --- .vscode/launch.json | 2 +- src/lib/md/MDConsumer.cpp | 43 ++++++++++++++----- src/lib/md/MDConsumer.h | 5 ++- src/lib/md/MDConsumerHandle.cpp | 20 +++++---- src/lib/md/SecurityCache.cpp | 35 +++++++-------- src/lib/md/SecurityCache.h | 4 +- .../md/test/synthetic_book_merge_tests.cpp | 3 ++ 7 files changed, 71 insertions(+), 41 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index d3f66bbaf..cf754974d 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -209,7 +209,7 @@ "type": "cppdbg", "request": "launch", "program": "/home/bts/spark/build/ninja_debug/src/server/sparkd/sparkd", - "args": ["test","AuthToken"], + "args": ["test", "[synthetic]"], "stopAtEntry": false, "cwd": "/home/bts/spark/build/ninja_debug/src/server/sparkd/", "environment": [], diff --git a/src/lib/md/MDConsumer.cpp b/src/lib/md/MDConsumer.cpp index 5cc1d2abf..3a3eb8ab6 100644 --- a/src/lib/md/MDConsumer.cpp +++ b/src/lib/md/MDConsumer.cpp @@ -2,6 +2,7 @@ #include "MDConsumer.h" #include "MemorySegmentUtils.h" #include "PlatformLoader.h" +#include "SecurityCache.h" #include "lib/config/config.h" #include "lib/utils/SharedList.h" #include "lib/utils/WaitFreeQueue.h" @@ -530,8 +531,27 @@ private: // Pimple for MDConsumer /////////////////////////////////////////////////////////////////////////////////////////////////// class MDConsumerImpl { + typedef tbb::queuing_mutex Mutex; + Mutex _workmutex; + vector> _instances; + std::shared_ptr _sci; + map> _deferred_handles; + vector _callbacks; + SecurityCache _cache; + // co2::Platform _platform; + public: - MDConsumerImpl(const set& symbols, std::shared_ptr& ss, bool shared, co2::Platform platform = co2::PLATFORM_UNSET ) { + MDConsumerImpl( + const set& symbols, + std::shared_ptr& ss, + bool shared, co2::Platform platform = co2::PLATFORM_UNSET) : + _workmutex(), + _instances(), + _sci(), + _deferred_handles(), + _callbacks(), + _cache() + { std::ostringstream msg; for (const auto& symbol : symbols) { if (msg.tellp() > 0) { msg << ", "; } @@ -563,6 +583,11 @@ public: if (ss != nullptr) { _sci = std::make_shared(ss, [=](const set& keys) { return get_handles(keys); }); } + + // Add the SecurityCache to on_security so it can cache discovered securities + _callbacks.push_back([this](const SecurityPtr& security, MDConsumerHandle) { + _cache.add_live_security(security); + }); } ~MDConsumerImpl() { } @@ -766,7 +791,7 @@ public: } count += resolved_securities.size(); } - else /*!resolve_legs*/ { + else /* do NOT resolve_legs*/ { auto handles = get_handles(found_securities); for (const auto& security : found_securities) { auto find_result = handles.find(security->id()); @@ -801,6 +826,7 @@ public: } } + _cache.flush_to_disk(); return count; } @@ -843,6 +869,10 @@ public: scan_securities(resolve_legs); } + SecurityPtr get_security(SecurityKey key) { + return _cache.get_security(key); + } + MDConsumerChannel get_channel(const std::string& symbol, SecurityType type, Venue exchange) const { for (auto& inst : _instances) { if ((inst->symbol() == symbol && inst->type() == type && inst->exchange() == exchange) || @@ -869,14 +899,6 @@ public: return result; } -private: - typedef tbb::queuing_mutex Mutex; - Mutex _workmutex; - vector> _instances; - std::shared_ptr _sci; - map> _deferred_handles; - vector _callbacks; - // co2::Platform _platform; }; @@ -922,6 +944,7 @@ size_t MDConsumer::walk_security_list(SecurityCallback cb, const string& symbol, size_t MDConsumer::scan_securities(bool resolve_legs) { return _pimpl->scan_securities(resolve_legs); } void MDConsumer::on_security(SecurityCallback cb, bool resolve_legs) { _pimpl->on_security(cb, resolve_legs); } +SecurityPtr MDConsumer::get_security(SecurityKey key) { return _pimpl->get_security(key); } MDConsumerChannel MDConsumer::get_channel(const std::string& symbol, SecurityType type, Venue exchange) const { return _pimpl->get_channel(symbol, type, exchange); diff --git a/src/lib/md/MDConsumer.h b/src/lib/md/MDConsumer.h index 85e74f4b9..b1ea4f304 100644 --- a/src/lib/md/MDConsumer.h +++ b/src/lib/md/MDConsumer.h @@ -19,7 +19,6 @@ namespace bts { namespace md { -/* typedef void (security_callback)(const Security& security); */ typedef std::function SecurityCallback; using std::size_t; @@ -52,7 +51,9 @@ public: size_t scan_securities(bool resolve_legs = true); void on_security(SecurityCallback cb, bool resolve_legs = true); - // SecurityPtr get_security(SecurityKey key); + // Retrieve any security that has been seen in the past + // You still need to call scan_securities() periodically to discover new securities + SecurityPtr get_security(SecurityKey key); MDConsumerChannel get_channel(const std::string& symbol, SecurityType type, Venue exchange) const; MDConsumerChannel get_channel(const UserSymbol&) const; diff --git a/src/lib/md/MDConsumerHandle.cpp b/src/lib/md/MDConsumerHandle.cpp index 890a599c4..891f2ee90 100644 --- a/src/lib/md/MDConsumerHandle.cpp +++ b/src/lib/md/MDConsumerHandle.cpp @@ -223,6 +223,17 @@ void copy_synthetic_side(size_t count, api::Side side, const api::ContractSpecif } } +const spark::LegSpreaderParams& get_spreader_params(const SyntheticStrategy& synth, security_key key) { + auto strategy = synth.strategy.get(); + if (strategy) { + auto iter = strategy->leg_params_map().find(key); + if (iter != strategy->leg_params_map().end()) { + return iter->second; + } + } + return spark::LegSpreaderParams::default_instance(); +} + TimeStamp copy_synthetic_depth(const SyntheticStrategy& synth, size_t count, PriceSize* bid, PriceSize* ask) { small_vector bid_leg_travs; small_vector ask_leg_travs; @@ -232,14 +243,7 @@ TimeStamp copy_synthetic_depth(const SyntheticStrategy& synth, size_t count, Pri for (size_t x = 0; x < synth.legs.size(); ++x) { std::array bids, asks; TimeStamp t; - auto key = synth.legs[x].key(); - auto strategy = synth.strategy.get(); - auto iter = strategy->leg_params_map().find(key); - const spark::LegSpreaderParams& leg_params = - iter == strategy->leg_params_map().end() ? - spark::LegSpreaderParams::default_instance() : - iter->second; - + const spark::LegSpreaderParams& leg_params = get_spreader_params(synth, synth.legs[x].key()); auto mdtype = leg_params.market_data_type(); HandleReadResult r; if (mdtype == spark::LegSpreaderParams::DIRECT) { diff --git a/src/lib/md/SecurityCache.cpp b/src/lib/md/SecurityCache.cpp index 2675128bb..57a370221 100644 --- a/src/lib/md/SecurityCache.cpp +++ b/src/lib/md/SecurityCache.cpp @@ -13,28 +13,23 @@ struct SecurityCacheImpl { std::mutex cache_mutex; std::unordered_map cache; tbb::concurrent_queue update_queue; - void update_db(const SecurityPtr&); + void enqueue_update_db(const SecurityPtr&); + void flush(); }; -void SecurityCacheImpl::update_db(const SecurityPtr& security) { - // Batch updates to the database for performance - // algo: - // push item into queue - // post work to dispatcher with delay - // When dispatcher services request drain update_queue. - // This will cause one update per security but most will be no-op - // This is a little simpler than running a timer and trying to stop it after security discovery is done +void SecurityCacheImpl::enqueue_update_db(const SecurityPtr& security) { update_queue.push(security); - utils::Dispatcher::background().post([this]() { - if (!update_queue.empty()) { - std::vector work; - SecurityPtr temp; - while(update_queue.try_pop(temp)) { work.push_back(temp); } - if (!work.empty()) { - db::insert_or_replace(work); - } +} + +void SecurityCacheImpl::flush() { + if (!update_queue.empty()) { + std::vector work; + SecurityPtr temp; + while(update_queue.try_pop(temp)) { work.push_back(temp); } + if (!work.empty()) { + db::insert_or_replace(work); } - }, 1000); + } } ////////////////// @@ -83,8 +78,10 @@ bool SecurityCache::add_live_security(const SecurityPtr& security) { SecurityKey key(security->id()); // Insert but don't overwrite existing auto result = _me->cache.insert(std::make_pair(SecurityKey(security->id()), security)); - _me->update_db(security); + _me->enqueue_update_db(security); return result.second; } +void SecurityCache::flush_to_disk() { _me->flush(); } + }}//ns \ No newline at end of file diff --git a/src/lib/md/SecurityCache.h b/src/lib/md/SecurityCache.h index 388d66fd8..85de41b70 100644 --- a/src/lib/md/SecurityCache.h +++ b/src/lib/md/SecurityCache.h @@ -22,13 +22,15 @@ public: //Load security from cache with fallback to database SecurityPtr get_security(SecurityKey key); - // Get the security if it's in the cache otherwise return nullptr + // Get the security if it's in the cache otherwise return nullptr. Does not query security database. SecurityPtr try_get_security(SecurityKey key) const; // returns true if the cache was updated bool add_live_security(const SecurityPtr& security); SecurityPtr add_db_security(const SecurityPtr& security); + void flush_to_disk(); + }; }}//ns \ No newline at end of file diff --git a/src/lib/md/test/synthetic_book_merge_tests.cpp b/src/lib/md/test/synthetic_book_merge_tests.cpp index 822ebb907..87e8ce79a 100644 --- a/src/lib/md/test/synthetic_book_merge_tests.cpp +++ b/src/lib/md/test/synthetic_book_merge_tests.cpp @@ -135,6 +135,9 @@ tuple,vector> make_securities(c void set_leg_params(SyntheticStrategy& strat, uint32_t security_key, const spark::LegSpreaderParams& params) { + if (!strat.strategy) { + strat.strategy = std::make_unique(); + } auto* map = strat.strategy->mutable_leg_params_map(); map->insert({ security_key, params }); } From e36729814a31b3e11c4477e556a4ccd1fac48169 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 7 Jan 2019 16:53:04 -0500 Subject: [PATCH 17/85] SPARK-1731 Improve error handling in SecurityDB and implement db::find_security() --- src/lib/md/SecurityCache.cpp | 5 +- src/lib/md/SecurityDB.cpp | 94 ++++++++++++++++++++++-------------- src/lib/md/SecurityDB.h | 3 +- src/lib/utils/ResultEnum.h | 7 +++ 4 files changed, 72 insertions(+), 37 deletions(-) create mode 100644 src/lib/utils/ResultEnum.h diff --git a/src/lib/md/SecurityCache.cpp b/src/lib/md/SecurityCache.cpp index 57a370221..affe44c9e 100644 --- a/src/lib/md/SecurityCache.cpp +++ b/src/lib/md/SecurityCache.cpp @@ -27,7 +27,10 @@ void SecurityCacheImpl::flush() { SecurityPtr temp; while(update_queue.try_pop(temp)) { work.push_back(temp); } if (!work.empty()) { - db::insert_or_replace(work); + if (db::insert_or_replace(work) == Result::error) { + // If we fail to insert then retry later + for (auto& s : work) update_queue.push(std::move(s)); + } } } } diff --git a/src/lib/md/SecurityDB.cpp b/src/lib/md/SecurityDB.cpp index c374765a9..e16bb7584 100644 --- a/src/lib/md/SecurityDB.cpp +++ b/src/lib/md/SecurityDB.cpp @@ -56,15 +56,31 @@ datetime::Date sec_expiration_cut_off() return datetime::Date(pt.date().year(),pt.date().month(),pt.date().day()); } +void apply_compatibilit_mutations(Security& security) { + // Migrate from old tick stuff + if (security.has_glbx()) { + if (security.has_glbx()) { + if (security.contract_spec().main_fraction() == 0) { + security.mutable_contract_spec()->set_main_fraction(security.glbx().do_not_use_main_fraction()); + } + if (security.contract_spec().sub_fraction() == 0) { + security.mutable_contract_spec()->set_sub_fraction(security.glbx().do_not_use_sub_fraction()); + } + if (security.contract_spec().price_display_format() == 0) { + security.mutable_contract_spec()->set_price_display_format(security.glbx().do_not_use_price_display_format()); + } + } + } +} + size_t load_securities( const std::set& symbols, const std::function& callback) { size_t count = 0; - auto dbptr = open_db(); + auto db = open_db(); // Load all existing securities and call the callback - auto db = dbptr.get(); SQLite::Statement select_stmt(*db, "select key,symbol,venue,type,security_blob from securities where expiration > :exp"); select_stmt.bind(":exp", sec_expiration_cut_off().to_short()); @@ -77,21 +93,8 @@ size_t load_securities( UserSymbol us(symbol, venue, type); if (symbols.find(us) != symbols.end()) { auto blob = select_stmt.getColumn("security_blob"); - // auto data = blob.getBlob(); - // auto len = blob.getBytes(); - // Migrate from old tick stuff if (security.ParseFromArray(blob, blob.getBytes())) { - if (security.has_glbx()) { - if (security.contract_spec().main_fraction() == 0) { - security.mutable_contract_spec()->set_main_fraction(security.glbx().do_not_use_main_fraction()); - } - if (security.contract_spec().sub_fraction() == 0) { - security.mutable_contract_spec()->set_sub_fraction(security.glbx().do_not_use_sub_fraction()); - } - if (security.contract_spec().price_display_format() == 0) { - security.mutable_contract_spec()->set_price_display_format(security.glbx().do_not_use_price_display_format()); - } - } + apply_compatibilit_mutations(security); bts_assert(security_key == security.id()); count++; callback(security); @@ -106,29 +109,50 @@ size_t load_securities( } SecurityPtr find_security(SecurityKey key) { - //todo: - (void)key; + try { + auto db = open_db(); + SQLite::Statement select(*db, "select security_blob from securities where key = :key"); + select.bind(":key", key.value()); + if(select.tryExecuteStep() == SQLITE_ROW) { + auto result = std::make_shared(); + auto blob = select.getColumn(0); + if(result->ParseFromArray(blob, blob.getBytes())) { + apply_compatibilit_mutations(*result); + return result; + } + } + } + catch(const SQLite::Exception& e) { + LOG(md,Error, "Error reading security database. {}", std::string(e.what())); + } return nullptr; } -void insert_or_replace(const std::vector& securities) { - auto db = open_db(); - SQLite::Statement insert(*db,"insert or replace into securities (key, symbol, venue, type, security_blob, expiration) values (?,?,?,?,?,?)"); - SQLite::Transaction transaction(*db); - std::string buffer; - for (const auto& psecurity : securities) { - if (psecurity && psecurity->SerializeToString(&buffer)) { - insert.bind(1, psecurity->id()); - insert.bind(2, psecurity->symbol()); - insert.bind(3, psecurity->exchange()); - insert.bind(4, securitytype_of(*psecurity)); - insert.bind(5, buffer); - insert.bind(6, expiration(*psecurity).to_short()); - insert.exec(); - insert.reset(); +Result insert_or_replace(const std::vector& securities) { + try { + auto db = open_db(); + SQLite::Statement insert(*db,"insert or replace into securities (key, symbol, venue, type, security_blob, expiration) values (?,?,?,?,?,?)"); + SQLite::Transaction transaction(*db); + std::string buffer; + for (const auto& psecurity : securities) { + if (psecurity && psecurity->SerializeToString(&buffer)) { + insert.bind(1, psecurity->id()); + insert.bind(2, psecurity->symbol()); + insert.bind(3, psecurity->exchange()); + insert.bind(4, securitytype_of(*psecurity)); + insert.bind(5, buffer); + insert.bind(6, expiration(*psecurity).to_short()); + insert.exec(); + insert.reset(); + } } + transaction.commit(); } - transaction.commit(); + catch(const SQLite::Exception& e) { + LOG(md,Warn, "Error writing to security db (retry pending): {}", std::string(e.what())); + return Result::error; + } + return Result::ok; } @@ -175,7 +199,7 @@ size_t load_securities( const std::function& callback) { return 0; } -void insert_or_replace(const std::vector& handles) {} +Result insert_or_replace(const std::vector& handles) { return Result::error; } SecurityPtr find_security(SecurityKey) { return nullptr; } #endif diff --git a/src/lib/md/SecurityDB.h b/src/lib/md/SecurityDB.h index 77948dcce..b04f355da 100644 --- a/src/lib/md/SecurityDB.h +++ b/src/lib/md/SecurityDB.h @@ -3,6 +3,7 @@ #include "lib/domain_model/common.h" #include "lib/domain_model/Security.pb.h" #include "lib/utils/datetime.h" +#include "lib/utils/ResultEnum.h" #include "lib/md/MDProducerHandle.h" #include "lib/api/services/SyntheticSecurityService.pb.h" #include "common.h" @@ -29,7 +30,7 @@ size_t load_securities( SecurityPtr find_security(SecurityKey); -void insert_or_replace(const std::vector&); +Result insert_or_replace(const std::vector&); // metadata std::vector load_metadata(); diff --git a/src/lib/utils/ResultEnum.h b/src/lib/utils/ResultEnum.h new file mode 100644 index 000000000..3509ce0b2 --- /dev/null +++ b/src/lib/utils/ResultEnum.h @@ -0,0 +1,7 @@ +#pragma once +namespace bts { + enum class Result { + ok, + error + }; +} \ No newline at end of file From 2020f6af6d0d99ed99f40d0f6ec7e5d7e14c4a2b Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 8 Jan 2019 11:28:19 -0500 Subject: [PATCH 18/85] SPARK-1951 Add SecurityCache access to QueryServer --- src/lib/api/services/MarketDataRelay.proto | 8 +++++++- src/lib/domain_model/Security.proto | 4 ++++ src/lib/md/MDConsumer.cpp | 6 +++--- src/lib/md/MDConsumer.h | 2 +- src/lib/md/relay/QueryServer.cpp | 19 +++++++++++++++++++ 5 files changed, 34 insertions(+), 5 deletions(-) diff --git a/src/lib/api/services/MarketDataRelay.proto b/src/lib/api/services/MarketDataRelay.proto index d93256c7c..d83daf33b 100644 --- a/src/lib/api/services/MarketDataRelay.proto +++ b/src/lib/api/services/MarketDataRelay.proto @@ -14,14 +14,20 @@ message ExpirationNameList { bts.api.Product product = 1; map names = 2; } - message ExpirationNameListReply { repeated ExpirationNameList securities = 2; } +message SecurityKeyList { + repeated uint32 keys = 1; +} + service MarketDataRelay { option (bts.api.service_description)="Poll the market data service for updates"; rpc requestmd(bts.co2.MDRequest) returns (bts.co2.MDReply); + rpc request_products(bts.api.VoidMessage) returns (ProductListReply); rpc request_names(bts.api.VoidMessage) returns (ExpirationNameListReply); + + rpc get_securities(SecurityKeyList) returns (bts.api.SecurityList); } diff --git a/src/lib/domain_model/Security.proto b/src/lib/domain_model/Security.proto index 5e921021e..8ee1db886 100644 --- a/src/lib/domain_model/Security.proto +++ b/src/lib/domain_model/Security.proto @@ -1,6 +1,7 @@ syntax="proto3"; package bts.api; option csharp_namespace="bts.api"; +option cc_enable_arenas=true; import "lib/api/apioptions.proto"; enum Venue { @@ -163,6 +164,9 @@ message Security { ICEExchangeData ice = 230; } } +message SecurityList { + repeated Security security = 1; +} message Product { bts.api.Venue exchange = 1; diff --git a/src/lib/md/MDConsumer.cpp b/src/lib/md/MDConsumer.cpp index 3a3eb8ab6..449bfe420 100644 --- a/src/lib/md/MDConsumer.cpp +++ b/src/lib/md/MDConsumer.cpp @@ -537,7 +537,7 @@ class MDConsumerImpl { std::shared_ptr _sci; map> _deferred_handles; vector _callbacks; - SecurityCache _cache; + mutable SecurityCache _cache; // co2::Platform _platform; public: @@ -869,7 +869,7 @@ public: scan_securities(resolve_legs); } - SecurityPtr get_security(SecurityKey key) { + SecurityPtr get_security(SecurityKey key) const { return _cache.get_security(key); } @@ -944,7 +944,7 @@ size_t MDConsumer::walk_security_list(SecurityCallback cb, const string& symbol, size_t MDConsumer::scan_securities(bool resolve_legs) { return _pimpl->scan_securities(resolve_legs); } void MDConsumer::on_security(SecurityCallback cb, bool resolve_legs) { _pimpl->on_security(cb, resolve_legs); } -SecurityPtr MDConsumer::get_security(SecurityKey key) { return _pimpl->get_security(key); } +SecurityPtr MDConsumer::get_security(SecurityKey key) const { return _pimpl->get_security(key); } MDConsumerChannel MDConsumer::get_channel(const std::string& symbol, SecurityType type, Venue exchange) const { return _pimpl->get_channel(symbol, type, exchange); diff --git a/src/lib/md/MDConsumer.h b/src/lib/md/MDConsumer.h index b1ea4f304..a53b33665 100644 --- a/src/lib/md/MDConsumer.h +++ b/src/lib/md/MDConsumer.h @@ -53,7 +53,7 @@ public: // Retrieve any security that has been seen in the past // You still need to call scan_securities() periodically to discover new securities - SecurityPtr get_security(SecurityKey key); + SecurityPtr get_security(SecurityKey key) const; MDConsumerChannel get_channel(const std::string& symbol, SecurityType type, Venue exchange) const; MDConsumerChannel get_channel(const UserSymbol&) const; diff --git a/src/lib/md/relay/QueryServer.cpp b/src/lib/md/relay/QueryServer.cpp index 57ab06e89..6c3e2e457 100644 --- a/src/lib/md/relay/QueryServer.cpp +++ b/src/lib/md/relay/QueryServer.cpp @@ -388,6 +388,16 @@ public: } } + void get_securities(google::protobuf::RepeatedField keys, api::SecurityList* result) const { + bts_assert(result); + for (auto k : keys) { + auto ptr = _consumer.get_security(SecurityKey(k)); + if (ptr) { + result->add_security()->CopyFrom(*ptr); + } + } + } + }; struct MDRelayServer : bts::co2::AuthenticatedMarketDataRelayBase { @@ -474,6 +484,15 @@ struct MDRelayServer : bts::co2::AuthenticatedMarketDataRelayBase { reply(*preply); }); } + + virtual void get_securities(const api::AuthToken&, const SecurityKeyList& keys, const std::function& reply) override { + _threadpool.post([=]() { + google::protobuf::Arena arena; + api::SecurityList* preply = google::protobuf::Arena::CreateMessage(&arena); + _db.get_securities(keys.keys(), preply); + reply(*preply); + }); + } }; struct QueryServer { From 20fe97bbbaf02023a1892aa8a99f6501d456a135 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 8 Jan 2019 11:45:18 -0500 Subject: [PATCH 19/85] Move ConsoleEx.cs because it's in the wrong place --- src/client/spark_core/{TestRunner => utils}/ConsoleEx.cs | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/client/spark_core/{TestRunner => utils}/ConsoleEx.cs (100%) diff --git a/src/client/spark_core/TestRunner/ConsoleEx.cs b/src/client/spark_core/utils/ConsoleEx.cs similarity index 100% rename from src/client/spark_core/TestRunner/ConsoleEx.cs rename to src/client/spark_core/utils/ConsoleEx.cs From aea38c168984d8be7be7b59be1a0e8a9b8f8a15a Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 8 Jan 2019 12:54:38 -0500 Subject: [PATCH 20/85] SPARK-1962 - Add SecurityStore.cs --- src/client/spark/Daemons/ShellServices.cs | 5 +- src/client/spark/Security/SecurityStore.cs | 92 ++++++++++++++++++++++ 2 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 src/client/spark/Security/SecurityStore.cs diff --git a/src/client/spark/Daemons/ShellServices.cs b/src/client/spark/Daemons/ShellServices.cs index eb9deee90..c51a6b0da 100644 --- a/src/client/spark/Daemons/ShellServices.cs +++ b/src/client/spark/Daemons/ShellServices.cs @@ -5,7 +5,7 @@ using bts.api.orderman; using bts.api.persistence; using bts.api.position; using bts.api.synthetic; -using bts.api.user; +using bts.co2; using bts.metrics; using bts.support; using System.ComponentModel.Composition; @@ -41,5 +41,8 @@ namespace Spark { [Export] public MarketDataAuxiliaryService md_auxiliary_service => context.locate(); + + [Export] + public MarketDataRelay md_replay => context.locate(); } } \ No newline at end of file diff --git a/src/client/spark/Security/SecurityStore.cs b/src/client/spark/Security/SecurityStore.cs new file mode 100644 index 000000000..3ef2673bf --- /dev/null +++ b/src/client/spark/Security/SecurityStore.cs @@ -0,0 +1,92 @@ +using bts.api; +using bts.co2; +using mbts; +using mbts.spark; +using System; +using System.Collections.Generic; +using System.ComponentModel.Composition; +using System.Threading.Tasks; +using System.Linq; +using security_key = System.UInt32; +using System.Collections.Concurrent; + +namespace bts { + [Export] + class SecurityStore : IDisposable { + readonly MarketDataRelay _md; + IDisposable _dispose; + IReadOnlyDictionary _live_securities; + readonly ConcurrentDictionary _cache = new ConcurrentDictionary(); + + [ImportingConstructor] + public SecurityStore(MarketDataRelay md, IBus bus) { + _md = md; + _dispose = bus.subscribe(SparkTags.SecuritiesTag).Subscribe(dict => _live_securities = dict); + } + + SecurityImmutable? try_get_local(security_key key) { + // Try live securities first + if (_live_securities != null) { + if (_live_securities.TryGetValue(key, out SecurityMD value)) { + return value.security; + } + } + + // Then try cached securities + if (_cache.TryGetValue(key, out SecurityImmutable security)) { + return security; + } + + return null; + } + + public async Task get_security(security_key key) { + var local_sec = try_get_local(key); + if (local_sec.HasValue) return local_sec; + + // Query the server for the value + var result = await _md.get_securities_async(new SecurityKeyList() { + Keys = { key } + }); + + if (result.success) { + var security = result.value.Security.FirstOrDefault(); + if (security != null) { + _cache.TryAdd(security.Id, new SecurityImmutable(security)); + return security; + } + } + + // Give up :( + Log.Md.warn($"Failed to get security for key {key}"); + return null; + } + + public async Task> get_securities(IEnumerable keys) { + var result = new Dictionary(); + var todo = new SecurityKeyList(); + foreach (var key in keys) { + var s = try_get_local(key); + if (s.HasValue) { + result.Add(key, s.Value); + } + else { + todo.Keys.Add(key); + } + } + if (todo.Keys.Count > 0) { + var res = await _md.get_securities_async(todo); + if (res.success) { + foreach (var security in res.value.Security) { + result.Add(security.Id, new SecurityImmutable(security)); + } + } + } + return result; + } + + public void Dispose() { + Disposable.dispose(ref _dispose); + } + } +} From 1abec0ab39b706ac7ea3fa6fab334c21a5331211 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 8 Jan 2019 16:32:57 -0500 Subject: [PATCH 21/85] Attempt to defer initial security publish until loading is complete so initial load is not bogged down by ticking security feed --- src/client/spark/Daemons/MDDaemon.cs | 12 +++++++----- src/client/spark_api/md/MDSource.cs | 24 ++++++++++++++++-------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/src/client/spark/Daemons/MDDaemon.cs b/src/client/spark/Daemons/MDDaemon.cs index 816faac4a..4231dd0f9 100644 --- a/src/client/spark/Daemons/MDDaemon.cs +++ b/src/client/spark/Daemons/MDDaemon.cs @@ -14,19 +14,21 @@ using SecurityKey = System.UInt32; using System.Collections.Concurrent; using System.Threading; using bts.api.user; +using bts.md; namespace bts { [Export(typeof(IInitializable))] - class MDDaemon : bts.md.MDSource, IObserver>, IInitializable, IDisposable { + class MDDaemon : md.MDSource, IObserver>, IInitializable, IDisposable { readonly Lazy _bus; readonly Lazy _token; readonly Lazy _ss; readonly Dictionary _deleted_synthetics = new Dictionary(); - static uint get_naggle() => mbts.config.Configuration.get().naggle_ms; + static uint get_naggle() => mbts.config.Configuration.get().naggle_ms; [ImportingConstructor] - public MDDaemon(Lazy bus, Lazy token, Lazy ss) : base(get_naggle(), true) { + public MDDaemon(Lazy bus, Lazy token, Lazy ss) : + base(get_naggle(), MDSourceOptions.DeferInit | MDSourceOptions.DeferPublish) { _bus = bus; _token = token; _ss = ss; @@ -44,14 +46,14 @@ namespace bts { if (!deleted_synthetic_keys.Contains(kvp.Key)) { _exchange_sec_map.TryAdd(kvp.Key, kvp.Value); _deleted_synthetics.Remove(kvp.Key); - _version += 1; + _version = DateTime.UtcNow; } } // Remove any newly deleted synthetics and store them for potential undelete foreach (var key in deleted_synthetic_keys) { if (_exchange_sec_map.TryRemove(key, out SecurityMD deleted)) { _deleted_synthetics[key] = deleted; - _version += 1; + _version = DateTime.UtcNow; } } // Post full dictionary of syntheics here diff --git a/src/client/spark_api/md/MDSource.cs b/src/client/spark_api/md/MDSource.cs index 9fdebb63b..d8559e799 100644 --- a/src/client/spark_api/md/MDSource.cs +++ b/src/client/spark_api/md/MDSource.cs @@ -15,20 +15,27 @@ using System.Reactive.Subjects; namespace bts.md { + [Flags] + enum MDSourceOptions { + DeferInit = 1, + DeferPublish = 2 + } class MDSource : IObserver>, IDisposable { protected readonly MDConsumer _consumer = new MDConsumer(); protected readonly Co2RelayControl _relay; - protected long _version = 0; + protected DateTime _version = DateTime.MinValue; protected readonly ConcurrentDictionary _exchange_sec_map = new ConcurrentDictionary(); protected List _dispose = new List(); + protected readonly MDSourceOptions _options; - public MDSource(uint naggle_ms, bool defer_init = false) - { + public MDSource(uint naggle_ms, MDSourceOptions options = 0) + { + _options = options; _relay = new Co2RelayControl(_consumer, naggle_ms); _relay.preload_securities((co2.MDRequest.Types.Flags)0xff).ignore(); _dispose.Add(_consumer.on_security.Subscribe(this)); - if(!defer_init) + if(options.HasFlag(MDSourceOptions.DeferInit)) initialize(); } @@ -57,19 +64,20 @@ namespace bts.md var smd = new SecurityMD(new SecurityImmutable(v.security), v.handle); _exchange_sec_map.AddOrUpdate(smd.security.Id, smd, (_, __) => smd); } - _version += 1; + _version = DateTime.UtcNow; } } public void Dispose() => Disposable.dispose(_dispose); async protected Task run() { - long run_version = 0; + DateTime run_version = DateTime.MinValue; + var minimum_change_delta = _options.HasFlag(MDSourceOptions.DeferPublish) ? TimeSpan.FromMilliseconds(200) : TimeSpan.Zero; while (true) { var exch = _exchange_sec_map; - long current_version = _version; - if (current_version > run_version) + var current_version = _version; + if (current_version - run_version > minimum_change_delta) { run_version = current_version; try From 2e542fb8f53c54de2dd2ee18c7f51a1213d36268 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 8 Jan 2019 16:33:41 -0500 Subject: [PATCH 22/85] Update.cs all should be a more generic type --- src/client/spark_api/api/Update.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/client/spark_api/api/Update.cs b/src/client/spark_api/api/Update.cs index 6f9604a01..d5f1b04fd 100644 --- a/src/client/spark_api/api/Update.cs +++ b/src/client/spark_api/api/Update.cs @@ -70,9 +70,9 @@ namespace bts.api.core { } public class Update where TKey : IEquatable { - private Func _key; + readonly Func _key; internal Update( - IReadOnlyCollection all, + IEnumerable all, IEnumerable added, IEnumerable> changes, IEnumerable removed, @@ -122,7 +122,7 @@ namespace bts.api.core { ); } - public IReadOnlyCollection all { get; } + public IEnumerable all { get; } public IEnumerable added { get; } public IEnumerable> changes { get; } public IEnumerable changed => changes.Select(c => c.New); From adf4476399d68513f63f84f72e7de5283ba551ae Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 8 Jan 2019 16:46:44 -0500 Subject: [PATCH 23/85] SPARK-1962 - Efficient observable join on all available securities --- src/client/spark/Security/SecurityStore.cs | 65 ++++++++++++++++++++-- 1 file changed, 59 insertions(+), 6 deletions(-) diff --git a/src/client/spark/Security/SecurityStore.cs b/src/client/spark/Security/SecurityStore.cs index 3ef2673bf..625697ae1 100644 --- a/src/client/spark/Security/SecurityStore.cs +++ b/src/client/spark/Security/SecurityStore.cs @@ -1,14 +1,17 @@ using bts.api; +using bts.api.core; using bts.co2; using mbts; using mbts.spark; using System; -using System.Collections.Generic; -using System.ComponentModel.Composition; -using System.Threading.Tasks; -using System.Linq; -using security_key = System.UInt32; using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.ComponentModel.Composition; +using System.Linq; +using System.Reactive.Linq; +using System.Threading.Tasks; +using security_key = System.UInt32; namespace bts { [Export] @@ -42,7 +45,9 @@ namespace bts { public async Task get_security(security_key key) { var local_sec = try_get_local(key); - if (local_sec.HasValue) return local_sec; + if (local_sec.HasValue) { + return local_sec; + } // Query the server for the value var result = await _md.get_securities_async(new SecurityKeyList() { @@ -85,6 +90,54 @@ namespace bts { return result; } + IObservable> join(IObservable> source) { + return Observable.Create>(observer => { + var all = ImmutableDictionary.Empty; + return source.Subscribe(async (update) => { + var key = update.key_extractor; + var relevant_securities = await get_securities(update.added.Concat(update.changed).Select(key)); + + var added = new List<(SecurityImmutable?, T)>(); + var changed = new List>(); + var removed = new List<(SecurityImmutable?, T)>(); + + foreach (var x in update.added) { + if (relevant_securities.TryGetValue(key(x), out SecurityImmutable s)) { + added.Add((s, x)); + } + else { + added.Add((null, x)); + } + } + foreach (var x in update.removed) { + if (all.TryGetValue(key(x), out (SecurityImmutable?, T) value)) { + removed.Add(value); + } + } + foreach (var ch in update.changes) { + SecurityImmutable? si = + relevant_securities.TryGetValue(key(ch.New), out SecurityImmutable s) ? + s : (SecurityImmutable?)null; + changed.Add(new Change<(SecurityImmutable?, T)>((si, ch.Old), (si, ch.New))); + } + + var builder = all.ToBuilder(); + builder.RemoveRange(update.removed.Select(key)); + builder.RemoveRange(changed.Select(ch => key(ch.Old.Item2))); + foreach (var a in added) { + builder.Add(key(a.Item2), a); + } + foreach (var ch in changed) { + builder.Add(key(ch.New.Item2), ch.New); + } + all = builder.ToImmutable(); + + observer.OnNext(new Update<(SecurityImmutable?, T), security_key>( + all.Values, added, changed, removed, i => key(i.Item2))); + }); + }); + } + public void Dispose() { Disposable.dispose(ref _dispose); } From 4991fa734c2a3b756f73e783e889e66b361c6c7c Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 8 Jan 2019 17:33:31 -0500 Subject: [PATCH 24/85] SPARK-1962 - Add operator so this join can be used inline --- src/client/spark/Security/SecurityStore.cs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/client/spark/Security/SecurityStore.cs b/src/client/spark/Security/SecurityStore.cs index 625697ae1..1167cddcf 100644 --- a/src/client/spark/Security/SecurityStore.cs +++ b/src/client/spark/Security/SecurityStore.cs @@ -15,7 +15,7 @@ using security_key = System.UInt32; namespace bts { [Export] - class SecurityStore : IDisposable { + public class SecurityStore : IDisposable { readonly MarketDataRelay _md; IDisposable _dispose; IReadOnlyDictionary _live_securities; @@ -90,12 +90,18 @@ namespace bts { return result; } - IObservable> join(IObservable> source) { + public void Dispose() { + Disposable.dispose(ref _dispose); + } + } + + public static class SecurityStoreExtensions { + public static IObservable> join(this IObservable> source, SecurityStore store) { return Observable.Create>(observer => { var all = ImmutableDictionary.Empty; return source.Subscribe(async (update) => { var key = update.key_extractor; - var relevant_securities = await get_securities(update.added.Concat(update.changed).Select(key)); + var relevant_securities = await store.get_securities(update.added.Concat(update.changed).Select(key)); var added = new List<(SecurityImmutable?, T)>(); var changed = new List>(); @@ -137,9 +143,5 @@ namespace bts { }); }); } - - public void Dispose() { - Disposable.dispose(ref _dispose); - } } } From 12f6c58c7cce4ddbbf5229812a7970c2170c4035 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 9 Jan 2019 15:24:45 -0500 Subject: [PATCH 25/85] Reconsidering 1abec0ab --- src/client/spark/Daemons/MDDaemon.cs | 2 +- src/client/spark_api/md/MDSource.cs | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/client/spark/Daemons/MDDaemon.cs b/src/client/spark/Daemons/MDDaemon.cs index 4231dd0f9..70361557b 100644 --- a/src/client/spark/Daemons/MDDaemon.cs +++ b/src/client/spark/Daemons/MDDaemon.cs @@ -28,7 +28,7 @@ namespace bts { [ImportingConstructor] public MDDaemon(Lazy bus, Lazy token, Lazy ss) : - base(get_naggle(), MDSourceOptions.DeferInit | MDSourceOptions.DeferPublish) { + base(get_naggle(), MDSourceOptions.DeferInit) { _bus = bus; _token = token; _ss = ss; diff --git a/src/client/spark_api/md/MDSource.cs b/src/client/spark_api/md/MDSource.cs index d8559e799..392fc6844 100644 --- a/src/client/spark_api/md/MDSource.cs +++ b/src/client/spark_api/md/MDSource.cs @@ -18,7 +18,6 @@ namespace bts.md [Flags] enum MDSourceOptions { DeferInit = 1, - DeferPublish = 2 } class MDSource : IObserver>, IDisposable { @@ -35,7 +34,7 @@ namespace bts.md _relay = new Co2RelayControl(_consumer, naggle_ms); _relay.preload_securities((co2.MDRequest.Types.Flags)0xff).ignore(); _dispose.Add(_consumer.on_security.Subscribe(this)); - if(options.HasFlag(MDSourceOptions.DeferInit)) + if(!options.HasFlag(MDSourceOptions.DeferInit)) initialize(); } @@ -72,12 +71,11 @@ namespace bts.md async protected Task run() { DateTime run_version = DateTime.MinValue; - var minimum_change_delta = _options.HasFlag(MDSourceOptions.DeferPublish) ? TimeSpan.FromMilliseconds(200) : TimeSpan.Zero; while (true) { var exch = _exchange_sec_map; var current_version = _version; - if (current_version - run_version > minimum_change_delta) + if (current_version > run_version) { run_version = current_version; try From e5ab062203957f5a1744c698bee9d51fe8736fd7 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 9 Jan 2019 18:32:52 -0500 Subject: [PATCH 26/85] SPARK-1962 Fix race bug identified in testing --- src/client/spark/Orders/OrdersModel.cs | 17 +- src/client/spark/Security/SecurityStore.cs | 224 +++++++++++++++------ 2 files changed, 179 insertions(+), 62 deletions(-) diff --git a/src/client/spark/Orders/OrdersModel.cs b/src/client/spark/Orders/OrdersModel.cs index eba27fea3..910ff14e6 100644 --- a/src/client/spark/Orders/OrdersModel.cs +++ b/src/client/spark/Orders/OrdersModel.cs @@ -34,7 +34,8 @@ namespace mbts.spark { [ImportingConstructor] private OrdersModel(IBus bus, OrderFactory factory, OrderService service, PositionService position_service, - SpreaderModel spreader, ErrorReporter error, DynamicOrderHelper dynamic_helper, AdminModel admin_model) { + SpreaderModel spreader, ErrorReporter error, DynamicOrderHelper dynamic_helper, AdminModel admin_model, + SecurityStore sstore) { _bus = bus; _factory = factory; _service = service; @@ -44,7 +45,13 @@ namespace mbts.spark { _dynamic_helper = dynamic_helper; var scheduler = ThreadPoolScheduler.Instance; var interval = TimeSpan.FromMilliseconds(100); - on_orders = _service.subscribei_orders().Naggle(interval, scheduler).PermissionFilter(admin_model.on_admin_data).Table().Cache(); + on_orders = + _service + .subscribei_orders() + .Naggle(interval, scheduler) + .PermissionFilter(admin_model.on_admin_data) + //.join(sstore) + .Table().Cache(); on_positions = _position_service.subscribei_position_data().Naggle(interval, scheduler).PermissionFilter(admin_model.on_admin_data).Table().Cache(); working_orders = on_orders.Scan( @@ -55,7 +62,11 @@ namespace mbts.spark { on_order_tuples = order_record_tuple_source.subscribe_incremental_update(); on_order_tuples_table = order_record_tuple_source.subscribe_incremental_table_update(); on_order_securities = order_record_tuple_source.subscribe_order_securities(); - on_fills = _service.subscribei_fills().Naggle(interval, scheduler).PermissionFilter(admin_model.on_admin_data).Table().Cache(); + on_fills = + _service.subscribei_fills() + .Naggle(interval, scheduler) + .PermissionFilter(admin_model.on_admin_data) + .Table().Cache(); _activity = new OrderActivitySummarizer(bus, on_orders); on_fill_tuples = FillTupleSource.create_observable(bus.subscribe(SparkTags.SecuritiesTag), on_orders, on_fills); _security_orders_demuxer = new TableUpdateDemuxer(o => o.Key, o => o.security_key()); diff --git a/src/client/spark/Security/SecurityStore.cs b/src/client/spark/Security/SecurityStore.cs index 1167cddcf..5e424beac 100644 --- a/src/client/spark/Security/SecurityStore.cs +++ b/src/client/spark/Security/SecurityStore.cs @@ -10,6 +10,7 @@ using System.Collections.Immutable; using System.ComponentModel.Composition; using System.Linq; using System.Reactive.Linq; +using System.Reactive.Subjects; using System.Threading.Tasks; using security_key = System.UInt32; @@ -17,20 +18,27 @@ namespace bts { [Export] public class SecurityStore : IDisposable { readonly MarketDataRelay _md; + readonly IBus _bus; IDisposable _dispose; IReadOnlyDictionary _live_securities; readonly ConcurrentDictionary _cache = new ConcurrentDictionary(); + readonly Subject> _cache_subject = new Subject>(); [ImportingConstructor] public SecurityStore(MarketDataRelay md, IBus bus) { _md = md; + _bus = bus; _dispose = bus.subscribe(SparkTags.SecuritiesTag).Subscribe(dict => _live_securities = dict); + _cache_subject.OnNext(_cache); } - SecurityImmutable? try_get_local(security_key key) { + IObservable> on_securities => _bus.subscribe(SparkTags.SecuritiesTag); + + SecurityImmutable? try_get_local(security_key key, IReadOnlyDictionary live=null) { // Try live securities first - if (_live_securities != null) { - if (_live_securities.TryGetValue(key, out SecurityMD value)) { + live = live ?? _live_securities; + if (live != null) { + if (live.TryGetValue(key, out SecurityMD value)) { return value.security; } } @@ -43,8 +51,8 @@ namespace bts { return null; } - public async Task get_security(security_key key) { - var local_sec = try_get_local(key); + async Task get_security(security_key key, IReadOnlyDictionary live) { + var local_sec = try_get_local(key, live); if (local_sec.HasValue) { return local_sec; } @@ -57,8 +65,7 @@ namespace bts { if (result.success) { var security = result.value.Security.FirstOrDefault(); if (security != null) { - _cache.TryAdd(security.Id, new SecurityImmutable(security)); - return security; + return try_cache_security(new SecurityImmutable(security)); } } @@ -67,81 +74,180 @@ namespace bts { return null; } - public async Task> get_securities(IEnumerable keys) { + async Task> get_securities(IEnumerable keys, IReadOnlyDictionary live) { var result = new Dictionary(); - var todo = new SecurityKeyList(); + var missing = new SecurityKeyList(); foreach (var key in keys) { var s = try_get_local(key); if (s.HasValue) { result.Add(key, s.Value); } else { - todo.Keys.Add(key); + missing.Keys.Add(key); } } - if (todo.Keys.Count > 0) { - var res = await _md.get_securities_async(todo); + if (missing.Keys.Count > 0) { + var res = await _md.get_securities_async(missing); if (res.success) { foreach (var security in res.value.Security) { - result.Add(security.Id, new SecurityImmutable(security)); + result.Add(security.Id, try_cache_security(new SecurityImmutable(security))); } } } return result; } + // Request any missing keys from the server but do not wait for them to arrive + IReadOnlyDictionary get_and_ensure_securities(IEnumerable keys) { + var result = new Dictionary(); + var missing = new SecurityKeyList(); + foreach (var key in keys) { + var security = try_get_local(key); + if (security.HasValue) { + result.Add(key, security.Value); + } + else { + missing.Keys.Add(key); + } + } + // Update cache in the background + Task.Run(async () => { + var r = await _md.get_securities_async(missing); + if (r.success) { + foreach (var security in r.value.Security) { + try_cache_security(new SecurityImmutable(security)); + } + } + }); + return result; + } + + SecurityImmutable try_cache_security(SecurityImmutable s) { + if (_live_securities == null || !_live_securities.TryGetValue(s.Id, out SecurityMD live)) { + _cache.TryAdd(s.Id, s); + _cache_subject.OnNext(_cache); + return s; + } + return live.security; + } + + /////////////// + // PUBLIC + /////////////// + #region public + + // Guarunteed to return a security if it has ever been seen by the system + public Task get_security(security_key key) { + return get_security(key, _live_securities); + } + // Guarunteed to return a security if it has ever been seen by the system + public Task> get_securities(IEnumerable keys) { + return get_securities(keys, _live_securities); + } public void Dispose() { Disposable.dispose(ref _dispose); } + + public IObservable> join( + IObservable> source) { + return Observable.Create>(observer => { + var state = ImmutableDictionary.Empty; + var gate = new object(); + Func keyfn = null; + + void on_securities_updated(object ambiguous) { + lock (gate) { + // If this is null we don't have any rows + if (keyfn == null) return; + var changed = new List>(); + foreach (var entry in state.Values) { + if (entry.s == null) { + var security = try_get_local(keyfn(entry.t), ambiguous as IReadOnlyDictionary); + if (security.HasValue) { + changed.Add( + new Change<(SecurityImmutable?, T)>(entry, (security, entry.t))); + } + } + } + if (changed.Count > 0) { + // update state + var builder = state.ToBuilder(); + foreach (var up in changed) { + builder[up.New.s.Value.Id] = up.New; + } + state = builder.ToImmutable(); + // fire update + observer.OnNext( + new Update<(SecurityImmutable?, T), security_key>( + state.Values, + Enumerable.Empty<(SecurityImmutable?,T)>(), + changed, + Enumerable.Empty<(SecurityImmutable?,T)>(), + i => keyfn(i.Item2))); + } + } + } + + return Disposable.Create( + // Fire update when source changes + source.Subscribe((update) => { + lock (gate) { + keyfn = update.key_extractor; + var relevant_securities = get_and_ensure_securities(update.added.Concat(update.changed).Select(keyfn)); + + var added = new List<(SecurityImmutable?, T)>(); + var changed = new List>(); + var removed = new List<(SecurityImmutable?, T)>(); + + foreach (var x in update.added) { + if (relevant_securities.TryGetValue(keyfn(x), out SecurityImmutable s)) { + added.Add((s, x)); + } + else { + added.Add((null, x)); + } + } + foreach (var x in update.removed) { + if (state.TryGetValue(keyfn(x), out (SecurityImmutable?, T) value)) { + removed.Add(value); + } + } + foreach (var ch in update.changes) { + SecurityImmutable? si = + relevant_securities.TryGetValue(keyfn(ch.New), out SecurityImmutable s) ? + s : (SecurityImmutable?)null; + changed.Add(new Change<(SecurityImmutable?, T)>((si, ch.Old), (si, ch.New))); + } + + var builder = state.ToBuilder(); + builder.RemoveRange(update.removed.Select(keyfn)); + builder.RemoveRange(changed.Select(ch => keyfn(ch.Old.Item2))); + foreach (var a in added) { + builder.Add(keyfn(a.Item2), a); + } + foreach (var ch in changed) { + builder.Add(keyfn(ch.New.Item2), ch.New); + } + state = builder.ToImmutable(); + + observer.OnNext(new Update<(SecurityImmutable?, T), security_key>( + state.Values, added, changed, removed, i => keyfn(i.Item2))); + } + }), + // Also fire update when securities change + on_securities.Subscribe(on_securities_updated), + // Also fire update when cache changes + _cache_subject.Subscribe(on_securities_updated) + ); + }); + } + #endregion } public static class SecurityStoreExtensions { - public static IObservable> join(this IObservable> source, SecurityStore store) { - return Observable.Create>(observer => { - var all = ImmutableDictionary.Empty; - return source.Subscribe(async (update) => { - var key = update.key_extractor; - var relevant_securities = await store.get_securities(update.added.Concat(update.changed).Select(key)); - - var added = new List<(SecurityImmutable?, T)>(); - var changed = new List>(); - var removed = new List<(SecurityImmutable?, T)>(); - - foreach (var x in update.added) { - if (relevant_securities.TryGetValue(key(x), out SecurityImmutable s)) { - added.Add((s, x)); - } - else { - added.Add((null, x)); - } - } - foreach (var x in update.removed) { - if (all.TryGetValue(key(x), out (SecurityImmutable?, T) value)) { - removed.Add(value); - } - } - foreach (var ch in update.changes) { - SecurityImmutable? si = - relevant_securities.TryGetValue(key(ch.New), out SecurityImmutable s) ? - s : (SecurityImmutable?)null; - changed.Add(new Change<(SecurityImmutable?, T)>((si, ch.Old), (si, ch.New))); - } - - var builder = all.ToBuilder(); - builder.RemoveRange(update.removed.Select(key)); - builder.RemoveRange(changed.Select(ch => key(ch.Old.Item2))); - foreach (var a in added) { - builder.Add(key(a.Item2), a); - } - foreach (var ch in changed) { - builder.Add(key(ch.New.Item2), ch.New); - } - all = builder.ToImmutable(); - - observer.OnNext(new Update<(SecurityImmutable?, T), security_key>( - all.Values, added, changed, removed, i => key(i.Item2))); - }); - }); + public static IObservable> join( + this IObservable> source, SecurityStore store) { + return store.join(source); } } } From a5be24aa7a6557897af5ce82a6682df2a35be6ca Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 22 Jan 2019 17:25:56 -0500 Subject: [PATCH 27/85] SPARK-1731 Add Key function to SecurityStore --- src/client/spark/Security/SecurityStore.cs | 51 +++++++++++----------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/src/client/spark/Security/SecurityStore.cs b/src/client/spark/Security/SecurityStore.cs index 5e424beac..558c588f4 100644 --- a/src/client/spark/Security/SecurityStore.cs +++ b/src/client/spark/Security/SecurityStore.cs @@ -148,21 +148,22 @@ namespace bts { Disposable.dispose(ref _dispose); } - public IObservable> join( - IObservable> source) { - return Observable.Create>(observer => { - var state = ImmutableDictionary.Empty; + public IObservable> join( + IObservable> source, Func skeyfn) where TKey : IEquatable { + return Observable.Create>(observer => { + var state = ImmutableDictionary.Empty; var gate = new object(); - Func keyfn = null; + // We get thie function from the first source update + Func<(SecurityImmutable?,T t), TKey> rowkeyfn = null; void on_securities_updated(object ambiguous) { + if (rowkeyfn == null) return; lock (gate) { // If this is null we don't have any rows - if (keyfn == null) return; var changed = new List>(); foreach (var entry in state.Values) { if (entry.s == null) { - var security = try_get_local(keyfn(entry.t), ambiguous as IReadOnlyDictionary); + var security = try_get_local(skeyfn(entry.t), ambiguous as IReadOnlyDictionary); if (security.HasValue) { changed.Add( new Change<(SecurityImmutable?, T)>(entry, (security, entry.t))); @@ -173,17 +174,17 @@ namespace bts { // update state var builder = state.ToBuilder(); foreach (var up in changed) { - builder[up.New.s.Value.Id] = up.New; + builder[rowkeyfn(up.New)] = up.New; } state = builder.ToImmutable(); // fire update observer.OnNext( - new Update<(SecurityImmutable?, T), security_key>( + new Update<(SecurityImmutable?, T), TKey>( state.Values, Enumerable.Empty<(SecurityImmutable?,T)>(), changed, Enumerable.Empty<(SecurityImmutable?,T)>(), - i => keyfn(i.Item2))); + rowkeyfn)); } } } @@ -192,15 +193,15 @@ namespace bts { // Fire update when source changes source.Subscribe((update) => { lock (gate) { - keyfn = update.key_extractor; - var relevant_securities = get_and_ensure_securities(update.added.Concat(update.changed).Select(keyfn)); + rowkeyfn = rowkeyfn ?? (i => update.key_extractor(i.Item2)); + var relevant_securities = get_and_ensure_securities(update.added.Concat(update.changed).Select(skeyfn)); var added = new List<(SecurityImmutable?, T)>(); - var changed = new List>(); + var changed = new List>(); var removed = new List<(SecurityImmutable?, T)>(); foreach (var x in update.added) { - if (relevant_securities.TryGetValue(keyfn(x), out SecurityImmutable s)) { + if (relevant_securities.TryGetValue(skeyfn(x), out SecurityImmutable s)) { added.Add((s, x)); } else { @@ -208,30 +209,30 @@ namespace bts { } } foreach (var x in update.removed) { - if (state.TryGetValue(keyfn(x), out (SecurityImmutable?, T) value)) { + if (state.TryGetValue(update.key_extractor(x), out (SecurityImmutable?, T) value)) { removed.Add(value); } } foreach (var ch in update.changes) { SecurityImmutable? si = - relevant_securities.TryGetValue(keyfn(ch.New), out SecurityImmutable s) ? + relevant_securities.TryGetValue(skeyfn(ch.New), out SecurityImmutable s) ? s : (SecurityImmutable?)null; changed.Add(new Change<(SecurityImmutable?, T)>((si, ch.Old), (si, ch.New))); } var builder = state.ToBuilder(); - builder.RemoveRange(update.removed.Select(keyfn)); - builder.RemoveRange(changed.Select(ch => keyfn(ch.Old.Item2))); + builder.RemoveRange(update.removed.Select(update.key_extractor)); + builder.RemoveRange(changed.Select(ch => rowkeyfn(ch.Old))); foreach (var a in added) { - builder.Add(keyfn(a.Item2), a); + builder.Add(rowkeyfn(a), a); } foreach (var ch in changed) { - builder.Add(keyfn(ch.New.Item2), ch.New); + builder.Add(rowkeyfn(ch.New), ch.New); } state = builder.ToImmutable(); - observer.OnNext(new Update<(SecurityImmutable?, T), security_key>( - state.Values, added, changed, removed, i => keyfn(i.Item2))); + observer.OnNext(new Update<(SecurityImmutable?, T t), TKey>( + state.Values, added, changed, removed, rowkeyfn)); } }), // Also fire update when securities change @@ -245,9 +246,9 @@ namespace bts { } public static class SecurityStoreExtensions { - public static IObservable> join( - this IObservable> source, SecurityStore store) { - return store.join(source); + public static IObservable> + join(this IObservable> source, SecurityStore store, Func skeyfn) where TKey : IEquatable { + return store.join(source, skeyfn); } } } From fd270a7ab57690790a319db9b93a79a3b82de03e Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 23 Jan 2019 08:59:43 -0500 Subject: [PATCH 28/85] Add the ability to *only* run sim from a replay file --- src/lib/config/protos/MarketData.proto | 5 +++-- src/server/sparkd/md/run_md.cpp | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/lib/config/protos/MarketData.proto b/src/lib/config/protos/MarketData.proto index 90b204f74..861b1ba08 100755 --- a/src/lib/config/protos/MarketData.proto +++ b/src/lib/config/protos/MarketData.proto @@ -4,8 +4,9 @@ package bts.md; message MarketDataConfig { // Allows you to turn off in-proc market data entirely optional bool use_md = 1 [default = true]; - optional bool use_replay = 2 [default=false]; - optional bool use_sim = 3 [default=true]; + optional bool use_onixs = 2 [default=true]; + optional bool use_replay = 3 [default=false]; + optional bool use_sim = 4 [default=true]; } message PlatformMap diff --git a/src/server/sparkd/md/run_md.cpp b/src/server/sparkd/md/run_md.cpp index c3a9e04d4..0a12bce66 100644 --- a/src/server/sparkd/md/run_md.cpp +++ b/src/server/sparkd/md/run_md.cpp @@ -79,10 +79,11 @@ void run_cleanup() { std::thread* _md_thread; void run_market_data_publishers_forever() { os::set_thread_name("md publisher"); - if (config::get().use_replay()) { + const auto& conf = config::get(); + if (conf.use_replay()) { run_replay(); } - else { + else if (conf.use_onixs()) { run_onixs(); } } From be13b812835205f3d5b84309c82580cd4d85a6ea Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 23 Jan 2019 11:35:13 -0500 Subject: [PATCH 29/85] db_dump needs stoul to handle SIM securitykeys --- src/server/sparkd/md/run_md.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server/sparkd/md/run_md.cpp b/src/server/sparkd/md/run_md.cpp index 0a12bce66..45806edb6 100644 --- a/src/server/sparkd/md/run_md.cpp +++ b/src/server/sparkd/md/run_md.cpp @@ -227,9 +227,9 @@ void dump_db(const vector& args) { if (arg == "all") { all = true; break; } - auto key = std::stoi(arg); + auto key = std::stoul(arg); if (key > 0) { - keys.insert(key); + keys.insert((uint)key); } } std::set symbols; From 90dd34945451bb531b5ad43fa65a31cfe03955c0 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 23 Jan 2019 16:18:52 -0500 Subject: [PATCH 30/85] SPARK-1973 1/2 use join for OrdersModel.on_order --- .../spark/Orders/OrderActivitySummarizer.cs | 46 ++++++++++--------- src/client/spark/Orders/OrdersModel.cs | 41 +++++++++-------- 2 files changed, 47 insertions(+), 40 deletions(-) diff --git a/src/client/spark/Orders/OrderActivitySummarizer.cs b/src/client/spark/Orders/OrderActivitySummarizer.cs index dd30e5962..3bcb6de13 100644 --- a/src/client/spark/Orders/OrderActivitySummarizer.cs +++ b/src/client/spark/Orders/OrderActivitySummarizer.cs @@ -11,6 +11,7 @@ using System.Linq; using System.Reactive.Linq; using System.Text; using System.Threading.Tasks; +using OrderPair = System.ValueTuple; namespace bts.spark { internal class Activity { @@ -26,7 +27,7 @@ namespace bts.spark { static readonly IEqualityComparer _id_comparer = bts.utils.EqualityComparer.Create(a => a.Key); - public OrderActivitySummarizer(IBus bus, IObservable> orders) { + public OrderActivitySummarizer(IBus bus, IObservable> orders) { _bus = bus; _dispose = Disposable.Create( bus.subscribe(SparkTags.SecuritiesTag).Subscribe(sec => _security = sec), @@ -38,11 +39,11 @@ namespace bts.spark { #region formatting - (SecurityImmutable?, ContractSpecificationImmutable?) get_security(uint security_key) { + SecurityImmutable? try_get_security(uint security_key) { if (_security.TryGetValue(security_key, out SecurityMD md)) { - return (md.security, md.security.ContractSpec); + return md.security; } - return (null, null); + return null; } public static string format_order_success(uint key, double price, uint size, Side side, ContractSpecificationImmutable? spec) { return $"({key}) {side.ToString().ToLower()} {size} @ {spec.format(price)}"; @@ -123,24 +124,23 @@ namespace bts.spark { #endregion - void orders_updated(TableUpdate update) { + void orders_updated(TableUpdate<(SecurityImmutable? sec, OrderRecordImmutable order), uint> update) { - foreach (var order in update.added) { - if (!string.IsNullOrEmpty(order.State.LastMsg) && order.State.Created.ToDateTime().Date == DateTime.Today) { - (var security, var spec) = get_security(order.Def.SecurityKey); - if (security.HasValue) { - log_status(ActivityLogCategory.warn, security, order.State.LastMsg); + foreach (var o in update.added) { + if (!string.IsNullOrEmpty(o.order.State.LastMsg) && o.order.State.Created.ToDateTime().Date == DateTime.Today) { + if (o.sec.HasValue) { + log_status(ActivityLogCategory.warn, o.sec.Value, o.order.State.LastMsg); } } } foreach (var order in update.changes) { - var neworder = order.New; - var oldorder = order.Old; - (var security, var spec) = get_security(neworder.Def.SecurityKey); + var neworder = order.New.order; + var oldorder = order.Old.order; + var security = order.New.sec ?? order.Old.sec; // // description - var mod_desc = modified_description(oldorder, neworder, spec); + var mod_desc = modified_description(oldorder, neworder, security?.ContractSpec); if (!string.IsNullOrEmpty(mod_desc)) { log_status(ActivityLogCategory.modify, security, mod_desc); } @@ -171,28 +171,30 @@ namespace bts.spark { } internal void on_send_order(OrderDefinitionImmutable order, double price, uint size, uint? key, string error) { - (var security, var spec) = get_security(order.SecurityKey); + var security = try_get_security(order.SecurityKey); log_status(ActivityLogCategory.order, security, key.HasValue ? - format_order_success(key.Value, price, size, order.Side, spec) : - format_order_fail(price, size, order.Side, spec, error)); + format_order_success(key.Value, price, size, order.Side, security?.ContractSpec) : + format_order_fail(price, size, order.Side, security?.ContractSpec, error)); } internal void on_cancel_order(OrderRecordImmutable order, bool success, string error) { - (var security, var spec) = get_security(order.Def.SecurityKey); + var security = try_get_security(order.Def.SecurityKey); log_status(ActivityLogCategory.cancel, security, success ? - format_cancel_success(order, spec): - format_cancel_error(order, spec, error)); + format_cancel_success(order, security?.ContractSpec): + format_cancel_error(order, security?.ContractSpec, error)); } internal void update_order(OrderRecordImmutable order, double price, uint size) { - (var security, var spec) = get_security(order.Def.SecurityKey); + var security = try_get_security(order.Def.SecurityKey); + var spec = security?.ContractSpec; log_status(ActivityLogCategory.modify, security, $"({order.Key}) change from {order.State.Quantity}@{spec.format(order.State.Price)} to {size}@{spec.format(price)}"); } internal void update_order_failed(OrderRecordImmutable order, double price, uint size) { - (var security, var spec) = get_security(order.Def.SecurityKey); + var security = try_get_security(order.Def.SecurityKey); + var spec = security?.ContractSpec; log_status(ActivityLogCategory.modify, security, $"({order.Key}) failed to change from {order.State.Quantity}@{spec.format(order.State.Price)} to {size}@{spec.format(price)}"); diff --git a/src/client/spark/Orders/OrdersModel.cs b/src/client/spark/Orders/OrdersModel.cs index 910ff14e6..2eab7f64a 100644 --- a/src/client/spark/Orders/OrdersModel.cs +++ b/src/client/spark/Orders/OrdersModel.cs @@ -16,6 +16,8 @@ using System.Linq; using System.Reactive.Concurrency; using System.Reactive.Linq; using System.Threading.Tasks; +using OrderPair = System.ValueTuple; +using OrderUpdate = bts.api.core.TableUpdate<(bts.api.SecurityImmutable? sec, bts.api.orderman.OrderRecordImmutable order), uint>; namespace mbts.spark { [Export] @@ -50,22 +52,24 @@ namespace mbts.spark { .subscribei_orders() .Naggle(interval, scheduler) .PermissionFilter(admin_model.on_admin_data) - //.join(sstore) + .join(sstore, order => order.security_key()) .Table().Cache(); on_positions = _position_service.subscribei_position_data().Naggle(interval, scheduler).PermissionFilter(admin_model.on_admin_data).Table().Cache(); working_orders = on_orders.Scan( - ImmutableDictionary.Empty, + ImmutableDictionary.Empty, (workers, table) => _update_working_orders(workers, table)) .Replay(1).RefCount(); var order_record_tuple_source = new OrderRecordTupleSource(bus.subscribe(SparkTags.SecuritiesTag), on_orders); on_order_tuples = order_record_tuple_source.subscribe_incremental_update(); on_order_tuples_table = order_record_tuple_source.subscribe_incremental_table_update(); on_order_securities = order_record_tuple_source.subscribe_order_securities(); + on_fills = _service.subscribei_fills() .Naggle(interval, scheduler) .PermissionFilter(admin_model.on_admin_data) + //.join(sstore, f => ) .Table().Cache(); _activity = new OrderActivitySummarizer(bus, on_orders); on_fill_tuples = FillTupleSource.create_observable(bus.subscribe(SparkTags.SecuritiesTag), on_orders, on_fills); @@ -87,13 +91,13 @@ namespace mbts.spark { public OrderFactory factory => _factory; public SpreaderModel spreader => _spreader; public DynamicOrderHelper dynamic_helper => _dynamic_helper; - public IObservable> on_orders { get; } + public IObservable on_orders { get; } public IObservable> on_order_tuples { get; } public IObservable> on_order_tuples_table { get; } public IObservable> on_order_securities { get; } public IObservable> on_fills { get; } public IObservable> on_fill_tuples { get; } - public IObservable> working_orders { get; } + public IObservable> working_orders { get; } public IObservable> on_positions { get; } public IObservable> on_security_orders(uint security) => @@ -164,24 +168,25 @@ namespace mbts.spark { public void Dispose() => Disposable.dispose(ref _dispose); - static ImmutableDictionary _update_working_orders(ImmutableDictionary target, TableUpdate table) { + static ImmutableDictionary + _update_working_orders(ImmutableDictionary target, OrderUpdate table) { var clone = target.ToBuilder(); foreach (var o in table.removed) { - if (o.is_active()) { - clone.Remove(o.Key); + if (o.order.is_active()) { + clone.Remove(o.order.Key); } } foreach (var o in table.changed) { - if (o.is_active()) { - clone[o.Key] = o; + if (o.order.is_active()) { + clone[o.order.Key] = o; } else { - clone.Remove(o.Key); + clone.Remove(o.order.Key); } } foreach (var o in table.added) { - if (o.is_active()) { - clone.Add(o.Key, o); + if (o.order.is_active()) { + clone.Add(o.order.Key, o); } } return clone.ToImmutableDictionary(); @@ -196,8 +201,8 @@ namespace mbts.spark { public static async Task cancel_order(this OrdersModel orders_model, uint order_id) { var tu = await orders_model.on_orders.FirstAsync(); - if (tu.state.TryGetValue(order_id, out OrderRecordImmutable order)) { - return await orders_model.cancel_order(order); + if (tu.state.TryGetValue(order_id, out var o)) { + return await orders_model.cancel_order(o.order); } return false; } @@ -206,8 +211,8 @@ namespace mbts.spark { var tu = await model.on_orders.FirstAsync(); var orders = tu.state.Values - .Where(o => o.is_non_terminal() && pred(o)) - .Select(async o => { return (o, await model.cancel_order(o)); } ); + .Where(o => o.order.is_non_terminal() && pred(o.order)) + .Select(async o => { return (o.order, await model.cancel_order(o.order)); } ); return await Task.WhenAll(orders); } @@ -315,8 +320,8 @@ namespace mbts.spark { public static async Task order_from_key(this OrdersModel orders_model, uint order_key) { (var tu, var success) = await orders_model.on_orders.FirstAsyncTask(t => t.state.ContainsKey(order_key), TimeSpan.FromSeconds(3)); - if (success && tu.state.TryGetValue(order_key, out OrderRecordImmutable rec)) { - return rec; + if (success && tu.state.TryGetValue(order_key, out var rec)) { + return rec.order; } return null; } From 9bcd8bcbdbda1debdf1913b5e94dbfab5c814ef6 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 24 Jan 2019 15:38:39 -0500 Subject: [PATCH 31/85] Add tests for IObservable.Select methods --- src/client/spark_api/api/Update.cs | 38 ++++ src/client/spark_test/api/ApiUpdateTests.cs | 193 +++++++++++++++----- 2 files changed, 184 insertions(+), 47 deletions(-) diff --git a/src/client/spark_api/api/Update.cs b/src/client/spark_api/api/Update.cs index d5f1b04fd..b2585f8ad 100644 --- a/src/client/spark_api/api/Update.cs +++ b/src/client/spark_api/api/Update.cs @@ -64,6 +64,10 @@ namespace bts.api.core { public Change merge(Change newer) { return new Change(Old, newer.New); } + public void Deconstruct(out T Old, out T New) { + Old = this.Old; + New = this.New; + } public override string ToString() { return $"{{{Old}=>{New}}}"; } @@ -415,5 +419,39 @@ namespace bts.api.core { changed = diffs; removed = rems; } + + // This method does everything lazily so it's cheap until the collections are enumerated + public static IObservable> + Select(this IObservable> src, Func map, Func key) where TKey : IEquatable { + return src.Select(update => { + return new Update( + update.all.Select(map), + update.added.Select(map), + update.changes.Select(ch => new Change(map(ch.Old), map(ch.New))), + update.removed.Select(map), + key); + }); + } + + // Constructs a new dictionary so not as cheap as Select on the Update<> type + public static IObservable> + Select(this IObservable> src, Func map, Func key) where TKey : IEquatable { + var cache = ImmutableDictionary.Empty; + return src.Select(update => { + var added = update.added.Select(map); + var removed = update.removed.Select(map); + var changes = update.changes.Select(ch => new Change(map(ch.Old), map(ch.New))); + var builder = cache.ToBuilder(); + builder.RemoveRange(removed.Select(key)); + builder.AddRange(added.Select(t => new KeyValuePair(key(t), t))); + foreach (var (o,n) in changes) { + Debug.Assert(key(o).Equals(key(n))); + builder[key(o)] = n; + } + cache = builder.ToImmutable(); + + return new TableUpdate(cache, added, changes, removed); + }); + } } } \ No newline at end of file diff --git a/src/client/spark_test/api/ApiUpdateTests.cs b/src/client/spark_test/api/ApiUpdateTests.cs index 791d9a773..e1f6f08b7 100644 --- a/src/client/spark_test/api/ApiUpdateTests.cs +++ b/src/client/spark_test/api/ApiUpdateTests.cs @@ -1,12 +1,12 @@ using bts.api.core; -using System.Collections.Generic; using bts.test; using System; -using System.Reactive.Linq; -using System.Threading.Tasks; +using System.Collections.Generic; using System.Linq; using System.Reactive.Concurrency; +using System.Reactive.Linq; using System.Reactive.Subjects; +using System.Threading.Tasks; namespace mbts.api.Test { [TestGroup] @@ -44,7 +44,7 @@ namespace mbts.api.Test { static int id(Item i) => i.id; Item mkitem() { - return new Item(++_id, Guid.NewGuid().ToString().Substring(0,8)); + return new Item(++_id, Guid.NewGuid().ToString().Substring(0, 8)); } IObservable> validate_incremental(IObservable> source) { @@ -68,7 +68,7 @@ namespace mbts.api.Test { // This will mostly be collapsing to add() for (var count = 1; count < total; ++count) { var orig = raw.Take(count).reconstitute(id).ToEnumerable().Last(); - var nag = await + var nag = await validate_incremental( raw .Take(count) @@ -80,7 +80,7 @@ namespace mbts.api.Test { // Do it again with TableUpdate for (var count = 1; count < total; ++count) { var orig = raw.Take(count).reconstitute(id).Table().ToEnumerable().Last(); - var nag = await + var nag = await validate_incremental( raw .Take(count) @@ -94,35 +94,32 @@ namespace mbts.api.Test { // collapse rolling window var update_list = raw.reconstitute(id).ToEnumerable().ToList(); var window = 10; - for (int x = 0; x+window < total; x += 5) { + for (int x = 0; x + window < total; x += 5) { var start = update_list[x]; var end = update_list[x + window]; - var super_update = update_list.Skip(x).Take(window+1).Aggregate((u1, u2) => u1.merge(u2)); + var super_update = update_list.Skip(x).Take(window + 1).Aggregate((u1, u2) => u1.merge(u2)); start.merge(super_update); Assert.SequenceEqual(end.all, super_update.all); } } - void validate_filter(IObservable> raw, IObservable> filtered, Func pred) - { + void validate_filter(IObservable> raw, IObservable> filtered, Func pred) { var total = raw.ToEnumerable().Count(); // Validate that update is right at each point - for (var count = 1; count < total; ++count) - { - var orig = raw.Take(count).reconstitute(id).ToEnumerable().Last(); + for (var count = 1; count < total; ++count) { + var orig = raw.Take(count).reconstitute(id).ToEnumerable().Last(); var next = filtered.Take(count).ToEnumerable().Last(); Assert.That(next.all.All(pred)); Assert.That(next.added.All(pred)); Assert.That(next.removed.All(pred)); - Assert.That(next.changed.All(pred)); + Assert.That(next.changed.All(pred)); } // Do it again with TableUpdate - for (var count = 1; count < total; ++count) - { + for (var count = 1; count < total; ++count) { var orig = raw.Take(count).reconstitute(id).Table().ToEnumerable().Last(); var next = filtered.Take(count).ToEnumerable().Last(); Assert.That(next.added.All(pred)); @@ -132,36 +129,32 @@ namespace mbts.api.Test { } - void validate_filter_sub(IObservable> raw, IObservable> filtered, BehaviorSubject> pred) - { - raw.reconstitute(x => x.id).CombineLatest(filtered, (r, f) => Tuple.Create(r, f)).Subscribe(tup => - { + void validate_filter_sub(IObservable> raw, IObservable> filtered, BehaviorSubject> pred) { + raw.reconstitute(x => x.id).CombineLatest(filtered, (r, f) => Tuple.Create(r, f)).Subscribe(tup => { Assert.That(tup.Item2.all.All(pred.Value)); Assert.That(tup.Item2.added.All(pred.Value)); - Assert.That(!tup.Item2.removed.Any(pred.Value)); + Assert.That(!tup.Item2.removed.Any(pred.Value)); Assert.That(tup.Item2.changed.All(pred.Value)); Dictionary rawd = tup.Item1.added.ToDictionary(x => x.id); Dictionary fd = tup.Item2.added.ToDictionary(x => x.id); - foreach (var kv in rawd) - { - if(pred.Value(kv.Value)) + foreach (var kv in rawd) { + if (pred.Value(kv.Value)) { Assert.That(fd.ContainsKey(kv.Key)); - else + } + else { Assert.That(!fd.ContainsKey(kv.Key)); + } } }); } [Test] - public void test_filter() - { - var raw = Observable.Create>(observer => - { + public void test_filter() { + var raw = Observable.Create>(observer => { RawUpdate rup = new RawUpdate(); - for (int x = 0; x < 50; ++x) - { + for (int x = 0; x < 50; ++x) { rup.Clear(); var item = _item_pool[x]; rup.added_items.Add(item); @@ -169,39 +162,35 @@ namespace mbts.api.Test { observer.OnNext(rup); } var rand = new Random(); - for (int r = 0; r < rand.Next(4, 30); r += rand.Next(1, 2)) - { + for (int r = 0; r < rand.Next(4, 30); r += rand.Next(1, 2)) { rup.Clear(); rup.removed_keys.Add(_item_pool[r].id); observer.OnNext(rup); } rup.Clear(); - for (int c = 31; c < rand.Next(32, 50); c += rand.Next(1, 2)) - { + for (int c = 31; c < rand.Next(32, 50); c += rand.Next(1, 2)) { var ch = new Item(_item_pool[c].id, _item_pool[c].name + "_change"); rup.changed_items.Add(ch); - rup.changed_keys.Add(ch.id); + rup.changed_keys.Add(ch.id); } observer.OnNext(rup); - observer.OnCompleted(); + observer.OnCompleted(); return Disposable.empty; }); Func evens = (x => x.id % 2 == 0); Func odds = (x => x.id % 2 == 1); - var predeven = Observable.Create>(observer => - { - observer.OnNext(evens); + var predeven = Observable.Create>(observer => { + observer.OnNext(evens); observer.OnCompleted(); return Disposable.empty; }); - var filtered = raw.reconstitute(x=> x.id).Filter(predeven); + var filtered = raw.reconstitute(x => x.id).Filter(predeven); validate_filter(raw, filtered, evens); - var predodd = Observable.Create>(observer => - { + var predodd = Observable.Create>(observer => { observer.OnNext(odds); observer.OnCompleted(); return Disposable.empty; @@ -212,13 +201,13 @@ namespace mbts.api.Test { - var sub= new BehaviorSubject>(evens); + var sub = new BehaviorSubject>(evens); filtered = raw.reconstitute(x => x.id).Filter(sub); validate_filter_sub(raw, filtered, sub); sub.OnNext(odds); sub.OnCompleted(); - } + } [Test] public async Task test_add() { @@ -249,7 +238,7 @@ namespace mbts.api.Test { rup.added_keys.AddRange(_item_pool.Take(20).Select(id)); observer.OnNext(rup); - for (int x = 0; x < 50; x+=2) { + for (int x = 0; x < 50; x += 2) { rup.Clear(); var item = _item_pool[x]; rup.removed_keys.Add(item.id); @@ -347,11 +336,121 @@ namespace mbts.api.Test { public void test_merge_begin_end_state() { int key(string s) => s[0]; var empty = new string[0]; - var a2 = new Update(new[] { "a2" }, empty, new[] { new Change("a1", "a2") }, empty, key); + var a2 = new Update(new[] { "a2" }, empty, new[] { new Change("a1", "a2") }, empty, key); var a3 = new Update(new[] { "a3" }, empty, new[] { new Change("a2", "a3") }, empty, key); var merged = a2.merge(a3); Assert.SequenceEqual(new[] { new Change("a1", "a3") }, merged.changes); } + T id(T t) => t; + + [Test,Ignore("SPARK-1988")] + public void test_reconstitute_twice() { + var empty = new int[0]; + RawUpdate create(int val) => new RawUpdate() { added_items = { val }, added_keys = { val } }; + var input = + Enumerable.Range(1, 10) + .Select(create) + .ToObservable() + .reconstitute(id); + + var expected = Enumerable.Range(1, 10).Select(i => Enumerable.Range(1, i)).SelectMany(id); + var a = input.ToEnumerable().SelectMany(u => u.all).ToList(); + var b = input.ToEnumerable().SelectMany(u => u.all).ToList(); + Assert.SequenceEqual(expected, a); + Assert.SequenceEqual(expected, b); + } + + [Test] + public void test_select_update() { + var empty = new int[0]; + RawUpdate create(int val) => new RawUpdate() { added_items = { val }, added_keys = { val } }; + int square(int i) => i * i; + var input = + Enumerable.Range(1, 10) + .Select(create) // IEnumerable + .ToObservable() // IObservable + .reconstitute(id); // IObservable + + var output = + input + .Select(square, id) // function under test + .ToEnumerable() + .ToList(); //REMOVE THIS WHEN SPARK-1988 is fixed. + + var expected = Enumerable.Range(1, 10).Select(square); + Assert.SequenceEqual(expected, output.SelectMany(u => u.added)); + + var expected_all = Enumerable.Range(1, 10).Select(i => Enumerable.Range(1, i).Select(square)).SelectMany(id); + var accumulated = output.SelectMany(u => u.all).ToList(); + Assert.SequenceEqual(expected_all, accumulated); + } + + [Test] + public void test_select_tableupdate() { + var empty = new int[0]; + RawUpdate create(int val) => new RawUpdate() { added_items = { val }, added_keys = { val } }; + int square(int i) => i * i; + var input = + Enumerable.Range(1, 10) + .Select(create) // IEnumerable + .ToObservable() // IObservable + .reconstitute(id); // IObservable + + var output = + input + .Table() + .Select(square, id) // function under test + .ToEnumerable() + .ToList(); //REMOVE THIS WHEN SPARK-1988 is fixed. + + var expected = Enumerable.Range(1, 10).Select(square); + Assert.SequenceEqual(expected, output.SelectMany(u => u.added)); + + var expected_all = Enumerable.Range(1, 10).Select(i => Enumerable.Range(1, i).Select(square)).SelectMany(id); + var accumulated = output.SelectMany(u => u.state.Values).ToList(); + Assert.SequenceEqual(expected_all, accumulated); + + } + + //public class Box { + // public int value; + // public override string ToString() { + // return value.ToString(); + // } + //} + + //[Test] + //public void test_scan() { + // var obs = + // Enumerable.Range(1, 5) + // .ToObservable() + // .Scan( + // new Box { value = 0 }, + // (box, i) => { + // box.value += i; + // return box; + // }); + // var a = obs.Select(box => box.value).ToEnumerable(); + // var b = obs.Select(box => box.value).ToEnumerable(); + // Assert.SequenceEqual(a, b); + //} + + //[Test] + //public void test_scan2() { + // var obs = + // Enumerable.Range(1, 5) + // .ToObservable() + // .Scan>( + // () => new Box { value = 0 }, + // (getbox, i) => { + // var box = getbox(); + // box.value += i; + // return () => box; + // }); + // var a = obs.Select(box => box().value).ToEnumerable().ToList(); + // var b = obs.Select(box => box().value).ToEnumerable().ToList(); + // Assert.SequenceEqual(a, b); + //} } } From b08bc0eb8b5f56c97deade9aae9f91bec8d4188e Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 25 Jan 2019 13:40:56 -0500 Subject: [PATCH 32/85] SPARK-1972 Join Securities to Orders in OrdersModel --- .../spark/Daemons/DeleteSyntheticDaemon.cs | 2 +- src/client/spark/Daemons/OrderTicketDaemon.cs | 20 +-- .../Trading/OrderBook/OrderBookColumns.cs | 86 +++++----- .../OrderBook/OrderBookRowCollection.cs | 24 +-- .../Trading/OrderBook/OrderBookRowVM.cs | 19 +-- .../Modules/Trading/OrderBook/OrderBookVM.cs | 17 +- .../Trading/OrderBook/OrderBookWindow.cs | 8 +- .../OrderBook/OrderBookWindowFactory.cs | 4 +- .../Notifications/INotificationService.cs | 28 ++-- .../INotificationServiceDataCache.cs | 33 ++-- .../spark/Notifications/ToastNotification.cs | 8 +- .../spark/Orders/ExistingOrderTicketVM.cs | 2 +- src/client/spark/Orders/FillSecurity.cs | 25 +++ src/client/spark/Orders/OrderRecordTuple.cs | 47 ------ .../spark/Orders/OrderRecordTupleSource.cs | 147 ------------------ src/client/spark/Orders/OrderSecurity.cs | 25 +++ src/client/spark/Orders/OrdersModel.cs | 29 ++-- .../Events/OrderCanceledEventPredicate.cs | 19 +-- .../Events/OrderCompletedEventPredicate.cs | 14 +- .../Data/Events/OrderEventPredicateHelper.cs | 24 +-- .../Data/Events/OrderFilledEventPredicate.cs | 47 +++--- .../Data/OrderRecordTuplePredicate.cs | 24 +-- src/client/spark/Security/SecurityStore.cs | 3 +- .../core/GlobalExtensions/GlobalExtensions.cs | 1 + .../core/Utility/TableUpdateTransform.cs | 39 ++++- .../DIYGrid/Settings/GridSettingsManager.cs | 3 +- .../spark/view/Spark/Core/SparkUtils.cs | 11 +- .../spark/view/Spark/Ladder/LadderVM.cs | 17 +- src/client/spark_api/api/Update.cs | 14 +- .../Extensions/DictionaryExtensions.cs | 23 +++ .../spark_core/utils/FilteredDictionary.cs | 86 ++++++++++ .../spark_core/utils/MappedDictionary.cs | 64 ++++++++ 32 files changed, 508 insertions(+), 405 deletions(-) create mode 100644 src/client/spark/Orders/FillSecurity.cs delete mode 100644 src/client/spark/Orders/OrderRecordTuple.cs delete mode 100644 src/client/spark/Orders/OrderRecordTupleSource.cs create mode 100644 src/client/spark/Orders/OrderSecurity.cs create mode 100644 src/client/spark_core/utils/FilteredDictionary.cs create mode 100644 src/client/spark_core/utils/MappedDictionary.cs diff --git a/src/client/spark/Daemons/DeleteSyntheticDaemon.cs b/src/client/spark/Daemons/DeleteSyntheticDaemon.cs index 15bc8d00c..52819f4e3 100644 --- a/src/client/spark/Daemons/DeleteSyntheticDaemon.cs +++ b/src/client/spark/Daemons/DeleteSyntheticDaemon.cs @@ -32,7 +32,7 @@ namespace bts { if (!orders_success) { _error_reporter.report_error($"Unable to delete synthetic security ({id}): unable to determine whether or not security ({id}) has working orders (deletion is not permitted while there are working orders)."); Log.Ui.warn($"Ignoring request to delete synthetic security ({id}) because the orders lookup timed out (must confirm no working orders)."); - } else if (orders.state.Values.Any(o => o.Def.SecurityKey == id && o.is_non_terminal())) { + } else if (orders.state.Values.Any(o => o.order.Def.SecurityKey == id && o.order.is_non_terminal())) { Log.Ui.warn($"Ignoring request to delete synthetic security ({id}) because non-terminal-orders were detected."); } else { ExecutionStrategyImmutable? strategy = await _orders_model.spreader.lookup_strategy_async(id); diff --git a/src/client/spark/Daemons/OrderTicketDaemon.cs b/src/client/spark/Daemons/OrderTicketDaemon.cs index f0dfc0583..5b8ac95c3 100644 --- a/src/client/spark/Daemons/OrderTicketDaemon.cs +++ b/src/client/spark/Daemons/OrderTicketDaemon.cs @@ -19,7 +19,7 @@ namespace bts.shell { readonly OrdersModel _model; readonly ErrorReporter _error_reporter; IReadOnlyDictionary _md_map; - TableUpdate _orders; + TableUpdate<(SecurityImmutable? sec, OrderRecordImmutable order), uint> _orders; IDisposable _dispose; [ImportingConstructor] @@ -57,14 +57,16 @@ namespace bts.shell { } void _show_modify_order(uint order_key) { - if (_orders.state.TryGetValue(order_key, out OrderRecordImmutable ori) && _md_map.TryGetValue(ori.Def.SecurityKey, out SecurityMD smd) - && (ori.HasDef && ori.Def.HasCustomDef && !(ori.Def.CustomDef.HasStop || ori.Def.CustomDef.HasStopLimit))) - _show_order_window(new OrderModificationDialog { - DataContext = new ExistingOrderTicketVM(_model, - Observable.Return(new OrderRecordTuple(ori, smd)), - Observable.Return(_model.factory.default_ladder_settings(smd.security).calculator_buttons) - ) - }); + if (_orders.state.TryGetValue(order_key, out var ori) && ori.sec.HasValue + && (ori.order.HasDef && ori.order.Def.HasCustomDef && !(ori.order.Def.CustomDef.HasStop || ori.order.Def.CustomDef.HasStopLimit))) { + var md = _md_map.TryGetValue(ori.order.security_key()); + _show_order_window(new OrderModificationDialog { + DataContext = new ExistingOrderTicketVM(_model, + Observable.Return((md,ori.order)), + Observable.Return(_model.factory.default_ladder_settings(ori.sec.Value).calculator_buttons) + ) + }); + } } } } \ No newline at end of file diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookColumns.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookColumns.cs index acb550f67..209416179 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookColumns.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookColumns.cs @@ -30,7 +30,7 @@ namespace mbts.spark { interface ICVOrderBookColumn : IKeyedColumn, IGridColumn { string sort_member_path { get; } ListSortDirection? SortDirection { get; set; } - IDataGridColumnFilter filter { get; } + IDataGridColumnFilter filter { get; } IComparer comparer { get; } } @@ -65,9 +65,9 @@ namespace mbts.spark { public OrderBookColumnCollection(IBus bus, OrdersModel model, IObservable<(IEnumerable added, IEnumerable removed)> order_securities) { // security Add(security_column = new SecurityColumn( - DiscreteValuesFilterVM.Create( + DiscreteValuesFilterVM.Create( order_securities, - ort => ort.security_md.HasValue ? ort.security_md.Value.security : SecurityDataHelper.Defaults.SecurityImmutable, + ort => ort.security ?? SecurityDataHelper.Defaults.SecurityImmutable, SecurityCriteria.from_securities, SecurityCriteria.get_to_securities(bus), nameof(OrderBookRowVM.security), @@ -81,9 +81,9 @@ namespace mbts.spark { nameof(OrderBookRowVM.symbol), r => r.symbol, DefaultColumnWidths.Symbol, - DiscreteValuesFilterVM.Create( + DiscreteValuesFilterVM.Create( mbts.co2.UserSymbol.get_all().Select(u => u.symbol).Distinct().ToList(), - ort => ort.security_md.HasValue ? ort.security_md.Value.security.Symbol : string.Empty, + ort => ort.security?.Symbol ?? string.Empty, nameof(OrderBookRowVM.symbol) ), DataGridFilterKnownTemplates.DiscreteValues, @@ -95,7 +95,7 @@ namespace mbts.spark { nameof(OrderBookRowVM.time_created), r => r.time_created_text, DefaultColumnWidths.Time, - new TimeFilterVM( + new TimeFilterVM( ort => ort.order.created(), nameof(OrderBookRowVM.time_created)), DataGridFilterKnownTemplates.TimeFilter, @@ -107,7 +107,7 @@ namespace mbts.spark { nameof(OrderBookRowVM.last_change), r => r.last_change_text, DefaultColumnWidths.Time, - new TimeFilterVM( + new TimeFilterVM( ort => ort.order.last_change(), nameof(OrderBookRowVM.last_change)), DataGridFilterKnownTemplates.TimeFilter, @@ -119,7 +119,7 @@ namespace mbts.spark { nameof(OrderBookRowVM.user), r => r.user, DefaultColumnWidths.User, - DiscreteValuesFilterVM.Create( + DiscreteValuesFilterVM.Create( model.factory.subscribe_users().Select(d => d.Values), ort => ort.order.user(), nameof(OrderBookRowVM.user)), @@ -132,7 +132,7 @@ namespace mbts.spark { nameof(OrderBookRowVM.account), r => r.account, DefaultColumnWidths.Account, - DiscreteValuesFilterVM.Create( + DiscreteValuesFilterVM.Create( model.factory.subscribe_accounts().Select(d => d.Values), ort => ort.order.account(), vals => vals.ToArray(), @@ -147,9 +147,9 @@ namespace mbts.spark { nameof(OrderBookRowVM.exchange), r => r.exchange.ToString(), DefaultColumnWidths.Exchange, - DiscreteValuesFilterVM.Create( + DiscreteValuesFilterVM.Create( EnumHelper.get_values_except(Venue.Unset).ToList(), - ort => ort.security_md.HasValue ? ort.security_md.Value.security.Exchange : Venue.Unset, + ort => ort.security.exchange_or_default(), nameof(OrderBookRowVM.exchange)), DataGridFilterKnownTemplates.EnumValues, _wrap_comparer(r => r.exchange))); @@ -160,7 +160,7 @@ namespace mbts.spark { nameof(OrderBookRowVM.time_in_force), r => r.time_in_force.ToString(), DefaultColumnWidths.TimeInForce, - DiscreteValuesFilterVM.Create( + DiscreteValuesFilterVM.Create( EnumHelper.get_values().ToList(), ort => ort.order.time_in_force(), nameof(OrderBookRowVM.time_in_force)), @@ -168,7 +168,7 @@ namespace mbts.spark { _wrap_comparer(r => r.time_in_force))); // order type Add(order_type_column = new OrderBookOrderTypeColumn( - DiscreteValuesFilterVM.Create( + DiscreteValuesFilterVM.Create( utilities.OrderFactory.all_order_types, ort => ort.order.order_type(), nameof(OrderBookRowVM.order_type)))); @@ -179,7 +179,7 @@ namespace mbts.spark { nameof(OrderBookRowVM.order_sub_type), r => r.order_sub_type_text, 80, - DiscreteValuesFilterVM.Create( + DiscreteValuesFilterVM.Create( EnumHelper.get_values().ToList(), ort => ort.order.sub_type(), nameof(OrderBookRowVM.order_sub_type)), @@ -187,16 +187,16 @@ namespace mbts.spark { _wrap_comparer(r => r.order_sub_type))); // side Add(side_column = new SideColumn( - DiscreteValuesFilterVM.Create( + DiscreteValuesFilterVM.Create( new[] { Side.Buy, Side.Sell }, ort=> ort.order.side(), nameof(OrderBookRowVM.side)))); // quantity Add(qty_column = new SizeColumn( - new UIntMinMax(ort => ort.order.quantity(), nameof(OrderBookRowVM.quantity)))); + new UIntMinMax(ort => ort.order.quantity(), nameof(OrderBookRowVM.quantity)))); // filled quantity Add(filled_qty_column = new FilledColumn( - new UIntMinMax(ort => ort.order.filled(), nameof(OrderBookRowVM.filled_quantity)))); + new UIntMinMax(ort => ort.order.filled(), nameof(OrderBookRowVM.filled_quantity)))); // price Add(price_column = new SimpleTextColumn( Columns.Price, @@ -204,7 +204,7 @@ namespace mbts.spark { nameof(OrderBookRowVM.price), r => r.price_text, DefaultColumnWidths.Price, - new DoubleMinMax( + new DoubleMinMax( ort => ort.order.price(), nameof(OrderBookRowVM.price)), DataGridFilterKnownTemplates.DoubleMinMax, @@ -216,7 +216,7 @@ namespace mbts.spark { nameof(OrderBookRowVM.avg_price), r => r.avg_price_text, DefaultColumnWidths.Price, - new DoubleMinMax( + new DoubleMinMax( ort => ort.order.avg_price(), nameof(OrderBookRowVM.avg_price)), DataGridFilterKnownTemplates.DoubleMinMax, @@ -228,7 +228,7 @@ namespace mbts.spark { nameof(OrderBookRowVM.last_price), r => r.last_price_text, DefaultColumnWidths.Price, - new DoubleMinMax( + new DoubleMinMax( ort => ort.order.last_price(), nameof(OrderBookRowVM.last_price)), DataGridFilterKnownTemplates.DoubleMinMax, @@ -240,7 +240,7 @@ namespace mbts.spark { nameof(OrderBookRowVM.status), r => r.status.ToString(), 80, - DiscreteValuesFilterVM.Create( + DiscreteValuesFilterVM.Create( EnumHelper.get_values().ToList(), ort => ort.order.status(), nameof(OrderBookRowVM.status)), @@ -248,7 +248,7 @@ namespace mbts.spark { _wrap_comparer(r => r.status))); // last message Add(last_message_column = new OrderBookLastMessageColumn( - new RegexFilterVM( + new RegexFilterVM( ort => ort.order.last_msg(), nameof(OrderBookRowVM.last_msg)))); // security key @@ -258,7 +258,7 @@ namespace mbts.spark { nameof(OrderBookRowVM.security_key), r => r.security_key.ToString(), DefaultColumnWidths.SecurityKey, - new RegexFilterVM( + new RegexFilterVM( ort => ort.order.security_key().ToString(), nameof(OrderBookRowVM.security_key)), DataGridFilterKnownTemplates.Regex, @@ -270,7 +270,7 @@ namespace mbts.spark { nameof(OrderBookRowVM.order_key), r => r.order_key.ToString(), DefaultColumnWidths.OrderKey, - new RegexFilterVM( + new RegexFilterVM( ort => ort.order.Key.ToString(), nameof(OrderBookRowVM.order_key)), DataGridFilterKnownTemplates.Regex, @@ -282,7 +282,7 @@ namespace mbts.spark { nameof(OrderBookRowVM.parent_order_key), r => r.parent_order_key.ToString(), DefaultColumnWidths.ParentKey, - new RegexFilterVM( + new RegexFilterVM( ort => ort.order.ParentKey.ToString(), nameof(OrderBookRowVM.parent_order_key)), DataGridFilterKnownTemplates.Regex, @@ -378,9 +378,9 @@ namespace mbts.spark { } } - public Func get_filter() { + public Func get_filter() { var active_filters = this.OfType().Where(c => c.filter.active.GetValueOrDefault()).Select(c => c.filter).ToArray(); - return active_filters.Length == 0 ? null : new Func(r => active_filters.All(f => f.filter(r))); + return active_filters.Length == 0 ? null : new Func(r => active_filters.All(f => f.filter(r))); } public OrderBookSortingData get_sorting_data() { @@ -470,7 +470,7 @@ namespace mbts.spark { GlyphContext _root_glyph_context, _child_glyph_context; protected OrderbookColumnBase(Columns col_key, string header, Func get_text, string sort_prop, double width, - IDataGridColumnPersistentFilter filter, string filter_template, IComparer comparer) : base(col_key) { + IDataGridColumnPersistentFilter filter, string filter_template, IComparer comparer) : base(col_key) { id = key.ToString(); Width = width; Header = header; @@ -488,10 +488,10 @@ namespace mbts.spark { } } - public IDataGridColumnPersistentFilter filter { get; } + public IDataGridColumnPersistentFilter filter { get; } public IComparer comparer { get; } - IDataGridColumnFilter ICVOrderBookColumn.filter => filter; + IDataGridColumnFilter ICVOrderBookColumn.filter => filter; IDataGridColumnFilter IFilterableColumn.filter => filter; string IFilterableColumn.filter_template => _filter_template; ListSortDirection? ICVOrderBookColumn.SortDirection { get => _cached_sort_direction; set => SortDirection = value; } @@ -505,7 +505,7 @@ namespace mbts.spark { public class SimpleTextColumn : OrderbookColumnBase, IRenderColumn { public SimpleTextColumn(Columns key, string header, string sort_prop, Func get_text, double width, - IDataGridColumnPersistentFilter filter, string filter_template, IComparer comparer) + IDataGridColumnPersistentFilter filter, string filter_template, IComparer comparer) : base(key, header, get_text, sort_prop, width, filter, filter_template, comparer) { } public void draw(OrderBookRowVM row, DrawingContext dc, ColumnGeometry position, double row_height) => @@ -515,7 +515,7 @@ namespace mbts.spark { #region Explicit Column Defs public class SideColumn : OrderbookColumnBase, IRenderColumn { - public SideColumn(IDataGridColumnPersistentFilter filter) + public SideColumn(IDataGridColumnPersistentFilter filter) : base(Columns.Side, "B/S", r => r.side.ToString(), nameof(OrderBookRowVM.side), DefaultColumnWidths.Side, filter, DataGridFilterKnownTemplates.EnumValues, _wrap_comparer(r => r.side)) { } public void draw(OrderBookRowVM row, DrawingContext dc, ColumnGeometry position, double row_height) => @@ -524,7 +524,7 @@ namespace mbts.spark { } public class SizeColumn : OrderbookColumnBase, IRenderColumn { - public SizeColumn(IDataGridColumnPersistentFilter filter) + public SizeColumn(IDataGridColumnPersistentFilter filter) : base(Columns.Size, "Qty", r => r.quantity.ToString(), nameof(OrderBookRowVM.quantity), DefaultColumnWidths.Size, filter, DataGridFilterKnownTemplates.UIntMinMax, _wrap_comparer(r => r.quantity)) { sort_member_path = nameof(OrderBookRowVM.quantity); } @@ -535,7 +535,7 @@ namespace mbts.spark { } public class FilledColumn : OrderbookColumnBase, IRenderColumn { - public FilledColumn(IDataGridColumnPersistentFilter filter) + public FilledColumn(IDataGridColumnPersistentFilter filter) : base(Columns.Filled, "Filled Qty", r => r.filled_quantity.ToString(), nameof(OrderBookRowVM.filled_quantity), DefaultColumnWidths.Size, filter, DataGridFilterKnownTemplates.UIntMinMax, _wrap_comparer(r => r.filled_quantity)) { sort_member_path = nameof(OrderBookRowVM.filled_quantity); } @@ -547,7 +547,7 @@ namespace mbts.spark { public class OrderBookOrderTypeColumn : OrderTypeColumn, ICVOrderBookColumn { ListSortDirection? _cached_sort_direction; - public OrderBookOrderTypeColumn(IDataGridColumnPersistentFilter filter) + public OrderBookOrderTypeColumn(IDataGridColumnPersistentFilter filter) : base(Columns.OrderType, new Binding(nameof(OrderBookRowVM.order_type)), new Binding(nameof(OrderBookRowVM.order_metadata)), filter) { id = key.ToString(); this.filter = filter; @@ -556,10 +556,10 @@ namespace mbts.spark { _cached_sort_direction = SortDirection; } - public new IDataGridColumnPersistentFilter filter { get; } + public new IDataGridColumnPersistentFilter filter { get; } public IComparer comparer => _wrap_comparer(r => r.order_type); - IDataGridColumnFilter ICVOrderBookColumn.filter => filter; + IDataGridColumnFilter ICVOrderBookColumn.filter => filter; ListSortDirection? ICVOrderBookColumn.SortDirection { get => _cached_sort_direction; set => SortDirection = value; } protected override void OnSortDirectionChanged(ListSortDirection? new_value) { @@ -571,7 +571,7 @@ namespace mbts.spark { public class OrderBookLastMessageColumn : TextColumn, ICVOrderBookColumn { ListSortDirection? _cached_sort_direction; - public OrderBookLastMessageColumn(IDataGridColumnPersistentFilter filter) + public OrderBookLastMessageColumn(IDataGridColumnPersistentFilter filter) : base(Columns.LastMessage, new Binding(nameof(OrderBookRowVM.last_msg)), filter, DataGridFilterKnownTemplates.Regex) { id = key.ToString(); this.filter = filter; @@ -581,10 +581,10 @@ namespace mbts.spark { tooltip_binding = new Binding(nameof(OrderBookRowVM.last_msg)); } - public new IDataGridColumnPersistentFilter filter { get; } + public new IDataGridColumnPersistentFilter filter { get; } public IComparer comparer => _wrap_comparer(r => r.last_msg); - IDataGridColumnFilter ICVOrderBookColumn.filter => filter; + IDataGridColumnFilter ICVOrderBookColumn.filter => filter; ListSortDirection? ICVOrderBookColumn.SortDirection { get => _cached_sort_direction; set => SortDirection = value; } protected override void OnSortDirectionChanged(ListSortDirection? new_value) { @@ -600,7 +600,7 @@ namespace mbts.spark { ListSortDirection? _cached_sort_direction; - public SecurityColumn(IDataGridColumnPersistentFilter filter) : base(Columns.Security) { + public SecurityColumn(IDataGridColumnPersistentFilter filter) : base(Columns.Security) { id = key.ToString(); this.filter = filter; Header = DefaultColumnHeaders.Security; @@ -609,10 +609,10 @@ namespace mbts.spark { _cached_sort_direction = SortDirection; } - public IDataGridColumnPersistentFilter filter { get; } + public IDataGridColumnPersistentFilter filter { get; } public IComparer comparer => _wrap_comparer(r => r.security, SecurityHelper.Comparer.InstanceImmutable.to_nullable()); - IDataGridColumnFilter ICVOrderBookColumn.filter => filter; + IDataGridColumnFilter ICVOrderBookColumn.filter => filter; IDataGridColumnFilter IFilterableColumn.filter => filter; string IFilterableColumn.filter_template => DataGridFilterKnownTemplates.DiscreteValues; ListSortDirection? ICVOrderBookColumn.SortDirection { get => _cached_sort_direction; set => SortDirection = value; } diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookRowCollection.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookRowCollection.cs index b572530fc..f65aeb4be 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookRowCollection.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookRowCollection.cs @@ -1,5 +1,7 @@ using bts; +using bts.api; using bts.api.core; +using bts.api.orderman; using bts.collections; using bts.utils; using mbts.utils; @@ -13,26 +15,26 @@ using System.Reactive.Linq; namespace mbts.spark { class OrderBookRowCollection : ObservableCollection { static readonly IEqualityComparer> _node_equality_comparer = EqualityComparer.Create, uint>(n => n.value.order_key); - readonly Func _row_factory; + readonly Func _row_factory; /// Critical: Any and all row mutations must be executed through . /// Mutations can change the sort-position of a row, and will be broken if its elements aren't properly sorted (it relies heavily on binary-search logic). SortedTree _tree; IComparer _comparer; - Func _filter; + Func _filter; bool _is_refresh_pending; int _deferment_count = 0; /// When processing changed-items, the method is used to update the changed row. It is not expected to fail (return false), /// but if it does, we don't want to overwhelm the logs with a bunch of useless noise in case it starts happening repeatedly, hence the int fields below. - /// , , . + /// , , . const int _max_replace_misses_logged = 10; int _n_replace_misses_logged; - public OrderBookRowCollection(Func row_factory, IComparer comparer) { + public OrderBookRowCollection(Func row_factory, IComparer comparer) { _row_factory = row_factory; _tree = _mk_tree(_comparer = comparer); } - public Func filter { + public Func filter { get => _filter; set { _filter = value; @@ -144,24 +146,24 @@ namespace mbts.spark { }); } - public void update(IIncrementalTableUpdateTransform table) { + public void update(TableUpdate<(SecurityImmutable? sec,OrderRecordImmutable order),uint> table) { IEnumerable> impacted_branch_roots; int n_removed = table.removed.Count(); if (n_removed == total_row_count) { // full refresh (most likely this is the first update or a server restart) - _tree = _mk_tree(_comparer, table.added.Select(_row_factory)); + _tree = _mk_tree(_comparer, table.added.Select(x => _row_factory(x))); impacted_branch_roots = _tree.RootNodes; } else { - foreach (OrderRecordTuple ort in table.removed) + foreach (OrderSecurity ort in table.removed) _tree.Remove(ort.order.Key); - foreach (Change change in table.changes) { + foreach (var change in table.changes) { if (!_tree.TryReplace(change.New.order.Key, r => { r.update(change.New); return r; })) { // uh oh, replace failed! Let's log this, but not a million times (this seems like the kind of error that, if it happens once, could happen a lot). if (_n_replace_misses_logged++ < _max_replace_misses_logged) - Log.Ui.error($"{nameof(OrderBookRowCollection)} failed to process Changed {nameof(OrderRecordTuple)} in update (old:{change.Old.key}, new:{change.New.key}). This is the '{_n_replace_misses_logged}' time this has happened in this OrderBook."); + Log.Ui.error($"{nameof(OrderBookRowCollection)} failed to process Changed {nameof(OrderSecurity)} in update (old:{change.Old.order.Key}, new:{change.New.order.Key}). This is the '{_n_replace_misses_logged}' time this has happened in this OrderBook."); } } - foreach (OrderRecordTuple ort in table.added) + foreach (OrderSecurity ort in table.added) _tree.Insert(_row_factory(ort)); impacted_branch_roots = table.added.Concat(table.changed).Select(ort => ort.order.Key).Select(_tree.GetRootNode) diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs index 5a5213efd..dc41be7ac 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs @@ -16,9 +16,10 @@ namespace mbts.spark { IRaiseCanExecuteChangedCommand _show_order_modify_cmd, _toggle_expanded_cmd; bool _is_parent, _is_expanded; int _depth_level; - SecurityMD? _security_md => order_tuple.security_md; + SecurityImmutable? _security => order_tuple.security; + - public OrderBookRowVM(OrderBookVM owner, OrderRecordTuple order_tuple) { + public OrderBookRowVM(OrderBookVM owner, OrderSecurity order_tuple) { _owner = owner; this.order_tuple = order_tuple; is_visible = !is_child; @@ -27,16 +28,16 @@ namespace mbts.spark { public event Action invalidated; // Core - public OrderRecordTuple order_tuple { get; private set; } + public OrderSecurity order_tuple { get; private set; } public OrderRecordImmutable order => order_tuple.order; public OrderMetadata order_metadata => _owner.get_metadata(this); public uint order_key => order.Key; public uint parent_order_key => order.ParentKey; public uint security_key => order.security_key(); - public SecurityImmutable? security => _security_md.security(); + public SecurityImmutable? security => _security; public string security_text => security.display_name(); - public Venue exchange => _security_md.exchange(); - public string symbol => _security_md.symbol(); + public Venue exchange => _security.exchange_or_default(); + public string symbol => _security.symbol_or_default(); // Def public CustomOrderDefinition.CustomOneofCase order_type => order.order_type(); public OrderSubType order_sub_type => order.sub_type(); @@ -84,9 +85,9 @@ namespace mbts.spark { internal void expand_internal() => Set(ref _is_expanded, true); internal void collapse_internal() => Set(ref _is_expanded, false); - public void update(OrderRecordTuple order_tuple) { + public void update(OrderSecurity order_tuple) { Log.assert(order_tuple.order.Key == this.order.Key, "OrderBookRowVM: update_order was called with an order with the wrong key."); - bool had_security = _security_md.HasValue; + bool had_security = _security.HasValue; // _show_order_modify_cmd is lazy (instantiated in the default_command getter); it might be null, don't not force instantiation bool? could_modify_execute = _show_order_modify_cmd?.CanExecute(null); OrderRecordImmutable old_order = order; @@ -103,7 +104,7 @@ namespace mbts.spark { // however, the order-type and security cells DO use databindings - if these cells have changed, the PropertyChanged event must be raised regardless of whether or not invalidated is raised if (old_order.order_type() != order.order_type()) RaisePropertyChanged(nameof(order_type), nameof(order_metadata)); - if (!had_security && _security_md.HasValue) { + if (!had_security && _security.HasValue) { RaisePropertyChanged(nameof(security), nameof(security_text)); // need to raise invalidated to refresh the price cells (the security.ContractSpec determines how they are formatted - Ticks!) invalidate = true; diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs index d86ea35bf..1b4a5afbf 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs @@ -3,7 +3,6 @@ using bts.api; using bts.api.orderman; using bts.shell; using bts.spark; -using bts.utils; using mbts.persist; using mbts.ui; using mbts.ui.controls; @@ -20,9 +19,15 @@ using System.Windows.Controls; using System.Windows.Input; using static bts.shell.OrderBookWindowFactory; using static mbts.spark.OrderBookView; -using _GridSettingsManager = mbts.ui.GridSettingsManager; +using _GridSettingsManager = + mbts.ui.GridSettingsManager< + mbts.spark.OrderBookRowVM, + bts.api.orderman.OrderSecurity, + bts.shell.INotificationServiceDataCache, + mbts.OrderRecordTuplePredicate, + bts.shell.OrderBookWindowFactory.OrderBookGridSettings>; -namespace mbts.spark { +namespace mbts.spark { partial class OrderBookVM : ViewModelBase, IDisposable, IHandleRowSelection, IHandleGridSorting, IWillLetYouKnowWhenIDispose { readonly IBus _bus; readonly OrdersModel _model; @@ -46,7 +51,7 @@ namespace mbts.spark { expand_all_rows_command = new DelegateCommand(_row_collection.expand_all); grid_settings_manager = _create_grid_settings_manager(owner_title, bus, notification_service, model, rows, columns); _resources = new List { - model.on_order_tuples_table.OnDispatcher().Subscribe(_row_collection.update), + model.on_orders.OnDispatcher().Subscribe(_row_collection.update), settings_manager.subscribe_current().OnDispatcher().Subscribe(grid_settings_manager), grid_settings_manager, Disposable.Create(() => { @@ -175,13 +180,13 @@ namespace mbts.spark { return new _GridSettingsManager( owner_title, bus, notification_service, model, rows, columns, () => { - var pvm = PredicateBuilderVMWrapper.create( + var pvm = PredicateBuilderVMWrapper.create( OrderRecordTuplePredicateVMFactory.get_oneofs(bus, model), rp => rp.predicate_type_case); pvm.selected_predicate_type = pvm.predicate_types.FirstOrDefault(); return pvm; }, col_id => { - var pvm = PredicateBuilderVMWrapper.create( + var pvm = PredicateBuilderVMWrapper.create( OrderRecordTuplePredicateVMFactory.get_oneofs(bus, model), rp => rp.predicate_type_case); pvm.selected_predicate_type = _corresponding_predicate_type(EnumHelper.parse_or_default(col_id)); return pvm; diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookWindow.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookWindow.cs index 1bbf0173e..8b0efabb6 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookWindow.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookWindow.cs @@ -16,7 +16,7 @@ using System.Windows.Data; using System.Windows.Media; //using IOrderBookWSM = mbts.persist.IWindowSettingsManager>; using OrderBookGridSettings = bts.shell.OrderBookWindowFactory.OrderBookGridSettings; -using OrderBookGridSettingsVM = mbts.spark.GridSettingsVM; +using OrderBookGridSettingsVM = mbts.spark.GridSettingsVM; namespace mbts.spark { internal class OrderBookWindow : SparkWindow, ISharedWindow { @@ -51,7 +51,7 @@ namespace mbts.spark { _settings_manager = settings_manager; } - public override OrderBookWindowFactory.OrderBookStartupData SaveState => _state = _vm.create(); + public override OrderBookWindowFactory.OrderBookStartupData SaveState => _state = _vm?.create(); public override string window_type => _window_type; public override UIElement Content => _view; @@ -86,12 +86,12 @@ namespace mbts.spark { ); _cancel_buy_orders_command = SparkCommands.CancelBuyOrders.complete( - _model.working_orders.Select(os => os.Values.Where(o => o.side() == Side.Buy).Select(o => o.Key)), + _model.working_orders.Select(os => os.Values.Where(o => o.order.side() == Side.Buy).Select(o => o.order.Key)), wbids => wbids.Any() ); _cancel_sell_orders_command = SparkCommands.CancelSellOrders.complete( - _model.working_orders.Select(os => os.Values.Where(o => o.side() == Side.Sell).Select(o => o.Key)), + _model.working_orders.Select(os => os.Values.Where(o => o.order.side() == Side.Sell).Select(o => o.order.Key)), wasks => wasks.Any() ); diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookWindowFactory.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookWindowFactory.cs index 77f1a5b14..52c632be9 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookWindowFactory.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookWindowFactory.cs @@ -215,7 +215,7 @@ namespace bts.shell { } [ProtoContract] - public class OrderBookGridSettings : IGridSettingsWithNotifications, IEquatable { + public class OrderBookGridSettings : IGridSettingsWithNotifications, IEquatable { [ProtoMember(1)] public GridSettings general { get; set; } [ProtoMember(2)] @@ -230,7 +230,7 @@ namespace bts.shell { } public IReadOnlyList predicate_animations => _predicate_animations ?? (_predicate_animations = new List()); - IEnumerable> IGridSettingsWithNotifications.predicate_animations => predicate_animations; + IEnumerable> IGridSettingsWithNotifications.predicate_animations => predicate_animations; public OrderBookGridSettings with(GridSettings general) => new OrderBookGridSettings(general ?? this.general, predicate_animations); diff --git a/src/client/spark/Notifications/INotificationService.cs b/src/client/spark/Notifications/INotificationService.cs index a01dd2506..eb3823cc7 100644 --- a/src/client/spark/Notifications/INotificationService.cs +++ b/src/client/spark/Notifications/INotificationService.cs @@ -1,4 +1,6 @@ using bts.api; +using bts.api.core; +using bts.api.orderman; using bts.spark; using mbts; using mbts.collections; @@ -9,6 +11,8 @@ using System; using System.Collections.Generic; using System.ComponentModel.Composition; using System.Linq; +using System.Reactive.Linq; +using OrderTuple = System.ValueTuple; namespace bts.shell { interface INotificationService { @@ -16,13 +20,13 @@ namespace bts.shell { INotificationServiceDataCache data_cache { get; } IDisposable link_to_activity(IPredicate predicate, SoundNotification sound); IDisposable link_to_activity(INotificationPredicate toast_notification_predicate); - IDisposable link_to_order(IPredicate predicate, SoundNotification sound); - IDisposable link_to_order(INotificationPredicate toast_notification_predicate); + IDisposable link_to_order(IPredicate predicate, SoundNotification sound); + IDisposable link_to_order(INotificationPredicate toast_notification_predicate); } static class NotificationServiceExtensions { public static IDisposable link(this INotificationService notification_service, IPredicate predicate, SoundNotification sound) { - if (predicate is IPredicate orp) + if (predicate is IPredicate orp) return notification_service.link_to_order(orp, sound); if (predicate is IPredicate ap) return notification_service.link_to_activity(ap, sound); @@ -30,7 +34,7 @@ namespace bts.shell { } public static IDisposable link(this INotificationService notification_service, INotificationPredicate toast_notification_predicate) { - if (toast_notification_predicate is INotificationPredicate orp) + if (toast_notification_predicate is INotificationPredicate orp) return notification_service.link_to_order(orp); if (toast_notification_predicate is INotificationPredicate ap) return notification_service.link_to_activity(ap); @@ -41,7 +45,7 @@ namespace bts.shell { [Export(typeof(INotificationService))] partial class NotificationService : INotificationService, IDisposable { readonly NotificationProcessor _activity_notifications; - readonly NotificationProcessor _order_notifications; + readonly NotificationProcessor _order_notifications; readonly DataCache _data_cache = new DataCache(); IDisposable _dispose; @@ -50,10 +54,10 @@ namespace bts.shell { public NotificationService(IBus bus, INotifier notifier, OrdersModel orders_model) { this.notifier = notifier; _activity_notifications = new NotificationProcessor(notifier); - _order_notifications = new NotificationProcessor(notifier); + _order_notifications = new NotificationProcessor(notifier); _dispose = Disposable.Create( bus.subscribe(SparkTags.LogActivity).Subscribe(_on_activity), - orders_model.on_order_tuples.Subscribe(_on_orders) + orders_model.on_orders.Subscribe(_on_orders) ); } @@ -63,17 +67,17 @@ namespace bts.shell { public IDisposable link_to_activity(IPredicate predicate, SoundNotification sound) => _activity_notifications.link(predicate, sound); public IDisposable link_to_activity(INotificationPredicate toast_notification_predicate) => _activity_notifications.link(toast_notification_predicate); - public IDisposable link_to_order(IPredicate predicate, SoundNotification sound) => _order_notifications.link(predicate, sound); - public IDisposable link_to_order(INotificationPredicate toast_notification_predicate) => _order_notifications.link(toast_notification_predicate); + public IDisposable link_to_order(IPredicate predicate, SoundNotification sound) => _order_notifications.link(predicate, sound); + public IDisposable link_to_order(INotificationPredicate toast_notification_predicate) => _order_notifications.link(toast_notification_predicate); public void Dispose() => Disposable.dispose(ref _dispose); void _on_activity(Activity activity) => _activity_notifications.process_item(activity); - void _on_orders(IIncrementalUpdate update) { + void _on_orders(TableUpdate<(SecurityImmutable?,OrderRecordImmutable),uint> update) { // must update notification_service_data_cache first, some of the predicate/triggers (esp. ToastNotifications) will use the notification_service_data_cache to lookup order-history - _data_cache.update(update); - foreach (var o in update.added.Values) + _data_cache.update(update.as_iincremental()); + foreach (var o in update.added) _order_notifications.process_item(o); } diff --git a/src/client/spark/Notifications/INotificationServiceDataCache.cs b/src/client/spark/Notifications/INotificationServiceDataCache.cs index aed90fbdb..fac9deeed 100644 --- a/src/client/spark/Notifications/INotificationServiceDataCache.cs +++ b/src/client/spark/Notifications/INotificationServiceDataCache.cs @@ -1,4 +1,6 @@ -using mbts.spark; +using bts.api; +using bts.api.orderman; +using mbts.spark; using mbts.utils; using System; using System.Collections.Generic; @@ -13,15 +15,21 @@ namespace bts.shell { interface INotificationServiceDataCache { DateTime last_order_reset_time { get; } bool is_first_order_update { get; } - IReadOnlyDictionary>> order_history { get; } + IReadOnlyDictionary>> order_history { get; } } static class NotificationServiceDataCacheExtensions { - public static OrderRecordTuple get_previous_order_state(this INotificationServiceDataCache notification_service_data_cache, uint key) => - notification_service_data_cache.order_history.TryGetValue(key, out List> l) && l.Count > 1 ? l[l.Count - 2].data : null; + public static OrderSecurity? get_previous_order_state(this INotificationServiceDataCache notification_service_data_cache, uint key) { + if (notification_service_data_cache.order_history.TryGetValue(key, out List> l)) { + if (l.Count > 1) { + return l[l.Count - 2].data; + } + } + return null; + } } - class TimeStampedData { + struct TimeStampedData { public readonly T data; public readonly DateTime time_stamp; @@ -34,14 +42,15 @@ namespace bts.shell { partial class NotificationService : INotificationService, IDisposable { class DataCache : INotificationServiceDataCache { const int _buffer_size = 10; - readonly Dictionary _order_cache = new Dictionary(); - readonly Dictionary>> _order_history = new Dictionary>>(); + readonly Dictionary _order_cache = new Dictionary(); + readonly Dictionary>> _order_history = new Dictionary>>(); + + public IReadOnlyDictionary>> order_history => _order_history; - public IReadOnlyDictionary>> order_history => _order_history; public DateTime last_order_reset_time { get; private set; } public bool is_first_order_update { get; private set; } - public void update(IIncrementalUpdate data) { + internal void update(IIncrementalUpdate<(SecurityImmutable? sec, OrderRecordImmutable order), uint> data) { DateTime now = DateTime.Now; if (data.removed.Count == _order_cache.Count) { last_order_reset_time = now; @@ -51,7 +60,7 @@ namespace bts.shell { foreach (var kv in data.removed) { if (data.added.ContainsKey(kv.Key)) { _order_cache[kv.Key] = kv.Value; - _order_history[kv.Key].Add(new TimeStampedData(kv.Value, now)); + _order_history[kv.Key].Add(new TimeStampedData(kv.Value, now)); } else { _order_cache.Remove(kv.Key); // TODO: maybe these shouldn't be removed? @@ -65,9 +74,9 @@ namespace bts.shell { continue; _order_cache.Add(kv.Key, kv.Value); _order_history.AddOrUpdate(kv.Key, - k => new List> { new TimeStampedData(kv.Value, now) }, + k => new List> { new TimeStampedData(kv.Value, now) }, (k, l) => { - l.Add(new TimeStampedData(kv.Value, now)); + l.Add(new TimeStampedData(kv.Value, now)); if (l.Count >= 10) l.RemoveAt(0); }); diff --git a/src/client/spark/Notifications/ToastNotification.cs b/src/client/spark/Notifications/ToastNotification.cs index f2590e5db..892a42cb4 100644 --- a/src/client/spark/Notifications/ToastNotification.cs +++ b/src/client/spark/Notifications/ToastNotification.cs @@ -60,14 +60,14 @@ namespace bts.shell { public static IToastNotification order_filled(OrderRecordImmutable oldval, OrderRecordImmutable newval, SecurityImmutable? security) => new OrderFilledToastNotification(oldval, newval, security); - public static IToastNotification order_filled(OrderRecordTuple oldval, OrderRecordTuple newval) => - new OrderFilledToastNotification(oldval.order, newval.order, newval.security_md?.security); + public static IToastNotification order_filled(OrderSecurity oldval, OrderSecurity newval) => + new OrderFilledToastNotification(oldval.order, newval.order, newval.security); public static IToastNotification order_completed(OrderRecordImmutable order, SecurityImmutable? security) => new OrderCompletedToastNotification(order, security); - public static IToastNotification order_completed(OrderRecordTuple order) => - new OrderCompletedToastNotification(order.order, order.security_md?.security); + public static IToastNotification order_completed(OrderSecurity order) => + new OrderCompletedToastNotification(order.order, order.security); public static IToastNotification order_status_changed(uint key, OrderStatus? old_status, OrderStatus new_status, SecurityImmutable? security) => new OrderStatusChangeToastNotification(key, old_status, new_status, security); diff --git a/src/client/spark/Orders/ExistingOrderTicketVM.cs b/src/client/spark/Orders/ExistingOrderTicketVM.cs index 18583a019..ddbf8f6f4 100644 --- a/src/client/spark/Orders/ExistingOrderTicketVM.cs +++ b/src/client/spark/Orders/ExistingOrderTicketVM.cs @@ -19,7 +19,7 @@ namespace mbts.spark { readonly Property _current_order_type; readonly Property _price_dirty, _size_dirty, _account_dirty, _time_in_force_dirty, _order_type_dirty; - public ExistingOrderTicketVM(OrdersModel model, IObservable order_source, IObservable calculator_buttons) { + public ExistingOrderTicketVM(OrdersModel model, IObservable<(SecurityMD? security_md, OrderRecordImmutable order)> order_source, IObservable calculator_buttons) { uint? key = null; var security_md_source = order_source.Where(ot => ot.security_md.HasValue).Select(ot => ot.security_md.Value); // diff --git a/src/client/spark/Orders/FillSecurity.cs b/src/client/spark/Orders/FillSecurity.cs new file mode 100644 index 000000000..01d8d0a20 --- /dev/null +++ b/src/client/spark/Orders/FillSecurity.cs @@ -0,0 +1,25 @@ +using System; + +namespace bts.api.orderman { + public readonly struct FillSecurity : IEquatable { + public FillSecurity(SecurityImmutable? sec, FillImmutable fill) { + this.sec = sec; + this.fill = fill; + } + public FillSecurity((SecurityImmutable? sec, FillImmutable fill) t) { + sec = t.sec; + fill = t.fill; + } + public static implicit operator FillSecurity(ValueTuple x) => new FillSecurity(x); + public SecurityImmutable? sec { get; } + public FillImmutable fill { get; } + + public override bool Equals(object obj) => (obj is FillSecurity os) && Equals(os); + public bool Equals(FillSecurity other) => fill.FillKey == other.fill.FillKey && sec == other.sec; + public override int GetHashCode() => (sec, fill).GetHashCode(); + + public static bool operator ==(FillSecurity left, FillSecurity right) => left.Equals(right); + public static bool operator !=(FillSecurity left, FillSecurity right) => !left.Equals(right); + + } +} \ No newline at end of file diff --git a/src/client/spark/Orders/OrderRecordTuple.cs b/src/client/spark/Orders/OrderRecordTuple.cs deleted file mode 100644 index 4e6255364..000000000 --- a/src/client/spark/Orders/OrderRecordTuple.cs +++ /dev/null @@ -1,47 +0,0 @@ -using bts.api.orderman; -using bts.utils; -using mbts.utils; -using System; - -namespace mbts.spark { - struct OrderRecordTupleKey : IEquatable { - public readonly uint order_key, security_key; - - public OrderRecordTupleKey(uint order_key, uint security_key) { - this.order_key = order_key; - this.security_key = security_key; - } - - public bool Equals(OrderRecordTupleKey other) => - other.order_key == order_key && - other.security_key == security_key; - - public override bool Equals(object obj) => - obj is OrderRecordTupleKey k && Equals(k); - - public override int GetHashCode() => - EqualityComparer.aggregate_hash( - (int)order_key, - (int)security_key); - } - - class OrderRecordTuple : IEquatable { - public readonly OrderRecordImmutable order; - public readonly SecurityMD? security_md; - public readonly OrderRecordTupleKey key; - - public OrderRecordTuple(OrderRecordImmutable order, SecurityMD? security_md) { - this.order = order; - this.security_md = security_md; - key = new OrderRecordTupleKey(order.Key, security_md.HasValue ? security_md.Value.security.Id : 0u); - } - - public bool Equals(OrderRecordTuple other) => other != null - && other.order.Equals(order) - && other.security_md.HasValue == security_md.HasValue; - - public override bool Equals(object obj) => Equals(obj as OrderRecordTuple); - public override int GetHashCode() => key.GetHashCode(); - public override string ToString() => $"order:{order.Key}, sec:{security_md.display_name()}"; - } -} \ No newline at end of file diff --git a/src/client/spark/Orders/OrderRecordTupleSource.cs b/src/client/spark/Orders/OrderRecordTupleSource.cs deleted file mode 100644 index 0c8aeaccf..000000000 --- a/src/client/spark/Orders/OrderRecordTupleSource.cs +++ /dev/null @@ -1,147 +0,0 @@ -using bts.api.core; -using bts.api.orderman; -using mbts.collections; -using mbts.utils; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Reactive.Linq; -using System.Reactive.Subjects; - -namespace mbts.spark { - class OrderRecordTupleSource : IDisposable { - readonly object _lock = new object(); - // subjects - readonly Subject> _subject = new Subject>(); - readonly Subject> _table_subject = new Subject>(); - readonly Subject> _order_securities_subject = new Subject>(); - // data cache - readonly Dictionary _state = new Dictionary(); - readonly SetMap _security_orders = new SetMap(); - readonly IDictionary _security_md_cache = new Dictionary(0); - IReadOnlyDictionary _orders_cache = new Dictionary(0); - IDisposable _dispose; - - public OrderRecordTupleSource(IObservable> securities, IObservable> orders) => - _dispose = Disposable.Create( - securities.Subscribe(_update_securities), - orders.Subscribe(_update_orders)); - - public IObservable> subscribe_incremental_update() => - Observable.Create>(observer => { - lock (_lock) { - observer.OnNext(new IncrementalUpdate(new Dictionary(_state), new Dictionary(0))); - return _subject.Subscribe(observer); - } - }); - - public IObservable> subscribe_incremental_table_update() => - Observable.Create>(observer => { - lock (_lock) { - observer.OnNext(IncrementalTableUpdateTranform.init(_state.Values)); - return _table_subject.Subscribe(observer); - } - }); - - public IObservable> subscribe_order_securities() => - Observable.Create>(observer => { - lock (_lock) { - var init = new Dictionary(); - foreach (uint sec_key in _security_orders.Keys) { - if (_security_md_cache.TryGetValue(sec_key, out SecurityMD smd)) - init.Add(sec_key, smd); - } - observer.OnNext(new IncrementalUpdate(init, new Dictionary(0))); - return _order_securities_subject.Subscribe(observer); - } - }); - - public void Dispose() => Disposable.dispose(ref _dispose); - - void _update_orders(TableUpdate orders) { - lock (_lock) { - Dictionary removed = new Dictionary(), added = new Dictionary(); - Dictionary added_securities = new Dictionary(), removed_securities = new Dictionary(); - foreach (OrderRecordImmutable order in orders.removed) { - uint sec_key = order.security_key(); - OrderRecordTuple old_ft = new OrderRecordTuple(order, _get_security_md(sec_key)); - removed.Add(old_ft.order.Key, old_ft); - if (_security_orders.removes_key(sec_key, order.Key) && old_ft.security_md.HasValue) - removed_securities.Add(sec_key, old_ft.security_md.Value); - } - foreach (Change ch in orders.changes) { - // orders cannot change securities; ch.New can use the same security_md as ch.Old - var sec_md = _get_security_md(ch.Old.security_key()); - removed.Add(ch.Old.Key, new OrderRecordTuple(ch.Old, sec_md)); - added.Add(ch.New.Key, new OrderRecordTuple(ch.New, sec_md)); - } - foreach (OrderRecordImmutable order in orders.added) { - uint sec_key = order.security_key(); - OrderRecordTuple new_ort = new OrderRecordTuple(order, _get_security_md(sec_key)); - added.Add(new_ort.order.Key, new_ort); - if (_security_orders.adds_key(sec_key, order.Key) && new_ort.security_md.HasValue) - added_securities.Add(sec_key, new_ort.security_md.Value); - } - _orders_cache = orders.state; - _apply_changes(added, removed, added_securities, removed_securities); - } - } - - void _update_securities(IReadOnlyDictionary securities) { - lock (_lock) { - Dictionary added = new Dictionary(), removed = new Dictionary(); - Dictionary added_securities = new Dictionary(), removed_securities = new Dictionary(); - _security_md_cache.Keys.difference(securities.Keys, out IEnumerable added_keys, out IEnumerable removed_keys); - /// Warning: must NOT be reassigned with ; the latter is always the same object. - /// This would mean that the - /// call above would literally be calculating the difference between the keys of the same dictionary! - foreach (var k in removed_keys.ToList()) { - if (_security_orders.try_lookup(k, out ISet order_keys)) { - SecurityMD old_sec_md = _security_md_cache[k]; - removed_securities.Add(k, old_sec_md); - IEnumerable orders = order_keys.Select(fk => _orders_cache[fk]); - foreach (OrderRecordImmutable order in orders) { - OrderRecordTuple old_ft = new OrderRecordTuple(order, old_sec_md); - removed.Add(old_ft.order.Key, old_ft); - OrderRecordTuple new_ft = new OrderRecordTuple(order, null); - added.Add(new_ft.order.Key, new_ft); - } - } - _security_md_cache.Remove(k); - } - foreach (var k in added_keys.ToList()) { - SecurityMD new_sec_md = securities[k]; - if (_security_orders.try_lookup(k, out ISet order_keys)) { - added_securities.Add(k, new_sec_md); - IEnumerable orders = order_keys.Select(fk => _orders_cache[fk]); - foreach (OrderRecordImmutable order in orders) { - OrderRecordTuple old_ft = new OrderRecordTuple(order, null); - removed.Add(old_ft.order.Key, old_ft); - OrderRecordTuple new_ft = new OrderRecordTuple(order, new_sec_md); - added.Add(new_ft.order.Key, new_ft); - } - } - _security_md_cache.Add(k, new_sec_md); - } - _apply_changes(added, removed, added_securities, removed_securities); - } - } - - SecurityMD? _get_security_md(uint key) => - _security_md_cache.TryGetValue(key, out SecurityMD smd) ? smd : new SecurityMD?(); - - OrderRecordImmutable? _get_order(uint key) => - _orders_cache.TryGetValue(key, out OrderRecordImmutable o) ? o : new OrderRecordImmutable?(); - - void _apply_changes(Dictionary added, Dictionary removed, Dictionary added_securities, Dictionary removed_securities) { - foreach (var kv in removed) - _state.Remove(kv.Key); - foreach (var kv in added) - _state.Add(kv.Key, kv.Value); - var update = new IncrementalUpdate(added, removed); - _subject.OnNext(update); - _table_subject.OnNext(update.as_table()); - _order_securities_subject.OnNext(new IncrementalUpdate(added_securities, removed_securities)); - } - } -} \ No newline at end of file diff --git a/src/client/spark/Orders/OrderSecurity.cs b/src/client/spark/Orders/OrderSecurity.cs new file mode 100644 index 000000000..dd9936162 --- /dev/null +++ b/src/client/spark/Orders/OrderSecurity.cs @@ -0,0 +1,25 @@ +using System; + +namespace bts.api.orderman { + public readonly struct OrderSecurity: IEquatable { + public OrderSecurity(SecurityImmutable? sec, OrderRecordImmutable order) { + this.security = sec; + this.order = order; + } + public OrderSecurity((SecurityImmutable? sec, OrderRecordImmutable order) t) { + security = t.sec; + order = t.order; + } + public static implicit operator OrderSecurity(ValueTuple x) => new OrderSecurity(x); + public SecurityImmutable? security { get; } + public OrderRecordImmutable order { get; } + + public override bool Equals(object obj) => (obj is OrderSecurity os) && Equals(os); + public bool Equals(OrderSecurity other) => order.Key == other.order.Key && security == other.security; + public override int GetHashCode() => (security, order).GetHashCode(); + + public static bool operator == (OrderSecurity left, OrderSecurity right) => left.Equals(right); + public static bool operator != (OrderSecurity left, OrderSecurity right) => !left.Equals(right); + + } +} diff --git a/src/client/spark/Orders/OrdersModel.cs b/src/client/spark/Orders/OrdersModel.cs index 2eab7f64a..bbf337718 100644 --- a/src/client/spark/Orders/OrdersModel.cs +++ b/src/client/spark/Orders/OrdersModel.cs @@ -16,10 +16,10 @@ using System.Linq; using System.Reactive.Concurrency; using System.Reactive.Linq; using System.Threading.Tasks; -using OrderPair = System.ValueTuple; using OrderUpdate = bts.api.core.TableUpdate<(bts.api.SecurityImmutable? sec, bts.api.orderman.OrderRecordImmutable order), uint>; namespace mbts.spark { + [Export] class OrdersModel : IDisposable { readonly IBus _bus; @@ -57,13 +57,15 @@ namespace mbts.spark { on_positions = _position_service.subscribei_position_data().Naggle(interval, scheduler).PermissionFilter(admin_model.on_admin_data).Table().Cache(); working_orders = on_orders.Scan( - ImmutableDictionary.Empty, + ImmutableDictionary.Empty, (workers, table) => _update_working_orders(workers, table)) .Replay(1).RefCount(); - var order_record_tuple_source = new OrderRecordTupleSource(bus.subscribe(SparkTags.SecuritiesTag), on_orders); - on_order_tuples = order_record_tuple_source.subscribe_incremental_update(); - on_order_tuples_table = order_record_tuple_source.subscribe_incremental_table_update(); - on_order_securities = order_record_tuple_source.subscribe_order_securities(); + + on_order_securities = + on_orders. + as_iincremental() + .filter(t => t.sec.HasValue) + .Select(t => t.sec.Value); on_fills = _service.subscribei_fills() @@ -72,13 +74,12 @@ namespace mbts.spark { //.join(sstore, f => ) .Table().Cache(); _activity = new OrderActivitySummarizer(bus, on_orders); - on_fill_tuples = FillTupleSource.create_observable(bus.subscribe(SparkTags.SecuritiesTag), on_orders, on_fills); + on_fill_tuples = FillTupleSource.create_observable(bus.subscribe(SparkTags.SecuritiesTag), on_orders.Select(i => i.order, order => order.Key), on_fills); _security_orders_demuxer = new TableUpdateDemuxer(o => o.Key, o => o.security_key()); _security_positions_demuxer = new TableUpdateDemuxer(o => o.RecordId, p => p.SecurityId); _dispose = Disposable.Create( - on_orders.AddRemoveOnly().Subscribe(_security_orders_demuxer), + //on_orders.AddRemoveOnly().Subscribe(_security_orders_demuxer), on_positions.AddRemoveOnly().Subscribe(_security_positions_demuxer), - order_record_tuple_source, _bus.subscribe(SparkTags.CreateOrder).Subscribe(a => send_order(a.definition, a.price, a.size).ignore()), _bus.subscribe(SparkTags.RecreateOrder).Subscribe(a => a.ForEach(x => send_order(x.definition, x.price, x.size).ignore())), _bus.subscribe(SparkTags.CancelOrders).Subscribe(x => cancel_orders(x).ignore()), @@ -92,12 +93,10 @@ namespace mbts.spark { public SpreaderModel spreader => _spreader; public DynamicOrderHelper dynamic_helper => _dynamic_helper; public IObservable on_orders { get; } - public IObservable> on_order_tuples { get; } - public IObservable> on_order_tuples_table { get; } - public IObservable> on_order_securities { get; } + public IObservable> on_order_securities { get; } public IObservable> on_fills { get; } public IObservable> on_fill_tuples { get; } - public IObservable> working_orders { get; } + public IObservable> working_orders { get; } public IObservable> on_positions { get; } public IObservable> on_security_orders(uint security) => @@ -168,8 +167,8 @@ namespace mbts.spark { public void Dispose() => Disposable.dispose(ref _dispose); - static ImmutableDictionary - _update_working_orders(ImmutableDictionary target, OrderUpdate table) { + static ImmutableDictionary + _update_working_orders(ImmutableDictionary target, OrderUpdate table) { var clone = target.ToBuilder(); foreach (var o in table.removed) { if (o.order.is_active()) { diff --git a/src/client/spark/Predicates/Data/Events/OrderCanceledEventPredicate.cs b/src/client/spark/Predicates/Data/Events/OrderCanceledEventPredicate.cs index 4546dfd6d..9ed71bb6e 100644 --- a/src/client/spark/Predicates/Data/Events/OrderCanceledEventPredicate.cs +++ b/src/client/spark/Predicates/Data/Events/OrderCanceledEventPredicate.cs @@ -26,7 +26,7 @@ namespace mbts { } [ProtoContract] - class OrderCanceledEventPredicate : IPredicateBuilder, IEquatable { + class OrderCanceledEventPredicate : IPredicateBuilder, IEquatable { public enum PredicateTypeOneofCase { [Description("(any order)")] AnyOrder, @@ -45,13 +45,13 @@ namespace mbts { } } - public IPredicate create(object obj) => + public IPredicate create(object obj) => create_predicate((INotificationServiceDataCache)obj); - public IPredicate create_predicate(INotificationServiceDataCache data_cache) => + public IPredicate create_predicate(INotificationServiceDataCache data_cache) => OrderEventPredicateHelper.create_status_transition_predicate(OrderStatus.Canceled, security, data_cache); - public INotificationPredicate create_toast(INotificationServiceDataCache data_cache) => + public INotificationPredicate create_toast(INotificationServiceDataCache data_cache) => new MyToaster(security?.create(data_cache), data_cache); public bool Equals(OrderCanceledEventPredicate other) => other != null && other.security.safe_equals(other.security); @@ -61,7 +61,7 @@ namespace mbts { static bool _is_canceled(OrderRecordImmutable o) => o.HasState && o.State.Status == OrderStatus.Canceled; - class MyToaster : INotificationPredicate, IEquatable { + class MyToaster : INotificationPredicate, IEquatable { readonly IPredicate _security_predicate; readonly INotificationServiceDataCache _data_cache; @@ -70,12 +70,13 @@ namespace mbts { _data_cache = data_cache; } - public IToastNotification trigger(OrderRecordTuple order) => - OrderEventPredicateHelper.check_order_transition_trigger(order, _is_canceled, _security_predicate, _data_cache, out OrderRecordTuple oldval) ? - ToastNotifications.order_status_changed(order.order.Key, OrderEventPredicateHelper.get_nullable_status(oldval), OrderStatus.Canceled, order.security_md.HasValue ? order.security_md.Value.security : new SecurityImmutable?()) : + public IToastNotification trigger(OrderSecurity order) => + OrderEventPredicateHelper.check_order_transition_trigger(order, _is_canceled, _security_predicate, _data_cache, out var oldval) ? + ToastNotifications.order_status_changed(order.order.Key, + OrderEventPredicateHelper.get_nullable_status(oldval), OrderStatus.Canceled, order.security) : ToastNotifications.empty; - bool IPredicate.trigger(OrderRecordTuple order) => + bool IPredicate.trigger(OrderSecurity order) => OrderEventPredicateHelper.check_order_transition_predicate(order, _is_canceled, _security_predicate, _data_cache); public bool Equals(MyToaster other) => diff --git a/src/client/spark/Predicates/Data/Events/OrderCompletedEventPredicate.cs b/src/client/spark/Predicates/Data/Events/OrderCompletedEventPredicate.cs index 881201de5..f6942d908 100644 --- a/src/client/spark/Predicates/Data/Events/OrderCompletedEventPredicate.cs +++ b/src/client/spark/Predicates/Data/Events/OrderCompletedEventPredicate.cs @@ -26,7 +26,7 @@ namespace mbts { } [ProtoContract] - class OrderCompletedEventPredicate : IPredicateBuilder, IEquatable { + class OrderCompletedEventPredicate : IPredicateBuilder, IEquatable { public enum PredicateTypeOneofCase { [Description("(any order)")] AnyOrder, @@ -45,13 +45,13 @@ namespace mbts { } } - public IPredicate create(object obj) => + public IPredicate create(object obj) => create_predicate((INotificationServiceDataCache)obj); - public IPredicate create_predicate(INotificationServiceDataCache data_cache) => + public IPredicate create_predicate(INotificationServiceDataCache data_cache) => OrderEventPredicateHelper.create_status_transition_predicate(OrderStatus.Completed, security, data_cache); - public INotificationPredicate create_toast(INotificationServiceDataCache data_cache) => + public INotificationPredicate create_toast(INotificationServiceDataCache data_cache) => new MyToaster(security?.create(data_cache), data_cache); public bool Equals(OrderCompletedEventPredicate other) => other != null && other.security.safe_equals(other.security); @@ -61,7 +61,7 @@ namespace mbts { static bool _is_completed(OrderRecordImmutable o) => o.HasState && o.State.Status == OrderStatus.Completed; - class MyToaster : INotificationPredicate, IEquatable { + class MyToaster : INotificationPredicate, IEquatable { readonly IPredicate _security_predicate; readonly INotificationServiceDataCache _data_cache; @@ -70,11 +70,11 @@ namespace mbts { _data_cache = data_cache; } - public IToastNotification trigger(OrderRecordTuple order) => + public IToastNotification trigger(OrderSecurity order) => OrderEventPredicateHelper.check_order_transition_predicate(order, _is_completed, _security_predicate, _data_cache) ? ToastNotifications.order_completed(order) : ToastNotifications.empty; - bool IPredicate.trigger(OrderRecordTuple order) => + bool IPredicate.trigger(OrderSecurity order) => OrderEventPredicateHelper.check_order_transition_predicate(order, _is_completed, _security_predicate, _data_cache); public bool Equals(MyToaster other) => diff --git a/src/client/spark/Predicates/Data/Events/OrderEventPredicateHelper.cs b/src/client/spark/Predicates/Data/Events/OrderEventPredicateHelper.cs index 25c019787..f9e27e62f 100644 --- a/src/client/spark/Predicates/Data/Events/OrderEventPredicateHelper.cs +++ b/src/client/spark/Predicates/Data/Events/OrderEventPredicateHelper.cs @@ -11,16 +11,16 @@ namespace mbts { /// /// Checks if the order has transitioned from 'not passing the predicate, or not existing' to 'passing the predicate.' /// - public static bool check_order_transition_predicate(OrderRecordTuple order, Func order_predicate_core, IPredicate security_predicate, INotificationServiceDataCache data_cache) => - check_order_transition_trigger(order, order_predicate_core, security_predicate, data_cache, out OrderRecordTuple _); + public static bool check_order_transition_predicate(OrderSecurity order, Func order_predicate_core, IPredicate security_predicate, INotificationServiceDataCache data_cache) => + check_order_transition_trigger(order, order_predicate_core, security_predicate, data_cache, out var _); /// /// Checks if the order has transitioned from 'not passing the predicate, or not existing' to 'passing the predicate.' Oldval will be null if the order did not previously exist, or does not currently pass the predicate. /// - public static bool check_order_transition_trigger(OrderRecordTuple order, Func order_predicate_core, IPredicate security_predicate, INotificationServiceDataCache data_cache, out OrderRecordTuple oldval) { + public static bool check_order_transition_trigger(OrderSecurity order, Func order_predicate_core, IPredicate security_predicate, INotificationServiceDataCache data_cache, out OrderSecurity? oldval) { if (check_security_predicate(order, security_predicate) && order_predicate_core(order.order)) { oldval = data_cache.get_previous_order_state(order.order.Key); - return oldval == null ? !data_cache.is_first_order_update : !order_predicate_core(oldval.order); + return oldval == null ? !data_cache.is_first_order_update : !order_predicate_core(oldval.Value.order); } oldval = null; return false; @@ -29,19 +29,19 @@ namespace mbts { /// /// Checks if the 'security predicate is null,' or 'it is not null, and the order has a non-null security which passes.' /// - public static bool check_security_predicate(OrderRecordTuple order, IPredicate security_predicate) => - security_predicate == null || (order.security_md.HasValue && security_predicate.trigger(order.security_md.Value.security)); + public static bool check_security_predicate(OrderSecurity order, IPredicate security_predicate) => + security_predicate == null || (order.security.HasValue && security_predicate.trigger(order.security.Value)); - public static OrderStatus? get_nullable_status(OrderRecordTuple order) { - if (order == null && order.order.HasState) - return order.order.State.Status; + public static OrderStatus? get_nullable_status(OrderSecurity? order) { + if (order?.order.HasState == true) + return order?.order.State.Status; return null; } - public static IPredicate create_status_transition_predicate(OrderStatus status, SecurityPredicate security_predicate, INotificationServiceDataCache data_cache) => + public static IPredicate create_status_transition_predicate(OrderStatus status, SecurityPredicate security_predicate, INotificationServiceDataCache data_cache) => new OrderStatusTransitionPredicate(status, security_predicate?.create(data_cache), data_cache); - class OrderStatusTransitionPredicate : IPredicate, IEquatable { + class OrderStatusTransitionPredicate : IPredicate, IEquatable { readonly IPredicate _security_predicate; readonly INotificationServiceDataCache _data_cache; readonly OrderStatus _status; @@ -52,7 +52,7 @@ namespace mbts { _data_cache = data_cache; } - public bool trigger(OrderRecordTuple value) => + public bool trigger(OrderSecurity value) => check_order_transition_predicate(value, o => o.HasState && o.State.Status == _status, _security_predicate, _data_cache); public bool Equals(OrderStatusTransitionPredicate other) => diff --git a/src/client/spark/Predicates/Data/Events/OrderFilledEventPredicate.cs b/src/client/spark/Predicates/Data/Events/OrderFilledEventPredicate.cs index 9b67bd56f..b6b1f749f 100644 --- a/src/client/spark/Predicates/Data/Events/OrderFilledEventPredicate.cs +++ b/src/client/spark/Predicates/Data/Events/OrderFilledEventPredicate.cs @@ -1,4 +1,5 @@ using bts.api; +using bts.api.orderman; using bts.shell; using mbts.spark; using mbts.ui.settings; @@ -14,15 +15,15 @@ namespace mbts { { OrderFilledEventPredicate.PredicateTypeOneofCase.SecurityOrder, SecurityCriteriaVMFactory.create(bus, o => o.security?.is_value ?? new SecurityCriteria(), (p, s) => { if (p.security == null) - p.security = new SecurityPredicate(); - p.security.is_value = s; +{ p.security = new SecurityPredicate(); +} p.security.is_value = s; }) } }; } [ProtoContract] - class OrderFilledEventPredicate : IPredicateBuilder, IEquatable { + class OrderFilledEventPredicate : IPredicateBuilder, IEquatable { public enum PredicateTypeOneofCase { [PredicateCase("(any order)")] AnyOrder, @@ -35,19 +36,21 @@ namespace mbts { public PredicateTypeOneofCase predicate_type_case { get { - if (security != null) + if (security != null) { return PredicateTypeOneofCase.SecurityOrder; + } + return PredicateTypeOneofCase.AnyOrder; } } - public IPredicate create(object obj) => + public IPredicate create(object obj) => create_predicate((INotificationServiceDataCache)obj); - public IPredicate create_predicate(INotificationServiceDataCache notification_service_data_cache) => + public IPredicate create_predicate(INotificationServiceDataCache notification_service_data_cache) => new FillTrigger(_create_model(notification_service_data_cache)); - public INotificationPredicate create_toast(INotificationServiceDataCache notification_service_data_cache) => + public INotificationPredicate create_toast(INotificationServiceDataCache notification_service_data_cache) => new FillToastNotificationTrigger(_create_model(notification_service_data_cache)); public bool Equals(OrderFilledEventPredicate other) => other != null && other.security.safe_equals(other.security); @@ -66,35 +69,43 @@ namespace mbts { _security_predicate = security_predicate; } - public (bool trigger, OrderRecordTuple oldval, OrderRecordTuple newval) get_filled_order_args(OrderRecordTuple ort) { + // Please review carfully + public (OrderSecurity oldval, OrderSecurity newval)? get_filled_order_args(OrderSecurity ort) { if (ort.order.is_exchange_order() && OrderEventPredicateHelper.check_security_predicate(ort, _security_predicate)) { - if (_data_cache.get_previous_order_state(ort.order.Key) is OrderRecordTuple oldval) - return (ort.order.filled() > oldval.order.filled(), oldval, ort); + if (_data_cache.get_previous_order_state(ort.order.Key) is var oldval && oldval.HasValue) { + if (ort.order.filled() > oldval?.order.filled()) { + return (oldval.Value, ort); + } + } else if (!_data_cache.is_first_order_update && ort.order.filled() > 0) { // old order will be a copy of new order, except with filledQuantity set to 0 var old_state = ort.order.State.CloneToMutable(); old_state.FilledQuantity = 0; - return (true, new OrderRecordTuple(ort.order.with(state: old_state), ort.security_md), ort); + return (new OrderSecurity(ort.security, ort.order.with(state: old_state)), ort); } } - return (false, null, null); + return null; } } - class FillTrigger : IPredicate { + class FillTrigger : IPredicate { protected readonly FillTriggerModel _model; public FillTrigger(FillTriggerModel model) => _model = model; - public bool trigger(OrderRecordTuple order) => _model.get_filled_order_args(order).trigger; + public bool trigger(OrderSecurity order) => _model.get_filled_order_args(order).HasValue; } - class FillToastNotificationTrigger : FillTrigger, INotificationPredicate { + class FillToastNotificationTrigger : FillTrigger, INotificationPredicate { public FillToastNotificationTrigger(FillTriggerModel model) : base(model) { } - public new IToastNotification trigger(OrderRecordTuple value) { - (var trigger, var oldval, var newval) = _model.get_filled_order_args(value); - return trigger ? ToastNotifications.order_filled(oldval, newval) : ToastNotifications.empty; + public new IToastNotification trigger(OrderSecurity value) { + var args = _model.get_filled_order_args(value); + if (args.HasValue) { + return ToastNotifications.order_filled(args.Value.oldval, args.Value.newval); + } + return ToastNotifications.empty; + //return args ? ToastNotifications.order_filled(oldval, newval) : ToastNotifications.empty; } } } diff --git a/src/client/spark/Predicates/Data/OrderRecordTuplePredicate.cs b/src/client/spark/Predicates/Data/OrderRecordTuplePredicate.cs index 1a938f50a..e12c4d480 100644 --- a/src/client/spark/Predicates/Data/OrderRecordTuplePredicate.cs +++ b/src/client/spark/Predicates/Data/OrderRecordTuplePredicate.cs @@ -61,7 +61,7 @@ namespace mbts { } [ProtoContract] - class OrderRecordTuplePredicate : INotificationPredicateBuilder, IEquatable { + class OrderRecordTuplePredicate : INotificationPredicateBuilder, IEquatable { SecurityPredicate _security; SidePredicate _side; DoublePredicate _price, _last_price, _avg_price; @@ -206,14 +206,14 @@ namespace mbts { } } - public IPredicate create_predicate(INotificationServiceDataCache data_cache) { + public IPredicate create_predicate(INotificationServiceDataCache data_cache) { switch (predicate_type_case) { - case PTOC.Security: return wrap(_security, r => r.security_md.HasValue ? r.security_md.Value.security : SecurityDataHelper.Defaults.SecurityImmutable); + case PTOC.Security: return wrap(_security, r => r.security ?? SecurityDataHelper.Defaults.SecurityImmutable); case PTOC.Time: return wrap(_time_created, r => r.order.created()); case PTOC.LastChange: return wrap(_last_change, r => r.order.last_change()); case PTOC.User: return wrap(_user, r => r.order.user_id()); case PTOC.Account: return wrap(_account, r => r.order.account_id()); - case PTOC.Exchange: return wrap(_exchange, r => r.security_md.HasValue ? r.security_md.Value.security.Exchange : Venue.Unset); + case PTOC.Exchange: return wrap(_exchange, r => r.security.exchange_or_default()); case PTOC.TimeInForce: return wrap(_time_in_force, r => r.order.time_in_force()); case PTOC.OrderType: return wrap(_order_type, r => r.order.order_type()); case PTOC.SubType: return wrap(_order_sub_type, r => r.order.sub_type()); @@ -234,14 +234,14 @@ namespace mbts { default: return null; } // - IPredicate wrap(IPredicateBuilder trigger, Func select) => - trigger.create(data_cache).wrap(select); + IPredicate wrap(IPredicateBuilder trigger, Func select) => + trigger.create(data_cache).wrap(select); } - IPredicate IPredicateBuilder.create(object obj) => + IPredicate IPredicateBuilder.create(object obj) => create_predicate((INotificationServiceDataCache)obj); - public INotificationPredicate create_notification_predicate(INotificationServiceDataCache notification_service_data_cache) { + public INotificationPredicate create_notification_predicate(INotificationServiceDataCache notification_service_data_cache) { if (!is_toast_enabled) return null; switch (predicate_type_case) { @@ -252,7 +252,7 @@ namespace mbts { } } - INotificationPredicate INotificationPredicateBuilder.create_notification_predicate(object obj) => + INotificationPredicate INotificationPredicateBuilder.create_notification_predicate(object obj) => create_notification_predicate((INotificationServiceDataCache)obj); public bool Equals(OrderRecordTuplePredicate other) { @@ -325,8 +325,8 @@ namespace mbts { return EqualityComparer.phase_hash(raw_hash, (int)ptoc); } - static string _get_order_placed_toast_msg(OrderRecordTuple order) => - $"Order Sent: {order.security_md.display_name()} {order.order.short_description(order.security_md.spec_or_default())}"; + static string _get_order_placed_toast_msg(OrderSecurity order) => + $"Order Sent: {order.security.display_name()} {order.order.short_description(order.security.spec_or_default())}"; void _set(ref T field, T value) { if (value != null) { @@ -359,7 +359,7 @@ namespace mbts { } [ProtoContract] - class OrderRecordTuplePredicateAnimations : IPredicateAnimationsData { + class OrderRecordTuplePredicateAnimations : IPredicateAnimationsData { [ProtoMember(1)] OrderRecordTuplePredicate _predicate; [ProtoMember(2)] diff --git a/src/client/spark/Security/SecurityStore.cs b/src/client/spark/Security/SecurityStore.cs index 558c588f4..3b0f53794 100644 --- a/src/client/spark/Security/SecurityStore.cs +++ b/src/client/spark/Security/SecurityStore.cs @@ -15,6 +15,7 @@ using System.Threading.Tasks; using security_key = System.UInt32; namespace bts { + [Export] public class SecurityStore : IDisposable { readonly MarketDataRelay _md; @@ -104,7 +105,7 @@ namespace bts { foreach (var key in keys) { var security = try_get_local(key); if (security.HasValue) { - result.Add(key, security.Value); + result.TryAdd(key, security.Value); } else { missing.Keys.Add(key); diff --git a/src/client/spark/core/GlobalExtensions/GlobalExtensions.cs b/src/client/spark/core/GlobalExtensions/GlobalExtensions.cs index 93ba83022..36c545789 100644 --- a/src/client/spark/core/GlobalExtensions/GlobalExtensions.cs +++ b/src/client/spark/core/GlobalExtensions/GlobalExtensions.cs @@ -233,6 +233,7 @@ namespace mbts return has_value; } + } } diff --git a/src/client/spark/core/Utility/TableUpdateTransform.cs b/src/client/spark/core/Utility/TableUpdateTransform.cs index 6dee3f511..fbd3da11f 100644 --- a/src/client/spark/core/Utility/TableUpdateTransform.cs +++ b/src/client/spark/core/Utility/TableUpdateTransform.cs @@ -26,9 +26,6 @@ namespace mbts.utils { } class IncrementalUpdate : IIncrementalUpdate where TKey : IEquatable { - public IncrementalUpdate() - : this(new Dictionary(0), new Dictionary(0)) { } - public IncrementalUpdate(IReadOnlyDictionary added, IReadOnlyDictionary removed) { this.added = added; this.removed = removed; @@ -305,6 +302,40 @@ namespace mbts.utils { state_clone.Add(kv.Key, kv.Value); return new PivotTableUpdateTransform(state_clone, added.Values.ToList(), removed.Values.ToList(), _key_map, _selector, _result_key, _key); } - } + } + + public static IObservable> as_iincremental(this IObservable> src) where TKey : IEquatable { + return src.Select(as_iincremental); + } + + public static IIncrementalUpdate as_iincremental(this TableUpdate update) where TKey : IEquatable { + var key = update.key_extractor; + var added = update.added.ToDictionary(key); + var removed = update.removed.ToDictionary(key); + foreach (var ch in update.changes) { + added[key(ch.New)] = ch.New; + removed[key(ch.Old)] = ch.Old; + } + return new IncrementalUpdate(added, removed); + } + + public static IObservable> + Select(this IObservable> src, Func map) where TKey : IEquatable { + return src.Select(update => { + return new IncrementalUpdate( + new MappedDictionary(update.added, map), + new MappedDictionary(update.removed, map)); + }); + } + + public static IObservable> + Where(this IObservable> src, Func where) where TKey : IEquatable { + return src.Select(update => { + return new IncrementalUpdate( + new FilteredDictionary(update.added, where), + new FilteredDictionary(update.removed, where)); + }); + } + } } \ No newline at end of file diff --git a/src/client/spark/view/Controls/DIYGrid/Settings/GridSettingsManager.cs b/src/client/spark/view/Controls/DIYGrid/Settings/GridSettingsManager.cs index e4eccfc55..5d5e60b52 100644 --- a/src/client/spark/view/Controls/DIYGrid/Settings/GridSettingsManager.cs +++ b/src/client/spark/view/Controls/DIYGrid/Settings/GridSettingsManager.cs @@ -17,7 +17,8 @@ using System.Windows.Media; using static mbts.ui.controls.DIYGrid; namespace mbts.ui { - class GridSettingsManager : ViewModelBase, IValueFactory, IObserver, IDisposable + class GridSettingsManager : + ViewModelBase, IValueFactory, IObserver, IDisposable where TRow : IRowVM where TPredicateBuilder : IPredicateBuilder, new() where TSettingsData : class, IGridSettingsWithNotifications { diff --git a/src/client/spark/view/Spark/Core/SparkUtils.cs b/src/client/spark/view/Spark/Core/SparkUtils.cs index fb620cb3a..3474ea79f 100644 --- a/src/client/spark/view/Spark/Core/SparkUtils.cs +++ b/src/client/spark/view/Spark/Core/SparkUtils.cs @@ -48,9 +48,6 @@ namespace mbts.spark { } #endregion - public static string display_name(this SecurityMD? security_md) => - (security_md?.security).display_name(); - public static string display_name(this SecurityImmutable? security) => security.HasValue ? security.Value.ToString() : "unknown security"; @@ -59,11 +56,11 @@ namespace mbts.spark { public static SecurityImmutable? security(this SecurityMD? security_md) => security_md.HasValue ? new SecurityImmutable?(security_md.Value.security) : new SecurityImmutable?(); - public static Venue exchange(this SecurityMD? security_md) => - security_md.HasValue ? security_md.Value.security.Exchange : Venue.Unset; + public static Venue exchange_or_default(this SecurityImmutable? security) => + security?.Exchange ?? Venue.Unset; - public static string symbol(this SecurityMD? security_md) => - security_md.HasValue ? security_md.Value.security.Symbol : "unknown"; + public static string symbol_or_default(this SecurityImmutable? security) => + security?.Symbol ?? "unknown"; public static async Task get_security_async(IBus bus, uint security_key) => await bus.subscribe(SparkTags.SecuritiesTag).Where(map => map.ContainsKey(security_key)).Select(map => map[security_key].security).FirstAsync(); diff --git a/src/client/spark/view/Spark/Ladder/LadderVM.cs b/src/client/spark/view/Spark/Ladder/LadderVM.cs index 4b7528808..034725b4a 100644 --- a/src/client/spark/view/Spark/Ladder/LadderVM.cs +++ b/src/client/spark/view/Spark/Ladder/LadderVM.cs @@ -134,8 +134,14 @@ namespace mbts.spark { // Here we manage N orders to 1 price slot in the ladder Dictionary> orders_used_in_this_update = new Dictionary>(_equality); - var ladders_orders = working.Values.Where(o => o.Def.SecurityKey == security.Id && (o.HasState && o.State.DisplayStatus == OrderState.Types.DisplayStatus.Visible)); - foreach (var wo in ladders_orders) { + var ladders_orders = + working.Values + .Where(o => + o.order.Def.SecurityKey == security.Id && + (o.order.HasState && o.order.State.DisplayStatus == OrderState.Types.DisplayStatus.Visible)); + + foreach (var osec in ladders_orders) { + var wo = osec.order; var slot_price_target = wo.price(); var order_type = wo.order_type(); var filled = wo.filled(); @@ -186,10 +192,9 @@ namespace mbts.spark { var traversal_candidate = only_order; bool exit_early = false; while (!exit_early && traversal_candidate.ParentKey != 0) { - var found_parent = working.Where(o => o.Key == traversal_candidate.ParentKey); - if (found_parent.Count() == 1) - traversal_candidate = found_parent.First().Value; - else + if (working.TryGetFirst(o => o.Key == traversal_candidate.ParentKey, out var first)) + traversal_candidate = first.Value.order; + else exit_early = true; } if (!exit_early) diff --git a/src/client/spark_api/api/Update.cs b/src/client/spark_api/api/Update.cs index b2585f8ad..e97fc196b 100644 --- a/src/client/spark_api/api/Update.cs +++ b/src/client/spark_api/api/Update.cs @@ -212,11 +212,13 @@ namespace bts.api.core { ImmutableDictionary init, IEnumerable added, IEnumerable> changed, - IEnumerable removed) { + IEnumerable removed, + Func maybe_key_extractor) { _state = init; this.added = added; this.changes = changed; this.removed = removed; + this.key_extractor = maybe_key_extractor; } public IReadOnlyDictionary state { get { return _state; } } @@ -233,12 +235,13 @@ namespace bts.api.core { foreach (var i in update.changed) { builder[update.key(i)] = i; } - return new TableUpdate(builder.ToImmutableDictionary(), update.added, update.changes, update.removed); + return new TableUpdate(builder.ToImmutableDictionary(), update.added, update.changes, update.removed, update.key_extractor); } internal TableUpdate as_snapshot() { - return new TableUpdate(_state, _state.Values.ToList(), new Change[0], new T[0]); + return new TableUpdate(_state, _state.Values.ToList(), new Change[0], new T[0], key_extractor); } + internal Func key_extractor { get; private set; } } internal class LegacyTableUpdate where TKey : IEquatable { @@ -287,7 +290,7 @@ namespace bts.api.core { public static IObservable> Table(this IObservable> source) where TKey : IEquatable => source.Scan( - new TableUpdate(ImmutableDictionary.Empty, new T[0], new Change[0], new T[0]), + new TableUpdate(ImmutableDictionary.Empty, new T[0], new Change[0], new T[0], null), (table, update) => TableUpdate.update(table,update)); // Fold updates and emit at max 1/interval @@ -450,8 +453,9 @@ namespace bts.api.core { } cache = builder.ToImmutable(); - return new TableUpdate(cache, added, changes, removed); + return new TableUpdate(cache, added, changes, removed, key); }); } + } } \ No newline at end of file diff --git a/src/client/spark_core/Collections/Extensions/DictionaryExtensions.cs b/src/client/spark_core/Collections/Extensions/DictionaryExtensions.cs index 5f7576ad8..415cb0ef3 100644 --- a/src/client/spark_core/Collections/Extensions/DictionaryExtensions.cs +++ b/src/client/spark_core/Collections/Extensions/DictionaryExtensions.cs @@ -9,6 +9,14 @@ namespace System.Collections.Generic { else update_value(key, value); } + public static bool TryAdd(this IDictionary dict, TKey key, TValue value) { + if (dict.ContainsKey(key)) return false; + else { + dict[key] = value; + return true; + } + } + public static void AddOrUpdate(this IDictionary dictionary, TKey key, Func add_value_factory, Func update_value) { if (!dictionary.TryGetValue(key, out TValue value)) dictionary.Add(key, add_value_factory(key)); @@ -77,5 +85,20 @@ namespace System.Collections.Generic { /// true if the dictionary contains an element with the specified key, AND returns true; otherwise, false. public static bool TryPopValueIf(this IDictionary dictionary, TKey key, Func should_pop) => dictionary.TryGetValue(key, out TValue result) && should_pop(result) && dictionary.Remove(key); + + + public static T? TryGetValue(this IDictionary src, TKey key) where T : struct { + if (src.TryGetValue(key, out T value)) { + return value; + } + return null; + } + public static T? TryGetValue(this IReadOnlyDictionary src, TKey key) where T : struct { + if (src.TryGetValue(key, out T value)) { + return value; + } + return null; + } + } } \ No newline at end of file diff --git a/src/client/spark_core/utils/FilteredDictionary.cs b/src/client/spark_core/utils/FilteredDictionary.cs new file mode 100644 index 000000000..9ca45bbc4 --- /dev/null +++ b/src/client/spark_core/utils/FilteredDictionary.cs @@ -0,0 +1,86 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; + +namespace mbts.utils { + public class FilteredDictionary : IReadOnlyDictionary { + readonly IReadOnlyDictionary _dict; + readonly Func _filter; + int? _count; + + public FilteredDictionary(IReadOnlyDictionary src, Func filter) { + _dict = src; + _filter = filter; + _count = null; + } + + public T this[TKey key] { + get { + var result = _dict[key]; + if (_filter(result)) { + return result; + } + throw new KeyNotFoundException(key.ToString()); + } + } + + public IEnumerable Keys => _dict.Where(kvp => _filter(kvp.Value)).Select(kvp => kvp.Key); + + public IEnumerable Values => _dict.Values.Where(_filter); + + public int Count { + get { + if (_count.HasValue) return _count.Value; + _count = Values.Count(); + return _count.Value; + } + } + + public bool ContainsKey(TKey key) => Keys.Contains(key); + + public IEnumerator> GetEnumerator() { + return new FilteredEnumerator>(_dict.GetEnumerator(), kvp => _filter(kvp.Value)); + } + + public bool TryGetValue(TKey key, out T value) { + if (_dict.TryGetValue(key, out var val)) { + if (_filter(val)) { + value = val; + return true; + } + } + value = default; + return false; + } + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } + + public class FilteredEnumerator : IEnumerator { + readonly Func _filter; + readonly IEnumerator _src; + + public FilteredEnumerator(IEnumerator src, Func filter) { + _filter = filter; + _src = src; + } + public T Current => _src.Current; + + object IEnumerator.Current => Current; + + public void Dispose() => _src.Dispose(); + + public bool MoveNext() { + do { + var ok = _src.MoveNext(); + if (!ok) return false; + } while (!_filter(_src.Current)); + return true; + } + + public void Reset() { _src.MoveNext(); } + } + + +} \ No newline at end of file diff --git a/src/client/spark_core/utils/MappedDictionary.cs b/src/client/spark_core/utils/MappedDictionary.cs new file mode 100644 index 000000000..904c69252 --- /dev/null +++ b/src/client/spark_core/utils/MappedDictionary.cs @@ -0,0 +1,64 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; + +namespace mbts.utils { + internal class MappedDictionary : IReadOnlyDictionary { + readonly IReadOnlyDictionary _dict; + readonly Func _map; + + public MappedDictionary(IReadOnlyDictionary src, Func map) { + _dict = src; + _map = map; + } + + public TRes this[TKey key] => _map(_dict[key]); + + public IEnumerable Keys => _dict.Keys; + + public IEnumerable Values => _dict.Values.Select(_map); + + public int Count => _dict.Count; + + public bool ContainsKey(TKey key) => _dict.ContainsKey(key); + + public IEnumerator> GetEnumerator() { + return new MappedEnumerator, KeyValuePair>( + _dict.GetEnumerator(), + kvp => new KeyValuePair(kvp.Key, _map(kvp.Value))); + } + + public bool TryGetValue(TKey key, out TRes value) { + if (_dict.TryGetValue(key, out TSrc src)) { + value = _map(src); + return true; + } + value = default; + return false; + } + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } + + public class MappedEnumerator : IEnumerator { + readonly IEnumerator _src; + readonly Func _map; + + public MappedEnumerator(IEnumerator src, Func map) { + _src = src; + _map = map; + } + + public TRes Current => _map(_src.Current); + + object IEnumerator.Current => Current; + + public void Dispose() { _src.Dispose(); } + + public bool MoveNext() => _src.MoveNext(); + + public void Reset() { _src.Reset(); } + } + +} \ No newline at end of file From c7b890eeb0fd14d832a71e3b3fdba65ff04ab115 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 28 Jan 2019 09:41:55 -0500 Subject: [PATCH 33/85] Add tests for Filtered/Mapped dictionary types --- .../Modules/Trading/OrderBook/OrderBookVM.cs | 3 +- src/client/spark/Orders/OrdersModel.cs | 4 +- .../core/Utility/TableUpdateTransform.cs | 1 + .../spark_core/TestRunner/TestAssert.cs | 20 +++++ .../spark_core/utils/FilteredDictionary.cs | 2 +- .../spark_core/utils/MappedDictionary.cs | 2 +- .../Utils/FilteredDictionaryTests.cs | 85 +++++++++++++++++++ .../spark_test/Utils/MappedDictionaryTests.cs | 71 ++++++++++++++++ 8 files changed, 183 insertions(+), 5 deletions(-) create mode 100644 src/client/spark_test/Utils/FilteredDictionaryTests.cs create mode 100644 src/client/spark_test/Utils/MappedDictionaryTests.cs diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs index 1b4a5afbf..534bf0a02 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs @@ -43,7 +43,8 @@ namespace mbts.spark { _bus = bus; _model = model; _settings_manager = settings_manager; - _columns = new OrderBookColumnCollection(bus, model, model.on_order_securities.Select(u => (u.added.Values.Select(smd => smd.security), u.removed.Values.Select(smd => smd.security)))); + _columns = new OrderBookColumnCollection(bus, model, + model.on_order_securities.Select(x => (x.added.Values, x.removed.Values))); _row_collection = new OrderBookRowCollection(ort => new OrderBookRowVM(this, ort), _columns.get_sorting_data().comparer) { filter = _columns.get_filter() }; _current_row = backing(nameof(current_row)); _columns.filter_changed += on_filter_changed; diff --git a/src/client/spark/Orders/OrdersModel.cs b/src/client/spark/Orders/OrdersModel.cs index bbf337718..c5892f4a4 100644 --- a/src/client/spark/Orders/OrdersModel.cs +++ b/src/client/spark/Orders/OrdersModel.cs @@ -62,8 +62,8 @@ namespace mbts.spark { .Replay(1).RefCount(); on_order_securities = - on_orders. - as_iincremental() + on_orders + .as_iincremental() .filter(t => t.sec.HasValue) .Select(t => t.sec.Value); diff --git a/src/client/spark/core/Utility/TableUpdateTransform.cs b/src/client/spark/core/Utility/TableUpdateTransform.cs index fbd3da11f..c733d1202 100644 --- a/src/client/spark/core/Utility/TableUpdateTransform.cs +++ b/src/client/spark/core/Utility/TableUpdateTransform.cs @@ -1,4 +1,5 @@ using bts.api.core; +using bts.collections; using mbts.collections; using System; using System.Collections.Generic; diff --git a/src/client/spark_core/TestRunner/TestAssert.cs b/src/client/spark_core/TestRunner/TestAssert.cs index 5bc09ae02..0306073cc 100644 --- a/src/client/spark_core/TestRunner/TestAssert.cs +++ b/src/client/spark_core/TestRunner/TestAssert.cs @@ -65,5 +65,25 @@ namespace bts.test { $"Expected: {item} in [{aa}]. {msg}", file, line); } } + + public static void throws(Action runme, string msg="", [CallerFilePath] string file="", [CallerLineNumber] int line=0) where T : Exception { + bool threw = false; + try { + runme(); + } + catch (T) { + threw = true; + } + catch (Exception e) { + throw new AssertFailture( + $"Expected '{typeof(T).Name}' but threw '{e.GetType().Name}'", file, line); + } + + if (!threw) { + throw new AssertFailture( + msg ?? $"Expected '{typeof(T).Name}' exception but ran successfully.", file, line); + } + + } } } \ No newline at end of file diff --git a/src/client/spark_core/utils/FilteredDictionary.cs b/src/client/spark_core/utils/FilteredDictionary.cs index 9ca45bbc4..cb0ca57f1 100644 --- a/src/client/spark_core/utils/FilteredDictionary.cs +++ b/src/client/spark_core/utils/FilteredDictionary.cs @@ -3,7 +3,7 @@ using System.Collections; using System.Collections.Generic; using System.Linq; -namespace mbts.utils { +namespace bts.collections { public class FilteredDictionary : IReadOnlyDictionary { readonly IReadOnlyDictionary _dict; readonly Func _filter; diff --git a/src/client/spark_core/utils/MappedDictionary.cs b/src/client/spark_core/utils/MappedDictionary.cs index 904c69252..d66ce775f 100644 --- a/src/client/spark_core/utils/MappedDictionary.cs +++ b/src/client/spark_core/utils/MappedDictionary.cs @@ -3,7 +3,7 @@ using System.Collections; using System.Collections.Generic; using System.Linq; -namespace mbts.utils { +namespace bts.collections { internal class MappedDictionary : IReadOnlyDictionary { readonly IReadOnlyDictionary _dict; readonly Func _map; diff --git a/src/client/spark_test/Utils/FilteredDictionaryTests.cs b/src/client/spark_test/Utils/FilteredDictionaryTests.cs new file mode 100644 index 000000000..8a99f1c7a --- /dev/null +++ b/src/client/spark_test/Utils/FilteredDictionaryTests.cs @@ -0,0 +1,85 @@ +using bts.collections; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace bts.test { + + [TestGroup] + public class FilteredDictionaryTests { + IEnumerable range = Enumerable.Range(5,6); + Dictionary source; + IReadOnlyDictionary filtered; + + public FilteredDictionaryTests() { + // Create a source dictionary + source = Enumerable.Range(1, 10).ToDictionary(x => x); + // create a filtered dictionary + filtered = new FilteredDictionary(source, x => x >= 5); + } + + void ignore(T t) { } + + [Test] + public void index() { + foreach (var i in Enumerable.Range(-10,20)) { + if (range.Contains(i)) { + Assert.AreEqual(i, filtered[i]); + } + else { + Assert.throws(() => ignore(filtered[i])); + } + } + } + [Test] + public void keys() { + Assert.SequenceEqual(range, filtered.Keys); + } + [Test] + public void values() { + Assert.SequenceEqual(range, filtered.Values); + } + [Test] + public void count() { + Assert.AreEqual(range.Count(), filtered.Count); + Assert.AreEqual(range.Count(), filtered.Count()); + } + [Test] + public void contains_key() { + foreach (var i in Enumerable.Range(-10, 20)) { + if (range.Contains(i)) { + Assert.IsTrue(filtered.ContainsKey(i)); + } + else { + Assert.IsFalse(filtered.ContainsKey(i)); + } + } + } + [Test] + public void enumerator() { + Assert.SequenceEqual( + range.Select(i => new KeyValuePair(i, i)), filtered); + } + [Test] + public void try_get_value() { + foreach (var i in Enumerable.Range(-10, 20)) { + if (range.Contains(i)) { + Assert.IsTrue(filtered.TryGetValue(i, out var _)); + } + else { + Assert.IsFalse(filtered.TryGetValue(i, out var _)); + } + } + } + + [Test] + public void FilterAndMap() { + string transform(int i) => (i * i).ToString(); + var mapped = new MappedDictionary(filtered, transform); + Assert.SequenceEqual( + range.Select(transform), + mapped.Values); + + } + } +} diff --git a/src/client/spark_test/Utils/MappedDictionaryTests.cs b/src/client/spark_test/Utils/MappedDictionaryTests.cs new file mode 100644 index 000000000..e426de542 --- /dev/null +++ b/src/client/spark_test/Utils/MappedDictionaryTests.cs @@ -0,0 +1,71 @@ +using bts.collections; +using System.Collections.Generic; +using System.Linq; + +namespace bts.test { + + [TestGroup] + public class MappedDictionaryTests { + IEnumerable range = Enumerable.Range(1, 10); + Dictionary source; + IReadOnlyDictionary mapped; + + public MappedDictionaryTests() { + // Create a source dictionary + source = range.ToDictionary(x => x); + // create a filtered dictionary + mapped = new MappedDictionary(source, x => x.ToString()); + } + + [Test] + public void index() { + foreach(var i in range) { + Assert.AreEqual(i.ToString(), mapped[i]); + } + } + [Test] + public void keys() { + Assert.SequenceEqual(range, mapped.Keys); + } + [Test] + public void values() { + Assert.SequenceEqual(range.Select(x => x.ToString()), mapped.Values); + } + [Test] + public void count() { + Assert.AreEqual(range.Count(), mapped.Count); + } + [Test] + public void contains_key() { + foreach (var key in range) { + Assert.IsTrue(mapped.ContainsKey(key)); + } + Assert.IsFalse(mapped.ContainsKey(range.Max() + 1)); + Assert.IsFalse(mapped.ContainsKey(range.Min() - 1)); + } + [Test] + public void enumerator() { + Assert.SequenceEqual(range, mapped.Select(kvp => kvp.Key)); + Assert.SequenceEqual( + range.Select(x => x.ToString()), + mapped.Select(kvp => kvp.Value)); + } + [Test] + public void try_get_value() { + foreach (var i in Enumerable.Range(-20, 20)) { + Assert.AreEqual( + range.Contains(i), + mapped.TryGetValue(i, out string _)); + } + } + [Test] + public void MapAndFilter() { + var set = new HashSet(new[] { "4", "5" }); + var filtered = new FilteredDictionary(mapped, set.Contains); + Assert.AreEqual(2, filtered.Count); + Assert.SequenceEqual( + Enumerable.Range(4, 2).Select(i => (i, i.ToString())), + filtered.Select(kvp => (kvp.Key, kvp.Value))); + } + } +} From 5cc408557727828ad09536b625dbf1f1cb6fc7ff Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 28 Jan 2019 16:50:27 -0500 Subject: [PATCH 34/85] Try to ensure Dispatcher::background is terminated --- src/lib/utils/init_spark_lib.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib/utils/init_spark_lib.cpp b/src/lib/utils/init_spark_lib.cpp index d9dc6fab9..772d62c7a 100644 --- a/src/lib/utils/init_spark_lib.cpp +++ b/src/lib/utils/init_spark_lib.cpp @@ -2,6 +2,7 @@ #include "lib/logging/logging.h" #include "lib/utils/bts_assert.h" #include "lib/utils/thread_manager.h" +#include "lib/utils/Dispatcher.h" namespace bts { namespace utils { @@ -20,6 +21,7 @@ void finit_spark_lib() if (poller) { delete poller; } + utils::Dispatcher::background().terminate(); } }} From 51704be22a870565c97f6bd36bb9215592325b09 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 28 Jan 2019 17:27:42 -0500 Subject: [PATCH 35/85] Avoid creating task that does no work. --- src/client/spark/Security/SecurityStore.cs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/client/spark/Security/SecurityStore.cs b/src/client/spark/Security/SecurityStore.cs index 3b0f53794..731ca169f 100644 --- a/src/client/spark/Security/SecurityStore.cs +++ b/src/client/spark/Security/SecurityStore.cs @@ -112,14 +112,16 @@ namespace bts { } } // Update cache in the background - Task.Run(async () => { - var r = await _md.get_securities_async(missing); - if (r.success) { - foreach (var security in r.value.Security) { - try_cache_security(new SecurityImmutable(security)); + if (missing.Keys.Count > 0) { + Task.Run(async () => { + var r = await _md.get_securities_async(missing); + if (r.success) { + foreach (var security in r.value.Security) { + try_cache_security(new SecurityImmutable(security)); + } } - } - }); + }); + } return result; } From 6b3e2f396d4840c563330a23ff4b36a0f25aa283 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 28 Jan 2019 17:27:53 -0500 Subject: [PATCH 36/85] Update ReplayRecorder version --- src/lib/md/relay/RelayRecorder.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/lib/md/relay/RelayRecorder.cpp b/src/lib/md/relay/RelayRecorder.cpp index a6f453e73..6b6d3f020 100644 --- a/src/lib/md/relay/RelayRecorder.cpp +++ b/src/lib/md/relay/RelayRecorder.cpp @@ -26,6 +26,7 @@ namespace bts { namespace md { const char* magic = "Co2"; +const uint32_t recorder_version = 3; template void writebuf(ofstream& out, const T& pb) { @@ -61,7 +62,7 @@ RelayRecorder::RelayRecorder(const std::string& base_path, const std::string& fi CoconutReplayHeader header; header.set_magic(magic); - header.set_version(3); + header.set_version(recorder_version); header.set_timestamp(datetime::ll_now()); header.set_host(config::get().config_host()); header.set_naggle_ms(naggle); @@ -132,7 +133,7 @@ void replay_recording(istream& in, IRelayWriter& writer, int32_t naggle_ms) { #ifndef _WIN32 md::SecurityIDRecorder idrec {{}}; #endif - if (readbuf(in, &header) && header.magic() == magic && header.version() == 3) { + if (readbuf(in, &header) && header.magic() == magic && header.version() == recorder_version) { co2::MDReply message; while (readbuf(in, &message)) { bool has_securities = false; @@ -226,7 +227,7 @@ void unprint_recording(const std::string& src, const std::string& dest) { // Write out the header CoconutReplayHeader header; header.set_magic(magic); - header.set_version(2); + header.set_version(recorder_version); header.set_timestamp(datetime::ll_now()); header.set_host(config::get().host_name()); writebuf(out, header); From ab1fe94b88da7b1c994f8a81c6bcdb4a7b93fcd2 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 30 Jan 2019 10:40:23 -0500 Subject: [PATCH 37/85] SPARK-1955 Introduce MDClient and SecurityOracle types --- src/lib/md/MDClient.cpp | 74 +++++++++++++++++++++++++++++++++++ src/lib/md/MDClient.h | 26 ++++++++++++ src/lib/md/SecurityOracle.cpp | 31 +++++++++++++++ src/lib/md/SecurityOracle.h | 19 +++++++++ 4 files changed, 150 insertions(+) create mode 100644 src/lib/md/MDClient.cpp create mode 100644 src/lib/md/MDClient.h create mode 100644 src/lib/md/SecurityOracle.cpp create mode 100644 src/lib/md/SecurityOracle.h diff --git a/src/lib/md/MDClient.cpp b/src/lib/md/MDClient.cpp new file mode 100644 index 000000000..0463f6810 --- /dev/null +++ b/src/lib/md/MDClient.cpp @@ -0,0 +1,74 @@ +#include "MDClient.h" +#include "get_symbols.h" +#include + +namespace bts { +namespace md { + +struct MDClientImpl{ + utils::Dispatcher& _dispatcher; + std::shared_ptr _ss; + utils::DispatcherTimer _security_timer; + utils::DispatcherTimer _recovery_timer; + MDConsumer _consumer; + utils::Event _event; + + MDClientImpl(utils::Dispatcher& d, std::shared_ptr s): + _dispatcher(d), + _ss(s), + _security_timer(d, config::get().security_scan_interval(),[this]() { scan_securities(); }), + _recovery_timer(d, config::get().recovery_interval(), [this]() { check_heartbeats(); }), + _consumer(s,true), + _event() + { + _security_timer.start(); + _recovery_timer.start(); + } + + std::set symbols() { + std::set all_symbols; + Venue venue; + bts_assert(api::Venue_Parse(config::get().exchange(), &venue)); + SecurityType type; + bts_assert(api::SecurityType_Parse(config::get().type(), &type)); + get_system_md_symbols(venue, type, all_symbols); + return all_symbols; + } + + void check_heartbeats() { + if (!_consumer.all_heartbeats_ok()) { + MDConsumer temp(symbols(), _ss); + if (temp.all_heartbeats_ok()) { + _consumer.swap(temp); + LOG(md,Info, "Recovering from bad MD. Old consumer {} new consumer {}", _consumer.id(), temp.id()); + // Subscribers are responsible for keeping the old consumer reference if they need it + _event.publish(_consumer); + } + } + } + + void scan_securities() { + if (_consumer.all_heartbeats_ok()) { + _consumer.scan_securities(); + } + } +}; + + +MDClient::MDClient(utils::Dispatcher& d, std::shared_ptr s) : _self(new MDClientImpl(d,s)) { + +} +MDClient::~MDClient() { +} + +utils::EventSubscriber& MDClient::on_consumer() { + return _self->_event; +} +MDConsumer MDClient::current_consumer() const { + return _self->_consumer; +} +SecurityOracle MDClient::oracle() { + return SecurityOracle(*this); +} + +}}//namespace \ No newline at end of file diff --git a/src/lib/md/MDClient.h b/src/lib/md/MDClient.h new file mode 100644 index 000000000..b82549ae7 --- /dev/null +++ b/src/lib/md/MDClient.h @@ -0,0 +1,26 @@ +#pragma once +#include "MDConsumer.h" +#include "lib/utils/Dispatcher.h" +#include "lib/utils/Event.h" +#include "ISyntheticSecurityService.h" +#include "SecurityOracle.h" +#include + +namespace bts { +namespace md { + +struct MDClientImpl; +class MDClient { + std::unique_ptr _self; + +public: + MDClient(utils::Dispatcher&, std::shared_ptr); + ~MDClient(); + + utils::EventSubscriber& on_consumer(); + MDConsumer current_consumer() const; + SecurityOracle oracle(); +}; + + +}}//namespace \ No newline at end of file diff --git a/src/lib/md/SecurityOracle.cpp b/src/lib/md/SecurityOracle.cpp new file mode 100644 index 000000000..4d49affe6 --- /dev/null +++ b/src/lib/md/SecurityOracle.cpp @@ -0,0 +1,31 @@ +#include "SecurityOracle.h" +#include "MDClient.h" + +namespace bts { +namespace md { + +struct SecurityOracleImpl { + SecurityOracleImpl(MDClient& client): + client(client), + consumer(client.current_consumer()) + { + sub = client.on_consumer().subscribe([this](const MDConsumer& c) { + consumer = c; + }); + } + ~SecurityOracleImpl() { + client.on_consumer().unsubscribe(sub); + } + MDClient& client; + MDConsumer consumer; + int sub; + +}; + +SecurityOracle::SecurityOracle(MDClient& client): _self(new SecurityOracleImpl(client)) { } +SecurityOracle::~SecurityOracle() { } +SecurityPtr SecurityOracle::get_security(SecurityKey key) const { + return _self->consumer.get_security(key); +} + +}}//namespace \ No newline at end of file diff --git a/src/lib/md/SecurityOracle.h b/src/lib/md/SecurityOracle.h new file mode 100644 index 000000000..fc9ec3ea6 --- /dev/null +++ b/src/lib/md/SecurityOracle.h @@ -0,0 +1,19 @@ +#pragma once +#include "SecurityPtr.h" +#include + +namespace bts { +namespace md { + +class MDClient; + +struct SecurityOracleImpl; +class SecurityOracle { + std::unique_ptr _self; +public: + SecurityOracle(MDClient&); + ~SecurityOracle(); + SecurityPtr get_security(SecurityKey key) const; +}; + +}}//namespace \ No newline at end of file From 791b0fdbc1916928f7e2cb8f450f1a8c681496c4 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 31 Jan 2019 09:18:03 -0500 Subject: [PATCH 38/85] Makd SecurityCache external to MDConsumer so it live beyond md recovery --- src/lib/md/Diag/MDPeriodicDumper.cpp | 4 +- src/lib/md/MDClient.cpp | 6 ++- src/lib/md/MDConsumer.cpp | 23 ++++++------ src/lib/md/MDConsumer.h | 12 +++--- src/lib/md/MDRecoverableConsumer.cpp | 2 +- src/lib/md/MDRecoverableConsumer.h | 5 ++- src/lib/md/SecurityCache.cpp | 2 +- src/lib/md/SecurityCache.h | 2 +- src/lib/md/SecurityOracle.cpp | 2 +- src/lib/md/SecurityOracle.h | 2 +- src/lib/md/relay/QueryServer.cpp | 8 +++- src/lib/md/test/md2_tests.cpp | 37 +++++++++++++------ .../orderman/server/OrderServiceServer.cpp | 3 +- 13 files changed, 68 insertions(+), 40 deletions(-) diff --git a/src/lib/md/Diag/MDPeriodicDumper.cpp b/src/lib/md/Diag/MDPeriodicDumper.cpp index bc964116c..05cdadb06 100644 --- a/src/lib/md/Diag/MDPeriodicDumper.cpp +++ b/src/lib/md/Diag/MDPeriodicDumper.cpp @@ -1,5 +1,6 @@ #include "MDPeriodicDumper.h" #include "lib/md/MDConsumer.h" +#include "lib/md/SecurityCache.h" #include "lib/utils/TimeStamp.h" #include "lib/utils/Dispatcher.h" #include "lib/utils/datetime.h" @@ -158,7 +159,8 @@ void run_periodic_dumper(const std::set& symbols, uint32_t seq_mod, auto thread_name = "tail-" + qualifier; threading::configure_current_thread(thread_name); - MDConsumer consumer(symbols, nullptr); + SecurityCache cache; + MDConsumer consumer(symbols, nullptr, cache); // read all securities into a vector std::vector entries; diff --git a/src/lib/md/MDClient.cpp b/src/lib/md/MDClient.cpp index 0463f6810..a0f5313b0 100644 --- a/src/lib/md/MDClient.cpp +++ b/src/lib/md/MDClient.cpp @@ -1,5 +1,6 @@ #include "MDClient.h" #include "get_symbols.h" +#include "SecurityCache.h" #include namespace bts { @@ -8,6 +9,7 @@ namespace md { struct MDClientImpl{ utils::Dispatcher& _dispatcher; std::shared_ptr _ss; + SecurityCache _cache; utils::DispatcherTimer _security_timer; utils::DispatcherTimer _recovery_timer; MDConsumer _consumer; @@ -18,7 +20,7 @@ struct MDClientImpl{ _ss(s), _security_timer(d, config::get().security_scan_interval(),[this]() { scan_securities(); }), _recovery_timer(d, config::get().recovery_interval(), [this]() { check_heartbeats(); }), - _consumer(s,true), + _consumer(s, _cache), _event() { _security_timer.start(); @@ -37,7 +39,7 @@ struct MDClientImpl{ void check_heartbeats() { if (!_consumer.all_heartbeats_ok()) { - MDConsumer temp(symbols(), _ss); + MDConsumer temp(_ss,_cache); if (temp.all_heartbeats_ok()) { _consumer.swap(temp); LOG(md,Info, "Recovering from bad MD. Old consumer {} new consumer {}", _consumer.id(), temp.id()); diff --git a/src/lib/md/MDConsumer.cpp b/src/lib/md/MDConsumer.cpp index 449bfe420..d01908032 100644 --- a/src/lib/md/MDConsumer.cpp +++ b/src/lib/md/MDConsumer.cpp @@ -537,20 +537,21 @@ class MDConsumerImpl { std::shared_ptr _sci; map> _deferred_handles; vector _callbacks; - mutable SecurityCache _cache; + SecurityCache& _cache; // co2::Platform _platform; public: MDConsumerImpl( const set& symbols, std::shared_ptr& ss, + SecurityCache& cache, bool shared, co2::Platform platform = co2::PLATFORM_UNSET) : _workmutex(), _instances(), _sci(), _deferred_handles(), _callbacks(), - _cache() + _cache(cache) { std::ostringstream msg; for (const auto& symbol : symbols) { @@ -906,27 +907,27 @@ public: // MDConsumer public interface, forwards to pimple /////////////////////////////////////////////////////////////////////////////////////////////////// -MDConsumer::MDConsumer(std::shared_ptr& synt, bool shared) : _pimpl() { +MDConsumer::MDConsumer(std::shared_ptr& synt, SecurityCache& cache, bool shared) : _pimpl() { set all_symbols; get_system_md_symbols(VENUE_UNSET, SECURITYTYPE_UNSET, all_symbols); - _pimpl.reset(new MDConsumerImpl(all_symbols, synt, shared, co2::PLATFORM_UNSET)); + _pimpl.reset(new MDConsumerImpl(all_symbols, synt, cache, shared, co2::PLATFORM_UNSET)); } -MDConsumer::MDConsumer(const UserSymbol& s, std::shared_ptr synt, bool shared) : +MDConsumer::MDConsumer(const UserSymbol& s, std::shared_ptr synt, SecurityCache& cache, bool shared) : _pimpl() { set symbols; symbols.insert(s); - _pimpl.reset(new MDConsumerImpl(symbols, synt, shared, co2::PLATFORM_UNSET)); + _pimpl.reset(new MDConsumerImpl(symbols, synt, cache, shared, co2::PLATFORM_UNSET)); } -MDConsumer::MDConsumer(set const& symbols, std::shared_ptr s, bool shared, co2::Platform platform) : - _pimpl(new MDConsumerImpl(symbols, s, shared, platform)) {} +MDConsumer::MDConsumer(set const& symbols, std::shared_ptr s, SecurityCache& cache, bool shared, co2::Platform platform) : + _pimpl(new MDConsumerImpl(symbols, s, cache, shared, platform)) {} -MDConsumer::MDConsumer(api::Venue ven, co2::Platform platform) : _pimpl() { +MDConsumer::MDConsumer(SecurityCache& cache, api::Venue ven, co2::Platform platform) : _pimpl() { set all_symbols; get_system_md_symbols(ven, SECURITYTYPE_UNSET, all_symbols); std::shared_ptr null = nullptr; - _pimpl.reset(new MDConsumerImpl(all_symbols, null, true, platform)); + _pimpl.reset(new MDConsumerImpl(all_symbols, null, cache, true, platform)); } MDConsumer::~MDConsumer() {} @@ -944,7 +945,7 @@ size_t MDConsumer::walk_security_list(SecurityCallback cb, const string& symbol, size_t MDConsumer::scan_securities(bool resolve_legs) { return _pimpl->scan_securities(resolve_legs); } void MDConsumer::on_security(SecurityCallback cb, bool resolve_legs) { _pimpl->on_security(cb, resolve_legs); } -SecurityPtr MDConsumer::get_security(SecurityKey key) const { return _pimpl->get_security(key); } +const api::Security* MDConsumer::get_security(SecurityKey key) const { return _pimpl->get_security(key).get(); } MDConsumerChannel MDConsumer::get_channel(const std::string& symbol, SecurityType type, Venue exchange) const { return _pimpl->get_channel(symbol, type, exchange); diff --git a/src/lib/md/MDConsumer.h b/src/lib/md/MDConsumer.h index a53b33665..dff515edc 100644 --- a/src/lib/md/MDConsumer.h +++ b/src/lib/md/MDConsumer.h @@ -25,15 +25,17 @@ using std::size_t; class MDConsumerImpl; class MDConsumerInst; class MDConsumerChannel; +class SecurityCache; + class MDConsumer { public: // Consumer for all symbols - MDConsumer(std::shared_ptr&, bool shared=true); - MDConsumer(const UserSymbol&, std::shared_ptr, bool shared=true); + MDConsumer(std::shared_ptr&, SecurityCache&, bool shared=true); + MDConsumer(const UserSymbol&, std::shared_ptr, SecurityCache&, bool shared=true); // Consumer for a given set of symbols - MDConsumer(std::set const& symbols, std::shared_ptr, bool shared=true, co2::Platform platform=co2::PLATFORM_UNSET); - MDConsumer(api::Venue, co2::Platform); + MDConsumer(std::set const& symbols, std::shared_ptr, SecurityCache&, bool shared=true, co2::Platform platform=co2::PLATFORM_UNSET); + MDConsumer(SecurityCache&, api::Venue, co2::Platform); ~MDConsumer(); bool all_heartbeats_ok() const; @@ -53,7 +55,7 @@ public: // Retrieve any security that has been seen in the past // You still need to call scan_securities() periodically to discover new securities - SecurityPtr get_security(SecurityKey key) const; + const api::Security* get_security(SecurityKey key) const; MDConsumerChannel get_channel(const std::string& symbol, SecurityType type, Venue exchange) const; MDConsumerChannel get_channel(const UserSymbol&) const; diff --git a/src/lib/md/MDRecoverableConsumer.cpp b/src/lib/md/MDRecoverableConsumer.cpp index 9e6250388..5b5378033 100644 --- a/src/lib/md/MDRecoverableConsumer.cpp +++ b/src/lib/md/MDRecoverableConsumer.cpp @@ -45,7 +45,7 @@ MDRecoverableConsumer::MDRecoverableConsumer(std::set symbols, std:: MDRecoverableConsumer::~MDRecoverableConsumer() {} unique_ptr MDRecoverableConsumer::init(std::set& symbols, std::shared_ptr& ss) { - return std::make_unique(symbols, ss); + return std::make_unique(symbols, ss, _cache); } size_t MDRecoverableConsumer::walk_security_list(SecurityCallback callback) { diff --git a/src/lib/md/MDRecoverableConsumer.h b/src/lib/md/MDRecoverableConsumer.h index 95068587d..c773b5d4d 100644 --- a/src/lib/md/MDRecoverableConsumer.h +++ b/src/lib/md/MDRecoverableConsumer.h @@ -5,6 +5,7 @@ #include "MDConsumer.h" #include "MDConsumerHandle.h" #include "UserSymbol.h" +#include "SecurityCache.h" #include #include #include @@ -45,7 +46,8 @@ private: struct MDConsumerGroup { MDConsumerGroup( std::set& symbols, - std::shared_ptr& ss) : consumer(symbols, ss), handles() {} + std::shared_ptr& ss, + SecurityCache& cache) : consumer(symbols, ss, cache), handles() {} MDConsumer consumer; std::vector> handles; }; @@ -53,6 +55,7 @@ private: std::set _symbols; std::shared_ptr _ss; + SecurityCache _cache; std::unique_ptr _current; std::vector> _old; diff --git a/src/lib/md/SecurityCache.cpp b/src/lib/md/SecurityCache.cpp index affe44c9e..6d5fd1b43 100644 --- a/src/lib/md/SecurityCache.cpp +++ b/src/lib/md/SecurityCache.cpp @@ -37,7 +37,7 @@ void SecurityCacheImpl::flush() { ////////////////// -SecurityCache::SecurityCache() : _me(new SecurityCacheImpl()) {} +SecurityCache::SecurityCache() : _me(std::make_shared()) {} SecurityCache::~SecurityCache() {} SecurityPtr SecurityCache::try_get_security(SecurityKey key) const { diff --git a/src/lib/md/SecurityCache.h b/src/lib/md/SecurityCache.h index 85de41b70..e4c2ef2c4 100644 --- a/src/lib/md/SecurityCache.h +++ b/src/lib/md/SecurityCache.h @@ -13,7 +13,7 @@ namespace md { // Store security+handle struct SecurityCacheImpl; class SecurityCache { - std::unique_ptr _me; + std::shared_ptr _me; public: SecurityCache(); diff --git a/src/lib/md/SecurityOracle.cpp b/src/lib/md/SecurityOracle.cpp index 4d49affe6..6a7f81ea1 100644 --- a/src/lib/md/SecurityOracle.cpp +++ b/src/lib/md/SecurityOracle.cpp @@ -24,7 +24,7 @@ struct SecurityOracleImpl { SecurityOracle::SecurityOracle(MDClient& client): _self(new SecurityOracleImpl(client)) { } SecurityOracle::~SecurityOracle() { } -SecurityPtr SecurityOracle::get_security(SecurityKey key) const { +const api::Security* SecurityOracle::get_security(SecurityKey key) const { return _self->consumer.get_security(key); } diff --git a/src/lib/md/SecurityOracle.h b/src/lib/md/SecurityOracle.h index fc9ec3ea6..da0210f18 100644 --- a/src/lib/md/SecurityOracle.h +++ b/src/lib/md/SecurityOracle.h @@ -13,7 +13,7 @@ class SecurityOracle { public: SecurityOracle(MDClient&); ~SecurityOracle(); - SecurityPtr get_security(SecurityKey key) const; + const api::Security* get_security(SecurityKey key) const; }; }}//namespace \ No newline at end of file diff --git a/src/lib/md/relay/QueryServer.cpp b/src/lib/md/relay/QueryServer.cpp index 6c3e2e457..783c355e6 100644 --- a/src/lib/md/relay/QueryServer.cpp +++ b/src/lib/md/relay/QueryServer.cpp @@ -13,6 +13,7 @@ #include "lib/config/protos/system.pb.h" #include "lib/md/protos/co2.pb.h" +#include "lib/md/SecurityCache.h" #include "lib/api/apicore.pb.h" #include "lib/api/services/MarketDataRelay.server.pb.h" #include "lib/api/public/ServiceContext.h" @@ -162,6 +163,7 @@ void copy_misc(IntPrice settlement, MiscData misc, co2::MiscMessage* pmisc) { // Handles market data queries and responds class QueryServerImpl : public IQueryServer { + SecurityCache _cache; MDConsumer _consumer; const std::set _symbols; const std::shared_ptr _ss; @@ -169,7 +171,8 @@ class QueryServerImpl : public IQueryServer { public: QueryServerImpl(const std::set& symbols, std::shared_ptr& ss, metrics::MetricsContainer& stats) : - _consumer(symbols, ss), + _cache(), + _consumer(symbols, ss, _cache), _symbols(symbols), _ss(ss), _groups() @@ -211,7 +214,8 @@ public: // not-reentrant void check_heartbeats(utils::Dispatcher& dispatcher) { if (!_consumer.all_heartbeats_ok()) { - MDConsumer temp(_symbols, _ss); + SecurityCache cache; + MDConsumer temp(_symbols, _ss, cache); if (temp.all_heartbeats_ok()) { _consumer.swap(temp); LOG(md,Info, "Recovering from bad MD. Old consumer {} new consumer {}", _consumer.id(), temp.id()); diff --git a/src/lib/md/test/md2_tests.cpp b/src/lib/md/test/md2_tests.cpp index 9d6ef027e..35c8a828e 100644 --- a/src/lib/md/test/md2_tests.cpp +++ b/src/lib/md/test/md2_tests.cpp @@ -3,6 +3,7 @@ #include "../SettlementRecorder.h" #include "../coconut2.h" #include "../relay/QueryServer.h" +#include "../SecurityCache.h" #include "Co2TestInstrument.h" #include "lib/config/config.h" #include "lib/os/os.h" @@ -368,7 +369,8 @@ public: symbols.insert(future_symbol); UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); - MDConsumer consumer(symbols, nullptr); + SecurityCache cache; + MDConsumer consumer(symbols, nullptr, cache); vector consumer_securities; // Poll for security list - none visible until they have keys @@ -438,7 +440,8 @@ public: symbols.insert(future_symbol); UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); - MDConsumer consumer(symbols, nullptr); + SecurityCache cache; + MDConsumer consumer(symbols, nullptr, cache); // Add the on_security callback vector securities; @@ -477,7 +480,8 @@ public: symbols.insert(future_symbol); UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); - MDConsumer consumer(symbols, nullptr); + SecurityCache cache; + MDConsumer consumer(symbols, nullptr, cache); // Get a handle to 24 consumers which haven't been created yet vector consumers; @@ -534,7 +538,8 @@ public: symbols.insert(future_symbol); UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); - MDConsumer consumer(symbols, nullptr); + SecurityCache cache; + MDConsumer consumer(symbols, nullptr, cache); // Get a handle to 24 consumers which haven't been created yet vector consumers; @@ -594,7 +599,8 @@ public: // Symbol list for the consumer auto syms = get_test_symbols(); - MDConsumer consumer = MDConsumer(syms, nullptr); + SecurityCache cache; + MDConsumer consumer = MDConsumer(syms, nullptr, cache); // Add the on_security callback - callback fires for existing securities vector securities; @@ -737,7 +743,8 @@ public: symbols.insert(future_symbol); UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); - MDConsumer consumer(symbols, nullptr); + SecurityCache cache; + MDConsumer consumer(symbols, nullptr, cache); set keys; consumer.walk_security_list([&](const SecurityPtr& security, MDConsumerHandle) { keys.insert(security->id()); }); @@ -781,7 +788,8 @@ public: csymbols.insert(UserSymbol("S2", VENUE_BTS, SECURITYTYPE_SPREAD)); csymbols.insert(UserSymbol("S3", VENUE_BTS, SECURITYTYPE_SPREAD)); csymbols.insert(UserSymbol("S4", VENUE_BTS, SECURITYTYPE_SPREAD)); - MDConsumer consumer(csymbols, nullptr); + SecurityCache cache; + MDConsumer consumer(csymbols, nullptr, cache); auto check_sec = [&](const SecurityPtr& security, MDConsumerHandle handle) { REQUIRE(security->id() == handle.key()); @@ -827,7 +835,8 @@ public: csymbols.insert(UserSymbol("FF", VENUE_BTS, SECURITYTYPE_FUTURE)); csymbols.insert(UserSymbol("OO", VENUE_BTS, SECURITYTYPE_OPTION)); csymbols.insert(UserSymbol("SS", VENUE_BTS, SECURITYTYPE_SPREAD)); - MDConsumer consumer(csymbols, nullptr); + SecurityCache cache; + MDConsumer consumer(csymbols, nullptr, cache); auto check_sec = [&](const SecurityPtr& security, MDConsumerHandle handle) { REQUIRE(security->id() == handle.key()); @@ -878,7 +887,8 @@ public: set consumer_symbols; consumer_symbols.insert(UserSymbol("OO", VENUE_BTS, SECURITYTYPE_OPTION)); consumer_symbols.insert(UserSymbol("SS", VENUE_BTS, SECURITYTYPE_SPREAD)); - MDConsumer consumer(consumer_symbols, nullptr); + SecurityCache cache; + MDConsumer consumer(consumer_symbols, nullptr, cache); size_t security_count = 0; // Indirection for check function @@ -946,7 +956,8 @@ public: auto ph2 = *add_security(o2); set consumer_symbols; consumer_symbols.insert(UserSymbol("OO", VENUE_BTS, SECURITYTYPE_OPTION)); - MDConsumer consumer(consumer_symbols, nullptr); + SecurityCache cache; + MDConsumer consumer(consumer_symbols, nullptr, cache); auto h1 = consumer.get_handle(k1); REQUIRE(h1.is_initialized()); auto h2 = consumer.get_handle(k2); @@ -1212,6 +1223,7 @@ struct Co2IndexTests { } ; struct Co2RelayTests { + SecurityCache cache; Co2RelayTests() : _stats() {} void setup() { config::internal::initialize_for_test( @@ -1230,7 +1242,7 @@ struct Co2RelayTests { symbols.insert(future_symbol); UserSymbol option_symbol("FOOX", VENUE_BTS, SECURITYTYPE_OPTION); symbols.insert(option_symbol); - MDConsumer consumer(symbols, nullptr); + MDConsumer consumer(symbols, nullptr, cache); return consumer; } @@ -1587,7 +1599,8 @@ namespace ImpliedTests { symbols.insert(future_symbol); UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); - MDConsumer consumer(symbols, nullptr); + SecurityCache cache; + MDConsumer consumer(symbols, nullptr, cache); vector consumer_securities; // Poll for security list - none visible until they have keys size_t count = consumer.walk_security_list([&](const SecurityPtr& security, MDConsumerHandle) {consumer_securities.push_back(*security);}); diff --git a/src/lib/orderman/server/OrderServiceServer.cpp b/src/lib/orderman/server/OrderServiceServer.cpp index 9259a2a7c..0cee13d43 100644 --- a/src/lib/orderman/server/OrderServiceServer.cpp +++ b/src/lib/orderman/server/OrderServiceServer.cpp @@ -92,7 +92,7 @@ struct OrderServiceServer::PImpl { DynamicOrderRegistry const &d, metrics::MetricsContainer &stats, trace::PerfLog &log) : service_bridge(), service_ready(false), known_orders(), known_fills(), publish_orders(), publish_fills(), - admin_service(ctx.locate()), md_consumer(ss), + admin_service(ctx.locate()), cache(), md_consumer(ss, cache), stats(stats), dyn_order_registry(d),log(log) {} ServiceBridge *service_bridge; @@ -106,6 +106,7 @@ struct OrderServiceServer::PImpl { OrderServiceServer::OrdersPublisher publish_orders; OrderServiceServer::FillsPublisher publish_fills; api::admin::AdminService admin_service; + md::SecurityCache cache; bts::md::MDConsumer md_consumer; metrics::MetricsContainer &stats; DynamicOrderRegistry const &dyn_order_registry; From bf72122edccfbb32440e700db8c7b8a28e69ea58 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 31 Jan 2019 11:06:19 -0500 Subject: [PATCH 39/85] SPARK 1731 - Simplify SecurityOracle impl --- src/lib/md/MDClient.cpp | 17 ++++++++++------- src/lib/md/MDClient.h | 6 +++--- src/lib/md/MDConsumerInterop.cpp | 6 ++++-- src/lib/md/SecurityOracle.cpp | 22 ++-------------------- src/lib/md/SecurityOracle.h | 7 ++----- 5 files changed, 21 insertions(+), 37 deletions(-) diff --git a/src/lib/md/MDClient.cpp b/src/lib/md/MDClient.cpp index a0f5313b0..dbe3eab5b 100644 --- a/src/lib/md/MDClient.cpp +++ b/src/lib/md/MDClient.cpp @@ -1,6 +1,7 @@ #include "MDClient.h" #include "get_symbols.h" #include "SecurityCache.h" +#include "SecurityOracle.h" #include namespace bts { @@ -14,14 +15,16 @@ struct MDClientImpl{ utils::DispatcherTimer _recovery_timer; MDConsumer _consumer; utils::Event _event; + SecurityOracle _oracle; - MDClientImpl(utils::Dispatcher& d, std::shared_ptr s): + MDClientImpl(utils::Dispatcher& d, std::shared_ptr s, SecurityOracle o): _dispatcher(d), _ss(s), _security_timer(d, config::get().security_scan_interval(),[this]() { scan_securities(); }), _recovery_timer(d, config::get().recovery_interval(), [this]() { check_heartbeats(); }), _consumer(s, _cache), - _event() + _event(), + _oracle(o) { _security_timer.start(); _recovery_timer.start(); @@ -57,8 +60,8 @@ struct MDClientImpl{ }; -MDClient::MDClient(utils::Dispatcher& d, std::shared_ptr s) : _self(new MDClientImpl(d,s)) { - +MDClient::MDClient(utils::Dispatcher& d, std::shared_ptr s) : + _self(new MDClientImpl(d,s, SecurityOracle(*this))) { } MDClient::~MDClient() { } @@ -66,11 +69,11 @@ MDClient::~MDClient() { utils::EventSubscriber& MDClient::on_consumer() { return _self->_event; } -MDConsumer MDClient::current_consumer() const { +MDConsumer& MDClient::current_consumer() const { return _self->_consumer; } -SecurityOracle MDClient::oracle() { - return SecurityOracle(*this); +SecurityOracle& MDClient::oracle() const { + return _self->_oracle; } }}//namespace \ No newline at end of file diff --git a/src/lib/md/MDClient.h b/src/lib/md/MDClient.h index b82549ae7..e80bb4a1e 100644 --- a/src/lib/md/MDClient.h +++ b/src/lib/md/MDClient.h @@ -3,12 +3,12 @@ #include "lib/utils/Dispatcher.h" #include "lib/utils/Event.h" #include "ISyntheticSecurityService.h" -#include "SecurityOracle.h" #include namespace bts { namespace md { +class SecurityOracle; struct MDClientImpl; class MDClient { std::unique_ptr _self; @@ -18,8 +18,8 @@ public: ~MDClient(); utils::EventSubscriber& on_consumer(); - MDConsumer current_consumer() const; - SecurityOracle oracle(); + MDConsumer& current_consumer() const; + SecurityOracle& oracle() const; }; diff --git a/src/lib/md/MDConsumerInterop.cpp b/src/lib/md/MDConsumerInterop.cpp index 7048354c5..142270e6f 100644 --- a/src/lib/md/MDConsumerInterop.cpp +++ b/src/lib/md/MDConsumerInterop.cpp @@ -2,6 +2,7 @@ #include "common.h" #include "UserSymbol.h" #include "MDConsumer.h" +#include "SecurityCache.h" #include "get_symbols.h" #include "relay/RelayClient.h" #include "lib/utils/Dispatcher.h" @@ -27,15 +28,16 @@ struct BlitUserSymbol { CAPI MDConsumer* create_md_consumer(BlitUserSymbol* usersymbols, uint32_t count, bool shared) { std::shared_ptr pnull = nullptr; + SecurityCache cache; if (usersymbols == nullptr || count == 0) { - return new MDConsumer(pnull, shared); + return new MDConsumer(pnull, cache, shared); } else { std::set symbols; for (size_t x = 0; x < count; ++x) { symbols.insert(UserSymbol(usersymbols[x].symbol, usersymbols[x].exchange, usersymbols[x].type)); } - return new MDConsumer(symbols, pnull, shared); + return new MDConsumer(symbols, pnull, cache, shared); } } diff --git a/src/lib/md/SecurityOracle.cpp b/src/lib/md/SecurityOracle.cpp index 6a7f81ea1..7691f6e4c 100644 --- a/src/lib/md/SecurityOracle.cpp +++ b/src/lib/md/SecurityOracle.cpp @@ -4,28 +4,10 @@ namespace bts { namespace md { -struct SecurityOracleImpl { - SecurityOracleImpl(MDClient& client): - client(client), - consumer(client.current_consumer()) - { - sub = client.on_consumer().subscribe([this](const MDConsumer& c) { - consumer = c; - }); - } - ~SecurityOracleImpl() { - client.on_consumer().unsubscribe(sub); - } - MDClient& client; - MDConsumer consumer; - int sub; - -}; - -SecurityOracle::SecurityOracle(MDClient& client): _self(new SecurityOracleImpl(client)) { } +SecurityOracle::SecurityOracle(const MDClient& client): _client(client) {} SecurityOracle::~SecurityOracle() { } const api::Security* SecurityOracle::get_security(SecurityKey key) const { - return _self->consumer.get_security(key); + return _client.current_consumer().get_security(key); } }}//namespace \ No newline at end of file diff --git a/src/lib/md/SecurityOracle.h b/src/lib/md/SecurityOracle.h index da0210f18..e45ba062f 100644 --- a/src/lib/md/SecurityOracle.h +++ b/src/lib/md/SecurityOracle.h @@ -4,14 +4,11 @@ namespace bts { namespace md { - class MDClient; - -struct SecurityOracleImpl; class SecurityOracle { - std::unique_ptr _self; + const MDClient& _client; public: - SecurityOracle(MDClient&); + SecurityOracle(const MDClient&); ~SecurityOracle(); const api::Security* get_security(SecurityKey key) const; }; From 20f42749dd1a8be2f2a9955732991e8cc5ff7573 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 31 Jan 2019 11:16:25 -0500 Subject: [PATCH 40/85] SPARK-1997 Update OrderServiceServer/FillStore to use SecurityOracle --- src/lib/orderman/actors/FillStore.cpp | 115 +++++++----------- src/lib/orderman/actors/FillStore.h | 3 +- src/lib/orderman/actors/SODPosition.cpp | 10 +- src/lib/orderman/actors/SODPosition.h | 3 +- .../orderman/server/OrderServiceServer.cpp | 15 +-- src/lib/orderman/server/OrderServiceServer.h | 3 +- src/server/sparkd/cmds/Run.cpp | 7 +- 7 files changed, 68 insertions(+), 88 deletions(-) diff --git a/src/lib/orderman/actors/FillStore.cpp b/src/lib/orderman/actors/FillStore.cpp index a7d2fab18..76f1c62a0 100644 --- a/src/lib/orderman/actors/FillStore.cpp +++ b/src/lib/orderman/actors/FillStore.cpp @@ -15,100 +15,71 @@ namespace orderman { auto const fills_ns = "fills"; -enum class FillStore::State { WAIT_SEC, READY }; - struct FillStore::UnresolvedFills { std::vector fills; api::Security const *security; }; struct FillStore::PImpl { - PImpl() - : state(State::WAIT_SEC), store(new persistence::SqliteStore("fills")), - known_orders(), fills_with_unresolved_accounts(), sod_pos_map() {} + PImpl(md::SecurityOracle& oracle) + : store(new persistence::SqliteStore("fills")), + known_orders(), fills_with_unresolved_accounts(), sod_pos_map(), oracle(oracle) {} - State state; std::unique_ptr store; std::unordered_set known_fills; std::unordered_map known_orders; std::unordered_map fills_with_unresolved_accounts; mercury::Stream fill_stream; - mercury::Actor::SubHandle stored_sec_sub; SODPosition sod_pos_map; + md::SecurityOracle oracle; }; -FillStore::FillStore() : me(new PImpl()) {} +FillStore::FillStore(md::SecurityOracle& o) : me(new PImpl(o)) {} void FillStore::init() { - me->state = State::WAIT_SEC; - me->stored_sec_sub = subscribe([this]( - auto const &update) { - if (update.events) { - if (me->state == State::WAIT_SEC) { + // Subscribe to NewExecutionReportEvent for new fills + subscribe( + [this](auto const &ev) { this->on_execution_report(ev); }); - LOG(OrderMan, Info, - "Fill store received initial list of {} securities", - update.events->size()); + // Load stored fills, resolve their securities and publish the lot + api::orderman::Fill fill; + std::vector stored_batch; + LOG(persist, Info, "Loading load all fills"); + auto all = me->store->retrieve_all()[fills_ns]; + LOG(persist, Info, "Finished loading all {} fills", all.size()); + for (auto const &kvp : all) + if (utils::pb::from_string(kvp.second, + &fill)) { - // don't need securities anymore, NewExecutionReportEvent - // events will carry the security with them - unsubscribe(me->stored_sec_sub); + auto sec = me->oracle.get_security(SecurityKey(fill.security_key())); + if (sec) { + auto fk = kvp.first; + if (!me->sod_pos_map.add_fill_to_sod_pos(fill)) + stored_batch.emplace_back(fill, *sec); + me->known_fills.insert(fk); + } else + LOG(persist, Error, + "Stored fill {} contains unresolved security {}. ", + kvp.first, fill.security_key()); + } else + LOG(persist, Error, + "Could not recover fill {} from storage. " + "This should be investigated", + kvp.first); + me->fill_stream = make_stream(); + LOG(OrderMan, Info, + "Fill Store published {} fills recoved from disk out of {} " + "total fills", + stored_batch.size(), all.size()); + me->sod_pos_map.add_sod_pos_as_fill_events(stored_batch, me->oracle); + publish_batch(me->fill_stream, stored_batch); - // Subscribe to NewExecutionReportEvent for new fills - subscribe( - [this](auto const &ev) { this->on_execution_report(ev); }); - - // index securities - std::map sec_idx; - for (auto const &ev : *(update.events)) { - auto sec_ptr = ev.security; - sec_idx[SecurityKey(sec_ptr->id())] = sec_ptr; - } - - // Load stored fills, resolve their securities and publish the lot - api::orderman::Fill fill; - std::vector stored_batch; - LOG(persist, Info, "Loading load all fills"); - auto all = me->store->retrieve_all()[fills_ns]; - LOG(persist, Info, "Finished loading all {} fills", all.size()); - for (auto const &kvp : all) - if (utils::pb::from_string(kvp.second, - &fill)) { - - auto sec = sec_idx[SecurityKey(fill.security_key())]; - if (sec) { - auto fk = kvp.first; - if (!me->sod_pos_map.add_fill_to_sod_pos(fill)) - stored_batch.emplace_back(fill, *sec); - me->known_fills.insert(fk); - } else - LOG(persist, Diag, - "Stored fill {} contains unresolved security {}. ", - kvp.first, fill.security_key()); - } else - LOG(persist, Error, - "Could not recover fill {} from storage. " - "This should be investigated", - kvp.first); - me->fill_stream = make_stream(); - LOG(OrderMan, Info, - "Fill Store published {} fills recoved from disk out of {} " - "total fills", - stored_batch.size(), all.size()); - me->sod_pos_map.add_sod_pos_as_fill_events(stored_batch, sec_idx); - publish_batch(me->fill_stream, stored_batch); - - // subscribe to order updates so we can resolve - // account ids for future execution reports - subscribe( - [this](auto const &ev) { this->on_order_state_update(ev); }); - - me->state = State::READY; - } - } - }); + // subscribe to order updates so we can resolve + // account ids for future execution reports + subscribe( + [this](auto const &ev) { this->on_order_state_update(ev); }); } FillStore::~FillStore() {} diff --git a/src/lib/orderman/actors/FillStore.h b/src/lib/orderman/actors/FillStore.h index 7d1c7b909..d4309e734 100644 --- a/src/lib/orderman/actors/FillStore.h +++ b/src/lib/orderman/actors/FillStore.h @@ -2,6 +2,7 @@ #include #include "lib/mercury/actors/Actor.h" +#include "lib/md/SecurityOracle.h" namespace bts { namespace orderman { @@ -14,7 +15,7 @@ class OrderStateUpdateEvent; class FillStore : public mercury::Actor { public: - FillStore(); + FillStore(md::SecurityOracle&); ~FillStore(); void init() override; diff --git a/src/lib/orderman/actors/SODPosition.cpp b/src/lib/orderman/actors/SODPosition.cpp index ee9515be7..18f6d83c4 100644 --- a/src/lib/orderman/actors/SODPosition.cpp +++ b/src/lib/orderman/actors/SODPosition.cpp @@ -171,22 +171,22 @@ bool SODPosition::add_fill_to_sod_pos(const Fill &fill) { void SODPosition::add_sod_pos_as_fill_events( std::vector &fe, - std::map sec_map) { + md::SecurityOracle& oracle) { if (_enabled) for (auto &acct_p : _sod_map) { for (auto &sec_p : acct_p.second) { Fill &long_pos = sec_p.second.first; Fill &short_pos = sec_p.second.second; bts_assert(long_pos.security_key() == short_pos.security_key()); - auto sec_it = sec_map.find(SecurityKey(long_pos.security_key())); - if (sec_it == sec_map.end()) + auto sec = oracle.get_security(SecurityKey(long_pos.security_key())); + if (!sec) LOG(persist, Error, "Could not recover fill for key:{} from storage. " "This should be investigated", long_pos.security_key()); else { - fe.emplace_back(long_pos, *sec_it->second); - fe.emplace_back(short_pos, *sec_it->second); + fe.emplace_back(long_pos, *sec); + fe.emplace_back(short_pos, *sec); } } } diff --git a/src/lib/orderman/actors/SODPosition.h b/src/lib/orderman/actors/SODPosition.h index d31e39446..860e0cf2d 100644 --- a/src/lib/orderman/actors/SODPosition.h +++ b/src/lib/orderman/actors/SODPosition.h @@ -4,6 +4,7 @@ #include "lib/api/services/OrderService.pb.h" #include "lib/mercury/actors/Actor.h" #include "lib/orderman/events/StoredFillEvent.h" +#include "lib/md/SecurityOracle.h" using namespace bts::api::orderman; @@ -34,7 +35,7 @@ public: static timeval get_order_cutoff_time(const std::map& ordermap); bool add_fill_to_sod_pos(const Fill& fill); void add_sod_pos_as_fill_events(std::vector& fill_events, - std::map sec_map); + md::SecurityOracle&); }; }} diff --git a/src/lib/orderman/server/OrderServiceServer.cpp b/src/lib/orderman/server/OrderServiceServer.cpp index 0cee13d43..caf1c67b2 100644 --- a/src/lib/orderman/server/OrderServiceServer.cpp +++ b/src/lib/orderman/server/OrderServiceServer.cpp @@ -5,6 +5,7 @@ #include "lib/config/protos/jetfix.pb.h" #include "lib/config/protos/orderman_cfg.pb.h" #include "lib/md/MDConsumer.h" +#include "lib/md/MDClient.h" #include "lib/mercury/actors/Timer.h" #include "lib/mercury/executor/Executor.h" #include "lib/metrics/metrics.h" @@ -88,11 +89,11 @@ ActorThread *find_thread_for_actor( struct OrderServiceServer::PImpl { PImpl(bts::api::ServiceContext &ctx, - std::shared_ptr &ss, + md::MDClient& mdc, DynamicOrderRegistry const &d, metrics::MetricsContainer &stats, trace::PerfLog &log) : service_bridge(), service_ready(false), known_orders(), known_fills(), publish_orders(), publish_fills(), - admin_service(ctx.locate()), cache(), md_consumer(ss, cache), + admin_service(ctx.locate()), md_consumer(mdc.current_consumer()), stats(stats), dyn_order_registry(d),log(log) {} ServiceBridge *service_bridge; @@ -106,8 +107,7 @@ struct OrderServiceServer::PImpl { OrderServiceServer::OrdersPublisher publish_orders; OrderServiceServer::FillsPublisher publish_fills; api::admin::AdminService admin_service; - md::SecurityCache cache; - bts::md::MDConsumer md_consumer; + bts::md::MDConsumer& md_consumer; metrics::MetricsContainer &stats; DynamicOrderRegistry const &dyn_order_registry; trace::PerfLog &log; @@ -115,10 +115,10 @@ struct OrderServiceServer::PImpl { OrderServiceServer::OrderServiceServer( bts::api::ServiceContext &ctx, - std::shared_ptr &ss, + md::MDClient& mdc, DynamicOrderRegistry const &d, metrics::MetricsContainer &stats, trace::PerfLog &log) - : api::orderman::OrderServiceBase(ctx), me(new PImpl(ctx, ss, d, stats,log)) { + : api::orderman::OrderServiceBase(ctx), me(new PImpl(ctx, mdc, d, stats,log)) { me->publish_orders = make_orders_publisher(); me->publish_fills = make_fills_publisher(); @@ -138,6 +138,7 @@ OrderServiceServer::OrderServiceServer( bts_assert_msg(thread, "Could not find a configured thread for MDSource. See the " "OrdermanCfg key"); + thread->executor.spawn(me->md_consumer, me->stats,log); // OrderManager @@ -173,7 +174,7 @@ OrderServiceServer::OrderServiceServer( bts_assert_msg(thread, "Could not find a configured thread for FilllStore. See " "the OrdermanCfg key"); - thread->executor.spawn(); + thread->executor.spawn(mdc.oracle()); // SecuritySrc thread = find_thread_for_actor("SecuritySrc", me->actor_threads); diff --git a/src/lib/orderman/server/OrderServiceServer.h b/src/lib/orderman/server/OrderServiceServer.h index 4c23c13f9..75c558932 100644 --- a/src/lib/orderman/server/OrderServiceServer.h +++ b/src/lib/orderman/server/OrderServiceServer.h @@ -3,6 +3,7 @@ #include "lib/api/services/OrderService.pb.h" #include "lib/api/services/OrderService.server.pb.h" #include "lib/mercury/actors/IPollableActor.h" +#include "lib/md/MDClient.h" #include namespace bts { @@ -19,7 +20,7 @@ class DynamicOrderRegistry; class OrderServiceServer : public api::orderman::OrderServiceBase { public: OrderServiceServer(bts::api::ServiceContext &context, - std::shared_ptr &, + md::MDClient&, DynamicOrderRegistry const&, metrics::MetricsContainer &, trace::PerfLog &); diff --git a/src/server/sparkd/cmds/Run.cpp b/src/server/sparkd/cmds/Run.cpp index a73561dec..a1840cf77 100644 --- a/src/server/sparkd/cmds/Run.cpp +++ b/src/server/sparkd/cmds/Run.cpp @@ -11,6 +11,7 @@ #include "lib/utils/Dispatcher.h" #include "lib/config/protos/toggles.pb.h" +#include "lib/md/MDClient.h" #include "lib/md/SyntheticSecurityService.h" #include "lib/md/relay/QueryServer.h" #include "lib/metrics/metrics.h" @@ -56,8 +57,12 @@ int Run::execute(std::vector const &) { persistence::PersistenceServiceServer settings_saver(sc, "settings"); user::AuthServiceServer login_server(sc); md::run_market_datapublishers(); + std::shared_ptr synthetic_service = std::make_shared(sc); + md::MDClient mdclient(utils::Dispatcher::current(), synthetic_service); + + md::QueryServerPtr md_query_server( md::run_query_server(sc, synthetic_service, stats)); @@ -68,7 +73,7 @@ int Run::execute(std::vector const &) { // finished initializing order plugins orderman::OrderServiceServer order_service_server( - sc, synthetic_service, dyn_order_registry, stats, perf_log); + sc, mdclient, dyn_order_registry, stats, perf_log); support::ErrorReportServer error_report_service(sc); admin::AdminServiceServer admin_service(sc); md::auxiliary::MarketDataAuxiliaryServiceServer md_aux_server(sc); From d99d3bccabca409cb817da840d2550d0c068d89b Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 31 Jan 2019 11:52:36 -0500 Subject: [PATCH 41/85] PR feedback #1 --- src/lib/md/MDClient.cpp | 2 +- src/lib/md/MDClient.h | 2 +- src/lib/orderman/actors/FillStore.cpp | 3 +-- src/lib/orderman/actors/FillStore.h | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/lib/md/MDClient.cpp b/src/lib/md/MDClient.cpp index dbe3eab5b..851509d20 100644 --- a/src/lib/md/MDClient.cpp +++ b/src/lib/md/MDClient.cpp @@ -72,7 +72,7 @@ utils::EventSubscriber& MDClient::on_consumer() { MDConsumer& MDClient::current_consumer() const { return _self->_consumer; } -SecurityOracle& MDClient::oracle() const { +SecurityOracle MDClient::oracle() const { return _self->_oracle; } diff --git a/src/lib/md/MDClient.h b/src/lib/md/MDClient.h index e80bb4a1e..f9c6b8d3a 100644 --- a/src/lib/md/MDClient.h +++ b/src/lib/md/MDClient.h @@ -19,7 +19,7 @@ public: utils::EventSubscriber& on_consumer(); MDConsumer& current_consumer() const; - SecurityOracle& oracle() const; + SecurityOracle oracle() const; }; diff --git a/src/lib/orderman/actors/FillStore.cpp b/src/lib/orderman/actors/FillStore.cpp index 76f1c62a0..f5f114eb6 100644 --- a/src/lib/orderman/actors/FillStore.cpp +++ b/src/lib/orderman/actors/FillStore.cpp @@ -2,7 +2,6 @@ #include "lib/api/services/OrderService.pb.h" #include "lib/orderman/actors/SODPosition.h" #include "lib/orderman/events/NewExecutionReportEvent.h" -#include "lib/orderman/events/NewSecuritiesEvent.h" #include "lib/orderman/events/OrderStateUpdateEvent.h" #include "lib/orderman/events/StoredFillEvent.h" #include "lib/persistence/SqliteStore.h" @@ -35,7 +34,7 @@ struct FillStore::PImpl { md::SecurityOracle oracle; }; -FillStore::FillStore(md::SecurityOracle& o) : me(new PImpl(o)) {} +FillStore::FillStore(md::SecurityOracle o) : me(new PImpl(o)) {} void FillStore::init() { diff --git a/src/lib/orderman/actors/FillStore.h b/src/lib/orderman/actors/FillStore.h index d4309e734..1d35e032c 100644 --- a/src/lib/orderman/actors/FillStore.h +++ b/src/lib/orderman/actors/FillStore.h @@ -15,7 +15,7 @@ class OrderStateUpdateEvent; class FillStore : public mercury::Actor { public: - FillStore(md::SecurityOracle&); + FillStore(md::SecurityOracle); ~FillStore(); void init() override; From 3541ae5a8558d9c76905b6570c53028080078d2a Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 31 Jan 2019 17:05:22 -0500 Subject: [PATCH 42/85] SPARK-1996 QueryServer and constructor refactoring --- src/lib/md/Diag/MDPeriodicDumper.cpp | 2 +- src/lib/md/MDClient.cpp | 77 +++++---- src/lib/md/MDClient.h | 13 +- src/lib/md/MDConsumer.cpp | 39 ++--- src/lib/md/MDConsumer.h | 12 +- src/lib/md/MDConsumerInterop.cpp | 14 +- src/lib/md/MDRecoverableConsumer.cpp | 157 ------------------- src/lib/md/MDRecoverableConsumer.h | 69 -------- src/lib/md/PlatformLoader.cpp | 6 + src/lib/md/SecurityCache.cpp | 1 + src/lib/md/SyntheticSecurityService.h | 6 +- src/lib/md/coconut2.h | 1 - src/lib/md/get_symbols.h | 1 + src/lib/md/relay/QueryServer.cpp | 100 ++++-------- src/lib/md/relay/QueryServer.h | 6 +- src/lib/md/test/md2_tests.cpp | 96 +++++++----- src/lib/orderman/events/NewSecuritiesEvent.h | 4 + src/lib/orderman/server/OrderServiceServer.h | 3 - src/server/sparkd/cmds/Run.cpp | 7 +- src/server/sparkd/cmds/Web.cpp | 12 +- 20 files changed, 202 insertions(+), 424 deletions(-) delete mode 100644 src/lib/md/MDRecoverableConsumer.cpp delete mode 100644 src/lib/md/MDRecoverableConsumer.h diff --git a/src/lib/md/Diag/MDPeriodicDumper.cpp b/src/lib/md/Diag/MDPeriodicDumper.cpp index 05cdadb06..46e5bfbcd 100644 --- a/src/lib/md/Diag/MDPeriodicDumper.cpp +++ b/src/lib/md/Diag/MDPeriodicDumper.cpp @@ -160,7 +160,7 @@ void run_periodic_dumper(const std::set& symbols, uint32_t seq_mod, threading::configure_current_thread(thread_name); SecurityCache cache; - MDConsumer consumer(symbols, nullptr, cache); + MDConsumer consumer(cache, nullptr, &symbols); // read all securities into a vector std::vector entries; diff --git a/src/lib/md/MDClient.cpp b/src/lib/md/MDClient.cpp index 851509d20..6822c4d20 100644 --- a/src/lib/md/MDClient.cpp +++ b/src/lib/md/MDClient.cpp @@ -2,47 +2,47 @@ #include "get_symbols.h" #include "SecurityCache.h" #include "SecurityOracle.h" +#include "lib/utils/TimeStamp.h" #include namespace bts { namespace md { struct MDClientImpl{ - utils::Dispatcher& _dispatcher; - std::shared_ptr _ss; + utils::Dispatcher* _dispatcher; + api::ISyntheticSecurityService* _ss; SecurityCache _cache; - utils::DispatcherTimer _security_timer; - utils::DispatcherTimer _recovery_timer; + std::unique_ptr _security_timer; + std::unique_ptr _recovery_timer; + std::set _symbols; MDConsumer _consumer; utils::Event _event; - SecurityOracle _oracle; - MDClientImpl(utils::Dispatcher& d, std::shared_ptr s, SecurityOracle o): + MDClientImpl( + utils::Dispatcher* d, + api::ISyntheticSecurityService* s, + std::set* symbols): _dispatcher(d), _ss(s), - _security_timer(d, config::get().security_scan_interval(),[this]() { scan_securities(); }), - _recovery_timer(d, config::get().recovery_interval(), [this]() { check_heartbeats(); }), - _consumer(s, _cache), - _event(), - _oracle(o) + _security_timer(), + _recovery_timer(), + _symbols(symbols ? *symbols : get_system_md_symbols()), + _consumer(_cache, _ss, &_symbols), + _event() { - _security_timer.start(); - _recovery_timer.start(); - } - - std::set symbols() { - std::set all_symbols; - Venue venue; - bts_assert(api::Venue_Parse(config::get().exchange(), &venue)); - SecurityType type; - bts_assert(api::SecurityType_Parse(config::get().type(), &type)); - get_system_md_symbols(venue, type, all_symbols); - return all_symbols; + if (d) { + _security_timer = + std::make_unique(*d, config::get().security_scan_interval(),[this]() { scan_securities(); }), + _recovery_timer = + std::make_unique(*d, config::get().recovery_interval(), [this]() { check_heartbeats(); }), + _security_timer->start(); + _recovery_timer->start(); + } } void check_heartbeats() { if (!_consumer.all_heartbeats_ok()) { - MDConsumer temp(_ss,_cache); + MDConsumer temp(_cache,_ss); if (temp.all_heartbeats_ok()) { _consumer.swap(temp); LOG(md,Info, "Recovering from bad MD. Old consumer {} new consumer {}", _consumer.id(), temp.id()); @@ -52,20 +52,34 @@ struct MDClientImpl{ } } - void scan_securities() { + size_t scan_securities() { if (_consumer.all_heartbeats_ok()) { - _consumer.scan_securities(); + TimeStamp start = TimeStamp::now(); + auto count = _consumer.scan_securities(); + auto delta = (TimeStamp::now() - start); + if (delta.milliseconds() > 10) { + LOG(md,Diag,"Scanned handles: {}ms found: {} securities. " , + delta.milliseconds(),count); + } + return count; } + return 0; } }; -MDClient::MDClient(utils::Dispatcher& d, std::shared_ptr s) : - _self(new MDClientImpl(d,s, SecurityOracle(*this))) { -} +MDClient::MDClient( + utils::Dispatcher* d, + api::ISyntheticSecurityService* s, + std::set* symbols) : + _self(new MDClientImpl(d,s, symbols)) + { } MDClient::~MDClient() { } +size_t MDClient::scan_securities() { return _self->scan_securities(); } +void MDClient::check_heartbeats() { return _self->check_heartbeats(); } + utils::EventSubscriber& MDClient::on_consumer() { return _self->_event; } @@ -73,7 +87,10 @@ MDConsumer& MDClient::current_consumer() const { return _self->_consumer; } SecurityOracle MDClient::oracle() const { - return _self->_oracle; + return SecurityOracle(*this); +} +std::set MDClient::symbols() const { + return _self->_symbols; } }}//namespace \ No newline at end of file diff --git a/src/lib/md/MDClient.h b/src/lib/md/MDClient.h index f9c6b8d3a..6de826de1 100644 --- a/src/lib/md/MDClient.h +++ b/src/lib/md/MDClient.h @@ -3,7 +3,9 @@ #include "lib/utils/Dispatcher.h" #include "lib/utils/Event.h" #include "ISyntheticSecurityService.h" +#include "UserSymbol.h" #include +#include namespace bts { namespace md { @@ -14,12 +16,21 @@ class MDClient { std::unique_ptr _self; public: - MDClient(utils::Dispatcher&, std::shared_ptr); + // Will check for recovery + MDClient( + utils::Dispatcher* d=nullptr, + api::ISyntheticSecurityService* ss=nullptr, + std::set* s=nullptr); ~MDClient(); + // Check heartbeats and attempt recovery if they are bad + void check_heartbeats(); + size_t scan_securities(); + utils::EventSubscriber& on_consumer(); MDConsumer& current_consumer() const; SecurityOracle oracle() const; + std::set symbols() const; }; diff --git a/src/lib/md/MDConsumer.cpp b/src/lib/md/MDConsumer.cpp index d01908032..a532cdf1a 100644 --- a/src/lib/md/MDConsumer.cpp +++ b/src/lib/md/MDConsumer.cpp @@ -307,7 +307,7 @@ class SyntheticConsumerInst { // LOCK only public methods Mutex _workmutex; - shared_ptr _ss; + api::ISyntheticSecurityService* _ss; map _data; vector> _pending; set _seen; @@ -318,7 +318,7 @@ class SyntheticConsumerInst { std::vector> _old_strategies; public: - SyntheticConsumerInst(std::shared_ptr& ss, const IGetHandle& gh) : + SyntheticConsumerInst(api::ISyntheticSecurityService* ss, const IGetHandle& gh) : _workmutex(), _ss(ss), _data(), @@ -543,9 +543,9 @@ class MDConsumerImpl { public: MDConsumerImpl( const set& symbols, - std::shared_ptr& ss, SecurityCache& cache, - bool shared, co2::Platform platform = co2::PLATFORM_UNSET) : + api::ISyntheticSecurityService* ss, + bool shared, co2::Platform platform) : _workmutex(), _instances(), _sci(), @@ -907,27 +907,20 @@ public: // MDConsumer public interface, forwards to pimple /////////////////////////////////////////////////////////////////////////////////////////////////// -MDConsumer::MDConsumer(std::shared_ptr& synt, SecurityCache& cache, bool shared) : _pimpl() { - set all_symbols; - get_system_md_symbols(VENUE_UNSET, SECURITYTYPE_UNSET, all_symbols); +MDConsumer::MDConsumer( + SecurityCache& cache, + api::ISyntheticSecurityService* ss, + const std::set* symbols, + bool shared, + co2::Platform platform) : _pimpl() { - _pimpl.reset(new MDConsumerImpl(all_symbols, synt, cache, shared, co2::PLATFORM_UNSET)); -} -MDConsumer::MDConsumer(const UserSymbol& s, std::shared_ptr synt, SecurityCache& cache, bool shared) : - _pimpl() -{ - set symbols; - symbols.insert(s); - _pimpl.reset(new MDConsumerImpl(symbols, synt, cache, shared, co2::PLATFORM_UNSET)); -} -MDConsumer::MDConsumer(set const& symbols, std::shared_ptr s, SecurityCache& cache, bool shared, co2::Platform platform) : - _pimpl(new MDConsumerImpl(symbols, s, cache, shared, platform)) {} + set all_symbols; + if (symbols == nullptr || symbols->empty()) { + all_symbols = get_system_md_symbols(); + symbols = &all_symbols; + } -MDConsumer::MDConsumer(SecurityCache& cache, api::Venue ven, co2::Platform platform) : _pimpl() { - set all_symbols; - get_system_md_symbols(ven, SECURITYTYPE_UNSET, all_symbols); - std::shared_ptr null = nullptr; - _pimpl.reset(new MDConsumerImpl(all_symbols, null, cache, true, platform)); + _pimpl.reset(new MDConsumerImpl(*symbols, cache, ss, shared, platform)); } MDConsumer::~MDConsumer() {} diff --git a/src/lib/md/MDConsumer.h b/src/lib/md/MDConsumer.h index dff515edc..02c762a8f 100644 --- a/src/lib/md/MDConsumer.h +++ b/src/lib/md/MDConsumer.h @@ -30,12 +30,12 @@ class SecurityCache; class MDConsumer { public: - // Consumer for all symbols - MDConsumer(std::shared_ptr&, SecurityCache&, bool shared=true); - MDConsumer(const UserSymbol&, std::shared_ptr, SecurityCache&, bool shared=true); - // Consumer for a given set of symbols - MDConsumer(std::set const& symbols, std::shared_ptr, SecurityCache&, bool shared=true, co2::Platform platform=co2::PLATFORM_UNSET); - MDConsumer(SecurityCache&, api::Venue, co2::Platform); + MDConsumer( + SecurityCache&, + api::ISyntheticSecurityService* ss, + const std::set* sym = nullptr, + bool shared=true, + co2::Platform platform=co2::PLATFORM_UNSET); ~MDConsumer(); bool all_heartbeats_ok() const; diff --git a/src/lib/md/MDConsumerInterop.cpp b/src/lib/md/MDConsumerInterop.cpp index 142270e6f..5027f3d75 100644 --- a/src/lib/md/MDConsumerInterop.cpp +++ b/src/lib/md/MDConsumerInterop.cpp @@ -27,18 +27,12 @@ struct BlitUserSymbol { }; CAPI MDConsumer* create_md_consumer(BlitUserSymbol* usersymbols, uint32_t count, bool shared) { - std::shared_ptr pnull = nullptr; SecurityCache cache; - if (usersymbols == nullptr || count == 0) { - return new MDConsumer(pnull, cache, shared); - } - else { - std::set symbols; - for (size_t x = 0; x < count; ++x) { - symbols.insert(UserSymbol(usersymbols[x].symbol, usersymbols[x].exchange, usersymbols[x].type)); - } - return new MDConsumer(symbols, pnull, cache, shared); + std::set symbols; + for (size_t x = 0; x < count; ++x) { + symbols.insert(UserSymbol(usersymbols[x].symbol, usersymbols[x].exchange, usersymbols[x].type)); } + return new MDConsumer(cache, nullptr/*synthetics*/, &symbols, shared); } CAPI void destroy_md_consumer(MDConsumer* consumer) { diff --git a/src/lib/md/MDRecoverableConsumer.cpp b/src/lib/md/MDRecoverableConsumer.cpp deleted file mode 100644 index 5b5378033..000000000 --- a/src/lib/md/MDRecoverableConsumer.cpp +++ /dev/null @@ -1,157 +0,0 @@ -#ifndef NO_CPP0X -#include "MDRecoverableConsumer.h" -#include "PlatformLoader.h" -#include "lib/logging/logging.h" -#include - - -using namespace std; -namespace bts { -namespace md { - -set get_symbols(Venue v=api::VENUE_UNSET); - -MDRecoverableConsumer::MDRecoverableConsumer(std::shared_ptr ss) : - _symbols(get_symbols()), - _ss(ss), - _current(init(_symbols, ss)), - _old(), - _callbacks(), - _seen(), - _lock () -{} - -MDRecoverableConsumer::MDRecoverableConsumer(Venue exchange, std::shared_ptr ss) : // (visual studio 2012) MDRecoverableConsumer(get_symbols(exchange)) - _symbols(get_symbols(exchange)), - _ss(ss), - _current(init(_symbols, ss)), - _old(), - _callbacks(), - _seen(), - _lock () -{} - -MDRecoverableConsumer::MDRecoverableConsumer(std::set symbols, std::shared_ptr ss) : - _symbols(symbols), - _ss(ss), - _current(init(symbols, ss)), - _old(), - _callbacks(), - _seen(), - _lock () -{ -} - -MDRecoverableConsumer::~MDRecoverableConsumer() {} - -unique_ptr MDRecoverableConsumer::init(std::set& symbols, std::shared_ptr& ss) { - return std::make_unique(symbols, ss, _cache); -} - -size_t MDRecoverableConsumer::walk_security_list(SecurityCallback callback) { - return _current->consumer.walk_security_list(callback); -} - -void MDRecoverableConsumer::on_security(SecurityCallback cb) { - // Add a filter to the callback to avoid calling the on_security callback with duplicates - auto filtered_cb = [this,cb](const SecurityPtr& security, MDConsumerHandle handle) { - auto key = security->id(); - if (_seen.find(key) == _seen.end()) { - _seen.insert(key); - cb(security, handle); - } - }; - _current->consumer.on_security(filtered_cb); - Mutex::scoped_lock l (_lock); - _callbacks.push_back(filtered_cb); -} - -size_t MDRecoverableConsumer::scan_securities() { - return _current->consumer.scan_securities(); -} - -std::shared_ptr MDRecoverableConsumer::get_managed_handle(security_key key) { - Mutex::scoped_lock l (_lock); - auto handle = _current->consumer.get_managed_handle(key); - _current->handles.push_back(handle); - return handle; -} - -bool MDRecoverableConsumer::all_heartbeats_ok() const { - return _current->consumer.all_heartbeats_ok(); -} - -bool MDRecoverableConsumer::try_recover() { - Mutex::scoped_lock l (_lock); - // Replace current with a new one (if the new one is better) - auto temp = init(_symbols, _ss); - if (!temp->consumer.all_heartbeats_ok()) return false; - - std::swap(temp, _current); - // push current to old - _old.push_back(std::move(temp)); - // copy old callbacks so we can reinstate them - auto old_callbacks = _callbacks; - - // Add an on_security callback to current to check the old handles - l.release(); - - // re-instante old callbacks - for (auto &cb : old_callbacks) { - _current->consumer.on_security(cb); - } - - _current->consumer.on_security([&](const SecurityPtr& security, MDConsumerHandle newhandle) { - Mutex::scoped_lock l (_lock); - for (auto& group : _old) { - for (auto& handle : group->handles) { - if (handle->key() == security->id()) { - // update the handle! (if this is called guarunteed success) - *handle = newhandle; - bts_assert(handle->is_initialized()); - _current->handles.push_back(move(handle)); - } - } - } - cleanup(); - }); - - return true; -} - -// cleanup should only ever be called while _lock is locked -void MDRecoverableConsumer::cleanup() { - for (auto& group : _old) { - auto& handles = group->handles; - handles.erase( - remove_if( - handles.begin(), - handles.end(), - [](shared_ptr& handle) { return handle == nullptr; }), - handles.end()); - } - auto old_sz = _old.size (); - _old.erase( - remove_if( - _old.begin(), - _old.end(), - [](unique_ptr& pgroup) -> bool { return pgroup->handles.empty(); }), - _old.end()); - if (old_sz > _old.size ()) { - LOG (md,Info,"Cleaned up {} MDConsumer instances" ,old_sz - _old.size () ); - } -} - - -// -// Utility functions -set get_symbols(Venue exchange) { - set all_symbols; - get_system_md_symbols(exchange, api::SECURITYTYPE_UNSET, all_symbols); - return all_symbols; -} - - -}}//namespace -#endif - diff --git a/src/lib/md/MDRecoverableConsumer.h b/src/lib/md/MDRecoverableConsumer.h deleted file mode 100644 index c773b5d4d..000000000 --- a/src/lib/md/MDRecoverableConsumer.h +++ /dev/null @@ -1,69 +0,0 @@ -#pragma once - -#include "lib/utils/bts_no_copy.h" -#include "common.h" -#include "MDConsumer.h" -#include "MDConsumerHandle.h" -#include "UserSymbol.h" -#include "SecurityCache.h" -#include -#include -#include - -namespace bts { -namespace md { - -class BTS_EXPORT MDRecoverableConsumer -{ -public: - MDRecoverableConsumer(std::shared_ptr ss); - // Map only a certain exchange for more granular recovery - MDRecoverableConsumer(Venue, std::shared_ptr ss); - MDRecoverableConsumer(std::set symbols, std::shared_ptr ss); - ~MDRecoverableConsumer(); - - // - // Security functions - size_t walk_security_list(SecurityCallback callback); - void on_security(SecurityCallback cb); - // Scans for added securities - updates managed handles - size_t scan_securities(); - - // - // Data Access - std::shared_ptr get_managed_handle(security_key key); - - // - // Recover from publisher failure - bool all_heartbeats_ok() const; - bool try_recover(); - -private: - BTS_NO_COPY(MDRecoverableConsumer); - - void cleanup(); - - struct MDConsumerGroup { - MDConsumerGroup( - std::set& symbols, - std::shared_ptr& ss, - SecurityCache& cache) : consumer(symbols, ss, cache), handles() {} - MDConsumer consumer; - std::vector> handles; - }; - std::unique_ptr init(std::set& symbols, std::shared_ptr& ss); - - std::set _symbols; - std::shared_ptr _ss; - SecurityCache _cache; - std::unique_ptr _current; - std::vector> _old; - - std::vector _callbacks; - std::set _seen; - typedef tbb::mutex Mutex; - Mutex _lock; -}; - -}}//namespace - diff --git a/src/lib/md/PlatformLoader.cpp b/src/lib/md/PlatformLoader.cpp index acd1ace54..717e8e88c 100644 --- a/src/lib/md/PlatformLoader.cpp +++ b/src/lib/md/PlatformLoader.cpp @@ -172,6 +172,12 @@ size_t get_system_md_symbols(const md::PlatformConfig &config, return get_system_md_symbols(v, st, out_result); } +std::set get_system_md_symbols() { + std::set all_symbols; + get_system_md_symbols(VENUE_UNSET, SECURITYTYPE_UNSET, all_symbols); + return all_symbols; +} + extern "C" BTS_EXPORT Platform platform_for_exchange(Venue v) { const auto &pmap = config::get(); co2::Platform platform; diff --git a/src/lib/md/SecurityCache.cpp b/src/lib/md/SecurityCache.cpp index 6d5fd1b43..160633145 100644 --- a/src/lib/md/SecurityCache.cpp +++ b/src/lib/md/SecurityCache.cpp @@ -60,6 +60,7 @@ SecurityPtr SecurityCache::get_security(SecurityKey key) { lock.unlock(); auto security = db::find_security(key); if (security) { + LOG(md,Info, "Loaded security from database: {} {}", security->symbol(), security->id()); return add_db_security(security); } return security; diff --git a/src/lib/md/SyntheticSecurityService.h b/src/lib/md/SyntheticSecurityService.h index 0f07c0080..32b169192 100644 --- a/src/lib/md/SyntheticSecurityService.h +++ b/src/lib/md/SyntheticSecurityService.h @@ -20,9 +20,9 @@ class SyntheticSecurityService : struct Pimpl; std::unique_ptr _me; - public: - SyntheticSecurityService(api::ServiceContext& context); - ~SyntheticSecurityService(); +public: + SyntheticSecurityService(api::ServiceContext& context); + ~SyntheticSecurityService(); const std::unordered_map securities() override; utils::EventSubscriber& on_securities() override; diff --git a/src/lib/md/coconut2.h b/src/lib/md/coconut2.h index 79230eb4c..4195a0a40 100644 --- a/src/lib/md/coconut2.h +++ b/src/lib/md/coconut2.h @@ -9,5 +9,4 @@ #include "SettlementRecorder.h" // TODO: separate coconut2.h into consumer and producer sides #include "SecurityIndex.h" -#include "MDRecoverableConsumer.h" diff --git a/src/lib/md/get_symbols.h b/src/lib/md/get_symbols.h index 3088e1b77..d0226432d 100644 --- a/src/lib/md/get_symbols.h +++ b/src/lib/md/get_symbols.h @@ -8,6 +8,7 @@ namespace bts { namespace md { +std::set get_system_md_symbols(); size_t get_system_md_symbols(api::Venue, api::SecurityType, std::vector* result); size_t get_system_md_symbols(api::Venue, api::SecurityType, std::set& out_result); size_t get_system_md_symbols(const md::PlatformConfig& config, std::set& out_result); diff --git a/src/lib/md/relay/QueryServer.cpp b/src/lib/md/relay/QueryServer.cpp index 783c355e6..5b44513d4 100644 --- a/src/lib/md/relay/QueryServer.cpp +++ b/src/lib/md/relay/QueryServer.cpp @@ -7,6 +7,7 @@ #include "lib/utils/AuthToken.h" #include "../MDConsumer.h" +#include "../MDClient.h" #include "../PlatformLoader.h" #include "lib/logging/logging.h" @@ -163,27 +164,30 @@ void copy_misc(IntPrice settlement, MiscData misc, co2::MiscMessage* pmisc) { // Handles market data queries and responds class QueryServerImpl : public IQueryServer { - SecurityCache _cache; + MDClient& _client; MDConsumer _consumer; - const std::set _symbols; - const std::shared_ptr _ss; + int _sub; boost::container::flat_map> _groups; public: - QueryServerImpl(const std::set& symbols, std::shared_ptr& ss, metrics::MetricsContainer& stats) : - _cache(), - _consumer(symbols, ss, _cache), - _symbols(symbols), - _ss(ss), + QueryServerImpl(MDClient& client, metrics::MetricsContainer& stats) : + _client(client), + _consumer(client.current_consumer()), + _sub(0), _groups() { - for (const auto& sym : symbols) { + for (const auto& sym : client.symbols()) { _groups.emplace(std::make_pair(sym, std::make_unique(sym, _consumer, stats))); } _groups.shrink_to_fit(); + _sub = client.on_consumer().subscribe([=](const MDConsumer& c) { handle_recovery(c); }); subscribe_securities(); } + QueryServerImpl(const QueryServerImpl&)=delete; + ~QueryServerImpl() { + _client.on_consumer().unsubscribe(_sub); + } void subscribe_securities() { auto id = _consumer.id(); @@ -199,32 +203,16 @@ public: } // not-reentrant - void scan_securities() { - if (_consumer.all_heartbeats_ok()) { - TimeStamp start = TimeStamp::now(); - size_t count = _consumer.scan_securities(false); - auto delta = (TimeStamp::now() - start); - if (delta.milliseconds() > 10) { - LOG(md,Diag,"Scanned handles: {}ms found: {} securities. " , - delta.milliseconds(),count); - } - } - } - - // not-reentrant - void check_heartbeats(utils::Dispatcher& dispatcher) { - if (!_consumer.all_heartbeats_ok()) { - SecurityCache cache; - MDConsumer temp(_symbols, _ss, cache); - if (temp.all_heartbeats_ok()) { - _consumer.swap(temp); - LOG(md,Info, "Recovering from bad MD. Old consumer {} new consumer {}", _consumer.id(), temp.id()); - for (auto& pair : _groups) { - pair.second->set_consumer(_consumer); - } - subscribe_securities(); - dispatcher.post([temp]() { LOG(md,Info,"QServer destroying consumer {}", temp.id()); }, 60000); + void handle_recovery(const MDConsumer& consumer) { + MDConsumer temp(consumer); + if (temp.all_heartbeats_ok()) { + _consumer.swap(temp); + LOG(md,Info, "Recovering from bad MD. Old consumer {} new consumer {}", _consumer.id(), temp.id()); + for (auto& pair : _groups) { + pair.second->set_consumer(_consumer); } + subscribe_securities(); + utils::Dispatcher::background().post([temp]() { LOG(md,Info,"QServer destroying consumer {}", temp.id()); }, 60000); } } @@ -500,53 +488,25 @@ struct MDRelayServer : bts::co2::AuthenticatedMarketDataRelayBase { }; struct QueryServer { - QueryServer(std::set symbols, - utils::Dispatcher& threadpool, + QueryServer(utils::Dispatcher& threadpool, const bts::api::ServiceContext& ctx, - std::shared_ptr& ss, + MDClient& client, metrics::MetricsContainer& stats): - db(symbols, ss, stats), + db(client, stats), relay(db, threadpool, ctx, stats), - admin_client(ctx.locate()), - security_timer(), - health_timer() + admin_client(ctx.locate()) { } QueryServerImpl db; MDRelayServer relay; api::admin::AdminService admin_client; - std::unique_ptr security_timer; - std::unique_ptr health_timer; rxcpp::composite_subscription entitlement_sub; }; -QueryServerPtr run_query_server(const bts::api::ServiceContext& ctx, std::shared_ptr& ss, metrics::MetricsContainer& stats) { +QueryServerPtr run_query_server(const bts::api::ServiceContext& ctx, MDClient& client, metrics::MetricsContainer& stats) { auto& threadpool = utils::Dispatcher::get_named("MDQuery"); - std::set all_symbols; - Venue venue; - bts_assert(api::Venue_Parse(config::get().exchange(), &venue)); - SecurityType type; - bts_assert(api::SecurityType_Parse(config::get().type(), &type)); - get_system_md_symbols(venue, type, all_symbols); + QueryServerPtr container = std::make_shared(threadpool, ctx, client, stats); - // auto admin = ctx.locate(); - // admin.subscribe_md_entitlement_data().subscribe( - - QueryServerPtr container = std::make_shared(all_symbols, threadpool, ctx, ss, stats); - auto& db = container->db; - - // Scan for new securities every half second - container->security_timer = std::make_unique(threadpool, config::get().security_scan_interval(), [&db]() { - db.scan_securities(); - }); - container->security_timer->start(); - - // Scan for dead producers every 30 seconds - container->health_timer = std::make_unique(threadpool, config::get().recovery_interval(), [&db, &threadpool]() { - db.check_heartbeats(threadpool); - }); - container->health_timer->start(); - // Subscribe to entitlement data container->entitlement_sub = container->admin_client.subscribe_md_entitlement_data().subscribe([=](const api::admin::Md_entitlement_dataUpdate& update) { @@ -587,8 +547,8 @@ QueryServerPtr run_query_server(const bts::api::ServiceContext& ctx, std::shared return container; } -std::unique_ptr create_query_server(const std::set& symbols, std::shared_ptr& ss, metrics::MetricsContainer& stats) { - return std::make_unique(symbols, ss, stats); +std::unique_ptr create_query_server(MDClient& mdc, metrics::MetricsContainer& mets) { + return std::make_unique(mdc, mets); } }}//namespace diff --git a/src/lib/md/relay/QueryServer.h b/src/lib/md/relay/QueryServer.h index 13a8d056e..186089658 100644 --- a/src/lib/md/relay/QueryServer.h +++ b/src/lib/md/relay/QueryServer.h @@ -14,18 +14,18 @@ namespace bts { namespace api { class ServiceContext; } namespace metrics { class MetricsContainer; } namespace md { +class MDClient; typedef boost::container::flat_set Entitlements; struct IQueryServer { virtual ~IQueryServer() {} - virtual void scan_securities() = 0; virtual void query(const co2::MDRequest& request, co2::MDReply* reply, const Entitlements*, bool) const = 0; }; -std::unique_ptr create_query_server(const std::set&, std::shared_ptr&, metrics::MetricsContainer&); +std::unique_ptr create_query_server(MDClient&, metrics::MetricsContainer&); struct QueryServer; typedef std::shared_ptr QueryServerPtr; -QueryServerPtr run_query_server(const bts::api::ServiceContext&, std::shared_ptr&, metrics::MetricsContainer&); +QueryServerPtr run_query_server(const bts::api::ServiceContext&, MDClient&, metrics::MetricsContainer&); }}//namespace diff --git a/src/lib/md/test/md2_tests.cpp b/src/lib/md/test/md2_tests.cpp index 35c8a828e..69413197a 100644 --- a/src/lib/md/test/md2_tests.cpp +++ b/src/lib/md/test/md2_tests.cpp @@ -1,9 +1,9 @@ - #include "catch/catch.hpp" #include "../SettlementRecorder.h" #include "../coconut2.h" #include "../relay/QueryServer.h" #include "../SecurityCache.h" +#include "../MDClient.h" #include "Co2TestInstrument.h" #include "lib/config/config.h" #include "lib/os/os.h" @@ -370,7 +370,7 @@ public: UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); SecurityCache cache; - MDConsumer consumer(symbols, nullptr, cache); + MDConsumer consumer(cache, nullptr, &symbols, true, TEST_PLATFORM); vector consumer_securities; // Poll for security list - none visible until they have keys @@ -441,7 +441,7 @@ public: UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); SecurityCache cache; - MDConsumer consumer(symbols, nullptr, cache); + MDConsumer consumer(cache, nullptr, &symbols, true, TEST_PLATFORM); // Add the on_security callback vector securities; @@ -481,7 +481,7 @@ public: UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); SecurityCache cache; - MDConsumer consumer(symbols, nullptr, cache); + MDConsumer consumer(cache, nullptr, &symbols, true, TEST_PLATFORM); // Get a handle to 24 consumers which haven't been created yet vector consumers; @@ -539,7 +539,7 @@ public: UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); SecurityCache cache; - MDConsumer consumer(symbols, nullptr, cache); + MDConsumer consumer(cache, nullptr, &symbols, true, TEST_PLATFORM); // Get a handle to 24 consumers which haven't been created yet vector consumers; @@ -598,9 +598,9 @@ public: assign_ids(producers.begin(), producers.end()); // Symbol list for the consumer - auto syms = get_test_symbols(); + auto symbols = get_test_symbols(); SecurityCache cache; - MDConsumer consumer = MDConsumer(syms, nullptr, cache); + MDConsumer consumer(cache, nullptr, &symbols, true, TEST_PLATFORM); // Add the on_security callback - callback fires for existing securities vector securities; @@ -626,20 +626,23 @@ public: for (auto& pro : producers) { pro.update_bbo(TimeStamp::now(), PriceSize({1, 0}, 1), PriceSize({2, 0}, 2)); } // Symbol list for the consumer - MDRecoverableConsumer consumer(get_test_symbols(), nullptr); - map> products; + SecurityCache cache; + auto symbols = get_test_symbols(); + MDClient client(nullptr, nullptr, &symbols); + MDConsumer consumer = client.current_consumer(); + map products; consumer.on_security([&](const SecurityPtr& security, MDConsumerHandle handle) { REQUIRE(security->id() == handle.key()); - products.insert(make_pair(security->id(), consumer.get_managed_handle(security->id()))); + products.insert(make_pair(security->id(), handle)); }); - consumer.scan_securities(); + client.scan_securities(); REQUIRE(products.size() == producers.size()); - // Choose a handle + // Check handles for (auto& pair : products) { auto& handle = pair.second; - REQUIRE(handle->is_initialized()); - REQUIRE(handle->is_heart_beating()); + REQUIRE(handle.is_initialized()); + REQUIRE(handle.is_heart_beating()); } // Delete the producer @@ -661,15 +664,15 @@ public: try { for (auto& pair : products) { auto& handle = pair.second; - REQUIRE(handle->is_initialized()); - REQUIRE(handle->is_heart_beating() == false); + REQUIRE(handle.is_initialized()); + REQUIRE(handle.is_heart_beating() == false); } } catch(std::exception&) { LOG(md,Error,"Failed to validate heartbeat while producer is down."); REQUIRE(false); } - // Create a new publisher and attempt to recover + // Create a new producer and attempt to recover shared_ptr producer2; try { producer2 = make_shared(TEST_PLATFORM); @@ -689,16 +692,30 @@ public: } // Attempt to recover the managed handles - REQUIRE(consumer.try_recover()); + int on_consumer_count = 0; + auto handle = client.on_consumer().subscribe([&](const MDConsumer& c) { + ++on_consumer_count; + consumer = c; + products.clear(); + }); + client.check_heartbeats(); + REQUIRE(on_consumer_count == 1); + REQUIRE(products.empty()); + // Consumer has been updated + consumer.on_security([&](const SecurityPtr& security, MDConsumerHandle handle) { + REQUIRE(security->id() == handle.key()); + products.insert(make_pair(security->id(), handle)); + }); + client.on_consumer().unsubscribe(handle); - // Since we know that both securities exist they should have been updated to the new section + // Check that the new product handles have been updated for (auto& pair : products) { auto& handle = pair.second; - REQUIRE(handle->is_initialized()); - REQUIRE(handle->is_heart_beating()); + REQUIRE(handle.is_initialized()); + REQUIRE(handle.is_heart_beating()); TimeStamp tzero; PriceSize bid, ask; - REQUIRE(handle->get_direct_book(tzero, 1, &bid, &ask) == HandleReadResult::SUCCESS); + REQUIRE(handle.get_direct_book(tzero, 1, &bid, &ask) == HandleReadResult::SUCCESS); REQUIRE(bid.price() == 4.0); REQUIRE(bid.size() == 5); REQUIRE(ask.price() == 6.0); @@ -744,7 +761,7 @@ public: UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); SecurityCache cache; - MDConsumer consumer(symbols, nullptr, cache); + MDConsumer consumer(cache, nullptr, &symbols, true, TEST_PLATFORM); set keys; consumer.walk_security_list([&](const SecurityPtr& security, MDConsumerHandle) { keys.insert(security->id()); }); @@ -789,7 +806,7 @@ public: csymbols.insert(UserSymbol("S3", VENUE_BTS, SECURITYTYPE_SPREAD)); csymbols.insert(UserSymbol("S4", VENUE_BTS, SECURITYTYPE_SPREAD)); SecurityCache cache; - MDConsumer consumer(csymbols, nullptr, cache); + MDConsumer consumer(cache, nullptr, &csymbols, true, TEST_PLATFORM); auto check_sec = [&](const SecurityPtr& security, MDConsumerHandle handle) { REQUIRE(security->id() == handle.key()); @@ -836,7 +853,7 @@ public: csymbols.insert(UserSymbol("OO", VENUE_BTS, SECURITYTYPE_OPTION)); csymbols.insert(UserSymbol("SS", VENUE_BTS, SECURITYTYPE_SPREAD)); SecurityCache cache; - MDConsumer consumer(csymbols, nullptr, cache); + MDConsumer consumer(cache, nullptr, &csymbols, true, TEST_PLATFORM); auto check_sec = [&](const SecurityPtr& security, MDConsumerHandle handle) { REQUIRE(security->id() == handle.key()); @@ -888,7 +905,7 @@ public: consumer_symbols.insert(UserSymbol("OO", VENUE_BTS, SECURITYTYPE_OPTION)); consumer_symbols.insert(UserSymbol("SS", VENUE_BTS, SECURITYTYPE_SPREAD)); SecurityCache cache; - MDConsumer consumer(consumer_symbols, nullptr, cache); + MDConsumer consumer(cache, nullptr, &consumer_symbols, true, TEST_PLATFORM); size_t security_count = 0; // Indirection for check function @@ -957,7 +974,7 @@ public: set consumer_symbols; consumer_symbols.insert(UserSymbol("OO", VENUE_BTS, SECURITYTYPE_OPTION)); SecurityCache cache; - MDConsumer consumer(consumer_symbols, nullptr, cache); + MDConsumer consumer(cache, nullptr, &consumer_symbols, true, TEST_PLATFORM); auto h1 = consumer.get_handle(k1); REQUIRE(h1.is_initialized()); auto h2 = consumer.get_handle(k2); @@ -1242,7 +1259,8 @@ struct Co2RelayTests { symbols.insert(future_symbol); UserSymbol option_symbol("FOOX", VENUE_BTS, SECURITYTYPE_OPTION); symbols.insert(option_symbol); - MDConsumer consumer(symbols, nullptr, cache); + SecurityCache cache; + MDConsumer consumer(cache, nullptr, &symbols); return consumer; } @@ -1257,7 +1275,13 @@ struct Co2RelayTests { } metrics::MetricsContainer _stats; - std::unique_ptr setup_query_server(MDProducer& producer, Entitlements* out_entitlements, vector* out_producers) { + + std::unique_ptr setup_query_server( + MDClient& client, + MDProducer& producer, + Entitlements* out_entitlements, + vector* out_producers) { + // Create a bunch of securities for (size_t x = 0; x < 100; ++x) { // That's right. Expirations daily. @@ -1277,10 +1301,9 @@ struct Co2RelayTests { for (const auto& s : symbols) { out_entitlements->insert(s.symbol()); } - shared_ptr empty; - auto server = create_query_server(symbols, empty, _stats); + auto server = create_query_server(client, _stats); REQUIRE(server); - server->scan_securities(); + client.scan_securities(); return server; } @@ -1293,7 +1316,8 @@ struct Co2RelayTests { // initialize data vector producers; - auto server = setup_query_server(producer, &entitlements, &producers); + MDClient client; + auto server = setup_query_server(client, producer, &entitlements, &producers); // assign prices PriceSize price35 = PriceSize_from_double<2>(3.5, 5); @@ -1423,7 +1447,8 @@ struct Co2RelayTests { // initialize data vector producers; Entitlements entitlements; - auto server = setup_query_server(producer, &entitlements, &producers); + MDClient client; + auto server = setup_query_server(client, producer, &entitlements, &producers); // assign prices to all levels auto set_direct = [&](PriceSize bid, PriceSize ask) { @@ -1600,7 +1625,7 @@ namespace ImpliedTests { UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); SecurityCache cache; - MDConsumer consumer(symbols, nullptr, cache); + MDConsumer consumer(cache, nullptr, &symbols); vector consumer_securities; // Poll for security list - none visible until they have keys size_t count = consumer.walk_security_list([&](const SecurityPtr& security, MDConsumerHandle) {consumer_securities.push_back(*security);}); @@ -2373,4 +2398,3 @@ TEST_CASE("IntPrice", "[md]") { }}} - diff --git a/src/lib/orderman/events/NewSecuritiesEvent.h b/src/lib/orderman/events/NewSecuritiesEvent.h index fddcc0e6d..be5f3c505 100644 --- a/src/lib/orderman/events/NewSecuritiesEvent.h +++ b/src/lib/orderman/events/NewSecuritiesEvent.h @@ -7,6 +7,10 @@ namespace bts { namespace orderman { + //todo: + // anywhere that uses this + // get a SecurityOracle + // which keeps a shared_ptr and uses it class NewSecuritiesEvent : public mercury::Event { diff --git a/src/lib/orderman/server/OrderServiceServer.h b/src/lib/orderman/server/OrderServiceServer.h index 75c558932..999b46790 100644 --- a/src/lib/orderman/server/OrderServiceServer.h +++ b/src/lib/orderman/server/OrderServiceServer.h @@ -8,9 +8,6 @@ namespace bts { namespace trace {class PerfLog;} -namespace api { -class ISyntheticSecurityService; -} namespace metrics { class MetricsContainer; } diff --git a/src/server/sparkd/cmds/Run.cpp b/src/server/sparkd/cmds/Run.cpp index a1840cf77..2c13a48a9 100644 --- a/src/server/sparkd/cmds/Run.cpp +++ b/src/server/sparkd/cmds/Run.cpp @@ -58,13 +58,12 @@ int Run::execute(std::vector const &) { user::AuthServiceServer login_server(sc); md::run_market_datapublishers(); - std::shared_ptr synthetic_service = - std::make_shared(sc); - md::MDClient mdclient(utils::Dispatcher::current(), synthetic_service); + api::synthetic::SyntheticSecurityService synthetic_service(sc); + md::MDClient mdclient(&utils::Dispatcher::background(), &synthetic_service); md::QueryServerPtr md_query_server( - md::run_query_server(sc, synthetic_service, stats)); + md::run_query_server(sc, mdclient, stats)); // initialize order plugins orderman::DynamicOrderRegistry dyn_order_registry; diff --git a/src/server/sparkd/cmds/Web.cpp b/src/server/sparkd/cmds/Web.cpp index 8cc3ad12e..029ae34bc 100644 --- a/src/server/sparkd/cmds/Web.cpp +++ b/src/server/sparkd/cmds/Web.cpp @@ -10,6 +10,7 @@ #include "lib/metrics/system_metrics.h" #include "server/sparkd/http/WebServer.h" #include "lib/md/relay/QueryServer.h" +#include "lib/md/MDClient.h" #include "lib/md/SyntheticSecurityService.h" #include "server/sparkd/user/AdminServiceServer.h" #include "server/sparkd/user/AuthServiceServer.h" @@ -39,17 +40,14 @@ int Web::execute(const std::vector& args) { md::run_market_datapublishers(); } - std::shared_ptr synthetic_service = - std::make_shared(sc); + api::synthetic::SyntheticSecurityService synthetic_service(sc); + md::MDClient mdclient(&utils::Dispatcher::current(), &synthetic_service); md::QueryServerPtr md_query_server( - md::run_query_server(sc, synthetic_service, stats)); + md::run_query_server(sc, mdclient, stats)); admin::AdminServiceServer admin_service(sc); - - - std::this_thread::sleep_until( - std::chrono::time_point::max()); + utils::Dispatcher::current().run(); return 0; } From 8d240bff602a3c14ec041e6a75c0dbdc0bfd81a7 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 1 Feb 2019 17:14:14 -0500 Subject: [PATCH 43/85] Fix fucked up curlies --- .../Predicates/Data/Events/OrderFilledEventPredicate.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/client/spark/Predicates/Data/Events/OrderFilledEventPredicate.cs b/src/client/spark/Predicates/Data/Events/OrderFilledEventPredicate.cs index b6b1f749f..80767eec6 100644 --- a/src/client/spark/Predicates/Data/Events/OrderFilledEventPredicate.cs +++ b/src/client/spark/Predicates/Data/Events/OrderFilledEventPredicate.cs @@ -14,9 +14,10 @@ namespace mbts { { OrderFilledEventPredicate.PredicateTypeOneofCase.AnyOrder, PredicateBuilderVMWrapper.create(p => p.security = null) }, { OrderFilledEventPredicate.PredicateTypeOneofCase.SecurityOrder, SecurityCriteriaVMFactory.create(bus, o => o.security?.is_value ?? new SecurityCriteria(), (p, s) => { - if (p.security == null) -{ p.security = new SecurityPredicate(); -} p.security.is_value = s; + if (p.security == null) { + p.security = new SecurityPredicate(); + } + p.security.is_value = s; }) } }; From 48849fc631ab5ae3f71e84aba5474163aff217a3 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 1 Feb 2019 17:14:26 -0500 Subject: [PATCH 44/85] Fix md unit tests --- .vscode/launch.json | 2 +- src/lib/md/MDClient.cpp | 4 ++-- src/lib/md/MDClient.h | 2 +- src/lib/md/test/md2_tests.cpp | 27 ++++++++++++++++++++------- 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index cf754974d..0fdb97700 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -209,7 +209,7 @@ "type": "cppdbg", "request": "launch", "program": "/home/bts/spark/build/ninja_debug/src/server/sparkd/sparkd", - "args": ["test", "[synthetic]"], + "args": ["test", "[md]"], "stopAtEntry": false, "cwd": "/home/bts/spark/build/ninja_debug/src/server/sparkd/", "environment": [], diff --git a/src/lib/md/MDClient.cpp b/src/lib/md/MDClient.cpp index 6822c4d20..1cb454971 100644 --- a/src/lib/md/MDClient.cpp +++ b/src/lib/md/MDClient.cpp @@ -21,7 +21,7 @@ struct MDClientImpl{ MDClientImpl( utils::Dispatcher* d, api::ISyntheticSecurityService* s, - std::set* symbols): + const std::set* symbols): _dispatcher(d), _ss(s), _security_timer(), @@ -71,7 +71,7 @@ struct MDClientImpl{ MDClient::MDClient( utils::Dispatcher* d, api::ISyntheticSecurityService* s, - std::set* symbols) : + const std::set* symbols) : _self(new MDClientImpl(d,s, symbols)) { } MDClient::~MDClient() { diff --git a/src/lib/md/MDClient.h b/src/lib/md/MDClient.h index 6de826de1..47925bc88 100644 --- a/src/lib/md/MDClient.h +++ b/src/lib/md/MDClient.h @@ -20,7 +20,7 @@ public: MDClient( utils::Dispatcher* d=nullptr, api::ISyntheticSecurityService* ss=nullptr, - std::set* s=nullptr); + const std::set* s=nullptr); ~MDClient(); // Check heartbeats and attempt recovery if they are bad diff --git a/src/lib/md/test/md2_tests.cpp b/src/lib/md/test/md2_tests.cpp index 69413197a..9f0bf2aca 100644 --- a/src/lib/md/test/md2_tests.cpp +++ b/src/lib/md/test/md2_tests.cpp @@ -1276,6 +1276,14 @@ struct Co2RelayTests { metrics::MetricsContainer _stats; + const std::set setup_symbols() { + std::set result; + for (const auto& symbol : { "FOO", "FOOF" }) { + result.insert({symbol, VENUE_BTS, SECURITYTYPE_FUTURE}); + } + return result; + } + std::unique_ptr setup_query_server( MDClient& client, MDProducer& producer, @@ -1310,13 +1318,16 @@ struct Co2RelayTests { void relay_nans() { MDProducer producer(TEST_PLATFORM); - UserSymbol symbol("FOO", VENUE_BTS, SECURITYTYPE_FUTURE); Entitlements entitlements; // initialize data vector producers; - MDClient client; + // + // todo: setup_query_server should use client.symbols() instead of it's own list + // + const auto symbols = setup_symbols(); + MDClient client(nullptr, nullptr, &symbols); auto server = setup_query_server(client, producer, &entitlements, &producers); // assign prices @@ -1328,14 +1339,15 @@ struct Co2RelayTests { // Request via query server co2::MDRequest request; - for (const auto& symbol : { "FOO", "FOOF" }) { + for (const auto& symbol : symbols) { auto* chan = request.add_channels(); - chan->set_type(SECURITYTYPE_FUTURE); - chan->set_symbol(symbol); - chan->set_exchange(VENUE_BTS); + chan->set_type(symbol.type()); + chan->set_symbol(symbol.symbol()); + chan->set_exchange(symbol.exchange()); chan->set_flags(co2::MDRequest::BBO | co2::MDRequest::SECURITY); chan->set_entitled(true); } + // This request should include everything co2::MDReply reply; @@ -1447,7 +1459,8 @@ struct Co2RelayTests { // initialize data vector producers; Entitlements entitlements; - MDClient client; + const auto symbols = setup_symbols(); + MDClient client(nullptr, nullptr, &symbols); auto server = setup_query_server(client, producer, &entitlements, &producers); // assign prices to all levels From dd6b415ce77e7e7ba71cb1ebfc6dd087da9e6eb9 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 5 Feb 2019 11:06:55 -0600 Subject: [PATCH 45/85] remove trade key column from Time and Sales --- .../spark/Modules/Market/TimeAndSales/TimeAndSalesVM.cs | 7 +------ .../Modules/Market/TimeAndSales/TimeAndSalesView.xaml.cs | 2 +- .../spark/Modules/Market/TimeAndSalesWindowFactory.cs | 6 ++---- .../spark/view/Controls/DIYGrid/Columns/ColumnBase.cs | 2 -- 4 files changed, 4 insertions(+), 13 deletions(-) diff --git a/src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesVM.cs b/src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesVM.cs index b95459cc6..d566f7441 100644 --- a/src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesVM.cs +++ b/src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesVM.cs @@ -38,7 +38,6 @@ namespace mbts.spark { price_filter.load_state(filters.price); size_filter.load_state(filters.size); aggressor_filter.load_state(filters.aggressor_sides); - trade_key_filter.load_state(filters.trade_key); security_key_filter.load_state(filters.security_key); } // Columns @@ -47,7 +46,6 @@ namespace mbts.spark { add_column(TimeAndSalesView.Columns.Price, DefaultColumnHeaders.Price, DefaultColumnWidths.Price, nameof(Row.price), price_filter, DataGridFilterKnownTemplates.DoubleMinMax, System.Windows.HorizontalAlignment.Right); add_column(TimeAndSalesView.Columns.Size, DefaultColumnHeaders.Size, DefaultColumnWidths.Size, nameof(Row.size), size_filter, DataGridFilterKnownTemplates.UIntMinMax, System.Windows.HorizontalAlignment.Right); add_column(TimeAndSalesView.Columns.Aggressor, DefaultColumnHeaders.Aggressor, DefaultColumnWidths.Aggressor, nameof(Row.aggressor_side), aggressor_filter, DataGridFilterKnownTemplates.EnumValues); - add_column(TimeAndSalesView.Columns.TradeKey, DefaultColumnHeaders.TradeKey, DefaultColumnWidths.TradeKey, nameof(Row.trade_key), trade_key_filter, DataGridFilterKnownTemplates.Regex); add_column(TimeAndSalesView.Columns.SecurityKey, DefaultColumnHeaders.SecurityKey, DefaultColumnWidths.SecurityKey, nameof(Row.security_key), security_key_filter, DataGridFilterKnownTemplates.Regex); void add_column(TimeAndSalesView.Columns col_key, string header, double width, string sort_member_path, IDataGridColumnFilter filter, string filter_template, System.Windows.HorizontalAlignment h_alignment = System.Windows.HorizontalAlignment.Left) => columns.Add(new TimeAndSalesColumn( @@ -66,7 +64,7 @@ namespace mbts.spark { _resources.AddRange( show_ladder, DataGridFilterHelper.hook_filters( - rows_view, time_filter, security_filter, price_filter, size_filter, aggressor_filter, trade_key_filter, security_key_filter), + rows_view, time_filter, security_filter, price_filter, size_filter, aggressor_filter, security_key_filter), settings_manager.subscribe_current().OnDispatcher().Subscribe(grid_settings_manager) ); rows_view.set_sort_descriptions(startup_data.sorting_criterion, columns); @@ -103,7 +101,6 @@ namespace mbts.spark { public IDataGridColumnPersistentFilter size_filter { get; } = new UIntMinMax(r => r.size, nameof(Row.size)); public IDataGridColumnPersistentFilter aggressor_filter { get; } = DiscreteValuesFilterVM.Create(EnumHelper.get_values().ToArray(), r => r.aggressor_side, nameof(Row.aggressor_side)); public IDataGridColumnPersistentFilter security_key_filter { get; } = new RegexFilterVM(r => r.security_key.ToString(), nameof(Row.security_key)); - public IDataGridColumnPersistentFilter trade_key_filter { get; } = new RegexFilterVM(r => r.trade_key.ToString(), nameof(Row.trade_key)); FilterStates _get_filter_states() => new FilterStates { @@ -111,7 +108,6 @@ namespace mbts.spark { security = security_filter.get_state(), price = price_filter.get_state(), size = size_filter.get_state(), - trade_key = trade_key_filter.get_state(), security_key = security_key_filter.get_state(), aggressor_sides = aggressor_filter.get_state() }; @@ -175,7 +171,6 @@ namespace mbts.spark { case TimeAndSalesView.Columns.Price: return _price_text; case TimeAndSalesView.Columns.Size: return _size_text; case TimeAndSalesView.Columns.Aggressor: return _aggressor_text; - case TimeAndSalesView.Columns.TradeKey: return _trade_key_text; case TimeAndSalesView.Columns.SecurityKey: return _security_key_text; default: return null; } diff --git a/src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesView.xaml.cs b/src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesView.xaml.cs index fff762c5c..ab066f2dd 100644 --- a/src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesView.xaml.cs +++ b/src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesView.xaml.cs @@ -7,7 +7,7 @@ namespace mbts.ui.spark { /// Interaction logic for TimeAndSalesView.xaml /// partial class TimeAndSalesView : UserControl { - public enum Columns { Time, Security, Price, Size, OrderType, TradeKey, SecurityKey, Aggressor } + public enum Columns { Time, Security, Price, Size, OrderType, SecurityKey, Aggressor } public TimeAndSalesView() => InitializeComponent(); diff --git a/src/client/spark/Modules/Market/TimeAndSalesWindowFactory.cs b/src/client/spark/Modules/Market/TimeAndSalesWindowFactory.cs index 16b1ba341..a5719bdbc 100644 --- a/src/client/spark/Modules/Market/TimeAndSalesWindowFactory.cs +++ b/src/client/spark/Modules/Market/TimeAndSalesWindowFactory.cs @@ -84,7 +84,6 @@ namespace bts.shell { public static GridColumnSettings price => new GridColumnSettings(col_id(Columns.Price), DefaultColumnHeaders.Price, width: DefaultColumnWidths.Price, horizontal_alignment: System.Windows.HorizontalAlignment.Right); public static GridColumnSettings size => new GridColumnSettings(col_id(Columns.Size), DefaultColumnHeaders.Size, width: DefaultColumnWidths.Size, horizontal_alignment: System.Windows.HorizontalAlignment.Right); public static GridColumnSettings aggressor => new GridColumnSettings(col_id(Columns.Aggressor), DefaultColumnHeaders.Aggressor, width: DefaultColumnWidths.Aggressor); - public static GridColumnSettings trade_key => new GridColumnSettings(col_id(Columns.TradeKey), DefaultColumnHeaders.TradeKey, width: DefaultColumnWidths.TradeKey); public static GridColumnSettings security_key => new GridColumnSettings(col_id(Columns.SecurityKey), DefaultColumnHeaders.SecurityKey, width: DefaultColumnWidths.SecurityKey); public static IEnumerable default_sorting_criterion { @@ -100,7 +99,6 @@ namespace bts.shell { yield return price; yield return size; yield return aggressor; - yield return trade_key; yield return security_key; } } @@ -117,8 +115,8 @@ namespace bts.shell { public DoubleMinMaxState price; [ProtoMember(4)] public UIntMinMaxState size; - [ProtoMember(5)] - public string trade_key; + // [ProtoMember(5)] + // ProtoMember 5 cannot be used further [ProtoMember(6)] public string security_key; [ProtoMember(7)] diff --git a/src/client/spark/view/Controls/DIYGrid/Columns/ColumnBase.cs b/src/client/spark/view/Controls/DIYGrid/Columns/ColumnBase.cs index 02ff43b49..336861a5c 100644 --- a/src/client/spark/view/Controls/DIYGrid/Columns/ColumnBase.cs +++ b/src/client/spark/view/Controls/DIYGrid/Columns/ColumnBase.cs @@ -36,7 +36,6 @@ namespace mbts.ui.controls { Symbol = 78, Time = 112, TimeInForce = 47, - TradeKey = 76, User = 53, Aggressor = 80, Volume = 74, @@ -71,7 +70,6 @@ namespace mbts.ui.controls { ImpliedSize = "Qty (I)", Symbol = "Symbol", TimeInForce = "TIF", - TradeKey = "Trade Key", User = "User", Time = "Time", Aggressor = "Aggressor", From 5b40b4b4ca60d205c89676143757d09443e213a4 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 5 Feb 2019 11:14:40 -0600 Subject: [PATCH 46/85] Hiding Security Key column in default settings in all grids --- .../spark/Modules/Market/MarketView/MarketViewWindowFactory.cs | 2 +- src/client/spark/Modules/Market/TimeAndSalesWindowFactory.cs | 2 +- .../spark/Modules/Spreader/SyntheticManagementWindowFactory.cs | 2 +- src/client/spark/Modules/Trading/Fills/FillsWindowFactory.cs | 2 +- .../spark/Modules/Trading/OrderBook/OrderBookWindowFactory.cs | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/client/spark/Modules/Market/MarketView/MarketViewWindowFactory.cs b/src/client/spark/Modules/Market/MarketView/MarketViewWindowFactory.cs index a5d80795c..44759c0c5 100644 --- a/src/client/spark/Modules/Market/MarketView/MarketViewWindowFactory.cs +++ b/src/client/spark/Modules/Market/MarketView/MarketViewWindowFactory.cs @@ -96,7 +96,7 @@ namespace bts.shell { public static GridColumnSettings day_change => new GridColumnSettings(col_id(Columns.DayChange), DefaultColumnHeaders.DayChange, width: DefaultColumnWidths.Price, horizontal_alignment: System.Windows.HorizontalAlignment.Right); public static GridColumnSettings session_low => new GridColumnSettings(col_id(Columns.SessionLow), DefaultColumnHeaders.SessionLow, width: DefaultColumnWidths.Price, horizontal_alignment: System.Windows.HorizontalAlignment.Right); public static GridColumnSettings session_high => new GridColumnSettings(col_id(Columns.SessionHigh), DefaultColumnHeaders.SessionHigh, width: DefaultColumnWidths.Price, horizontal_alignment: System.Windows.HorizontalAlignment.Right); - public static GridColumnSettings security_key => new GridColumnSettings(col_id(Columns.SecurityKey), DefaultColumnHeaders.SecurityKey, width: DefaultColumnWidths.SecurityKey); + public static GridColumnSettings security_key => new GridColumnSettings(col_id(Columns.SecurityKey), DefaultColumnHeaders.SecurityKey, is_visible: false, width: DefaultColumnWidths.SecurityKey); diff --git a/src/client/spark/Modules/Market/TimeAndSalesWindowFactory.cs b/src/client/spark/Modules/Market/TimeAndSalesWindowFactory.cs index a5719bdbc..efb01ab83 100644 --- a/src/client/spark/Modules/Market/TimeAndSalesWindowFactory.cs +++ b/src/client/spark/Modules/Market/TimeAndSalesWindowFactory.cs @@ -84,7 +84,7 @@ namespace bts.shell { public static GridColumnSettings price => new GridColumnSettings(col_id(Columns.Price), DefaultColumnHeaders.Price, width: DefaultColumnWidths.Price, horizontal_alignment: System.Windows.HorizontalAlignment.Right); public static GridColumnSettings size => new GridColumnSettings(col_id(Columns.Size), DefaultColumnHeaders.Size, width: DefaultColumnWidths.Size, horizontal_alignment: System.Windows.HorizontalAlignment.Right); public static GridColumnSettings aggressor => new GridColumnSettings(col_id(Columns.Aggressor), DefaultColumnHeaders.Aggressor, width: DefaultColumnWidths.Aggressor); - public static GridColumnSettings security_key => new GridColumnSettings(col_id(Columns.SecurityKey), DefaultColumnHeaders.SecurityKey, width: DefaultColumnWidths.SecurityKey); + public static GridColumnSettings security_key => new GridColumnSettings(col_id(Columns.SecurityKey), DefaultColumnHeaders.SecurityKey, is_visible:false, width: DefaultColumnWidths.SecurityKey); public static IEnumerable default_sorting_criterion { get { diff --git a/src/client/spark/Modules/Spreader/SyntheticManagementWindowFactory.cs b/src/client/spark/Modules/Spreader/SyntheticManagementWindowFactory.cs index afb327aa7..e676dd306 100644 --- a/src/client/spark/Modules/Spreader/SyntheticManagementWindowFactory.cs +++ b/src/client/spark/Modules/Spreader/SyntheticManagementWindowFactory.cs @@ -66,7 +66,7 @@ namespace bts.shell { public static GridColumnSettings description => new GridColumnSettings(col_id(Columns.Description), DefaultColumnHeaders.Description, width: DefaultColumnWidths.Description); public static GridColumnSettings n_legs => new GridColumnSettings(col_id(Columns.n_legs), DefaultColumnHeaders.N_Legs, width: DefaultColumnWidths.N_Legs); public static GridColumnSettings n_working_orders => new GridColumnSettings(col_id(Columns.n_working_orders), DefaultColumnHeaders.N_Working_Orders, width: DefaultColumnWidths.N_Working_Orders); - public static GridColumnSettings security_key => new GridColumnSettings(col_id(Columns.SecurityKey), DefaultColumnHeaders.SecurityKey, width: DefaultColumnWidths.SecurityKey); + public static GridColumnSettings security_key => new GridColumnSettings(col_id(Columns.SecurityKey), DefaultColumnHeaders.SecurityKey, is_visible: false, width: DefaultColumnWidths.SecurityKey); public static IEnumerable default_sorting_criterion { get { diff --git a/src/client/spark/Modules/Trading/Fills/FillsWindowFactory.cs b/src/client/spark/Modules/Trading/Fills/FillsWindowFactory.cs index b4e789c2b..69c9df1dc 100644 --- a/src/client/spark/Modules/Trading/Fills/FillsWindowFactory.cs +++ b/src/client/spark/Modules/Trading/Fills/FillsWindowFactory.cs @@ -86,7 +86,7 @@ namespace bts.shell { public static GridColumnSettings fill_kind => new GridColumnSettings(col_id(Columns.FillKind), DefaultColumnHeaders.FillKind, width: DefaultColumnWidths.FillKind); public static GridColumnSettings fill_key => new GridColumnSettings(col_id(Columns.FillKey), DefaultColumnHeaders.FillKey, width: DefaultColumnWidths.FillKey); public static GridColumnSettings order_key => new GridColumnSettings(col_id(Columns.OrderKey), DefaultColumnHeaders.OrderKey, width: DefaultColumnWidths.OrderKey); - public static GridColumnSettings security_key => new GridColumnSettings(col_id(Columns.SecurityKey), DefaultColumnHeaders.SecurityKey, width: DefaultColumnWidths.SecurityKey); + public static GridColumnSettings security_key => new GridColumnSettings(col_id(Columns.SecurityKey), DefaultColumnHeaders.SecurityKey, is_visible: false, width: DefaultColumnWidths.SecurityKey); public static IEnumerable default_sorting_criterion { get { yield return new SortingCriteria(fill_time.id, false); diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookWindowFactory.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookWindowFactory.cs index 5912ac397..0733cb42b 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookWindowFactory.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookWindowFactory.cs @@ -112,7 +112,7 @@ namespace bts.shell { public static GridColumnSettings last_price => new GridColumnSettings(Columns.LastPrice.ToString(), "Last Filled", width: DIYGrid.DefaultColumnWidths.LastPrice, horizontal_alignment: HorizontalAlignment.Right); public static GridColumnSettings status => new GridColumnSettings(Columns.Status.ToString(), "Status", width: 80); public static GridColumnSettings last_message => new GridColumnSettings(Columns.LastMessage.ToString(), "Last Message", width: DIYGrid.DefaultColumnWidths.LastMessage); - public static GridColumnSettings security_key => new GridColumnSettings(Columns.SecurityKey.ToString(), DIYGrid.DefaultColumnHeaders.SecurityKey, width: DIYGrid.DefaultColumnWidths.SecurityKey); + public static GridColumnSettings security_key => new GridColumnSettings(Columns.SecurityKey.ToString(), DIYGrid.DefaultColumnHeaders.SecurityKey, is_visible: false, width: DIYGrid.DefaultColumnWidths.SecurityKey); public static GridColumnSettings order_key => new GridColumnSettings(Columns.OrderKey.ToString(), DIYGrid.DefaultColumnHeaders.OrderKey, width: DIYGrid.DefaultColumnWidths.OrderKey); public static GridColumnSettings parent_key => new GridColumnSettings(Columns.ParentKey.ToString(), "Parent Key", width: DIYGrid.DefaultColumnWidths.ParentKey); From 1864c27584759e3af163c0a130ea49c03bf43437 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 5 Feb 2019 14:35:10 -0600 Subject: [PATCH 47/85] Anything that used to hit get_cme_gen_open now hits get_start_of_trading_day --- .../spark/core/data/Proto/DateTimeProvider.cs | 2 +- .../DataGridEx/Filters/VM/TimeFilterVM.cs | 2 +- .../spark/view/Spark/PAndL/PnLCalculator.cs | 42 +++++++++---------- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/client/spark/core/data/Proto/DateTimeProvider.cs b/src/client/spark/core/data/Proto/DateTimeProvider.cs index 8224670bb..e0df81f72 100644 --- a/src/client/spark/core/data/Proto/DateTimeProvider.cs +++ b/src/client/spark/core/data/Proto/DateTimeProvider.cs @@ -51,7 +51,7 @@ namespace bts.proto { static class DateTimeProviderExtensions { public static DateTime get_date_time(this DateTimeProvider dtp) { switch (dtp.mode) { - case DateTimeProviderMode.CMEOpen: return MarketSchedule.get_cme_gen_open(); + case DateTimeProviderMode.CMEOpen: return MarketSchedule.get_start_of_trading_day(); case DateTimeProviderMode.Today: return DateTime.Today; case DateTimeProviderMode.Explicit: default: return dtp.explicit_date.GetValueOrDefault(); diff --git a/src/client/spark/view/Controls/DataGridEx/Filters/VM/TimeFilterVM.cs b/src/client/spark/view/Controls/DataGridEx/Filters/VM/TimeFilterVM.cs index c89353f5a..da48d458d 100644 --- a/src/client/spark/view/Controls/DataGridEx/Filters/VM/TimeFilterVM.cs +++ b/src/client/spark/view/Controls/DataGridEx/Filters/VM/TimeFilterVM.cs @@ -14,7 +14,7 @@ namespace mbts.spark.filters { TimeFilterMode _mode; TimeTextTuple _begin, _end; DateTime? _begin_dt, _end_dt; - readonly DateTime _cme_open = MarketSchedule.get_cme_gen_open(); + readonly DateTime _cme_open = MarketSchedule.get_start_of_trading_day(); public TimeFilterVM(Func get_row_val, string prop_name) : base(get_row_val, prop_name) => _set_default_values(); diff --git a/src/client/spark/view/Spark/PAndL/PnLCalculator.cs b/src/client/spark/view/Spark/PAndL/PnLCalculator.cs index 58c41f9ba..f0fb40328 100644 --- a/src/client/spark/view/Spark/PAndL/PnLCalculator.cs +++ b/src/client/spark/view/Spark/PAndL/PnLCalculator.cs @@ -28,31 +28,10 @@ namespace mbts.spark { static readonly int sod_hh = (int)config.Configuration.get().sod_hh; static readonly int sod_mm = (int)config.Configuration.get().sod_mm; static readonly TimeZoneInfo central_tzi = TimeZoneInfo.FindSystemTimeZoneById(cst_id); - static readonly DateTime _start_of_trading_day_utc; - static readonly DateTime _adjusted_central_sod; - - - static readonly DateTime unadjusted_central_sod = TimeZoneInfo.ConvertTimeFromUtc( - DateTime.UtcNow.Date.Add(new TimeSpan(sod_hh, sod_mm, 0)), - central_tzi - ); static MarketSchedule() { var cme_schedule = config.Configuration.get(); _cme_gen_open_hrs = cme_schedule?.gen_open_hour ?? 8.5; - - _adjusted_central_sod = !central_tzi.IsDaylightSavingTime(unadjusted_central_sod) ? unadjusted_central_sod.AddHours(1) : unadjusted_central_sod; - - // Note that this isn't the acutal start of the trading day - // but it's close enough for pnl purposes. - // In reality, instruments have different schedules - - // if now is less than the current sod, then we me move the sod back a day to account for the trading day start - if (_adjusted_central_sod >= TimeZoneInfo.ConvertTime(DateTime.Now, central_tzi)) { - _adjusted_central_sod = _adjusted_central_sod.AddDays(-1); - } - var local_time_zone_sod = TimeZoneInfo.ConvertTime(_adjusted_central_sod, central_tzi, TimeZoneInfo.Local); - _start_of_trading_day_utc = local_time_zone_sod.ToUniversalTime(); } // for object initializers public MarketSchedule() { } @@ -73,6 +52,27 @@ namespace mbts.spark { return last_open; } + // DateTime? used here b/c we can't set a default parameter with DateTime.Now + // since it's now + public static DateTime get_start_of_trading_day(DateTime? target_datetime = null, DateTimeKind dtk = DateTimeKind.Local ) { + if (target_datetime == null) target_datetime = DateTime.Now; + var dst_adjusted_central_sod = new DateTimeOffset( + new DateTime(target_datetime.Value.Date.Add(new TimeSpan(sod_hh, sod_mm, 0)).Ticks, DateTimeKind.Unspecified), + central_tzi.BaseUtcOffset + ); + + // if we're not currently past the start of the trading day, then the start must be yesterday + var temp = dst_adjusted_central_sod; + if (temp >= TimeZoneInfo.ConvertTime(target_datetime.Value, central_tzi)) { + temp = dst_adjusted_central_sod.AddDays(-1); + } + var local_time_zone_sod = TimeZoneInfo.ConvertTime(temp.DateTime, central_tzi, TimeZoneInfo.Local); + if (dtk == DateTimeKind.Utc) { + return local_time_zone_sod.ToUniversalTime(); + } + return local_time_zone_sod; + } + public static DateTime get_utctimestamp_as_localtime(string s) { return DateTimeOffset.ParseExact(s, "yyyyMMddHHmmssffffff", null, System.Globalization.DateTimeStyles.AssumeUniversal).ToLocalTime().LocalDateTime; } From fb059e557190a897b0b955f39dfcdcb628898154 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 5 Feb 2019 14:36:51 -0600 Subject: [PATCH 48/85] refine comment --- src/client/spark/view/Spark/PAndL/PnLCalculator.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/spark/view/Spark/PAndL/PnLCalculator.cs b/src/client/spark/view/Spark/PAndL/PnLCalculator.cs index f0fb40328..0966c538d 100644 --- a/src/client/spark/view/Spark/PAndL/PnLCalculator.cs +++ b/src/client/spark/view/Spark/PAndL/PnLCalculator.cs @@ -53,7 +53,7 @@ namespace mbts.spark { } // DateTime? used here b/c we can't set a default parameter with DateTime.Now - // since it's now + // since it's not a compile-time constant public static DateTime get_start_of_trading_day(DateTime? target_datetime = null, DateTimeKind dtk = DateTimeKind.Local ) { if (target_datetime == null) target_datetime = DateTime.Now; var dst_adjusted_central_sod = new DateTimeOffset( From 4264928cc2106bb40127115e2cefd0084cea7485 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 5 Feb 2019 15:44:40 -0500 Subject: [PATCH 49/85] Use SecurityOracle in AsyncFactory,ServiceBridge,OrderServicesData --- src/lib/md/SecurityOracle.h | 3 +- src/lib/orderman/actors/AsyncFactory.cpp | 13 ++------- src/lib/orderman/actors/AsyncFactory.h | 3 +- src/lib/orderman/actors/ServiceBridge.cpp | 29 ++++--------------- src/lib/orderman/actors/ServiceBridge.h | 7 ++--- .../actors/order_manager/OrderManager.cpp | 11 ++++--- .../actors/order_manager/OrderManager.h | 3 +- .../order_manager/OrderServicesData.cpp | 4 +-- .../actors/order_manager/OrderServicesData.h | 4 +-- src/lib/orderman/orders/NativeOrder.cpp | 1 + src/lib/orderman/securities/ISecurityDB.h | 1 - src/lib/orderman/securities/MapSecurityDB.h | 13 ++++----- .../orderman/server/OrderServiceServer.cpp | 6 ++-- 13 files changed, 36 insertions(+), 62 deletions(-) diff --git a/src/lib/md/SecurityOracle.h b/src/lib/md/SecurityOracle.h index e45ba062f..82163a1e1 100644 --- a/src/lib/md/SecurityOracle.h +++ b/src/lib/md/SecurityOracle.h @@ -1,8 +1,9 @@ #pragma once -#include "SecurityPtr.h" +#include "lib/domain_model/SecurityKey.h" #include namespace bts { +namespace api {class Security;} namespace md { class MDClient; class SecurityOracle { diff --git a/src/lib/orderman/actors/AsyncFactory.cpp b/src/lib/orderman/actors/AsyncFactory.cpp index 91b76045e..f6543a5b2 100644 --- a/src/lib/orderman/actors/AsyncFactory.cpp +++ b/src/lib/orderman/actors/AsyncFactory.cpp @@ -7,7 +7,6 @@ #include "lib/orderman/events/CreateOrderReplyEvent.h" #include "lib/orderman/events/CreateOrderRequestEvent.h" #include "lib/orderman/events/NewAdminDataEvent.h" -#include "lib/orderman/events/NewSecuritiesEvent.h" #include "lib/orderman/order_factory/OrderFactory.h" #include "lib/orderman/order_router/OrderRouter.h" #include "lib/orderman/securities/MapSecurityDB.h" @@ -16,8 +15,8 @@ namespace bts { namespace orderman { struct AsyncFactory::PImpl { - PImpl(DynamicOrderRegistry const &b) - : orders_stream(), fix_buffer_stream(), db(), order_router(), + PImpl(DynamicOrderRegistry const &b, md::SecurityOracle o) + : orders_stream(), fix_buffer_stream(), db(o), order_router(), order_factory(db, order_router, b) {} mercury::Stream orders_stream; @@ -27,7 +26,7 @@ struct AsyncFactory::PImpl { OrderFactory order_factory; }; -AsyncFactory::AsyncFactory(DynamicOrderRegistry const &b) : me(new PImpl(b)) {} +AsyncFactory::AsyncFactory(DynamicOrderRegistry const &b, md::SecurityOracle o) : me(new PImpl(b, o)) {} AsyncFactory::~AsyncFactory() {} void AsyncFactory::init() { @@ -40,12 +39,6 @@ void AsyncFactory::init() { me->order_router.update(*ev.admin_data); }); - subscribe([this](auto const &u) { - if (u.events) - for (auto const &ev : *(u.events)) - me->db.securities[SecurityKey(ev.security->id())] = ev.security; - }); - subscribe([this](const auto &u) { if (u.events) for (auto const &ev : *u.events) { diff --git a/src/lib/orderman/actors/AsyncFactory.h b/src/lib/orderman/actors/AsyncFactory.h index 76214701c..d66aebab8 100644 --- a/src/lib/orderman/actors/AsyncFactory.h +++ b/src/lib/orderman/actors/AsyncFactory.h @@ -4,6 +4,7 @@ #include "lib/orderman/orderman_utils/orderman_common.h" #include "lib/mercury/actors/Actor.h" #include "lib/config/protos/jetfix.pb.h" +#include "lib/md/SecurityOracle.h" namespace bts { namespace orderman { @@ -13,7 +14,7 @@ class DynamicOrderRegistry; class AsyncFactory : public mercury::Actor { public: - AsyncFactory(DynamicOrderRegistry const&); + AsyncFactory(DynamicOrderRegistry const&, md::SecurityOracle); ~AsyncFactory(); void init() override; diff --git a/src/lib/orderman/actors/ServiceBridge.cpp b/src/lib/orderman/actors/ServiceBridge.cpp index 62a5ba336..81e96c7b8 100644 --- a/src/lib/orderman/actors/ServiceBridge.cpp +++ b/src/lib/orderman/actors/ServiceBridge.cpp @@ -6,7 +6,6 @@ #include "lib/orderman/events/CancelOrderEvent.h" #include "lib/orderman/events/CrackOrderEvent.h" #include "lib/orderman/events/NewAdminDataEvent.h" -#include "lib/orderman/events/NewSecuritiesEvent.h" #include "lib/orderman/events/OrderManagerReadyEvent.h" #include "lib/orderman/events/StoredFillEvent.h" #include "lib/orderman/events/StoredOrderEvent.h" @@ -29,10 +28,10 @@ struct ServiceBridge::PImpl { PImpl(OnReady const &on_ready, OnOrders const &on_orders, OnFills const &on_fills, DynamicOrderRegistry const &dyn_order_registry, - mercury::BrokerProxy &service_call_marshaller) + mercury::BrokerProxy &service_call_marshaller, md::SecurityOracle o) : on_ready(on_ready), on_orders(on_orders), on_fills(on_fills), - have_orders(false), have_securities(false), order_manager_ready(false), - sec_db(), order_router(), + have_orders(false), order_manager_ready(false), + sec_db(o), order_router(), order_factory(sec_db, order_router, dyn_order_registry), known_orders(), service_call_marshaller(service_call_marshaller) {} @@ -40,7 +39,6 @@ struct ServiceBridge::PImpl { OnOrders on_orders; OnFills on_fills; bool have_orders; - bool have_securities; bool order_manager_ready; securities::MapSecurityDB sec_db; OrderRouter order_router; @@ -59,9 +57,9 @@ struct ServiceBridge::PImpl { ServiceBridge::ServiceBridge(OnReady const &on_ready, OnOrders const &on_orders, OnFills const &on_fills, DynamicOrderRegistry const &dyn_order_registry, - mercury::BrokerProxy &service_call_marshaller) + mercury::BrokerProxy &service_call_marshaller, md::SecurityOracle o) : me(new PImpl(on_ready, on_orders, on_fills, dyn_order_registry, - service_call_marshaller)) {} + service_call_marshaller, o)) {} ServiceBridge::~ServiceBridge() {} @@ -69,7 +67,6 @@ void ServiceBridge::block_until_ready() { me->ready.get_future().wait(); } void ServiceBridge::init() { subscribe([this](auto const &u) { this->on(u); }); - subscribe([this](auto const &u) { this->on(u); }); subscribe([this](auto const &u) { this->on(u); }); subscribe([this](auto const &u) { this->on(u); }); subscribe([this](auto const &u) { this->on(u); }); @@ -108,20 +105,6 @@ void ServiceBridge::on(mercury::Update const &update) { } } -void ServiceBridge::on(mercury::Update const &update) { - if (update.events) { - - // update our security db - for (auto ev : *(update.events)) { - auto sec_ptr = ev.security; - me->sec_db.securities[SecurityKey(sec_ptr->id())] = sec_ptr; - } - bool was_ready = is_ready(); - me->have_securities = true; - maybe_signal_ready(was_ready); - } -} - void ServiceBridge::on(mercury::Update const &update) { if (update.events) { bool was_ready = is_ready(); @@ -247,7 +230,7 @@ void ServiceBridge::update_admin_data( } bool ServiceBridge::is_ready() const { - return me->have_orders && me->have_securities && me->order_manager_ready; + return me->have_orders && me->order_manager_ready; } void ServiceBridge::maybe_signal_ready(bool old_ready) { diff --git a/src/lib/orderman/actors/ServiceBridge.h b/src/lib/orderman/actors/ServiceBridge.h index f78f0a6c6..1c949f061 100644 --- a/src/lib/orderman/actors/ServiceBridge.h +++ b/src/lib/orderman/actors/ServiceBridge.h @@ -2,6 +2,7 @@ #include "lib/config/protos/jetfix.pb.h" #include "lib/mercury/actors/Actor.h" +#include "lib/md/SecurityOracle.h" #include // fwds @@ -25,7 +26,6 @@ class Crack; namespace orderman { class StoredFillEvent; class StoredOrderEvent; -class NewSecuritiesEvent; class OrderManagerReadyEvent; class NewAdminDataEvent; class DynamicOrderRegistry; @@ -50,8 +50,8 @@ class ServiceBridge : public mercury::Actor { ServiceBridge(OnReady const &, OnOrders const &, OnFills const &, DynamicOrderRegistry const &, - - mercury::BrokerProxy &service_call_marshaller); + mercury::BrokerProxy &service_call_marshaller, + md::SecurityOracle o); void init() override; ~ServiceBridge(); @@ -74,7 +74,6 @@ class ServiceBridge : public mercury::Actor { void on(mercury::Update const &); void on(mercury::Update const &); - void on(mercury::Update const &); void on(mercury::Update const &); void on(mercury::Update const &); }; diff --git a/src/lib/orderman/actors/order_manager/OrderManager.cpp b/src/lib/orderman/actors/order_manager/OrderManager.cpp index 68a99d941..4a8ed4482 100644 --- a/src/lib/orderman/actors/order_manager/OrderManager.cpp +++ b/src/lib/orderman/actors/order_manager/OrderManager.cpp @@ -8,6 +8,7 @@ #include "lib/orderman/events/OrderManagerReadyEvent.h" #include "lib/orderman/events/StoredFillEvent.h" #include "lib/orderman/events/StoredOrderEvent.h" +#include "lib/orderman/events/NewSecuritiesEvent.h" #include "lib/orderman/order_factory/OrderFactory.h" #include "lib/orderman/orders/Order.h" #include "lib/orderman/securities/ISecurityDB.h" @@ -31,8 +32,8 @@ enum class OrderManager::State { }; struct OrderManager::PImpl { - PImpl(OrderManager &owner, DynamicOrderRegistry const &b, trace::PerfLog &l) - : state(State::WAIT_ADMIN), services_data(owner, b, l) {} + PImpl(OrderManager &owner, DynamicOrderRegistry const &b, trace::PerfLog &l, md::SecurityOracle o) + : state(State::WAIT_ADMIN), services_data(owner, b, l, o) {} State state; std::unordered_map orders; @@ -42,8 +43,8 @@ struct OrderManager::PImpl { mercury::Stream pub_ready; }; -OrderManager::OrderManager(DynamicOrderRegistry const &b, trace::PerfLog &l) - : me(new PImpl(*this, b, l)) {} +OrderManager::OrderManager(DynamicOrderRegistry const &b, trace::PerfLog &l, md::SecurityOracle o) + : me(new PImpl(*this, b, l, o)) {} OrderManager::~OrderManager() {} void OrderManager::init() { @@ -85,8 +86,6 @@ void OrderManager::on_securities( for (auto const &ev : *(update.events)) { auto sec_ptr = ev.security; secs.push_back(sec_ptr); - me->services_data.sec_db.securities[SecurityKey(sec_ptr->id())] = - sec_ptr; } me->services_data.limiter.add_securities(secs); diff --git a/src/lib/orderman/actors/order_manager/OrderManager.h b/src/lib/orderman/actors/order_manager/OrderManager.h index a8ec11582..4bee86449 100644 --- a/src/lib/orderman/actors/order_manager/OrderManager.h +++ b/src/lib/orderman/actors/order_manager/OrderManager.h @@ -4,6 +4,7 @@ #include "lib/jetfix/common/OutFIXPtr.h" #include "lib/mercury/actors/Actor.h" #include "lib/mercury/actors/IPollableActor.h" +#include "lib/md/SecurityOracle.h" #include "lib/orderman/events/ActivateOrderEvent.h" #include "lib/orderman/events/CancelOrderEvent.h" #include "lib/orderman/events/CrackOrderEvent.h" @@ -60,7 +61,7 @@ namespace orderman { class OrderManager : public mercury::Actor, public mercury::IPollableActor { public: - OrderManager(DynamicOrderRegistry const &dyn_order_builder, trace::PerfLog &); + OrderManager(DynamicOrderRegistry const &dyn_order_builder, trace::PerfLog &, md::SecurityOracle); ~OrderManager(); void init() override; diff --git a/src/lib/orderman/actors/order_manager/OrderServicesData.cpp b/src/lib/orderman/actors/order_manager/OrderServicesData.cpp index 5b19f6e44..253f56317 100644 --- a/src/lib/orderman/actors/order_manager/OrderServicesData.cpp +++ b/src/lib/orderman/actors/order_manager/OrderServicesData.cpp @@ -7,8 +7,8 @@ namespace orderman { OrderServicesData::OrderServicesData(OrderManager &owner, DynamicOrderRegistry const &b, - trace::PerfLog &l) - : owner(owner), log(l), order_router(), + trace::PerfLog &l, md::SecurityOracle o) + : owner(owner), log(l), sec_db(o), order_router(), order_factory(sec_db, order_router, b), last_timer_ev( Timer::now(), diff --git a/src/lib/orderman/actors/order_manager/OrderServicesData.h b/src/lib/orderman/actors/order_manager/OrderServicesData.h index e664aec0a..21afdbc4d 100644 --- a/src/lib/orderman/actors/order_manager/OrderServicesData.h +++ b/src/lib/orderman/actors/order_manager/OrderServicesData.h @@ -8,6 +8,7 @@ #include "lib/orderman/securities/MapSecurityDB.h" #include "lib/mercury/events/TimerEvent.h" +#include "lib/md/SecurityOracle.h" #include "lib/orderman/actors/ExecutionGateway.h" #include "lib/orderman/events/ActivateOrderEvent.h" #include "lib/orderman/events/BookUpdateEvent.h" @@ -22,7 +23,6 @@ #include "lib/orderman/events/GatewayStatusChangeEvent.h" #include "lib/orderman/events/MDRequestEvent.h" #include "lib/orderman/events/NewExecutionReportEvent.h" -#include "lib/orderman/events/NewSecuritiesEvent.h" #include "lib/orderman/events/OrderStateUpdateEvent.h" #include "lib/orderman/events/SettlementRequestEvent.h" #include "lib/orderman/events/SettlementUpdateEvent.h" @@ -75,7 +75,7 @@ class OrderServicesData { public: OrderServicesData(OrderManager &owner, DynamicOrderRegistry const &b, - trace::PerfLog &l); + trace::PerfLog &l, md::SecurityOracle); OrderManager &owner; trace::PerfLog &log; diff --git a/src/lib/orderman/orders/NativeOrder.cpp b/src/lib/orderman/orders/NativeOrder.cpp index 60e1a615f..9ff30477d 100644 --- a/src/lib/orderman/orders/NativeOrder.cpp +++ b/src/lib/orderman/orders/NativeOrder.cpp @@ -4,6 +4,7 @@ #include "lib/jetfix/codec/FIXFragment.h" #include "lib/jetfix/codec/FieldDefinitions.h" #include "lib/logging/logging.h" +#include "lib/domain_model/Security.h" #include "lib/orderman/order_factory/IOrderFactory.h" #include "lib/orderman/order_router/OrderRouter.h" #include "lib/orderman/orderman_utils/ExecutionReport.h" diff --git a/src/lib/orderman/securities/ISecurityDB.h b/src/lib/orderman/securities/ISecurityDB.h index eb7a23fad..427d1171b 100644 --- a/src/lib/orderman/securities/ISecurityDB.h +++ b/src/lib/orderman/securities/ISecurityDB.h @@ -14,7 +14,6 @@ public: virtual ~ISecurityDB() {}; virtual api::Security const *resolve(SecurityKey const&k) const = 0; using SecurityMap = std::unordered_map ; - virtual SecurityMap retrieve_all() const = 0; }; }} diff --git a/src/lib/orderman/securities/MapSecurityDB.h b/src/lib/orderman/securities/MapSecurityDB.h index 2c447d6f9..b2c80252c 100644 --- a/src/lib/orderman/securities/MapSecurityDB.h +++ b/src/lib/orderman/securities/MapSecurityDB.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include "lib/md/SecurityOracle.h" #include "lib/orderman/securities/ISecurityDB.h" namespace bts { @@ -9,15 +9,12 @@ namespace securities { struct MapSecurityDB : public securities::ISecurityDB { + MapSecurityDB(md::SecurityOracle oracle): _oracle(oracle) {} api::Security const *resolve(SecurityKey const&k) const override { - auto it = securities.find(k); - if (it != securities.end()) - return it->second; - else - return nullptr; + return _oracle.get_security(k); } - SecurityMap retrieve_all() const override {return securities;} - SecurityMap securities; + private: + md::SecurityOracle _oracle; }; }} diff --git a/src/lib/orderman/server/OrderServiceServer.cpp b/src/lib/orderman/server/OrderServiceServer.cpp index caf1c67b2..1782d39f7 100644 --- a/src/lib/orderman/server/OrderServiceServer.cpp +++ b/src/lib/orderman/server/OrderServiceServer.cpp @@ -146,7 +146,7 @@ OrderServiceServer::OrderServiceServer( bts_assert_msg(thread, "Could not find a configured thread for OrderManager. See " "the OrdermanCfg key"); - thread->executor.spawn(d,me->log); + thread->executor.spawn(d,me->log, mdc.oracle()); // Timer thread = find_thread_for_actor("Timer", me->actor_threads); @@ -160,7 +160,7 @@ OrderServiceServer::OrderServiceServer( bts_assert_msg(thread, "Could not find a configured thread for AsyncFactory. See " "the OrdermanCfg key"); - thread->executor.spawn(d); + thread->executor.spawn(d, mdc.oracle()); // OrderStore thread = find_thread_for_actor("OrderStore", me->actor_threads); @@ -206,7 +206,7 @@ OrderServiceServer::OrderServiceServer( [this, &ctx](auto const &fill) { ctx.post([this, fill] { this->on_fills(fill); }); }, - me->dyn_order_registry, thread->executor.make_proxy()); + me->dyn_order_registry, thread->executor.make_proxy(),mdc.oracle()); thread->executor.activate(me->service_bridge); From b39812aad0c8879fe58a2676d20399b6fc33b072 Mon Sep 17 00:00:00 2001 From: Kevin Darby Date: Tue, 5 Feb 2019 17:50:08 -0500 Subject: [PATCH 50/85] add RelativeSide enum for OCO. Fix SPARK-2022 --- remote-sparkvm.code-workspace | 27 +++++++++++++++++++ .../Orders/OrderTypeVMs/DynamicOrderVM.cs | 2 ++ .../spark/Orders/OrderTypeVMs/OCOOrderVM.cs | 8 +++--- .../Order/OrderDetailsPopupControl.xaml | 8 +++--- src/lib/api/services/OrderService.proto | 2 +- src/lib/domain_model/Security.proto | 8 ++++++ .../orderman/orders/book_algos/OCOOrder.cpp | 14 +++++++--- 7 files changed, 58 insertions(+), 11 deletions(-) create mode 100644 remote-sparkvm.code-workspace diff --git a/remote-sparkvm.code-workspace b/remote-sparkvm.code-workspace new file mode 100644 index 000000000..03c875df7 --- /dev/null +++ b/remote-sparkvm.code-workspace @@ -0,0 +1,27 @@ +{ + "folders": [ + { + "path": "." + }, + { + "uri": "sftp://bts:BT5u53r@sparkvm:22/home/bts/", + "name": "sparkvm" + } + ], + "settings": { + "files.associations": { + "utility": "cpp", + "initializer_list": "cpp", + "list": "cpp", + "type_traits": "cpp", + "vector": "cpp", + "xhash": "cpp", + "xstring": "cpp", + "system_error": "cpp", + "array": "cpp", + "string_view": "cpp", + "xutility": "cpp", + "*.ipp": "cpp" + } + } +} \ No newline at end of file diff --git a/src/client/spark/Orders/OrderTypeVMs/DynamicOrderVM.cs b/src/client/spark/Orders/OrderTypeVMs/DynamicOrderVM.cs index 172c23162..6bb6419cc 100644 --- a/src/client/spark/Orders/OrderTypeVMs/DynamicOrderVM.cs +++ b/src/client/spark/Orders/OrderTypeVMs/DynamicOrderVM.cs @@ -126,6 +126,8 @@ namespace mbts.spark return _child_order_choices; } } + + public bool has_child => _order_def_tags.Any(); public IOrderTypeVM child_order { diff --git a/src/client/spark/Orders/OrderTypeVMs/OCOOrderVM.cs b/src/client/spark/Orders/OrderTypeVMs/OCOOrderVM.cs index 393b0a25f..3bd35629f 100644 --- a/src/client/spark/Orders/OrderTypeVMs/OCOOrderVM.cs +++ b/src/client/spark/Orders/OrderTypeVMs/OCOOrderVM.cs @@ -28,7 +28,7 @@ namespace mbts.spark this.other_order_types = new InlineUserProperty>(child_order_types, true); other_ticks = new InlineUserProperty(1, false); - other_side = new InlineUserProperty(bts.api.Side.Undefined, false); + other_side = new InlineUserProperty(bts.api.RelativeSide.Opposite, false); } public readonly InlineUserProperty selected_child_order_type; public readonly InlineUserProperty> child_order_types; @@ -36,7 +36,7 @@ namespace mbts.spark public readonly InlineUserProperty> other_order_types; public readonly InlineUserProperty other_ticks; - public readonly InlineUserProperty other_side; + public readonly InlineUserProperty other_side; } @@ -55,7 +55,7 @@ namespace mbts.spark IDisposable _other_order_errors_sub; public int other_ticks { get; set; } = 1; - public bts.api.Side other_side { get; set; } = bts.api.Side.Undefined; + public bts.api.RelativeSide other_side { get; set; } = bts.api.RelativeSide.Opposite; public OCOOrderVM(OrdersModel model, IObservable security_md_source, string id, string display_name) : this(model, security_md_source, security_md_source.Select(smd => @@ -120,7 +120,7 @@ namespace mbts.spark } } - public IEnumerable other_side_types => EnumHelper.get_values(); + public IEnumerable other_side_types => EnumHelper.get_values_except(bts.api.RelativeSide.Undefined); public ICollection child_order_types { diff --git a/src/client/spark/view/Controls/Order/OrderDetailsPopupControl.xaml b/src/client/spark/view/Controls/Order/OrderDetailsPopupControl.xaml index 3e611e39a..b06aa00ea 100644 --- a/src/client/spark/view/Controls/Order/OrderDetailsPopupControl.xaml +++ b/src/client/spark/view/Controls/Order/OrderDetailsPopupControl.xaml @@ -397,7 +397,7 @@ Other Side - + - Child + Child mutable_custom_def() = oco_def.other(); - if(me->other_side == api::SIDE_UNDEFINED) + bts_assert_msg(me->other_side != api::RELATIVESIDE_UNDEFINED, "undefined relativeside"); + + if(oco_def.other_side() == api::RELATIVESIDE_BUY) + out_hedge->set_side(api::SIDE_BUY); + else if(oco_def.other_side() == api::RELATIVESIDE_SELL) + out_hedge->set_side(api::SIDE_SELL); + else if(oco_def.other_side() == api::RELATIVESIDE_SAME) + out_hedge->set_side(definition().side()); + else if(oco_def.other_side() == api::RELATIVESIDE_OPPOSITE) { if(definition().side() == api::SIDE_BUY) out_hedge->set_side(api::SIDE_SELL); @@ -53,7 +61,7 @@ bool OCOOrder::setup_hedge(const api::orderman::OrderDefinition& this_def, const out_hedge->set_side(api::SIDE_BUY); } else - out_hedge->set_side(me->other_side); + bts_assert_msg(false, "unknown relativeside"); return true; } else From 00ce3adde533459487f1779863ae6aec0a38aa14 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 6 Feb 2019 13:10:18 -0500 Subject: [PATCH 51/85] Add ApiThreading man page (draft) --- doc/user/docs/ApiThreading.md | 74 +++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 doc/user/docs/ApiThreading.md diff --git a/doc/user/docs/ApiThreading.md b/doc/user/docs/ApiThreading.md new file mode 100644 index 000000000..9b8b05a33 --- /dev/null +++ b/doc/user/docs/ApiThreading.md @@ -0,0 +1,74 @@ + +# API Threading + +BTS API provides two types of calls. There are RPC (remote procedure calls) which accept an argument and return a value and there are streams of tabular data which we call subscriptions. All communication to and from the server happens on a connection thread owned by the ServiceContext object and data must be marshaled back to the right context when the call is complete. + +## RPC + +### C# + +All API calls provide both a synchronous and [asynconous](https://docs.microsoft.com/en-us/dotnet/csharp/language-reference/keywords/async) version. If you are calling the API from a WPF or Windows Forms application you should always prefer the async version of the method because it will efficiently make the call and resume execution on the correct thread without blocking or lagging user interactions. The syncronous calls are intended for use in console applications where nothing else needs use of the thread until the data is available. + +The async version of the method will return a `Result` type which is used to signal if an error occurred during the call. You should test the Result.success to see if the call succeeded before using Result.value. + +```csharp + var s = context.locate(); + var result = await s.get_users_async(new GetUsersRequest()); + if (result.success) { + Console.WriteLine(result.value); + } + else { + Console.WriteLine($"Error {result.error_message}"); + } +``` + +### C++ + +All C++ API calls also provide both a blocking synchronous version and a callback based asynchronous version. In the asynchronous case the callback will execute on the ServiceContext thread and you will need to handle marshaling that data to the appropriate thread. Just like in c# the callback will receive a `Result` which will indicate success or failure. `Result.value` returns an `std::shared_ptr` so you have control over the lifetime of the value. + +## Subscriptions + +BTS provides streaming updates for some types of data. These updates are incremental in nature and will provide a snapshot of the current state on a new subscription. Streaming updates use the [ReactiveX library](http://reactivex.io) to manage subscribing and unsubscribing. This library provides a rich set of operators that you can use to slide and dice the data streams based on your specific application requirements. + +### C# + +The subscription callback will arrive on the ServiceContext thread but it can be easily marshaled to the thread of your choice using the [ObserveOn](http://reactivex.io/documentation/operators/observeon.html) operator. For a WPF or Windows Forms application the most common use of this will be [ObserveOnDispatcher]() which will simply execute the callback in the context of the original calling thread. By your choice of where to place the `ObserveOnDispatcher` call in your reactive call chain you can choose whether other operators execute before after switching threads. + +In this example the `Where` and `SelectMany` operators will execute on `ServiceContext` connection thread and only the resulting matches will be marshaled across to the calling thread. It is safe to unsubscribe by disposing `sub` from any thread. + +```csharp + var orders = context.locate(); + ObservableCollection orders_with_errors; + IDisposable sub = + orders + .subscribe_orders() + .SelectMany(update => update.added) + .Where(o => o.State.Status == OrderStatus.Error) + .ObserveOnDispatcher() + .Subscribe(o => { + orders_with_errors.Add(o); + }); +``` + +### C++ + +The C++ subscription callbacks use the [RxCpp](https://github.com/ReactiveX/RxCpp) library which implements the [ReactiveX](http://reactivex.io) operators. The subscribe call will return an `rxcpp::composite_subscription` which implements [RAII](https://en.wikipedia.org/wiki/Resource_acquisition_is_initialization) and will unsubscribe when it is destroyed. + +```cpp +#include "lib/api/public/ServiceContext.h" +#include "lib/api/services/OrderService.client.pb.h" +#include +#include +//... +bts::api::ServiceContext ctx; +auto client = ctx.locate(); +auto sub = + client.subscribe_orders().subscribe([=](const bts::api::orderman::OrdersUpdate& update) { + std::string result; + for(const auto& order : update.added()) { + auto status = google::protobuf::util::MessageToJsonString(*order, &result, {}); + std::cout << "New order: " << result << std::endl; + } + }); + +``` \ No newline at end of file From e08cd7bdbf1ea33cd99bce890c56b01f21b202d8 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 6 Feb 2019 14:25:07 -0500 Subject: [PATCH 52/85] Use SecurityOracle in OrderStore --- src/lib/orderman/actors/OrderStore.cpp | 118 +++++++----------- src/lib/orderman/actors/OrderStore.h | 3 +- .../orderman/server/OrderServiceServer.cpp | 2 +- 3 files changed, 48 insertions(+), 75 deletions(-) diff --git a/src/lib/orderman/actors/OrderStore.cpp b/src/lib/orderman/actors/OrderStore.cpp index db6dd64b4..4e5f43b38 100644 --- a/src/lib/orderman/actors/OrderStore.cpp +++ b/src/lib/orderman/actors/OrderStore.cpp @@ -5,7 +5,6 @@ #include "lib/config/protos/orders_fills_cleanup.pb.h" #include "lib/config/protos/toggles.pb.h" #include "lib/orderman/actors/SODPosition.h" -#include "lib/orderman/events/NewSecuritiesEvent.h" #include "lib/orderman/events/OrderStateUpdateEvent.h" #include "lib/orderman/events/StoredOrderEvent.h" #include "lib/persistence/SqliteStore.h" @@ -18,99 +17,72 @@ namespace orderman { static auto const orders_ns = "orders"; -enum class OrderStore::State { WAIT_SEC, READY }; - struct OrderStore::PImpl { - PImpl() - : state(State::WAIT_SEC), known_orders(), + PImpl(md::SecurityOracle o) + : known_orders(), store(new persistence::SqliteStore("orders")), order_stream(), soft_cleanup_enabled( - config::get().enabled()) {} + config::get().enabled()), + oracle(o) {} - State state; const GatewayName gw_name; std::unordered_map known_orders; std::unique_ptr store; mercury::Stream order_stream; - mercury::Actor::SubHandle stored_sec_sub; bool soft_cleanup_enabled; + md::SecurityOracle oracle; timeval cutoff_time; }; -OrderStore::OrderStore() : me(new PImpl()) {} +OrderStore::OrderStore(md::SecurityOracle o) : me(new PImpl(o)) {} OrderStore::~OrderStore() {} void OrderStore::init() { - me->state = State::WAIT_SEC; - me->stored_sec_sub = subscribe([this]( - auto const &update) { - if (update.events) { - if (me->state == State::WAIT_SEC) { - LOG(OrderMan, Info, - "Order store received initial list of {} securities", - update.events->size()); + // subscribe to order updates so we can store them + subscribe( + [this](auto const &ev) { this->on_order_state_update(ev); }); - // don't need securities anymore, we just need the initial - // batch so we can filter out orders on expired securities - unsubscribe(me->stored_sec_sub); + // now load orders from disk, for each order check if + // it reffers to a valid security. if so publish it. + api::orderman::OrderRecord rec; + std::vector stored_batch; - // subscribe to order updates so we can store them - subscribe( - [this](auto const &ev) { this->on_order_state_update(ev); }); + LOG(persist, Info, "Loading all orders"); + auto all = me->store->retrieve_all()[orders_ns]; + LOG(persist, Info, "Finished loading all {} orders", all.size()); - // index securities - std::map sec_idx; - for (auto const &ev : *(update.events)) { - auto sec_ptr = ev.security; - sec_idx[SecurityKey(sec_ptr->id())] = sec_ptr; + me->cutoff_time = SODPosition::get_order_cutoff_time(all); + + for (auto const &kvp : all) + if (utils::pb::from_string( + kvp.second, &rec)) { + auto ok = OrderKey(std::stoi(kvp.first)); + auto sk = SecurityKey(rec.def().security_key()); + auto sec = me->oracle.get_security(sk); + + if (config::get() + .enabled() && sec == nullptr) { + LOG(OrderMan, Diag, + "Order {} refers to unresolved security {}", + ok.value(), sk.value()); + } else { + if (!_order_cutoff(rec)) + stored_batch.emplace_back(rec); + me->known_orders[ok] = rec.state(); } + } else + LOG(OrderMan, Error, + "Could not recover order {} from storage. " + "This should be investigated", + kvp.first); - // now load orders from disk, for each order check if - // it reffers to a valid security. if so publish it. - api::orderman::OrderRecord rec; - std::vector stored_batch; - - LOG(persist, Info, "Loading all orders"); - auto all = me->store->retrieve_all()[orders_ns]; - LOG(persist, Info, "Finished loading all {} orders", all.size()); - - me->cutoff_time = SODPosition::get_order_cutoff_time(all); - - for (auto const &kvp : all) - if (utils::pb::from_string( - kvp.second, &rec)) { - auto ok = OrderKey(std::stoi(kvp.first)); - auto sk = SecurityKey(rec.def().security_key()); - - if (config::get< - config::toggles::FilterOrdersWithDanglingSecurities>() - .enabled() && - sec_idx.find(sk) == sec_idx.end()) { - LOG(OrderMan, Diag, - "Order {} refers to unresolved security {}", - ok.value(), sk.value()); - } else { - if (!_order_cutoff(rec)) - stored_batch.emplace_back(rec); - me->known_orders[ok] = rec.state(); - } - } else - LOG(OrderMan, Error, - "Could not recover order {} from storage. " - "This should be investigated", - kvp.first); - - me->order_stream = make_stream(); - LOG(OrderMan, Info, - "Order Store published {} orders recoved from disk out of {} " - "total orders", - stored_batch.size(), all.size()); - publish_batch(me->order_stream, stored_batch); - me->state = State::READY; - } - } - }); + me->order_stream = make_stream(); + LOG(OrderMan, Info, + "Order Store published {} orders recoved from disk out of {} " + "total orders", + stored_batch.size(), all.size()); + publish_batch(me->order_stream, stored_batch); } void OrderStore::on_order_state_update( diff --git a/src/lib/orderman/actors/OrderStore.h b/src/lib/orderman/actors/OrderStore.h index 535a6cc24..6b46df8ce 100644 --- a/src/lib/orderman/actors/OrderStore.h +++ b/src/lib/orderman/actors/OrderStore.h @@ -2,6 +2,7 @@ #include #include "lib/mercury/actors/Actor.h" +#include "lib/md/SecurityOracle.h" #include "lib/orderman/orderman_utils/orderman_common.h" namespace bts { @@ -15,7 +16,7 @@ class OrderStateUpdateEvent; class OrderStore : public mercury::Actor { public: - OrderStore(); + OrderStore(md::SecurityOracle); ~OrderStore(); void init() override; diff --git a/src/lib/orderman/server/OrderServiceServer.cpp b/src/lib/orderman/server/OrderServiceServer.cpp index 1782d39f7..7c157ca4f 100644 --- a/src/lib/orderman/server/OrderServiceServer.cpp +++ b/src/lib/orderman/server/OrderServiceServer.cpp @@ -167,7 +167,7 @@ OrderServiceServer::OrderServiceServer( bts_assert_msg(thread, "Could not find a configured thread for OrderStore. See " "the OrdermanCfg key"); - thread->executor.spawn(); + thread->executor.spawn(mdc.oracle()); // FillStore thread = find_thread_for_actor("FillStore", me->actor_threads); From a7d71fe17055a286d66f2cdcd04d6d2c3df9c0a5 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 6 Feb 2019 14:31:04 -0500 Subject: [PATCH 53/85] Move ApiThreading doc --- doc/{user/docs => sdk}/ApiThreading.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename doc/{user/docs => sdk}/ApiThreading.md (100%) diff --git a/doc/user/docs/ApiThreading.md b/doc/sdk/ApiThreading.md similarity index 100% rename from doc/user/docs/ApiThreading.md rename to doc/sdk/ApiThreading.md From 03fe75cfd2f1391b66451453a4e1aa997e3d9165 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Wed, 6 Feb 2019 13:38:40 -0600 Subject: [PATCH 54/85] commit button now unselects row --- .../Modules/Trading/OrderBook/OrderBookRowVM.cs | 17 ++++++++++++++--- .../Modules/Trading/OrderBook/OrderBookVM.cs | 14 +++++++++++++- .../Trading/OrderBook/OrderBookWindow.cs | 2 -- 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs index 76dc8341f..5323e629f 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs @@ -130,7 +130,10 @@ namespace mbts.spark { public ICommand modify_order_quantity => _modify_order_quantity ?? (_modify_order_quantity = new DelegateCommand( - (desired_quantity) => _owner.modify_order_quantity(this, (uint)desired_quantity), + (desired_quantity) => { + _owner.modify_order_quantity(this, (uint)desired_quantity); + _unselect_me(); + }, (desired_quantity) => (desired_quantity != null && this.status == OrderStatus.Working && (uint)desired_quantity != this.quantity) || is_qty_dirty @@ -139,12 +142,20 @@ namespace mbts.spark { public ICommand modify_order_price => _modify_order_price ?? (_modify_order_price = new DelegateCommand( - (desired_price) => _owner.modify_order_price(this, (double)desired_price), + (desired_price) => { + _owner.modify_order_price(this, (double)desired_price); + _unselect_me(); + }, (desired_price) => (desired_price != null && this.status == OrderStatus.Working && (double)desired_price != this.price) || is_price_dirty ) - ); + ); + + private void _unselect_me() { + this.is_selected = false; + _owner.current_row = null; + } // These fns present a way for the orderbook to avoid noisy, intermittent collection changes during "Expand/Collapse All" ops, // when incremental changes are ignored in favor of a full reset, after 'is_expanded' is updated for all rows. diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs index b0fc0609d..5b2e9ec19 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookVM.cs @@ -76,7 +76,19 @@ namespace mbts.spark { public ObservableAppendCollection columns { get; } public ObservableCollection core_columns => _columns; public ObservableCollection rows => _row_collection; - public OrderBookRowVM current_row { get => _current_row.Value; set => _current_row.Value = value ?? _selected_rows.FirstOrDefault(); } + public OrderBookRowVM current_row { + get => _current_row.Value; + set { + _current_row.Value = value; + // this is so that when the current_row is set to null from + // the row_vm, we propagate the change up to the OrderBookWindow + // so that the cancel and modify commands can_execute is updated. + // Note that this is really only for if/when the edit_xxx columns + // from the mini-ob are present in regular orderbook, else this is + // a no-op. + selected_order_changed?.Invoke(current_order); + } + } public OrderRecordImmutable? current_order => current_row?.order; public IEnumerable selected_orders => _selected_rows.Select(r => r.order); public _GridSettingsManager grid_settings_manager { get; } diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookWindow.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookWindow.cs index 3c4c08060..3b62863a4 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookWindow.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookWindow.cs @@ -167,11 +167,9 @@ namespace mbts.spark { } private void _vm_selected_order_changed(OrderRecordImmutable? e) { - if (e.HasValue) { _modify_order_command.raiseCanExecuteChanged(); _cancel_selected_orders_command.raiseCanExecuteChanged(); _recreate_selected_orders_command.raiseCanExecuteChanged(); - } } } } From 99f1d1fa010d22f2a2220f2e647fc7995f9fd38d Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 6 Feb 2019 14:48:18 -0500 Subject: [PATCH 55/85] Review feedback --- doc/sdk/ApiThreading.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sdk/ApiThreading.md b/doc/sdk/ApiThreading.md index 9b8b05a33..1ca91214b 100644 --- a/doc/sdk/ApiThreading.md +++ b/doc/sdk/ApiThreading.md @@ -1,5 +1,5 @@ -# API Threading +# API Client Threading BTS API provides two types of calls. There are RPC (remote procedure calls) which accept an argument and return a value and there are streams of tabular data which we call subscriptions. All communication to and from the server happens on a connection thread owned by the ServiceContext object and data must be marshaled back to the right context when the call is complete. @@ -7,7 +7,7 @@ BTS API provides two types of calls. There are RPC (remote procedure calls) whi ### C# -All API calls provide both a synchronous and [asynconous](https://docs.microsoft.com/en-us/dotnet/csharp/language-reference/keywords/async) version. If you are calling the API from a WPF or Windows Forms application you should always prefer the async version of the method because it will efficiently make the call and resume execution on the correct thread without blocking or lagging user interactions. The syncronous calls are intended for use in console applications where nothing else needs use of the thread until the data is available. +All API calls provide both a synchronous and [asynchronous](https://docs.microsoft.com/en-us/dotnet/csharp/language-reference/keywords/async) version. If you are calling the API from a WPF or Windows Forms application you should always prefer the async version of the method because it will efficiently make the call and resume execution on the correct thread without blocking or lagging user interactions. The synchronous calls are intended for use in console applications where nothing else needs use of the thread until the data is available. The async version of the method will return a `Result` type which is used to signal if an error occurred during the call. You should test the Result.success to see if the call succeeded before using Result.value. From 7b9071a1b23050e770a01bcd6911e441d83f01ed Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 6 Feb 2019 14:49:17 -0500 Subject: [PATCH 56/85] Review feedback 2 --- doc/sdk/ApiThreading.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sdk/ApiThreading.md b/doc/sdk/ApiThreading.md index 1ca91214b..ab79e821d 100644 --- a/doc/sdk/ApiThreading.md +++ b/doc/sdk/ApiThreading.md @@ -1,5 +1,5 @@ -# API Client Threading +# Client API Threading BTS API provides two types of calls. There are RPC (remote procedure calls) which accept an argument and return a value and there are streams of tabular data which we call subscriptions. All communication to and from the server happens on a connection thread owned by the ServiceContext object and data must be marshaled back to the right context when the call is complete. From cff32ad09a8a59921934251dbdb897c44737a2d0 Mon Sep 17 00:00:00 2001 From: Kevin Darby Date: Wed, 6 Feb 2019 17:03:23 -0500 Subject: [PATCH 57/85] for now I'd like to keep these client and server docs separate since they are two repos --- src/client/spark_sdk/README.md | 44 +++++--- src/client/spark_sdk/doc/Overview.md | 103 ++++++++++++++++++ .../client/spark_sdk/doc/Threading.md | 0 src/client/spark_sdk/spark_sdk.csproj | 4 +- src/sdk/README.md | 33 ++++-- src/sdk/doc/algos.md | 40 +++++++ src/sdk/make_sdk.sh | 1 + 7 files changed, 202 insertions(+), 23 deletions(-) create mode 100644 src/client/spark_sdk/doc/Overview.md rename doc/sdk/ApiThreading.md => src/client/spark_sdk/doc/Threading.md (100%) create mode 100644 src/sdk/doc/algos.md diff --git a/src/client/spark_sdk/README.md b/src/client/spark_sdk/README.md index 49c539fa8..a5367bce8 100644 --- a/src/client/spark_sdk/README.md +++ b/src/client/spark_sdk/README.md @@ -1,8 +1,31 @@ +# BTS Spark + + +Our offering consists of a client UI [Spark Client](https://www.bluetradingsystems.com/images/Images/spark/02.06.19/Ladder.png), a high performance professional futures front end, connected via our proprietary protocol over TCP to our Spark Server, which in turn connects to execution venues via their native protocols (typically FIX/TCP for execution and some proprietary protocol over UDP multicast for market data). + +![alt text](https://www.bluetradingsystems.com/images/Images/api/API.png) + + # BTS Spark Client SDK -#### Getting Started +We currently offer two API extension points. This repository, our BTS **Spark Client SDK** allows users to control our Spark Server externally, that is, out of process, using the exact same protocol our own Spark Client does. In other words, a client application developed via the Spark Client SDK, say a VB .NET UI, would be able to provide any functionality our own UI provides. -## Git +We currently offer bindings for the Spark protocol to C++, C# and Excel clients. Since the underlying transport is TCP, clients developed via the Spark Client SDK can be located anywhere in the world, as long as they can reach the Spark Server via TCP. We do require a VPN or a secure network, since the Spark Protocol is currently not yet encrypted. + +Our second API extension point is intended for users that require adding functionality to our Spark Server. If you are interested in extending the server process through custom order types, market data sources, or execution gateways, please see the [Spark Server SDK](https://github.com/BlueTradingSystems/spark_sdk). + + +### In Depth + + * [Interface Overview](doc/Overview.md) + * [Threading Notes](doc/Threading.md) + + + + +## Getting Started + +### Git * First clone the spark_client_sdk repo @@ -11,7 +34,7 @@ ``` -## Build +### Build * Use msbuild to target windows / .net 4.71 * Install studio here if you don't have it: https://visualstudio.microsoft.com/downloads/ * Use VSCode for editing, it's better: https://code.visualstudio.com/ @@ -24,7 +47,7 @@ spark_client_sdk\src\spark_sdk> msbuild * spark_sdk binaries will be produced in the bin\Debug directory -## Connecting to Spark Server +### Connecting to Spark Server * Start your exe using the -ConfigServer parameter to point at a running Spark Server ``` @@ -35,7 +58,7 @@ spark_client_sdk\src\spark_sdk> msbuild include: "http://location:8000/config/" ``` - ## Examples +### Examples * Make sure you can connect to a running server * run spark_sdk.exe with no parameters to see help ``` @@ -58,22 +81,15 @@ Arguments: * [Driver](src/spark_sdk/samples/SDKMain.cs) - ## Excel + ### Excel ``` spark_sdk\bin\Debug\net471> start spark_excel-addin64-packed.xll ``` * This will start excel and load the bts addin * Check the [```excel```](excel/) directory for an example spreadsheet - -## Notes -* Our client side API is based on Google's protocol buffers; the message types are all protos. https://developers.google.com/protocol-buffers/ - -* We extended the protoc compiler to generate c#, c++, excel and js bindings to the services provided by our server process (https://github.com/BlueTradingSystems/spark_sdk) - - -* ### Dotnet core +### Dotnet core * Right now we only support c# Windows 10 clients, but the package targets both net471 and .net core 2.1.6, so in theory one could run a .net core or c++ client on linux with a little work. diff --git a/src/client/spark_sdk/doc/Overview.md b/src/client/spark_sdk/doc/Overview.md new file mode 100644 index 000000000..6514552a8 --- /dev/null +++ b/src/client/spark_sdk/doc/Overview.md @@ -0,0 +1,103 @@ + +## Spark Client SDK Overview + +The high level description of our order entry interface is as follows: + +``` +service OrderService { + // RPCs + rpc create_order(Create) returns (Response) + rpc cancel_order(Cancel) returns (Response) + rpc update_order(Update) returns (Response) + // Streams + rpc orders (OrdersSub) returns (stream OrderRecord); + rpc fills (FillsSub) returns (stream Fill); +} +``` + +This description uses the Google Protocol Buffers syntax to define the server interface (we use our own little compiler to generate bindings to this interface in C#, C++ and Excel). Using this server interface we can create, cancel or update orders, using the Remote Procedure Calls (RPCs) defined these orders can be exchange native orders or our own high level algo orders, or new order types created by users via the Spark Server SDK). We can monitor order state by subscribing to the orders stream above, which produces a a stream of OrderRecord objects, each capturing the latest state of a particular order, as it goes through it life cycle. + +An OrderRecord looks like this: + +``` +message OrderRecord { + uint32 key; + uint32 parent_key; + OrderDefinition def; + OrderState state; +} +``` + +We have an order key, a possibly null parent order key, an OrderDefinition record, which defines the basic elements of the order, namely its type, side, security, account, etc, and an OrderState record, which captures the latest state of the order: + +``` +message OrderState { + // Standard order state, shared by all order types + OrderStatus status ; + double price ; + uint32 quantity ; + uint32 filled_quantity; + double last_price; + double avg_price; + string last_msg; + google.protobuf.Timestamp created; + google.protobuf.Timestamp last_change; + // Custom order state, specific to the order type in question + oneof custom { + // native order types + MarketOrderState market; + LimitOrderState limit; + StopOrderState stop; + StopLimitOrderState stop_limit; + // algo order types + IcebergOrderState iceberg; + SpreaderOrderState spreader; + TimeBoxOrderState time_box; + TickOrderState tick; + LeanOrderState lean; + JoinOrderState join; + TopOrderState top;evident + TriggerOrderState trigger; + CoverOrderState cover; + OffsetOrderState offset; + DynamicOrderState dynamic; + } +} +``` + +This OrderState record consists of a common part, applicable to all order types, and a custom part that is specific to the particular order type the record is associated with. Most of the standard fields should be self-explanatory, the latest price, the latest quantity, the filled quantity, etc. The status enumerate field tells you which state the order is in. We currently define the following states: + +``` +enum OrderStatus { + // The order has been constructed but not acted on. Inactive orders + // can be abandoned without consequences. This state should only be + // used internally by the order manager, i.e. it should never appear + // in externally visible orders. + ORDER_STATUS_INACTIVE = 0; + // The order is currently working, i.e. its algorithm is operating. + ORDER_STATUS_WORKING = 1; + // The order has completed successfully and accomplished its + // goal. This is a terminal state. + ORDER_STATUS_COMPLETED = 2; + // The order was terminated orderly before it had a chance to + // complete. This is a terminal state. + ORDER_STATUS_CANCELED = 3; + // The order has encountered an unexpected condition from which it + // cannot recover. This a terminal state. + ORDER_STATUS_ERROR = 4; + // The last known state for this order was ORDER_STATUS_WORKING, but + // since then the execution gateway has signaled a disconnect. This + // is not a terminal state. + ORDER_STATUS_DISCONNECTED = 5; + // The order is initializiYou can ng. It cannot be modified at this + point. + ORDER_STATUS_PENDING_WORKING = 6; + // The order is invalid and was rejected by Spark (not the + // exchange), on creation. This is a terminal state. + ORDER_STATUS_REJECTED_BY_SPARK= 7; +} +``` + + +### Threading + Check out our API Threading Doc [here](Threading.md). \ No newline at end of file diff --git a/doc/sdk/ApiThreading.md b/src/client/spark_sdk/doc/Threading.md similarity index 100% rename from doc/sdk/ApiThreading.md rename to src/client/spark_sdk/doc/Threading.md diff --git a/src/client/spark_sdk/spark_sdk.csproj b/src/client/spark_sdk/spark_sdk.csproj index 26ff14c6c..0579a5960 100644 --- a/src/client/spark_sdk/spark_sdk.csproj +++ b/src/client/spark_sdk/spark_sdk.csproj @@ -2,7 +2,8 @@ netcoreapp2.1;net471 Exe - true + + true $(OutputPath)\net471 true true @@ -61,6 +62,7 @@ + diff --git a/src/sdk/README.md b/src/sdk/README.md index d6fdaed69..51dbde907 100644 --- a/src/sdk/README.md +++ b/src/sdk/README.md @@ -1,8 +1,23 @@ -# BTS Spark SDK +# BTS Spark -#### Getting Started +Our offering consists of a client UI ([Spark Client](https://www.bluetradingsystems.com/bts-spark#ladder-algo-screen)), a high performance professional futures front end, connected via our proprietary protocol over TCP to our Spark Server, which in turn connects to execution venues via their native protocols (typically FIX/TCP for execution and some proprietary protocol over UDP multicast for market data). -## Docker +![alt text](https://www.bluetradingsystems.com/images/Images/api/API.png) + + +# BTS Spark Server SDK + +This API extension point is intended for users that require adding functionality to our Spark Server. Using the **Spark Server SDK** users can create [new order types](https://github.com/BlueTradingSystems/spark_sdk/blob/master/examples/order_plugin_example.cpp), [new market data sources](https://github.com/BlueTradingSystems/spark_sdk/blob/master/examples/md_publisher_example.cpp) and soon new execution gateways. In this case users implement the new functionality in C++, producing a shared library that is loaded at run time by our Spark Server. + +If you are interested in sending orders or otherwise controlling a spark server remotely please see the BTS [Spark Client SDK](https://github.com/BlueTradingSystems/spark_client_sdk) repo. + +### In Depth + + * [Spark Algo Descriptions](doc/algos.md) + +## Getting Started + +### Docker We use docker to provide a full dev environment with all the tools users need to develop their own algos. @@ -35,7 +50,7 @@ $ docker start --attach -i bts_sdk * If you prefer to run docker as a daemon, check out [``~/spark_sdk/sdk_shell.sh``](sdk_shell.sh) -## Git +### Git * We assume the git you are using is git-lfs compatible; if you're using the git that comes with our docker, you're all set. @@ -50,7 +65,7 @@ $ cd spark_sdk ``` -## Development +### Development Look at [examples/md_publisher_example.cpp](examples/md_publisher_example.cpp) to look at how to build a custom market data producer @@ -62,7 +77,7 @@ run ```build_examples.sh``` to make sure you can build the examples, this will p $ cd ~/spark_sdk/examples $ ./build_examples.sh ``` -### Server Layout +#### Server Layout * Spark servers reside on a port, usually 8000, but your production instance could be on a different $PORT ##### SDK Package Layout @@ -75,8 +90,10 @@ $ ./build_examples.sh * Production binaries are in ``/sparkbin`` inside the docker container * Production algo plugins should go into the ``/sparkbin/plugins`` directory -### Recorded market data -* If you want to see prices, before you start sparkd, you'll have to extract the futures.coconut.tar.gz found at the root of the sdk repo and place futures.coconut in ``~/sparkdata/data/replay/`` +#### Recorded market data +* We first check for http://mdreplay/spark/futures.coconut + +* Then fall back to ``./data_files/ relative to the sparkd binary`` ## Debugging diff --git a/src/sdk/doc/algos.md b/src/sdk/doc/algos.md new file mode 100644 index 000000000..1264b8de5 --- /dev/null +++ b/src/sdk/doc/algos.md @@ -0,0 +1,40 @@ +# BTS Basic Algos + +## Lean + +The goal of the lean order is to watch the opposite-inside market size and cross when that sizegoes below the threshold (this also implies a cross if that size goes to zero and the markettrades away from you)If the market trades toward you, you can opt into several different behaviors + +##### Parameters + * Cross Thresh - size thresh in absolute or ratio terms + * Max Chase Ticks - max levels to pay up, in ticks + + +## Join +The goal of the join order is to watch the opposite-inside market and join when the market flips (that is, when the best offer becomes the best bid) and has at least threshold size. + +##### Parameters +* Join Thresh - size thresh in absolute or ratio terms + + +## Top + +The goal of the top order is to not trade by yourself, but the intent at the time of order entry is toget filled.If the order is on the market (say current bid price) AND the quantity on that bid price is belowtop thresh then cancel the order else remain working. + +##### Parameters + * Top Thresh - size thresh in absolute or ratio terms + + +## Cover +A Cover order’s purpose is to fire a hedge order (of any type) in response to fills from it’s childorder on the opposite side of the market N ticks away + +##### Parameters + * Profit Ticks - ticks above or below fill price to place profit / cover order + + +## OCO +An OCO order’s purpose is to join a 'stop' or 'other' order to a main order wherein when one of these is cancelled, both cancel, or when one is filled, the other order's size is debited in the amount of the fill. + +Composing this order with Cover simulates scalping functionality + +##### Parameters + * Stop Ticks - ticks below or above main order to place stop / other order \ No newline at end of file diff --git a/src/sdk/make_sdk.sh b/src/sdk/make_sdk.sh index 945b69c2e..53112d13e 100755 --- a/src/sdk/make_sdk.sh +++ b/src/sdk/make_sdk.sh @@ -25,6 +25,7 @@ ln -sf $1/server $1/sdk/lib mkdir $1/sdk/examples cp $2/examples/* $1/sdk/examples +cp $2/doc/* $1/sdk/doc cp $2/sdk_shell.sh $1/sdk/sdk_shell.sh cp $2/update_github.sh $1/sdk/update_github.sh cp $2/README.md $1/sdk/README.md From 82a8d007bfd1296ac8ec6363b1bb302ff411e351 Mon Sep 17 00:00:00 2001 From: Kevin Darby Date: Wed, 6 Feb 2019 17:06:22 -0500 Subject: [PATCH 58/85] only generate package on release build --- src/client/spark_sdk/spark_sdk.csproj | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/client/spark_sdk/spark_sdk.csproj b/src/client/spark_sdk/spark_sdk.csproj index 0579a5960..b1995ae48 100644 --- a/src/client/spark_sdk/spark_sdk.csproj +++ b/src/client/spark_sdk/spark_sdk.csproj @@ -2,8 +2,7 @@ netcoreapp2.1;net471 Exe - - true + true $(OutputPath)\net471 true true From 9390fb06390fbe534e16779c0afb6083dee3d210 Mon Sep 17 00:00:00 2001 From: Kevin Darby Date: Wed, 6 Feb 2019 17:41:24 -0500 Subject: [PATCH 59/85] per Brian standardizing on Stop Ticks, Stop Type, Profit Ticks, Profit Type --- .../Order/OrderDetailsPopupControl.xaml | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/client/spark/view/Controls/Order/OrderDetailsPopupControl.xaml b/src/client/spark/view/Controls/Order/OrderDetailsPopupControl.xaml index b06aa00ea..6c464260b 100644 --- a/src/client/spark/view/Controls/Order/OrderDetailsPopupControl.xaml +++ b/src/client/spark/view/Controls/Order/OrderDetailsPopupControl.xaml @@ -334,16 +334,16 @@ Orientation="Horizontal" Grid.Row="1" > - Hedge Ticks - + Profit Ticks + - Hedge - + Profit Type + - Other Ticks - + Stop Ticks + - Other Side - + Stop Side + - Other - + Stop Type + Date: Thu, 7 Feb 2019 10:32:13 -0500 Subject: [PATCH 60/85] Update fills to use SecurityStore.join --- .../spark/Modules/Trading/Fills/FillsVM.cs | 5 +- src/client/spark/Orders/FillTuple.cs | 10 +- src/client/spark/Orders/FillTupleSource.cs | 99 ++++++------------- src/client/spark/Orders/OrdersModel.cs | 6 +- src/client/spark/Security/MDLookup.cs | 30 ++++++ src/client/spark/Security/SecurityStore.cs | 4 +- .../spark/view/Spark/PAndL/PAndLModel.cs | 9 +- src/client/spark/view/Spark/PAndL/PnLData.cs | 14 ++- 8 files changed, 85 insertions(+), 92 deletions(-) create mode 100644 src/client/spark/Security/MDLookup.cs diff --git a/src/client/spark/Modules/Trading/Fills/FillsVM.cs b/src/client/spark/Modules/Trading/Fills/FillsVM.cs index 595108167..f3586e228 100644 --- a/src/client/spark/Modules/Trading/Fills/FillsVM.cs +++ b/src/client/spark/Modules/Trading/Fills/FillsVM.cs @@ -39,7 +39,7 @@ namespace mbts.spark { user_filter = DiscreteValuesFilterVM.Create(_model.factory.subscribe_users().Select(d => d.Values), r => r.user, nameof(Row.user)); security_filter = DiscreteValuesFilterVM.Create( fills_source.with_respect_to( - ft => ft.security_md.HasValue ? ft.security_md.Value.security : SecurityDataHelper.Defaults.SecurityImmutable, + ft => ft.security ?? SecurityDataHelper.Defaults.SecurityImmutable, s => s.Id, ft => ft.key ).Select(t => (t.added.Values, t.removed.Values)), r => r.security, @@ -186,7 +186,8 @@ namespace mbts.spark { readonly FillImmutable _fill; readonly OrderRecordImmutable _order; - public Row(FillTuple fill_tuple, OrdersModel model) : base(fill_tuple.security_md.HasValue ? fill_tuple.security_md.Value.security : SecurityDataHelper.Defaults.SecurityImmutable) { + public Row(FillTuple fill_tuple, OrdersModel model) : + base(fill_tuple.security ?? SecurityDataHelper.Defaults.SecurityImmutable) { _fill = fill_tuple.fill; _order = fill_tuple.order ?? OrderDataHelper.Defaults.OrderRecordImmutable; _model = model; diff --git a/src/client/spark/Orders/FillTuple.cs b/src/client/spark/Orders/FillTuple.cs index 4481d2ffa..80f7eb946 100644 --- a/src/client/spark/Orders/FillTuple.cs +++ b/src/client/spark/Orders/FillTuple.cs @@ -10,14 +10,14 @@ namespace mbts.spark { class FillTuple : IEquatable { public readonly FillImmutable fill; public readonly OrderRecordImmutable? order; - public readonly SecurityMD? security_md; + public readonly SecurityImmutable? security; public readonly FillTupleKey key; - public FillTuple(FillImmutable fill, OrderRecordImmutable? order, SecurityMD? security_md) { + public FillTuple(FillImmutable fill, OrderRecordImmutable? order, SecurityImmutable? security) { this.fill = fill; this.order = order; - this.security_md = security_md; - key = new FillTupleKey(fill.FillKey, order.HasValue ? order.Value.Key : 0u, security_md.HasValue ? security_md.Value.security.Id : 0u); + this.security = security; + key = new FillTupleKey(fill.FillKey, order.HasValue ? order.Value.Key : 0u, security?.Id ?? 0); } public string account { @@ -31,7 +31,7 @@ namespace mbts.spark { public bool Equals(FillTuple other) => other != null && other.order.HasValue == order.HasValue - && other.security_md.HasValue == security_md.HasValue + && other.security.HasValue == security.HasValue && other.fill.Equals(fill); public override bool Equals(object obj) => Equals(obj as FillTuple); diff --git a/src/client/spark/Orders/FillTupleSource.cs b/src/client/spark/Orders/FillTupleSource.cs index 18629764e..dc8962b70 100644 --- a/src/client/spark/Orders/FillTupleSource.cs +++ b/src/client/spark/Orders/FillTupleSource.cs @@ -1,4 +1,5 @@ -using bts.api.core; +using bts.api; +using bts.api.core; using bts.api.orderman; using mbts.collections; using mbts.utils; @@ -14,15 +15,14 @@ namespace mbts.spark { // returns a ref-count Observable.Create-wrapper that initializes/destroys the [hot-observable] FillTupleSource. // (initializes when the first subscription is made, and destroys when the last subscription is disposed). public static IObservable> create_observable( - IObservable> securities, - IObservable> orders, - IObservable> fills) { + IObservable> orders, + IObservable> fills) { FillTupleSource source = null; int ref_count = 0; return Observable.Create>(observer => { if (Interlocked.Increment(ref ref_count) == 1) { SparkUtils.lazy_assert(source == null, () => fmt_assert_msg("source should be null, there must be a leak.")); - source = new FillTupleSource(securities, orders, fills); + source = new FillTupleSource(orders, fills); } IDisposable sub = source.Subscribe(observer); return Disposable.Create(() => { @@ -38,19 +38,16 @@ namespace mbts.spark { readonly Dictionary _state = new Dictionary(); readonly SetMap _order_fills = new SetMap(); readonly SetMap _security_fills = new SetMap(); - IReadOnlyDictionary _fills_cache = new Dictionary(0); - IReadOnlyDictionary _orders_cache = new Dictionary(0); - IReadOnlyDictionary _security_md_cache = new Dictionary(0); + IReadOnlyDictionary _fills_cache = new Dictionary(); + IReadOnlyDictionary _orders_cache = new Dictionary(); IDisposable _dispose; object _lock = new object(); FillTupleSource( - IObservable> securities, - IObservable> orders, - IObservable> fills) => + IObservable> orders, + IObservable> fills) => // _dispose = Disposable.Create( - securities.Subscribe(_update_securities), orders.Subscribe(_update_orders), fills.AddRemoveOnly().Subscribe(_update_fills)); @@ -63,19 +60,19 @@ namespace mbts.spark { public void Dispose() => Disposable.dispose(ref _dispose); - void _update_fills(LegacyTableUpdate fills) { + void _update_fills(LegacyTableUpdate<(SecurityImmutable?,FillImmutable), string> fills) { lock (_lock) { var removed = new Dictionary(); var added = new Dictionary(); - foreach (FillImmutable fill in fills.removed) { - FillTuple old_ft = new FillTuple(fill, _get_order(fill.OrderKey), _get_security_md(fill.SecurityKey)); + foreach (var (security,fill) in fills.removed) { + FillTuple old_ft = new FillTuple(fill, _get_order(fill.OrderKey), security); removed.Add(old_ft.key, old_ft); _security_fills.remove(fill.SecurityKey, fill.FillKey); _order_fills.remove(fill.OrderKey, fill.FillKey); } - foreach (FillImmutable fill in fills.added) { - FillTuple new_ft = new FillTuple(fill, _get_order(fill.OrderKey), _get_security_md(fill.SecurityKey)); + foreach (var (security,fill) in fills.added) { + FillTuple new_ft = new FillTuple(fill, _get_order(fill.OrderKey), security); added.Add(new_ft.key, new_ft); _security_fills.add(fill.SecurityKey, fill.FillKey); _order_fills.add(fill.OrderKey, fill.FillKey); @@ -85,35 +82,33 @@ namespace mbts.spark { } } - void _update_orders(TableUpdate orders) { + void _update_orders(TableUpdate<(SecurityImmutable?,OrderRecordImmutable), uint> orders) { lock (_lock) { Dictionary added = new Dictionary(), removed = new Dictionary(); HashSet changed_keys = new HashSet(); - foreach (var o in orders.removed) { + foreach (var (_,o) /*winking owl*/ in orders.removed) { if (_order_fills.try_lookup(o.Key, out ISet fill_keys)) { OrderRecordImmutable? curr_order = null; - if (orders.try_get(o.Key, out OrderRecordImmutable newval)) { - curr_order = newval; + if (orders.try_get(o.Key, out var newval)) { + curr_order = newval.Item2; changed_keys.Add(o.Key); } - IEnumerable fills = fill_keys.Select(fk => _fills_cache[fk]); - foreach (FillImmutable fill in fills) { - SecurityMD? sec_md = _get_security_md(fill.SecurityKey); - FillTuple old_ft = new FillTuple(fill, o, sec_md); + var fills = fill_keys.Select(fk => _fills_cache[fk]); + foreach (var (security, fill) in fills) { + FillTuple old_ft = new FillTuple(fill, o, security); removed.Add(old_ft.key, old_ft); - FillTuple new_ft = new FillTuple(fill, curr_order, sec_md); + FillTuple new_ft = new FillTuple(fill, curr_order, security); added.Add(new_ft.key, new_ft); } } } - foreach (var o in orders.added) { + foreach (var (_,o) in orders.added) { if (!changed_keys.Contains(o.Key) && _order_fills.try_lookup(o.Key, out ISet fill_keys)) { - IEnumerable fills = fill_keys.Select(fk => _fills_cache[fk]); - foreach (FillImmutable fill in fills) { - SecurityMD? sec_md = _get_security_md(fill.SecurityKey); - FillTuple old_ft = new FillTuple(fill, null, sec_md); + var fills = fill_keys.Select(fk => _fills_cache[fk]); + foreach (var (security,fill) in fills) { + FillTuple old_ft = new FillTuple(fill, null, security); removed.Add(old_ft.key, old_ft); - FillTuple new_ft = new FillTuple(fill, o, sec_md); + FillTuple new_ft = new FillTuple(fill, o, security); added.Add(new_ft.key, new_ft); } } @@ -123,46 +118,8 @@ namespace mbts.spark { } } - void _update_securities(IReadOnlyDictionary securities) { - lock (_lock) { - Dictionary added = new Dictionary(), removed = new Dictionary(); - _security_md_cache.Keys.difference(securities.Keys, out IEnumerable added_keys, out IEnumerable removed_keys); - foreach (var k in removed_keys) { - if (_security_fills.try_lookup(k, out ISet fill_keys)) { - IEnumerable fills = fill_keys.Select(fk => _fills_cache[fk]); - SecurityMD old_sec_md = _security_md_cache[k]; - foreach (FillImmutable fill in fills) { - OrderRecordImmutable? order = _get_order(fill.OrderKey); - FillTuple old_ft = new FillTuple(fill, order, old_sec_md); - removed.Add(old_ft.key, old_ft); - FillTuple new_ft = new FillTuple(fill, order, null); - added.Add(new_ft.key, new_ft); - } - } - } - foreach (var k in added_keys) { - if (_security_fills.try_lookup(k, out ISet fill_keys)) { - IEnumerable fills = fill_keys.Select(fk => _fills_cache[fk]); - SecurityMD new_sec_md = securities[k]; - foreach (FillImmutable fill in fills) { - OrderRecordImmutable? order = _get_order(fill.OrderKey); - FillTuple old_ft = new FillTuple(fill, order, null); - removed.Add(old_ft.key, old_ft); - FillTuple new_ft = new FillTuple(fill, order, new_sec_md); - added.Add(new_ft.key, new_ft); - } - } - } - _security_md_cache = securities; - _apply_changes(added, removed); - } - } - - SecurityMD? _get_security_md(uint key) => - _security_md_cache.TryGetValue(key, out SecurityMD smd) ? smd : new SecurityMD?(); - OrderRecordImmutable? _get_order(uint key) => - _orders_cache.TryGetValue(key, out OrderRecordImmutable o) ? o : new OrderRecordImmutable?(); + _orders_cache.TryGetValue(key, out var o) ? o.Item2 : new OrderRecordImmutable?(); void _apply_changes(Dictionary added, Dictionary removed) { foreach (var kv in removed) diff --git a/src/client/spark/Orders/OrdersModel.cs b/src/client/spark/Orders/OrdersModel.cs index c5892f4a4..0503e569e 100644 --- a/src/client/spark/Orders/OrdersModel.cs +++ b/src/client/spark/Orders/OrdersModel.cs @@ -71,10 +71,10 @@ namespace mbts.spark { _service.subscribei_fills() .Naggle(interval, scheduler) .PermissionFilter(admin_model.on_admin_data) - //.join(sstore, f => ) + .join(sstore, f => f.SecurityKey) .Table().Cache(); _activity = new OrderActivitySummarizer(bus, on_orders); - on_fill_tuples = FillTupleSource.create_observable(bus.subscribe(SparkTags.SecuritiesTag), on_orders.Select(i => i.order, order => order.Key), on_fills); + on_fill_tuples = FillTupleSource.create_observable(on_orders, on_fills); _security_orders_demuxer = new TableUpdateDemuxer(o => o.Key, o => o.security_key()); _security_positions_demuxer = new TableUpdateDemuxer(o => o.RecordId, p => p.SecurityId); _dispose = Disposable.Create( @@ -94,7 +94,7 @@ namespace mbts.spark { public DynamicOrderHelper dynamic_helper => _dynamic_helper; public IObservable on_orders { get; } public IObservable> on_order_securities { get; } - public IObservable> on_fills { get; } + public IObservable> on_fills { get; } public IObservable> on_fill_tuples { get; } public IObservable> working_orders { get; } public IObservable> on_positions { get; } diff --git a/src/client/spark/Security/MDLookup.cs b/src/client/spark/Security/MDLookup.cs new file mode 100644 index 000000000..e7949dda7 --- /dev/null +++ b/src/client/spark/Security/MDLookup.cs @@ -0,0 +1,30 @@ +using mbts; +using System; +using System.Collections.Generic; +using System.ComponentModel.Composition; +using System.Reactive.Linq; + +namespace bts { + class MDLookup : IDisposable { + IReadOnlyDictionary _state; + IDisposable _dispose; + + [ImportingConstructor] + public MDLookup(IObservable> src) { + _dispose = src.Subscribe(s => _state = s); + } + + public void Dispose() { Disposable.dispose(ref _dispose); } + + public SecurityMD? get_md(uint key) { + if (_state?.TryGetValue(key, out var md) == true) { + return md; + } + return null; + } + public SecurityMD? get_md(uint? key) { + if (key.HasValue) return get_md(key.Value); + return null; + } + } +} diff --git a/src/client/spark/Security/SecurityStore.cs b/src/client/spark/Security/SecurityStore.cs index 731ca169f..cf2dd306d 100644 --- a/src/client/spark/Security/SecurityStore.cs +++ b/src/client/spark/Security/SecurityStore.cs @@ -139,11 +139,11 @@ namespace bts { /////////////// #region public - // Guarunteed to return a security if it has ever been seen by the system + // Guaranteed to return a security if it has ever been seen by the system public Task get_security(security_key key) { return get_security(key, _live_securities); } - // Guarunteed to return a security if it has ever been seen by the system + // Guaranteed to return a security if it has ever been seen by the system public Task> get_securities(IEnumerable keys) { return get_securities(keys, _live_securities); } diff --git a/src/client/spark/view/Spark/PAndL/PAndLModel.cs b/src/client/spark/view/Spark/PAndL/PAndLModel.cs index 558f2305a..9ffa2d95b 100644 --- a/src/client/spark/view/Spark/PAndL/PAndLModel.cs +++ b/src/client/spark/view/Spark/PAndL/PAndLModel.cs @@ -1,4 +1,5 @@ -using bts.api; +using bts; +using bts.api; using mbts.utils; using System; using System.ComponentModel.Composition; @@ -8,18 +9,18 @@ namespace mbts.spark { class PAndLModel : IDisposable { readonly IBus _bus; readonly OrdersModel _order_model; - //readonly AccountService _account_service; + readonly MDLookup _mdlookup; IDisposable _dispose; [ImportingConstructor] private PAndLModel(IBus bus, OrdersModel order_model/*, AccountService account_service*/) { _bus = bus; _order_model = order_model; - //_account_service = account_service; + _mdlookup = new MDLookup(bus.subscribe(SparkTags.SecuritiesTag)); } public IObservable> on_pnl_data => - _order_model.on_fill_tuples.convert(ft => new PnLData(ft)).table(pd => pd.key); + _order_model.on_fill_tuples.Select(ft => new PnLData(ft, _mdlookup.get_md(ft.security?.Id))).table(pd => pd.key); //public async Task get_accounts() { // var request = await _account_service.get_accounts_async(new GetAccountsRequest()); diff --git a/src/client/spark/view/Spark/PAndL/PnLData.cs b/src/client/spark/view/Spark/PAndL/PnLData.cs index 99c441c16..5b44015a5 100644 --- a/src/client/spark/view/Spark/PAndL/PnLData.cs +++ b/src/client/spark/view/Spark/PAndL/PnLData.cs @@ -28,19 +28,23 @@ namespace mbts.spark { class PnLData { readonly FillTuple _fill_tuple; + readonly SecurityMD? _md; - public PnLData(FillTuple fill_tuple) => + public PnLData(FillTuple fill_tuple, SecurityMD? md) { + // It is entirely possible for the FillTuple to have a security but for no SecrityMD to be available. For example: expired security + _md = md; _fill_tuple = fill_tuple; + } public FillTupleKey key => _fill_tuple.key; public FillImmutable fill => _fill_tuple.fill; - public SecurityMD? sec_md => _fill_tuple.security_md; + public SecurityMD? sec_md => _md; + public SecurityImmutable? security => _fill_tuple.security; public OrderRecordImmutable? order => _fill_tuple.order; public string account => _fill_tuple.account; - public Venue? exchange => sec_md?.security.Exchange; - public string symbol => sec_md?.security.Symbol; - public SecurityImmutable? security => sec_md?.security; + public Venue? exchange => security?.Exchange; + public string symbol => security?.Symbol; public double buy_quantity => (fill.Side == Side.Buy) ? fill.LastShares : 0; public double sell_quantity => (fill.Side == Side.Sell) ? fill.LastShares : 0; public SecurityType? security_type => security?.security_type(); From 1697a1a0159f5dabba0aa37c413285415656c2b2 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 7 Feb 2019 11:25:37 -0500 Subject: [PATCH 61/85] Cease barbarism --- src/client/spark/Orders/FillTupleSource.cs | 4 ++-- src/lib/CMakeDeps_win.cmake | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/client/spark/Orders/FillTupleSource.cs b/src/client/spark/Orders/FillTupleSource.cs index dc8962b70..679ca6a1f 100644 --- a/src/client/spark/Orders/FillTupleSource.cs +++ b/src/client/spark/Orders/FillTupleSource.cs @@ -82,7 +82,7 @@ namespace mbts.spark { } } - void _update_orders(TableUpdate<(SecurityImmutable?,OrderRecordImmutable), uint> orders) { + void _update_orders(TableUpdate<(SecurityImmutable?,OrderRecordImmutable order), uint> orders) { lock (_lock) { Dictionary added = new Dictionary(), removed = new Dictionary(); HashSet changed_keys = new HashSet(); @@ -90,7 +90,7 @@ namespace mbts.spark { if (_order_fills.try_lookup(o.Key, out ISet fill_keys)) { OrderRecordImmutable? curr_order = null; if (orders.try_get(o.Key, out var newval)) { - curr_order = newval.Item2; + curr_order = newval.order; changed_keys.Add(o.Key); } var fills = fill_keys.Select(fk => _fills_cache[fk]); diff --git a/src/lib/CMakeDeps_win.cmake b/src/lib/CMakeDeps_win.cmake index afd768425..f3e9584f4 100644 --- a/src/lib/CMakeDeps_win.cmake +++ b/src/lib/CMakeDeps_win.cmake @@ -1 +1 @@ -set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp;utils/stats.cpp;utils/tests/AuthTokenTests.cpp;utils/thread_manager.cpp) \ No newline at end of file +set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/AuthService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/perflog_cfg.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MDClient.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityCache.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SecurityOracle.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp;utils/stats.cpp;utils/tests/AuthTokenTests.cpp;utils/thread_manager.cpp) \ No newline at end of file From cdf13268a60356a1e8b06b8985bd8ce274ac029b Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Thu, 7 Feb 2019 14:41:51 -0600 Subject: [PATCH 62/85] OrderTypeColumn will now display the DynamicOrderType --- src/client/spark/Modules/Trading/Fills/FillsVM.cs | 3 ++- .../spark/Modules/Trading/OrderBook/OrderBookColumns.cs | 2 +- .../spark/Modules/Trading/OrderBook/OrderBookRowVM.cs | 1 + .../Columns/FrameworkElementColumns/OrderTypeColumn.cs | 6 ++++-- src/client/spark/view/Spark/Core/OrderUtils.cs | 3 +++ 5 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/client/spark/Modules/Trading/Fills/FillsVM.cs b/src/client/spark/Modules/Trading/Fills/FillsVM.cs index 595108167..a7abb9199 100644 --- a/src/client/spark/Modules/Trading/Fills/FillsVM.cs +++ b/src/client/spark/Modules/Trading/Fills/FillsVM.cs @@ -56,7 +56,7 @@ namespace mbts.spark { add_col(Columns.Account, DefaultColumnHeaders.Account, r => r.account, nameof(Row.account), account_filter, DataGridFilterKnownTemplates.DiscreteValues, DefaultColumnWidths.Account); add_col(Columns.Exchange, DefaultColumnHeaders.Exchange, r => r.exchange.ToString(), nameof(Row.exchange), exchange_filter, DataGridFilterKnownTemplates.DiscreteValues, DefaultColumnWidths.Exchange); add_col(Columns.TimeInForce, DefaultColumnHeaders.TimeInForce, r => r.time_in_force.ToString(), nameof(Row.time_in_force), time_in_force_filter, DataGridFilterKnownTemplates.DiscreteValues, DefaultColumnWidths.TimeInForce); - columns.Add(new OrderTypeColumn(Columns.OrderType, new Binding(nameof(Row.order_type)), new Binding(nameof(Row.order_metadata)), order_type_filter) { id = col_id(Columns.OrderType), sort_member_path = nameof(Row.order_type) }); + columns.Add(new OrderTypeColumn(Columns.OrderType, new Binding(nameof(Row.order_type)), new Binding(nameof(Row.order_metadata)), new Binding(nameof(Row.dynamic_order_type_display_name)), order_type_filter) { id = col_id(Columns.OrderType), sort_member_path = nameof(Row.order_type) }); columns.Add(new SideColumn(Columns.Side, r => r.side, side_filter) { id = col_id(Columns.Side), sort_member_path = nameof(Row.side) }); add_col(Columns.Size, DefaultColumnHeaders.Size, r => r.size.ToString(), nameof(Row.size), size_filter, DataGridFilterKnownTemplates.UIntMinMax, DefaultColumnWidths.Size); add_col(Columns.Price, DefaultColumnHeaders.Price, r => r.format_price(r.price), nameof(Row.price), price_filter, DataGridFilterKnownTemplates.DoubleMinMax, DefaultColumnWidths.Price); @@ -196,6 +196,7 @@ namespace mbts.spark { public DateTime fill_time => _fill.Timestamp.ToDateTimeOffset().LocalDateTime; public TimeInForce time_in_force => _order.time_in_force(); public OrderType order_type => _order.order_type(); + public string dynamic_order_type_display_name => _order.dynamic_order_type_display_name(); public OrderMetadata order_metadata => _model.factory.get_metadata(_order.Def.CustomDef.CustomCase); public string account { get; private set; } public string user => _order.user(); diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookColumns.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookColumns.cs index 58a62d2b4..5986961e8 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookColumns.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookColumns.cs @@ -559,7 +559,7 @@ namespace mbts.spark { public class OrderBookOrderTypeColumn : OrderTypeColumn, ICVOrderBookColumn { ListSortDirection? _cached_sort_direction; public OrderBookOrderTypeColumn(IDataGridColumnPersistentFilter filter) - : base(Columns.OrderType, new Binding(nameof(OrderBookRowVM.order_type)), new Binding(nameof(OrderBookRowVM.order_metadata)), filter) { + : base(Columns.OrderType, new Binding(nameof(OrderBookRowVM.order_type)), new Binding(nameof(OrderBookRowVM.order_metadata)), new Binding(nameof(OrderBookRowVM.dynamic_order_type_display_name)), filter) { id = key.ToString(); this.filter = filter; sort_member_path = nameof(OrderBookRowVM.order_type); diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs index 76dc8341f..97270e0f0 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs @@ -62,6 +62,7 @@ namespace mbts.spark { public string symbol => _security_md.symbol(); // Def public CustomOrderDefinition.CustomOneofCase order_type => order.order_type(); + public string dynamic_order_type_display_name => order.dynamic_order_type_display_name(); public OrderSubType order_sub_type => order.sub_type(); public string order_sub_type_text => _fmt_order_sub_type(order_sub_type); public Side side => order.side(); diff --git a/src/client/spark/view/Controls/DIYGrid/Columns/FrameworkElementColumns/OrderTypeColumn.cs b/src/client/spark/view/Controls/DIYGrid/Columns/FrameworkElementColumns/OrderTypeColumn.cs index ac2dce3f4..a26aa53f5 100644 --- a/src/client/spark/view/Controls/DIYGrid/Columns/FrameworkElementColumns/OrderTypeColumn.cs +++ b/src/client/spark/view/Controls/DIYGrid/Columns/FrameworkElementColumns/OrderTypeColumn.cs @@ -6,11 +6,12 @@ using System.Windows.Data; namespace mbts.ui.controls { partial class DIYGrid { internal class OrderTypeColumn : FrameworkElementColumnBase, IFilterableColumn { - readonly BindingBase _order_type_binding, _metadata_binding; + readonly BindingBase _order_type_binding, _metadata_binding, _display_name_binding; - public OrderTypeColumn(TColKey key, BindingBase order_type_binding, BindingBase metadata_binding, IDataGridColumnFilter filter) : base(key) { + public OrderTypeColumn(TColKey key, BindingBase order_type_binding, BindingBase metadata_binding, BindingBase display_name_binding, IDataGridColumnFilter filter) : base(key) { _order_type_binding = order_type_binding; _metadata_binding = metadata_binding; + _display_name_binding = display_name_binding; this.filter = filter; Header = DefaultColumnHeaders.OrderType; Width = DefaultColumnWidths.OrderType; @@ -28,6 +29,7 @@ namespace mbts.ui.controls { }; v.SetBinding(OrderTypeItem.OrderTypeProperty, _order_type_binding); v.SetBinding(OrderTypeItem.MetadataProperty, _metadata_binding); + v.SetBinding(OrderTypeItem.DisplayNameProperty, _display_name_binding); if (font_weight_binding != null) { v.SetBinding(FontWeightProperty, font_weight_binding); } else { diff --git a/src/client/spark/view/Spark/Core/OrderUtils.cs b/src/client/spark/view/Spark/Core/OrderUtils.cs index 6b588658b..c61f22eca 100644 --- a/src/client/spark/view/Spark/Core/OrderUtils.cs +++ b/src/client/spark/view/Spark/Core/OrderUtils.cs @@ -69,6 +69,9 @@ namespace mbts.spark { public static CustomOrderDefinition.CustomOneofCase order_type(this OrderRecordImmutable order) => order.HasDef && order.Def.HasCustomDef ? order.Def.CustomDef.CustomCase : CustomOrderDefinition.CustomOneofCase.None; + public static string dynamic_order_type_display_name(this OrderRecordImmutable order) => + order.HasDef && order.Def.HasCustomDef && order.Def.CustomDef.HasDynamic ? order.Def.CustomDef.Dynamic.DynamicOrderType : string.Empty; + public static OrderSubType sub_type(this OrderRecordImmutable order) => order.HasDef ? order.Def.SubType : OrderSubType.None; From cdce7cd2422dfc87f8b745e904c353709577b1bf Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 7 Feb 2019 17:36:38 -0500 Subject: [PATCH 63/85] Bugfix: SecurityCache is shared_ptr pimpl and should be stored by value instead of by ref even though this messes with const --- src/lib/md/MDConsumer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/md/MDConsumer.cpp b/src/lib/md/MDConsumer.cpp index a532cdf1a..d1659d628 100644 --- a/src/lib/md/MDConsumer.cpp +++ b/src/lib/md/MDConsumer.cpp @@ -537,7 +537,7 @@ class MDConsumerImpl { std::shared_ptr _sci; map> _deferred_handles; vector _callbacks; - SecurityCache& _cache; + mutable SecurityCache _cache; // co2::Platform _platform; public: From 785839327ebe31cef07fc90fe2dfaf67138097c1 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 7 Feb 2019 17:36:56 -0500 Subject: [PATCH 64/85] Add extra safety check that a given security is not serialized more than once --- src/client/spark/Security/SecurityStore.cs | 12 ++++++++---- src/lib/md/relay/QueryServer.cpp | 4 +++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/client/spark/Security/SecurityStore.cs b/src/client/spark/Security/SecurityStore.cs index cf2dd306d..ed848ef2b 100644 --- a/src/client/spark/Security/SecurityStore.cs +++ b/src/client/spark/Security/SecurityStore.cs @@ -101,21 +101,25 @@ namespace bts { // Request any missing keys from the server but do not wait for them to arrive IReadOnlyDictionary get_and_ensure_securities(IEnumerable keys) { var result = new Dictionary(); - var missing = new SecurityKeyList(); + var missing = new HashSet(); foreach (var key in keys) { var security = try_get_local(key); if (security.HasValue) { result.TryAdd(key, security.Value); } else { - missing.Keys.Add(key); + missing.Add(key); } } // Update cache in the background - if (missing.Keys.Count > 0) { + if (missing.Count > 0) { Task.Run(async () => { - var r = await _md.get_securities_async(missing); + var skl = new SecurityKeyList(); + skl.Keys.AddRange(missing); + var r = await _md.get_securities_async(skl); if (r.success) { + var retrieved = r.value.Security.Select(s => s.Id); + Log.Md.info($"Loaded securities from cold storage [{string.Join(",", retrieved)}]"); foreach (var security in r.value.Security) { try_cache_security(new SecurityImmutable(security)); } diff --git a/src/lib/md/relay/QueryServer.cpp b/src/lib/md/relay/QueryServer.cpp index 5b44513d4..e8331308b 100644 --- a/src/lib/md/relay/QueryServer.cpp +++ b/src/lib/md/relay/QueryServer.cpp @@ -382,7 +382,9 @@ public: void get_securities(google::protobuf::RepeatedField keys, api::SecurityList* result) const { bts_assert(result); - for (auto k : keys) { + // Ensure keys are unique + std::set unique_keys(keys.begin(), keys.end()); + for (auto k : unique_keys) { auto ptr = _consumer.get_security(SecurityKey(k)); if (ptr) { result->add_security()->CopyFrom(*ptr); From 9132b4a1789fceba98eb9e044e083c1cdb62ab0d Mon Sep 17 00:00:00 2001 From: Michael Harding Date: Thu, 7 Feb 2019 12:37:10 -0500 Subject: [PATCH 65/85] Fix clip limits test --- src/client/spark_test/System/LimitsTests.cs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/client/spark_test/System/LimitsTests.cs b/src/client/spark_test/System/LimitsTests.cs index b517765bd..36a2235fe 100644 --- a/src/client/spark_test/System/LimitsTests.cs +++ b/src/client/spark_test/System/LimitsTests.cs @@ -124,10 +124,16 @@ namespace bts.tests over_limit_order_key = over_limit_order_result.value.OrderKey; no_limit_order_key = no_limit_order_result.value.OrderKey; + bool waiting_for_orders = true; + var orders = _order_service.subscribei_orders().Subscribe(ords => { + waiting_for_orders = false; + IEnumerable filtered_orders = ords.all.Where(x => x.Key == under_limit_order_key || x.Key == over_limit_order_key || x.Key == no_limit_order_key); + Assert.IsTrue(filtered_orders.Count() == 3, "There should be 3 matching orders... there are not"); + foreach(var o in filtered_orders) { if(o.Key == under_limit_order_key) { @@ -142,6 +148,9 @@ namespace bts.tests } }); + while(waiting_for_orders) { + await Task.Delay(100); + } // Restore the Limits to the original values bool success = false; From 02defef93d902f4b39adc48046da6bbc53c5785c Mon Sep 17 00:00:00 2001 From: Kevin Darby Date: Fri, 8 Feb 2019 14:55:16 -0500 Subject: [PATCH 66/85] oco testing --- .../spark/Orders/OrderTypeVMs/OCOOrderVM.cs | 4 +- .../Order/OrderDetailsPopupControl.xaml | 4 +- .../orderman/orders/book_algos/OCOOrder.cpp | 25 +++++----- testing/manual/algos/oco.txt | 46 +++++++++++++++++++ 4 files changed, 63 insertions(+), 16 deletions(-) create mode 100644 testing/manual/algos/oco.txt diff --git a/src/client/spark/Orders/OrderTypeVMs/OCOOrderVM.cs b/src/client/spark/Orders/OrderTypeVMs/OCOOrderVM.cs index 3bd35629f..c40150ae3 100644 --- a/src/client/spark/Orders/OrderTypeVMs/OCOOrderVM.cs +++ b/src/client/spark/Orders/OrderTypeVMs/OCOOrderVM.cs @@ -28,7 +28,7 @@ namespace mbts.spark this.other_order_types = new InlineUserProperty>(child_order_types, true); other_ticks = new InlineUserProperty(1, false); - other_side = new InlineUserProperty(bts.api.RelativeSide.Opposite, false); + other_side = new InlineUserProperty(bts.api.RelativeSide.Same, false); } public readonly InlineUserProperty selected_child_order_type; public readonly InlineUserProperty> child_order_types; @@ -55,7 +55,7 @@ namespace mbts.spark IDisposable _other_order_errors_sub; public int other_ticks { get; set; } = 1; - public bts.api.RelativeSide other_side { get; set; } = bts.api.RelativeSide.Opposite; + public bts.api.RelativeSide other_side { get; set; } = bts.api.RelativeSide.Same; public OCOOrderVM(OrdersModel model, IObservable security_md_source, string id, string display_name) : this(model, security_md_source, security_md_source.Select(smd => diff --git a/src/client/spark/view/Controls/Order/OrderDetailsPopupControl.xaml b/src/client/spark/view/Controls/Order/OrderDetailsPopupControl.xaml index 6c464260b..5ddcc2eab 100644 --- a/src/client/spark/view/Controls/Order/OrderDetailsPopupControl.xaml +++ b/src/client/spark/view/Controls/Order/OrderDetailsPopupControl.xaml @@ -342,7 +342,7 @@ Orientation="Horizontal" Grid.Row="3" > - Profit Type + Profit Order - Stop Type + Stop Order namespace bts { @@ -70,25 +71,25 @@ bool OCOOrder::setup_hedge(const api::orderman::OrderDefinition& this_def, const void OCOOrder::process_child_state_change(const api::orderman::OrderState& orig_state, const OrderStateUpdateEvent& new_state) { - uint32_t fill_qty = new_state.filled_quantity - orig_state.filled_quantity(); + uint32_t fill_qty = new_state.filled_quantity - orig_state.filled_quantity(); const auto& the_state = hedges_state(); auto& the_hedges = hedges(); for(const auto& i : the_state) { const auto& hedge_state = i.second; - uint32_t orig_qty = hedge_state.quantity(); - uint32_t newqty = orig_qty - std::min(fill_qty, orig_qty); - bts_assert(newqty <= orig_qty); + uint32_t working_qty = hedge_state.quantity() - hedge_state.filled_quantity(); + uint32_t newqty = working_qty - std::min(fill_qty, working_qty); + bts_assert(newqty <= working_qty); auto it = the_hedges.find(i.first); if(it != the_hedges.end()) { - if(newqty != orig_qty) + if(newqty != working_qty) { if(newqty > 0) { - LOG(OrderMan, Info, "OCO reducing hedge qty from {} to {}", hedge_state.quantity(), newqty); + LOG(OrderMan, Info, "OCO reducing hedge qty from {} to {}", working_qty, newqty); it->second.update(Price(hedge_state.price()), Quantity(newqty)); } else @@ -110,15 +111,15 @@ void OCOOrder::process_child_state_change(const api::orderman::OrderState& orig_ void OCOOrder::process_hedge_state_change(const api::orderman::OrderState& orig_state, const OrderStateUpdateEvent& new_state) { - uint32_t fill_qty = new_state.filled_quantity - orig_state.filled_quantity(); - uint32_t orig_qty = child_state().quantity(); - uint32_t newqty = orig_qty - std::min(fill_qty, orig_qty); - bts_assert(newqty <= orig_qty); - if(newqty != orig_qty) + uint32_t fill_qty = new_state.filled_quantity - orig_state.filled_quantity(); + uint32_t working_qty = child_state().quantity() - child_state().filled_quantity(); + uint32_t newqty = working_qty - std::min(fill_qty, working_qty); + bts_assert(newqty <= working_qty); + if(newqty != working_qty) { if(newqty > 0) { - LOG(OrderMan, Info, "OCO reducing child qty from {} to {}", child_state().quantity(), newqty); + LOG(OrderMan, Info, "OCO reducing child qty from {} to {}", working_qty, newqty); update_child(Price(child_state().price()), Quantity(newqty)); } else diff --git a/testing/manual/algos/oco.txt b/testing/manual/algos/oco.txt new file mode 100644 index 000000000..ffd5770ed --- /dev/null +++ b/testing/manual/algos/oco.txt @@ -0,0 +1,46 @@ +- Set +ExchangeSim {value {merge_with_md:false}} + +in your spark.conf + +- Use the order ticket to put in a 10 lot 55 dollar bid +- Open up a ladder for SIM:CLZ9 and put in 10 lot offers at 55.10, 55.11, 55.12 and 1 lot bids at 54.99 and 54.98 +- Select OCO order from the menu. + +-Set Stop Side to Same, Stop Type and Child type to Limit +-Set Stop Ticks to -1 +-Put in a 1 lot bid at 54.93, you should see another 1 lot bid go in one tick worse +-Move these orders independently (but don't get filled) +-Cancel one order, the other should cancel +-Check that an order goes in 1 tick better than where you click if you set the Stop Ticks to 1 + + +-Now set Stop Side to Opposite and Stop ticks to 3, send a 5 lot bid at 55.02 +-You should see a 5 lot buy go in at 55.02 and a 5 lot sell at 55.05. +-Move the bid up to 55.03, the offer should not move +-Move the offer up to 55.06, the bid should stay put +- Cancel either the bid or the offer, the other will cancel. Set it up again and cancel the other side. + +-Keep Stop Side at Opposite and Stop ticks to 3, send a 5 lot bid at 55.03 +-Switch to normal limit orders, send a 1 lot buy at 55.06. You should see a fill at 06 and the 5 lot bid at 03 should reduce to 4 +-Still with limit orders, send a 2 lot sell at 55.03. You should see a fill at 04 and the 4 lot offer at 06 should reduce to 2 +- (you might see the sell order have 1/2 in the badge. This means 1 fill / 2 total quantity, however I think this is a bug because we are working 2 still and that is correct) +-Finally send a 2 lot buy (limit) at 55.06, you should see both the sell side OCO fill and the buy side OCO cancel +-Flip this section around buy vs sell and make sure it does the same thing + +Now compose a bracket order (extra credit if you make a template for it in the new guis) +Select Cover +Set Profit ticks to 1, +Profit Order = OCO +Child=Limit + +For the OCO +Stop Ticks = 3 +Stop Side = Same +Stop Order = Trigger +Child = Limit + +Send in a buy of this order at 55.05 +Fill a 1 lot of this order at 55.05 (by putting in a Limit 55.05 sell) +You should see a 1 lot sell order go in at 55.06 and a 1 lot sell trigger at 55.03 +Cancel one of those orders and the other should go away From 49e5eab7d3c75546da440cd23840290d47622813 Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Mon, 11 Feb 2019 08:55:10 -0500 Subject: [PATCH 67/85] Updated release notes. --- doc/user/docs/release_notes.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/user/docs/release_notes.md b/doc/user/docs/release_notes.md index 110ba5c53..566027819 100644 --- a/doc/user/docs/release_notes.md +++ b/doc/user/docs/release_notes.md @@ -1,5 +1,9 @@ # Release Notes +## 1.5.0 +### Bug Fixes + - Fixed the "Since Open" time filter to use the exchange open time rather than 8:30 AM + ## 1.4.11 ### New Features - Floating Order Book From 1301130afbafaa0c564ff6a6d877eab8252583dc Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Mon, 11 Feb 2019 08:58:00 -0500 Subject: [PATCH 68/85] Quieted the build. --- src/data_files/copy_data_files.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/src/data_files/copy_data_files.sh b/src/data_files/copy_data_files.sh index b3b880d74..cdce7beaa 100644 --- a/src/data_files/copy_data_files.sh +++ b/src/data_files/copy_data_files.sh @@ -6,7 +6,6 @@ cp -r $2/files/* $1 for filename in $1/*; do if [[ $filename == *.xz ]]; then - echo Decompressing $filename unxz $filename fi done From be6124e72f5a6f4596876d5a09729f9ce72e4d9d Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Mon, 11 Feb 2019 09:06:05 -0500 Subject: [PATCH 69/85] Updated release notes. --- doc/user/docs/release_notes.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/user/docs/release_notes.md b/doc/user/docs/release_notes.md index ecd2233da..d3b1f2a87 100644 --- a/doc/user/docs/release_notes.md +++ b/doc/user/docs/release_notes.md @@ -3,7 +3,9 @@ ## 1.5.0 ### Bug Fixes - Fixed the "Since Open" time filter to use the exchange open time rather than 8:30 AM - +### SDK + - Improved SDK documentation and usability + ## 1.4.11 ### New Features - Floating Order Book From 10e174c5b4519d9b564204b786587d35fc9394c7 Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Mon, 11 Feb 2019 09:11:55 -0500 Subject: [PATCH 70/85] Updated release notes. --- doc/user/docs/release_notes.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/doc/user/docs/release_notes.md b/doc/user/docs/release_notes.md index d3b1f2a87..5621f6bca 100644 --- a/doc/user/docs/release_notes.md +++ b/doc/user/docs/release_notes.md @@ -1,11 +1,16 @@ # Release Notes ## 1.5.0 -### Bug Fixes +### User Visible Changes +#### Bug Fixes - Fixed the "Since Open" time filter to use the exchange open time rather than 8:30 AM -### SDK +#### SDK - Improved SDK documentation and usability - +### Internal Changes + - Improved robustness of limit system test + - Extensive refactoring to the market data subsystem to allow for + updates of security definitions + ## 1.4.11 ### New Features - Floating Order Book From ee21b4f597a772140c93def8dadde37bcb89e2be Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Mon, 11 Feb 2019 09:42:37 -0500 Subject: [PATCH 71/85] Updated release notes. --- doc/user/docs/release_notes.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/doc/user/docs/release_notes.md b/doc/user/docs/release_notes.md index d3b1f2a87..ec048c3e1 100644 --- a/doc/user/docs/release_notes.md +++ b/doc/user/docs/release_notes.md @@ -1,10 +1,19 @@ # Release Notes ## 1.5.0 -### Bug Fixes +### User Visible Changes +#### User Interface Improvements + - Removed security key field from most grids + - Removed trade key field from time and sales grid - Fixed the "Since Open" time filter to use the exchange open time rather than 8:30 AM -### SDK +#### SDK - Improved SDK documentation and usability +### Internal Changes + - Improved robustness of limit system test + - Extensive refactoring to the market data subsystem to allow for + updates of security definitions + + ## 1.4.11 ### New Features From 924b654f5f28508adb3bb99a612c6fab93ea6c03 Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Mon, 11 Feb 2019 09:53:30 -0500 Subject: [PATCH 72/85] Update release notes. --- doc/user/docs/release_notes.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/user/docs/release_notes.md b/doc/user/docs/release_notes.md index ec048c3e1..a08414425 100644 --- a/doc/user/docs/release_notes.md +++ b/doc/user/docs/release_notes.md @@ -6,14 +6,14 @@ - Removed security key field from most grids - Removed trade key field from time and sales grid - Fixed the "Since Open" time filter to use the exchange open time rather than 8:30 AM + - Fixed the floating order book so that rows are no longer + highlighted after an order is modified #### SDK - Improved SDK documentation and usability ### Internal Changes - Improved robustness of limit system test - - Extensive refactoring to the market data subsystem to allow for - updates of security definitions + - Extensive market data refactoring allow for security definition updates - ## 1.4.11 ### New Features From 37f595ed9a19863bbef697cc14c179e843abdd99 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 11 Feb 2019 09:55:18 -0500 Subject: [PATCH 73/85] Add startup delay for security discovery --- src/lib/config/defaults/MarketData.conf | 13 +++++++++++++ src/lib/config/protos/MarketData.proto | 11 +++++++++++ src/lib/utils/datetime.cpp | 21 +++++++++++++++++++++ src/lib/utils/datetime.h | 4 ++++ src/server/sparkd/md/run_md.cpp | 24 ++++++++++++++++++++++-- 5 files changed, 71 insertions(+), 2 deletions(-) diff --git a/src/lib/config/defaults/MarketData.conf b/src/lib/config/defaults/MarketData.conf index 17d800d6b..0a11b62c6 100644 --- a/src/lib/config/defaults/MarketData.conf +++ b/src/lib/config/defaults/MarketData.conf @@ -35,3 +35,16 @@ CMEOptions { } } +MDBootDelayWindow { + value { + delay_seconds: 120 + standard { + start: "16:30" + end: "16:40" + } + sunday { + start: "14:00" + end: "14:10" + } + } +} \ No newline at end of file diff --git a/src/lib/config/protos/MarketData.proto b/src/lib/config/protos/MarketData.proto index 08b397958..aba6237c6 100755 --- a/src/lib/config/protos/MarketData.proto +++ b/src/lib/config/protos/MarketData.proto @@ -15,6 +15,17 @@ message PlatformMap optional string glbx_platform = 1 [default = "PLATFORM_SIM"]; } +// LOCAL TIME +message MDBootDelayWindow { + message TimeSpan { + optional string start = 1 [default = "16:30"]; + optional string end = 2 [default = "16:40"]; + } + optional uint32 delay_seconds = 1 [default = 120]; + optional TimeSpan standard = 2; + optional TimeSpan sunday = 3; +} + // // Options that apply to market data server diff --git a/src/lib/utils/datetime.cpp b/src/lib/utils/datetime.cpp index 5e776e159..4cd7fe092 100755 --- a/src/lib/utils/datetime.cpp +++ b/src/lib/utils/datetime.cpp @@ -1,5 +1,6 @@ #include "datetime.h" + #include "bts_assert.h" #include "TimeStamp.h" @@ -317,6 +318,26 @@ Date ll_to_date(LLDateTime time) { return date; } + +bool is_local_time_sunday() { + auto now = local_now(); + return now.date().day_of_week() == boost::date_time::Sunday; +} +bool is_local_time_between(int hh_start, int mm_start, int hh_end, int mm_end) { + auto now = local_now(); + auto hh_now = now.time_of_day().hours(); + auto mm_now = now.time_of_day().minutes(); + + if (hh_now > hh_start && hh_now < hh_end) return true; + if (hh_now == hh_start || hh_now == hh_end) { + if (mm_now >= mm_start && mm_now <= mm_end) { + return true; + } + } + return false; +} + + struct UtcTimeStamp { int hour; int min; diff --git a/src/lib/utils/datetime.h b/src/lib/utils/datetime.h index e6095ad82..133555edc 100755 --- a/src/lib/utils/datetime.h +++ b/src/lib/utils/datetime.h @@ -76,11 +76,15 @@ BTS_EXPORT bool is_at_least_days_old (PTime const &time, uint64_t days); BTS_EXPORT LLDateTime ll_from_unix_epoch(long); BTS_EXPORT LLDateTime ll_from_unix_epoch_ms(long long); + extern "C" BTS_EXPORT void ll_to_parts(LLDateTime, int& year, int& month, int& day, int& hour, int& minute, int& second, int& millisecond); BTS_EXPORT Date ll_to_date(LLDateTime); extern "C" BTS_EXPORT LLDateTime ll_from_parts(int year, int month, int day, int hour, int minute, int second, int millisecond); BTS_EXPORT LLDateTime ll_from_date(const Date& date); +bool is_local_time_sunday(); +bool is_local_time_between(int hh_start, int mm_start, int hh_end, int mm_end); + #ifndef NO_CPP0X extern "C" BTS_EXPORT LLDateTime ll_today(); BTS_EXPORT Date today(); diff --git a/src/server/sparkd/md/run_md.cpp b/src/server/sparkd/md/run_md.cpp index 6bb2a2ef5..56e2c2762 100644 --- a/src/server/sparkd/md/run_md.cpp +++ b/src/server/sparkd/md/run_md.cpp @@ -90,6 +90,26 @@ void run_market_data_publishers_forever() { } } +void boot_sleep() { + auto conf = config::get(); + const auto* schedule = &conf.standard(); + if (datetime::is_local_time_sunday()) { + schedule = &conf.sunday(); + } + bts_assert(schedule); + int hh_start = 0, mm_start = 0, hh_end = 0, mm_end = 0; + sscanf(schedule->start().c_str(), "%i:%i", &hh_start, &mm_start); + sscanf(schedule->end().c_str(), "%i:%i", &hh_end, &mm_end); + if (datetime::is_local_time_between(hh_start, mm_start, hh_end, mm_end)) { + std::this_thread::sleep_for( + std::chrono::seconds( + conf.delay_seconds())); + } + else { + std::this_thread::sleep_for(2s); + } +} + void run_market_datapublishers() { bts_assert(_md_thread == nullptr); run_cleanup(); @@ -103,10 +123,10 @@ void run_market_datapublishers() { threading::make_thread("ExchangeSim", []() { sim::start_exchange_sim(); }) .detach(); } - - std::this_thread::sleep_for(2s); + boot_sleep(); } + void run_record(const std::string& arg){ boost::filesystem::path p(arg); record_everything(p.parent_path().string(), p.filename().string()); From 883373f7e515770486e91d7b048515ee31d1e204 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 11 Feb 2019 09:58:30 -0500 Subject: [PATCH 74/85] Bugfix: SyntheticStrategy.strategy is now null initially --- src/lib/md/MDConsumer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/md/MDConsumer.cpp b/src/lib/md/MDConsumer.cpp index d1659d628..e296fc1d6 100644 --- a/src/lib/md/MDConsumer.cpp +++ b/src/lib/md/MDConsumer.cpp @@ -483,7 +483,7 @@ private: if (legs.size() == (size_t)strategy.security->synthetic().legs_size()) { auto s = make_shared(); s->security.CopyFrom(*strategy.security); - s->strategy->CopyFrom(strategy.strategy); + s->strategy = std::make_unique(strategy.strategy); s->legs = legs; s->updated = TimeStamp::now(); result.insert( From 97d5e2e14917337b3429e03b0a354651ee455c85 Mon Sep 17 00:00:00 2001 From: Kevin Darby Date: Mon, 11 Feb 2019 12:03:19 -0500 Subject: [PATCH 75/85] added doc for OCOOrder --- doc/user/docs/release_notes.md | 3 +++ src/sdk/doc/algos.md | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/user/docs/release_notes.md b/doc/user/docs/release_notes.md index a08414425..e11484d8b 100644 --- a/doc/user/docs/release_notes.md +++ b/doc/user/docs/release_notes.md @@ -8,6 +8,9 @@ - Fixed the "Since Open" time filter to use the exchange open time rather than 8:30 AM - Fixed the floating order book so that rows are no longer highlighted after an order is modified +#### Algos + - Added OCO Algo + - An OCO order creates a relationshiop between a child order and a 'stop' or 'other' order such that when one of these is cancelled, both cancel, or when one is filled, the other order's size is debited in the amount of the fill. Composing this order with Cover simulates scalping functionality #### SDK - Improved SDK documentation and usability ### Internal Changes diff --git a/src/sdk/doc/algos.md b/src/sdk/doc/algos.md index 1264b8de5..11a91894a 100644 --- a/src/sdk/doc/algos.md +++ b/src/sdk/doc/algos.md @@ -32,7 +32,7 @@ A Cover order’s purpose is to fire a hedge order (of any type) in response to ## OCO -An OCO order’s purpose is to join a 'stop' or 'other' order to a main order wherein when one of these is cancelled, both cancel, or when one is filled, the other order's size is debited in the amount of the fill. +An OCO order creates a relationshiop between a child order and a 'stop' or 'other' order such that when one of these is cancelled, both cancel, or when one is filled, the other order's size is debited in the amount of the fill. Composing this order with Cover simulates scalping functionality From 87bc8a067425d493ec36ef742649e8dcd3ac9510 Mon Sep 17 00:00:00 2001 From: Kevin Darby Date: Mon, 11 Feb 2019 12:09:46 -0500 Subject: [PATCH 76/85] typo --- doc/user/docs/release_notes.md | 2 +- src/sdk/doc/algos.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/user/docs/release_notes.md b/doc/user/docs/release_notes.md index e11484d8b..e8a80be34 100644 --- a/doc/user/docs/release_notes.md +++ b/doc/user/docs/release_notes.md @@ -10,7 +10,7 @@ highlighted after an order is modified #### Algos - Added OCO Algo - - An OCO order creates a relationshiop between a child order and a 'stop' or 'other' order such that when one of these is cancelled, both cancel, or when one is filled, the other order's size is debited in the amount of the fill. Composing this order with Cover simulates scalping functionality + - An OCO order creates a relationship between a child order and a 'stop' or 'other' order such that when one of these is cancelled, both cancel, or when one is filled, the other order's size is debited in the amount of the fill. Composing this order with Cover simulates scalping functionality #### SDK - Improved SDK documentation and usability ### Internal Changes diff --git a/src/sdk/doc/algos.md b/src/sdk/doc/algos.md index 11a91894a..e6200cca2 100644 --- a/src/sdk/doc/algos.md +++ b/src/sdk/doc/algos.md @@ -32,7 +32,7 @@ A Cover order’s purpose is to fire a hedge order (of any type) in response to ## OCO -An OCO order creates a relationshiop between a child order and a 'stop' or 'other' order such that when one of these is cancelled, both cancel, or when one is filled, the other order's size is debited in the amount of the fill. +An OCO order creates a relationship between a child order and a 'stop' or 'other' order such that when one of these is cancelled, both cancel, or when one is filled, the other order's size is debited in the amount of the fill. Composing this order with Cover simulates scalping functionality From 4944aaa3fa3ae1cf857215016e06b614f3d9a23c Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Mon, 11 Feb 2019 13:48:22 -0500 Subject: [PATCH 77/85] Updated release notes and tweaked DynamicIceberg name. --- doc/user/docs/release_notes.md | 1 + src/sdk/examples/order_plugin_example.cpp | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/user/docs/release_notes.md b/doc/user/docs/release_notes.md index ec048c3e1..fcf6f02bb 100644 --- a/doc/user/docs/release_notes.md +++ b/doc/user/docs/release_notes.md @@ -8,6 +8,7 @@ - Fixed the "Since Open" time filter to use the exchange open time rather than 8:30 AM #### SDK - Improved SDK documentation and usability + - Dynamic orders are now properly identified in the order book ### Internal Changes - Improved robustness of limit system test - Extensive refactoring to the market data subsystem to allow for diff --git a/src/sdk/examples/order_plugin_example.cpp b/src/sdk/examples/order_plugin_example.cpp index 3202d102a..03de5fa67 100644 --- a/src/sdk/examples/order_plugin_example.cpp +++ b/src/sdk/examples/order_plugin_example.cpp @@ -363,7 +363,7 @@ DynamicOrderDescriptor get_dynamic_iceberg_order_desc() DynamicOrderDescriptor desc; // order type name - desc.set_dynamic_order_type("DynamicIcebergOrder"); + desc.set_dynamic_order_type("dice"); // show size auto param = desc.add_param_desc(); From 3b43df458adb87a7e116957cbf347fd16dd3af3b Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Mon, 11 Feb 2019 13:09:45 -0600 Subject: [PATCH 78/85] reverting to how this was previously; I was overzealous in my horizontal alignment tinkering --- src/client/spark/Modules/Trading/OrderBook/OrderBookColumns.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookColumns.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookColumns.cs index 58a62d2b4..f0532a1fc 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookColumns.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookColumns.cs @@ -651,7 +651,6 @@ namespace mbts.spark { grid.ColumnDefinitions.Add(new ColumnDefinition { Width = new GridLength(1, GridUnitType.Star) }); grid.Children.Add(toggle_button); grid.Children.Add(security_view); - grid.HorizontalAlignment = HorizontalAlignment.Center; grid.VerticalAlignment = VerticalAlignment.Center; // var border = new Border { Child = grid }; From 2925cf50927aded02ad76cdc4f34d221306ba1f9 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 11 Feb 2019 16:10:51 -0500 Subject: [PATCH 79/85] CR Feedback --- src/lib/config/defaults/MarketData.conf | 7 +++++++ src/lib/utils/datetime.cpp | 15 ++++----------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/lib/config/defaults/MarketData.conf b/src/lib/config/defaults/MarketData.conf index 0a11b62c6..08603497a 100644 --- a/src/lib/config/defaults/MarketData.conf +++ b/src/lib/config/defaults/MarketData.conf @@ -35,6 +35,13 @@ CMEOptions { } } +MDBootDelayWindow { + rule: "DeploymentType.type == 'dev'" + value { + delay_seconds: 0 + } +} + MDBootDelayWindow { value { delay_seconds: 120 diff --git a/src/lib/utils/datetime.cpp b/src/lib/utils/datetime.cpp index 4cd7fe092..8c33965e4 100755 --- a/src/lib/utils/datetime.cpp +++ b/src/lib/utils/datetime.cpp @@ -324,20 +324,13 @@ bool is_local_time_sunday() { return now.date().day_of_week() == boost::date_time::Sunday; } bool is_local_time_between(int hh_start, int mm_start, int hh_end, int mm_end) { + using namespace boost::posix_time; auto now = local_now(); - auto hh_now = now.time_of_day().hours(); - auto mm_now = now.time_of_day().minutes(); - - if (hh_now > hh_start && hh_now < hh_end) return true; - if (hh_now == hh_start || hh_now == hh_end) { - if (mm_now >= mm_start && mm_now <= mm_end) { - return true; - } - } - return false; + auto start = ptime(now.date(), hours(hh_start) + minutes(mm_start)); + auto end = ptime(now.date(), hours(hh_end) + minutes(mm_end)); + return (now >= start && now <= end); } - struct UtcTimeStamp { int hour; int min; From 0fc7887ecd297efe38a0fb5efac432ca6e8db9ff Mon Sep 17 00:00:00 2001 From: Kevin Darby Date: Mon, 11 Feb 2019 16:27:24 -0500 Subject: [PATCH 80/85] fix release mode build warning --- src/lib/orderman/actors/order_manager/ChildOrder.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/orderman/actors/order_manager/ChildOrder.h b/src/lib/orderman/actors/order_manager/ChildOrder.h index ca7b2e477..5e6c36d2a 100644 --- a/src/lib/orderman/actors/order_manager/ChildOrder.h +++ b/src/lib/orderman/actors/order_manager/ChildOrder.h @@ -11,7 +11,8 @@ namespace orderman { class ChildOrder { public: ChildOrder() - : _order(nullptr), _parent_order_services(nullptr), _active(false) {} + : _order(nullptr), _parent_order_services(nullptr), + _child_order_services(nullptr), _active(false) {} ChildOrder(Order &o, OrderServices &services) : _order(&o), _parent_order_services(&services), From f6f9605a83a1df4fd94131bbf952018b386303a2 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 12 Feb 2019 12:26:11 -0500 Subject: [PATCH 81/85] Revert "Try to ensure Dispatcher::background is terminated" This reverts commit 5cc408557727828ad09536b625dbf1f1cb6fc7ff. --- src/lib/utils/init_spark_lib.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/lib/utils/init_spark_lib.cpp b/src/lib/utils/init_spark_lib.cpp index 772d62c7a..d9dc6fab9 100644 --- a/src/lib/utils/init_spark_lib.cpp +++ b/src/lib/utils/init_spark_lib.cpp @@ -2,7 +2,6 @@ #include "lib/logging/logging.h" #include "lib/utils/bts_assert.h" #include "lib/utils/thread_manager.h" -#include "lib/utils/Dispatcher.h" namespace bts { namespace utils { @@ -21,7 +20,6 @@ void finit_spark_lib() if (poller) { delete poller; } - utils::Dispatcher::background().terminate(); } }} From 15750383652b6c960fe0edbab2ed0604755084b3 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 13 Feb 2019 08:27:28 -0500 Subject: [PATCH 82/85] Fix equality on OrderSecurity type --- src/client/spark/Orders/OrderSecurity.cs | 2 +- src/client/spark/Orders/OrdersModel.cs | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/client/spark/Orders/OrderSecurity.cs b/src/client/spark/Orders/OrderSecurity.cs index dd9936162..a703e4c2b 100644 --- a/src/client/spark/Orders/OrderSecurity.cs +++ b/src/client/spark/Orders/OrderSecurity.cs @@ -15,7 +15,7 @@ namespace bts.api.orderman { public OrderRecordImmutable order { get; } public override bool Equals(object obj) => (obj is OrderSecurity os) && Equals(os); - public bool Equals(OrderSecurity other) => order.Key == other.order.Key && security == other.security; + public bool Equals(OrderSecurity other) => order.Equals(other.order) && security == other.security; public override int GetHashCode() => (security, order).GetHashCode(); public static bool operator == (OrderSecurity left, OrderSecurity right) => left.Equals(right); diff --git a/src/client/spark/Orders/OrdersModel.cs b/src/client/spark/Orders/OrdersModel.cs index 0503e569e..b41edc81f 100644 --- a/src/client/spark/Orders/OrdersModel.cs +++ b/src/client/spark/Orders/OrdersModel.cs @@ -175,12 +175,12 @@ namespace mbts.spark { clone.Remove(o.order.Key); } } - foreach (var o in table.changed) { - if (o.order.is_active()) { - clone[o.order.Key] = o; + foreach (var o in table.changes) { + if (o.Old.order.is_active()) { + clone.Remove(o.Old.order.Key); } - else { - clone.Remove(o.order.Key); + if (o.New.order.is_active()) { + clone.Add(o.New.order.Key, o.New); } } foreach (var o in table.added) { From 2cb485a883e55670a2fc9853404e6deba5a8de09 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 13 Feb 2019 13:18:36 -0500 Subject: [PATCH 83/85] Add note for ui_perf manual test --- testing/manual/ladder/ui_performance.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/testing/manual/ladder/ui_performance.txt b/testing/manual/ladder/ui_performance.txt index 9fb54cc18..ee27662c8 100644 --- a/testing/manual/ladder/ui_performance.txt +++ b/testing/manual/ladder/ui_performance.txt @@ -14,3 +14,4 @@ This test needs to run against production market data or high quality recorded d 4. Repeat 2. +5. If there is *any* doubt that the UI is snappy bring up the previous release and compare. At minimum we need to not regress UI performance. From 0f8b996c8ca8d9a6a0b0c1b54ea20ae5d58af7d5 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 13 Feb 2019 13:44:07 -0500 Subject: [PATCH 84/85] Reduce config connection timeout to 6 seconds from 30 --- src/lib/config/ConfigDownloader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/config/ConfigDownloader.cpp b/src/lib/config/ConfigDownloader.cpp index 2c6e94af8..da23232eb 100755 --- a/src/lib/config/ConfigDownloader.cpp +++ b/src/lib/config/ConfigDownloader.cpp @@ -53,7 +53,7 @@ ConfigDownloader::Result ConfigDownloader::get(const string& url, stringstream& curl_easy_setopt(_curl, CURLOPT_WRITEDATA, &out_stream); curl_easy_setopt(_curl, CURLOPT_VERBOSE, 0); curl_easy_setopt(_curl, CURLOPT_FAILONERROR, 1); - curl_easy_setopt(_curl, CURLOPT_CONNECTTIMEOUT, 30); + curl_easy_setopt(_curl, CURLOPT_CONNECTTIMEOUT, 6); curl_easy_setopt(_curl, CURLOPT_SOCKOPTFUNCTION, no_linger); CURLcode res = curl_easy_perform(_curl); From 7212049c0cb890330c6665fc2b48b4b634cb952c Mon Sep 17 00:00:00 2001 From: Kevin Darby Date: Thu, 14 Feb 2019 12:37:34 -0500 Subject: [PATCH 85/85] ignore *.code-workspace files --- .gitignore | 1 + remote-sparkvm.code-workspace | 27 --------------------------- 2 files changed, 1 insertion(+), 27 deletions(-) delete mode 100644 remote-sparkvm.code-workspace diff --git a/.gitignore b/.gitignore index d87a92c1b..a8d93ca09 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,7 @@ src/server/sparkd/orderman/perf_analysis/logs/ src/server/sparkd/orderman/perf_analysis/trace_data/ .fake *.fsx.lock +*.code-workspace src/lib/orderman/perf_analysis/logs/latency_data/ trace_data/ /src/lib/orderman/perf_analysis/last_pretty_log.csv diff --git a/remote-sparkvm.code-workspace b/remote-sparkvm.code-workspace deleted file mode 100644 index 03c875df7..000000000 --- a/remote-sparkvm.code-workspace +++ /dev/null @@ -1,27 +0,0 @@ -{ - "folders": [ - { - "path": "." - }, - { - "uri": "sftp://bts:BT5u53r@sparkvm:22/home/bts/", - "name": "sparkvm" - } - ], - "settings": { - "files.associations": { - "utility": "cpp", - "initializer_list": "cpp", - "list": "cpp", - "type_traits": "cpp", - "vector": "cpp", - "xhash": "cpp", - "xstring": "cpp", - "system_error": "cpp", - "array": "cpp", - "string_view": "cpp", - "xutility": "cpp", - "*.ipp": "cpp" - } - } -} \ No newline at end of file