From 99fd77c1f10a629ed708b8049a61a9fec521ac35 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 25 Oct 2018 13:01:38 -0400 Subject: [PATCH 01/29] Add emumba platform --- src/lib/md/protos/co2.proto | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/md/protos/co2.proto b/src/lib/md/protos/co2.proto index bfb145d5c..c30037170 100644 --- a/src/lib/md/protos/co2.proto +++ b/src/lib/md/protos/co2.proto @@ -10,6 +10,7 @@ enum Platform { PLATFORM_SIM = 1; PLATFORM_MDP3ONIXS = 10; PLATFORM_CELOXICA = 13; + PLATFORM_MDP3EMUMBA = 14; } enum TradingStatus { From a84e5e346094b92cac48f5b44acee667574ddef1 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Mon, 5 Nov 2018 11:03:02 -0600 Subject: [PATCH 02/29] SPARK-1805 Removing a synthetic disables the remove menu for the life of the window --- src/client/spark/core/Utility/KeyDrivenObservable.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/client/spark/core/Utility/KeyDrivenObservable.cs b/src/client/spark/core/Utility/KeyDrivenObservable.cs index 9e3b668ed..e0e3e27b8 100644 --- a/src/client/spark/core/Utility/KeyDrivenObservable.cs +++ b/src/client/spark/core/Utility/KeyDrivenObservable.cs @@ -32,8 +32,11 @@ namespace mbts.utils { // need to lock bc simultaneous callbacks on diff threads could generate 2 subscriptions at once, // only 1 of which would get disposed in the next callback. Disposable.dispose(ref _current_key_sub); - if (!_is_disposed) - _current_key_sub = _selector(key)?.Subscribe(_value_subject); + if (!_is_disposed) { + // IMPORTANT: do NOT subscribe _value_subject directly to the inner sequences. Subscribe with a callback delegate instead (e.g. _value_subject.OnNext). + // Reason: If a sequence 'completes' while _value_subject's subscription is active, it will enter a 'Terminated' state, and forever cease notifying observers of subsequent updates. + _current_key_sub = _selector(key)?.Subscribe(_value_subject.OnNext); + } } } From efa31059d9c874a0d1b44dc7a3e3218b24117788 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Mon, 5 Nov 2018 17:00:07 -0600 Subject: [PATCH 03/29] SPARK-1870 Orderbook - if an order's security arrives after the order, it is always displayed as 'unknown-security' --- .../spark/Orders/OrderRecordTupleSource.cs | 16 ++++++++++------ .../spark_test/Properties/launchSettings.json | 7 +++++++ 2 files changed, 17 insertions(+), 6 deletions(-) create mode 100644 src/client/spark_test/Properties/launchSettings.json diff --git a/src/client/spark/Orders/OrderRecordTupleSource.cs b/src/client/spark/Orders/OrderRecordTupleSource.cs index 2bb3ca29a..9bcda2694 100644 --- a/src/client/spark/Orders/OrderRecordTupleSource.cs +++ b/src/client/spark/Orders/OrderRecordTupleSource.cs @@ -18,8 +18,8 @@ namespace mbts.spark { // data cache readonly Dictionary _state = new Dictionary(); readonly SetMap _security_orders = new SetMap(); + readonly IDictionary _security_md_cache = new Dictionary(0); IReadOnlyDictionary _orders_cache = new Dictionary(0); - IReadOnlyDictionary _security_md_cache = new Dictionary(0); IDisposable _dispose; public OrderRecordTupleSource(IObservable> securities, IObservable> orders) => @@ -92,11 +92,14 @@ namespace mbts.spark { Dictionary added = new Dictionary(), removed = new Dictionary(); Dictionary added_securities = new Dictionary(), removed_securities = new Dictionary(); _security_md_cache.Keys.difference(securities.Keys, out IEnumerable added_keys, out IEnumerable removed_keys); + /// Warning: must NOT be reassigned with ; the latter is always the same object. + /// This would mean that the + /// call above would literally be caluclating the difference between the keys of the same dictionary! foreach (var k in removed_keys) { if (_security_orders.try_lookup(k, out ISet order_keys)) { - removed_securities.Add(k, _security_md_cache[k]); - IEnumerable orders = order_keys.Select(fk => _orders_cache[fk]); SecurityMD old_sec_md = _security_md_cache[k]; + removed_securities.Add(k, old_sec_md); + IEnumerable orders = order_keys.Select(fk => _orders_cache[fk]); foreach (OrderRecordImmutable order in orders) { OrderRecordTuple old_ft = new OrderRecordTuple(order, old_sec_md); removed.Add(old_ft.order.Key, old_ft); @@ -104,12 +107,13 @@ namespace mbts.spark { added.Add(new_ft.order.Key, new_ft); } } + _security_md_cache.Remove(k); } foreach (var k in added_keys) { + SecurityMD new_sec_md = securities[k]; if (_security_orders.try_lookup(k, out ISet order_keys)) { - added_securities.Add(k, securities[k]); + added_securities.Add(k, new_sec_md); IEnumerable orders = order_keys.Select(fk => _orders_cache[fk]); - SecurityMD new_sec_md = securities[k]; foreach (OrderRecordImmutable order in orders) { OrderRecordTuple old_ft = new OrderRecordTuple(order, null); removed.Add(old_ft.order.Key, old_ft); @@ -117,8 +121,8 @@ namespace mbts.spark { added.Add(new_ft.order.Key, new_ft); } } + _security_md_cache.Add(k, new_sec_md); } - _security_md_cache = securities; _apply_changes(added, removed, added_securities, removed_securities); } } diff --git a/src/client/spark_test/Properties/launchSettings.json b/src/client/spark_test/Properties/launchSettings.json new file mode 100644 index 000000000..ad77d29ce --- /dev/null +++ b/src/client/spark_test/Properties/launchSettings.json @@ -0,0 +1,7 @@ +{ + "profiles": { + "spark_test": { + "commandName": "Project" + } + } +} \ No newline at end of file From 306cbaa837b7a9813d4c6f807786b66918197004 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Mon, 5 Nov 2018 17:48:40 -0600 Subject: [PATCH 04/29] SPARK-1870 / SPARK-1871 if an order's security arrives late, the security cell view doesn't automatically refresh --- src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs b/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs index dbcd9cdc7..5a5213efd 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookRowVM.cs @@ -104,7 +104,7 @@ namespace mbts.spark { if (old_order.order_type() != order.order_type()) RaisePropertyChanged(nameof(order_type), nameof(order_metadata)); if (!had_security && _security_md.HasValue) { - RaisePropertyChanged(nameof(security)); + RaisePropertyChanged(nameof(security), nameof(security_text)); // need to raise invalidated to refresh the price cells (the security.ContractSpec determines how they are formatted - Ticks!) invalidate = true; } From 8115f9df744e33332e025782436951109ca7fa84 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Mon, 5 Nov 2018 17:52:23 -0600 Subject: [PATCH 05/29] remove launchSettings.json (added by accident) --- src/client/spark_test/Properties/launchSettings.json | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 src/client/spark_test/Properties/launchSettings.json diff --git a/src/client/spark_test/Properties/launchSettings.json b/src/client/spark_test/Properties/launchSettings.json deleted file mode 100644 index ad77d29ce..000000000 --- a/src/client/spark_test/Properties/launchSettings.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "profiles": { - "spark_test": { - "commandName": "Project" - } - } -} \ No newline at end of file From 1ca5ed4520c123a23c6083bd38c67f2153431708 Mon Sep 17 00:00:00 2001 From: John Colvin Date: Mon, 5 Nov 2018 17:57:43 -0600 Subject: [PATCH 06/29] fix typeo in comment --- src/client/spark/Orders/OrderRecordTupleSource.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/spark/Orders/OrderRecordTupleSource.cs b/src/client/spark/Orders/OrderRecordTupleSource.cs index 9bcda2694..51ffa7f78 100644 --- a/src/client/spark/Orders/OrderRecordTupleSource.cs +++ b/src/client/spark/Orders/OrderRecordTupleSource.cs @@ -94,7 +94,7 @@ namespace mbts.spark { _security_md_cache.Keys.difference(securities.Keys, out IEnumerable added_keys, out IEnumerable removed_keys); /// Warning: must NOT be reassigned with ; the latter is always the same object. /// This would mean that the - /// call above would literally be caluclating the difference between the keys of the same dictionary! + /// call above would literally be calculating the difference between the keys of the same dictionary! foreach (var k in removed_keys) { if (_security_orders.try_lookup(k, out ISet order_keys)) { SecurityMD old_sec_md = _security_md_cache[k]; From 1ffda9c413addec6218e6e758e0ee23fdb2902ae Mon Sep 17 00:00:00 2001 From: Kevin Darby Date: Tue, 6 Nov 2018 10:39:55 -0500 Subject: [PATCH 07/29] clarified algo tests for new functionality --- testing/manual/algos/join.txt | 8 +------- testing/manual/algos/lean.txt | 3 ++- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/testing/manual/algos/join.txt b/testing/manual/algos/join.txt index 4b194f7e6..82799b450 100644 --- a/testing/manual/algos/join.txt +++ b/testing/manual/algos/join.txt @@ -3,7 +3,6 @@ - Make sure you have limits to trade this future - Select Join order from the menu Label 1) Set join thresh to absolute 10 -Label 2) Set initial tick offset to 2 The goal of the join order is to watch the opposite-inside market and join when the market *flips and @@ -12,14 +11,9 @@ has at least threshold size *flip is when the best offer becomes the best bid, or the market trades away from you (the line moves up in the gui) Label 3) Put an order in to way down in the bid side of the book -- Because of the parameter in label 2, the order should instead join two levels below the best bid - Watch the opposite inside market, if it flips make sure the order moves itself to join the new best bid (former offer) -Goto Label 2 and set tick offset to 0 - { - Orders should then go in where you put them - } Goto Label l and test the ratio threshold { - Use a threshold of at least 1.0 @@ -28,7 +22,7 @@ Goto Label 3 and try a sell order { } -Now do this test again with "Enable Hedge" checked and hedge ticks set to 1, hedge type set to limit +Now do this test again but use a Cover of Join (that's a Cover order with a Join Child and a Limit Hedge, HedgeTicks = 1) This should modify the behavior such that the algorithm will send a limit order on the opposite side of the market, one tick away from fill price in a quantity equivalent to the amount of the fill diff --git a/testing/manual/algos/lean.txt b/testing/manual/algos/lean.txt index cffa08e52..2e6a0ccbf 100644 --- a/testing/manual/algos/lean.txt +++ b/testing/manual/algos/lean.txt @@ -6,11 +6,12 @@ Label 1) Set cross thresh to absolute 10 The goal of the lean order is to watch the opposite-inside market size and *cross when that size goes below the threshold (this also implies a cross if that size goes to zero and the market trades away from you) -(if the market trades toward you, we just start watching the new ask) + *cross means take an order working on the bid and 'cross' by buying into the asks. This is also called 'paying up' - Open up order book and fills +- Make sure OnToward behavior is set to No Change (not Cancel) Label 2) Put an order in to Join the best bid - Watch the opposite inside market, if it dips below the threshold (or trades away from you) make sure the order moves itself to cross - Verify that the order filled at the offer price From c7fb636e49fbeff018625800f74087c6758a8100 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 6 Nov 2018 10:52:18 -0500 Subject: [PATCH 08/29] Factor out MD argument parsing --- src/lib/CMakeDeps.cmake | 2 +- src/lib/md/Diag/MDPeriodicDumper.cpp | 26 ++++++ src/lib/md/Diag/MDPeriodicDumper.h | 11 +++ src/lib/md/Diag/MDPriceLantencyTester.cpp | 69 ---------------- src/lib/md/Diag/MDPriceLantencyTester.h | 15 ---- src/server/sparkd/md/run_md.cpp | 99 ++++++++++++++--------- 6 files changed, 99 insertions(+), 123 deletions(-) create mode 100644 src/lib/md/Diag/MDPeriodicDumper.cpp create mode 100644 src/lib/md/Diag/MDPeriodicDumper.h delete mode 100644 src/lib/md/Diag/MDPriceLantencyTester.cpp delete mode 100644 src/lib/md/Diag/MDPriceLantencyTester.h diff --git a/src/lib/CMakeDeps.cmake b/src/lib/CMakeDeps.cmake index 255a29c21..1552418c6 100644 --- a/src/lib/CMakeDeps.cmake +++ b/src/lib/CMakeDeps.cmake @@ -1 +1 @@ -set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/services/UserService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/Executor.conf;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPriceLantencyTester.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp;utils/stats.cpp;utils/thread_manager.cpp) \ No newline at end of file +set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataAuxiliaryService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/services/UserService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/Executor.conf;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cme_reference_data.conf;config/defaults/edge.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/threads.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cme_reference_data.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/mercury_cfg.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/threads.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPeriodicDumper.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;utils/AuthToken.cpp;utils/Dispatcher.cpp;utils/GrowableSPSCQueue.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp;utils/stats.cpp;utils/thread_manager.cpp) \ No newline at end of file diff --git a/src/lib/md/Diag/MDPeriodicDumper.cpp b/src/lib/md/Diag/MDPeriodicDumper.cpp new file mode 100644 index 000000000..222594bab --- /dev/null +++ b/src/lib/md/Diag/MDPeriodicDumper.cpp @@ -0,0 +1,26 @@ +#include "MDPeriodicDumper.h" +#include "lib/md/MDConsumer.h" + +#include + +namespace bts { +namespace md { + +struct Handle { + utils::TimeStamp time; + MDConsumerHandle handle; +}; + +void run_dumper(const std::set& symbols) { + MDConsumer consumer(symbols, nullptr); + + // read all securities into a vector + + + // spin around on the vector detecting changes + // if change read book+ sequence + // If seq % n == 0 + // print book to stdout (async?) +} + +}}//ns \ No newline at end of file diff --git a/src/lib/md/Diag/MDPeriodicDumper.h b/src/lib/md/Diag/MDPeriodicDumper.h new file mode 100644 index 000000000..4c3f95719 --- /dev/null +++ b/src/lib/md/Diag/MDPeriodicDumper.h @@ -0,0 +1,11 @@ +#pragma once +#include "lib/md/UserSymbol.h" +#include + +namespace bts { +namespace md { + +void run_dumper(const std::set&); + + +}}//ns \ No newline at end of file diff --git a/src/lib/md/Diag/MDPriceLantencyTester.cpp b/src/lib/md/Diag/MDPriceLantencyTester.cpp deleted file mode 100644 index c15319110..000000000 --- a/src/lib/md/Diag/MDPriceLantencyTester.cpp +++ /dev/null @@ -1,69 +0,0 @@ -#ifndef NO_CPP0X -#include "MDPriceLantencyTester.h" -#include "../MDConsumer.h" -#include "lib/utils/Dispatcher.h" - -#include -#include -#include - -using namespace std; -namespace bts { -namespace md { - -struct HandleData { - MDConsumerHandle handle; - utils::TimeStamp time; - utils::TimeStamp last_price_change; - float bid, ask; -}; - -void run_md_price_latency_test(UserSymbol symbol) { - set symbols; - symbols.insert(symbol); - std::shared_ptr null; - MDConsumer consumer(symbols, null); - - vector handles; - - consumer.walk_security_list([&](const Security&, MDConsumerHandle handle) { - HandleData entry = { handle, TimeStamp(), TimeStamp(), 0, 0 }; - if (!entry.handle.is_initialized()) { - cout << "Please start the md publisher before running this command." << endl; - return; - } - handles.push_back(entry); - }); - - cout << "key, sequence, handover, delta, bid price, ask price" << endl; - while (true) { - for(auto& entry : handles) { - PriceSize bid, ask; - if (entry.handle.get_direct_book(entry.time, 1, &bid, &ask) == HandleReadResult::SUCCESS) { - auto bid_price = bid.price(); - auto ask_price = ask.price(); - if (bid_price != entry.bid || ask_price != entry.ask) { - auto now = TimeStamp::now(); - auto handover = (now - entry.time); - auto delta = (now - entry.last_price_change); - entry.last_price_change = entry.time; - entry.bid = bid_price; - entry.ask = ask_price; - utils::Dispatcher::background().post([=]() { - cout << - entry.handle.key() << ',' - << entry.handle.sequence() << ',' - << handover.microseconds() << ',' - << delta.microseconds() << ',' - << bid_price << ',' << ask_price - << endl; - }); - } - } - } - } -} - - -}}//namespace -#endif diff --git a/src/lib/md/Diag/MDPriceLantencyTester.h b/src/lib/md/Diag/MDPriceLantencyTester.h deleted file mode 100644 index 6f169e4d3..000000000 --- a/src/lib/md/Diag/MDPriceLantencyTester.h +++ /dev/null @@ -1,15 +0,0 @@ -#pragma once - -#include "lib/utils/bts_export.h" -#include "../common.h" -#include "../UserSymbol.h" -#include - -namespace bts { -namespace md { - -BTS_EXPORT void run_md_price_latency_test(UserSymbol); - - -}}//namespace - diff --git a/src/server/sparkd/md/run_md.cpp b/src/server/sparkd/md/run_md.cpp index e2a06f7c7..c2136992d 100644 --- a/src/server/sparkd/md/run_md.cpp +++ b/src/server/sparkd/md/run_md.cpp @@ -8,6 +8,7 @@ #include "lib/md/get_symbols.h" #include "lib/md/UserSymbol.h" #include "lib/md/Diag/ShmDumper.h" +#include "lib/md/Diag/MDPeriodicDumper.h" #include "platforms/mdp3_onixs/MDP3Main.h" #include "lib/md/SecurityDB.h" #include "server/sparkd/exchange_sim/ExchangeSim.h" @@ -106,56 +107,71 @@ void run_record(const std::string& arg){ record_everything(p.parent_path().string(), p.filename().string()); } + + string& uppercase(string& s) { transform(s.begin(), s.end(), s.begin(), ::toupper); return s; } template -T parse_enum(const string& value, const string& prefix, T2 parse, T default_value) { +bool parse_enum(const string& value, const string& prefix, T2 parse, T default_value, T* presult) { string temp = value; if (temp.find(prefix) == string::npos) { temp = prefix + "_" + temp; } - T result; - if (parse(temp, &result)) { - return result; - } - return default_value; -} -void run_dump(const vector& args) { - // either - // name, [hot|cold] - // symbol type exchange [venue|hot] [hot|cold] - - string name_or_symbol = default_arg(args, 0, ""); - string hotcold_or_type = default_arg(args, 1, "cold"); - string venue = default_arg(args, 2, "glbx"); - string platform = default_arg(args, 3, ""); - bool hot = std::find(args.begin(), args.end(), "hot") != args.end(); - if (name_or_symbol.empty()) return; - if (name_or_symbol.find("PLATFORM") == string::npos /*heuristic*/) { - Platform p = parse_enum(uppercase(platform), "PLATFORM", co2::Platform_Parse, co2::PLATFORM_UNSET); - if (p == co2::PLATFORM_UNSET) { - api::Venue v = parse_enum(uppercase(venue), "VENUE", api::Venue_Parse, api::VENUE_UNSET); - p = platform_for_exchange(v); - } - stringstream ss; - ss << uppercase(name_or_symbol) - << "_SECURITYTYPE_" << uppercase(hotcold_or_type) - << "_VENUE_" << uppercase(venue) << "_" - << co2::Platform_Name(p); - name_or_symbol = ss.str(); - } - if (name_or_symbol.find("bts_edge_") == string::npos) { - name_or_symbol = "bts_edge_" + config::get().shared_memory_namespace() + "_" + name_or_symbol; - } - ShmDumper dumper(name_or_symbol); - if (!hot || hotcold_or_type == "cold" || hotcold_or_type == "c") { - dumper.dump_cold(); + if(parse(temp, presult)) { + return true; } else { + *presult = default_value; + return false; + } +} + +enum class HotCold { Tepid, Hot, Cold }; +std::tuple parse_md_args(const vector& args) { + using namespace api; + using namespace co2; + auto venue = VENUE_UNSET; // Default to glbx. If you change this change below (used) also + auto platform = PLATFORM_UNSET; + auto type = SECURITYTYPE_UNSET; + auto temperature = HotCold::Tepid; + std::string symbol; + for (auto str : args) { + uppercase(str); + bool used = + (venue == VENUE_UNSET && parse_enum(str, "VENUE", Venue_Parse, venue, &venue)) || + (platform == PLATFORM_UNSET && parse_enum(str, "PLATFORM", Platform_Parse, platform, &platform)) || + parse_enum(str, "SECURITYTYPE", SecurityType_Parse, type, &type); + if (str == "HOT") { temperature = HotCold::Hot; used = true; } + else if (str == "COLD") { temperature = HotCold::Cold; used = true; } + if (!used) { + symbol = str; + } + } + if (venue == VENUE_UNSET) venue = VENUE_GLBX; + if (platform == PLATFORM_UNSET) platform = platform_for_exchange(venue); + return { venue, platform, type, symbol, temperature }; +} + +void run_dump(const vector& args) { + using std::get; + + auto p = parse_md_args(args); + stringstream ss; + ss << "bts_edge_" << config::get().shared_memory_namespace() << "_" + << uppercase(get(p)) << "_" + << SecurityType_Name(get(p)) << "_" + << Venue_Name(get(p)) << "_" + << Platform_Name(get(p)); + + ShmDumper dumper(ss.str()); + if (get(p) == HotCold::Hot) { dumper.dump_hot(); } + else { + dumper.dump_cold(); + } } @@ -178,7 +194,7 @@ void run_rectool(const vector args) { } } -void dump_db(const vector args) { +void dump_db(const vector& args) { set keys; bool all = false; for (const auto& arg : args) { @@ -208,6 +224,12 @@ void dump_db(const vector args) { }); } +void race(const vector& args) { + // args: % + auto _ug = parse_md_args(args); + // platform +} + string help() { @@ -229,6 +251,7 @@ void md_tool(std::vector const &args) { else if (cmd == "relay") relay_everything(); else if (cmd == "test_relay") relay_everything(); else if (cmd == "dumpsecurity" || cmd == "dumpsecurities") dump_db(vector(args.begin()+1, args.end())); + else if (cmd == "race") race(args); else printhelp(); } From 3630bf5b9a3f0fd17a0e33506b4d2a0e342081bb Mon Sep 17 00:00:00 2001 From: Michael Harding Date: Tue, 6 Nov 2018 14:51:29 -0500 Subject: [PATCH 09/29] Test updates --- testing/manual/positions_and_fills/PAndL.txt | 47 +++++++++----------- testing/manual/positions_and_fills/SOD.txt | 21 ++++----- testing/manual/settings/GridSettings.txt | 2 +- 3 files changed, 32 insertions(+), 38 deletions(-) diff --git a/testing/manual/positions_and_fills/PAndL.txt b/testing/manual/positions_and_fills/PAndL.txt index 295fb1c9c..a863aed2f 100644 --- a/testing/manual/positions_and_fills/PAndL.txt +++ b/testing/manual/positions_and_fills/PAndL.txt @@ -16,13 +16,13 @@ and you may also want to clear your orders for a clean slate: Throughout the test we'll refer to accounts ACC A and ACC A1. These specific accounts aren't necessary, but rather that we have fills in multiple distinct accounts. -Bring up the pnl form (empty) . +Bring up the pnl form (empty). Bring up a SIM:SI ladder form, pick your favorite expiration. It should be completely empty since we're controlling the md, and the ACC A account should be selected in the account drop down list. Bring up an order ticket for your corresponding ladder place a buy order with price 14.000 and qty 10 your ladder should populate now place a sell order with price 14.010 at qty 10 - this will give you a valid Midpoint for you PnL Price in the PnL + this will give you a valid Midpoint for your PnL Price in the PnL Sell a 1 lot at your bid (14.000) The Start Net Pos should be 0 Start Pnl should be 0 @@ -33,9 +33,9 @@ Sell a 1 lot at your bid (14.000) PnL Price should be at the midpoint of your bid and ask (14.000 + 14.010)/2 = 14.005. Improve the bid Under ACC A1 account (this is important for the next step), enter a 10 lot bid at 14.005 - Observe the PnL Price reflect the current Midpoint (14.0075) - Switch back to ACC A -Sell a 1 lot at the current bid + Observe the PnL Price reflect the current Midpoint (14.0075 displayed as 14.010) +Switch back to ACC A + Sell a 1 lot at the current bid You should now have rows for both the ACC A and ACC A1 account ACC A Day Long 1 @@ -44,7 +44,7 @@ Sell a 1 lot at the current bid (14.005 [fill price] - 14.0075 [driver price]) * 1 [qty] * 5000 [cash per point] = -12.50 Total Net Pos -1 Total PnL = -12.5 - PnL Price = 14.0075 + PnL Price = 14.0075 (displayed as 14.010) ACC A1 Day Long 1 Day Short 0 @@ -52,7 +52,7 @@ Sell a 1 lot at the current bid (14.0075 - 14.005) * 1 * 5000 Move the market Move your working sell order up to 14.020 - The PnL Price should update to 14.0125 ((14.020 + 14.005) /2) + The PnL Price should update to 14.0125 ((14.020 + 14.005) /2) (displayed as 14.015) The Day and Total PnLs should update to according to the same calculation as earlier ACC A: -37.5 ACC A1: 37.5 @@ -64,60 +64,57 @@ Check the Totals Footer Force the SOD process to aggregate the fills Configure your sod time to be after your last fill and before now. Note that SODTime is currently expressed in UTC, so you'll have to make the appropriate conversion. - + Here is a sample config: - + SODTime{ value { sod_hh: 15 sod_mm: 00 } } - + Restart sparkd and the SOD process should consolidate your day fills so... - - ACC A: + + ACC A: Start Net Pos -1 - Start PnL -12.5 - (14.005 - 14.0125) * 2 * 5000 = -75 - (14.0125 - 14.000) * 1 * 5000 = 62.5 - -75 + 62.5 = -12.5 + Start PnL blank Day Long 0 Day Short 0 Day PnL 0 Total Net Pos -1 Total PnL -12.5 - PnL Price 14.0125 - ACC A1: + PnL Price 14.0125 (displayed as 14.015) + ACC A1: Start Net Pos 1 - Start PnL 37.5 + Start PnL blank Day Long 0 Day Short 0 Day PnL 0 Total Net Pos 1 Total PnL 37.5 (14.0125 - 14.005) * 1 * 5000 - PnL Price 14.0125 + PnL Price 14.0125 (displayed as 14.015) Sell a 1 lot as the current bid in ACC A ACC A: Start Net Pos -1 (no change) - Start PnL -12.5 (no change) + Start PnL -37.5 Day Long 0 Day Short 1 Day PnL -37.5 Total Net Pos -2 Total PnL -50 - PnL Price 14.0125 - ACC A1: + PnL Price 14.0125 (displayed as 14.015) + ACC A1: Start Net Pos 1 (no change) - Start PnL 37.5 (no change) + Start PnL 37.5 Day Long 1 Day Short 0 Day PnL 37.5 Total Net Pos 2 Total PnL 75 - PnL Price 14.0125 + PnL Price 14.0125 (displayed as 14.015) Test sorting and filtering diff --git a/testing/manual/positions_and_fills/SOD.txt b/testing/manual/positions_and_fills/SOD.txt index 95406c323..6d8eee837 100644 --- a/testing/manual/positions_and_fills/SOD.txt +++ b/testing/manual/positions_and_fills/SOD.txt @@ -1,23 +1,22 @@ clear your fills and orders database: rm ~/sparkdata/data/databases/fills/* rm ~/sparkdata/data/databases/orders/* -and bounce your server. +and bounce your server. -0. We will force fills across for 3 securities, across 2 accounts. Lets pick 2 futures and -one exchange traded spread (make sure there are no overlaps). Choose these securities -such that there is market data (sim or cert) that you can get fills against (maybe bring up -the market view and select securities from there). Bring up the 6 ladders. Make sure the position -button the ladder displays - +0. We will force fills across for 3 securities, across 2 accounts. + a. Pick 2 futures and one exchange traded spread (make sure there are no overlaps). + b. Choose these securities such that there is market data (sim or cert) that you can get fills against (maybe bring up the market view and select securities from there). + c. Bring up the 6 ladders. Make sure the position button the ladder displays 1. Force buy and sell fills across the 6 ladders. Keep track of the long and short positions and verify that that the position button in the ladder is reflecting the net position carefully. Open up the PNL and make sure the Day Long and Day Short positions match what you expect. -The Start Net Pos should be 0. (The Total Net Pos in the PNL should match the pos button +The Start Net Pos should be 0. (The Total Net Pos in the PNL should match the pos button in the ladder) -2. Now comes the SOD part. The SOD is what demarkates a day. Find the most recent time on your fills, select 1 minute past that and convert it to UTC and override the SOD time in your spark.conf +2. Now comes the SOD part. The SOD is what demarcates a day. Find the most recent time on your fills, select 1 minute past that and convert it to UTC and override the SOD time in your spark.conf Here's an example of what the key looks like: @@ -30,10 +29,8 @@ SODTime{ 3. Now restart the server. Verify that all rows in the fills have a Fill Kind 'Sod' and the time matches the SODTime (converted to your local time zone) -4. Verify that the ladder is displaying the correct position (it should be the same as before -the server start) +4. Verify that the ladder is displaying the correct position (it should be the same as before the server start) 5. Verify that the PNL is displaying the correct Start Net Pos and the Day part is 0. -6. Now force a few more fills. Verify that the ladder position box and the PNL is -correctly counting them. \ No newline at end of file +6. Now force a few more fills. Verify that the ladder position box and the PNL is correctly counting them. \ No newline at end of file diff --git a/testing/manual/settings/GridSettings.txt b/testing/manual/settings/GridSettings.txt index 43e096259..db8cb0072 100644 --- a/testing/manual/settings/GridSettings.txt +++ b/testing/manual/settings/GridSettings.txt @@ -1,7 +1,7 @@ ----Grid Settings Test (11 minutes) --------- 1) Open the Orderbook -2) Click the Window Settings button at the bottom of the Spark menu to open the settings dialog +2) Click the Settings button at the bottom of the Spark menu to open the settings dialog ----Columns---------------------------------- From a83f97e1ef18b20a18c1ddbbed2ce0cc8c75434a Mon Sep 17 00:00:00 2001 From: John Colvin Date: Tue, 6 Nov 2018 13:54:25 -0600 Subject: [PATCH 10/29] SPARK-1560 Synthetic Management browser maintain order --- .../spark/Modules/Spreader/VM/SyntheticEditorVM.cs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/client/spark/Modules/Spreader/VM/SyntheticEditorVM.cs b/src/client/spark/Modules/Spreader/VM/SyntheticEditorVM.cs index 2b162676a..7f294ab6c 100644 --- a/src/client/spark/Modules/Spreader/VM/SyntheticEditorVM.cs +++ b/src/client/spark/Modules/Spreader/VM/SyntheticEditorVM.cs @@ -2,7 +2,6 @@ using bts.shell; using bts.spark; using bts.utils; -using mbts.spark.utilities; using mbts.ui; using mbts.ui.shell; using mbts.ui.spark; @@ -110,7 +109,7 @@ namespace mbts.spark { _validate_leg_count(); }; - async Task commit_async(){ + async Task commit_async() { RequestUpdate?.Invoke(); bool valid = true; if (string.IsNullOrWhiteSpace(name)) { @@ -125,8 +124,7 @@ namespace mbts.spark { valid = false; } if (valid) { - _synthetic.Synthetic.Legs.Clear(); - _synthetic.Synthetic.Legs.AddRange(leg_tuples.Select(l => l.leg).OrderBy(l => l.Security, SecurityHelper.Comparer.Instance)); + _synthetic.Synthetic.Legs.SetItems(leg_tuples.Select(l => l.leg)); _synthetic.Symbol = _deduce_symbol(_synthetic); double tick_frac = _tick_factors.tick_unit / _tick_factors.tick_base; _synthetic.ContractSpec.LowTick = tick_frac; @@ -296,7 +294,6 @@ namespace mbts.spark { static IEnumerable _get_legs(Security synthetic, ExecutionStrategyImmutable strategy) => synthetic.Synthetic.Legs - .OrderBy(l => l.Security, SecurityHelper.Comparer.Instance) .Select(l => new LegSpreaderParamsTuple(l, strategy.LegParamsMap[l.Security.Id].Clone())); void _close_new_leg_popup() { @@ -339,4 +336,4 @@ namespace mbts.spark { else clear_error(nameof(leg_count)); } } -} +} \ No newline at end of file From 8a9ddfded70bf8ef73e59dab5d1ea93fd81a0067 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 6 Nov 2018 13:59:21 -0600 Subject: [PATCH 11/29] SPARK-1869 - x of y everywhere --- .../Market/MarketView/MarketViewView.xaml | 14 ++++++-------- .../Market/TimeAndSales/TimeAndSalesView.xaml | 8 +------- .../Spreader/SyntheticManagementView.xaml | 14 ++++++-------- .../spark/Modules/Trading/Fills/FillsView.xaml | 14 ++++++-------- .../Trading/OrderBook/OrderBookView.xaml | 18 ++++++------------ 5 files changed, 25 insertions(+), 43 deletions(-) diff --git a/src/client/spark/Modules/Market/MarketView/MarketViewView.xaml b/src/client/spark/Modules/Market/MarketView/MarketViewView.xaml index 7031a7151..3ded0304e 100644 --- a/src/client/spark/Modules/Market/MarketView/MarketViewView.xaml +++ b/src/client/spark/Modules/Market/MarketView/MarketViewView.xaml @@ -48,14 +48,12 @@ - - - - - - - - + + + + + + diff --git a/src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesView.xaml b/src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesView.xaml index f3552d905..aa2fb3998 100644 --- a/src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesView.xaml +++ b/src/client/spark/Modules/Market/TimeAndSales/TimeAndSalesView.xaml @@ -49,13 +49,7 @@ - - - - - - - + diff --git a/src/client/spark/Modules/Spreader/SyntheticManagementView.xaml b/src/client/spark/Modules/Spreader/SyntheticManagementView.xaml index 0aa2c43ba..78e6570b9 100644 --- a/src/client/spark/Modules/Spreader/SyntheticManagementView.xaml +++ b/src/client/spark/Modules/Spreader/SyntheticManagementView.xaml @@ -46,14 +46,12 @@ - - - - - - - - + + + + + + diff --git a/src/client/spark/Modules/Trading/Fills/FillsView.xaml b/src/client/spark/Modules/Trading/Fills/FillsView.xaml index 983a4deba..690bdffed 100644 --- a/src/client/spark/Modules/Trading/Fills/FillsView.xaml +++ b/src/client/spark/Modules/Trading/Fills/FillsView.xaml @@ -49,14 +49,12 @@ - - - - - - - - + + + + + + diff --git a/src/client/spark/Modules/Trading/OrderBook/OrderBookView.xaml b/src/client/spark/Modules/Trading/OrderBook/OrderBookView.xaml index 48ebb5f0e..b5698bbca 100644 --- a/src/client/spark/Modules/Trading/OrderBook/OrderBookView.xaml +++ b/src/client/spark/Modules/Trading/OrderBook/OrderBookView.xaml @@ -155,18 +155,12 @@ - - - - - - - - - - - - + + + + + + From 96da8e32762271d90d1de17c5dce9983eb167f36 Mon Sep 17 00:00:00 2001 From: Jake Norbut Date: Tue, 6 Nov 2018 14:46:40 -0600 Subject: [PATCH 12/29] SOD tm_isdst bug regarding our defaults; new defaults --- src/lib/config/protos/orders_fills_cleanup.proto | 2 +- src/lib/orderman/actors/SODPosition.cpp | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/config/protos/orders_fills_cleanup.proto b/src/lib/config/protos/orders_fills_cleanup.proto index 9c0f5815a..7c15118c7 100644 --- a/src/lib/config/protos/orders_fills_cleanup.proto +++ b/src/lib/config/protos/orders_fills_cleanup.proto @@ -7,6 +7,6 @@ message OrdersSoftCleanupCutoff{ message SODTime{ // utc time - optional uint32 sod_hh = 20 [default=15]; + optional uint32 sod_hh = 20 [default=16]; optional uint32 sod_mm = 30 [default=30]; } \ No newline at end of file diff --git a/src/lib/orderman/actors/SODPosition.cpp b/src/lib/orderman/actors/SODPosition.cpp index 4a13d4837..e5a09e0ca 100644 --- a/src/lib/orderman/actors/SODPosition.cpp +++ b/src/lib/orderman/actors/SODPosition.cpp @@ -31,6 +31,7 @@ timeval SODPosition::_to_timeval(uint16_t hh, uint16_t mm, uint16_t ss){ ptm->tm_hour=hh; ptm->tm_min=mm; ptm->tm_sec=ss; + ptm->tm_isdst=-1; secs=mktime(ptm); timeval rv; rv.tv_sec=secs; From 6aeba4f599df4cb11d8dadb9d12b3755219e22dd Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 7 Nov 2018 15:59:07 -0500 Subject: [PATCH 13/29] Remove unused MarketData config field --- src/lib/config/protos/MarketData.proto | 1 - 1 file changed, 1 deletion(-) diff --git a/src/lib/config/protos/MarketData.proto b/src/lib/config/protos/MarketData.proto index d5ed6cc8b..5ad80beea 100755 --- a/src/lib/config/protos/MarketData.proto +++ b/src/lib/config/protos/MarketData.proto @@ -76,7 +76,6 @@ message CMEOptions // Defines amount of time Feed Engine keeps cycling before going to sleep when no useful activity can be done optional uint32 spin_time = 4 [default = 1]; optional uint32 channels_per_thread = 5 [default = 3]; - optional string thread_group = 10 [default=""]; // if present this will be used to retrieve affinity settings from Threads config key optional uint32 udp_socket_buffer_size = 11 [default = 33554432]; //32mb } message CMEChannel { From 70a2feb92ad481c8dcef8dbd7dfd1f87aaabdb94 Mon Sep 17 00:00:00 2001 From: Kevin Darby Date: Thu, 8 Nov 2018 11:06:41 -0500 Subject: [PATCH 14/29] remove out of date test --- testing/manual/algos/tick.txt | 32 -------------------------------- 1 file changed, 32 deletions(-) delete mode 100644 testing/manual/algos/tick.txt diff --git a/testing/manual/algos/tick.txt b/testing/manual/algos/tick.txt deleted file mode 100644 index 0aafd36da..000000000 --- a/testing/manual/algos/tick.txt +++ /dev/null @@ -1,32 +0,0 @@ - -- Open up a ladder for a liquid future -- Make sure you have limits to trade this future -- Select Tick order from the menu -Label 1) Set cross thresh to absolute 10 -Label 2) Set initial tick offset to 1 - - -The goal of the tick order is to watch the opposite-inside market size and *cross when that size goes below the threshold -(this also implies a cross if that size goes to zero and the market trades away from you) -(if the market trades toward you, the algo will 'detach' or become a regular limit order by default) - -*cross means take an order working on the bid and 'cross' by buying into the asks. This is also called 'paying up' - -- Open up order book and fills -Label 3) Put an order in to way down in the bid side of the book -- Because of the parameter in label 2, the order should instead join at the best bid -- Watch the opposite inside market, if it dips below the threshold or the market trades away make sure the order moves itself to cross -- Verify that the order filled at the offer price -- If the opposite inside market does not dip below threshhold, you should expect to be eventually filled at the bid price -- If the opposite inside market trades toward you, you should see the algo detach and the limit be left working -- Verify that these fills come in at the proper price, crosses at the offer, working fills at the bid - - -Goto Label 2 and join the ask instead of the bid - { - } -Goto Label l and test the ratio threshold - { - - Use a threshold of at least 1.0 - - Test with a larger order (say 100) to trigger crosses more quickly and across different price levels - } From c8b3973b8ff1e18b398b0d8e3bb89d2db644b3f5 Mon Sep 17 00:00:00 2001 From: Michael Harding Date: Tue, 4 Sep 2018 13:15:06 -0400 Subject: [PATCH 15/29] Remove Beta from SparkAdmin From 088377582de5fbdc852cab30db214bbfeae59116 Mon Sep 17 00:00:00 2001 From: Michael Harding Date: Tue, 18 Sep 2018 15:06:18 -0400 Subject: [PATCH 16/29] SPARK-1678: Small code tweaks based on Eric's suggestions --- src/web/ui/src/Accounts/State.fs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/web/ui/src/Accounts/State.fs b/src/web/ui/src/Accounts/State.fs index e1d11ae06..724f58af9 100644 --- a/src/web/ui/src/Accounts/State.fs +++ b/src/web/ui/src/Accounts/State.fs @@ -67,7 +67,6 @@ let applySIMUpdate (data:SIMAccountData) = function | CTI v -> { data with cti = v } | Origin v -> { data with origin = v } | ClearingAccount v -> { data with clearingAccount = v } - | _ -> data let saveAccount (admin:IAdminService) (account:BTSAccount) = promise { let! result = admin.update_account { completeData = account } @@ -174,11 +173,11 @@ let update msg (model:Model) : Model * Cmd = | CancelOrders orders -> model, Cmd.ofExistingPromise (cancelOrders model.admin model.scratch.name orders) id (box>>ErrorOccured) | SelectOrder (chckd, orders) -> - let so = match chckd, orders.Length > 1 with - | true, false -> (orders.[0]::model.SelectedOrders) - | false, false -> List.filter (fun elm -> (orderIsEqual elm orders.[0]) |> not) model.SelectedOrders - | true, true -> orders - | false, true -> [] + let so = match chckd, orders with + | true, h::m::tail -> orders + | true, h::_ -> h::model.SelectedOrders + | false, h::_ -> List.filter (fun elm -> (orderIsEqual elm h) |> not) model.SelectedOrders + | false, _ -> [] {model with SelectedOrders = so}, Cmd.Empty | SelectAccount account -> { (clearSelection model) with previousScratch = model.scratch; scratch = account }, Cmd.Empty From 79ec41556e0eff0d60d628677e7323360d72afbe Mon Sep 17 00:00:00 2001 From: Michael Harding Date: Thu, 20 Sep 2018 13:59:30 -0400 Subject: [PATCH 17/29] SPARK-1678: Refresh orders after one second (cherry picked) --- src/web/ui/src/Accounts/State.fs | 10 +++++++++- src/web/ui/src/Accounts/Types.fs | 3 ++- src/web/ui/src/Accounts/View.fs | 11 +++++++++-- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/web/ui/src/Accounts/State.fs b/src/web/ui/src/Accounts/State.fs index 724f58af9..273f1b82c 100644 --- a/src/web/ui/src/Accounts/State.fs +++ b/src/web/ui/src/Accounts/State.fs @@ -7,6 +7,7 @@ open Fable.PowerPack.PromiseImpl open Fable.Import open bts.co2 open Fable +open Fable.PowerPack let isNewAccount (account:BTSAccount) = account.id = 0u && account.isPresent @@ -171,7 +172,14 @@ let update msg (model:Model) : Model * Cmd = | UpdateOrdersData summary -> {model with ordersData = List.ofArray summary.units}, Cmd.Empty | CancelOrders orders -> - model, Cmd.ofExistingPromise (cancelOrders model.admin model.scratch.name orders) id (box>>ErrorOccured) + { model with ConfirmModal = ("", Msg.SelectTab model.tab); SelectedOrders = [] }, + Cmd.batch [ + Cmd.ofExistingPromise (cancelOrders model.admin model.scratch.name orders) id (box>>ErrorOccured) + Cmd.ofExistingPromise (promise { + do! Promise.sleep 1000 + return GetOrdersData + }) id (fun _ -> GetOrdersData) + ] | SelectOrder (chckd, orders) -> let so = match chckd, orders with | true, h::m::tail -> orders diff --git a/src/web/ui/src/Accounts/Types.fs b/src/web/ui/src/Accounts/Types.fs index 402f009e8..e5df9d082 100644 --- a/src/web/ui/src/Accounts/Types.fs +++ b/src/web/ui/src/Accounts/Types.fs @@ -43,11 +43,12 @@ type SIMExchangeField = type Msg = | UpdateAdminData of AdminData + | NoAdminData of string + // Orders | GetOrdersData | UpdateOrdersData of OrdersSummary | SelectOrder of bool * OrdersSummaryUnit list | CancelOrders of OrdersSummaryUnit list - | NoAdminData of string | SelectAccount of BTSAccount | DeleteAccount of BTSAccount | AddAccount diff --git a/src/web/ui/src/Accounts/View.fs b/src/web/ui/src/Accounts/View.fs index ed74b8a34..a8d05350e 100644 --- a/src/web/ui/src/Accounts/View.fs +++ b/src/web/ui/src/Accounts/View.fs @@ -78,7 +78,6 @@ let generalView model dispatch = ] let GetSymbolArray (model:Model) = - Array.map (fun a -> a.symbol ) model.mdRelayProductData |> Array.distinct @@ -125,7 +124,15 @@ let ordersView (model:Model) dispatch = } ] - let cancelButton = Button.button "Cancel Selected Orders" "is-danger" "times" Button.Enabled (fun _ -> model.SelectedOrders |> CancelOrders |> dispatch) + let enableCancel = + match model.SelectedOrders.Length with + | 0 -> Button.Disabled + | _ -> Button.Enabled + + let cancelMessage = model.SelectedOrders |> CancelOrders + let confirmText = sprintf "Are you sure you want to cancel all orders in %s for users: %s?" model.scratch.name (String.concat ", " (List.map (fun (elm:OrdersSummaryUnit) -> elm.user) model.SelectedOrders)) + let cancelButton = Button.button "Cancel Selected Orders" "is-danger" "times" enableCancel (fun _ -> OpenConfirmModal (confirmText, cancelMessage) |> dispatch) + let reloadFun = fun _ -> dispatch (SelectTab model.tab) div [][ TableControl.createDataTable (Some headerRow) rws cancelButton From 19ae05cb7d98be9e07266f498eb0a33a1e20c55c Mon Sep 17 00:00:00 2001 From: Michael Harding Date: Thu, 20 Sep 2018 16:28:28 -0400 Subject: [PATCH 18/29] SPARK-1678: loading spinner (cherry picked) --- src/web/ui/src/Accounts/State.fs | 7 +++++-- src/web/ui/src/Accounts/Types.fs | 1 + src/web/ui/src/Accounts/View.fs | 11 ++++++++--- src/web/ui/src/Controls/Button.fs | 6 ++++++ 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/web/ui/src/Accounts/State.fs b/src/web/ui/src/Accounts/State.fs index 273f1b82c..ccd356ae6 100644 --- a/src/web/ui/src/Accounts/State.fs +++ b/src/web/ui/src/Accounts/State.fs @@ -33,6 +33,7 @@ let init (context:ServiceContext.ServiceContext) = ConfirmModal = "", (Msg.SelectTab General) SelectedClipLimits = [] SelectedPositionLimits = [] + ShowLoading = false }, Access.getAdminData admin UpdateAdminData NoAdminData let existingSelection (scratch:BTSAccount) (previousScratch:BTSAccount) (data:AdminData) = @@ -125,6 +126,8 @@ let clearSelection model = { model with SelectedClipLimits = [] SelectedPositionLimits = [] + SelectedOrders = [] + ShowLoading = false } let orderIsEqual (order1:OrdersSummaryUnit) (order2:OrdersSummaryUnit) = @@ -168,11 +171,11 @@ let update msg (model:Model) : Model * Cmd = Global.alert Global.Info msg model, Cmd.Empty | GetOrdersData -> - model, (Access.getAccountOrdersData model.admin UpdateOrdersData (box>>ErrorOccured)) + { model with ShowLoading = false}, (Access.getAccountOrdersData model.admin UpdateOrdersData (box>>ErrorOccured)) | UpdateOrdersData summary -> {model with ordersData = List.ofArray summary.units}, Cmd.Empty | CancelOrders orders -> - { model with ConfirmModal = ("", Msg.SelectTab model.tab); SelectedOrders = [] }, + { model with ConfirmModal = ("", Msg.SelectTab model.tab); SelectedOrders = []; ShowLoading = true }, Cmd.batch [ Cmd.ofExistingPromise (cancelOrders model.admin model.scratch.name orders) id (box>>ErrorOccured) Cmd.ofExistingPromise (promise { diff --git a/src/web/ui/src/Accounts/Types.fs b/src/web/ui/src/Accounts/Types.fs index e5df9d082..ffbc6ebe0 100644 --- a/src/web/ui/src/Accounts/Types.fs +++ b/src/web/ui/src/Accounts/Types.fs @@ -103,4 +103,5 @@ type Model = { SelectedPositionLimits: AccountProductPositionLimitUnit list OptionsMenuOpen:string ConfirmModal: string * Msg + ShowLoading: bool } \ No newline at end of file diff --git a/src/web/ui/src/Accounts/View.fs b/src/web/ui/src/Accounts/View.fs index a8d05350e..3d22692b1 100644 --- a/src/web/ui/src/Accounts/View.fs +++ b/src/web/ui/src/Accounts/View.fs @@ -127,15 +127,20 @@ let ordersView (model:Model) dispatch = let enableCancel = match model.SelectedOrders.Length with | 0 -> Button.Disabled - | _ -> Button.Enabled + | _ -> Button.boolToDisabledConverter model.ShowLoading + + let addLoadingString s b = + match b with + | true -> sprintf "is-loading %s" s + | _ -> s let cancelMessage = model.SelectedOrders |> CancelOrders let confirmText = sprintf "Are you sure you want to cancel all orders in %s for users: %s?" model.scratch.name (String.concat ", " (List.map (fun (elm:OrdersSummaryUnit) -> elm.user) model.SelectedOrders)) - let cancelButton = Button.button "Cancel Selected Orders" "is-danger" "times" enableCancel (fun _ -> OpenConfirmModal (confirmText, cancelMessage) |> dispatch) + let cancelButton = Button.button "Cancel Selected Orders" (addLoadingString "is-danger" model.ShowLoading) "times" enableCancel (fun _ -> OpenConfirmModal (confirmText, cancelMessage) |> dispatch) let reloadFun = fun _ -> dispatch (SelectTab model.tab) div [][ - TableControl.createDataTable (Some headerRow) rws cancelButton + TableControl.createDataTable (Some headerRow) rws (span [][ (Button.button "Refresh" "" "exchange" (Button.boolToDisabledConverter model.ShowLoading) reloadFun); nbsp.element; cancelButton ]) ] let limitFilter (positionLimit:AccountSecurityPositionLimitUnit option) (clipLimit:AccountSecurityClipLimitUnit option) allValueDisplay (array:ExpirationNameList []) = diff --git a/src/web/ui/src/Controls/Button.fs b/src/web/ui/src/Controls/Button.fs index 6275a64d2..7caf22245 100644 --- a/src/web/ui/src/Controls/Button.fs +++ b/src/web/ui/src/Controls/Button.fs @@ -12,6 +12,12 @@ let isDisabled = function | Disabled -> true | Enabled -> false +let boolToDisabledConverter d = + match d with + | true -> Disabled + | false -> Enabled + + let addButton disabled click = a [ ClassName "button fabutton " HTMLAttr.Disabled (isDisabled disabled) From ca81be38653454b34ea49c1b132de64d26beac25 Mon Sep 17 00:00:00 2001 From: Michael Harding Date: Thu, 8 Nov 2018 13:17:08 -0500 Subject: [PATCH 19/29] Add manual test for canceling orders in the Spark Admin --- testing/manual/admin/order_cancel.txt | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 testing/manual/admin/order_cancel.txt diff --git a/testing/manual/admin/order_cancel.txt b/testing/manual/admin/order_cancel.txt new file mode 100644 index 000000000..3a820bc04 --- /dev/null +++ b/testing/manual/admin/order_cancel.txt @@ -0,0 +1,18 @@ +This test is to ensure that orders for a user and account can be canceled through the Spark Admin + +This test refers to users bts and bts_2 and accounts ACC A and ACC A1, any two users and any two accounts can be used. + +1. Login to the shell as bts +2. Open at least one ladder with market data +3. Enter a few working limit orders and few of your favorite algos in ACC A +4. Logout of bts and in as bts_2 +5. Repeat steps 2 and 3 for this user + +6. Open the Spark Admin for your VM in your preferred non Microsoft web browser +7. Go to the Orders tab under the Accounts module for ACC A; the number of orders you entered for each user should be displayed in the table +8. Check the box in the header to select all rows and click the big red button + +9. Go back to the shell an make sure all of your orders have been canceled +10. Logout of bts_2 and log back in as bts +11. Repeat step 9 +12. Repeat step 3,6,7,8, and 9 for account ACC A1 From 4d993f2ef7869976618cc1e48535ac7438601705 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Wed, 7 Nov 2018 12:42:12 -0500 Subject: [PATCH 20/29] SPARK-1798 - Tool to benchmark market data by dumping to stdout periodically --- .vscode/launch.json | 3 +- src/lib/md/Diag/MDPeriodicDumper.cpp | 129 +++++++++++++++++++++++++-- src/lib/md/Diag/MDPeriodicDumper.h | 2 +- src/lib/md/MDConsumerHandle.cpp | 11 --- src/lib/md/MDConsumerHandle.h | 12 ++- src/server/sparkd/md/run_md.cpp | 64 +++++++++---- 6 files changed, 183 insertions(+), 38 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index c8d1c78ce..3a4b76ad2 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -209,7 +209,8 @@ "type": "cppdbg", "request": "launch", "program": "/home/bts/spark/build/ninja_debug/src/server/sparkd/sparkd", - "args": ["test","Co2RelayTests"], + // "args": ["test","Co2RelayTests"], + "args": ["md", "race", "100"], "stopAtEntry": false, "cwd": "/home/bts/spark/build/ninja_debug/src/server/sparkd/", "environment": [], diff --git a/src/lib/md/Diag/MDPeriodicDumper.cpp b/src/lib/md/Diag/MDPeriodicDumper.cpp index 222594bab..f70654f60 100644 --- a/src/lib/md/Diag/MDPeriodicDumper.cpp +++ b/src/lib/md/Diag/MDPeriodicDumper.cpp @@ -1,26 +1,143 @@ #include "MDPeriodicDumper.h" #include "lib/md/MDConsumer.h" +#include "lib/utils/TimeStamp.h" +#include "lib/utils/Dispatcher.h" +#include "lib/utils/datetime.h" #include +#include namespace bts { namespace md { -struct Handle { - utils::TimeStamp time; +/* + * Design considerations + * + * - We want this test to be 'fair' + * - We want a manageable amount of data + * - +*/ + +struct Entry { + uint32_t seq; MDConsumerHandle handle; + std::unique_ptr security; }; -void run_dumper(const std::set& symbols) { +std::string print_book( + const Entry& e, + utils::TimeStamp twrite, + utils::TimeStamp tread, + uint32_t seq, + bool ambiguous, + const std::array& bid, + const std::array& ask) { + + std::stringstream ss; + ss << std::endl; + ss << e.security->glbx().security_desc() << " (" << e.handle.key() << ")" << std::endl; + ss << "Sequence: " << seq; + if (ambiguous) ss << "*"; + ss << std::endl; + ss << "TWrit: " << twrite.to_uint64() << " ns" << std::endl; + ss << "TRead: " << tread.to_uint64() << " ns" << std::endl; + char buffer[512]; + + snprintf(buffer, 512, "%-4s %-10s | %-10s %-4s\n", "sz", "bid", "ask", "sz"); + ss << buffer; + ss << "---------- ---------- ----------\n"; + for (auto x = COCONUT_MAX_DEPTH; x != 0; --x) { + auto a = ask[x-1]; + if (!os::is_nan(a.price())) { + snprintf(buffer, 512, "%4s %10s | %10.7f %4d\n", "", "", a.price(), a.size()); + ss << buffer; + } + } + ss << "---------- ---------- ----------\n"; + for (uint32_t x = 0; x < COCONUT_MAX_DEPTH; ++x) { + auto b = bid[x]; + if (!os::is_nan(b.price())) { + snprintf(buffer, 512, "%4d %10.7f | \n", b.size(), b.price()); + ss << buffer; + } + } + return ss.str(); +} + +datetime::Date expiration(const api::Security& sec) { + auto f = sec.future(); + return { (uint16_t)f.expiration_year(), (uint8_t)f.expiration_month(), (uint8_t)f.expiration_day() }; +} + +std::vector filter_entries(std::vector& entries) { + // Choose the front month for all symbols + auto today = datetime::today(); + std::map map; + for (auto& e : entries) { + auto type = securitytype_of(*e.security); + // Only futures because front month is complicated + if (type != api::SECURITYTYPE_FUTURE) continue; + UserSymbol key { e.security->symbol(), e.security->exchange(), type }; + auto& current_entry = map[key]; + if (current_entry == nullptr) { + current_entry = &e; + } + else { + auto cur = expiration(*current_entry->security); + auto next = expiration(*e.security); + if (next > today && next < cur) { + current_entry = &e; + } + } + } + + std::vector result; + for (const auto& pair : map) { + result.push_back(std::move(*pair.second)); + } + return result; +} + +void run_periodic_dumper(const std::set& symbols, uint32_t seq_mod) { + + std::cout << + "Dumping every sequence % " << seq_mod << " == 0 for\n"; + MDConsumer consumer(symbols, nullptr); // read all securities into a vector + std::vector entries; + consumer.walk_security_list([&](const api::Security& sec, MDConsumerHandle handle) { + entries.push_back({ 0, handle, std::make_unique(sec) }); + }); + entries = filter_entries(entries); + + std::cout << "Selected securities: " << std::endl; + for (const auto& e : entries) { + std::cout << e.security->glbx().security_desc() << std::endl; + } + + auto& disp = utils::Dispatcher::background(); // spin around on the vector detecting changes - // if change read book+ sequence - // If seq % n == 0 - // print book to stdout (async?) + std::array bid,ask; + while(true) { + for(auto& e : entries) { + auto sequence = e.handle.sequence(); + if (sequence != e.seq && sequence % seq_mod == 0) { + e.seq = sequence; + auto tread = TimeStamp::now(); + utils::TimeStamp twrite; + e.handle.get_book(twrite, COCONUT_MAX_DEPTH, bid.data(), ask.data()); + auto seq2 = e.handle.sequence(); + // print book to stdout (async?) + disp.post([=,&e]() { + std::cout << print_book(e, twrite, tread, sequence, (sequence != seq2), bid, ask); + }); + } + } + } } }}//ns \ No newline at end of file diff --git a/src/lib/md/Diag/MDPeriodicDumper.h b/src/lib/md/Diag/MDPeriodicDumper.h index 4c3f95719..c3cbc278e 100644 --- a/src/lib/md/Diag/MDPeriodicDumper.h +++ b/src/lib/md/Diag/MDPeriodicDumper.h @@ -5,7 +5,7 @@ namespace bts { namespace md { -void run_dumper(const std::set&); +void run_periodic_dumper(const std::set&, uint32_t seq_mod); }}//ns \ No newline at end of file diff --git a/src/lib/md/MDConsumerHandle.cpp b/src/lib/md/MDConsumerHandle.cpp index a078dbef1..6f11dcece 100644 --- a/src/lib/md/MDConsumerHandle.cpp +++ b/src/lib/md/MDConsumerHandle.cpp @@ -305,17 +305,6 @@ CoconutSecurityStatus MDConsumerHandle::status() const { } never; } -uint16_t MDConsumerHandle::sequence() const { - switch(_type) { - case HANDLETYPE_VOID: return 0; - case HANDLETYPE_DEPTH: return book.hot->sequence; - case HANDLETYPE_BBO: return bbo.hot->sequence; - case HANDLETYPE_SYNTHETIC: - bts_assert(!synthetic.def->legs.empty()); - return sane::min_by(synthetic.def->legs, [](const MDConsumerHandle& h) { return h.last_market_update(); })->sequence(); - } - never; -} }}//namespace diff --git a/src/lib/md/MDConsumerHandle.h b/src/lib/md/MDConsumerHandle.h index 7f1097d73..904a44e21 100644 --- a/src/lib/md/MDConsumerHandle.h +++ b/src/lib/md/MDConsumerHandle.h @@ -159,7 +159,17 @@ public: bool is_heart_beating() const; TimeStamp heartbeat() const; CoconutSecurityStatus status() const; - uint16_t sequence() const; + uint32_t sequence() const { + switch(_type) { + case HANDLETYPE_VOID: return 0; + case HANDLETYPE_DEPTH: return book.hot->sequence; + case HANDLETYPE_BBO: return bbo.hot->sequence; + case HANDLETYPE_SYNTHETIC: + bts_assert(!synthetic.def->legs.empty()); + return sane::min_by(synthetic.def->legs, [](const MDConsumerHandle& h) { return h.last_market_update(); })->sequence(); + } + bts_assert(false); + } security_key key() const { switch(_type) { case HANDLETYPE_VOID: return invalid_security_key; diff --git a/src/server/sparkd/md/run_md.cpp b/src/server/sparkd/md/run_md.cpp index c2136992d..e630d83ac 100644 --- a/src/server/sparkd/md/run_md.cpp +++ b/src/server/sparkd/md/run_md.cpp @@ -107,8 +107,6 @@ void run_record(const std::string& arg){ record_everything(p.parent_path().string(), p.filename().string()); } - - string& uppercase(string& s) { transform(s.begin(), s.end(), s.begin(), ::toupper); return s; @@ -128,14 +126,36 @@ bool parse_enum(const string& value, const string& prefix, T2 parse, T default_v } } -enum class HotCold { Tepid, Hot, Cold }; -std::tuple parse_md_args(const vector& args) { +bool int_of_string(const std::string& str, long* result) { + try { + size_t pos = 0; + int i = std::stol(str, &pos); + if (pos == str.size()) { + *result = i; + return true; + } + } + catch(const std::exception&) { } + return false; +} + +typedef std::string Symbol; +struct MDArgs { + api::Venue venue; + co2::Platform platform; + api::SecurityType type; + Symbol symbol; + bool hot; + long number; +}; +MDArgs parse_md_args(const vector& args) { using namespace api; using namespace co2; auto venue = VENUE_UNSET; // Default to glbx. If you change this change below (used) also auto platform = PLATFORM_UNSET; auto type = SECURITYTYPE_UNSET; - auto temperature = HotCold::Tepid; + bool hot = false; + long number = 0; std::string symbol; for (auto str : args) { uppercase(str); @@ -143,15 +163,18 @@ std::tuple p (venue == VENUE_UNSET && parse_enum(str, "VENUE", Venue_Parse, venue, &venue)) || (platform == PLATFORM_UNSET && parse_enum(str, "PLATFORM", Platform_Parse, platform, &platform)) || parse_enum(str, "SECURITYTYPE", SecurityType_Parse, type, &type); - if (str == "HOT") { temperature = HotCold::Hot; used = true; } - else if (str == "COLD") { temperature = HotCold::Cold; used = true; } + if (str == "HOT") { hot = true; used = true; } + else if (str == "COLD") { hot = false; used = true; } + else { + used |= int_of_string(str, &number); + } if (!used) { symbol = str; } } if (venue == VENUE_UNSET) venue = VENUE_GLBX; if (platform == PLATFORM_UNSET) platform = platform_for_exchange(venue); - return { venue, platform, type, symbol, temperature }; + return { venue, platform, type, symbol, hot, number }; } void run_dump(const vector& args) { @@ -160,13 +183,13 @@ void run_dump(const vector& args) { auto p = parse_md_args(args); stringstream ss; ss << "bts_edge_" << config::get().shared_memory_namespace() << "_" - << uppercase(get(p)) << "_" - << SecurityType_Name(get(p)) << "_" - << Venue_Name(get(p)) << "_" - << Platform_Name(get(p)); + << uppercase(p.symbol) << "_" + << SecurityType_Name(p.type) << "_" + << Venue_Name(p.venue) << "_" + << Platform_Name(p.platform); ShmDumper dumper(ss.str()); - if (get(p) == HotCold::Hot) { + if (p.hot) { dumper.dump_hot(); } else { @@ -224,10 +247,15 @@ void dump_db(const vector& args) { }); } -void race(const vector& args) { - // args: % - auto _ug = parse_md_args(args); - // platform +void run_race(const vector& args) { + std::set products; + auto conf = parse_md_args(args); + PlatformConfig config; + config.set_type(api::SecurityType_Name(api::SECURITYTYPE_FUTURE)); + config.set_platform(co2::Platform_Name(conf.platform)); + get_system_md_symbols(config, products); + auto mod = conf.number > 0 ? conf.number : 1000; + md::run_periodic_dumper(products, mod); } @@ -251,7 +279,7 @@ void md_tool(std::vector const &args) { else if (cmd == "relay") relay_everything(); else if (cmd == "test_relay") relay_everything(); else if (cmd == "dumpsecurity" || cmd == "dumpsecurities") dump_db(vector(args.begin()+1, args.end())); - else if (cmd == "race") race(args); + else if (cmd == "race") run_race(args); else printhelp(); } From de83f19388f389664dfcab79e1dd49bbef5ac64c Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Thu, 8 Nov 2018 13:33:23 -0500 Subject: [PATCH 21/29] Adjust column widths --- src/lib/md/Diag/MDPeriodicDumper.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/lib/md/Diag/MDPeriodicDumper.cpp b/src/lib/md/Diag/MDPeriodicDumper.cpp index f70654f60..dca51eafe 100644 --- a/src/lib/md/Diag/MDPeriodicDumper.cpp +++ b/src/lib/md/Diag/MDPeriodicDumper.cpp @@ -43,21 +43,21 @@ std::string print_book( ss << "TRead: " << tread.to_uint64() << " ns" << std::endl; char buffer[512]; - snprintf(buffer, 512, "%-4s %-10s | %-10s %-4s\n", "sz", "bid", "ask", "sz"); + snprintf(buffer, 512, "%-5s %-14s | %-14s %-5s\n", "sz", "bid", "ask", "sz"); ss << buffer; - ss << "---------- ---------- ----------\n"; + ss << "-------------------- - -------------------- \n"; for (auto x = COCONUT_MAX_DEPTH; x != 0; --x) { auto a = ask[x-1]; if (!os::is_nan(a.price())) { - snprintf(buffer, 512, "%4s %10s | %10.7f %4d\n", "", "", a.price(), a.size()); + snprintf(buffer, 512, "%5s %14s | %14.7f %5d\n", "", "", a.price(), a.size()); ss << buffer; } } - ss << "---------- ---------- ----------\n"; + ss << "-------------------- - -------------------- \n"; for (uint32_t x = 0; x < COCONUT_MAX_DEPTH; ++x) { auto b = bid[x]; if (!os::is_nan(b.price())) { - snprintf(buffer, 512, "%4d %10.7f | \n", b.size(), b.price()); + snprintf(buffer, 512, "%5d %14.7f | \n", b.size(), b.price()); ss << buffer; } } From a4bc34e633d463b113212ab19d1047ec5c489266 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 9 Nov 2018 12:15:19 -0500 Subject: [PATCH 22/29] SPARK-1798 Add CSV output option (default) --- .vscode/launch.json | 2 +- src/lib/md/Diag/MDPeriodicDumper.cpp | 105 +++++++++++++++++++-------- src/lib/md/Diag/MDPeriodicDumper.h | 7 +- src/server/sparkd/md/run_md.cpp | 8 +- 4 files changed, 87 insertions(+), 35 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 3a4b76ad2..7882b7088 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -210,7 +210,7 @@ "request": "launch", "program": "/home/bts/spark/build/ninja_debug/src/server/sparkd/sparkd", // "args": ["test","Co2RelayTests"], - "args": ["md", "race", "100"], + "args": ["md", "tail", "10"], "stopAtEntry": false, "cwd": "/home/bts/spark/build/ninja_debug/src/server/sparkd/", "environment": [], diff --git a/src/lib/md/Diag/MDPeriodicDumper.cpp b/src/lib/md/Diag/MDPeriodicDumper.cpp index dca51eafe..10e449ed9 100644 --- a/src/lib/md/Diag/MDPeriodicDumper.cpp +++ b/src/lib/md/Diag/MDPeriodicDumper.cpp @@ -6,6 +6,8 @@ #include #include +#include +#include namespace bts { namespace md { @@ -25,6 +27,7 @@ struct Entry { }; std::string print_book( + PrintFormat format, const Entry& e, utils::TimeStamp twrite, utils::TimeStamp tread, @@ -34,32 +37,59 @@ std::string print_book( const std::array& ask) { std::stringstream ss; - ss << std::endl; - ss << e.security->glbx().security_desc() << " (" << e.handle.key() << ")" << std::endl; - ss << "Sequence: " << seq; - if (ambiguous) ss << "*"; - ss << std::endl; - ss << "TWrit: " << twrite.to_uint64() << " ns" << std::endl; - ss << "TRead: " << tread.to_uint64() << " ns" << std::endl; - char buffer[512]; + if(format == PrintFormat::Human) { + ss << std::endl; + ss << e.security->glbx().security_desc() << " (" << e.handle.key() << ")" << std::endl; + ss << "Sequence: " << seq; + if (ambiguous) ss << "*"; + ss << std::endl; + ss << "TWrit: " << twrite.to_uint64() << " ns" << std::endl; + ss << "TRead: " << tread.to_uint64() << " ns" << std::endl; + char buffer[512]; - snprintf(buffer, 512, "%-5s %-14s | %-14s %-5s\n", "sz", "bid", "ask", "sz"); - ss << buffer; - ss << "-------------------- - -------------------- \n"; - for (auto x = COCONUT_MAX_DEPTH; x != 0; --x) { - auto a = ask[x-1]; - if (!os::is_nan(a.price())) { - snprintf(buffer, 512, "%5s %14s | %14.7f %5d\n", "", "", a.price(), a.size()); - ss << buffer; + snprintf(buffer, 512, "%-5s %-14s | %-14s %-5s\n", "sz", "bid", "ask", "sz"); + ss << buffer; + ss << "-------------------- - -------------------- \n"; + for (auto x = COCONUT_MAX_DEPTH; x != 0; --x) { + auto a = ask[x-1]; + if (!os::is_nan(a.price())) { + snprintf(buffer, 512, "%5s %14s | %14.7f %5d\n", "", "", a.price(), a.size()); + ss << buffer; + } + } + ss << "-------------------- - -------------------- \n"; + for (uint32_t x = 0; x < COCONUT_MAX_DEPTH; ++x) { + auto b = bid[x]; + if (!os::is_nan(b.price())) { + snprintf(buffer, 512, "%5d %14.7f | \n", b.size(), b.price()); + ss << buffer; + } } } - ss << "-------------------- - -------------------- \n"; - for (uint32_t x = 0; x < COCONUT_MAX_DEPTH; ++x) { - auto b = bid[x]; - if (!os::is_nan(b.price())) { - snprintf(buffer, 512, "%5d %14.7f | \n", b.size(), b.price()); - ss << buffer; + else if (format == PrintFormat::CSV) { + // Symbol, Id, Sequence, Ambiguous, TWrite, TRead, BidBook, AskBook + ss << e.security->glbx().security_desc() << ", " + << e.security->id() << ", " + << seq << ", " + << (ambiguous ? "true, " : "false, ") + << twrite.to_uint64() << ", " + << tread.to_uint64() << ","; + for (size_t x = 0; x < COCONUT_MAX_DEPTH; ++x) { + if (!bid[x].is_nan()) { + ss << " " << std::setprecision(11) << bid[x].price() << "-"; + if (bid[x].implied_size() > 0) { ss << "*"; } + ss << bid[x].size(); + } } + ss << ","; + for (size_t x = 0; x < COCONUT_MAX_DEPTH; ++x) { + if (!ask[x].is_nan()) { + ss << " " << std::setprecision(11) << ask[x].price() << "-"; + if (ask[x].implied_size() > 0) { ss << "*"; } + ss << ask[x].size(); + } + } + ss << std::endl; } return ss.str(); } @@ -98,10 +128,28 @@ std::vector filter_entries(std::vector& entries) { return result; } -void run_periodic_dumper(const std::set& symbols, uint32_t seq_mod) { +void write_header(PrintFormat format, long seq_mod, const std::vector& entries) { + if (format == PrintFormat::Human) { + std::cout << "Dumping every sequence % " << seq_mod << " == 0 for\n"; + std::cout << "Selected securities: " << std::endl; + for (const auto& e : entries) { + std::cout << e.security->glbx().security_desc() << std::endl; + } + } + else { + // CSV header + std::cout << "Symbol, Id, Sequence, Ambiguous, TWrite, TRead, BidBook, AskBook"; + // for (size_t x = 0; x < COCONUT_MAX_DEPTH; ++x) { + // std::cout << ", " << "Bid" << x << ", DirectBidSize" << x << ", ImpliedBidSize" << x; + // } + // for (size_t x = 0; x < COCONUT_MAX_DEPTH; ++x) { + // std::cout << ", " << "Ask" << x << ", AskSize" << x << ", ImpliedAskSize" << x; + // } + std::cout << std::endl; + } +} - std::cout << - "Dumping every sequence % " << seq_mod << " == 0 for\n"; +void run_periodic_dumper(const std::set& symbols, uint32_t seq_mod, PrintFormat format) { MDConsumer consumer(symbols, nullptr); @@ -113,10 +161,7 @@ void run_periodic_dumper(const std::set& symbols, uint32_t seq_mod) entries = filter_entries(entries); - std::cout << "Selected securities: " << std::endl; - for (const auto& e : entries) { - std::cout << e.security->glbx().security_desc() << std::endl; - } + write_header(format, seq_mod, entries); auto& disp = utils::Dispatcher::background(); @@ -133,7 +178,7 @@ void run_periodic_dumper(const std::set& symbols, uint32_t seq_mod) auto seq2 = e.handle.sequence(); // print book to stdout (async?) disp.post([=,&e]() { - std::cout << print_book(e, twrite, tread, sequence, (sequence != seq2), bid, ask); + std::cout << print_book(format, e, twrite, tread, sequence, (sequence != seq2), bid, ask); }); } } diff --git a/src/lib/md/Diag/MDPeriodicDumper.h b/src/lib/md/Diag/MDPeriodicDumper.h index c3cbc278e..e59a201f0 100644 --- a/src/lib/md/Diag/MDPeriodicDumper.h +++ b/src/lib/md/Diag/MDPeriodicDumper.h @@ -5,7 +5,12 @@ namespace bts { namespace md { -void run_periodic_dumper(const std::set&, uint32_t seq_mod); +enum class PrintFormat { + Human, + CSV +}; + +void run_periodic_dumper(const std::set&, uint32_t seq_mod, PrintFormat format); }}//ns \ No newline at end of file diff --git a/src/server/sparkd/md/run_md.cpp b/src/server/sparkd/md/run_md.cpp index e630d83ac..76e337de1 100644 --- a/src/server/sparkd/md/run_md.cpp +++ b/src/server/sparkd/md/run_md.cpp @@ -247,7 +247,7 @@ void dump_db(const vector& args) { }); } -void run_race(const vector& args) { +void run_tail(const vector& args) { std::set products; auto conf = parse_md_args(args); PlatformConfig config; @@ -255,7 +255,9 @@ void run_race(const vector& args) { config.set_platform(co2::Platform_Name(conf.platform)); get_system_md_symbols(config, products); auto mod = conf.number > 0 ? conf.number : 1000; - md::run_periodic_dumper(products, mod); + PrintFormat fmt = PrintFormat::CSV; + for (const auto& a : args) { if (a == "longform") fmt = PrintFormat::Human; } + md::run_periodic_dumper(products, mod, fmt); } @@ -279,7 +281,7 @@ void md_tool(std::vector const &args) { else if (cmd == "relay") relay_everything(); else if (cmd == "test_relay") relay_everything(); else if (cmd == "dumpsecurity" || cmd == "dumpsecurities") dump_db(vector(args.begin()+1, args.end())); - else if (cmd == "race") run_race(args); + else if (cmd == "tail") run_tail(args); else printhelp(); } From d780e2aab20b9894f10afde74ef732d6c81c3646 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Fri, 9 Nov 2018 12:29:15 -0500 Subject: [PATCH 23/29] SPARK-1798 sqlite import does not want spaces after comma in csv --- src/lib/md/Diag/MDPeriodicDumper.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/lib/md/Diag/MDPeriodicDumper.cpp b/src/lib/md/Diag/MDPeriodicDumper.cpp index 10e449ed9..f1ff98fe7 100644 --- a/src/lib/md/Diag/MDPeriodicDumper.cpp +++ b/src/lib/md/Diag/MDPeriodicDumper.cpp @@ -68,11 +68,11 @@ std::string print_book( } else if (format == PrintFormat::CSV) { // Symbol, Id, Sequence, Ambiguous, TWrite, TRead, BidBook, AskBook - ss << e.security->glbx().security_desc() << ", " - << e.security->id() << ", " - << seq << ", " - << (ambiguous ? "true, " : "false, ") - << twrite.to_uint64() << ", " + ss << e.security->glbx().security_desc() << "," + << e.security->id() << "," + << seq << "," + << (ambiguous ? "true," : "false,") + << twrite.to_uint64() << "," << tread.to_uint64() << ","; for (size_t x = 0; x < COCONUT_MAX_DEPTH; ++x) { if (!bid[x].is_nan()) { @@ -138,12 +138,12 @@ void write_header(PrintFormat format, long seq_mod, const std::vector& en } else { // CSV header - std::cout << "Symbol, Id, Sequence, Ambiguous, TWrite, TRead, BidBook, AskBook"; + std::cout << "Symbol,Id,Sequence,Ambiguous,TWrite,TRead,BidBook,AskBook"; // for (size_t x = 0; x < COCONUT_MAX_DEPTH; ++x) { - // std::cout << ", " << "Bid" << x << ", DirectBidSize" << x << ", ImpliedBidSize" << x; + // std::cout << "," << "Bid" << x << ", DirectBidSize" << x << ", ImpliedBidSize" << x; // } // for (size_t x = 0; x < COCONUT_MAX_DEPTH; ++x) { - // std::cout << ", " << "Ask" << x << ", AskSize" << x << ", ImpliedAskSize" << x; + // std::cout << "," << "Ask" << x << ", AskSize" << x << ", ImpliedAskSize" << x; // } std::cout << std::endl; } From 88e48ebcf7df4bd270f313be6d7402a4a5c2836c Mon Sep 17 00:00:00 2001 From: Michael Harding Date: Fri, 9 Nov 2018 17:31:03 -0500 Subject: [PATCH 24/29] SPARK-1703: Add tool-tips for lean, join, cover, top --- .../Order/OrderDetailsPopupControl.xaml | 134 +++++++++++++----- 1 file changed, 101 insertions(+), 33 deletions(-) diff --git a/src/client/spark/view/Controls/Order/OrderDetailsPopupControl.xaml b/src/client/spark/view/Controls/Order/OrderDetailsPopupControl.xaml index a5ee0b0cf..09759a533 100644 --- a/src/client/spark/view/Controls/Order/OrderDetailsPopupControl.xaml +++ b/src/client/spark/view/Controls/Order/OrderDetailsPopupControl.xaml @@ -223,7 +223,7 @@ - + @@ -231,13 +231,37 @@ - Cross Thresh + + Cross Thresh + + - Max Chase Ticks + + Max Chase Ticks + + - On Toward - - Child + + On Toward + + + + + Child + + - Join Thresh + + Join Thresh + + - Child + + Child + + - Top Thresh + + Top Thresh + + - Child + + Child + + + - Hedge Ticks + + Hedge Ticks + + - Hedge + + Hedge + + - Child + + Child + + + - + @@ -458,7 +526,7 @@ Price - + @@ -634,8 +702,8 @@ - - + + @@ -647,9 +715,9 @@ - - - + + + @@ -660,9 +728,9 @@ - - - + + + @@ -673,9 +741,9 @@ - - - + + + @@ -686,9 +754,9 @@ - - - + + + @@ -699,11 +767,11 @@ - - - + + + - + Child Date: Mon, 12 Nov 2018 09:33:44 -0500 Subject: [PATCH 25/29] Switch tbb::queueing_mutex to std::mutex --- src/lib/md/MDProducer.cpp | 6 +++--- src/lib/md/MDProducer.h | 5 ++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/lib/md/MDProducer.cpp b/src/lib/md/MDProducer.cpp index b519f52d9..aba89688b 100644 --- a/src/lib/md/MDProducer.cpp +++ b/src/lib/md/MDProducer.cpp @@ -102,7 +102,7 @@ private: // MDProducer::MDProducer(Platform platform, bool shared) : - _mutex(new Mutex()), + _mutex(new std::mutex()), _producers(), _platform(platform), _shared(shared) @@ -143,7 +143,7 @@ MDProducerImpl& MDProducer::get_producer(const char* symbol, SecurityType type, MDProducerHandle MDProducer::add_security(const FixedSecurity& security) { bts_assert_d(strlen(security.symbol()) > 0); bts_assert(security.size >= sizeof(FixedSecurity)); - Mutex::scoped_lock lock(*_mutex); + std::lock_guard lock(*_mutex); auto& impl = get_producer(security.symbol(), security.type, security.exchange); auto result = impl.add_security(security); if (security.key != invalid_security_key) { result.set_key(security.key); } @@ -151,7 +151,7 @@ MDProducerHandle MDProducer::add_security(const FixedSecurity& security) { } MDProducerChannel MDProducer::get_channel(const char* symbol, SecurityType type, Venue exchange) { - Mutex::scoped_lock lock(*_mutex); + std::lock_guard lock(*_mutex); return MDProducerChannel(get_producer(symbol, type, exchange)); } diff --git a/src/lib/md/MDProducer.h b/src/lib/md/MDProducer.h index 4123b59e6..29ef0feee 100644 --- a/src/lib/md/MDProducer.h +++ b/src/lib/md/MDProducer.h @@ -7,9 +7,9 @@ #include "MDProducerHandle.h" #include "MDProducerChannel.h" #include "Shm/FixedSecurity.h" -#include "tbb/queuing_mutex.h" #include #include +#include namespace bts { namespace md { @@ -39,8 +39,7 @@ private: MDProducerImpl& get_producer(const char* symbol, SecurityType type, Venue exchange); - typedef tbb::queuing_mutex Mutex; - std::unique_ptr _mutex; + std::unique_ptr _mutex; std::vector> _producers; Platform _platform; bool _shared; From 986a7918018096ed36b2e65ccd57ff35158d4c33 Mon Sep 17 00:00:00 2001 From: Michael Harding Date: Mon, 12 Nov 2018 14:29:35 -0500 Subject: [PATCH 26/29] Update Release Notes --- doc/user/docs/Trading.md | 16 ++++++++-------- doc/user/docs/release_notes.md | 9 +++++++++ 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/doc/user/docs/Trading.md b/doc/user/docs/Trading.md index 4b994d85c..9a7472a88 100644 --- a/doc/user/docs/Trading.md +++ b/doc/user/docs/Trading.md @@ -54,22 +54,22 @@ ![](images/image4.png) -- ![](images/image48.png)][Fills - +- ![](images/image48.png)Fills - clicking on this will launch a fills window -- ![](images/image57.png)][ Modify - +- ![](images/image57.png) Modify - clicking on this will launch an order ticket for a live order highlighted in the Order Book. -- ![](images/image73.png)][Cancel +- ![](images/image73.png)Cancel selected - clicking on this will cancel the selected orders you have highlighted in the order book.  You can choose more than one by holding hold the Ctrl key when click on each row of orders -- ![](images/image74.png)][Cancel +- ![](images/image74.png)Cancel buys - clicking on this will cancel all the buy orders you have in the order book -- ![](images/image33.png)][Cancel +- ![](images/image33.png)Cancel sells - clicking on this will cancel all the sell orders you have in the order book -- ![](images/image2.png)][Cancel +- ![](images/image2.png)Cancel all - clicking on this will cancel all working orders you have the order book @@ -168,10 +168,10 @@ - Iceberg - This is a buy or sell order where the order is divided into a - visible or “show” portion which is seen at the exchange.  ][The + visible or “show” portion which is seen at the exchange.  The rest of the order is hidden and will remain hidden until the show quantity is filled, essentially reloading the order back to the show - quantity.  You will need to ][define what type of order that will + quantity.  You will need to define what type of order that will be used for the child order to be sent. ![](images/image47.png) diff --git a/doc/user/docs/release_notes.md b/doc/user/docs/release_notes.md index 5dda1897a..cd9cede36 100644 --- a/doc/user/docs/release_notes.md +++ b/doc/user/docs/release_notes.md @@ -1,5 +1,14 @@ # Release Notes +## 1.3.8 + +### Speed Enhancements + - Further speed enhancements +### Conditional (new quoting parameter) + - Reduces re-quoting updates by only quoting if N ticks away from the opposite inside market. + - Quotes are cancelled when it gets further away. + - "Price Type" dictates if this logic applies to the quoting market or to the synthetic spread market. + ## 1.3.7 ### Net Change pricing From 7eb8b21ceae44bb658a436071246df02a9950841 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 12 Nov 2018 16:58:06 -0500 Subject: [PATCH 27/29] Simplify SecurityIDRecorder api for 3rd party consumption --- .vscode/launch.json | 2 +- src/lib/md/MDProducer.cpp | 17 +++++++- src/lib/md/MDProducer.h | 3 ++ src/lib/md/SecurityIDRecorder.cpp | 39 ++++++++++--------- src/lib/md/SecurityIDRecorder.h | 17 +++++--- src/lib/md/SyntheticSecurityService.cpp | 5 +-- .../md/platforms/mdp3_onixs/MDP3Listener.cpp | 9 +---- .../md/platforms/mdp3_onixs/MDP3Listener.h | 2 - .../md/platforms/mdp3_onixs/MDP3Main.cpp | 10 +---- .../sparkd/md/platforms/mdp3_onixs/MDP3Main.h | 1 - 10 files changed, 56 insertions(+), 49 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 7882b7088..0eaae7c1f 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -210,7 +210,7 @@ "request": "launch", "program": "/home/bts/spark/build/ninja_debug/src/server/sparkd/sparkd", // "args": ["test","Co2RelayTests"], - "args": ["md", "tail", "10"], + "args": ["md", "run", "10"], "stopAtEntry": false, "cwd": "/home/bts/spark/build/ninja_debug/src/server/sparkd/", "environment": [], diff --git a/src/lib/md/MDProducer.cpp b/src/lib/md/MDProducer.cpp index aba89688b..1971fd647 100644 --- a/src/lib/md/MDProducer.cpp +++ b/src/lib/md/MDProducer.cpp @@ -5,6 +5,7 @@ #include "lib/utils/WaitFreeQueue.h" #include "Shm/MDStructure.h" #include "MemorySegmentUtils.h" +#include "SecurityIDRecorder.h" using namespace std; @@ -104,10 +105,19 @@ private: MDProducer::MDProducer(Platform platform, bool shared) : _mutex(new std::mutex()), _producers(), + _id_recorder(), _platform(platform), _shared(shared) -{ -} +{ } +#ifndef WIN32 +MDProducer::MDProducer(Platform platform, SecurityIDRecorder recorder, bool shared) : + _mutex(new std::mutex()), + _producers(), + _id_recorder(std::make_unique(recorder)), + _platform(platform), + _shared(shared) +{ } +#endif MDProducer::MDProducer(MDProducer&& other) : _mutex(std::move(other._mutex)), _producers(std::move(other._producers)), @@ -147,6 +157,9 @@ MDProducerHandle MDProducer::add_security(const FixedSecurity& security) { auto& impl = get_producer(security.symbol(), security.type, security.exchange); auto result = impl.add_security(security); if (security.key != invalid_security_key) { result.set_key(security.key); } + else if(_id_recorder) { + _id_recorder->ensure_key(result); + } return result; } diff --git a/src/lib/md/MDProducer.h b/src/lib/md/MDProducer.h index 29ef0feee..aaad1ee1a 100644 --- a/src/lib/md/MDProducer.h +++ b/src/lib/md/MDProducer.h @@ -15,11 +15,13 @@ namespace bts { namespace md { class MDProducerImpl; +class SecurityIDRecorder; class BTS_EXPORT MDProducer { public: MDProducer(Platform platform, bool shared=true); + MDProducer(Platform platform, SecurityIDRecorder recorder, bool shared=true); MDProducer(MDProducer&& other); ~MDProducer(); @@ -41,6 +43,7 @@ private: std::unique_ptr _mutex; std::vector> _producers; + std::unique_ptr _id_recorder; Platform _platform; bool _shared; }; diff --git a/src/lib/md/SecurityIDRecorder.cpp b/src/lib/md/SecurityIDRecorder.cpp index 00c66c222..8b4fc9711 100644 --- a/src/lib/md/SecurityIDRecorder.cpp +++ b/src/lib/md/SecurityIDRecorder.cpp @@ -1,7 +1,6 @@ #ifndef WIN32 #include "SecurityIDRecorder.h" -#include "lib/utils/defer.h" #include "lib/config/config.h" #include "lib/utils/Sqlite_utils.h" @@ -32,15 +31,12 @@ struct SecurityIDRecorderState { SecurityIDRecorderState() : work(), pending_handle(), - db(nullptr, sqlite3_close), - select_stmt(nullptr, sqlite3_finalize), - insert_stmt(nullptr, sqlite3_finalize) + db(nullptr, sqlite3_close) {} tbb::concurrent_queue> work; tbb::concurrent_queue pending_handle; sqlite3_ptr db; - sqlite3_stmt_ptr select_stmt; - sqlite3_stmt_ptr insert_stmt; + unique_ptr timer; }; SecurityIDRecorder::SecurityIDRecorder(): @@ -49,6 +45,16 @@ SecurityIDRecorder::SecurityIDRecorder(): init(); } +SecurityIDRecorder::SecurityIDRecorder(utils::Dispatcher& dispatcher): + _self(new SecurityIDRecorderState()) +{ + init(); + _self->timer = std::make_unique( + dispatcher, 250, + [=]() { this->do_work(); }); + _self->timer->start(); +} + SecurityIDRecorder::~SecurityIDRecorder() { } void SecurityIDRecorder::init() { @@ -66,10 +72,6 @@ void SecurityIDRecorder::init() { sqlite3_exec_nofail(_self->db, "PRAGMA journal_mode=WAL"); sqlite3_exec_nofail(_self->db, "PRAGMA syncronous=0"); sqlite3_exec_nofail(_self->db, "CREATE TABLE IF NOT EXISTS security_keys (descriptor STRING NOT NULL PRIMARY KEY, type int);"); - - // Prepare statements - _self->insert_stmt = prepare_query(_self->db, "INSERT INTO security_keys (descriptor, type) values (?, ?)"); - _self->select_stmt = prepare_query(_self->db, "SELECT rowid from security_keys where descriptor=? LIMIT 1;"); } void SecurityIDRecorder::ensure_key(MDProducerHandle handle) { @@ -85,6 +87,7 @@ void SecurityIDRecorder::assign_key(const api::Security& security, const Securit } void SecurityIDRecorder::do_work() { + LOG(md, Info, "do_work"); work_handles(); work_db(); } @@ -226,20 +229,20 @@ void SecurityIDRecorder::bulk_insert(const std::vector& da } security_key SecurityIDRecorder::search(const std::string& descriptor) { - sqlite3_bind_text(_self->select_stmt.get(), 1, descriptor.c_str(), descriptor.size(), SQLITE_STATIC); - DEFER(sqlite3_reset(_self->select_stmt.get())); - int result = sqlite3_step(_self->select_stmt.get()); + auto select_stmt = prepare_query(_self->db, "SELECT rowid from security_keys where descriptor=? LIMIT 1;"); + sqlite3_bind_text(select_stmt.get(), 1, descriptor.c_str(), descriptor.size(), SQLITE_STATIC); + int result = sqlite3_step(select_stmt.get()); if (result == SQLITE_DONE || result == SQLITE_ROW) { - return sqlite3_column_int(_self->select_stmt.get(), 0); + return sqlite3_column_int(select_stmt.get(), 0); } return invalid_security_key; } security_key SecurityIDRecorder::insert(const std::string& descriptor, SecurityType type) { - sqlite3_bind_text(_self->insert_stmt.get(), 1, descriptor.c_str(), descriptor.size(), SQLITE_STATIC); - sqlite3_bind_int(_self->insert_stmt.get(), 2, (int)type); - DEFER(sqlite3_reset(_self->insert_stmt.get())); - int result = sqlite3_step(_self->insert_stmt.get()); + auto insert_stmt = prepare_query(_self->db, "INSERT INTO security_keys (descriptor, type) values (?, ?)"); + sqlite3_bind_text(insert_stmt.get(), 1, descriptor.c_str(), descriptor.size(), SQLITE_STATIC); + sqlite3_bind_int(insert_stmt.get(), 2, (int)type); + int result = sqlite3_step(insert_stmt.get()); if (result == SQLITE_DONE || result == SQLITE_ROW) { return sqlite3_last_insert_rowid(_self->db.get()); } diff --git a/src/lib/md/SecurityIDRecorder.h b/src/lib/md/SecurityIDRecorder.h index 34e1a2f91..e42aab2b6 100644 --- a/src/lib/md/SecurityIDRecorder.h +++ b/src/lib/md/SecurityIDRecorder.h @@ -8,7 +8,6 @@ #include #include #include -#include namespace bts { namespace md { @@ -23,6 +22,8 @@ class SecurityIDRecorder { public: SecurityIDRecorder(); + // Automatically call do_work on this dispatcher + SecurityIDRecorder(utils::Dispatcher&); ~SecurityIDRecorder(); typedef std::function Callback; @@ -32,13 +33,12 @@ public: void assign_key(const api::Security& security, const Callback& key_assigned); // Someone must call work periodically to do lookups and flush the reserved ids to disk. - // returne true if there is more work to be done void do_work(); void bulk_insert(const std::vector&, const char* prefix); private: - std::unique_ptr _self; + std::shared_ptr _self; void init(); security_key search(const std::string& descriptor); @@ -46,9 +46,16 @@ private: void work_handles(); void work_db(); - - BTS_NO_COPY(SecurityIDRecorder); }; }}//namespace +#else +#include "MDProducerHandle.h" +namespace bts { +namespace md { + class SecurityIDRecorder { + public: + void ensure_key(MDProducerHandle){}; + }; +}} #endif diff --git a/src/lib/md/SyntheticSecurityService.cpp b/src/lib/md/SyntheticSecurityService.cpp index 84be15e0e..8d6ca60aa 100644 --- a/src/lib/md/SyntheticSecurityService.cpp +++ b/src/lib/md/SyntheticSecurityService.cpp @@ -25,15 +25,12 @@ typedef SyntheticSecurityServiceBase::MetadataStatefulPublisher MetadataPublishe struct SyntheticSecurityService::Pimpl { Pimpl(const MetadataStatefulPublisher& fn) : - recorder(), - timer(utils::Dispatcher::background(), 200, [&]() { recorder.do_work(); }), + recorder(utils::Dispatcher::background()), synthetics(), on_security(), meta(), on_metadata(), metadata_publisher(fn) { - timer.start(); } md::SecurityIDRecorder recorder; - utils::DispatcherTimer timer; std::unordered_map synthetics; utils::Event on_security; std::unordered_map meta; diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp index de7fd2d39..b788c6a2a 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp @@ -15,14 +15,12 @@ namespace md { MDP3Listener::MDP3Listener( CMESecurityIndex& index, MDProducer& producer, - SecurityIDRecorder& recorder, // co2::SettlementRecorder& , const std::vector channels) : _index(index), _producer(producer), _packet_arrived(), _latest_sequence(0), - _security_recorder(recorder), // _settlement_recorder(settles), _symbol_filter_storage(), _symbol_filter(), @@ -227,12 +225,7 @@ void MDP3Listener::add_security(const FixedSecurity& security) { auto handle = _producer.add_security(security); bts_assert(security.numeric_id != 0); // Add it to the index so it can receive messages - SecurityData* data = _index.add_security(security.numeric_id, handle); - - // Assign a security key - if (data->handle.key() == invalid_security_key) { - _security_recorder.ensure_key(data->handle); - } + _index.add_security(security.numeric_id, handle); } void MDP3Listener::flush_books() { diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h index b95f3ab5e..1a54c9bd0 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h @@ -31,7 +31,6 @@ class MDP3Listener : public: MDP3Listener(CMESecurityIndex& index, MDProducer&, - SecurityIDRecorder&, // co2::SettlementRecorder&, const std::vector); virtual ~MDP3Listener(); @@ -185,7 +184,6 @@ private: MDProducer& _producer; utils::TimeStamp _packet_arrived; uint32_t _latest_sequence; - SecurityIDRecorder& _security_recorder; // co2::SettlementRecorder& _settlement_recorder; std::vector _symbol_filter_storage; std::set _symbol_filter; diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp index a0e4bb3d5..679f92f3c 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp @@ -26,8 +26,7 @@ namespace bts { namespace md { MDP3Main::MDP3Main() : - _producer(PLATFORM_MDP3ONIXS), - _security_recorder(), + _producer(PLATFORM_MDP3ONIXS, { utils::Dispatcher::current() }), // _settlement_recorder(PLATFORM_MDP3ONIXS), _engines(), _handlers(), @@ -153,11 +152,6 @@ void MDP3Main::run(std::vector& symbols) { // Timers auto& disp = utils::Dispatcher::current(); - utils::DispatcherTimer security_timer(disp, 500, [&]() { - _security_recorder.do_work(); - }); - security_timer.start(); - utils::DispatcherTimer spread_resolve_timer(disp, 1000, [&]() { _index.resolve_spreads(); }); @@ -346,7 +340,7 @@ unique_ptr MDP3Main::prepare_listener(onixs::Handler& handler, con const auto& options = channel_settings_for(channel.id); - unique_ptr listener(new MDP3Listener(_index, _producer, _security_recorder, /*_settlement_recorder,*/ producer_channels)); + unique_ptr listener(new MDP3Listener(_index, _producer, /*_settlement_recorder,*/ producer_channels)); for (auto& symbol : channel.symbols) { listener->add_filter_symbol(symbol.symbol()); } diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h index 1ebfdb99b..a14825fdc 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h @@ -39,7 +39,6 @@ private: CMESecurityIndex _index; MDProducer _producer; - SecurityIDRecorder _security_recorder; // co2::SettlementRecorder _settlement_recorder; std::map> _engines; std::map> _handlers; From ffbda10200ef57eceec49e8a24d3d224101f4c8f Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 13 Nov 2018 14:14:42 -0500 Subject: [PATCH 28/29] Fix capture of transient 'this' pointer --- src/lib/md/MDProducerHandle.h | 2 +- src/lib/md/SecurityIDRecorder.cpp | 65 ++++++++++++++++++++----------- src/lib/md/SecurityIDRecorder.h | 5 --- 3 files changed, 44 insertions(+), 28 deletions(-) diff --git a/src/lib/md/MDProducerHandle.h b/src/lib/md/MDProducerHandle.h index 050dec46d..5f5e5f52b 100644 --- a/src/lib/md/MDProducerHandle.h +++ b/src/lib/md/MDProducerHandle.h @@ -118,7 +118,7 @@ private: // Private default constructor allows friend classes to store this class in a concurrent_queue MDProducerHandle(); friend class tbb::concurrent_queue < MDProducerHandle > ; - friend class SecurityIDRecorder; + friend class SecurityIDRecorderState; friend struct bts::co2::Work; HotEntry& hdata() const { diff --git a/src/lib/md/SecurityIDRecorder.cpp b/src/lib/md/SecurityIDRecorder.cpp index 8b4fc9711..192ca68e2 100644 --- a/src/lib/md/SecurityIDRecorder.cpp +++ b/src/lib/md/SecurityIDRecorder.cpp @@ -31,12 +31,31 @@ struct SecurityIDRecorderState { SecurityIDRecorderState() : work(), pending_handle(), - db(nullptr, sqlite3_close) + db(nullptr, sqlite3_close), + timer(), + id((size_t)this) {} tbb::concurrent_queue> work; tbb::concurrent_queue pending_handle; sqlite3_ptr db; unique_ptr timer; + size_t id; + + void run(utils::Dispatcher& dispatcher) { + timer = std::make_unique( + dispatcher, 500, + [=]() { do_work(); }); + timer->start(); + } + void do_work() { + work_handles(); + work_db(); + } + void work_handles(); + void work_db(); + void assign_key(const api::Security& security, const SecurityIDRecorder::Callback& key_assigned); + security_key search(const std::string& descriptor); + security_key insert(const std::string& descriptor, SecurityType); }; SecurityIDRecorder::SecurityIDRecorder(): @@ -49,10 +68,7 @@ SecurityIDRecorder::SecurityIDRecorder(utils::Dispatcher& dispatcher): _self(new SecurityIDRecorderState()) { init(); - _self->timer = std::make_unique( - dispatcher, 250, - [=]() { this->do_work(); }); - _self->timer->start(); + _self->run(dispatcher); } SecurityIDRecorder::~SecurityIDRecorder() { } @@ -77,19 +93,22 @@ void SecurityIDRecorder::init() { void SecurityIDRecorder::ensure_key(MDProducerHandle handle) { _self->pending_handle.push(handle); } + void SecurityIDRecorder::assign_key(const api::Security& security, const SecurityIDRecorder::Callback& callback) { + _self->assign_key(security, callback); +} + +void SecurityIDRecorderState::assign_key(const api::Security& security, const SecurityIDRecorder::Callback& callback) { if (security.id() == invalid_security_key) { auto work = std::make_shared(); work->security = security; work->callback = callback; - _self->work.push(work); + this->work.push(work); } } void SecurityIDRecorder::do_work() { - LOG(md, Info, "do_work"); - work_handles(); - work_db(); + _self->do_work(); } bool has_all_leg_keys(const MDProducerHandle& handle) { @@ -105,11 +124,12 @@ bool has_all_leg_keys(const MDProducerHandle& handle) { return true; } -void SecurityIDRecorder::work_handles() { +void SecurityIDRecorderState::work_handles() { Security security; MDProducerHandle handle; vector retry; - while (_self->pending_handle.try_pop(handle)) { + int count = 0; + while (pending_handle.try_pop(handle)) { security.Clear(); // if a spread does not have all of its legs we need to wait if (!has_all_leg_keys(handle)) { @@ -117,6 +137,7 @@ void SecurityIDRecorder::work_handles() { } else { convert_security(handle.security(), &security); + ++count; assign_key(security, [handle](const api::Security& security) mutable { handle.set_key(security.id()); }); @@ -124,11 +145,11 @@ void SecurityIDRecorder::work_handles() { } for(const auto& h : retry) { - _self->pending_handle.push(h); + pending_handle.push(h); } } -void SecurityIDRecorder::work_db() { +void SecurityIDRecorderState::work_db() { // Drain the work queue // Query the database for each security. If it has an ID set it // If it does not have an ID add it to a list @@ -142,7 +163,7 @@ void SecurityIDRecorder::work_db() { vector new_descriptors; size_t count = 0; string descriptor; - while (_self->work.try_pop(temp)) { + while (work.try_pop(temp)) { descriptor = security_descriptor(temp->security); // If we're unable to create a descriptor try again later @@ -170,12 +191,12 @@ void SecurityIDRecorder::work_db() { } for (auto& d : defer) { - _self->work.push(move(d)); + work.push(move(d)); } if (!new_entries.empty()) { - sqlite3_exec_nofail(_self->db.get(), "BEGIN TRANSACTION"); + sqlite3_exec_nofail(db.get(), "BEGIN TRANSACTION"); for (size_t x = 0; x < new_entries.size(); ++x) { Security& security = new_entries[x]->security; @@ -184,7 +205,7 @@ void SecurityIDRecorder::work_db() { security.set_id(key); } - sqlite3_exec_nofail(_self->db.get(), "COMMIT TRANSACTION"); + sqlite3_exec_nofail(db.get(), "COMMIT TRANSACTION"); } // record the securities @@ -228,8 +249,8 @@ void SecurityIDRecorder::bulk_insert(const std::vector& da sqlite3_exec_nofail(_self->db.get(), "COMMIT TRANSACTION"); } -security_key SecurityIDRecorder::search(const std::string& descriptor) { - auto select_stmt = prepare_query(_self->db, "SELECT rowid from security_keys where descriptor=? LIMIT 1;"); +security_key SecurityIDRecorderState::search(const std::string& descriptor) { + auto select_stmt = prepare_query(db, "SELECT rowid from security_keys where descriptor=? LIMIT 1;"); sqlite3_bind_text(select_stmt.get(), 1, descriptor.c_str(), descriptor.size(), SQLITE_STATIC); int result = sqlite3_step(select_stmt.get()); if (result == SQLITE_DONE || result == SQLITE_ROW) { @@ -238,13 +259,13 @@ security_key SecurityIDRecorder::search(const std::string& descriptor) { return invalid_security_key; } -security_key SecurityIDRecorder::insert(const std::string& descriptor, SecurityType type) { - auto insert_stmt = prepare_query(_self->db, "INSERT INTO security_keys (descriptor, type) values (?, ?)"); +security_key SecurityIDRecorderState::insert(const std::string& descriptor, SecurityType type) { + auto insert_stmt = prepare_query(db, "INSERT INTO security_keys (descriptor, type) values (?, ?)"); sqlite3_bind_text(insert_stmt.get(), 1, descriptor.c_str(), descriptor.size(), SQLITE_STATIC); sqlite3_bind_int(insert_stmt.get(), 2, (int)type); int result = sqlite3_step(insert_stmt.get()); if (result == SQLITE_DONE || result == SQLITE_ROW) { - return sqlite3_last_insert_rowid(_self->db.get()); + return sqlite3_last_insert_rowid(db.get()); } // Just in case our insert failed due to duplicate row auto key = search(descriptor); diff --git a/src/lib/md/SecurityIDRecorder.h b/src/lib/md/SecurityIDRecorder.h index e42aab2b6..f63731ff1 100644 --- a/src/lib/md/SecurityIDRecorder.h +++ b/src/lib/md/SecurityIDRecorder.h @@ -41,11 +41,6 @@ private: std::shared_ptr _self; void init(); - security_key search(const std::string& descriptor); - security_key insert(const std::string& descriptor, SecurityType); - - void work_handles(); - void work_db(); }; }}//namespace From 87218962e1a167256bfb6ad233f0a04fdeb416d9 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Tue, 13 Nov 2018 15:47:39 -0500 Subject: [PATCH 29/29] Update md publisher SDK sample - FixedSpreadSecurity --- src/sdk/examples/md_publisher_example.cpp | 29 +++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/src/sdk/examples/md_publisher_example.cpp b/src/sdk/examples/md_publisher_example.cpp index 4d09fd4d9..c81ebddb1 100644 --- a/src/sdk/examples/md_publisher_example.cpp +++ b/src/sdk/examples/md_publisher_example.cpp @@ -11,6 +11,7 @@ using namespace bts::md; void produce() { + // For Production use you should use provide a SecurityIDRecorder so security keys are generated md::MDProducer producer(co2::PLATFORM_SIM); // Create temporary security (on the stack) @@ -20,7 +21,6 @@ void produce() fsec.exchange = api::VENUE_GLBX; fsec.low_tick = 0.0078125; fsec.high_tick = 0.0078125; - fsec.cash_per_point = 0.0128; // display price is price * pow10(display_factor) fsec.display_factor = 0; fsec.expiration_year = 2018; @@ -34,11 +34,36 @@ void produce() fsec.cme_price_display_format = 3; fsec.cme_market_segment_id = 84; + md::FixedSpreadSecurityL fspread; + fspread.numeric_id = 1200; + fspread.symbol("ZF"); + fsec.security_desc("ZFU8-ZFU9"); + fsec.cfi_code("FMDXSX"); + fspread.exchange = api::VENUE_GLBX; + fspread.low_tick = 0.001; + fspread.high_tick = 0.001; + fspread.display_factor = 0; + fspread.set_leg_count(2); + for (size_t x = 0; x < 2; ++x) { + auto& leg = fspread.legs[x]; + leg.leg_price = 0.25;//optional LegPrice id=566 + leg.leg_delta = 0.1;//optional LegOptionDelta id=1017 + leg.leg_quantity = 1; // LegRatioQty id=623 + leg.leg_side = api::SIDE_BUY; // LegSide id=624 + } + std::vector producer_handles; // Add the security to the producer (this copies fsec) and get a handle to update it // This call is thread safe and may take a symbol specific lock internally - producer_handles.push_back(producer.add_security(fsec)); + MDProducerHandle future_handle = producer.add_security(fsec); + // future_handle is small and safe to copy + producer_handles.push_back(future_handle); + producer_handles.push_back(producer.add_security(fspread)); + + // The securities added though producer.add_security will not be visible to the rest of the system + // until a security key has been generated. If you supplied SecurityIDRecorder to the MDProducer constructor + // then this will happen automatically. // initialized bid side book data std::array bids {{