From 75568e6c7553e4109e26f28673fa7286389e683e Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Wed, 19 Sep 2018 18:08:03 +0000 Subject: [PATCH] Made thread allocation configurable for order management. --- src/lib/CMakeDeps.cmake | 2 +- src/lib/config/defaults/orderman_cfg.conf | 22 ++ src/lib/config/protos/orderman_cfg.proto | 8 +- src/lib/mercury/broker/AsyncEventBroker.h | 5 +- .../orderman/server/OrderServiceServer.cpp | 215 ++++++++++++------ 5 files changed, 179 insertions(+), 73 deletions(-) create mode 100644 src/lib/config/defaults/orderman_cfg.conf diff --git a/src/lib/CMakeDeps.cmake b/src/lib/CMakeDeps.cmake index 909d760bb..f3e3e8582 100644 --- a/src/lib/CMakeDeps.cmake +++ b/src/lib/CMakeDeps.cmake @@ -1 +1 @@ -set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/services/UserService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/Executor.conf;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cpp_config.conf;config/defaults/dynamic_orders.conf;config/defaults/edge.conf;config/defaults/expiration_display_offset.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cpp_config.proto;config/protos/dynamic_orders.proto;config/protos/expiration_display_offset.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPriceLantencyTester.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;mercury/actors/Actor.cpp;mercury/broker/AsyncEventBroker.cpp;mercury/broker/CoreEventBroker.cpp;mercury/broker/IEventBroker.cpp;mercury/events/TimerEvent.cpp;mercury/mercury_utils/Executor.cpp;mercury/mercury_utils/IPollableActor.cpp;mercury/mercury_utils/mercury_common.cpp;mercury/synopsis/SynopsisBuilder.cpp;mercury/topics/ITopic.cpp;metrics/metrics.cpp;metrics/system_metrics.cpp;orderman/actors/AsyncFactory.cpp;orderman/actors/ExecutionGateway.cpp;orderman/actors/FillStore.cpp;orderman/actors/OrderStore.cpp;orderman/actors/SODPosition.cpp;orderman/actors/SecuritySrc.cpp;orderman/actors/ServiceBridge.cpp;orderman/actors/order_manager/OrderBuilder.cpp;orderman/actors/order_manager/OrderManager.cpp;orderman/actors/order_manager/OrderServicesData.cpp;orderman/events/CancelRejectEvent.cpp;orderman/events/NewExecutionReportEvent.cpp;orderman/limiter/LimitStore.cpp;orderman/limiter/Limiter.cpp;orderman/limiter/ProductLimitNode.cpp;orderman/limiter/ProductLimitTree.cpp;orderman/limiter/SecurityLimitNode.cpp;orderman/limiter/SecurityLimitTree.cpp;orderman/limiter/tests/limit_tests.cpp;orderman/order_factory/OrderFactory.cpp;orderman/order_router/OrderRouter.cpp;orderman/orderman_utils/fix_order_codec.cpp;orderman/orders/DynamicOrder.cpp;orderman/orders/IcebergOrder.cpp;orderman/orders/NativeOrder.cpp;orderman/orders/Order.cpp;orderman/orders/TimeBoxOrder.cpp;orderman/orders/TriggerOrder.cpp;orderman/orders/book_algos/BookAlgo.cpp;orderman/orders/book_algos/CoverOrder.cpp;orderman/orders/book_algos/JoinOrder.cpp;orderman/orders/book_algos/LeanOrder.cpp;orderman/orders/book_algos/TickOrder.cpp;orderman/orders/book_algos/TopOrder.cpp;orderman/orders/spreader_orders/SpreaderOrder.cpp;orderman/orders/spreader_orders/spreader/LegPropertyData.cpp;orderman/orders/spreader_orders/spreader/QuoteLegPriceQuantityState.cpp;orderman/orders/spreader_orders/spreader/SingleFillHedger.cpp;orderman/orders/spreader_orders/spreader/SingleLegQuoter.cpp;orderman/orders/spreader_orders/spreader/SpreaderBookkeepingData.cpp;orderman/orders/spreader_orders/spreader/SpreaderChildOrderState.cpp;orderman/orders/spreader_orders/spreader/SpreaderLegData.cpp;orderman/orders/spreader_orders/spreader/SpreaderUtils.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderBuilder.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderServices.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderPreActivationOrderServices.cpp;orderman/orders/spreader_orders/spreader/hedger/UnhedgedDelta.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckCancelledStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWorkingStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSet.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSetHistory.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteLegDriver.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckCancelledWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForIOCWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForSizeStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWorkingStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderEndStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderStartStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWaitingForCancelAllStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWorkingStateWorker.cpp;orderman/orders/spreader_orders/tests/spreader_tests.cpp;orderman/server/OrderServiceServer.cpp;orderman/tests/fat_wrapper_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;persistence/IStore.cpp;persistence/PersistenceServiceServer.cpp;persistence/SqliteStore.cpp;persistence/tests/PersistenceServiceTests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;utils/Dispatcher.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp) \ No newline at end of file +set(sources api/apicore.proto;api/apigateway.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/service_helpers/AdminServiceUtils.cpp;api/services/AdminService.proto;api/services/ApiUnitTestService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/PositionService.proto;api/services/ShellDiagnosticService.proto;api/services/ShellLatencyService.proto;api/services/SyntheticSecurityService.proto;api/services/UserService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/Executor.conf;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/cpp_config.conf;config/defaults/dynamic_orders.conf;config/defaults/edge.conf;config/defaults/expiration_display_offset.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/orderman_cfg.conf;config/defaults/orders_fills_cleanup.conf;config/defaults/spreader.conf;config/defaults/testkeys.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/ExchangeSimCfg.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/cpp_config.proto;config/protos/dynamic_orders.proto;config/protos/expiration_display_offset.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/orderman_cfg.proto;config/protos/orders_fills_cleanup.proto;config/protos/quickbus_cfg.proto;config/protos/spreader.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;domain_model/SecurityTests.cpp;domain_model/tests/domain_tests.cpp;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/sample_fix.cpp;jetfix/tests/store_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPriceLantencyTester.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/cme/MDP3Utils.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2ManagedTestHelper.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;mercury/actors/Actor.cpp;mercury/broker/AsyncEventBroker.cpp;mercury/broker/CoreEventBroker.cpp;mercury/broker/IEventBroker.cpp;mercury/events/TimerEvent.cpp;mercury/mercury_utils/Executor.cpp;mercury/mercury_utils/IPollableActor.cpp;mercury/mercury_utils/mercury_common.cpp;mercury/synopsis/SynopsisBuilder.cpp;mercury/topics/ITopic.cpp;metrics/metrics.cpp;metrics/system_metrics.cpp;orderman/actors/AsyncFactory.cpp;orderman/actors/ExecutionGateway.cpp;orderman/actors/FillStore.cpp;orderman/actors/OrderStore.cpp;orderman/actors/SODPosition.cpp;orderman/actors/SecuritySrc.cpp;orderman/actors/ServiceBridge.cpp;orderman/actors/order_manager/OrderBuilder.cpp;orderman/actors/order_manager/OrderManager.cpp;orderman/actors/order_manager/OrderServicesData.cpp;orderman/events/CancelRejectEvent.cpp;orderman/events/NewExecutionReportEvent.cpp;orderman/limiter/LimitStore.cpp;orderman/limiter/Limiter.cpp;orderman/limiter/ProductLimitNode.cpp;orderman/limiter/ProductLimitTree.cpp;orderman/limiter/SecurityLimitNode.cpp;orderman/limiter/SecurityLimitTree.cpp;orderman/limiter/tests/limit_tests.cpp;orderman/order_factory/OrderFactory.cpp;orderman/order_router/OrderRouter.cpp;orderman/orderman_utils/fix_order_codec.cpp;orderman/orders/DynamicOrder.cpp;orderman/orders/IcebergOrder.cpp;orderman/orders/NativeOrder.cpp;orderman/orders/Order.cpp;orderman/orders/TimeBoxOrder.cpp;orderman/orders/TriggerOrder.cpp;orderman/orders/book_algos/BookAlgo.cpp;orderman/orders/book_algos/CoverOrder.cpp;orderman/orders/book_algos/JoinOrder.cpp;orderman/orders/book_algos/LeanOrder.cpp;orderman/orders/book_algos/TickOrder.cpp;orderman/orders/book_algos/TopOrder.cpp;orderman/orders/spreader_orders/SpreaderOrder.cpp;orderman/orders/spreader_orders/spreader/LegPropertyData.cpp;orderman/orders/spreader_orders/spreader/QuoteLegPriceQuantityState.cpp;orderman/orders/spreader_orders/spreader/SingleFillHedger.cpp;orderman/orders/spreader_orders/spreader/SingleLegQuoter.cpp;orderman/orders/spreader_orders/spreader/SpreaderBookkeepingData.cpp;orderman/orders/spreader_orders/spreader/SpreaderChildOrderState.cpp;orderman/orders/spreader_orders/spreader/SpreaderLegData.cpp;orderman/orders/spreader_orders/spreader/SpreaderUtils.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderBuilder.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderOrderServices.cpp;orderman/orders/spreader_orders/spreader/external_interfacing/SpreaderPreActivationOrderServices.cpp;orderman/orders/spreader_orders/spreader/hedger/UnhedgedDelta.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckCancelledStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWaitingForFirstAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/hedger/hedger_states/HedgerWorkingStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSet.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteDriverSetHistory.cpp;orderman/orders/spreader_orders/spreader/quoter/QuoteLegDriver.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterEndStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterStartStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForCancelAckStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckCancelledWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForFirstAckWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForIOCWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWaitingForSizeStateWorker.cpp;orderman/orders/spreader_orders/spreader/quoter/quoter_states/QuoterWorkingStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderEndStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderStartStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWaitingForCancelAllStateWorker.cpp;orderman/orders/spreader_orders/spreader_states/SpreaderWorkingStateWorker.cpp;orderman/orders/spreader_orders/tests/spreader_tests.cpp;orderman/server/OrderServiceServer.cpp;orderman/tests/fat_wrapper_tests.cpp;os/os.cpp;os/tests/os_tests.cpp;persistence/IStore.cpp;persistence/PersistenceServiceServer.cpp;persistence/SqliteStore.cpp;persistence/tests/PersistenceServiceTests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/catch_test_runner.cpp;utils/Dispatcher.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SecureLogon.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp) \ No newline at end of file diff --git a/src/lib/config/defaults/orderman_cfg.conf b/src/lib/config/defaults/orderman_cfg.conf new file mode 100644 index 000000000..ece1cf334 --- /dev/null +++ b/src/lib/config/defaults/orderman_cfg.conf @@ -0,0 +1,22 @@ +OrdermanCfg { + value { + thread { + name: "orderman" + busy_poll: true + core_affinity: 0 + hosted_actor: "MDSource" + hosted_actor: "OrderManager" + } + thread { + name: "gateway" + busy_poll: true + core_affinity: 1 + hosted_actor: "ExecutionGateway" + } + thread { + name: "slow" + busy_poll: false + hosted_actor: "*" + } + } +} \ No newline at end of file diff --git a/src/lib/config/protos/orderman_cfg.proto b/src/lib/config/protos/orderman_cfg.proto index d9ed4ad75..7dcdea246 100644 --- a/src/lib/config/protos/orderman_cfg.proto +++ b/src/lib/config/protos/orderman_cfg.proto @@ -2,5 +2,11 @@ syntax='proto2'; package bts.config; message OrdermanCfg { - optional int32 fast_thread_affinity = 1 [default = -1]; // -1 ==> no affinity + message OrdermanThread { + required string name = 1; + optional bool busy_poll = 2 [default = false]; + optional int32 core_affinity = 3 [default = -1]; + repeated string hosted_actor = 4; + } + repeated OrdermanThread thread = 1; } \ No newline at end of file diff --git a/src/lib/mercury/broker/AsyncEventBroker.h b/src/lib/mercury/broker/AsyncEventBroker.h index 97ec3debc..a5638ac75 100644 --- a/src/lib/mercury/broker/AsyncEventBroker.h +++ b/src/lib/mercury/broker/AsyncEventBroker.h @@ -29,9 +29,10 @@ class AsyncEventBroker : public IEventBroker { _external_pool_mtx(), _current_suid(0) {} void - init_neighbours(std::initializer_list const &neighbours) { + init_neighbours(std::vector const &neighbours) { for (auto const &n : neighbours) - _core_broker.add_neighbour(n); + if (_core_broker.id() != n->id()) + _core_broker.add_neighbour(n); } // ieventbroker methods, can be called from multiple threads diff --git a/src/lib/orderman/server/OrderServiceServer.cpp b/src/lib/orderman/server/OrderServiceServer.cpp index d561888ed..27b85caf6 100644 --- a/src/lib/orderman/server/OrderServiceServer.cpp +++ b/src/lib/orderman/server/OrderServiceServer.cpp @@ -18,29 +18,96 @@ #include "lib/orderman/actors/ServiceBridge.h" #include "lib/orderman/actors/order_manager/OrderManager.h" #include "lib/utils/bts_chrono.h" +#include "lib/utils/type_name.h" + +#include namespace bts { namespace orderman { +struct ActorThread { + ActorThread(const config::OrdermanCfg::OrdermanThread &cfg,metrics::MetricsContainer &stats) + : cfg(cfg), thread(), event_broker(new mercury::AsyncEventBroker()), + executor(*event_broker), loop_metric(), done(false) { + std::array buckets; + auto i = 0; + for (auto &b : buckets) + b = (i += 1000); + + loop_metric = std::make_shared>( + fmt::format("spark_metrics_{}_thread_loop_ns", cfg.name()), buckets); + stats.publish(loop_metric); + } + + void start() { + thread = std::thread([this]() { + os::set_thread_name(cfg.name()); + + const auto inner_iter = 1000; + while (!done) { + auto start = Timer::now(); + for (auto i = 0; i < inner_iter; i++) + executor.do_work(); + auto stop = Timer::now(); + loop_metric->observe(ns_diff(stop, start) / inner_iter); + if (!cfg.busy_poll()) + std::this_thread::sleep_for(10ms); + } + }); + + if (cfg.core_affinity() >= 0) { + LOG(OrderMan, Info, "Setting {} core affinity to {}", cfg.name(), + cfg.core_affinity()); + cpu_set_t cpuset; + CPU_ZERO(&cpuset); + CPU_SET(cfg.core_affinity(), &cpuset); + int rc = pthread_setaffinity_np(thread.native_handle(), + sizeof(cpu_set_t), &cpuset); + if (rc != 0) + LOG(OrderMan, Error, + "Could not set core affinity to {} for {}. Error " + "code={}", + cfg.core_affinity(), cfg.name(), rc); + } + } + + ~ActorThread() { + done= true; + thread.join(); + } + + config::OrdermanCfg::OrdermanThread cfg; + std::thread thread; + std::shared_ptr event_broker; + mercury::Executor executor; + std::shared_ptr> loop_metric; + std::atomic done; +}; + +ActorThread *find_thread_for_actor( + std::string const &actor_name, + std::vector> const &all_threads) { + for (auto const &at : all_threads) + for (auto const &hosted : at->cfg.hosted_actor()) + if (hosted == "*" || boost::algorithm::to_lower_copy(hosted) == + boost::algorithm::to_lower_copy(actor_name)) + return at.get(); + return nullptr; +} + struct OrderServiceServer::PImpl { PImpl(bts::api::ServiceContext &ctx, std::shared_ptr &ss, metrics::MetricsContainer &stats) - : fast_broker(std::make_shared()), - slow_broker(std::make_shared()), - service_bridge(), fast_exe(*fast_broker), slow_exe(*slow_broker), - fast_thread(), slow_thread(), service_ready(false), known_orders(), - known_fills(), publish_orders(), publish_fills(), + : service_bridge(), service_ready(false), known_orders(), known_fills(), + publish_orders(), publish_fills(), admin_service(ctx.locate()), md_consumer(ss), stats(stats) {} - std::shared_ptr fast_broker; - std::shared_ptr slow_broker; std::shared_ptr service_bridge; - mercury::Executor fast_exe; - mercury::Executor slow_exe; - std::thread fast_thread; - std::thread slow_thread; + + std::vector> actor_threads; + std::atomic service_ready; std::unordered_set known_orders; std::unordered_set known_fills; @@ -72,66 +139,76 @@ OrderServiceServer::OrderServiceServer( ctx.post([this, fill] { this->on_fills(fill); }); }); - me->fast_broker->init_neighbours({me->slow_broker}); - me->slow_broker->init_neighbours({me->fast_broker}); - - // fast actors - me->fast_exe.spawn(me->md_consumer, me->stats); - me->fast_exe.spawn(); - - auto &all_connections = config::get().connection(); - for (auto const &c : all_connections) - me->fast_exe.spawn(c, stats); - - me->fast_thread = std::thread([this]() { - os::set_thread_name("fast_exe"); - - std::array buckets; - auto i = 1; - for (auto &b : buckets) - b = (i *= 2); - - auto loop_timer = std::make_shared>( - "spark_metrics_fast_thread_loop_ns", buckets); - - me->stats.publish(loop_timer); - for (;;) { - auto start = Timer::now(); - me->fast_exe.do_work(); - auto stop = Timer::now(); - loop_timer->observe(ns_diff(stop, start)); - } - }); - - auto fast_thread_core = - config::get().fast_thread_affinity(); - if (fast_thread_core >= 0) { - LOG(OrderMan, Info, "Setting fast thread core affinity to {}", - fast_thread_core); - cpu_set_t cpuset; - CPU_ZERO(&cpuset); - CPU_SET(fast_thread_core, &cpuset); - int rc = pthread_setaffinity_np(me->fast_thread.native_handle(), - sizeof(cpu_set_t), &cpuset); - if (rc != 0) - LOG(OrderMan, Error, - "Could not set core affinity to {} for fast thread. Error code={}", - fast_thread_core, rc); + std::vector all_brokers; + for (auto const &thread_cfg : config::get().thread()) { + auto at = std::make_unique(thread_cfg,stats); + all_brokers.push_back(at->event_broker); + me->actor_threads.push_back(std::move(at)); } - // slow actors - me->slow_exe.activate(me->service_bridge); - me->slow_exe.spawn(); - me->slow_exe.spawn(); - me->slow_exe.spawn(); - me->slow_exe.spawn(); - me->slow_exe.spawn(me->md_consumer); - me->slow_thread = std::thread([this]() { - os::set_thread_name("slow_exe"); - for (;;) - if (!me->slow_exe.do_work()) - std::this_thread::sleep_for(10ms); - }); + for (auto &t : me->actor_threads) + t->event_broker->init_neighbours(all_brokers); + + // MDSource + auto thread = find_thread_for_actor("MDSource", me->actor_threads); + bts_assert_msg(thread, "Could not find a configured thread for MDSource. See the " + "OrdermanCfg key"); + thread->executor.spawn(me->md_consumer, me->stats); + + // OrderManager + thread = find_thread_for_actor("OrderManager", me->actor_threads); + bts_assert_msg(thread, "Could not find a configured thread for OrderManager. See " + "the OrdermanCfg key"); + thread->executor.spawn(); + + // ExecutionGateway's + thread = find_thread_for_actor("ExecutionGateway", me->actor_threads); + bts_assert_msg(thread, "Could not find a configured thread for ExecutionGateway " + "instances. See the OrdermanCfg key"); + auto &all_connections = config::get().connection(); + for (auto const &c : all_connections) + thread->executor.spawn(c, stats); + + // Timer + thread = find_thread_for_actor("Timer", me->actor_threads); + bts_assert_msg( + thread, + "Could not find a configured thread for Timer. See the OrdermanCfg key"); + thread->executor.spawn(); + + // AsyncFactory + thread = find_thread_for_actor("AsyncFactory", me->actor_threads); + bts_assert_msg(thread, "Could not find a configured thread for AsyncFactory. See " + "the OrdermanCfg key"); + thread->executor.spawn(); + + // OrderStore + thread = find_thread_for_actor("OrderStore", me->actor_threads); + bts_assert_msg(thread, "Could not find a configured thread for OrderStore. See " + "the OrdermanCfg key"); + thread->executor.spawn(); + + // FillStore + thread = find_thread_for_actor("FillStore", me->actor_threads); + bts_assert_msg(thread, "Could not find a configured thread for FilllStore. See " + "the OrdermanCfg key"); + thread->executor.spawn(); + + // SecuritySrc + thread = find_thread_for_actor("SecuritySrc", me->actor_threads); + bts_assert_msg(thread, "Could not find a configured thread for SecuritySrc. See " + "the OrdermanCfg key"); + thread->executor.spawn(me->md_consumer); + + // ServiceBridge + thread = find_thread_for_actor("ServiceBridge", me->actor_threads); + bts_assert_msg(thread, "Could not find a configured thread for ServiceBridge. See " + "the OrdermanCfg key"); + thread->executor.activate(me->service_bridge); + + // spin up threads + for (auto &t : me->actor_threads) + t->start(); // wait for the service bridge to be ready me->service_bridge->block_until_ready();