From 950e7b7e2d72cb3b467d2b679406a86a52530fff Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Mon, 7 Aug 2017 17:21:12 -0400 Subject: [PATCH] Init --- .gitignore | 2 + CMakeLists.txt | 25 + src/client/tools/CMakeLists.txt | 30 + src/client/tools/main.cpp | 4 + src/lib/CMakeLists.txt | 193 +++ src/lib/api/apicore.proto | 37 + src/lib/api/apioptions.proto | 15 + src/lib/api/core/ApiClientInterop.cpp | 212 +++ src/lib/api/core/ApiClientInterop.h | 25 + src/lib/api/core/ApiInteropCommon.h | 32 + src/lib/api/core/ApiServerInterop.cpp | 82 + src/lib/api/core/ApiServerInterop.h | 19 + src/lib/api/public/ApiException.h | 20 + src/lib/api/public/ApiServer.cpp | 14 + src/lib/api/public/ApiServer.h | 65 + src/lib/api/public/ApiUtils.h | 28 + src/lib/api/public/ClientChannel.cpp | 13 + src/lib/api/public/ClientChannel.h | 117 ++ src/lib/api/public/QuickbusClientChannel.cpp | 48 + src/lib/api/public/QuickbusClientChannel.h | 26 + src/lib/api/public/RawUpdate.h | 32 + src/lib/api/public/Result.h | 75 + src/lib/api/public/ServerChannel.cpp | 63 + src/lib/api/public/ServerChannel.h | 27 + src/lib/api/public/ServiceContext.cpp | 29 + src/lib/api/public/ServiceContext.h | 37 + src/lib/api/public/Update.h | 79 + src/lib/api/public/api.h | 2 + .../api/service_compiler/apioptions_pb2.py | 74 + .../api/service_compiler/apioptions_pb2.pyc | Bin 0 -> 3133 bytes src/lib/api/service_compiler/plugin.cmd | 3 + src/lib/api/service_compiler/plugin.py | 226 +++ .../templates/c#/client.pb.cs | 44 + .../templates/c#/immutable.pb.cs | 36 + .../templates/c#/server.pb.cs | 172 ++ .../templates/cpp/client.pb.cc | 67 + .../templates/cpp/client.pb.h | 58 + .../templates/cpp/server.pb.cc | 167 ++ .../templates/cpp/server.pb.h | 56 + src/lib/api/services/Hello.proto | 34 + src/lib/api/test/ApiTests.cpp | 90 + src/lib/api/test/HelloServiceServer.cpp | 66 + src/lib/api/test/HelloServiceServer.h | 39 + src/lib/config/ConfigDownloader.cpp | 82 + src/lib/config/ConfigDownloader.h | 32 + src/lib/config/ConfigErrorCollector.cpp | 72 + src/lib/config/ConfigErrorCollector.h | 53 + src/lib/config/ConfigLog.cpp | 105 ++ src/lib/config/ConfigLog.h | 72 + src/lib/config/ConfigPropertyEvaluator.cpp | 296 ++++ src/lib/config/ConfigPropertyEvaluator.h | 31 + src/lib/config/ConfigValueQueue.cpp | 294 ++++ src/lib/config/ConfigValueQueue.h | 101 ++ src/lib/config/ConfigurationException.cpp | 37 + src/lib/config/ConfigurationException.h | 75 + src/lib/config/ConfigurationState.cpp | 944 ++++++++++ src/lib/config/ConfigurationState.h | 123 ++ src/lib/config/PreprocessInputStream.cpp | 70 + src/lib/config/PreprocessInputStream.h | 34 + src/lib/config/config.cpp | 129 ++ src/lib/config/config.h | 70 + src/lib/config/configure_proto_files.h.in | 3 + src/lib/config/defaults/edge.conf | 4 + src/lib/config/defaults/logging.conf | 10 + src/lib/config/defaults/testkeys.conf | 6 + src/lib/config/doc/README.txt | 40 + src/lib/config/extensions/extensions.proto | 15 + src/lib/config/invalid/invalid.proto | 10 + src/lib/config/protos/MarketData.proto | 248 +++ src/lib/config/protos/Ports.proto | 14 + src/lib/config/protos/Securities.proto | 13 + src/lib/config/protos/Symbols.proto | 13 + src/lib/config/protos/jetfix.proto | 89 + src/lib/config/protos/logging.proto | 19 + src/lib/config/protos/quickbus_cfg.proto | 20 + src/lib/config/protos/system.proto | 38 + src/lib/config/protos/testkeys.proto | 84 + src/lib/config/tests/HTTPServer.cpp | 104 ++ src/lib/config/tests/HTTPServer.h | 53 + src/lib/config/tests/config_ref_tests.cpp | 932 ++++++++++ src/lib/config/tests/config_tests.cpp | 578 +++++++ src/lib/config/text_format/strutil.h | 878 ++++++++++ src/lib/config/text_format/text_format.cpp | 1386 +++++++++++++++ src/lib/config/text_format/text_format.h | 320 ++++ src/lib/expression/BooleanGrammar.cpp | 28 + src/lib/expression/BooleanGrammar.h | 276 +++ src/lib/expression/ExpressionEvaluator.cpp | 478 +++++ src/lib/expression/ExpressionEvaluator.h | 39 + src/lib/expression/IPropertyEvaluator.h | 23 + src/lib/expression/ParserExceptions.cpp | 87 + src/lib/expression/ParserExceptions.h | 55 + src/lib/expression/RecursionGuard.h | 91 + src/lib/expression/Value.cpp | 94 + src/lib/expression/Value.h | 44 + .../tests/expression_parser_tests.cpp | 179 ++ src/lib/logging/Common.h | 29 + src/lib/logging/LogRec.h | 42 + src/lib/logging/Logger.h | 134 ++ src/lib/logging/LoggerHub.cpp | 201 +++ src/lib/logging/LoggerHub.h | 35 + src/lib/logging/LoggingConfig.h | 26 + src/lib/logging/MsgPacking.h | 45 + src/lib/logging/Sink.cpp | 86 + src/lib/logging/Sink.h | 31 + src/lib/logging/logging.cpp | 140 ++ src/lib/logging/logging.h | 57 + src/lib/logging/tests/logging_tests.cpp | 78 + src/lib/os/Win32Exception.cpp | 52 + src/lib/os/Win32Exception.h | 32 + src/lib/os/os.cpp | 1095 ++++++++++++ src/lib/os/os.h | 86 + src/lib/os/tests/os_tests.cpp | 37 + src/lib/quickbus/core/Advertisement.h | 18 + src/lib/quickbus/core/LocalNode.cpp | 121 ++ src/lib/quickbus/core/LocalNode.h | 58 + src/lib/quickbus/core/Quickbus.cpp | 55 + src/lib/quickbus/core/Quickbus.h | 51 + src/lib/quickbus/core/SubscriptionID.cpp | 6 + src/lib/quickbus/core/SubscriptionID.h | 12 + src/lib/quickbus/core/TableID.cpp | 11 + src/lib/quickbus/core/TableID.h | 22 + src/lib/quickbus/core/Topic.h | 10 + src/lib/quickbus/message_model/Change.h | 30 + src/lib/quickbus/message_model/Record.h | 10 + src/lib/quickbus/message_model/RecordKey.h | 21 + src/lib/quickbus/message_model/RecordValue.h | 0 src/lib/quickbus/message_model/Update.cpp | 6 + src/lib/quickbus/message_model/Update.h | 10 + .../quickbus/message_model/merge_updates.cpp | 42 + .../quickbus/message_model/merge_updates.h | 16 + src/lib/quickbus/network/QBConnection.cpp | 446 +++++ src/lib/quickbus/network/QBConnection.h | 58 + .../quickbus/network/QBConnectionManager.cpp | 126 ++ .../quickbus/network/QBConnectionManager.h | 33 + .../quickbus/network/endpoint_description.cpp | 36 + .../quickbus/network/endpoint_description.h | 10 + src/lib/quickbus/network/quickbus_msgs.proto | 89 + src/lib/quickbus/pubsub/TableEvent.h | 27 + src/lib/quickbus/pubsub/TableFilter.h | 18 + src/lib/quickbus/pubsub/TablePublisher.cpp | 138 ++ src/lib/quickbus/pubsub/TablePublisher.h | 41 + src/lib/quickbus/pubsub/TableSubscriber.cpp | 147 ++ src/lib/quickbus/pubsub/TableSubscriber.h | 39 + .../quickbus/pubsub/tests/qb_pubsub_tests.cpp | 231 +++ src/lib/quickbus/quickbus.h | 11 + src/lib/quickbus/routing/INode.cpp | 8 + src/lib/quickbus/routing/INode.h | 27 + src/lib/quickbus/routing/INodePtr.h | 9 + src/lib/quickbus/routing/NodeID.cpp | 11 + src/lib/quickbus/routing/NodeID.h | 11 + src/lib/quickbus/routing/Router.cpp | 367 ++++ src/lib/quickbus/routing/Router.h | 38 + src/lib/quickbus/rpc/OnRPCReply.h | 11 + src/lib/quickbus/rpc/OnRPCRequest.h | 14 + src/lib/quickbus/rpc/RPCClient.cpp | 319 ++++ src/lib/quickbus/rpc/RPCClient.h | 40 + src/lib/quickbus/rpc/RPCError.h | 10 + src/lib/quickbus/rpc/RPCReply.h | 10 + src/lib/quickbus/rpc/RPCRequest.h | 10 + src/lib/quickbus/rpc/RPCServer.cpp | 175 ++ src/lib/quickbus/rpc/RPCServer.h | 31 + src/lib/quickbus/rpc/RPCServiceName.h | 13 + src/lib/quickbus/rpc/RPCSessionID.cpp | 9 + src/lib/quickbus/rpc/RPCSessionID.h | 23 + src/lib/quickbus/rpc/rpc_utils.cpp | 34 + src/lib/quickbus/rpc/rpc_utils.h | 18 + src/lib/quickbus/rpc/tests/qb_rpc_tests.cpp | 272 +++ src/lib/quickbus/tests/quickbus_tests.cpp | 244 +++ src/lib/security_model/SecurityHelper.cpp | 435 +++++ src/lib/security_model/SecurityHelper.h | 63 + src/lib/security_model/security.proto | 172 ++ src/lib/utils/CallTimer.h | 78 + src/lib/utils/Dispatcher.cpp | 299 ++++ src/lib/utils/Dispatcher.h | 116 ++ src/lib/utils/HTTPStream.cpp | 100 ++ src/lib/utils/HTTPStream.h | 30 + src/lib/utils/PBUtils.cpp | 17 + src/lib/utils/PBUtils.h | 17 + src/lib/utils/SharedList.h | 309 ++++ src/lib/utils/SharedMemory.cpp | 201 +++ src/lib/utils/SharedMemory.h | 30 + src/lib/utils/Sqlite_utils.cpp | 35 + src/lib/utils/Sqlite_utils.h | 24 + src/lib/utils/TempDir.h | 60 + src/lib/utils/TempFile.h | 84 + src/lib/utils/TimeStamp.h | 170 ++ src/lib/utils/Uuid.cpp | 98 ++ src/lib/utils/Uuid.h | 29 + src/lib/utils/ValueWrapper.h | 31 + src/lib/utils/WaitFreeQueue.h | 178 ++ src/lib/utils/benchmarking_utils.h | 23 + src/lib/utils/bts_array_size.h | 6 + src/lib/utils/bts_assert.h | 23 + src/lib/utils/bts_chrono.h | 37 + src/lib/utils/bts_debugger.h | 16 + src/lib/utils/bts_export.h | 5 + src/lib/utils/bts_no_copy.h | 10 + src/lib/utils/catch_test_runner.cpp | 20 + src/lib/utils/catch_test_runner.h | 5 + src/lib/utils/checked_cast.h | 14 + src/lib/utils/datetime.cpp | 427 +++++ src/lib/utils/datetime.h | 84 + src/lib/utils/defer.h | 38 + src/lib/utils/file_utils.cpp | 25 + src/lib/utils/file_utils.h | 9 + src/lib/utils/helper_macros.h | 4 + src/lib/utils/is_unittest_environment.cpp | 23 + src/lib/utils/is_unittest_environment.h | 8 + src/lib/utils/meta_programming.h | 163 ++ src/lib/utils/mpmc_bounded_queue.h | 166 ++ src/lib/utils/proto_helper.h | 250 +++ src/lib/utils/type_name.h | 17 + src/lib/utils/warn_ctrl.h | 19 + src/lib/utils/windows_wrapper.h | 11 + src/server/sparkd/CMakeLists.txt | 92 + src/server/sparkd/interconnect/Actor.cpp | 10 + src/server/sparkd/interconnect/Actor.h | 282 +++ src/server/sparkd/interconnect/Domain.cpp | 192 +++ src/server/sparkd/interconnect/Domain.h | 30 + src/server/sparkd/interconnect/Executor.cpp | 326 ++++ src/server/sparkd/interconnect/Executor.h | 50 + src/server/sparkd/interconnect/IActor.cpp | 5 + src/server/sparkd/interconnect/IActor.h | 25 + src/server/sparkd/interconnect/IDomain.cpp | 5 + src/server/sparkd/interconnect/IDomain.h | 23 + src/server/sparkd/interconnect/IPollable.h | 7 + src/server/sparkd/interconnect/Msg.h | 151 ++ src/server/sparkd/interconnect/RPC.h | 53 + src/server/sparkd/interconnect/Topic.h | 18 + .../sparkd/interconnect/TopicDirectory.cpp | 5 + .../sparkd/interconnect/TopicDirectory.h | 35 + .../tests/baseline_perf_interconnect_test.cpp | 432 +++++ .../interconnect/tests/interconnect_test.cpp | 168 ++ src/server/sparkd/jetfix/codec/FIXArray.cpp | 1 + src/server/sparkd/jetfix/codec/FIXArray.h | 31 + src/server/sparkd/jetfix/codec/FIXBuffer.h | 145 ++ .../sparkd/jetfix/codec/FIXEncodingStream.h | 224 +++ src/server/sparkd/jetfix/codec/FIXFragment.h | 201 +++ src/server/sparkd/jetfix/codec/FIXMsg.h | 104 ++ src/server/sparkd/jetfix/codec/FIXParser.h | 63 + src/server/sparkd/jetfix/codec/FIXWriter.h | 20 + .../sparkd/jetfix/codec/FieldDefinitions.h | 129 ++ src/server/sparkd/jetfix/codec/Fields.h | 209 +++ src/server/sparkd/jetfix/codec/Group.h | 135 ++ src/server/sparkd/jetfix/codec/IFIXMsg.h | 16 + src/server/sparkd/jetfix/codec/IFIXParser.h | 18 + src/server/sparkd/jetfix/codec/IFieldReader.h | 18 + .../sparkd/jetfix/codec/MessageDefinitions.h | 46 + src/server/sparkd/jetfix/codec/MessageTypes.h | 54 + src/server/sparkd/jetfix/codec/ReaderCursor.h | 72 + src/server/sparkd/jetfix/codec/WriterCursor.h | 163 ++ src/server/sparkd/jetfix/codec/f_toa.h | 42 + src/server/sparkd/jetfix/codec/i32_toa.h | 218 +++ src/server/sparkd/jetfix/codec/sum_bytes.h | 16 + src/server/sparkd/jetfix/common/InMsgBuffer.h | 172 ++ src/server/sparkd/jetfix/common/JetFIXError.h | 98 ++ src/server/sparkd/jetfix/common/OutFIX.cpp | 8 + src/server/sparkd/jetfix/common/OutFIX.h | 69 + src/server/sparkd/jetfix/common/OutFIXPtr.h | 9 + .../sparkd/jetfix/common/OutMsgVecPtr.h | 13 + .../sparkd/jetfix/connection/AsyncReader.cpp | 162 ++ .../sparkd/jetfix/connection/AsyncReader.h | 30 + .../sparkd/jetfix/connection/IConnection.cpp | 3 + .../sparkd/jetfix/connection/IConnection.h | 20 + .../jetfix/connection/IConnectionObserver.h | 22 + .../jetfix/connection/SocketConnection.cpp | 127 ++ .../jetfix/connection/SocketConnection.h | 29 + .../sparkd/jetfix/gateway/CoreGateway.cpp | 537 ++++++ .../sparkd/jetfix/gateway/CoreGateway.h | 67 + .../jetfix/gateway/ICoreGatewayEventSink.h | 21 + .../jetfix/gateway/ICoreGatewayObserver.h | 19 + .../sparkd/jetfix/gateway/JetFIXGateway.cpp | 9 + .../sparkd/jetfix/gateway/JetFIXGateway.h | 116 ++ src/server/sparkd/jetfix/gateway/Session.cpp | 802 +++++++++ src/server/sparkd/jetfix/gateway/Session.h | 78 + src/server/sparkd/jetfix/jetfix.h | 5 + src/server/sparkd/jetfix/store/FileStore.cpp | 326 ++++ src/server/sparkd/jetfix/store/FileStore.h | 31 + src/server/sparkd/jetfix/store/IStore.h | 27 + .../sparkd/jetfix/store/session_name.cpp | 140 ++ src/server/sparkd/jetfix/store/session_name.h | 49 + .../jetfix/tests/async_reader_tests.cpp | 158 ++ .../sparkd/jetfix/tests/decoding_tests.cpp | 161 ++ .../tests/fix_encoding_stream_tests.cpp | 105 ++ .../jetfix/tests/fix_fragment_tests.cpp | 199 +++ .../jetfix/tests/jetfix_performance_tests.cpp | 261 +++ .../sparkd/jetfix/tests/store_tests.cpp | 283 +++ src/server/sparkd/main.cpp | 20 + .../sparkd/md/Diag/MDPriceLantencyTester.cpp | 69 + .../sparkd/md/Diag/MDPriceLantencyTester.h | 15 + src/server/sparkd/md/Diag/ShmDumper.cpp | 248 +++ src/server/sparkd/md/Diag/ShmDumper.h | 24 + src/server/sparkd/md/MDConsumer.cpp | 681 ++++++++ src/server/sparkd/md/MDConsumer.h | 98 ++ src/server/sparkd/md/MDConsumerHandle.cpp | 107 ++ src/server/sparkd/md/MDConsumerHandle.h | 96 ++ src/server/sparkd/md/MDConsumerInterop.cpp | 189 ++ src/server/sparkd/md/MDProducer.cpp | 157 ++ src/server/sparkd/md/MDProducer.h | 48 + src/server/sparkd/md/MDProducerChannel.h | 23 + src/server/sparkd/md/MDProducerHandle.cpp | 311 ++++ src/server/sparkd/md/MDProducerHandle.h | 85 + .../sparkd/md/MDRecoverableConsumer.cpp | 155 ++ src/server/sparkd/md/MDRecoverableConsumer.h | 63 + src/server/sparkd/md/MemorySegmentUtils.cpp | 27 + src/server/sparkd/md/MemorySegmentUtils.h | 14 + src/server/sparkd/md/PlatformLoader.cpp | 125 ++ src/server/sparkd/md/PlatformLoader.h | 39 + src/server/sparkd/md/SecurityConstructor.h | 56 + src/server/sparkd/md/SecurityIDRecorder.cpp | 198 +++ src/server/sparkd/md/SecurityIDRecorder.h | 56 + src/server/sparkd/md/SecurityIndex.h | 204 +++ src/server/sparkd/md/SettlementRecorder.cpp | 140 ++ src/server/sparkd/md/SettlementRecorder.h | 31 + src/server/sparkd/md/SpreadResolver.h | 79 + .../sparkd/md/Structure/FixedSecurity.cpp | 283 +++ .../sparkd/md/Structure/FixedSecurity.h | 197 +++ src/server/sparkd/md/Structure/PriceSize.cpp | 21 + src/server/sparkd/md/Structure/PriceSize.h | 171 ++ src/server/sparkd/md/Structure/readme.txt | 8 + src/server/sparkd/md/Structure/structure.h | 217 +++ src/server/sparkd/md/UserSymbol.h | 55 + src/server/sparkd/md/coconut2.h | 13 + src/server/sparkd/md/common.cpp | 43 + src/server/sparkd/md/common.h | 56 + src/server/sparkd/md/get_symbols.h | 15 + .../platforms/mdp3_onixs/CMESecurityIndex.h | 19 + .../md/platforms/mdp3_onixs/MDP3Listener.cpp | 596 +++++++ .../md/platforms/mdp3_onixs/MDP3Listener.h | 131 ++ .../md/platforms/mdp3_onixs/MDP3Main.cpp | 318 ++++ .../sparkd/md/platforms/mdp3_onixs/MDP3Main.h | 50 + .../md/platforms/mdp3_onixs/MDP3Utils.cpp | 128 ++ .../md/platforms/mdp3_onixs/MDP3Utils.h | 23 + .../mdp3_onixs/licenses/Onixs.BTS.lic | 21 + .../sparkd/md/platforms/mdp3_onixs/onixs.h | 4 + .../platforms/mdp3_onixs/tests/mdp3_tests.cpp | 210 +++ src/server/sparkd/md/protos/co2.proto | 126 ++ src/server/sparkd/md/protos/coconut.proto | 334 ++++ src/server/sparkd/md/readme.txt | 96 ++ src/server/sparkd/md/relay/QueryServer.cpp | 382 ++++ src/server/sparkd/md/relay/QueryServer.h | 24 + src/server/sparkd/md/relay/RelayClient.cpp | 417 +++++ src/server/sparkd/md/relay/RelayClient.h | 37 + src/server/sparkd/md/relay/RelayRecorder.cpp | 236 +++ src/server/sparkd/md/relay/RelayRecorder.h | 34 + .../sparkd/md/test/Co2TestInstrument.cpp | 60 + src/server/sparkd/md/test/Co2TestInstrument.h | 29 + src/server/sparkd/md/test/md2_tests.cpp | 1533 +++++++++++++++++ tools/Protobuf.cmake | 119 ++ tools/setup_builds.sh | 33 + 350 files changed, 38599 insertions(+) create mode 100644 .gitignore create mode 100644 CMakeLists.txt create mode 100644 src/client/tools/CMakeLists.txt create mode 100644 src/client/tools/main.cpp create mode 100644 src/lib/CMakeLists.txt create mode 100644 src/lib/api/apicore.proto create mode 100644 src/lib/api/apioptions.proto create mode 100644 src/lib/api/core/ApiClientInterop.cpp create mode 100644 src/lib/api/core/ApiClientInterop.h create mode 100644 src/lib/api/core/ApiInteropCommon.h create mode 100644 src/lib/api/core/ApiServerInterop.cpp create mode 100644 src/lib/api/core/ApiServerInterop.h create mode 100644 src/lib/api/public/ApiException.h create mode 100644 src/lib/api/public/ApiServer.cpp create mode 100644 src/lib/api/public/ApiServer.h create mode 100644 src/lib/api/public/ApiUtils.h create mode 100644 src/lib/api/public/ClientChannel.cpp create mode 100644 src/lib/api/public/ClientChannel.h create mode 100644 src/lib/api/public/QuickbusClientChannel.cpp create mode 100644 src/lib/api/public/QuickbusClientChannel.h create mode 100644 src/lib/api/public/RawUpdate.h create mode 100644 src/lib/api/public/Result.h create mode 100644 src/lib/api/public/ServerChannel.cpp create mode 100644 src/lib/api/public/ServerChannel.h create mode 100644 src/lib/api/public/ServiceContext.cpp create mode 100644 src/lib/api/public/ServiceContext.h create mode 100644 src/lib/api/public/Update.h create mode 100644 src/lib/api/public/api.h create mode 100644 src/lib/api/service_compiler/apioptions_pb2.py create mode 100644 src/lib/api/service_compiler/apioptions_pb2.pyc create mode 100755 src/lib/api/service_compiler/plugin.cmd create mode 100755 src/lib/api/service_compiler/plugin.py create mode 100644 src/lib/api/service_compiler/templates/c#/client.pb.cs create mode 100644 src/lib/api/service_compiler/templates/c#/immutable.pb.cs create mode 100644 src/lib/api/service_compiler/templates/c#/server.pb.cs create mode 100644 src/lib/api/service_compiler/templates/cpp/client.pb.cc create mode 100644 src/lib/api/service_compiler/templates/cpp/client.pb.h create mode 100644 src/lib/api/service_compiler/templates/cpp/server.pb.cc create mode 100644 src/lib/api/service_compiler/templates/cpp/server.pb.h create mode 100644 src/lib/api/services/Hello.proto create mode 100644 src/lib/api/test/ApiTests.cpp create mode 100644 src/lib/api/test/HelloServiceServer.cpp create mode 100644 src/lib/api/test/HelloServiceServer.h create mode 100755 src/lib/config/ConfigDownloader.cpp create mode 100755 src/lib/config/ConfigDownloader.h create mode 100755 src/lib/config/ConfigErrorCollector.cpp create mode 100755 src/lib/config/ConfigErrorCollector.h create mode 100755 src/lib/config/ConfigLog.cpp create mode 100755 src/lib/config/ConfigLog.h create mode 100755 src/lib/config/ConfigPropertyEvaluator.cpp create mode 100755 src/lib/config/ConfigPropertyEvaluator.h create mode 100755 src/lib/config/ConfigValueQueue.cpp create mode 100755 src/lib/config/ConfigValueQueue.h create mode 100755 src/lib/config/ConfigurationException.cpp create mode 100755 src/lib/config/ConfigurationException.h create mode 100755 src/lib/config/ConfigurationState.cpp create mode 100755 src/lib/config/ConfigurationState.h create mode 100755 src/lib/config/PreprocessInputStream.cpp create mode 100755 src/lib/config/PreprocessInputStream.h create mode 100755 src/lib/config/config.cpp create mode 100755 src/lib/config/config.h create mode 100644 src/lib/config/configure_proto_files.h.in create mode 100755 src/lib/config/defaults/edge.conf create mode 100644 src/lib/config/defaults/logging.conf create mode 100755 src/lib/config/defaults/testkeys.conf create mode 100755 src/lib/config/doc/README.txt create mode 100755 src/lib/config/extensions/extensions.proto create mode 100755 src/lib/config/invalid/invalid.proto create mode 100755 src/lib/config/protos/MarketData.proto create mode 100644 src/lib/config/protos/Ports.proto create mode 100644 src/lib/config/protos/Securities.proto create mode 100644 src/lib/config/protos/Symbols.proto create mode 100644 src/lib/config/protos/jetfix.proto create mode 100644 src/lib/config/protos/logging.proto create mode 100644 src/lib/config/protos/quickbus_cfg.proto create mode 100755 src/lib/config/protos/system.proto create mode 100755 src/lib/config/protos/testkeys.proto create mode 100755 src/lib/config/tests/HTTPServer.cpp create mode 100755 src/lib/config/tests/HTTPServer.h create mode 100755 src/lib/config/tests/config_ref_tests.cpp create mode 100755 src/lib/config/tests/config_tests.cpp create mode 100644 src/lib/config/text_format/strutil.h create mode 100755 src/lib/config/text_format/text_format.cpp create mode 100755 src/lib/config/text_format/text_format.h create mode 100755 src/lib/expression/BooleanGrammar.cpp create mode 100755 src/lib/expression/BooleanGrammar.h create mode 100755 src/lib/expression/ExpressionEvaluator.cpp create mode 100755 src/lib/expression/ExpressionEvaluator.h create mode 100755 src/lib/expression/IPropertyEvaluator.h create mode 100755 src/lib/expression/ParserExceptions.cpp create mode 100755 src/lib/expression/ParserExceptions.h create mode 100755 src/lib/expression/RecursionGuard.h create mode 100755 src/lib/expression/Value.cpp create mode 100755 src/lib/expression/Value.h create mode 100755 src/lib/expression/tests/expression_parser_tests.cpp create mode 100644 src/lib/logging/Common.h create mode 100644 src/lib/logging/LogRec.h create mode 100644 src/lib/logging/Logger.h create mode 100644 src/lib/logging/LoggerHub.cpp create mode 100644 src/lib/logging/LoggerHub.h create mode 100644 src/lib/logging/LoggingConfig.h create mode 100644 src/lib/logging/MsgPacking.h create mode 100644 src/lib/logging/Sink.cpp create mode 100644 src/lib/logging/Sink.h create mode 100644 src/lib/logging/logging.cpp create mode 100644 src/lib/logging/logging.h create mode 100644 src/lib/logging/tests/logging_tests.cpp create mode 100755 src/lib/os/Win32Exception.cpp create mode 100755 src/lib/os/Win32Exception.h create mode 100755 src/lib/os/os.cpp create mode 100755 src/lib/os/os.h create mode 100755 src/lib/os/tests/os_tests.cpp create mode 100644 src/lib/quickbus/core/Advertisement.h create mode 100644 src/lib/quickbus/core/LocalNode.cpp create mode 100644 src/lib/quickbus/core/LocalNode.h create mode 100644 src/lib/quickbus/core/Quickbus.cpp create mode 100644 src/lib/quickbus/core/Quickbus.h create mode 100644 src/lib/quickbus/core/SubscriptionID.cpp create mode 100644 src/lib/quickbus/core/SubscriptionID.h create mode 100644 src/lib/quickbus/core/TableID.cpp create mode 100644 src/lib/quickbus/core/TableID.h create mode 100644 src/lib/quickbus/core/Topic.h create mode 100644 src/lib/quickbus/message_model/Change.h create mode 100644 src/lib/quickbus/message_model/Record.h create mode 100644 src/lib/quickbus/message_model/RecordKey.h create mode 100644 src/lib/quickbus/message_model/RecordValue.h create mode 100644 src/lib/quickbus/message_model/Update.cpp create mode 100644 src/lib/quickbus/message_model/Update.h create mode 100644 src/lib/quickbus/message_model/merge_updates.cpp create mode 100644 src/lib/quickbus/message_model/merge_updates.h create mode 100644 src/lib/quickbus/network/QBConnection.cpp create mode 100644 src/lib/quickbus/network/QBConnection.h create mode 100644 src/lib/quickbus/network/QBConnectionManager.cpp create mode 100644 src/lib/quickbus/network/QBConnectionManager.h create mode 100644 src/lib/quickbus/network/endpoint_description.cpp create mode 100644 src/lib/quickbus/network/endpoint_description.h create mode 100644 src/lib/quickbus/network/quickbus_msgs.proto create mode 100644 src/lib/quickbus/pubsub/TableEvent.h create mode 100644 src/lib/quickbus/pubsub/TableFilter.h create mode 100644 src/lib/quickbus/pubsub/TablePublisher.cpp create mode 100644 src/lib/quickbus/pubsub/TablePublisher.h create mode 100644 src/lib/quickbus/pubsub/TableSubscriber.cpp create mode 100644 src/lib/quickbus/pubsub/TableSubscriber.h create mode 100644 src/lib/quickbus/pubsub/tests/qb_pubsub_tests.cpp create mode 100644 src/lib/quickbus/quickbus.h create mode 100644 src/lib/quickbus/routing/INode.cpp create mode 100644 src/lib/quickbus/routing/INode.h create mode 100644 src/lib/quickbus/routing/INodePtr.h create mode 100644 src/lib/quickbus/routing/NodeID.cpp create mode 100644 src/lib/quickbus/routing/NodeID.h create mode 100644 src/lib/quickbus/routing/Router.cpp create mode 100644 src/lib/quickbus/routing/Router.h create mode 100644 src/lib/quickbus/rpc/OnRPCReply.h create mode 100644 src/lib/quickbus/rpc/OnRPCRequest.h create mode 100644 src/lib/quickbus/rpc/RPCClient.cpp create mode 100644 src/lib/quickbus/rpc/RPCClient.h create mode 100644 src/lib/quickbus/rpc/RPCError.h create mode 100644 src/lib/quickbus/rpc/RPCReply.h create mode 100644 src/lib/quickbus/rpc/RPCRequest.h create mode 100644 src/lib/quickbus/rpc/RPCServer.cpp create mode 100644 src/lib/quickbus/rpc/RPCServer.h create mode 100644 src/lib/quickbus/rpc/RPCServiceName.h create mode 100644 src/lib/quickbus/rpc/RPCSessionID.cpp create mode 100644 src/lib/quickbus/rpc/RPCSessionID.h create mode 100644 src/lib/quickbus/rpc/rpc_utils.cpp create mode 100644 src/lib/quickbus/rpc/rpc_utils.h create mode 100644 src/lib/quickbus/rpc/tests/qb_rpc_tests.cpp create mode 100644 src/lib/quickbus/tests/quickbus_tests.cpp create mode 100755 src/lib/security_model/SecurityHelper.cpp create mode 100755 src/lib/security_model/SecurityHelper.h create mode 100755 src/lib/security_model/security.proto create mode 100755 src/lib/utils/CallTimer.h create mode 100755 src/lib/utils/Dispatcher.cpp create mode 100755 src/lib/utils/Dispatcher.h create mode 100755 src/lib/utils/HTTPStream.cpp create mode 100755 src/lib/utils/HTTPStream.h create mode 100755 src/lib/utils/PBUtils.cpp create mode 100755 src/lib/utils/PBUtils.h create mode 100644 src/lib/utils/SharedList.h create mode 100644 src/lib/utils/SharedMemory.cpp create mode 100644 src/lib/utils/SharedMemory.h create mode 100644 src/lib/utils/Sqlite_utils.cpp create mode 100644 src/lib/utils/Sqlite_utils.h create mode 100755 src/lib/utils/TempDir.h create mode 100755 src/lib/utils/TempFile.h create mode 100644 src/lib/utils/TimeStamp.h create mode 100755 src/lib/utils/Uuid.cpp create mode 100755 src/lib/utils/Uuid.h create mode 100644 src/lib/utils/ValueWrapper.h create mode 100644 src/lib/utils/WaitFreeQueue.h create mode 100644 src/lib/utils/benchmarking_utils.h create mode 100644 src/lib/utils/bts_array_size.h create mode 100644 src/lib/utils/bts_assert.h create mode 100644 src/lib/utils/bts_chrono.h create mode 100755 src/lib/utils/bts_debugger.h create mode 100644 src/lib/utils/bts_export.h create mode 100755 src/lib/utils/bts_no_copy.h create mode 100644 src/lib/utils/catch_test_runner.cpp create mode 100644 src/lib/utils/catch_test_runner.h create mode 100644 src/lib/utils/checked_cast.h create mode 100755 src/lib/utils/datetime.cpp create mode 100755 src/lib/utils/datetime.h create mode 100755 src/lib/utils/defer.h create mode 100644 src/lib/utils/file_utils.cpp create mode 100644 src/lib/utils/file_utils.h create mode 100644 src/lib/utils/helper_macros.h create mode 100644 src/lib/utils/is_unittest_environment.cpp create mode 100644 src/lib/utils/is_unittest_environment.h create mode 100755 src/lib/utils/meta_programming.h create mode 100644 src/lib/utils/mpmc_bounded_queue.h create mode 100755 src/lib/utils/proto_helper.h create mode 100644 src/lib/utils/type_name.h create mode 100644 src/lib/utils/warn_ctrl.h create mode 100755 src/lib/utils/windows_wrapper.h create mode 100644 src/server/sparkd/CMakeLists.txt create mode 100644 src/server/sparkd/interconnect/Actor.cpp create mode 100644 src/server/sparkd/interconnect/Actor.h create mode 100644 src/server/sparkd/interconnect/Domain.cpp create mode 100644 src/server/sparkd/interconnect/Domain.h create mode 100644 src/server/sparkd/interconnect/Executor.cpp create mode 100644 src/server/sparkd/interconnect/Executor.h create mode 100644 src/server/sparkd/interconnect/IActor.cpp create mode 100644 src/server/sparkd/interconnect/IActor.h create mode 100644 src/server/sparkd/interconnect/IDomain.cpp create mode 100644 src/server/sparkd/interconnect/IDomain.h create mode 100644 src/server/sparkd/interconnect/IPollable.h create mode 100644 src/server/sparkd/interconnect/Msg.h create mode 100644 src/server/sparkd/interconnect/RPC.h create mode 100644 src/server/sparkd/interconnect/Topic.h create mode 100644 src/server/sparkd/interconnect/TopicDirectory.cpp create mode 100644 src/server/sparkd/interconnect/TopicDirectory.h create mode 100644 src/server/sparkd/interconnect/tests/baseline_perf_interconnect_test.cpp create mode 100644 src/server/sparkd/interconnect/tests/interconnect_test.cpp create mode 100644 src/server/sparkd/jetfix/codec/FIXArray.cpp create mode 100644 src/server/sparkd/jetfix/codec/FIXArray.h create mode 100644 src/server/sparkd/jetfix/codec/FIXBuffer.h create mode 100644 src/server/sparkd/jetfix/codec/FIXEncodingStream.h create mode 100644 src/server/sparkd/jetfix/codec/FIXFragment.h create mode 100644 src/server/sparkd/jetfix/codec/FIXMsg.h create mode 100644 src/server/sparkd/jetfix/codec/FIXParser.h create mode 100644 src/server/sparkd/jetfix/codec/FIXWriter.h create mode 100644 src/server/sparkd/jetfix/codec/FieldDefinitions.h create mode 100644 src/server/sparkd/jetfix/codec/Fields.h create mode 100644 src/server/sparkd/jetfix/codec/Group.h create mode 100644 src/server/sparkd/jetfix/codec/IFIXMsg.h create mode 100644 src/server/sparkd/jetfix/codec/IFIXParser.h create mode 100644 src/server/sparkd/jetfix/codec/IFieldReader.h create mode 100644 src/server/sparkd/jetfix/codec/MessageDefinitions.h create mode 100644 src/server/sparkd/jetfix/codec/MessageTypes.h create mode 100644 src/server/sparkd/jetfix/codec/ReaderCursor.h create mode 100644 src/server/sparkd/jetfix/codec/WriterCursor.h create mode 100644 src/server/sparkd/jetfix/codec/f_toa.h create mode 100644 src/server/sparkd/jetfix/codec/i32_toa.h create mode 100644 src/server/sparkd/jetfix/codec/sum_bytes.h create mode 100644 src/server/sparkd/jetfix/common/InMsgBuffer.h create mode 100644 src/server/sparkd/jetfix/common/JetFIXError.h create mode 100644 src/server/sparkd/jetfix/common/OutFIX.cpp create mode 100644 src/server/sparkd/jetfix/common/OutFIX.h create mode 100644 src/server/sparkd/jetfix/common/OutFIXPtr.h create mode 100644 src/server/sparkd/jetfix/common/OutMsgVecPtr.h create mode 100644 src/server/sparkd/jetfix/connection/AsyncReader.cpp create mode 100644 src/server/sparkd/jetfix/connection/AsyncReader.h create mode 100644 src/server/sparkd/jetfix/connection/IConnection.cpp create mode 100644 src/server/sparkd/jetfix/connection/IConnection.h create mode 100644 src/server/sparkd/jetfix/connection/IConnectionObserver.h create mode 100644 src/server/sparkd/jetfix/connection/SocketConnection.cpp create mode 100644 src/server/sparkd/jetfix/connection/SocketConnection.h create mode 100644 src/server/sparkd/jetfix/gateway/CoreGateway.cpp create mode 100644 src/server/sparkd/jetfix/gateway/CoreGateway.h create mode 100644 src/server/sparkd/jetfix/gateway/ICoreGatewayEventSink.h create mode 100644 src/server/sparkd/jetfix/gateway/ICoreGatewayObserver.h create mode 100644 src/server/sparkd/jetfix/gateway/JetFIXGateway.cpp create mode 100644 src/server/sparkd/jetfix/gateway/JetFIXGateway.h create mode 100644 src/server/sparkd/jetfix/gateway/Session.cpp create mode 100644 src/server/sparkd/jetfix/gateway/Session.h create mode 100644 src/server/sparkd/jetfix/jetfix.h create mode 100644 src/server/sparkd/jetfix/store/FileStore.cpp create mode 100644 src/server/sparkd/jetfix/store/FileStore.h create mode 100644 src/server/sparkd/jetfix/store/IStore.h create mode 100644 src/server/sparkd/jetfix/store/session_name.cpp create mode 100644 src/server/sparkd/jetfix/store/session_name.h create mode 100644 src/server/sparkd/jetfix/tests/async_reader_tests.cpp create mode 100644 src/server/sparkd/jetfix/tests/decoding_tests.cpp create mode 100644 src/server/sparkd/jetfix/tests/fix_encoding_stream_tests.cpp create mode 100644 src/server/sparkd/jetfix/tests/fix_fragment_tests.cpp create mode 100644 src/server/sparkd/jetfix/tests/jetfix_performance_tests.cpp create mode 100644 src/server/sparkd/jetfix/tests/store_tests.cpp create mode 100644 src/server/sparkd/main.cpp create mode 100644 src/server/sparkd/md/Diag/MDPriceLantencyTester.cpp create mode 100644 src/server/sparkd/md/Diag/MDPriceLantencyTester.h create mode 100644 src/server/sparkd/md/Diag/ShmDumper.cpp create mode 100644 src/server/sparkd/md/Diag/ShmDumper.h create mode 100644 src/server/sparkd/md/MDConsumer.cpp create mode 100644 src/server/sparkd/md/MDConsumer.h create mode 100644 src/server/sparkd/md/MDConsumerHandle.cpp create mode 100644 src/server/sparkd/md/MDConsumerHandle.h create mode 100644 src/server/sparkd/md/MDConsumerInterop.cpp create mode 100644 src/server/sparkd/md/MDProducer.cpp create mode 100644 src/server/sparkd/md/MDProducer.h create mode 100644 src/server/sparkd/md/MDProducerChannel.h create mode 100644 src/server/sparkd/md/MDProducerHandle.cpp create mode 100644 src/server/sparkd/md/MDProducerHandle.h create mode 100644 src/server/sparkd/md/MDRecoverableConsumer.cpp create mode 100644 src/server/sparkd/md/MDRecoverableConsumer.h create mode 100644 src/server/sparkd/md/MemorySegmentUtils.cpp create mode 100644 src/server/sparkd/md/MemorySegmentUtils.h create mode 100644 src/server/sparkd/md/PlatformLoader.cpp create mode 100644 src/server/sparkd/md/PlatformLoader.h create mode 100644 src/server/sparkd/md/SecurityConstructor.h create mode 100644 src/server/sparkd/md/SecurityIDRecorder.cpp create mode 100644 src/server/sparkd/md/SecurityIDRecorder.h create mode 100644 src/server/sparkd/md/SecurityIndex.h create mode 100644 src/server/sparkd/md/SettlementRecorder.cpp create mode 100644 src/server/sparkd/md/SettlementRecorder.h create mode 100644 src/server/sparkd/md/SpreadResolver.h create mode 100644 src/server/sparkd/md/Structure/FixedSecurity.cpp create mode 100644 src/server/sparkd/md/Structure/FixedSecurity.h create mode 100644 src/server/sparkd/md/Structure/PriceSize.cpp create mode 100644 src/server/sparkd/md/Structure/PriceSize.h create mode 100644 src/server/sparkd/md/Structure/readme.txt create mode 100644 src/server/sparkd/md/Structure/structure.h create mode 100644 src/server/sparkd/md/UserSymbol.h create mode 100644 src/server/sparkd/md/coconut2.h create mode 100644 src/server/sparkd/md/common.cpp create mode 100644 src/server/sparkd/md/common.h create mode 100644 src/server/sparkd/md/get_symbols.h create mode 100644 src/server/sparkd/md/platforms/mdp3_onixs/CMESecurityIndex.h create mode 100644 src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp create mode 100644 src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h create mode 100644 src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp create mode 100644 src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h create mode 100644 src/server/sparkd/md/platforms/mdp3_onixs/MDP3Utils.cpp create mode 100644 src/server/sparkd/md/platforms/mdp3_onixs/MDP3Utils.h create mode 100644 src/server/sparkd/md/platforms/mdp3_onixs/licenses/Onixs.BTS.lic create mode 100644 src/server/sparkd/md/platforms/mdp3_onixs/onixs.h create mode 100644 src/server/sparkd/md/platforms/mdp3_onixs/tests/mdp3_tests.cpp create mode 100644 src/server/sparkd/md/protos/co2.proto create mode 100755 src/server/sparkd/md/protos/coconut.proto create mode 100644 src/server/sparkd/md/readme.txt create mode 100644 src/server/sparkd/md/relay/QueryServer.cpp create mode 100644 src/server/sparkd/md/relay/QueryServer.h create mode 100644 src/server/sparkd/md/relay/RelayClient.cpp create mode 100644 src/server/sparkd/md/relay/RelayClient.h create mode 100644 src/server/sparkd/md/relay/RelayRecorder.cpp create mode 100644 src/server/sparkd/md/relay/RelayRecorder.h create mode 100644 src/server/sparkd/md/test/Co2TestInstrument.cpp create mode 100644 src/server/sparkd/md/test/Co2TestInstrument.h create mode 100644 src/server/sparkd/md/test/md2_tests.cpp create mode 100644 tools/Protobuf.cmake create mode 100755 tools/setup_builds.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..7798ee773 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +build/ +.dir-locals.el diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 000000000..678ecc7c8 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,25 @@ +cmake_minimum_required(VERSION 3.5) +project(spark) + +if (WIN32) + include_directories ("c:/externals/include") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} \ + /MP \ + /WX /wd4251 /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 \ + /DCURL_STATICLIB + /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600") + + set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /MTd") + set (CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_DEBUG} /MT") + + add_subdirectory(src/lib) + add_subdirectory(src/client/tools) +else() + + set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} \ + -Wall -pedantic -Wextra -Werror -Weffc++ -Wno-missing-braces --std=c++14") + + set (CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_DEBUG} -DNDEBUG -O3") + add_subdirectory(src/lib) + add_subdirectory(src/server/sparkd) +endif() diff --git a/src/client/tools/CMakeLists.txt b/src/client/tools/CMakeLists.txt new file mode 100644 index 000000000..4f3702812 --- /dev/null +++ b/src/client/tools/CMakeLists.txt @@ -0,0 +1,30 @@ +cmake_minimum_required(VERSION 3.5) +project(spark_tool) + +# proto sources +set (Protos + ) + +include(${CMAKE_SOURCE_DIR}/tools/Protobuf.cmake) +protobuf_generate_cpp( Proto_Srcs PROTO_HDRS ${Protos} ) + +# cpp sources +set (Spark_Tool_Sources + main.cpp + ) + +# sparkd +add_executable(spark_tool ${Spark_Tool_Sources} ${Proto_Srcs}) + +#include paths +target_include_directories( spark_tool PUBLIC ${CMAKE_SOURCE_DIR}/src ${CMAKE_BINARY_DIR}/src) + +# linking +target_link_libraries( spark_tool + spark_lib + -static-libgcc + -static-libstdc++) + + + + diff --git a/src/client/tools/main.cpp b/src/client/tools/main.cpp new file mode 100644 index 000000000..f9235b04e --- /dev/null +++ b/src/client/tools/main.cpp @@ -0,0 +1,4 @@ +#include "lib/logging/logging.h" + +int main() +{return 0;} diff --git a/src/lib/CMakeLists.txt b/src/lib/CMakeLists.txt new file mode 100644 index 000000000..9e1f2a7c2 --- /dev/null +++ b/src/lib/CMakeLists.txt @@ -0,0 +1,193 @@ +cmake_minimum_required(VERSION 3.5) +project(spark_lib) + +# config protos +set (Config_Protos + config/protos/system.proto + config/protos/testkeys.proto + config/protos/logging.proto + config/protos/quickbus_cfg.proto + config/protos/jetfix.proto + config/protos/MarketData.proto + config/protos/Symbols.proto + config/protos/Ports.proto + ) + +# non config/non service protos +set (NonConfig_Protos + config/invalid/invalid.proto + config/extensions/extensions.proto + quickbus/network/quickbus_msgs.proto + security_model/security.proto + api/apicore.proto + api/apioptions.proto + ) + +# service protos +set (Service_Protos + # api/services/Hello.proto + ) + +include(${CMAKE_SOURCE_DIR}/tools/Protobuf.cmake) +protobuf_generate_cpp( PROTO_SRCS PROTO_HDRS ${NonConfig_Protos} ${Config_Protos}) +service_compiler_generate_cpp(SERVICE_COMPILER_SRCS SERVICE_COMPILER_HDRS ${Service_Protos}) + +# srcs for sparkd +set (SparkLib_Sources + + # api + api/core/ApiClientInterop.cpp + api/core/ApiServerInterop.cpp + api/public/ApiServer.cpp + api/public/ClientChannel.cpp + api/public/QuickbusClientChannel.cpp + api/public/ServerChannel.cpp + api/public/ServiceContext.cpp + # api/test/ApiTests.cpp + # api/test/HelloServiceServer.cpp + + # expression + expression/BooleanGrammar.cpp + expression/ExpressionEvaluator.cpp + expression/ParserExceptions.cpp + expression/Value.cpp + + # config + config/config.cpp + config/ConfigDownloader.cpp + config/ConfigErrorCollector.cpp + config/ConfigLog.cpp + config/ConfigPropertyEvaluator.cpp + config/ConfigurationException.cpp + config/ConfigurationState.cpp + config/ConfigValueQueue.cpp + config/PreprocessInputStream.cpp + config/text_format/text_format.cpp + config/tests/config_ref_tests.cpp + config/tests/HTTPServer.cpp + config/tests/config_tests.cpp + + # quickbus + quickbus/core/LocalNode.cpp + quickbus/core/Quickbus.cpp + quickbus/core/SubscriptionID.cpp + quickbus/core/TableID.cpp + quickbus/message_model/Update.cpp + quickbus/message_model/merge_updates.cpp + quickbus/network/QBConnection.cpp + quickbus/network/QBConnectionManager.cpp + quickbus/network/endpoint_description.cpp + quickbus/pubsub/TablePublisher.cpp + quickbus/pubsub/TableSubscriber.cpp + quickbus/pubsub/tests/qb_pubsub_tests.cpp + quickbus/routing/INode.cpp + quickbus/routing/NodeID.cpp + quickbus/routing/Router.cpp + quickbus/rpc/RPCClient.cpp + quickbus/rpc/RPCServer.cpp + quickbus/rpc/RPCSessionID.cpp + quickbus/rpc/rpc_utils.cpp + quickbus/rpc/tests/qb_rpc_tests.cpp + quickbus/tests/quickbus_tests.cpp + + # OS + os/os.cpp + + #logging + logging/logging.cpp + logging/tests/logging_tests.cpp + logging/LoggerHub.cpp + logging/Sink.cpp + + # misc utils + utils/is_unittest_environment.cpp + utils/datetime.cpp + utils/PBUtils.cpp + utils/Uuid.cpp + utils/file_utils.cpp + utils/HTTPStream.cpp + utils/SharedMemory.cpp + utils/Dispatcher.cpp + utils/Sqlite_utils.cpp + utils/catch_test_runner.cpp + + # security model + security_model/SecurityHelper.cpp + ) + + +# generate list of config protos +set(config_proto_file ${CMAKE_CURRENT_BINARY_DIR}/config_proto_files.h) +file(REMOVE ${config_proto_file}) +file(COPY config/defaults DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) +file(APPEND ${config_proto_file} "const char * config_proto_files[] = { // generated file\n") +foreach(protopath ${Config_Protos}) + get_filename_component(protoname ${protopath} NAME) + file(APPEND ${config_proto_file} "\"${protoname}\",\n") +endforeach(protopath) +file(APPEND ${config_proto_file} "};\n") + + +# spark_lib +if (WIN32) +add_library(spark_lib SHARED + ${SparkLib_Sources} + ${PROTO_SRCS} + ${SERVICE_COMPILER_SRCS} + ) +else () +add_library(spark_lib STATIC + ${SparkLib_Sources} + ${PROTO_SRCS} + ${SERVICE_COMPILER_SRCS} + ) +endif() + + +#include paths +target_include_directories( spark_lib PUBLIC + ${CMAKE_SOURCE_DIR}/src + ${CMAKE_BINARY_DIR}/src + ${Protobuf_INCLUDE_DIRS} + ${Boost_INCLUDE_DIRS} + ) + +if (WIN32) + + find_package(Boost REQUIRED COMPONENTS system thread filesystem regex chrono date_time atomic) + find_library (FMT NAMES fmt.lib) + find_library (SQLITE NAMES sqlite3.lib) + + if(CMAKE_BUILD_TYPE MATCHES "^Debug$" OR NOT DEFINED CMAKE_BUILD_TYPE) + target_include_directories (spark_lib PUBLIC "c:/externals/curl-debug/include") + target_link_libraries( spark_lib "c:/externals/curl-debug/lib/libcurl_a_debug.lib") + else() + target_include_directories (spark_lib PUBLIC "c:/externals/curl-release/include") + target_link_libraries( spark_lib "c:/externals/curl-release/lib/libcurl_a.lib") + endif() + + target_link_libraries( spark_lib + ${Boost_LIBRARIES} + ${Protobuf_Static_Lib} + ${FMT} + ${SQLITE} + ) +else () + set(Boost_USE_STATIC_LIBS ON) + find_package(Boost REQUIRED COMPONENTS system thread filesystem regex chrono date_time atomic) + + target_link_libraries( spark_lib + ${Boost_LIBRARIES} + ${Protobuf_Static_Lib} + curl.a + fmt.a + uuid.a + sqlite3.a + -lrt + -ldl + -static-libgcc + -static-libstdc++ + -pthread) +endif() + + diff --git a/src/lib/api/apicore.proto b/src/lib/api/apicore.proto new file mode 100644 index 000000000..97313b89e --- /dev/null +++ b/src/lib/api/apicore.proto @@ -0,0 +1,37 @@ +syntax="proto3"; +import "lib/api/apioptions.proto"; +package bts.api; +option csharp_namespace="bts.api"; + +message PingPong { + bytes payload = 1; +} + +message RequestHeader { + int32 request_id = 1; + int32 method = 2; +} + +enum ApiError { + SUCCESS = 0; + NO_SERVER = 1; + SERVER_DISCONNECT = 2; + MISSING_METHOD = 3; + SERVER_FAULT = 4; + UNEXPECTED_RESPONSE = 5; +} + +message ResponseHeader { + int32 request_id = 1; + ApiError code = 2; +} + +// Some Useful buffers for use as arguments/returns + +message VoidMessage {} +message BytesMessage { + bytes buffer = 1; +} +message BoolResultMessage { + bool success = 1; +} diff --git a/src/lib/api/apioptions.proto b/src/lib/api/apioptions.proto new file mode 100644 index 000000000..b682c2277 --- /dev/null +++ b/src/lib/api/apioptions.proto @@ -0,0 +1,15 @@ +syntax="proto3"; +import "google/protobuf/descriptor.proto"; +package bts.api; +option csharp_namespace="bts.api"; + +extend google.protobuf.MessageOptions { + string message_description = 8500; +} +extend google.protobuf.ServiceOptions { + string service_description = 8501; +} +extend google.protobuf.FieldOptions { + bool key = 8502; + string field_description = 8503; +} diff --git a/src/lib/api/core/ApiClientInterop.cpp b/src/lib/api/core/ApiClientInterop.cpp new file mode 100644 index 000000000..294523398 --- /dev/null +++ b/src/lib/api/core/ApiClientInterop.cpp @@ -0,0 +1,212 @@ +#include "ApiClientInterop.h" +#include "lib/quickbus/quickbus.h" +#include "lib/config/config.h" +#include "lib/logging/logging.h" + +#include "../public/ServiceContext.h" +#include "../public/Result.h" + +#include "lib/api/apicore.pb.h" +#include "lib/config/protos/quickbus_cfg.pb.h" +#include "lib/config/protos/system.pb.h" +#include "lib/config/protos/Ports.pb.h" + +#ifdef WIN32 +#include +#endif + +namespace bts { +namespace api { + +config::QuickbusNodeCfg get_client_node_cfg() +{ + auto cfg = config::get(); + config::QuickbusNodeCfg node_cfg; + node_cfg.set_reconnect_period_secs(cfg.reconnect_period_secs()); + node_cfg.set_hb_period_msecs(cfg.hb_period_msecs()); + node_cfg.set_max_missed_hbs(cfg.max_missed_hbs()); + node_cfg.set_parent_host( + cfg.has_qbus_host() ? + cfg.qbus_host() : + config::get().config_ip() + ); + node_cfg.set_parent_port( + cfg.has_qbus_port() ? + cfg.qbus_port() : + config::get().qbus_port() + ); + node_cfg.set_my_port(0); + return node_cfg; +} + +// +// Common + +BTS_EXPORT Quickbus* api_create_quickbus(const char* qbconfig, long length) { + // If config hasn't been initialized yet do it now and suppress exceptions. + // When we're called from a nuget package we may not have defaults which can cause errors + config::initialize_nothrow(nullptr, 0); + + config::QuickbusNodeCfg cfg; + if (length > 0 && qbconfig != nullptr && cfg.ParseFromArray(qbconfig, length)) { + return new Quickbus(cfg); + } + + // Note: Eventually api will probably want it's own instance of QB. For now we share + auto temp_conf = get_client_node_cfg(); + LOG(Api,Info,"Created Quickbus"); + return new Quickbus(temp_conf); +} + +BTS_EXPORT void api_destroy_quickbus(Quickbus* qbus) { + if (qbus != nullptr) { + delete qbus; + } +} + +std::string base_topic(const char* type_name, const char* service_qualifier) { + const std::string topic_base = "api-"; + if (service_qualifier == nullptr || + strlen(service_qualifier) == 0) { + return topic_base + type_name; + } + return topic_base + type_name + "-" + service_qualifier; +} + +qbus::RPCServiceName rpc_topic(const std::string& base_topic) { + return qbus::RPCServiceName(base_topic); +} + + +// +// Client + +struct ApiClient { + ApiClient(Quickbus& qbus, const std::string& topic) : + qbus(qbus), + service_name(topic), + rpc(qbus, rpc_topic(topic)) {} + + Quickbus& qbus; + std::string service_name; + qbus::RPCClient rpc; +}; + +BTS_EXPORT ApiClient* api_client_create(Quickbus* pqbus, const char* type_name, const char* service_qualifier) { + bts_assert(pqbus); + auto service_name = base_topic(type_name, service_qualifier); + LOG(Api,Info, "create client: {}",service_name); + ApiClient* pclient = new ApiClient(*pqbus, service_name); + return pclient; +} + +BTS_EXPORT void api_client_destroy(ApiClient* pclient) { + if (pclient) { + LOG(Api,Info, "destroy client: {}",pclient->service_name); + delete pclient; + } +} + +void reply_error(ApiError error, raw_response_callback cb, void* extra) { + ResponseHeader response; + response.set_code(error); + auto str = response.SerializeAsString(); + str = (char)(str.size()+1) + str; + cb(str.c_str(), str.size(), extra); +} + +BTS_EXPORT void api_client_send(ApiClient* pclient, const char* buffer, int buffer_length, raw_response_callback cb, void* extra) { + bts_assert(pclient); + std::string strbuf(buffer, buffer_length); + qbus::RPCRequest request(move(strbuf)); + pclient->rpc.wait_until_connected(1); + LOG(Api,Diag,"send rpc {}",((int64_t)buffer)); + pclient->rpc.send(std::move(request), [=](qbus::RPCError e, const qbus::RPCReplyPtr& reply) { + LOG(Api,Diag,"rpc reply {}",((int64_t)buffer)); + switch (e) { + case qbus::RPCError::RPC_OK: + cb(reply->value().c_str(), reply->value().size(), extra); + break; + case qbus::RPCError::NO_SERVER: + reply_error(ApiError::NO_SERVER, cb, extra); + break; + case qbus::RPCError::SERVER_DISCONNECT: + reply_error(ApiError::SERVER_DISCONNECT, cb, extra); + break; + default: + reply_error(ApiError::UNEXPECTED_RESPONSE, cb, extra); + } + }); +} + +BTS_EXPORT bool api_client_is_on_qb_thread(ApiClient* pclient) { + return pclient->qbus.on_qb_thread(); +} + + +void safe_call_subscription_callback(raw_subscription_callback cb, EventType type, const std::vector& updates, void* extra) { +#ifdef _MSC_VER + __try { +#endif + if (updates.size() > 0) { + cb(type, &updates[0], updates.size(), extra); + } +#ifdef _MSC_VER + } + __except(EXCEPTION_EXECUTE_HANDLER) { } +#endif +} + + +bts::qbus::Topic stream_topic(const std::string& base_topic, const char* method_and_qualifier) { + return qbus::Topic(base_topic + "-" + method_and_qualifier); +} + +BTS_EXPORT TableSubscription api_client_subscribe(ApiClient* pclient, const char* method_and_qualifier, raw_subscription_callback cb, void* extra) { + bts_assert(pclient); + static_assert((int)qbus::TableEvent::EventType::TABLE_CREATED == EventType::TABLE_CREATED, "Keep these Enums in sync"); + static_assert((int)qbus::TableEvent::EventType::TABLE_DESTROYED == EventType::TABLE_DESTROYED, ""); + static_assert((int)qbus::TableEvent::EventType::TABLE_UPDATED == EventType::TABLE_UPDATED, ""); + static_assert((int)qbus::Change::Type::REMOVE == StreamUpdate::REMOVE, ""); + static_assert((int)qbus::Change::Type::REPLACE == StreamUpdate::REPLACE, ""); + static_assert((int)qbus::Change::Type::CREATE == StreamUpdate::CREATE, ""); + + auto topic = stream_topic(pclient->service_name, method_and_qualifier); + LOG(Api,Info,"api subscribe: {}" ,topic.value()); + + auto result = new qbus::TableSubscriber(pclient->qbus, equal_topic(topic), [cb, extra](const qbus::TableEvent& ev) { + if (ev.event_type == qbus::TableEvent::EventType::TABLE_CREATED || + ev.event_type == qbus::TableEvent::EventType::TABLE_DESTROYED) { + cb((EventType)ev.event_type, nullptr, 0, extra); + } + else { + bts_assert(ev.update != nullptr); + size_t update_count = ev.update->size(); + std::vector bupdates(update_count); + // Copy structure for interop + for (size_t x = 0; x < update_count; ++x) { + const auto& update = (*ev.update)[x]; + bupdates[x].type = static_cast(update.type); + bupdates[x].key = update.key.value(); + const auto& val = update.value->value(); + bupdates[x].buffer_len = val.size(); + bupdates[x].buffer = val.c_str(); + } + safe_call_subscription_callback(cb, (EventType)ev.event_type, bupdates, extra); + } + }, true); + + return result; +} + +BTS_EXPORT void api_client_unsubscribe(ApiClient* pclient, TableSubscription pvtable) { + qbus::TableSubscriber* ptable = static_cast(pvtable); + if (pclient && ptable) { + ptable->disconnect(); + delete ptable; + } +} + +}}// namespace + + diff --git a/src/lib/api/core/ApiClientInterop.h b/src/lib/api/core/ApiClientInterop.h new file mode 100644 index 000000000..3df1d1ac9 --- /dev/null +++ b/src/lib/api/core/ApiClientInterop.h @@ -0,0 +1,25 @@ +#pragma once +#include "ApiInteropCommon.h" + +namespace bts { +namespace api { + +struct ApiClient; +BTS_EXPORT ApiClient* api_client_create(Quickbus*, const char* type_name, const char* service_qualifier); +BTS_EXPORT void api_client_destroy(ApiClient*); +typedef void(*raw_response_callback)(const char* buffer, size_t length, void* extra); +BTS_EXPORT void api_client_send(ApiClient* pclient, const char* buffer, int buffer_length, raw_response_callback cb, void* extra); +BTS_EXPORT bool api_client_is_on_qb_thread(ApiClient*); + +enum EventType { + TABLE_CREATED, + TABLE_UPDATED, + TABLE_DESTROYED +}; + +typedef void* TableSubscription; +typedef void(*raw_subscription_callback)(EventType event_type, const StreamUpdate* changes, int count, void* extra); +BTS_EXPORT TableSubscription api_client_subscribe(ApiClient* pclient, const char* method_and_qualifier, raw_subscription_callback cb, void* extra); +BTS_EXPORT void api_client_unsubscribe(ApiClient* pclient, TableSubscription ptable); + +}}//namespace diff --git a/src/lib/api/core/ApiInteropCommon.h b/src/lib/api/core/ApiInteropCommon.h new file mode 100644 index 000000000..9451efca8 --- /dev/null +++ b/src/lib/api/core/ApiInteropCommon.h @@ -0,0 +1,32 @@ +#pragma once + +#include "lib/utils/bts_export.h" +#include "lib/api/apicore.pb.h" + +#include +#include + +namespace bts { + namespace qbus { + class Quickbus; + } +namespace api { + +typedef qbus::Quickbus Quickbus; +BTS_EXPORT Quickbus* api_create_quickbus(const char* qbconfig, long length); +BTS_EXPORT void api_destroy_quickbus(Quickbus*); + +struct StreamUpdate { + enum Type { + REMOVE, + REPLACE, + CREATE + }; + Type type; + int key; + int buffer_len; + const char* buffer; +}; + + // Internal methods +}} diff --git a/src/lib/api/core/ApiServerInterop.cpp b/src/lib/api/core/ApiServerInterop.cpp new file mode 100644 index 000000000..aa39340c7 --- /dev/null +++ b/src/lib/api/core/ApiServerInterop.cpp @@ -0,0 +1,82 @@ +#include "ApiServerInterop.h" +#include "lib/quickbus/quickbus.h" +#include "lib/logging/logging.h" + +namespace bts { +namespace api { + +// defined in ApiClientInterop.h +std::string base_topic(const char* type_name, const char* service_qualifier); +bts::qbus::RPCServiceName rpc_topic(const std::string& base_topic); +bts::qbus::Topic stream_topic(const std::string& base_topic, const char* method_and_qualifier); + +struct ApiServerCore { + ApiServerCore(Quickbus& qbus, const std::string topic, const qbus::OnRPCRequest& handler) : + qbus(qbus), + topic(topic), + rpc(qbus, rpc_topic(topic), handler) + {} + + Quickbus& qbus; + std::string topic; + qbus::RPCServer rpc; +}; + +BTS_EXPORT ApiServerCore* api_server_create(Quickbus* pbus, const char* type_name, const char* service_qualifier, raw_rpc_request_callback cb, void* extra) { + bts_assert(pbus); + // Invalid characters in qualifier: + + for (auto ch : { '+', '_' }) { + bts_assert(strchr(service_qualifier, ch) == nullptr); + } + auto service_name = base_topic(type_name, service_qualifier); + LOG(Api,Info,"create server: {}",service_name); + return new ApiServerCore(*pbus, service_name, [=](qbus::RPCRequestPtr const& req, qbus::ReplyHandler const& reply) { + // Extend the lifetime of the ReplyHandler by moving it to the heap + qbus::ReplyHandler* phandler = new qbus::ReplyHandler(std::move(reply)); + LOG(Api,Info,"Received rpc request: {}",reinterpret_cast(phandler)); + cb((uint8_t*)(req->value().c_str()), req->value().size(), reinterpret_cast(phandler), extra); + }); +} + +BTS_EXPORT void api_server_call_callback(ReplyHandler** ppvhandler, const uint8_t* buffer, int length) { + bts_assert(ppvhandler); + bts_assert(*ppvhandler); + LOG(Api,Info,"Send rpc reply: {}",reinterpret_cast(*ppvhandler)); + qbus::ReplyHandler** pphandler = reinterpret_cast(ppvhandler); + std::string reply_buffer((const char*)buffer, length); + qbus::RPCReplyPtr replyptr = std::make_shared(move(reply_buffer)); + (**pphandler)(replyptr); + delete (*pphandler); + *pphandler = nullptr; +} + +BTS_EXPORT TableHandle api_server_create_table(ApiServerCore* pserver, const char* method_and_qualifier) { + bts_assert(pserver); + auto topic = stream_topic(pserver->topic, method_and_qualifier); + LOG(Api,Info,"Create table: {}",topic.value()); + return new qbus::TablePublisher(pserver->qbus, topic); +} + +BTS_EXPORT void api_server_update_table(TableHandle htable, StreamUpdate* raw_changes, int change_count) { + if (change_count > 0) { + bts_assert(htable); + qbus::TablePublisher* ptable = reinterpret_cast(htable); + std::vector changes(change_count); + for (int x = 0; x < change_count; ++x) { + StreamUpdate& ch = raw_changes[x]; + std::string buffer(ch.buffer, ch.buffer_len); + qbus::RecordPtr record = std::make_shared(std::move(buffer)); + changes[x] = qbus::Change((qbus::Change::Type)ch.type, qbus::RecordKey(ch.key), record); + } + ptable->publish(std::move(changes)); + } +} + +BTS_EXPORT void api_server_destroy_table(qbus::TablePublisher* ptable) { + if (ptable) { + ptable->disconnect(); + delete ptable; + } +} + +} }//namespace diff --git a/src/lib/api/core/ApiServerInterop.h b/src/lib/api/core/ApiServerInterop.h new file mode 100644 index 000000000..7cee3553f --- /dev/null +++ b/src/lib/api/core/ApiServerInterop.h @@ -0,0 +1,19 @@ +#pragma once +#include "ApiInteropCommon.h" + +namespace bts { +namespace api { + +struct ApiServerCore; +//struct ReplyHandler; +typedef void ReplyHandler; +typedef void(*raw_rpc_request_callback)(const uint8_t* buffer, size_t length, ReplyHandler* callback, void* extra); +BTS_EXPORT ApiServerCore* api_server_create(Quickbus* pbus, const char* type_name, const char* service_qualifier, raw_rpc_request_callback reply, void* extra); + +BTS_EXPORT void api_server_call_callback(ReplyHandler** pphandler, const uint8_t* buffer, int length); + +typedef void* TableHandle; +BTS_EXPORT TableHandle api_server_create_table(ApiServerCore* pserver, const char* method_and_qualifier); +BTS_EXPORT void api_server_update_table(TableHandle htable, StreamUpdate* raw_changes, int change_count); + +}}//namespace diff --git a/src/lib/api/public/ApiException.h b/src/lib/api/public/ApiException.h new file mode 100644 index 000000000..886d8485f --- /dev/null +++ b/src/lib/api/public/ApiException.h @@ -0,0 +1,20 @@ +#pragma once + +#include "lib/utils/bts_export.h" +#include +#include + +namespace bts { +namespace api { + +class ApiException : public std::exception +{ + std::string _what; +public: + ApiException(const std::string& what) : _what(what) {} + ~ApiException() {} + + const char* what() const throw() { return _what.c_str(); } +}; + +}}//namespace diff --git a/src/lib/api/public/ApiServer.cpp b/src/lib/api/public/ApiServer.cpp new file mode 100644 index 000000000..54b000ed3 --- /dev/null +++ b/src/lib/api/public/ApiServer.cpp @@ -0,0 +1,14 @@ +#ifndef NO_CPP0X +#include "ApiServer.h" +#include "ServiceContext.h" +namespace bts { +namespace api { + +ApiServer::ApiServer(const ServiceContext& ctx, const std::string& type, const std::string& qualifier) : + _channel(ctx.publish(*this, type, qualifier)) +{ } + +ApiServer::~ApiServer() { } + +}}//namespace +#endif diff --git a/src/lib/api/public/ApiServer.h b/src/lib/api/public/ApiServer.h new file mode 100644 index 000000000..e50be4d3a --- /dev/null +++ b/src/lib/api/public/ApiServer.h @@ -0,0 +1,65 @@ +#pragma once + +#include "lib/utils/bts_export.h" +#include "../core/ApiClientInterop.h" +#include "ApiUtils.h" +#include "ServerChannel.h" + +#include "lib/api/apicore.pb.h" + +#include +#include +#include +#include + +namespace bts { +namespace api { + +class ServiceContext; +class BTS_EXPORT ApiServer +{ +public: + ApiServer(const ServiceContext&, const std::string& type, const std::string& qualifier); + virtual ~ApiServer(); + + typedef std::function OnResponse; + virtual void dispatch(const RequestHeader&, google::protobuf::io::ZeroCopyInputStream&, OnResponse) = 0; + virtual const char* name() const = 0; + +protected: + ServerChannel _channel; + + template + void respond(int request_id, const T& response, OnResponse onreply) { + ResponseHeader header; + header.set_code(ApiError::SUCCESS); + header.set_request_id(request_id); + int header_size = header.ByteSize() + 1; + int body_size = response.ByteSize(); + int total_size = header_size + body_size; + if (total_size <= 1024) { + uint8_t buffer[1024]; + serialize(header_size, body_size, header, response, buffer); + onreply(buffer, total_size); + } + else { + std::vector buffer(total_size); + serialize(header_size, body_size, header, response, &buffer[0]); + onreply(&buffer[0], total_size); + } + } + + template + static void serialize(int header_size, int body_size, const ResponseHeader& header, const T& response, uint8_t* buffer) { + buffer[0] = (uint8_t)header_size; + header.SerializePartialToArray(&buffer[1], header_size - 1); + response.SerializePartialToArray(&buffer[header_size], body_size); + } + + static void serialize(int header_size, const ResponseHeader& header, uint8_t* buffer) { + buffer[0] = header_size; + header.SerializePartialToArray(&buffer[1], header_size - 1); + } +}; + +}}//namespace diff --git a/src/lib/api/public/ApiUtils.h b/src/lib/api/public/ApiUtils.h new file mode 100644 index 000000000..30ca5171d --- /dev/null +++ b/src/lib/api/public/ApiUtils.h @@ -0,0 +1,28 @@ +#pragma once + +#include +#include +#include + +namespace bts { +namespace api { + + template + const std::string format_qualifier(const char* type_name, const T& t) { + google::protobuf::util::JsonPrintOptions options; + options.add_whitespace = false; + options.always_print_primitive_fields = true; + std::string result; + auto status = google::protobuf::util::MessageToJsonString(t, &result, options); + assert(status.ok()); + result.erase( std::remove_if(result.begin(), result.end(), [](char c) { + return + c == '+' || + c == '_'; + }), result.end()); + return type_name + result; + } + + +}}//namespace + diff --git a/src/lib/api/public/ClientChannel.cpp b/src/lib/api/public/ClientChannel.cpp new file mode 100644 index 000000000..3d6f581c2 --- /dev/null +++ b/src/lib/api/public/ClientChannel.cpp @@ -0,0 +1,13 @@ +#ifndef NO_CPP0X +#include "ClientChannel.h" +namespace bts { +namespace api { + +ClientChannel::ClientChannel() : _id(1) +{ +} + +ClientChannel::~ClientChannel() { } + +}}//namespace +#endif diff --git a/src/lib/api/public/ClientChannel.h b/src/lib/api/public/ClientChannel.h new file mode 100644 index 000000000..a29e54a18 --- /dev/null +++ b/src/lib/api/public/ClientChannel.h @@ -0,0 +1,117 @@ +#pragma once + +#include "../core/ApiClientInterop.h" +#include "Result.h" +#include "ApiException.h" +#include "RawUpdate.h" +//#include "Update.h" +#include +#include +#include + +namespace bts { +namespace api { + + +class ClientChannel +{ + int _id; +public: + ClientChannel(); + virtual ~ClientChannel(); + + template + void send(int method_id, const Request& request, std::function)> on_response) { + RequestHeader header; + header.set_method(method_id); + header.set_request_id(++_id); + int header_size = header.ByteSize()+1; + assert(header_size < 128); + + std::string request_buffer; + request_buffer.reserve(header_size + request.ByteSize()); + request_buffer += (char)(header_size); + bool success = header.AppendToString(&request_buffer); + success = success && request.AppendToString(&request_buffer); + if (!success) throw ApiException("Failed to serialize request."); + + send_core(request_buffer, [on_response](const char* response_buffer, size_t response_size) { + assert(response_size > 1); + int header_size = response_buffer[0]; + assert(header_size > 0); + google::protobuf::io::ArrayInputStream header_stream(response_buffer+1, header_size-1); + ResponseHeader header; + if (header.ParseFromZeroCopyStream(&header_stream)) { + if (header.code() == ApiError::SUCCESS) { + auto response = std::make_shared(); + google::protobuf::io::ArrayInputStream data_stream(response_buffer + header_size, response_size - header_size); + if (response->ParseFromZeroCopyStream(&data_stream)) { + on_response(Result(move(response))); + return; + } + } + else { + on_response(Result((ApiError)header.code())); + } + } + else { + on_response(Result(ApiError::UNEXPECTED_RESPONSE)); + } + }); + } + + + template + rxcpp::observable>> subscribe(const std::string& method_and_qualifier) { + return rxcpp::observable<>::create>>([=](rxcpp::subscriber>> observer) { + bool first_update = true; + auto cb = [observer, first_update](EventType event_type, const StreamUpdate* changes, int change_count) mutable { + if (event_type == EventType::TABLE_UPDATED) { + std::vector> added_items; + std::vector added_keys, + removed_keys; + for (size_t x = 0; x < change_count; ++x) { + const StreamUpdate& change = changes[x]; + if (change.type != StreamUpdate::REMOVE) { + added_keys.reserve(change_count); + added_items.reserve(change_count); + added_keys.push_back(change.key); + google::protobuf::io::ArrayInputStream stream(change.buffer, change.buffer_len); + auto item = std::make_shared(); + if (item->ParseFromZeroCopyStream(&stream)) { + added_items.push_back(move(item)); + } + } + if (change.type != StreamUpdate::CREATE) { + removed_keys.reserve(change_count); + removed_keys.push_back(change.key); + } + } + observer.on_next(RawUpdate>(added_items, added_keys, removed_keys)); + } + else if (event_type == EventType::TABLE_CREATED) { + if (!first_update) { + observer.on_next(RawUpdate>::Reset()); + } + first_update = false; + } + }; + observer.add( + subscribe_core(method_and_qualifier, cb)); + }); + } + + virtual bool is_sync_call_ok() = 0; + +protected: + typedef std::function OnReply; + typedef std::function Unsubscribe; + typedef std::function OnSubscriptionCB; + virtual void send_core(const std::string& request_buffer, OnReply on_response) = 0; + virtual Unsubscribe subscribe_core(const std::string& method_and_qualifier, OnSubscriptionCB cb) = 0; + +private: + ClientChannel(const ClientChannel& other);// = delete; +}; + +}}//namespace diff --git a/src/lib/api/public/QuickbusClientChannel.cpp b/src/lib/api/public/QuickbusClientChannel.cpp new file mode 100644 index 000000000..688fe378c --- /dev/null +++ b/src/lib/api/public/QuickbusClientChannel.cpp @@ -0,0 +1,48 @@ +#include "QuickbusClientChannel.h" +#include "QuickbusClientChannel.h" +#include "ClientChannel.h" +namespace bts { +namespace api { + +QuickbusClientChannel::QuickbusClientChannel(Quickbus& bus, const char* type_name, const char* service_qualifier) : + _pclient(api_client_create(&bus, type_name, service_qualifier)) +{ +} + +QuickbusClientChannel::~QuickbusClientChannel() { + if (_pclient != nullptr) { + api_client_destroy(_pclient); + _pclient = nullptr; + } +} + +void QuickbusClientChannel::send_core(const std::string& request_envalope, OnReply on_response) { + raw_response_callback trampoline = [](const char* buffer, size_t length, void* extraptr) { + OnReply* on_response = static_cast(extraptr); + assert(on_response != nullptr); + (*on_response)(buffer, length); + delete on_response; + }; + OnReply* heap_on_response = new OnReply(move(on_response)); + api_client_send(_pclient, request_envalope.c_str(), request_envalope.size(), trampoline, heap_on_response); +} + +ClientChannel::Unsubscribe QuickbusClientChannel::subscribe_core(const std::string & method_and_qualifier, OnSubscriptionCB cb) { + raw_subscription_callback trampoline = [](EventType event_type, const StreamUpdate* changes, int count, void* extra) { + OnSubscriptionCB* heap_cb = static_cast(extra); + assert(heap_cb != nullptr); + (*heap_cb)(event_type, changes, count); + }; + OnSubscriptionCB* heap_cb = new OnSubscriptionCB(cb); + auto subscription = api_client_subscribe(_pclient, method_and_qualifier.c_str(), trampoline, heap_cb); + return [this,subscription,heap_cb]() { + api_client_unsubscribe(_pclient, subscription); + delete heap_cb; + }; +} + +bool QuickbusClientChannel::is_sync_call_ok() { + return !api_client_is_on_qb_thread(_pclient); +} + +}}//namespace diff --git a/src/lib/api/public/QuickbusClientChannel.h b/src/lib/api/public/QuickbusClientChannel.h new file mode 100644 index 000000000..41f221a51 --- /dev/null +++ b/src/lib/api/public/QuickbusClientChannel.h @@ -0,0 +1,26 @@ +#pragma once + +#include "lib/utils/bts_no_copy.h" +#include "ClientChannel.h" + +namespace bts { +namespace api { + +class QuickbusClientChannel : public ClientChannel +{ + ApiClient* _pclient; + +public: + QuickbusClientChannel(Quickbus&, const char* type_name, const char* service_qualifier); + ~QuickbusClientChannel(); + +private: + BTS_NO_COPY(QuickbusClientChannel); + + virtual void send_core(const std::string& request_envalope, OnReply on_response) override; + virtual Unsubscribe subscribe_core(const std::string& method_and_qualifier, OnSubscriptionCB cb) override; + virtual bool is_sync_call_ok() override; + +}; + +}}//namespace diff --git a/src/lib/api/public/RawUpdate.h b/src/lib/api/public/RawUpdate.h new file mode 100644 index 000000000..be79fa432 --- /dev/null +++ b/src/lib/api/public/RawUpdate.h @@ -0,0 +1,32 @@ +#pragma once +#include +#include +namespace bts { +namespace api { + + template + struct RawUpdate { + RawUpdate(std::vector& added_items, std::vector& added_keys, std::vector& removed_keys) : + is_reset(false), + added_items(std::move(added_items)), + added_keys(std::move(added_keys)), + removed_keys(std::move(removed_keys)) + {} + + static RawUpdate Reset() { + return RawUpdate(); + } + + bool is_reset; + std::vector added_items; + std::vector added_keys; + std::vector removed_keys; + + private: + RawUpdate() : + is_reset(true), + added_items(), added_keys(), removed_keys() + {} + }; + +} }//ns \ No newline at end of file diff --git a/src/lib/api/public/Result.h b/src/lib/api/public/Result.h new file mode 100644 index 000000000..70366d797 --- /dev/null +++ b/src/lib/api/public/Result.h @@ -0,0 +1,75 @@ +#pragma once + +#include "ApiException.h" + +#include "lib/api/apicore.pb.h" + +#include +#include +#include + + +namespace bts { +namespace api { + +template +class Result +{ + std::shared_ptr _t; + ApiError _code; + + static const char* error_string(ApiError code) { + switch (code) { + case ApiError::SUCCESS: return "Success"; + case ApiError::NO_SERVER: return "No server found."; + case ApiError::SERVER_DISCONNECT: return "Server disconnected."; + case ApiError::MISSING_METHOD: return "Method not found."; + case ApiError::SERVER_FAULT: return "Server error."; + case ApiError::UNEXPECTED_RESPONSE: return "Internal error"; + default: return "Unknown error"; + } + } + +public: + Result(std::shared_ptr& t): + _t(t), + _code(ApiError::SUCCESS) + { } + Result(std::shared_ptr&& t): + _t(t), + _code(ApiError::SUCCESS) + { } + Result(ApiError code) : + _t(), + _code(code) + { + assert(_code != ApiError::SUCCESS); + } + +#if _MSC_VER <= 1910 + // needed for std::future + Result() : + _t(), + _code(ApiError::UNEXPECTED_RESPONSE) + {} +#endif + + std::shared_ptr result() { + if (success()) return _t; + else throw ApiException(std::string(error_message())); + } + + bool success() { return _code == ApiError::SUCCESS; } + ApiError code() { return _code; } + const char* error_message() { return error_string(_code); } + + ~Result() {} + + // Is not a type?? + // typedef std::function)> OnResponse; +}; + +//template +//using OnResponse = std::function)>; + +}}//namespace diff --git a/src/lib/api/public/ServerChannel.cpp b/src/lib/api/public/ServerChannel.cpp new file mode 100644 index 000000000..a614806e8 --- /dev/null +++ b/src/lib/api/public/ServerChannel.cpp @@ -0,0 +1,63 @@ +#ifndef NO_CPP0X +#include "ServerChannel.h" +#include "../core/ApiServerInterop.h" +#include "ApiServer.h" +#include +namespace bts { +namespace api { + + +struct ServerChannel::ServerChannelImpl { + ServerChannelImpl(Quickbus& qbus, const std::string& service_type, const std::string& qualifier, ApiServer& server): + _server(server), + _core(api_server_create(&qbus, service_type.c_str(), qualifier.c_str(), dispatch_trampoline, this)) + { + } + //Need to guaruntee this object is stationary for the api_server_create trampoline + ServerChannelImpl(const ServerChannelImpl& ) = delete; + ServerChannelImpl(ServerChannelImpl&&) = delete; + + static void dispatch_trampoline(const uint8_t* buffer, size_t request_size, ReplyHandler* callback, void* extra) { + assert(extra != nullptr); + ServerChannelImpl* pimpl = static_cast(extra); + pimpl->dispatch(buffer, request_size, callback); + } + + void dispatch(const uint8_t* request_buffer, size_t request_length, ReplyHandler* callback) { + assert(request_buffer != nullptr); + assert(request_length > 1); + size_t header_length = request_buffer[0]; + assert(request_length >= header_length); + RequestHeader header; + if (header.ParsePartialFromArray(request_buffer + 1, header_length - 1)) { + google::protobuf::io::ArrayInputStream stream(request_buffer + header_length, request_length - header_length); + _server.dispatch(header, stream, [=](const uint8_t* buffer, size_t length) { + ReplyHandler* handler = callback; + api_server_call_callback(&handler, buffer, (int)length); + }); + } + } + + ApiServer& _server; + ApiServerCore* _core; +}; + + +ServerChannel::ServerChannel(Quickbus& qbus, const std::string& service_type, const std::string& qualifier, ApiServer& server) + :_me(new ServerChannelImpl(qbus, service_type, qualifier, server)) +{ } +ServerChannel::ServerChannel(ServerChannel&& other) : _me(std::move(other._me)) {} + +ServerChannel::~ServerChannel() { } + + +TableHandle ServerChannel::create_table(const std::string& method_and_qualifier) const { + return api_server_create_table(_me->_core, method_and_qualifier.c_str()); +} +void ServerChannel::publish(TableHandle table, StreamUpdate* changes, int change_count) const { + api_server_update_table(table, changes, change_count); +} + + +}}//namespace +#endif diff --git a/src/lib/api/public/ServerChannel.h b/src/lib/api/public/ServerChannel.h new file mode 100644 index 000000000..102db0bf3 --- /dev/null +++ b/src/lib/api/public/ServerChannel.h @@ -0,0 +1,27 @@ +#pragma once + +#include "../core/ApiClientInterop.h" +#include "../core/ApiServerInterop.h" +#include "lib/api/apicore.pb.h" + +namespace bts { +namespace api { + +class ApiServer; +class ServerChannel +{ +public: + ServerChannel(Quickbus& qbus, const std::string& service_type, const std::string& qualifier, ApiServer& server); + ServerChannel(ServerChannel&& other);// : _me(std::move(other._me)) {} + ~ServerChannel(); + + TableHandle create_table(const std::string& method_and_qualifier) const; + void publish(TableHandle table, StreamUpdate* changes, int change_count) const; + +private: + struct ServerChannelImpl; + std::unique_ptr _me; +}; + +}}//namespace + diff --git a/src/lib/api/public/ServiceContext.cpp b/src/lib/api/public/ServiceContext.cpp new file mode 100644 index 000000000..571209ee1 --- /dev/null +++ b/src/lib/api/public/ServiceContext.cpp @@ -0,0 +1,29 @@ +#ifndef NO_CPP0X +#include "ServiceContext.h" +#include "QuickbusClientChannel.h" +#include "ApiServer.h" +#include "lib/logging/logging.h" + +namespace bts { +namespace api { + +ServiceContext::ServiceContext(Quickbus* maybe_bus) : + _qbus(maybe_bus ? *maybe_bus : *api_create_quickbus(nullptr, 0)) +{ +} + +ServiceContext::~ServiceContext() +{ +} + +std::unique_ptr ServiceContext::create_channel(const char * type_name, const char * service_qualifier) { + LOG(Api,Info,"Locate Service: {}{}",type_name ,service_qualifier); + return std::unique_ptr(new QuickbusClientChannel(_qbus, type_name, service_qualifier)); +} + +ServerChannel ServiceContext::publish(ApiServer& service, const std::string& service_type, const std::string& qualifier) const { + return ServerChannel(_qbus, service_type, qualifier, service); +} + +}}//namespace +#endif diff --git a/src/lib/api/public/ServiceContext.h b/src/lib/api/public/ServiceContext.h new file mode 100644 index 000000000..0b929bffe --- /dev/null +++ b/src/lib/api/public/ServiceContext.h @@ -0,0 +1,37 @@ +#pragma once + +#include "lib/utils/bts_export.h" + +// client +#include "../core/ApiClientInterop.h" +#include "../core/ApiServerInterop.h" +#include "ClientChannel.h" +// server +#include "ServerChannel.h" + +#include +#include + +namespace bts { +namespace api { + +class BTS_EXPORT ServiceContext +{ + Quickbus& _qbus; + +public: + ServiceContext(Quickbus* maybe_bus=nullptr); + ~ServiceContext(); + + template + T locate(const char* qualifier="") { + return T(create_channel(T::name(), qualifier)); + } + +private: + friend class ApiServer; + std::unique_ptr create_channel(const char*, const char*); + ServerChannel publish(ApiServer& service, const std::string& service_type, const std::string& qualifier) const; +}; + +}}//namespace diff --git a/src/lib/api/public/Update.h b/src/lib/api/public/Update.h new file mode 100644 index 000000000..d37345679 --- /dev/null +++ b/src/lib/api/public/Update.h @@ -0,0 +1,79 @@ +#pragma once + +#include "RawUpdate.h" +#include +#include +#include +#include + +namespace bts { +namespace api { + + + template + struct Update { + Update(std::vector& all, std::vector& added, std::vector& removed) : + _all_items(std::move(all)), + _added_items(std::move(added)), + _removed_items(std::move(removed)) + {} + + Update(const Update& other) = delete; + Update(Update&& other) : + _all_items(std::move(other._all_items)), + _added_items(std::move(other._added_items)), + _removed_items(std::move(other._removed_items)) + {} + + TKey key(const T& t) const { return ServiceType::key(t); } + const std::vector& all() const { return _all_items; } + const std::vector& added() const { return _added_items; } + const std::vector& removed() const { return _removed_items; } + + private: + std::vector _all_items; + std::vector _added_items; + std::vector _removed_items; + }; + + template + std::function>(const rxcpp::observable>&)> reconstitute() { + return [](const rxcpp::observable>& input) { + auto state = std::make_shared>(); + return input.map([state](RawUpdate& raw) { + if (raw.is_reset) { + std::vector removed; + removed.reserve(state->size()); + for (auto& t : *state) { + removed.push_back(std::move(t.second)); + } + state->clear(); + std::vector all, added; + return Update(all, added, removed); + } + else { + std::vector removed; + removed.reserve(raw.removed_keys.size()); + for (int key : raw.removed_keys) { + auto iter = state->find(key); + if (iter != state->end()) { + removed.push_back(iter->second); + state->erase(iter); + } + } + for (size_t x = 0; x < raw.added_keys.size(); ++x) { + (*state)[raw.added_keys[x]] = raw.added_items[x]; + } + std::vector all; + all.reserve(state->size()); + for (const auto& item : *state) { + all.push_back(item.second); + } + return Update(all, raw.added_items, removed); + } + }); + }; + } + + +}}//namespace \ No newline at end of file diff --git a/src/lib/api/public/api.h b/src/lib/api/public/api.h new file mode 100644 index 000000000..1cb4a3753 --- /dev/null +++ b/src/lib/api/public/api.h @@ -0,0 +1,2 @@ +#pragma once +#include "public/ServiceContext.h" diff --git a/src/lib/api/service_compiler/apioptions_pb2.py b/src/lib/api/service_compiler/apioptions_pb2.py new file mode 100644 index 000000000..791b3e8ae --- /dev/null +++ b/src/lib/api/service_compiler/apioptions_pb2.py @@ -0,0 +1,74 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: apioptions.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +from google.protobuf import descriptor_pb2 +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from google.protobuf import descriptor_pb2 as google_dot_protobuf_dot_descriptor__pb2 + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='apioptions.proto', + package='bts.api', + syntax='proto3', + serialized_pb=_b('\n\x10\x61pioptions.proto\x12\x07\x62ts.api\x1a google/protobuf/descriptor.proto:=\n\x13message_description\x12\x1f.google.protobuf.MessageOptions\x18\xb4\x42 \x01(\t:=\n\x13service_description\x12\x1f.google.protobuf.ServiceOptions\x18\xb5\x42 \x01(\t:+\n\x03key\x12\x1d.google.protobuf.FieldOptions\x18\xb6\x42 \x01(\x08:9\n\x11\x66ield_description\x12\x1d.google.protobuf.FieldOptions\x18\xb7\x42 \x01(\tB\n\xaa\x02\x07\x62ts.apib\x06proto3') + , + dependencies=[google_dot_protobuf_dot_descriptor__pb2.DESCRIPTOR,]) +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + + +MESSAGE_DESCRIPTION_FIELD_NUMBER = 8500 +message_description = _descriptor.FieldDescriptor( + name='message_description', full_name='bts.api.message_description', index=0, + number=8500, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=True, extension_scope=None, + options=None) +SERVICE_DESCRIPTION_FIELD_NUMBER = 8501 +service_description = _descriptor.FieldDescriptor( + name='service_description', full_name='bts.api.service_description', index=1, + number=8501, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=True, extension_scope=None, + options=None) +KEY_FIELD_NUMBER = 8502 +key = _descriptor.FieldDescriptor( + name='key', full_name='bts.api.key', index=2, + number=8502, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=True, extension_scope=None, + options=None) +FIELD_DESCRIPTION_FIELD_NUMBER = 8503 +field_description = _descriptor.FieldDescriptor( + name='field_description', full_name='bts.api.field_description', index=3, + number=8503, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=True, extension_scope=None, + options=None) + +DESCRIPTOR.extensions_by_name['message_description'] = message_description +DESCRIPTOR.extensions_by_name['service_description'] = service_description +DESCRIPTOR.extensions_by_name['key'] = key +DESCRIPTOR.extensions_by_name['field_description'] = field_description + +google_dot_protobuf_dot_descriptor__pb2.MessageOptions.RegisterExtension(message_description) +google_dot_protobuf_dot_descriptor__pb2.ServiceOptions.RegisterExtension(service_description) +google_dot_protobuf_dot_descriptor__pb2.FieldOptions.RegisterExtension(key) +google_dot_protobuf_dot_descriptor__pb2.FieldOptions.RegisterExtension(field_description) + +DESCRIPTOR.has_options = True +DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\252\002\007bts.api')) +# @@protoc_insertion_point(module_scope) diff --git a/src/lib/api/service_compiler/apioptions_pb2.pyc b/src/lib/api/service_compiler/apioptions_pb2.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0de14023e7d0408cff2cdb7023549657ad0238e8 GIT binary patch literal 3133 zcmcImSyLNF5bl-CAt4ONN5B{ZK4STjV(^i;99Lo^Qd9zVt(;VOU$%BNBRpcYyR|cN zS*g4^za}qv$??PT1Cs6@tvC!xUO?6`-%NLZeNE%yU(@AZTiKg741bpK_gDDUei1<6 zKL;WJ4?qHR9S{Xr5JlJn$Z8^f*=%)_g*(5-@;qgzLu2M;CC(0Mj=0mMZRpMaRBK*a)}fVc!=5yWL) zmq1(ru?!L=SD0p%40TcKD)U{Vgo*3C-r)69kZA2D?=fZM$Bc)TZ-IB|0QGIAzQfc= z!2+j3jj`MX@A3iadrXa0L_Zr$eV3#DjHA8};z-mO`~M6R71zMKa)4`{xgIcAk-64R zC&O5y1uAoQ1;hn-Di?Eb8^KZz?jb-)E4GdRn`ITcS4o)~gzz=Siz9rU@zN2zZ1Fr9 z^KWp(SosQcZ&LeGLp~&l4+R$BMrhR@;X_k@b{wPVI{ zjRK?M`_@^)GHxeAGIkNdY5DF_W=1|+{xTj3soR-KO_JHO2|Vjbt%F_blp&!^b|TrP zR9K-IJkx_-m_)t^Ob`ZoD17#iMM|5YQpEw9oW;$4O6zHsn8XlW8npM2PSYOCQb!?9 zl^(<(zC$_NPY3#>qJ$UkE z`P?vGKfl8`tIKt(X{AuJzHPJe(ze9>Z_ihpT8Y&2#`$;kmTj+G{dcQ=r(E2VgX+>p zY@13(A{YLHgeRYTQJ(A2@gYb5L-kMQd0zgdkmn>EA7-&Hn`rztdS%NO^1q0G1y%pjGDZ1H1ir>RdThUB9ll#!x7 zHFLPU4KxPY3Hp)o-vv=$a`_LRpcrVp|))&*iOK zqliyECRs|vM>W?Ac$#Z|d`mBOzudK32$@q4G-v3Y%qUV{#ht_usULFGM;hLxmutw$ zQNJabOUdTa`NKrJG{vmg{+Mtn0z)s}KQbp=3Xl`JVQb5KlJZ4pZsMhN!xxG1^H<9t ze_l!E^I)M_vx$RxV>H2dR&6v}FWjwHuV1?6B1*V&S7{@&Ba$n4wcTvBzW%1^=Nelt zcl^z*=64(Z&iC8Tn=U7X-{AoYihnbRwB)%YBu#H_?42aGo7Y)iy3|F6NLo$zhpiVU zleqN1OU~4{%{PbWT~aq!aIsFE%J(Tzvm88}ClzPjfF3{YXBe#?=`v73YRC un}3Rx;%u?xjPs{bm~p0y<4(z`I`hsVt`*#$ge%A27p5Kl6owpylJgJ$RrEIi literal 0 HcmV?d00001 diff --git a/src/lib/api/service_compiler/plugin.cmd b/src/lib/api/service_compiler/plugin.cmd new file mode 100755 index 000000000..87c4b8f13 --- /dev/null +++ b/src/lib/api/service_compiler/plugin.cmd @@ -0,0 +1,3 @@ +@setlocal +@cd %~dp0 +@python -u plugin.py diff --git a/src/lib/api/service_compiler/plugin.py b/src/lib/api/service_compiler/plugin.py new file mode 100755 index 000000000..c9b1c92c3 --- /dev/null +++ b/src/lib/api/service_compiler/plugin.py @@ -0,0 +1,226 @@ +#!/usr/bin/python +import os, sys + + +# cd to scripts working directory +abspath = os.path.abspath(__file__) +dname = os.path.dirname(abspath) +os.chdir(dname) + +from google.protobuf.compiler import plugin_pb2 as plugin +from google.protobuf.descriptor_pb2 import FieldDescriptorProto +from pybars import Compiler, strlist +compiler = Compiler() + +## Logging +enable_logging = False +def log(msg): + if enable_logging: + from urllib import urlencode + import urllib2 + url = "http://localhost:8080/" + data = urlencode(dict(log=msg)) + req = urllib2.Request(url, data) + urllib2.urlopen(req).read() + +# required to resolve custom options +# generate with: protoc apioptions.proto --python_out=temp -Id:\blue\external\protobuf\protobuf-3.0.2\src -I. +## Note: +## If this import fails with something like "'module' object has no attribute 'Default'" you need to +## use `aptitude remove python-protobuf` and `pip install protobuf` +import apioptions_pb2 + +def loadTemplate(lang, name): + path = os.path.join("templates", lang, name) + text = open(path).read() + utext = unicode(text, "utf-8") + return compiler.compile(utext) + +def fix_type_name(proto): + for service in proto.service: + for method in service.method: + method.input_type = method.input_type.lstrip('.') + method.output_type = method.output_type.lstrip('.') + +def parseParams(params): + # log(params) + result = dict() + if params: + parts = params.split(',') + for p in parts: + a,b = p.split("=") + result[a] = b + if 'make_filename' in params and not 'package_name' in params: + result['package_name'] = result['make_filename'].lstrip('/').split('/')[1] + return result + +def generate_insertions(params, request, response): + # Include exports definition + if os.name == 'posix': + for file in request.file_to_generate: + f1, f2 = response.file.add(), response.file.add() + f1.name = f2.name = os.path.splitext(file)[0] + ".pb.h" + f1.insertion_point = "includes" + f1.content = """#ifdef __GNUC__ + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Weffc++" + #endif""" + f2.insertion_point = "global_scope" + f2.content = "#pragma GCC diagnostic pop" + if 'package_name' in params and 'project_name' in params: + for file in request.file_to_generate: + f = response.file.add() + f.name = os.path.splitext(file)[0] + ".pb.h" + # log("Inserting into " + f.name) + f.insertion_point = "includes" + f.content = "#include <{0}/{1}/{1}_exports.h>".format(params["package_name"], params["project_name"]) + +def keyfield(proto_file, item): + name = item.split('.')[-1:][0] + msg = next((x for x in proto_file.message_type if x.name == name), None) + key = apioptions_pb2.key + if msg != None: + field = next ((x for x in msg.field if x.options.HasExtension(key) and x.options.Extensions[key] == True), None) + if field != None: + return field + return None + +def keyname(proto_file, item): + result = keyfield(proto_file, item) + if result == None: return "id" + return result.name + +def camel_case(item): + if item.isupper() and not '_' in item: return item + return item.title().replace('_', '') + +def stupid_python_dict(o): + d = dict() + for attr in dir(o): + d[attr] = getattr(o, attr) + return d + +def field_type(field, parent): + try: + if field['type'] == FieldDescriptorProto.TYPE_INT32: return "int" + elif field['type'] == FieldDescriptorProto.TYPE_BOOL: return "bool" + elif field['type'] == FieldDescriptorProto.TYPE_INT64: return "long" + elif field['type'] == FieldDescriptorProto.TYPE_SINT32: return "int" + elif field['type'] == FieldDescriptorProto.TYPE_SINT64: return "long" + elif field['type'] == FieldDescriptorProto.TYPE_UINT64: return "UInt64" + elif field['type'] == FieldDescriptorProto.TYPE_UINT32: return "uint" + elif field['type'] == FieldDescriptorProto.TYPE_DOUBLE: return "double" + elif field['type'] == FieldDescriptorProto.TYPE_FLOAT: return "float" + elif field['type'] == FieldDescriptorProto.TYPE_STRING: return "string" + elif field['type'] == FieldDescriptorProto.TYPE_ENUM: + nested_enums = [x.name for x in parent['enum_type']] + full_name = field['type_name'].lstrip('.') + short_name = full_name.split('.')[-1:][0] + if short_name in nested_enums: + full_name = "{0}.Types.{1}".format(parent['name'], short_name) + return full_name + elif field['type'] == FieldDescriptorProto.TYPE_MESSAGE: + map_types = dict([(x.name, x) for x in parent['nested_type'] if x.options.map_entry == True]) + full_name = field['type_name'].lstrip('.') + short_name = full_name.split('.')[-1:][0] + immutable_postfix = "Immutable" + if full_name.startswith('google'): + immutable_postfix = "" + full_name = short_name + if short_name in map_types: + t = map_types[short_name] + key_field = [f for f in t.field if f.name == 'key'][0] + value_field = [f for f in t.field if f.name == 'value'][0] + key_type = field_type(stupid_python_dict(key_field), parent) + value_type = field_type(stupid_python_dict(value_field), parent) + return "IReadOnlyDictionary<{0},{1}>".format(key_type, value_type) + return full_name + immutable_postfix + except Exception as e: + log(e) + log(field) + return "???" + +def immutable_field_wrapper(field, parent): + type_name = field_type(field, parent) + is_repeat = field['label'] == FieldDescriptorProto.LABEL_REPEATED + is_dict = 'IReadOnlyDictionary' in type_name + field_name = camel_case(field['name']) + if is_dict or '???' in type_name: return "// todo dic " + field_name + if is_repeat: + return "public IEnumerable<{0}> {1} => _impl.{1}.Cast<{0}>();".format(type_name, field_name) + return "public {0} {1} => _impl.{1};".format(type_name, field_name) + +def key_type(proto_file, item, lang): + # trying to make these the same between c++/c# so we don't have to have 2 methods + field = keyfield(proto_file, item) + if field == None: return "int" + types = FieldDescriptorProto + if field.type == types.TYPE_INT32: return "int" + elif field.type == types.TYPE_INT64: return "long" + elif field.type == types.TYPE_SINT32: return "int" + elif field.type == types.TYPE_SINT64: return "long" + elif field.type == types.TYPE_UINT32 and lang == 'c++': return "unsigned" + elif field.type == types.TYPE_UINT32 and lang == 'c#': return "uint" + elif field.type == types.TYPE_DOUBLE: return "double" + elif field.type == types.TYPE_FLOAT: return "float" + elif field.type == types.TYPE_STRING and lang == 'cpp': return "std::string" + elif field.type == types.TYPE_STRING and lang == 'c#': return "string" + return "int" + +def has_complex_key(proto_file, this, options, item, lang): + kt = key_type(proto_file, item, lang) + if kt == 'int' or kt == 'uint' or kt == 'unsigned': + return options['inverse'](this) + else: + return options['fn'](this) + +def generate_code(request, response): + params = parseParams(request.parameter) + if not "lang" in params: + response.error = "You must provide a language for the service generator plugin" + return + lang = params["lang"] + if lang == "cpp": generate_insertions(params, request, response) + if not 'no_services' in params: + current_proto_file = None + templates = [(f, loadTemplate(lang, f)) for f in os.listdir("templates/" + lang)] + helpers = { 'upper': lambda this,item: item.upper(), + 'lower': lambda this,item: item.lower(), + 'cap': lambda this,item: item[0].upper() + item[1:], + 'filename': lambda this,item: os.path.splitext(item)[0], + 'ns-open': lambda this,item: "\n".join(["namespace {0} {{".format(x) for x in item.split('.')]), + 'ns-close': lambda this,item: "}" * len(item.split('.')), + 'type': lambda this,item: item.split('.')[-1:][0], + #'include_exports': lambda this: strlist("#include <{0}/{1}/{1}_exports.h>".format(params["package_name"], params["project_name"])), + #'api': lambda this: (params["project_name"].upper()+ "_API"), + 'camel': lambda _,f: camel_case(f), + 'immutable_field_wrapper': immutable_field_wrapper + # 'keyfield': * defined below * + } + for proto_file in request.proto_file: + if proto_file.name in request.file_to_generate: + fix_type_name(proto_file) + helpers['keyfield'] = lambda _,item: keyname(proto_file, item) + helpers['keyfield_camelcase'] = lambda _,item: camel_case(keyname(proto_file,item)) + helpers['key_type'] = lambda _,item: key_type(proto_file, item, lang) + helpers['has_complex_key'] = lambda this,options,item: has_complex_key(proto_file, this, options, item, lang) + # Path on response.file name is interpreted differently by the c# and c++ code gen + filename = ('strip_out_path' in params) and os.path.split(proto_file.name)[1] or proto_file.name + source_name = os.path.splitext(filename)[0] + for name,template in templates: + f = response.file.add() + f.name = source_name + "." + name + #log("creating: " + f.name) + f.content = template(proto_file, helpers) + +#### MAIN ######### +if __name__ == "__main__": + data = sys.stdin.read() + request = plugin.CodeGeneratorRequest() + #open("data", 'wb').write(data) + request.ParseFromString(data) + response = plugin.CodeGeneratorResponse() + generate_code(request, response) + output = response.SerializeToString() + sys.stdout.write(output) + diff --git a/src/lib/api/service_compiler/templates/c#/client.pb.cs b/src/lib/api/service_compiler/templates/c#/client.pb.cs new file mode 100644 index 000000000..62d66e9e3 --- /dev/null +++ b/src/lib/api/service_compiler/templates/c#/client.pb.cs @@ -0,0 +1,44 @@ +#region Generated code +using System; +using System.Threading.Tasks; +using bts.api.core; + +namespace {{package}} { +{{#each service}} +public sealed class {{name}} : IApiClient, IDisposable { + private readonly ClientChannel _channel; + private {{name}}(ClientChannel channel) { + _channel = channel; + } + + public void Dispose() { + _channel.Dispose(); + } + + {{#each method}} + {{#if server_streaming}} + public IObservable> subscribe_{{name}}({{input_type}} qualifier=null) { + return _channel.subscribe<{{output_type}}>(ApiServer.format_qualifier("{{name}}",qualifier), {{output_type}}.Parser).reconstitute(key); + } + private static {{key_type output_type}} key({{output_type}} row) { + return row.{{keyfield_camelcase output_type}}; + } + {{else}} + public Task> {{name}}_async({{input_type}} request) { + return _channel.send<{{input_type}},{{output_type}}>({{@index}}, request); + } + public {{output_type}} {{name}}({{input_type}} request) { + if (_channel.is_sync_call_ok()) { + return {{name}}_async(request).Result.result; + } + else { + throw new bts.api.core.ApiException("API blocking call cannot be made on this thread. Please use {{name}}_async instead."); + } + } + {{/if}} + {{/each}} +} + +{{/each}} +} +#endregion diff --git a/src/lib/api/service_compiler/templates/c#/immutable.pb.cs b/src/lib/api/service_compiler/templates/c#/immutable.pb.cs new file mode 100644 index 000000000..81f33e6b9 --- /dev/null +++ b/src/lib/api/service_compiler/templates/c#/immutable.pb.cs @@ -0,0 +1,36 @@ +#region Generated code +using System; +using System.Collections.Generic; +using System.Linq; +using Google.Protobuf; +using Google.Protobuf.Reflection; +using Google.Protobuf.WellKnownTypes; + +namespace {{package}} { + {{#each message_type}} + public partial struct {{name}}Immutable { + readonly {{name}} _impl; + + public static MessageDescriptor Descriptor => {{name}}.Descriptor; + public int CalculateSize() => _impl.CalculateSize(); + // Cannot implement IMessage because of this method + //public void MergeFrom(CodedInputStream input) => _impl.MergeFrom(input); + public void WriteTo(CodedOutputStream output) => _impl.WriteTo(output); + + public {{name}}Immutable({{name}} src, bool clone=true) { + _impl = clone ? src.Clone() : src; + } + + // Implicit conversion operator + public static implicit operator {{name}}Immutable({{name}} other) { + return new {{name}}Immutable(other, false); + } + + {{#each field}} + {{{immutable_field_wrapper ../}}} + {{/each}} + + } + {{/each}} +} +#endregion diff --git a/src/lib/api/service_compiler/templates/c#/server.pb.cs b/src/lib/api/service_compiler/templates/c#/server.pb.cs new file mode 100644 index 000000000..e70de4046 --- /dev/null +++ b/src/lib/api/service_compiler/templates/c#/server.pb.cs @@ -0,0 +1,172 @@ +#region Generated code + +using System; +using System.Threading.Tasks; +using System.Collections.Generic; +using bts.api.core; +using System.Linq; +using Google.Protobuf; +using System.Runtime.InteropServices; +using bts.api; + +namespace {{package}} +{ +{{#each service}} + +internal abstract class {{name}}Base : ApiServer, IDisposable { + readonly ServerChannel _channel; + public {{name}}Base(ServiceContext context, string qualifier="") { + _channel = context.publish(this, "{{name}}", qualifier); + } + + public virtual void Dispose() { + _channel.Dispose(); + } + + {{#each method}} + {{#if server_streaming}} + protected struct {{cap name}}Publisher { + readonly IntPtr _table; + {{#has_complex_key output_type}} + int _next_key; + Dictionary<{{key_type output_type}}, int> _keys; + int hash_key({{key_type output_type}} key) { + lock(_keys) { + int intkey; + if (_keys.TryGetValue(key, out intkey)) { + return intkey; + } + intkey = ++_next_key; + _keys[key] = intkey; + return intkey; + } + } + {{/has_complex_key}} + + public {{cap name}}Publisher(IntPtr table) { + _table = table; + {{#has_complex_key output_type}} + _next_key = 0; + _keys = new Dictionary<{{key_type output_type}}, int>(); + {{/has_complex_key}} + } + + ApiNativeMethods.StreamUpdate pack({{output_type}} item, bool is_change, List pinned_buffers) { + var data = item.ToByteArray(); + var handle = GCHandle.Alloc(data, GCHandleType.Pinned); + pinned_buffers.Add(handle); + return new ApiNativeMethods.StreamUpdate() { + type = is_change ? ApiNativeMethods.ChangeType.REPLACE : ApiNativeMethods.ChangeType.CREATE, + {{#has_complex_key output_type}} + key = hash_key({{name}}_key(item)), + {{else}} + key = (int){{name}}_key(item), + {{/has_complex_key}} + buffer_len = data.Length, + buffer = handle.AddrOfPinnedObject() + }; + } + public void publish( + List<{{output_type}}> added=null, + HashSet<{{key_type output_type}}> removed=null, + List<{{output_type}}> changed=null) { + int add_count = added == null ? 0 : added.Count; + int change_count = changed == null ? 0 : changed.Count; + int remove_count = removed == null ? 0 : removed.Count; + var changes = new ApiNativeMethods.StreamUpdate[add_count + change_count + remove_count]; + int change_index = 0; + var pinned_buffers = new List(add_count + change_count); + + try { + // Added + if (added != null) { + foreach (var a in added) { + var key = {{name}}_key(a); + var is_change = removed == null ? false : removed.Contains(key); + if (is_change) { removed.Remove(key); } + changes[change_index++] = pack(a, is_change, pinned_buffers); + } + } + // Changed + if (changed != null) { + foreach(var ch in changed) { + changes[change_index++] = pack(ch, true, pinned_buffers); + } + } + // Removed + if (removed != null) { + foreach(var r in removed) { + changes[change_index++] = new ApiNativeMethods.StreamUpdate() { + type = ApiNativeMethods.ChangeType.REMOVE, + {{#has_complex_key output_type}} + key = hash_key(r), + {{else}} + key = (int)r, + {{/has_complex_key}} + buffer_len = 0 + }; + } + } + ApiNativeMethods.api_server_update_table(_table, changes, change_index); + } + finally { + foreach (var handle in pinned_buffers) { + handle.Free(); + } + } + } + public void append({{output_type}} single) { + publish(added: new List<{{output_type}}> { single }); + } + } + + protected {{cap name}}Publisher make_{{name}}_publisher({{input_type}} qualifier=null) { + IntPtr table = _channel.create_table(format_qualifier("{{name}}", qualifier)); + return new {{cap name}}Publisher(table); + } + protected static {{key_type output_type}} {{name}}_key({{output_type}} row) { + return row.{{keyfield_camelcase output_type}}; + } + {{else}} + public abstract Task<{{output_type}}> {{name}}({{input_type}} request); + {{/if}} + {{/each}} + + enum ServiceMethods { + SYS_PING = -1, + {{#each method}} + {{upper name}} = {{@index}}, + {{/each}} + } + + public override void dispatch(RequestHeader header, System.IO.Stream request_stream, Action on_response) { + switch((ServiceMethods)header.Method) { + case ServiceMethods.SYS_PING: { + var msg = PingPong.Parser.ParseFrom(request_stream); + respond(header.RequestId, Task.FromResult(msg), on_response); + break; + } + {{#each method}} + {{#unless server_streaming}} + case ServiceMethods.{{upper name}}: { + var msg = {{input_type}}.Parser.ParseFrom(request_stream); + var result = {{name}}(msg); + var id = header.RequestId; + result.ContinueWith(t => respond(id, t, on_response)); + break; + } + {{/unless}} + {{/each}} + default: + var buffer = new ResponseHeader(){ + RequestId = header.RequestId, + Code = ApiError.MissingMethod + }.serialize(); + on_response(buffer); break; + } + } +} + +{{/each}} +} +#endregion diff --git a/src/lib/api/service_compiler/templates/cpp/client.pb.cc b/src/lib/api/service_compiler/templates/cpp/client.pb.cc new file mode 100644 index 000000000..c1725bddb --- /dev/null +++ b/src/lib/api/service_compiler/templates/cpp/client.pb.cc @@ -0,0 +1,67 @@ +{{#if service}} +#include "{{filename name}}.client.pb.h" +#include "lib/api/public/ApiUtils.h" +#include + +using namespace rxcpp; +using namespace rxcpp::sources; +using namespace rxcpp::operators; + +{{ns-open package}} +{{#each service}} + +// Constructor +{{name}}::{{name}}(std::unique_ptr pchannel) : _pchannel(move(pchannel)) {} +{{name}}::{{name}}({{name}}&& other) : _pchannel(move(other._pchannel)) { } + +{{#each method}} + {{#if server_streaming}} + rxcpp::observable<{{cap name}}Update> {{../name}}::subscribe_{{name}}(const {{type input_type}}& qualifier) { + return + _pchannel->subscribe<{{type output_type}}>(format_qualifier("{{name}}", qualifier)) + | (bts::api::reconstitute,const {{key_type output_type}}, {{../name}}>()); + } + rxcpp::observable<{{cap name}}Update> {{../name}}::subscribe_{{name}}() { + return + _pchannel->subscribe<{{type output_type}}>("{{name}}") + | (bts::api::reconstitute,const {{key_type output_type}}, {{../name}}>()); + } + + {{#has_complex_key output_type}} + const {{key_type output_type}}& {{../name}}::{{name}}_key(const {{type output_type}}& row) { + return row.{{keyfield output_type}}(); + } + {{else}} + {{key_type output_type}} {{../name}}::{{name}}_key(const {{type output_type}}& row) { + return row.{{keyfield output_type}}(); + } + {{/has_complex_key}} + + {{else}} + + void {{../name}}::{{name}}(const {{type input_type}}& input, std::function)> callback, uint32_t timeout_ms) { + _pchannel->send({{@index}}, input, callback); + } + + Result<{{type output_type}}> {{../name}}::{{name}}(const {{type input_type}}& input, uint32_t timeout_ms) { + if (_pchannel->is_sync_call_ok()) { + std::promise> p; + auto future = p.get_future(); + {{name}}(input, [&](Result<{{type output_type}}> res) { + p.set_value(res); + }, timeout_ms); + future.wait(); + return future.get(); + } + else { + throw bts::api::ApiException("API blocking call cannot be made on this thread. Please use {{name}}_async instead."); + } + } + + {{/if}} + +{{/each}} + +{{/each}} +{{ns-close package}} +{{/if}} diff --git a/src/lib/api/service_compiler/templates/cpp/client.pb.h b/src/lib/api/service_compiler/templates/cpp/client.pb.h new file mode 100644 index 000000000..8e8bc4dc6 --- /dev/null +++ b/src/lib/api/service_compiler/templates/cpp/client.pb.h @@ -0,0 +1,58 @@ +#pragma once +{{include_exports}} +#include "{{filename name}}.pb.h" +#include "lib/api/public/ServiceContext.h" +#include "lib/api/public/ClientChannel.h" +#include "lib/api/public/Result.h" +#include "lib/api/public/Update.h" +#include +#include +#include +#include +#include + +{{ns-open package}} + +// Forward declaractions +{{#each message_type}} +class {{name}}; +{{/each}} + +{{#each service}} + +{{#each method}}{{#if server_streaming}} +class {{../name}}; +typedef bts::api::Update, const {{key_type output_type}}, {{../name}}> {{cap name}}Update; +{{/if}}{{/each}} + +class {{api}} {{name}} { + + std::unique_ptr _pchannel; + friend class bts::api::ServiceContext; + {{name}}(std::unique_ptr pchannel); + // Key functions + {{#each method}} {{#if server_streaming}} + {{#has_complex_key output_type}} + static const {{key_type output_type}}& {{name}}_key(const {{type output_type}}& row); + {{else}} + static {{key_type output_type}} {{name}}_key(const {{type output_type}}& row); + {{/has_complex_key}} + {{/if}} {{/each}} + + public: + {{name}}({{name}}&&); + static const char* name() { return "{{name}}"; } + + {{#each method}} + {{#if server_streaming}} + rxcpp::observable<{{cap name}}Update> subscribe_{{name}}(const {{type input_type}}& qualifier); + rxcpp::observable<{{cap name}}Update> subscribe_{{name}}(); + {{else}} + void {{name}}(const {{type input_type}}&, std::function)>, uint32_t timeout_ms=0); + bts::api::Result<{{type output_type}}> {{name}}(const {{type input_type}}&, uint32_t timeout_ms=0); + {{/if}} + {{/each}} +}; +{{/each}} + +{{ns-close package}} diff --git a/src/lib/api/service_compiler/templates/cpp/server.pb.cc b/src/lib/api/service_compiler/templates/cpp/server.pb.cc new file mode 100644 index 000000000..24dd6de30 --- /dev/null +++ b/src/lib/api/service_compiler/templates/cpp/server.pb.cc @@ -0,0 +1,167 @@ +{{#if service}} +#include "{{filename name}}.server.pb.h" +#include "lib/api/public/ServiceContext.h" +#include "lib/api/public/ApiUtils.h" +#include "lib/api/core/ApiServerInterop.h" +#include +{{ns-open package}} +{{#each service}} + +{{#each method}}{{#if server_streaming}} +{{#has_complex_key output_type}} +const {{key_type output_type}}& {{name}}_key(const {{type output_type}}& row) { + return row.{{keyfield output_type}}(); +} +{{else}} +{{key_type output_type}} {{name}}_key(const {{type output_type}}& row) { + return row.{{keyfield output_type}}(); +} +{{/has_complex_key}} +{{/if}}{{/each}} + +{{name}}Base::{{name}}Base(bts::api::ServiceContext& context, const std::string& qualifier) : + bts::api::ApiServer(context, "{{name}}", qualifier) + {} + +const char* {{name}}Base::name() const { return "{{name}}"; }; + +void {{name}}Base::dispatch(const bts::api::RequestHeader& header, google::protobuf::io::ZeroCopyInputStream& stream, OnResponse on_response) { + switch((ServiceMethods)header.method()) { + case SYS_PING: { + bts::api::PingPong pp; + if(pp.ParsePartialFromZeroCopyStream(&stream)) { + respond(header.request_id(), pp, on_response); + } + } break; + {{#each method}} {{#unless server_streaming}} + case {{upper name}}: { + {{type input_type}} msg; + if (msg.ParsePartialFromZeroCopyStream(&stream)) { + auto id = header.request_id(); + {{name}}(msg, [this,id,on_response](const {{type output_type}}& result) { + respond(id, result, on_response); + }); + } + } break; + {{/unless}}{{/each}} + default: { + bts::api::ResponseHeader response; + response.set_request_id(header.request_id()); + response.set_code(ApiError::MISSING_METHOD); + uint8_t buffer[64]; + int header_size = response.ByteSize() + 1; + serialize(header_size, response, buffer); + on_response(buffer, header_size); + } break; + } +} + +{{#each method}}{{#if server_streaming}} + +{{../name}}Base::{{cap name}}Publisher {{../name}}Base::make_{{name}}_publisher(const {{type input_type}}& qualifier) const { + TableHandle handle = _channel.create_table(format_qualifier("{{name}}", qualifier)); + return make_{{name}}_publisher(handle); +} +{{../name}}Base::{{cap name}}Publisher {{../name}}Base::make_{{name}}_publisher() const { + TableHandle handle = _channel.create_table("{{name}}"); + return make_{{name}}_publisher(handle); +} +{{../name}}Base::{{cap name}}Publisher {{../name}}Base::make_{{name}}_publisher(TableHandle handle) const { + {{#has_complex_key output_type}} + std::unordered_map<{{key_type output_type}}, int> _map; + int _next_id; + std::shared_ptr pmutex = std::make_shared(); + auto get_key = [=](const {{key_type output_type}}& key) mutable -> int { + std::lock_guard guard(*pmutex); + auto iter = _map.find(key); + if (iter == _map.end()) { + int id = ++_next_id; + _map.insert(iter, std::make_pair(key, id)); + return id; + } + return iter->second; + }; + {{/has_complex_key}} + return [=]( + const std::vector<{{type output_type}}>* added, + const std::vector<{{type output_type}}>* changed, + std::set<{{key_type output_type}}>* removed) mutable { + + int add_count = added ? added->size() : 0; + int change_count = changed ? changed->size() : 0; + int remove_count = removed ? removed->size() : 0; + int total_updates = add_count + change_count + remove_count; + + // Optimize small collections + StreamUpdate stack_buffer[256]; + std::vector heap_buffer; + StreamUpdate* pupdates = stack_buffer; + if (total_updates > 256) { + heap_buffer.resize(total_updates); + pupdates = &heap_buffer[0]; + } + + int dest = 0; + if (added) { + for(const auto& item : *added) { + StreamUpdate& update = pupdates[dest++]; + update.type = StreamUpdate::CREATE; + if (removed != nullptr) { + auto iter = removed->find({{name}}_key(item)); + if (iter != removed->end()) { + removed->erase(iter); + update.type = StreamUpdate::REPLACE; + } + } + {{#has_complex_key output_type}} + update.key = get_key({{name}}_key(item)); + {{else}} + update.key = (int)({{name}}_key(item)); + {{/has_complex_key}} + update.buffer_len = item.ByteSize(); + char* buffer = new char[update.buffer_len]; + item.SerializePartialToArray(buffer, update.buffer_len); + update.buffer =buffer; + } + } + if (changed) { + for (const auto& item : *changed) { + StreamUpdate& update = pupdates[dest++]; + update.type = StreamUpdate::REPLACE; + {{#has_complex_key output_type}} + update.key = get_key({{name}}_key(item)); + {{else}} + update.key = (int)({{name}}_key(item)); + {{/has_complex_key}} + update.buffer_len = item.ByteSize(); + char* buffer = new char[update.buffer_len]; + item.SerializePartialToArray(buffer, update.buffer_len); + update.buffer =buffer; + } + } + if (removed) { + for (const auto& key : *removed) { + StreamUpdate& update = pupdates[dest++]; + update.type = StreamUpdate::REMOVE; + {{#has_complex_key output_type}} + update.key = get_key(key); + {{else}} + update.key = (int)(key); + {{/has_complex_key}} + update.buffer_len = 0; + update.buffer = nullptr; + } + } + _channel.publish(handle, pupdates, dest); + for (size_t x = 0; x < dest; ++x) { + if (pupdates[x].buffer != nullptr) { + delete[] pupdates[x].buffer; + } + } + }; +} +{{/if}} {{/each}} + +{{/each}} +{{ns-close package}} +{{/if}} diff --git a/src/lib/api/service_compiler/templates/cpp/server.pb.h b/src/lib/api/service_compiler/templates/cpp/server.pb.h new file mode 100644 index 000000000..90f1a9973 --- /dev/null +++ b/src/lib/api/service_compiler/templates/cpp/server.pb.h @@ -0,0 +1,56 @@ +#pragma once +#include "{{filename name}}.pb.h" +#include "lib/api/public/ApiServer.h" +{{include_exports}} +#include +#include +#include +#include +#include + +// #include +// #include +namespace bts { namespace api { + class ServiceContext; +}} + +{{ns-open package}} + +// Forward declaractions +{{#each message_type}} +class {{name}};{{/each}} + +{{#each service}} + +class {{name}}Base : public bts::api::ApiServer { +protected: + {{name}}Base(bts::api::ServiceContext& context, const std::string& qualifier=""); + + const char* name() const override; + + {{#each method}} + {{#if server_streaming}} + typedef std::function*, const std::vector<{{type output_type}}>*, std::set<{{key_type output_type}}>*)> {{cap name}}Publisher; + {{cap name}}Publisher make_{{name}}_publisher() const; + {{cap name}}Publisher make_{{name}}_publisher(const {{type input_type}}& qualifier) const; + {{else}} + virtual void {{name}}(const {{type input_type}}& request, std::function reply) = 0; + {{/if}} + {{/each}} + + enum ServiceMethods { + SYS_PING = -1, + {{#each method}} + {{upper name}} = {{@index}}, + {{/each}} + }; + +private: + void dispatch(const bts::api::RequestHeader&, google::protobuf::io::ZeroCopyInputStream&, OnResponse) override; + {{#each method}}{{#if server_streaming}} + {{cap name}}Publisher make_{{name}}_publisher(TableHandle) const; + {{/if}}{{/each}} +}; + +{{/each}} +{{ns-close package}} diff --git a/src/lib/api/services/Hello.proto b/src/lib/api/services/Hello.proto new file mode 100644 index 000000000..78e328153 --- /dev/null +++ b/src/lib/api/services/Hello.proto @@ -0,0 +1,34 @@ +syntax="proto3"; +package bts.api.test; +option csharp_namespace="bts.api.test"; +import "lib/api/apioptions.proto"; + +message HelloRequest { + string name = 1; +} +message HelloResponse { + string message = 1; +} +message HelloLog { + string myid = 1 [(bts.api.key)=true]; + string message = 10; +} +message HelloSubArgs { + string name = 1; +} +service HelloService { + rpc say_hello(HelloRequest) returns (HelloResponse); + rpc greetings(HelloSubArgs) returns (stream HelloLog); + rpc yell_hello(HelloRequest) returns (HelloResponse); +} + +message ShoutRequest { + string msg = 1; +} +message ShoutResponse { + string MSG = 1; +} +service ShoutService { + option (bts.api.service_description) = "Makes input message louder"; + rpc upper(ShoutRequest) returns (ShoutResponse); +} diff --git a/src/lib/api/test/ApiTests.cpp b/src/lib/api/test/ApiTests.cpp new file mode 100644 index 000000000..a609b4f14 --- /dev/null +++ b/src/lib/api/test/ApiTests.cpp @@ -0,0 +1,90 @@ +#include "../public/ServiceContext.h" +#include "../public/ApiUtils.h" +#include "lib/utils/bts_export.h" +#include "HelloServiceServer.h" + +#include "lib/api/services/ApiUnitTestService.server.pb.h" + +#include +#include + +using namespace std; + +namespace bts { +namespace api { +namespace test { + + struct ApiTestServiceServer: ApiTestServiceBase { + ServiceContext& _ctx; + unique_ptr _hello; + unique_ptr _shout; + + ApiTestServiceServer(ServiceContext& ctx) : + ApiTestServiceBase(ctx, "cpp"), + _ctx(ctx), + _hello(), + _shout() + {} + // Inherited via ApiTestServiceBase + virtual void format_qualifier(const QualifierArgument & request, function reply) override { + TestResult result; + switch (request.qualifier_case()) { + case QualifierArgument::kSimple: + result.set_str( + bts::api::format_qualifier("format_qualifier", request.simple())); + break; + case QualifierArgument::kComplex: + result.set_str( + bts::api::format_qualifier("format_qualifier", request.complex())); + break; + default: + result.set_str("Unknown qualifier test case"); + break; + } + reply(result); + } + + // Inherited via ApiTestServiceBase + virtual void test_empty_reply(const EmptyRequest&, std::function reply) override { + TestResult result; + reply(result); + } + + virtual void run_hello_service(const EmptyRequest&, std::function reply) { + _hello.reset(new HelloServiceServer(_ctx)); + reply(TestResult()); + } + virtual void stop_hello_service(const EmptyRequest&, std::function reply) { + _hello.reset(); + reply(TestResult()); + } + virtual void run_shout_service(const EmptyRequest&, std::function reply) { + _shout.reset(new ShoutServiceServer(_ctx)); + reply(TestResult()); + } + virtual void stop_shout_service(const EmptyRequest&, std::function reply) { + _shout.reset(); + reply(TestResult()); + } + + + }; + +struct ApiTestBundle { + ApiTestBundle() : ctx(), server() {} + unique_ptr ctx; + unique_ptr server; +}; + +extern "C" BTS_EXPORT ApiTestBundle* apitest_start_apitest_service(bts::qbus::Quickbus* bus) { + ApiTestBundle* pbundle = new ApiTestBundle(); + pbundle->ctx.reset(new ServiceContext(bus)); + pbundle->server.reset(new ApiTestServiceServer(*pbundle->ctx)); + return pbundle; +} + +extern "C" BTS_EXPORT void apitest_stop_apitest_service(ApiTestBundle* server) { + if (server != nullptr) delete server; +} + +}}}//namespace diff --git a/src/lib/api/test/HelloServiceServer.cpp b/src/lib/api/test/HelloServiceServer.cpp new file mode 100644 index 000000000..67653e2fd --- /dev/null +++ b/src/lib/api/test/HelloServiceServer.cpp @@ -0,0 +1,66 @@ +#include "HelloServiceServer.h" +#include +#include + +namespace bts { +namespace api { +namespace test { + +HelloServiceServer::HelloServiceServer(bts::api::ServiceContext & context) : + HelloServiceBase(context), + _id(0), + _shouter(context.locate()), + _gpub(make_greetings_publisher()) +{ } + +void HelloServiceServer::publish(const std::string& msg, int* to_remove) { + std::vector added; + HelloLog row; + char id[128]; + sprintf(id, "%i", ++_id); + row.set_myid(id); + row.set_message(msg); + added.push_back(row); + std::set removed; + if (to_remove) { + sprintf(id, "%i", *to_remove); + removed.insert(id); + } + _gpub(&added, nullptr, &removed); +} + +void HelloServiceServer::say_hello(const HelloRequest & request, std::function reply) { + HelloResponse response; + response.set_message("hello " + request.name()); + publish(response.message()); + reply(response); +} + +void HelloServiceServer::yell_hello(const HelloRequest & request, std::function reply) { + + HelloResponse response; + ShoutRequest sr; + sr.set_msg("hello " + request.name()); + _shouter.upper(sr, [=](Result response) { + if (response.success()) { + HelloResponse hr; + hr.set_message(response.result()->msg()); + int rmid = _id; + publish(hr.message(), &rmid); + reply(hr); + } + }); + +} + +ShoutServiceServer::ShoutServiceServer(bts::api::ServiceContext & context) : + ShoutServiceBase(context) +{ } +void test::ShoutServiceServer::upper(const ShoutRequest & request, std::function on_result) { + ShoutResponse response; + response.set_msg( + boost::to_upper_copy(request.msg())); + on_result(response); +} + +}}}//namespace diff --git a/src/lib/api/test/HelloServiceServer.h b/src/lib/api/test/HelloServiceServer.h new file mode 100644 index 000000000..b4bbeb57d --- /dev/null +++ b/src/lib/api/test/HelloServiceServer.h @@ -0,0 +1,39 @@ +#pragma once + +#include "lib/utils/bts_no_copy.h" +#include "lib/utils/bts_export.h" + +#include "lib/api/services/HelloService.server.pb.h" +#include "lib/api/services/HelloService.client.pb.h" + +namespace bts { +namespace api { +namespace test { + +struct BTS_SYS_API HelloServiceServer : public HelloServiceBase +{ +public: + HelloServiceServer(bts::api::ServiceContext& context); + +private: + BTS_NO_COPY(HelloServiceServer); + int _id; + ShoutService _shouter; + GreetingsPublisher _gpub; + + void publish(const std::string& msg, int * to_remove=nullptr); + + // Inherited via HelloServiceBase + virtual void say_hello(const HelloRequest & request, std::function on_result) override; + virtual void yell_hello(const HelloRequest & request, std::function on_result) override; +}; + +struct BTS_SYS_API ShoutServiceServer : public ShoutServiceBase +{ + ShoutServiceServer(bts::api::ServiceContext& context); + // Inherited via ShoutServiceBase + virtual void upper(const ShoutRequest & request, std::function on_result) override; +}; + +}}}//namespace + diff --git a/src/lib/config/ConfigDownloader.cpp b/src/lib/config/ConfigDownloader.cpp new file mode 100755 index 000000000..b25e73bda --- /dev/null +++ b/src/lib/config/ConfigDownloader.cpp @@ -0,0 +1,82 @@ +#include "ConfigDownloader.h" + +#include +#include +#include + +#include "lib/utils/bts_assert.h" + +using namespace std; + +namespace bts { +namespace config { +namespace internal { + +ConfigDownloader::ConfigDownloader() : + _curl(NULL) +{ + curl_global_init(CURL_GLOBAL_DEFAULT); + _curl = curl_easy_init(); + bts_assert(_curl); +} + +ConfigDownloader::~ConfigDownloader() +{ + curl_easy_cleanup(_curl); +} + +size_t curl_write(char* buffer, size_t size, size_t count, void* stream) { + stringstream* out = static_cast(stream); + bts_assert(out); + size_t bytes = size * count; + out->write(buffer, bytes); + return count; +} + +bool is_http(const string& url) { + return url.substr(0,7) == "http://"; +} + +int no_linger(void*, curl_socket_t socket, curlsocktype) { + linger l; + l.l_onoff = 1; + l.l_linger = 0; + setsockopt(socket, SOL_SOCKET, SO_LINGER, (char*)&l, sizeof(l)); + return CURL_SOCKOPT_OK; +} + +ConfigDownloader::Result ConfigDownloader::get(const string& url, stringstream& out_stream) { + curl_easy_setopt(_curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(_curl, CURLOPT_WRITEFUNCTION, curl_write); + curl_easy_setopt(_curl, CURLOPT_WRITEDATA, &out_stream); + curl_easy_setopt(_curl, CURLOPT_VERBOSE, 0); + curl_easy_setopt(_curl, CURLOPT_FAILONERROR, 1); + curl_easy_setopt(_curl, CURLOPT_CONNECTTIMEOUT, 30); + curl_easy_setopt(_curl, CURLOPT_SOCKOPTFUNCTION, no_linger); + + CURLcode res = curl_easy_perform(_curl); + + long http_code = 0; + CURLcode r2 = curl_easy_getinfo(_curl, CURLINFO_RESPONSE_CODE, &http_code); + bts_assert(r2 == CURLE_OK); + + bool used_http = is_http(url); + if (res == CURLE_OK && (http_code == 200 || !used_http)) { + return SUCCESS; + } else + if (res == CURLE_OPERATION_TIMEDOUT + || res == CURLE_COULDNT_RESOLVE_HOST + || res == CURLE_COULDNT_RESOLVE_PROXY + || res == CURLE_COULDNT_CONNECT + || res == CURLE_REMOTE_ACCESS_DENIED + || res == CURLE_FTP_CANT_GET_HOST + ) { + // Cannot rely on logging before config is initialized + // LOG(Error, Config) << "Permanent failure loading '" << url << "' - CURL error: " << curl_easy_strerror(res); + return PERMANENT_FAILURE; + } + return FAILURE; +} + +}}}//namespace + diff --git a/src/lib/config/ConfigDownloader.h b/src/lib/config/ConfigDownloader.h new file mode 100755 index 000000000..d86a0c777 --- /dev/null +++ b/src/lib/config/ConfigDownloader.h @@ -0,0 +1,32 @@ +#pragma once + +#include "lib/utils/bts_no_copy.h" +#include +#include + +namespace bts { +namespace config { +namespace internal { + + +class ConfigDownloader +{ +public: + ConfigDownloader(); + ~ConfigDownloader(); + + enum Result { + FAILURE, + SUCCESS, + PERMANENT_FAILURE + }; + + Result get(const std::string& url, std::stringstream& out_stream); + +private: + BTS_NO_COPY(ConfigDownloader); + CURL* _curl; +}; + +}}}//namespace + diff --git a/src/lib/config/ConfigErrorCollector.cpp b/src/lib/config/ConfigErrorCollector.cpp new file mode 100755 index 000000000..408fb59e0 --- /dev/null +++ b/src/lib/config/ConfigErrorCollector.cpp @@ -0,0 +1,72 @@ +#include "ConfigErrorCollector.h" + +#include + +namespace bts { +namespace config { +namespace internal { + +ConfigErrorCollector::ConfigErrorCollector(IConfigLog& log, const char* file): + _log(log), + _file(file), + _errors() +{ +} + +inline std::string format(const std::string& file, int line, int column, const std::string& message) +{ + std::stringstream buffer; + buffer << file.c_str() << ":" << line+1 << ":" << column+1 << ": error: " << message; + return buffer.str(); +} + +void ConfigErrorCollector::AddError(int line, int column, const std::string& message) +{ + _errors.push_back(format(_file, line, column, message)); +} + +void ConfigErrorCollector::AddWarning(int line, int column, const std::string& message) +{ + _log.warning(format(_file, line, column, message)); +} + +// For debugging purposes +void ConfigErrorCollector::AddError( + const std::string& , // File name in which the error occurred. + const std::string& , // Full name of the erroneous element. + const proto::Message* , // Descriptor of the erroneous element. + ErrorLocation , // One of the location constants, above. + const std::string& // Human-readable error message. + ) +{ + //_log.warning(format(filename, 0, 0, message)); +} + +// For debugging purposes +void ConfigErrorCollector::AddWarning( + const std::string& , // File name in which the error occurred. + const std::string& , // Full name of the erroneous element. + const proto::Message* , // Descriptor of the erroneous element. + ErrorLocation , // One of the location constants, above. + const std::string& // Human-readable error message. + ) +{ + //_log.warning(format(filename, 0, 0, message)); +} + +std::string ConfigErrorCollector::ToString() +{ + if (_errors.size() == 0) return ""; + + std::string error = _errors.front(); + for (size_t x = 1; x < _errors.size(); x++) + { + error += '\n'; + error += _errors[x]; + } + + return error; +} + +}}}//namespace + diff --git a/src/lib/config/ConfigErrorCollector.h b/src/lib/config/ConfigErrorCollector.h new file mode 100755 index 000000000..c4113e14f --- /dev/null +++ b/src/lib/config/ConfigErrorCollector.h @@ -0,0 +1,53 @@ +#pragma once + +#include "ConfigLog.h" + +#include +#include +#include +#include + +namespace proto = google::protobuf; + +namespace bts { +namespace config { +namespace internal { + +class ConfigErrorCollector : + public proto::io::ErrorCollector, + public proto::DescriptorPool::ErrorCollector +{ +public: + ConfigErrorCollector(IConfigLog& log, const char* file); + + void AddError(int line, int column, const std::string& message) override; + void AddWarning(int line, int column, const std::string& message) override; + + void AddError( + const std::string& filename, // File name in which the error occurred. + const std::string& element_name, // Full name of the erroneous element. + const proto::Message* descriptor, // Descriptor of the erroneous element. + ErrorLocation location, // One of the location constants, above. + const std::string& message // Human-readable error message. + ) override; + + void AddWarning( + const std::string& filename, // File name in which the error occurred. + const std::string& element_name, // Full name of the erroneous element. + const proto::Message* descriptor, // Descriptor of the erroneous element. + ErrorLocation location, // One of the location constants, above. + const std::string& message // Human-readable error message. + ) override; + + std::string ToString(); + +private: + BTS_NO_COPY(ConfigErrorCollector); + + IConfigLog& _log; + std::string _file; + std::vector _errors; +}; + +}}}//namespace + diff --git a/src/lib/config/ConfigLog.cpp b/src/lib/config/ConfigLog.cpp new file mode 100755 index 000000000..bcfe9991e --- /dev/null +++ b/src/lib/config/ConfigLog.cpp @@ -0,0 +1,105 @@ +#include "ConfigLog.h" + +#include "lib/logging/logging.h" + +namespace bts { +namespace config { +namespace internal { + +struct ConfigLog::PImpl { + PImpl() : + print_errors(true), + logger_hub("config_logger"), + logger(logger_hub.make_logger()) + {} + + bool print_errors; + log::LoggerHub logger_hub; + log::Logger &logger; +}; + +ConfigLog::ConfigLog() : + me (new PImpl()) +{ + log::LoggingConfig cfg; + cfg.level_names = { {Diag,"Diag"},{Info,"Info"}, + {Warn,"Warn"},{Error,"Error"}, + {Fatal,"Fatal"},{Sys,"Sys"} }; + cfg.subsystem_names = { {Gen,"Gen"},{Log,"Log"} }; + + cfg.logger_initial_queue_size = 10; + + log::Filter f; + + f[Cfg.ordinal] = Warn; + cfg.sinks["console"] = std::make_pair (log::console_sink(),f); + + f[Cfg.ordinal] = Info; + cfg.sinks["syslog"] = std::make_pair (log::syslog_sink(),f); + + me->logger_hub.config(cfg); +} + +ConfigLog::~ConfigLog() +{ + me->logger.mark_inactive(); +} + +bool ConfigLog::print_errors() const {return me->print_errors;} +void ConfigLog::set_print_errors(bool value) {me->print_errors = value;} + +void ConfigLog::info(const std::string& msg) +{ + if (me->print_errors) + me->logger.enqueue(Cfg,Info,__FILE__,__LINE__,"{0}",msg); +} + +void ConfigLog::error(const std::string& msg) +{ + if (me->print_errors) + me->logger.enqueue(Cfg,Error,__FILE__,__LINE__,"{0}",msg); +} + +void ConfigLog::warning(const std::string& msg) +{ + if (me->print_errors) + me->logger.enqueue(Cfg,Warn,__FILE__,__LINE__,"{0}",msg); +} + +// +// Log Aggregator + +LogAggregator::LogAggregator(IConfigLog& log) : + _log(log), + _buffer() +{ +} + +void LogAggregator::info(const std::string& msg) { + _buffer << msg; +#ifdef _WIN32 + _buffer << std::endl; +#else + _buffer << " "; +#endif +} + +void LogAggregator::error(const std::string& msg) { + _log.error(msg); +} + +void LogAggregator::warning(const std::string& msg) { + _log.warning(msg); + _buffer << "> Warning: " << msg << std::endl; +#ifdef _WIN32 + _buffer << std::endl; +#else + _buffer << " "; +#endif +} + +std::string LogAggregator::text() { return _buffer.str(); } + + +}}}//namespace + diff --git a/src/lib/config/ConfigLog.h b/src/lib/config/ConfigLog.h new file mode 100755 index 000000000..19510a8d3 --- /dev/null +++ b/src/lib/config/ConfigLog.h @@ -0,0 +1,72 @@ +#pragma once + +#include "lib/utils/bts_no_copy.h" + +#include +#include +#include + +namespace bts { +namespace config { +namespace internal { + +class IConfigLog +{ +public: + virtual void info(const std::string& msg) = 0; + virtual void error(const std::string& msg) = 0; + virtual void warning(const std::string& msg) = 0; + virtual ~IConfigLog() { } +}; + +// A back door though the logging system +// To allow config diagnostic logging +class ConfigLog : public IConfigLog +{ +public: + ConfigLog(); + ~ConfigLog(); + + void info(const std::string& msg); + void error(const std::string& msg); + void warning(const std::string& msg); + + bool print_errors() const ; + void set_print_errors(bool value) ; + +private: + struct PImpl; + std::unique_ptr me; +}; + +// Log aggregator aggregates Info and warning messages into one string +// but passes error message through to the log immediately +class LogAggregator : public IConfigLog +{ +public: + LogAggregator(IConfigLog& real_log); + void info(const std::string&); + void error(const std::string&); + void warning(const std::string&); + + std::string text(); + +private: + IConfigLog& _log; + std::ostringstream _buffer; +}; + +class ErrorsOnlyLog : public IConfigLog +{ +public: + ErrorsOnlyLog(IConfigLog& log) : _log(log) {} + ~ErrorsOnlyLog() {} + void info(const std::string&) {} + void error(const std::string& msg) { _log.error(msg); } + void warning(const std::string& msg) { _log.error(msg); } +private: + IConfigLog& _log; +}; + +}}}//namespace + diff --git a/src/lib/config/ConfigPropertyEvaluator.cpp b/src/lib/config/ConfigPropertyEvaluator.cpp new file mode 100755 index 000000000..4a2fb47e1 --- /dev/null +++ b/src/lib/config/ConfigPropertyEvaluator.cpp @@ -0,0 +1,296 @@ +#include "ConfigPropertyEvaluator.h" + +#include "lib/utils/windows_wrapper.h" +#include "lib/utils/bts_assert.h" +#include "lib/expression/ExpressionEvaluator.h" +#include "ConfigurationState.h" + +#include +#include + +namespace bts { +namespace config { +namespace internal { + +using namespace bts::expression; +using namespace boost; +using expression::Value; +namespace proto = google::protobuf; + +class ConfigListRuleEvaluatorShim : public expression::IPropertyEvaluator +{ + expression::IPropertyEvaluator& _eval; +public: + ConfigListRuleEvaluatorShim(expression::IPropertyEvaluator& eval) : _eval(eval) { } + virtual ~ConfigListRuleEvaluatorShim() {} + virtual bool get_value(const std::string& property, Value* out_value) { return _eval.get_value(property, out_value); } +}; + + +ConfigPropertyEvaluator::ConfigPropertyEvaluator(ConfigurationState& config) + : _config(config), _rule_engine() +{ + std::shared_ptr ptr(new ConfigListRuleEvaluatorShim(*this)); + _rule_engine.reset(new expression::ExpressionEvaluator(ptr)); +} + +ConfigPropertyEvaluator::~ConfigPropertyEvaluator() +{ +} + +void get_field_scalar(const proto::Message& message, const proto::FieldDescriptor* field_desc, + Value& out_value) +{ + const proto::Reflection* mirror = message.GetReflection(); + + switch(field_desc->cpp_type()) + { + case proto::FieldDescriptor::CPPTYPE_INT32: + out_value = Value((double) mirror->GetInt32(message, field_desc)); + break; + case proto::FieldDescriptor::CPPTYPE_INT64: + out_value = Value((double) mirror->GetInt64(message, field_desc)); + break; + case proto::FieldDescriptor::CPPTYPE_UINT32: + out_value = Value((double) mirror->GetUInt32(message, field_desc)); + break; + case proto::FieldDescriptor::CPPTYPE_UINT64: + out_value = Value((double) mirror->GetUInt64(message, field_desc)); + break; + case proto::FieldDescriptor::CPPTYPE_DOUBLE: + out_value = Value(mirror->GetDouble(message, field_desc)); + break; + case proto::FieldDescriptor::CPPTYPE_FLOAT: + out_value = Value((double) mirror->GetFloat(message, field_desc)); + break; + case proto::FieldDescriptor::CPPTYPE_BOOL: + out_value = Value(mirror->GetBool(message, field_desc)); + break; + case proto::FieldDescriptor::CPPTYPE_STRING: + out_value = Value(mirror->GetString(message, field_desc)); + break; + case proto::FieldDescriptor::CPPTYPE_ENUM: + out_value = Value(mirror->GetEnum(message, field_desc)->name()); + break; + case proto::FieldDescriptor::CPPTYPE_MESSAGE: + out_value = Value((void*)&mirror->GetMessage(message, field_desc)); + break; + default: + throw std::invalid_argument("Unsupported field type in Config rule"); + } +} + +void get_field_vector(const proto::Message& message, const proto::FieldDescriptor* field_desc, Value& out_value) +{ + const proto::Reflection* mirror = message.GetReflection(); + size_t field_size = mirror->FieldSize(message, field_desc); + + std::vector values; + + switch(field_desc->cpp_type()) + { + case proto::FieldDescriptor::CPPTYPE_INT32: + for(size_t index = 0; index < field_size; index++) + { + values.push_back( + Value((double) mirror->GetRepeatedInt32(message, field_desc, index))); + } + break; + case proto::FieldDescriptor::CPPTYPE_INT64: + for(size_t index = 0; index < field_size; index++) + { + values.push_back( + Value((double) mirror->GetRepeatedInt64(message, field_desc, index))); + } + break; + case proto::FieldDescriptor::CPPTYPE_UINT32: + for(size_t index = 0; index < field_size; index++) + { + values.push_back( + Value((double) mirror->GetRepeatedUInt32(message, field_desc, index))); + } + break; + case proto::FieldDescriptor::CPPTYPE_UINT64: + for(size_t index = 0; index < field_size; index++) + { + values.push_back( + Value((double) mirror->GetRepeatedUInt64(message, field_desc, index))); + } + break; + case proto::FieldDescriptor::CPPTYPE_DOUBLE: + for(size_t index = 0; index < field_size; index++) + { + values.push_back( + Value((double) mirror->GetRepeatedDouble(message, field_desc, index))); + } + break; + case proto::FieldDescriptor::CPPTYPE_FLOAT: + for(size_t index = 0; index < field_size; index++) + { + values.push_back( + Value((double) mirror->GetRepeatedFloat(message, field_desc, index))); + } + break; + case proto::FieldDescriptor::CPPTYPE_BOOL: + for(size_t index = 0; index < field_size; index++) + { + values.push_back( + Value(mirror->GetRepeatedBool(message, field_desc, index))); + } + break; + case proto::FieldDescriptor::CPPTYPE_STRING: + for(size_t index = 0; index < field_size; index++) + { + values.push_back( + Value(mirror->GetRepeatedString(message, field_desc, index))); + } + break; + case proto::FieldDescriptor::CPPTYPE_ENUM: + for (size_t index = 0; index < field_size; index++) { + values.push_back( + Value(mirror->GetRepeatedEnum(message, field_desc, index)->name())); + } + break; + case proto::FieldDescriptor::CPPTYPE_MESSAGE: + for (size_t index = 0; index < field_size; index++) { + values.push_back( + Value((void*)&mirror->GetRepeatedMessage(message, field_desc, index))); + } + break; + default: + throw std::invalid_argument("Unsupported field type in Config rule"); + } + + out_value = Value(values); +} + +const proto::Message* lookup_key(std::shared_ptr evaluator, + const proto::Message* message, const proto::FieldDescriptor* field, const std::string& search_key) { + bts_assert(message); + bts_assert(field); + bts_assert(!search_key.empty()); + bts_assert(field->is_repeated()); + bts_assert(field->type() == proto::FieldDescriptor::TYPE_MESSAGE); + + const proto::Reflection* mirror = message->GetReflection(); + int size = mirror->FieldSize(*message, field); + + const proto::FieldDescriptor* key_field = NULL; + const proto::FieldDescriptor* rule_field = NULL; + const proto::Reflection* key_reflection = NULL; + for (int x = 0; x < size; x++) { + // Iterate through the field instances + const proto::Message& cur = mirror->GetRepeatedMessage(*message, field, x); + if (x == 0) { + // Grab the reflection and field descriptor for the repeated type + key_reflection = cur.GetReflection(); + bts_assert(key_reflection); + // Use the reflection object to look for a field named "key" or "name" + key_field = cur.GetDescriptor()->FindFieldByLowercaseName("key"); + if (key_field == NULL) { + key_field = cur.GetDescriptor()->FindFieldByLowercaseName("name"); + if (key_field == NULL) { + // If there is no key abandon ship + break; + } + } + + if (key_field->type() != proto::FieldDescriptor::TYPE_STRING) { + // To keep things simple the key must be a string + break; + } + rule_field = cur.GetDescriptor()->FindFieldByLowercaseName("rule"); + } + + std::string key; + key = key_reflection->GetStringReference(cur, key_field, &key); + if (key == search_key) + { + if(rule_field != NULL) + { + std::string rule; + rule = key_reflection->GetStringReference(cur, rule_field, &rule); + if(rule != "" && !evaluator->eval(rule)) + return cur.New(); + } + return &cur; + } + } + + return NULL; +} + +bool ConfigPropertyEvaluator::get_value(const std::string& property, Value* out_value) +{ + bts_assert(out_value); + if (property[0] == '$') + { + *out_value = _config.get_macro(property); + return true; + } + + // 1) Get the proto::Message* from config + tokenizer > tokens(property, char_separator(".[]")); // Note: Split arbitrarily on [], enforcing matching brackets is done in the parser + tokenizer >::iterator iter = tokens.begin (), end = tokens.end (); + + // The first token is the key name + const proto::Message* key = _config.try_get_message(*iter); + if (!key) return false; + + // 2) Drill down using reflection to find the named property + do + { + ++iter; + + if (iter == end) { + // We're asking for a complete Message + *out_value = Value((void*)key); + return true; + } + + const proto::Descriptor* message_desc = key->GetDescriptor(); + std::string field_name = *iter; + const proto::FieldDescriptor* field_desc = message_desc->FindFieldByName(field_name); + + if (field_desc) + { + if (field_desc->is_repeated()) + { + ++iter; + if (field_desc->type() == proto::FieldDescriptor::TYPE_MESSAGE && iter != end) + { + // Treat the next token as a key into the list + key = lookup_key(_rule_engine, key, field_desc, *iter); + if (key == NULL) return false; + } + else + { + get_field_vector(*key, field_desc, *out_value); + return (iter == end); + } + } + else + { + if (field_desc->type() != proto::FieldDescriptor::TYPE_MESSAGE) + { + get_field_scalar(*key, field_desc, *out_value); + return (++iter == tokens.end()); + } + else + { + // continue iterating + const proto::Reflection* mirror = key->GetReflection(); + key = &mirror->GetMessage(*key, field_desc); + } + } + } + else { + break; + } + + } while(key && iter != end); + + return false; +} + +}}}//namespace diff --git a/src/lib/config/ConfigPropertyEvaluator.h b/src/lib/config/ConfigPropertyEvaluator.h new file mode 100755 index 000000000..6583479f1 --- /dev/null +++ b/src/lib/config/ConfigPropertyEvaluator.h @@ -0,0 +1,31 @@ +#ifndef CONFIGPROPERTYEVALUATOR_H_ +#define CONFIGPROPERTYEVALUATOR_H_ + +#include "lib/utils/bts_no_copy.h" +#include "lib/expression/Value.h" +#include "lib/expression/IPropertyEvaluator.h" + +namespace bts { +namespace expression { class ExpressionEvaluator;} +namespace config { +namespace internal { + +class ConfigurationState; + +class ConfigPropertyEvaluator : public expression::IPropertyEvaluator +{ +public: + ConfigPropertyEvaluator(ConfigurationState& config); + ~ConfigPropertyEvaluator(); + + bool get_value(const std::string& property, expression::Value* out_value) override; + +private: + BTS_NO_COPY(ConfigPropertyEvaluator); + + ConfigurationState& _config; + std::shared_ptr _rule_engine; +}; + +}}}//namespace +#endif diff --git a/src/lib/config/ConfigValueQueue.cpp b/src/lib/config/ConfigValueQueue.cpp new file mode 100755 index 000000000..1195eac08 --- /dev/null +++ b/src/lib/config/ConfigValueQueue.cpp @@ -0,0 +1,294 @@ + +#include "ConfigValueQueue.h" +#include "ConfigurationException.h" + +#include "lib/utils/proto_helper.h" +#include +#include +#include "config.h" + +namespace bts { +namespace config { +namespace internal { + +using namespace ::google::protobuf; +using namespace bts::expression; +using namespace bts::util::proto; + +ConfigValueQueue::ConfigValueQueue(IPropertyEvaluator& evaluator) : + _eval(evaluator), + _vals(), + _current_file() +{ +} + +ConfigValueQueue::~ConfigValueQueue() +{ +} + +void ConfigValueQueue::clear() { + _vals.clear(); +} + +void ConfigValueQueue::process() { + try { + process_fields(); + } catch(...) { + clear(); + throw; + } +} + +void ConfigValueQueue::defer_value(proto::Message* msg, + const proto::Reflection* reflection, + const proto::FieldDescriptor* field, + int index, int line, int col, + const std::string& source, + const std::string& destination, + const proto::Message* root) { + + bts_assert(msg); + bts_assert(reflection); + bts_assert(field); + bts_assert(!source.empty()); + //bts_assert(!destination.empty()); + //bts_assert_d(root != nullptr); + DeferredValue dv = {msg, reflection, field, index, line, col, _current_file, source, destination, root}; + _vals.push_back(dv); +} + +void ConfigValueQueue::process_fields() { + std::vector::iterator iter = _vals.begin(); + std::vector temp; + size_t len; + do { + temp.clear(); + len = _vals.size(); + for (size_t x = 0; x < len; ++x) { + DeferredValue& dv = _vals[x]; + if (can_process(dv.sourceval)) { + process_field(dv); + } else { + temp.push_back(dv); + } + } + _vals = temp; + // Continue as long as progress is being made + } while(_vals.size() > 0 && _vals.size() < len); + + // Error condition + if (!_vals.empty()) { + std::ostringstream msg; + msg << "Unable to processes config value references. " << _vals.size() << " errors detected.\n\n"; + for (iter = _vals.begin(); iter != _vals.end(); iter++) { + msg << "Cannot write " << iter->sourceval << " due to circular reference. "; + msg << iter->filename << ":" << iter->line << "," << iter->col << std::endl; + } + std::string error = msg.str(); + throw new RecursiveReferenceException(error.c_str()); + } +} + +bool ConfigValueQueue::can_process(const std::string& source) { + // Determine if source has a pending write + size_t len = _vals.size(); + for(size_t x = 0; x < len; ++x) { + // If there is a destination assignment underneath the source path + // Then we cannot copy the value yet + const string& dest = _vals[x].dest_key; + if (dest.compare(0, source.size(), source) == 0) { + if (dest.size() <= source.size() || dest[source.size()] == '.') { + return false; + } + } + } + + return true; +} + +bool is_relevant(const proto::Message* message) { + if (message) { + const proto::Descriptor* desc = message->GetDescriptor(); + bts_assert_d(desc); + const proto::Message& result = config::get_message(desc); + return &result == message; + } + return true; +} + +void ConfigValueQueue::process_field(const DeferredValue& dv) { + Value val; + if (_eval.get_value(dv.sourceval, &val)) { + assign_value_to_field(dv.message, dv.reflection, dv.field, val, dv.index); + } else { + if (is_relevant(dv.root_message)) { + std::string message = "Error retrieving value for " + dv.sourceval; + throw SelfReferenceException(message.c_str(), dv.filename.c_str(), dv.line, dv.col); + } + } +} + +template +void ConfigValueQueue::assign( + proto::Message* message, + const proto::Reflection* reflection, + const proto::FieldDescriptor* field, + const Value& value, int& index) { + if (value.which() == ValueType::list) { + vector v = boost::get >(value); + if (field->is_repeated()) { + for (vector::iterator i = v.begin(); i != v.end(); ++i) { + int size = reflection->FieldSize(*message, field); + add(message, reflection, field, (TProto)boost::get(*i)); + if (index != size) { + for (int i = index; i < size; ++i) { + reflection->SwapElements(message, field, i, size); + } + } + index++; + } + bump_indexes(message, field, v.size()); + } + else if (field->type() == proto::FieldDescriptor::Type::TYPE_STRING) { + reflection->SetString(message, field, value_to_string(value)); + } + else if (field->type() == proto::FieldDescriptor::Type::TYPE_BOOL) { + reflection->SetBool(message, field, value_to_bool(value)); + } + else { + string error = "Invalid type conversion in assignment to " + message->GetTypeName() + "." + field->name(); + throw ConfigurationException(error.c_str()); + } + } else { + set_field(message, reflection, field, (TProto)boost::get(value), index); + if (field->is_repeated()) { + bump_indexes(message, field, 1); + } + } +} + +template +void ConfigValueQueue::lexical_assign( + proto::Message* message, + const proto::Reflection* reflection, + const proto::FieldDescriptor* field, + const Value& value, int& index) { + if (value.which() == ValueType::list) { + vector v = boost::get >(value); + bts_assert(field->is_repeated()); + for (vector::iterator i = v.begin(); i != v.end(); ++i) { + int size = reflection->FieldSize(*message, field); + TProto result = boost::lexical_cast(boost::get(*i)); + add(message, reflection, field, result); + if (index != size) { + for (int i = index; i < size; ++i) { + reflection->SwapElements(message, field, i, size); + } + } + index++; + } + bump_indexes(message, field, v.size()); + } else { + TProto result = boost::lexical_cast(boost::get(value)); + set_field(message, reflection, field, result, index); + if (field->is_repeated()) { + bump_indexes(message, field, 1); + } + } +} + +void ConfigValueQueue::assign_value_to_field(proto::Message* message, + const proto::Reflection* reflection, + const proto::FieldDescriptor* field, + const Value& value, int index) throw() { +#define MY_ARGS message, reflection, field, value, index + + // Note: ConfigPropertyEvaluator puts every numeric type in as double so the right hand side must be double + switch(field->type()) { + case FieldDescriptor::TYPE_DOUBLE: + assign(MY_ARGS); break; + case FieldDescriptor::TYPE_FLOAT: + assign(MY_ARGS); break; + case FieldDescriptor::TYPE_FIXED64: + case FieldDescriptor::TYPE_INT64: + assign(MY_ARGS); break; + case FieldDescriptor::TYPE_UINT64: + assign(MY_ARGS); break; + case FieldDescriptor::TYPE_FIXED32: + case FieldDescriptor::TYPE_INT32: + if (value.which() == ValueType::string) { + lexical_assign(MY_ARGS); + } + else { + assign(MY_ARGS); + } + break; + case FieldDescriptor::TYPE_UINT32: + assign(MY_ARGS); break; + case FieldDescriptor::TYPE_BOOL: + assign(MY_ARGS); break; + case FieldDescriptor::TYPE_STRING: + if (value.which() == ValueType::integer || value.which() == ValueType::real) { + lexical_assign(MY_ARGS); + } + else { + assign(MY_ARGS); + } + break; + case FieldDescriptor::TYPE_MESSAGE: + assign(MY_ARGS); break; + case FieldDescriptor::TYPE_ENUM: { + if (value.which() == ValueType::list) { + vector v = boost::get >(value); + for (vector::iterator iter = v.begin(); iter != v.end(); iter++) { + int size = reflection->FieldSize(*message, field); + const EnumDescriptor* enum_type = field->enum_type(); + const EnumValueDescriptor* enum_value = enum_type->FindValueByName(boost::get(value)); + reflection->AddEnum(message, field, enum_value); + if (index != size) { + for (int i = index; i < size; ++i) { + reflection->SwapElements(message, field, i, size); + } + } + index++; + } + bump_indexes(message, field, v.size()); + } + else { + const EnumDescriptor* enum_type = field->enum_type(); + const EnumValueDescriptor* enum_value = enum_type->FindValueByName(boost::get(value)); + set_field(message, reflection, field, enum_value, index); + if (field->is_repeated()) { + bump_indexes(message, field, 1); + } + } + break; + } + case FieldDescriptor::TYPE_SFIXED32: + case FieldDescriptor::TYPE_SFIXED64: + case FieldDescriptor::TYPE_SINT32: + case FieldDescriptor::TYPE_SINT64: + case FieldDescriptor::TYPE_BYTES: + case FieldDescriptor::TYPE_GROUP: + default: + bts_assert_msg(false, "Unsupported assignment destination"); + } + +#undef MY_ARGS +} + +void ConfigValueQueue::bump_indexes(proto::Message* message, + const proto::FieldDescriptor* field, + int count) +{ + std::vector::iterator iter = _vals.begin(); + for (;iter != _vals.end(); iter++) { + DeferredValue& dv = *iter; + if (dv.message == message && dv.field == field) { + dv.index += count; + } + } +} + +}}}//namespace + diff --git a/src/lib/config/ConfigValueQueue.h b/src/lib/config/ConfigValueQueue.h new file mode 100755 index 000000000..963ca7029 --- /dev/null +++ b/src/lib/config/ConfigValueQueue.h @@ -0,0 +1,101 @@ +#pragma once + +#include "lib/utils/bts_no_copy.h" +#include "lib/expression/Value.h" +#include "lib/expression/IPropertyEvaluator.h" +#include "text_format/text_format.h" + +#include +#include + +namespace bts { +namespace config { +namespace internal { +namespace proto = ::google::protobuf; + + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Weffc++" +#endif +struct DeferredValue { + proto::Message* message; + const proto::Reflection* reflection; + const proto::FieldDescriptor* field; + int index; + int line; + int col; + std::string filename; + std::string sourceval; + std::string dest_key; + const proto::Message* root_message; +}; +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif + +class ConfigValueQueue : public google::protobuf::IDeferValueLookup +{ +public: + ConfigValueQueue(expression::IPropertyEvaluator& evaluator); + ~ConfigValueQueue(); + + // Process the queue of deferred values + void process(); + void clear(); + + // Add a lookup value + virtual void defer_value(proto::Message*, + const proto::Reflection* reflection, + const proto::FieldDescriptor* field, + int index, int line, int col, + const std::string& value, + const std::string& key_name, const proto::Message* root) override; + + void set_filename(const std::string& filename) { _current_file = filename; } + +private: // fields + expression::IPropertyEvaluator& _eval; + std::vector _vals; + std::string _current_file; + BTS_NO_COPY(ConfigValueQueue); + +private: //methods + void process_fields(); + bool can_process(const std::string& key_name); + void process_field(const DeferredValue&); + + void assign_value_to_field(proto::Message* message, + const proto::Reflection* reflection, + const proto::FieldDescriptor* field, + const expression::Value& value, int insert_index=0) throw(); + void bump_indexes(proto::Message* message, + const proto::FieldDescriptor* field, + int count); + + template + void assign( + proto::Message* message, + const proto::Reflection* reflection, + const proto::FieldDescriptor* field, + const expression::Value& value, int& index); + + template + void lexical_assign( + proto::Message* message, + const proto::Reflection* reflection, + const proto::FieldDescriptor* field, + const expression::Value& value, int& index); + + template + void assign( + proto::Message* message, + const proto::Reflection* reflection, + const proto::FieldDescriptor* field, + const expression::Value& value, int& index) { + assign(message, reflection, field, value, index); + } +}; + +}}}//namespace + diff --git a/src/lib/config/ConfigurationException.cpp b/src/lib/config/ConfigurationException.cpp new file mode 100755 index 000000000..500b5d197 --- /dev/null +++ b/src/lib/config/ConfigurationException.cpp @@ -0,0 +1,37 @@ +#include "ConfigurationException.h" + +namespace bts { +namespace config { + + // + // ConfigurationException + // + ConfigurationException::ConfigurationException(const char* what) + : std::logic_error(""), _what(what) + { + } + + ConfigurationException::ConfigurationException(const std::string& what) + : std::logic_error(""), _what(what) + { + } + + ConfigurationException::ConfigurationException(const ConfigurationException& other) + : std::logic_error(""), _what(other._what) + { + } + + ConfigurationException& ConfigurationException::operator=(const ConfigurationException& other) + { + _what = other._what; + return *this; + } + + const char* ConfigurationException::what() const throw() + { + return _what.c_str(); + } + + ConfigurationException::~ConfigurationException() throw() {} + +}}//namespace diff --git a/src/lib/config/ConfigurationException.h b/src/lib/config/ConfigurationException.h new file mode 100755 index 000000000..ab487a9ee --- /dev/null +++ b/src/lib/config/ConfigurationException.h @@ -0,0 +1,75 @@ +#ifndef BADCONFIGURATIONEXCEPTION_H_ +#define BADCONFIGURATIONEXCEPTION_H_ + +#include +#include +#include +#include "lib/utils/bts_export.h" + +namespace bts { +namespace config { + +class ConfigurationException : public std::logic_error +{ +public: + ConfigurationException(const char* what); + ConfigurationException(const std::string& what); + ConfigurationException(const ConfigurationException& other); + ConfigurationException& operator=(const ConfigurationException& other); + virtual ~ConfigurationException() throw(); + + BTS_EXPORT const char* what() const throw() override; + +protected: + std::string _what; +}; + +class InvalidDescriptorException : public ConfigurationException +{ +public: + InvalidDescriptorException(const char* what) + : ConfigurationException(what) + { + _what += " was not compiled in to config."; + } +}; + +class InvalidKeyException : public ConfigurationException +{ +public: + InvalidKeyException(const char* what) + : ConfigurationException(what) + { + _what += " is not a valid config type."; + } +}; + +class SelfReferenceException : public ConfigurationException +{ +public: + SelfReferenceException(const char* what, const char* file, int line, int col) : ConfigurationException(what) { + std::ostringstream out; + out << what; + out << " Location: " << file << ":" << line << "," << col; + _what = out.str(); + } +}; + +class RecursiveReferenceException : public ConfigurationException +{ +public: + RecursiveReferenceException(const char* what) : ConfigurationException(what) { } +}; + +class ServerUnreachableException : public ConfigurationException +{ +public: + ServerUnreachableException(const char* what) : ConfigurationException(what) + { + _what = "Unable to reach server "; + _what += what; + } +}; + +}} +#endif diff --git a/src/lib/config/ConfigurationState.cpp b/src/lib/config/ConfigurationState.cpp new file mode 100755 index 000000000..6ebd878db --- /dev/null +++ b/src/lib/config/ConfigurationState.cpp @@ -0,0 +1,944 @@ + +#include "lib/os/os.h" + +#include "ConfigurationState.h" +#include "ConfigurationException.h" +#include "ConfigPropertyEvaluator.h" +#include "ConfigErrorCollector.h" +#include "PreprocessInputStream.h" +#include "ConfigValueQueue.h" +#include "lib/os/os.h" +#include "lib/utils/bts_assert.h" +#include "lib/utils/is_unittest_environment.h" +#include "lib/utils/bts_array_size.h" +#include "config.h" + +#include "lib/config/protos/system.pb.h" + +#include // iequals +#include +#include +#include +#include +#include +#include + +#define URDL_HEADER_ONLY 1 +#define URDL_DISALE_SSL 1 +#include + +#ifdef WIN32 +#include +#include +#else +#include +#include +#include +#include +#endif + +#include "lib/config_proto_files.h" + +using namespace std; + +namespace bts { +namespace config { +namespace internal { + +const char* MESSAGE_STRING_RULE = "rule"; +const char* MESSAGE_STRING_VALUE = "value"; +const char* MESSAGE_STRING_INCLUDE = "include"; +const char* MESSAGE_STRING_INCLUDE_BACKUP = "fallback"; +const char* MESSAGE_STRING_MACRO_TYPE = "MacroType"; +const char* MESSAGE_STRING_MACRO = "macro"; + +const char* CONFIG_SERVER_ENV_VAR = "BTSConfig"; + +inline tuple hostname(const std::string& path); + +ConfigurationState::ConfigurationState(void) : + _is_initialized(false), + _initialization_mutex(), + _dynamic_pool(), + _prototype_factory(), + _prototype(nullptr), + _config_chain(), + _defaults(), + _properties(new ConfigPropertyEvaluator(*this)), + _rule_engine(_properties), + _macros(), + _deferred_queue(new ConfigValueQueue(*_properties.get())), + _log(), + _host(), + _port(80) +{ +} + +ConfigurationState::~ConfigurationState(void) +{ +} + +std::string ConfigurationState::defaults_path() +{ + std::string appdir = os::get_install_path(); + return appdir + "/defaults/"; +} + +std::string get_command_argument(const char* key) +{ + std::vector cmd_line = os::get_command_line(); + std::vector::iterator begin = cmd_line.begin(), end = cmd_line.end(); + for (; begin != end; begin++) + { + if (boost::iequals(*begin, key)) + { + begin++; + if (begin == end) { + break; + } + else { + os::setenv(CONFIG_SERVER_ENV_VAR, begin->c_str()); + return *begin; + } + } + } + return ""; +} + +string ConfigurationState::settings_path() +{ + // Look for the -ConfigServer option on the command line + std::string cmd_arg = get_command_argument("-ConfigServer"); + if (!cmd_arg.empty()) return cmd_arg; + + const char* env_server = getenv(CONFIG_SERVER_ENV_VAR); + if (env_server) return env_server; + + std::string path = os::get_install_path() + "/conf/"; + if (boost::filesystem::exists(path)) + { + return path; + } + else + { + path = os::bts_user_path() + "conf/"; + if (boost::filesystem::exists(path)) + { + return path; + } + else + { + path = os::bts_system_path(); + if (os::is_windows()) path += "conf/"; + if (boost::filesystem::exists(path)) + { + return path; + } + else + { + // Cannot bts_assert during config initialization + const char* error_message = "Local configuration path does not exist. This machine is not configured."; + std::cout << error_message << std::endl; + throw ConfigurationException(error_message); + } + } + } +} + +void ConfigurationState::initialize_defaults() +{ + std::lock_guard lock(_initialization_mutex); + if (_is_initialized) return; + + generate_all_config_proto(); + + std::shared_ptr new_message(_prototype->New()); + ErrorsOnlyLog log(_log); + load_message(defaults_path(), *new_message.get(), log, *_deferred_queue.get()); + _defaults = new_message; + + // If defaults has an include process that now + // FEATURE: Unlike settings loaded from ~/.bts/ these includes + // take precedence over the settings in the file which includes them. + // While this seems backwards it is always the behavior you want + std::string include = get_include(*_defaults.get()); + if (!include.empty()) + { + load(include, ""); + } + + _is_initialized = true; +} + + +std::string get_installation(const std::string& path) +{ + // "Installation" is defined as the directory where we are installed + // "bin" is special cased to "" as a backwards compatibility for developers + bts_assert(!path.empty()); + std::string parent_path = boost::filesystem::path(path).parent_path().string(); + size_t last = parent_path.find_last_of("/\\"); + if (last + 1 >= parent_path.size()) + return ""; + std::string installation = parent_path.substr(last + 1, parent_path.size() - last); + if (installation == "bin") + return ""; + return installation; +} + +void ConfigurationState::initialize_builtins() +{ + System sys; + + sys.set_host_name(os::host_name()); + sys.set_process_name(os::get_application_name()); + sys.set_process_path(os::get_application_path()); + sys.set_platform(os::platform_name()); + sys.set_qualifier(os::get_qualifier()); + sys.set_unittest_environment(utils::internal::is_unittest_environment()); + sys.set_data_path(os::bts_data_path()); + sys.set_installation(get_installation(sys.process_path())); + sys.set_installation_specific_data_path(sys.data_path() + sys.installation() + "/"); +// sys.set_build_number(BUILD_NUMBER); + sys.set_build_number(0); + sys.set_os_user(os::username()); +#ifdef DEBUG + sys.set_build_type("debug"); +#endif + sys.set_config_host(_host); + sys.set_config_port(_port); + if (!_host.empty()) { + hostent* host_entry = gethostbyname(_host.c_str()); + bts_assert_msg(host_entry != NULL, "Unable to resolve host name " + _host); + in_addr* address = (in_addr*) host_entry->h_addr_list[0]; + string ip_address = inet_ntoa(*address); + sys.set_config_ip(ip_address); + } +#ifdef _WIN32 + sys.set_is_remote_app(GetSystemMetrics(SM_REMOTESESSION) == 1); + + // Screen dpi, Assumes screen dpi is uniform in x and y + // Computes 2 different ratios depending - one will be right depending on whether the process calls SetDPIAware() + HDC screen = GetDC(0); + int log_pixels_x = GetDeviceCaps(screen, LOGPIXELSX); + auto ratio2 = log_pixels_x / 96.0; + int virtual_width = GetDeviceCaps(screen, HORZRES); + int physical_width = GetDeviceCaps(screen, DESKTOPHORZRES); + double ratio = (double)physical_width / (double)virtual_width; + sys.set_dpi(log_pixels_x * ratio); + sys.set_dpi_ratio(max(ratio, ratio2)); + ReleaseDC(0, screen); + +#endif + + std::vector args = os::get_command_line(); + for (std::vector::iterator iter = args.begin(); iter != args.end(); iter++) + { + sys.add_cmd_args(*iter); + } + + // Version string + std::string install_path = os::get_install_path(); + std::string app_version_file = install_path + "/edge.version"; + if (!boost::filesystem::exists(app_version_file)) + { + app_version_file = install_path + "/blue2.version"; + } + if (boost::filesystem::exists(app_version_file)) + { + std::ifstream input(app_version_file.c_str(), std::ifstream::in); + if (!input.fail()) + { + // Read the last line of the file + std::string version; + while(!input.eof()) + { + std::string temp; + std::getline(input, temp); + if (!temp.empty()) version = temp; + } + + sys.set_version(version); + input.close(); + } + } + + sys.set_num_cores(os::num_cores()); + + // + // Add the System value to the defaults + const proto::FieldDescriptor* field_desc = _defaults->GetDescriptor()->FindFieldByName("System"); + const proto::Descriptor* message_desc = field_desc->message_type(); + + // Create a meta message + proto::Message* new_message = _defaults->GetReflection()->AddMessage(_defaults.get(), field_desc); + const proto::FieldDescriptor* value_desc = message_desc->FindFieldByName("value"); + new_message->GetReflection()->MutableMessage(new_message, value_desc)->CopyFrom(sys); + + initialize_introspection(); +} + +void ConfigurationState::initialize_introspection() { + // With this config can be self describing enough + // that another language can request "all" of config + ConfigIntrospection ci; + proto::DescriptorPool::generated_pool(); + for (size_t x = 0; x < BTS_ARRAYSIZE(config_proto_files); x++) { + std::string proto_path = "core/bts_core/config/"; + proto_path += config_proto_files[x]; + const proto::FileDescriptor* file_desc = proto::DescriptorPool::generated_pool()->FindFileByName(proto_path); + if (file_desc != nullptr) { + ConfigIntrospection::ConfigProto* config_proto = ci.add_config(); + bts_assert(config_proto); + config_proto->set_name(file_desc->name()); + for (int y=0; y < file_desc->message_type_count(); y++) { + const proto::Descriptor* message_desc = file_desc->message_type(y); + if (message_desc != nullptr) { + config_proto->add_type(message_desc->full_name()); + } + } + } + } + + + // Add to defaults + const proto::FieldDescriptor* field_desc = _defaults->GetDescriptor()->FindFieldByName("ConfigIntrospection"); + const proto::Descriptor* message_desc = field_desc->message_type(); + proto::Message* new_message = _defaults->GetReflection()->AddMessage(_defaults.get(), field_desc); + const proto::FieldDescriptor* value_desc = message_desc->FindFieldByName("value"); + new_message->GetReflection()->MutableMessage(new_message, value_desc)->CopyFrom(ci); +} + +void ConfigurationState::initialize_macros() +{ + std::vector > :: iterator iter; + for (iter = _config_chain.begin(); iter != _config_chain.end(); ++iter) + { + const proto::Message* message = (*iter).get(); + add_macro(message); + } + add_macro(_defaults.get()); +} + +void ConfigurationState::add_macro(const proto::Message* message) +{ + bts_assert(message); + const proto::FieldDescriptor* field_desc = message->GetDescriptor()->FindFieldByName(MESSAGE_STRING_MACRO); + if (field_desc) + { + int size = message->GetReflection()->FieldSize(*message, field_desc); + for (int index = 0; index < size; index++) + { + // read "name" and "expression" fields + const proto::Message& meta_message = message->GetReflection()->GetRepeatedMessage(*message, field_desc, index); + const proto::FieldDescriptor* name_desc = meta_message.GetDescriptor()->FindFieldByName("name"); + const std::string name = '$' + meta_message.GetReflection()->GetString(meta_message, name_desc); + + const proto::FieldDescriptor* exp_desc = meta_message.GetDescriptor()->FindFieldByName("expression"); + const std::string expression = meta_message.GetReflection()->GetString(meta_message, exp_desc); + + if (_macros.find(name) == _macros.end()) + { + // eval expression to Value + //expression::Value result = _rule_engine.eval_to_value(expression); + _macros[name] = expression; + } + } + } +} + +expression::Value ConfigurationState::get_macro(const std::string& name) +{ + map::iterator iter = _macros.find(name); + if (iter == _macros.end()) { + std::string error = std::string("Macro '") + name + "' is undefined"; + throw ConfigurationException(error); + } + return _rule_engine.eval_to_value(iter->second); +} + +std::string ConfigurationState::get_macro_string(const std::string& name) { + try { + expression::Value result = get_macro(name); + return expression::value_to_string(result); + } + catch (const ConfigurationException& e) { return e.what(); } +} + +void check_rule(proto::Message* message, expression::ExpressionEvaluator& rule_engine) { + const proto::Reflection* reflection = message->GetReflection(); + const proto::Descriptor* descriptor = message->GetDescriptor(); + for (int findex = 0; findex < descriptor->field_count(); ++findex) { + const proto::FieldDescriptor* field = descriptor->field(findex); + if (field->is_repeated()) { + int field_size = reflection->FieldSize(*message, field); + for (int index = 0; index < field_size; ++index) { + const proto::Message& field_message = reflection->GetRepeatedMessage(*message, field, index); + const proto::FieldDescriptor* rule_descriptor = field_message.GetDescriptor()->FindFieldByName(MESSAGE_STRING_RULE); + if (rule_descriptor) { // Macros don't have rules + const std::string rule = field_message.GetReflection()->GetString(field_message, rule_descriptor); + rule_engine.eval(rule); + } + } + } + } +} + +void ConfigurationState::check_rules() { + initialize(); + std::vector > :: iterator iter; + for (iter = _config_chain.begin(); iter != _config_chain.end(); ++iter) + { + proto::Message* message = (*iter).get(); + check_rule(message, _rule_engine); + } + check_rule(_defaults.get(), _rule_engine); +} + +void ConfigurationState::initialize_settings() +{ + std::lock_guard lock(_initialization_mutex); + if (_is_initialized) return; + + initialize_defaults(); + + std::string settings = settings_path(); + std::string fallback = get_command_argument("-ConfigFallback"); + load(settings, fallback); + + initialize_builtins(); + initialize_macros(); + _deferred_queue->process(); +} + +void ConfigurationState::reset() +{ + std::lock_guard lock(_initialization_mutex); + + _config_chain.clear(); + _prototype = nullptr; + _defaults.reset(); + _prototype_factory.reset(); + _dynamic_pool.reset(); + _macros.clear(); + _deferred_queue->clear(); + _is_initialized = false; + _host.clear(); + _port = 80; + set_print_errors(true); +} + +int ConfigurationState::load(const std::string& url, const std::string& fallback_url) +{ + std::shared_ptr new_message(_prototype->New()); + + bool loaded = false; + try { + loaded = load_message(url, *new_message.get(), _log, *_deferred_queue.get()); + } + catch(ServerUnreachableException&) { + if (fallback_url.empty()) { + throw; + } + } + + if (!loaded && !fallback_url.empty()) + { + std::string message = "Failed to load: " + url + " using fallback."; + _log.warning(message); + loaded = load_message(fallback_url, *new_message.get(), _log, *_deferred_queue.get()); + if (_host.empty() || _host == "localhost") { tie(_host, _port) = hostname(fallback_url); } + } + if (!loaded) return 0; + + if (_host.empty() || _host == "localhost") { tie(_host, _port) = hostname(url); } + + std::string include = get_include(*new_message.get()); + std::string fallback = get_fallback(*new_message.get()); + + // insert at front? + _config_chain.push_back(new_message); + + // Recursively load includes + if (!include.empty()) + { + return 1 + load(include, fallback); + } + return 1; +} + +int ConfigurationState::load_string(const std::string& values) +{ + initialize_defaults(); + + std::shared_ptr new_message(_prototype->New()); + + proto::io::ArrayInputStream input_stream(values.c_str(), (int)values.length()); + parse_stream("test string", input_stream, *new_message.get(), _log, *_deferred_queue.get()); + + std::string include = get_include(*new_message.get()); + std::string fallback = get_fallback(*new_message.get()); + + _config_chain.push_back(new_message); + + int count = 0; + if (!include.empty()) + { + count += load(include, fallback); + } + + if (_host.empty()) { + _host = "localhost"; + } + + initialize_builtins(); + initialize_macros(); + _deferred_queue->process(); + return 1 + count; +} + +void ConfigurationState::add_metadata_field(proto::FileDescriptorProto* file_dp, proto::DescriptorProto* message_dp, const proto::Descriptor* base_field, int index) +{ + // Create the metadata message + proto::DescriptorProto* meta_message = file_dp->add_message_type(); + std::string meta_name = base_field->name() + "meta"; + meta_message->set_name(meta_name); + + // Add 'rule' + proto::FieldDescriptorProto* rule_field = meta_message->add_field(); + rule_field->set_name(MESSAGE_STRING_RULE); + rule_field->set_number(1); + rule_field->set_type(proto::FieldDescriptorProto_Type_TYPE_STRING); + rule_field->set_label(proto::FieldDescriptorProto_Label_LABEL_OPTIONAL); + rule_field->set_default_value("true"); + + // Add 'value' + proto::FieldDescriptorProto* value_field = meta_message->add_field(); + value_field->set_name(MESSAGE_STRING_VALUE); + value_field->set_number(2); + value_field->set_type_name(base_field->full_name()); + value_field->set_label(proto::FieldDescriptorProto_Label_LABEL_REQUIRED); + + // Now add a field to message_dp of that type + proto::FieldDescriptorProto* field_desc = message_dp->add_field(); + field_desc->set_name(base_field->name()); + field_desc->set_number(index); + field_desc->set_type_name(meta_name); + field_desc->set_label(proto::FieldDescriptorProto_Label_LABEL_REPEATED); +} + +// Construct a new Message that describes the save file format +// This can then be used with google::protobuf::TextFormat to load and save files +void ConfigurationState::generate_all_config_proto() +{ + const char* bts_config_classname = "__BTSConfig"; + int field_number = 0; + + proto::FileDescriptorProto bts_config_dp; + bts_config_dp.set_name("config.proto"); + proto::DescriptorProto* config_message = bts_config_dp.add_message_type(); + config_message->set_name(bts_config_classname); + + // Add the 'include' field + proto::FieldDescriptorProto* include_field = config_message->add_field(); + include_field->set_name(MESSAGE_STRING_INCLUDE); + include_field->set_number(++field_number); + include_field->set_type(proto::FieldDescriptorProto_Type_TYPE_STRING); + include_field->set_label(proto::FieldDescriptorProto_Label_LABEL_OPTIONAL); + + // Add the 'fallback' field + proto::FieldDescriptorProto* backup_field = config_message->add_field(); + backup_field->set_name(MESSAGE_STRING_INCLUDE_BACKUP); + backup_field->set_number(++field_number); + backup_field->set_type(proto::FieldDescriptorProto_Type_TYPE_STRING); + backup_field->set_label(proto::FieldDescriptorProto_Label_LABEL_OPTIONAL); + + // Add 'macro' message type + proto::DescriptorProto* macro_message = bts_config_dp.add_message_type(); + macro_message->set_name(MESSAGE_STRING_MACRO_TYPE); + proto::FieldDescriptorProto* name_field = macro_message->add_field(); + name_field->set_name("name"); + name_field->set_number(1); + name_field->set_type(proto::FieldDescriptorProto_Type_TYPE_STRING); + name_field->set_label(proto::FieldDescriptorProto_Label_LABEL_REQUIRED); + proto::FieldDescriptorProto* expression_field = macro_message->add_field(); + expression_field->set_name("expression"); + expression_field->set_number(2); + expression_field->set_type(proto::FieldDescriptorProto_Type_TYPE_STRING); + expression_field->set_label(proto::FieldDescriptorProto_Label_LABEL_REQUIRED); + + // add 'macro' message field + proto::FieldDescriptorProto* macro_field = config_message->add_field(); + macro_field->set_name(MESSAGE_STRING_MACRO); + macro_field->set_number(++field_number); + macro_field->set_type(proto::FieldDescriptorProto_Type_TYPE_MESSAGE); + macro_field->set_type_name(MESSAGE_STRING_MACRO_TYPE); + macro_field->set_label(proto::FieldDescriptorProto_Label_LABEL_REPEATED); + + // add a meta type and field for each generated message + proto::DescriptorPool::generated_pool(); + for (size_t x = 0; x < BTS_ARRAYSIZE(config_proto_files); x++) + { + std::string proto_path = "config/protos/"; + proto_path += config_proto_files[x]; + const proto::FileDescriptor* file_desc = proto::DescriptorPool::generated_pool()->FindFileByName(proto_path); + if (file_desc != nullptr) + { + bts_config_dp.add_dependency(file_desc->name()); + for (int y=0; y < file_desc->message_type_count(); y++) + { + const proto::Descriptor* message_desc = file_desc->message_type(y); + add_metadata_field(&bts_config_dp, config_message, message_desc, ++field_number); + } + } + else + { + throw InvalidDescriptorException(config_proto_files[x]); + } + } + + // Generate a proper FileDesctiptor from the FileDescriptorProto + // The newly created descriptors live in the pool + _dynamic_pool.reset(new proto::DescriptorPool(proto::DescriptorPool::generated_pool())); + ConfigErrorCollector errors(_log, ""); + const proto::FileDescriptor* file_desc = _dynamic_pool->BuildFileCollectingErrors(bts_config_dp, &errors); + if (!file_desc) { + std::string error_message = errors.ToString(); + if (!error_message.empty()) { + bts_assert_msg(file_desc, error_message); + } else { + bts_assert_msg(false, "Config: error building parser message. No errors logged by protobuf library.\n This could mean you have defined 2 config keys with the same name. Unfortunately that's all I know."); + } + } + const proto::Descriptor* bts_config_desc = file_desc->FindMessageTypeByName(bts_config_classname); + + _prototype_factory.reset(new proto::DynamicMessageFactory()); + _prototype_factory->SetDelegateToGeneratedFactory(true); + _prototype = _prototype_factory->GetPrototype(bts_config_desc); +} + +bool is_valid_key_descriptor(const proto::Descriptor* property_desc) +{ + std::string file_name = property_desc->file()->name(); + size_t file_name_start = file_name.find_last_of('/') + 1; + file_name = file_name.substr(file_name_start); + for (size_t x = 0; x < BTS_ARRAYSIZE(config_proto_files); x++) + { + std::string proto_name = config_proto_files[x]; + if (boost::iequals(file_name, proto_name)) + { + return true; + } + } + return false; +} + +const proto::Message* ConfigurationState::try_get_message(const std::string& property_name) +{ + initialize_settings(); + + const proto::Message* message; + + // Iterate through the loaded files + + std::vector > :: iterator iter; + for (iter = _config_chain.begin(); iter != _config_chain.end(); ++iter) + { + message = try_get_value(property_name, **iter); + if (message) + { + return message; + } + } + + // Fall back to defaults + message = try_get_value(property_name, *_defaults.get()); + if (message) + { + return message; + } + + return nullptr; +} + +const proto::Message& ConfigurationState::get_message(const proto::Descriptor* property_desc) +{ + if (!is_valid_key_descriptor(property_desc)) + { + throw InvalidKeyException(property_desc->name().c_str()); + } + + const proto::Message* message = try_get_message(property_desc->name()); + if (!message) + { + // Get the message prototype - will only have compiled in defaults + message = proto::MessageFactory::generated_factory()->GetPrototype(property_desc); + } + + return *message; +} + +const proto::Message* ConfigurationState::try_get_value(const std::string& field_name, const proto::Message& message) const +{ + // 1) Get the metadata descriptor + const proto::Descriptor* message_desc = message.GetDescriptor(); + const proto::FieldDescriptor* meta_desc = message_desc->FindFieldByName(field_name); + if (meta_desc) + { + // 2) Get the repeated metadata + int meta_size = message.GetReflection()->FieldSize(message, meta_desc); + for (int index = 0; index < meta_size; index++) + { + // 3) Iterate over repeated field and check rule + const proto::Message& meta_message = message.GetReflection()->GetRepeatedMessage(message, meta_desc, index); + const proto::FieldDescriptor* rule_desc = meta_message.GetDescriptor()->FindFieldByName(MESSAGE_STRING_RULE); + const std::string rule = meta_message.GetReflection()->GetString(meta_message, rule_desc); + + if (_rule_engine.eval(rule)) + { + // 4) return value + const proto::FieldDescriptor* value_desc = meta_message.GetDescriptor()->FindFieldByName(MESSAGE_STRING_VALUE); + const proto::Message& message_value = meta_message.GetReflection()->GetMessage(meta_message, value_desc, proto::MessageFactory::generated_factory()); + return &message_value; + } + else + { + // TODO: Consider a rule result cache + // pro: faster lookups after first parse + // con: right now config is read only and perfectly parallel after load + } + } + } + return nullptr; +} + +const google::protobuf::Descriptor* ConfigurationState::get_descriptor_by_name(const std::string& name) { + const proto::Message* pmsg = _defaults.get(); + if (pmsg) { + // Get the metadata descriptor (Meta = rule+value fields) + const proto::Descriptor* desc = pmsg->GetDescriptor(); + const proto::FieldDescriptor* meta_desc = desc->FindFieldByName(name); + if (meta_desc) { + // Find the "value" field descriptor + const proto::Descriptor* pmsg_desc = meta_desc->message_type(); + const proto::FieldDescriptor* value_desc = pmsg_desc->FindFieldByName(MESSAGE_STRING_VALUE); + if (value_desc) { + // return it's type + return value_desc->message_type(); + } + } + } + return nullptr; +} + +PathType::type get_path_type(std::string path) +{ + if (path.find("://") != std::string::npos) + { + return PathType::directory; + //const char last_char = path.at(path.length()-1); + //return last_char == '/' ? PathType::directory : PathType::file; + } + else + { + if (boost::filesystem::is_directory(path)) + { + return PathType::directory; + } + else if (boost::filesystem::exists(path)) + { + return PathType::file; + } + else + { + return PathType::not_found; + } + } +} + +urdl::url url_from_string(std::string& path, bool required) +{ + boost::system::error_code ec; + // attempt to parse the path as a URL - if that fails assume it is an absolute file path and append file:/// + urdl::url url = urdl::url::from_string(path, ec); + if (ec) + { + std::string file_prefix = "file:///"; + std::string abs_path = boost::filesystem::absolute(path).string(); + url = urdl::url::from_string(file_prefix + os::normalize_path(abs_path), ec); + } + bts_assert_msg(!required || !ec, ec.message().c_str()); + + return url; +} + +inline tuple hostname(const std::string& path) { + boost::system::error_code ec; + string host; + uint32_t port = 80; + // Try to parse a url with urdl + urdl::url url = urdl::url::from_string(path, ec); + if (!ec) { + host = url.host(); + port = url.port(); + } else { + // Check for a UNC path + string norm_path = path; + os::normalize_path(norm_path); + if (norm_path[0] == '/' && norm_path[1] == '/') { + size_t host_len = norm_path.find_first_of('/', 2) - 2; + if (host_len > 0) { + host = norm_path.substr(2, host_len); + } + } + } + + // If we are still empty return "localhost" + if (host.empty()) host = "localhost"; + + return std::make_tuple(host, port); +} + +// Loads a single message file or directory +bool load_message(std::string url, proto::Message& out_message, IConfigLog& log, ConfigValueQueue& lookup) +{ + ConfigDownloader downloader; + + bool result = false; + switch(get_path_type(url)) + { + case PathType::directory: + result = load_directory(url, downloader, out_message, log, lookup) > 0; + break; + case PathType::file: + result = load_file(url, downloader, out_message, log, lookup) == ConfigDownloader::SUCCESS; + break; + default: + std::ostringstream error; + error << "Error '" << url << "' is not a valid configuration path"; + std::string err_str = error.str(); + log.error(err_str); + throw ConfigurationException(err_str.c_str()); + break; + } + + return result; +} + +int load_directory(std::string url, ConfigDownloader& downloader, proto::Message& out_message, IConfigLog& log, ConfigValueQueue& lookup) +{ + LogAggregator aggregate_log(log); + aggregate_log.info("URL: " + url); + + const char last_char = url.at(url.length()-1); + if (last_char != '/') { + url = url + '/'; + } + + // If a file called "all" exists in the config directory we can short circuit ~60 http requests + ConfigDownloader::Result all_result = load_file(url + ".all", downloader, out_message, aggregate_log, lookup, false); + if (all_result == ConfigDownloader::SUCCESS) { + log.info(aggregate_log.text()); + return 1; + } + + // load edge.conf + int load_count = 0; + ConfigDownloader::Result result = load_file(url + "edge.conf", downloader, out_message, aggregate_log, lookup, false); + if (result == ConfigDownloader::SUCCESS) + { + load_count++; + } + else if (result == ConfigDownloader::PERMANENT_FAILURE) { + // If the server is down it doesn't make sense to suffer though the timeouts for the other files + throw ServerUnreachableException(url.c_str()); + } + + bool edge_has_include = !get_include(out_message).empty(); + + for(const char* proto_file: config_proto_files) + { + std::string config_file(proto_file); + if (config_file == "system.proto") continue; + if (config_file == "edge.proto") continue; + + const char proto_ext[] = ".proto"; + const char conf_ext[] = ".conf"; + size_t extension_pos = config_file.rfind(proto_ext); + // path_to_config_proto/foo.proto => foo.conf + config_file.replace(extension_pos, sizeof(proto_ext), conf_ext, sizeof(conf_ext)); + boost::filesystem::path p (config_file); + config_file = p.filename ().string (); + if (load_file(url + config_file, downloader, out_message, aggregate_log, lookup, false)) + { + load_count++; + if (!edge_has_include) { + bts_assert_msg(get_include(out_message).empty(), "Includes may only be specified in edge.conf"); + } + + // NOTE: This code allows any config key to go in any .conf file + // If this becomse a problem we could iterate over the loaded messages + // and ensure they came from the correct location. + } + } + + log.info(aggregate_log.text()); + + return load_count; +} + +ConfigDownloader::Result load_file(std::string path, ConfigDownloader& downloader, proto::Message& out_message, IConfigLog& log, ConfigValueQueue& lookup, bool required) +{ + stringstream url_stream; + string url = url_from_string(path, required).to_string(); + ConfigDownloader::Result result = downloader.get(url, url_stream); + if (!(result == ConfigDownloader::SUCCESS)) { + bts_assert_msg(!required || url_stream, (path + " not found.").c_str()); + return result; + } + + log.info("Loaded: " + path); + + proto::io::IstreamInputStream proto_stream(&url_stream); + parse_stream(path, proto_stream, out_message, log, lookup); + + return result; +} + +void parse_stream(const std::string& file, proto::io::ZeroCopyInputStream& input_stream, proto::Message& out_message, IConfigLog& log, ConfigValueQueue& lookup) +{ + ConfigErrorCollector error_collector(log, file.c_str()); + bts::google::protobuf::TextFormat::Parser parser; + parser.RecordErrorsTo(&error_collector); + parser.SetValueLookup(&lookup); + lookup.set_filename(file); + parser.AllowPartialMessage(true); + bool success = parser.Parse(&input_stream, &out_message); + if (!success) + { + std::string error_message = error_collector.ToString(); + log.error(error_message); + throw ConfigurationException(error_message); + } +} + +std::string get_field(const proto::Message& message, const char* name) +{ + const proto::Descriptor* message_descriptor = message.GetDescriptor(); + const proto::FieldDescriptor* include_field = message_descriptor->FindFieldByName(name); + const std::string include = message.GetReflection()->GetString(message, include_field); + + return include; +} + +std::string get_include(const proto::Message& message) +{ + return get_field(message, MESSAGE_STRING_INCLUDE); +} + +std::string get_fallback(const proto::Message& message) +{ + return get_field(message, MESSAGE_STRING_INCLUDE_BACKUP); +} + +}}}//end namespaces diff --git a/src/lib/config/ConfigurationState.h b/src/lib/config/ConfigurationState.h new file mode 100755 index 000000000..effee942f --- /dev/null +++ b/src/lib/config/ConfigurationState.h @@ -0,0 +1,123 @@ +#ifndef CONFIGURATIONSTATE_H_ +#define CONFIGURATIONSTATE_H_ + +#include "ConfigurationException.h" +#include "ConfigLog.h" +#include "lib/expression/ExpressionEvaluator.h" +#include "lib/expression/Value.h" +#include "ConfigDownloader.h" + +#include +#include "text_format/text_format.h" +#include +#include +#include + +#include +#include +#include +#include + + +namespace google { + namespace protobuf { + class Message; + class DescriptorPool; + class DynamicMessageFactory; + class Descriptor; + class FileDescriptorProto; + class DescriptorProto; + } +} + +namespace bts { +namespace config { + class System; +}} + +namespace proto = google::protobuf; + +namespace bts { +namespace config { +namespace internal { + + class ConfigPropertyEvaluator; + class ConfigValueQueue; + + class ConfigurationState + { + public: + ConfigurationState(void); + ~ConfigurationState(void); + + void initialize_defaults(); + void initialize_settings(); + + int load(const std::string& url, const std::string& fallback); + int load_string(const std::string& values); + void reset(); + + const proto::Message& get_message(const proto::Descriptor* property_desc); + const proto::Message* try_get_message(const std::string& property_name); + + bool is_initialized() { return _is_initialized; } + + bool print_errors() { return _log.print_errors(); } + void set_print_errors(bool value) { _log.set_print_errors(value); } + expression::Value get_macro(const std::string& name); + std::string get_macro_string(const std::string& name); + + void check_rules(); + + const google::protobuf::Descriptor* get_descriptor_by_name(const std::string& name); + + private: + BTS_NO_COPY(ConfigurationState); + + void initialize_builtins(); + void initialize_introspection(); + void initialize_macros(); + void add_macro(const proto::Message*); + void add_metadata_field(proto::FileDescriptorProto* file_dp, proto::DescriptorProto* message_dp, const proto::Descriptor* base_field, int index); + void generate_all_config_proto(); + const proto::Message* try_get_value(const std::string& field_name, const proto::Message& message) const; + + std::string defaults_path(); + std::string settings_path(); + + bool _is_initialized; + std::recursive_mutex _initialization_mutex; + + boost::scoped_ptr _dynamic_pool; + boost::scoped_ptr _prototype_factory; + const proto::Message* _prototype; // owned by prototype_factory + std::vector > _config_chain; + std::shared_ptr _defaults; + + std::shared_ptr _properties; + expression::ExpressionEvaluator _rule_engine; + std::map _macros; + std::shared_ptr _deferred_queue; + + ConfigLog _log; + std::string _host; + uint32_t _port; + }; + + std::string get_include(const proto::Message& messages); + std::string get_fallback(const proto::Message& message); + + struct PathType { + enum type { not_found, file, directory }; + }; + PathType::type get_path_type(std::string path); + + // Loads a file or directory and merges the values into one Message. + // Caller is responsible for processing includes + bool load_message(std::string url, proto::Message& out_message, IConfigLog&, ConfigValueQueue&); + int load_directory(std::string url, ConfigDownloader&, proto::Message& out_message, IConfigLog&, ConfigValueQueue&); + ConfigDownloader::Result load_file(std::string url, ConfigDownloader&, proto::Message& out_message, IConfigLog&, ConfigValueQueue&, bool required = true); + void parse_stream(const std::string& file, proto::io::ZeroCopyInputStream& stream, proto::Message& out_message, IConfigLog&, ConfigValueQueue& lookup); + +}}} +#endif diff --git a/src/lib/config/PreprocessInputStream.cpp b/src/lib/config/PreprocessInputStream.cpp new file mode 100755 index 000000000..fbaa5508d --- /dev/null +++ b/src/lib/config/PreprocessInputStream.cpp @@ -0,0 +1,70 @@ +#include "PreprocessInputStream.h" + +#include +#include + +using namespace std; + +namespace bts { +namespace config { + +void replace_all(string& line, const string& before, const string& after) { + + string::const_iterator end = line.end(); + string::const_iterator current = line.begin(); + string::const_iterator next = search(current, end, before.begin(), before.end()); + + if (next != end) { + std::string result; + while(next != end) { + result.append(current, next); + result.append(after); + current = next + before.size(); + next = search(current, end, before.begin(), before.end()); + } + result.append(current, next); + line = result; + } +} + +PreprocessInputStream::PreprocessInputStream(std::istream& input, map replacements) : + _buffer(), + _stream() +{ + char temp[4096]; + while(input.read(temp, sizeof(temp))) { + _buffer.append(temp, sizeof(temp)); + } + _buffer.append(temp, input.gcount()); + + for (map::iterator iter = replacements.begin(); iter != replacements.end(); iter++) { + const string& key = iter->first; + const string& value = iter->second; + replace_all(_buffer, key, value); + } + + _stream.reset(new proto::io::ArrayInputStream(_buffer.c_str(), _buffer.size())); +} + +PreprocessInputStream::~PreprocessInputStream() +{ +} + +bool PreprocessInputStream::Next(const void** data, int* size) { + return _stream->Next(data, size); +} + +void PreprocessInputStream::BackUp(int count) { + _stream->BackUp(count); +} + +bool PreprocessInputStream::Skip(int count) { + return _stream->Skip(count); +} + +proto::int64 PreprocessInputStream::ByteCount() const { + return _stream->ByteCount(); +} + +}}//namespace + diff --git a/src/lib/config/PreprocessInputStream.h b/src/lib/config/PreprocessInputStream.h new file mode 100755 index 000000000..aad2829e0 --- /dev/null +++ b/src/lib/config/PreprocessInputStream.h @@ -0,0 +1,34 @@ +#pragma once + +#include +#include +#include "lib/utils/bts_no_copy.h" + +#include +#include +#include + +namespace bts { +namespace config { + +namespace proto = google::protobuf; + +class PreprocessInputStream : public proto::io::ZeroCopyInputStream +{ +public: + PreprocessInputStream(std::istream& input, std::map replacements); + ~PreprocessInputStream(); + + bool Next(const void** data, int* size) override; + void BackUp(int count) override; + bool Skip(int count) override; + proto::int64 ByteCount() const override; + +private: + std::string _buffer; + boost::scoped_ptr _stream; + BTS_NO_COPY(PreprocessInputStream); +}; + +}}//namespace + diff --git a/src/lib/config/config.cpp b/src/lib/config/config.cpp new file mode 100755 index 000000000..f5cc2fa0e --- /dev/null +++ b/src/lib/config/config.cpp @@ -0,0 +1,129 @@ +#ifndef CONFIG_H_ +#define CONFIG_H_ + +#include "config.h" +#include "../os/os.h" +#include "ConfigurationState.h" + +#include +#include +#include +#include + +//#include + +namespace proto = google::protobuf; + +namespace bts { +namespace config { + +internal::ConfigurationState config_state; + +////////////////////////////////////////// +// Public API +////////////////////////////////////////// + +extern "C" BTS_EXPORT void initialize() +{ + config_state.initialize_settings(); +} + +extern "C" BTS_EXPORT bool initialize_nothrow(char* error_message, size_t len) +{ + try + { + initialize(); + } + catch(std::exception& e) + { + std::string msg = e.what(); + if (msg.size() < len) + { + os::strncpy(error_message, msg.c_str(), len); + } + return false; + } + return true; +} + +//extern "C" +const proto::Message& get_message(const google::protobuf::Descriptor* descriptor) +{ + return config_state.get_message(descriptor); +} + +namespace interop { + +////////////////////////////////////////////////// +// Functions used for interop to other languages +///////////////////////////////////////////////// + + +std::string getbuffer_serialized_buffer; +extern "C" BTS_EXPORT const char* get_buffer(const char* key_name, int& out_length) +{ + google::protobuf::DescriptorPool const* pool = google::protobuf::DescriptorPool::generated_pool(); + const proto::Descriptor* message_desc = pool->FindMessageTypeByName(key_name); + + const proto::Message& message = get_message(message_desc); + + getbuffer_serialized_buffer.clear(); + message.SerializePartialToString(&getbuffer_serialized_buffer); + + out_length = getbuffer_serialized_buffer.size(); + return getbuffer_serialized_buffer.c_str(); +} + +extern "C" BTS_EXPORT const proto::Message* get_named(const char* key_name) +{ + const auto* pmessage = config_state.try_get_message(key_name); + if (pmessage) { + return pmessage; + } + // If we can't find a value we should return a default instance + const auto* descriptor = config_state.get_descriptor_by_name(key_name); + if (descriptor) { + return proto::MessageFactory::generated_factory()->GetPrototype(descriptor); + } + return nullptr; +} + +BTS_EXPORT std::string get_macro_string(const char* name) { return config_state.get_macro_string(name); } + +} + +namespace internal { + +////////////////////////////////////////// +// Functions used in testing +////////////////////////////////////////// + +extern "C" BTS_EXPORT void initialize_for_test(const std::string& values, bool print_errors) +{ + config_state.reset(); + config_state.set_print_errors(print_errors); + config_state.load_string(values); +} + +extern "C" BTS_EXPORT void reset() +{ + config_state.reset(); +} + +extern "C" BTS_EXPORT bool check_rules(char* error_msg, size_t len) { + try { + config_state.check_rules(); + } + catch(std::exception& e) { + std::string msg = e.what(); + if (msg.size() < len) + { + os::strncpy(error_msg, msg.c_str(), len); + } + return false; + } + return true; +} + +}}} // end namespaces +#endif diff --git a/src/lib/config/config.h b/src/lib/config/config.h new file mode 100755 index 000000000..4cf0dc3bb --- /dev/null +++ b/src/lib/config/config.h @@ -0,0 +1,70 @@ +#ifndef BTS_CONFIG_H_ +#define BTS_CONFIG_H_ + +#include "lib/utils/bts_export.h" +#include "lib/utils/checked_cast.h" + +#include + +namespace google { + namespace protobuf { + class Descriptor; + class Message; + } +} + +// Config +// This API is intentionally kept flat and C-style to make interop from other languages simpler. + +namespace bts { +namespace config { + + // Load and initialize config. + // This will be automatically called on the first key request but since loading config + // can result in network requests this function is available for applications that want to + // manage pauses in execution. + // Ex: initialize() on a background thread so that config::get() is fast on a UI thread + extern "C" BTS_EXPORT void initialize(); + + // Calls initialize() but catches any thrown exception and copies it to the provided buffer + extern "C" BTS_EXPORT bool initialize_nothrow(char* error_msg, size_t length); + + // Gets a raw message pointer from config for the given descriptor + //extern "C" + BTS_EXPORT const google::protobuf::Message& get_message(const google::protobuf::Descriptor* descriptor); + + template + const T& get() + { + const google::protobuf::Descriptor* descriptor = T::descriptor(); + const google::protobuf::Message& message = get_message(descriptor); + const T& retval = utils::checked_cast(message); + return retval; + } + + namespace interop + { + // Returns a serialized protocol buffer of the given type + // The memory that this is serialized to may be re-used so you + // should imediately deserialize and copy the data. + extern "C" BTS_EXPORT const char* get_buffer(const char* key_name, int& out_length); + + extern "C" BTS_EXPORT const google::protobuf::Message* get_named(const char* key_name); + } + + namespace internal + { + // Initialize configuration state from a string instead of from the filesystem. defaults.conf will still be loaded. + // Calling this a second time clears all values from previous initialization + extern "C" BTS_EXPORT void initialize_for_test(const std::string& configuration, bool print_errors=true); + + // Releases all resources and returns config to factory defaults + extern "C" BTS_EXPORT void reset(); + + extern "C" BTS_EXPORT bool check_rules(char* error_msg, size_t length); + } +}} + + + +#endif diff --git a/src/lib/config/configure_proto_files.h.in b/src/lib/config/configure_proto_files.h.in new file mode 100644 index 000000000..91e3cc0a6 --- /dev/null +++ b/src/lib/config/configure_proto_files.h.in @@ -0,0 +1,3 @@ +//GENERATED FILE +const char * config_proto_files[] = { +}; diff --git a/src/lib/config/defaults/edge.conf b/src/lib/config/defaults/edge.conf new file mode 100755 index 000000000..b84c5774f --- /dev/null +++ b/src/lib/config/defaults/edge.conf @@ -0,0 +1,4 @@ +# This file 'catchall' configuration file +# If *every* site must include a specific url that should be listed here +# otherwise this file should be mostly empty. + diff --git a/src/lib/config/defaults/logging.conf b/src/lib/config/defaults/logging.conf new file mode 100644 index 000000000..49163e08f --- /dev/null +++ b/src/lib/config/defaults/logging.conf @@ -0,0 +1,10 @@ +Logging +{ + value + { + console { default_level: "Warn" } + syslog { default_level: "Info" } + } +} + + diff --git a/src/lib/config/defaults/testkeys.conf b/src/lib/config/defaults/testkeys.conf new file mode 100755 index 000000000..8b9561c01 --- /dev/null +++ b/src/lib/config/defaults/testkeys.conf @@ -0,0 +1,6 @@ +Foo { + rule: "true" + value { + email: "default@example.com" + } +} diff --git a/src/lib/config/doc/README.txt b/src/lib/config/doc/README.txt new file mode 100755 index 000000000..0f3360df5 --- /dev/null +++ b/src/lib/config/doc/README.txt @@ -0,0 +1,40 @@ +Structure: + + +To add a new config key +1) Do you want to add a new logical grouping of config keys? + If you do goto 2, otherwise goto 5. +2) Create a new file with the .proto extension and define the package (namespace) + to which your new keys should belong and import the common extensions.proto. + Then define your new keys using the protocol buffer message format - see + http://code.google.com/p/protobuf/ + + ex: + package bts.config.keys; + import "extensions.proto"; + + message MyNewExchange { + // Suggested: include a namespace to group your keys in the editor + option(name_space) = "MyKeys" + // Suggested: write a doc string for this key so you remember what it's for + option(doc_string) = "This key holds all the values that we need to know about when connecting to NewExchange" + + // Example of a string field with a default value + optional string login_name = 1 [default = "bts"]; + // example of a string field without a default value + optional string password = 2; + } +3) Open config_proto_file_list.h and add your new .profo file to the list. This is the list of + protobuf files that are considered config keys. +4) Add your new file to config.bs and run "bt update" to update the msbuild project. + You may also, optionally, add it as a 'document' or 'None' to the visual studio project so it is + displayed in the solution explorer. +5) Your message name is now a generated c++ type. If you added it to MyKeys.proto you can now #include "gen/core/config/mykeys.pb.h". + To request the value from config use the GetValue() method. Make multiple calls for multiple key values. + MyNewExchange* value = bts::config::GetValue(); + std::string login_name = value->login_name(); + + + + + diff --git a/src/lib/config/extensions/extensions.proto b/src/lib/config/extensions/extensions.proto new file mode 100755 index 000000000..1fae6c377 --- /dev/null +++ b/src/lib/config/extensions/extensions.proto @@ -0,0 +1,15 @@ +syntax='proto2'; +import "google/protobuf/descriptor.proto"; + +// Custom protobuf fields - Every other config proto file should include this + +extend google.protobuf.MessageOptions { + //optional string name_space = 50100; + optional string doc_string = 50101; +} + +//extend google.protobuf.FieldOptions { +// Consider allowing certain fields to be sent along with the HTTP request +//} + + \ No newline at end of file diff --git a/src/lib/config/invalid/invalid.proto b/src/lib/config/invalid/invalid.proto new file mode 100755 index 000000000..a61121f3d --- /dev/null +++ b/src/lib/config/invalid/invalid.proto @@ -0,0 +1,10 @@ +syntax='proto2'; + +// This protocol buffer is not considered a config key so requesting it from +// config is not allowed. + +package bts.config.test; + +message InvalidMessageType { +} + diff --git a/src/lib/config/protos/MarketData.proto b/src/lib/config/protos/MarketData.proto new file mode 100755 index 000000000..fd70f41ee --- /dev/null +++ b/src/lib/config/protos/MarketData.proto @@ -0,0 +1,248 @@ +syntax='proto2'; +package bts.md; +import "lib/security_model/security.proto"; + +// +// Properties that help the market data engine determine +// which platform should be used. +// +message PlatformMap +{ + message PlatformProps + { + repeated bts.security_model.Exchange.type exchange = 2; + } + optional PlatformProps sim = 1; + optional PlatformProps SRLabs = 2; + optional PlatformProps Spryware = 3; + optional PlatformProps CSM = 4; + optional PlatformProps CME = 5; + optional PlatformProps ICE = 6; + optional PlatformProps Replay = 7; + optional PlatformProps ACTIV = 8; + optional PlatformProps MDP3 = 9; + optional PlatformProps CSM_ONIXS = 10; + optional PlatformProps BLUE = 11; +} + +// +// Options that apply to market data server +// +message Channel { + enum type { + SECURITY = 1; + BBO = 2; + DEPTH = 3; + TRADE = 4; + MISC = 5; + STATUS = 6; + TRADELOG = 7; + RFQ = 8; + } +} + +message PublisherOptions +{ + // removed 2 + optional string network_interface = 3 [default = "0.0.0.0"]; + // Supply either network_interface -or- network_interface_[a,b] + optional string network_interface_a = 30; + optional string network_interface_b = 31; + // removed 4,5 + // removed 6,7,8 + optional string channel_prefix = 9 [default = ""]; + + // Co2 options + + // How old can the heartbeat be before it's considered dead + optional uint32 heartbeat_ms = 10 [default = 5000]; + optional bool disable_heartbeat = 11 [default = false]; + optional uint32 heartbeat_boot_seconds = 12 [default = 60]; // 1 minute grace period from shm creation +} + +message PlatformConfig +{ + required string platform = 1; + optional bts.security_model.Exchange.type exchange = 2 [default = TESTEXCHANGE]; + optional bts.security_model.SecurityType.type type = 3 [default = UNSET]; +} + +message StreamOptions +{ + optional uint32 trade_log_stream_size = 2 [default = 1024]; +} + +message PublisherNamespace { + optional string shared_memory_namespace = 1; +} + +// +// CBOE CSM options +// + +message CSMOptions +{ + optional bool use_backup_channels = 1 [default = false]; + optional bool always_run_security_feed = 2 [default = true]; + optional bool ignore_sequence_gap = 3 [default = false]; + + optional bool enable_gc = 4 [default = true]; + optional uint32 minimum_gc_cycle_seconds = 5 [default = 120]; + optional uint32 gc_after_cycles = 6 [default = 1]; + + optional uint32 udp_socket_buffer_size = 14 [default = 16777216]; // 16 mb + +} + +message CSMChannels +{ + message Feed + { + required string address = 2; + required uint32 port = 3; + } + + message Channel + { + optional string name = 1; + optional Feed A = 2; + optional Feed B = 3; + } + + optional Channel security_definition = 2; + repeated Channel market_data = 3; +} + +message CSMTests +{ + optional bool enabled = 1 [default = false]; +} + +// +// CME Options +// +message CMEOptions +{ + enum OnixSLogMode { + Disabled = 0; + Important = 1; + Regular = 2; + Debug = 3; + } + enum OnixSLogAdvanced { + LogNothing = 0; + LogFixMessages = 1; + LogFullBooks = 2; + LogEverything = 3; + } + message FeedEngineSettings { + optional string name = 1; + // If num_threads is set it will use that - otherwise it will use channels_per_thread and the number of threads configured for this channel to determine the number of threads + optional uint32 num_threads = 2 [default = 1]; + // Defines amount of time Feed Engine spends on socket waiting for I/O while running master processing loop. + optional uint32 data_wait = 3 [default = 10]; + // Defines amount of time Feed Engine keeps cycling before going to sleep when no useful activity can be done + optional uint32 spin_time = 4 [default = 1]; + optional uint32 channels_per_thread = 5 [default = 3]; + repeated uint32 affinity = 10; + } + message CMEChannel { + repeated string id = 2; + optional string feed_engine = 3 [default = "default"]; + + optional uint32 udp_socket_buffer_size = 10 [default = 33554432]; //32mb + optional uint32 max_gap_recovery_cache_size = 11 [default = 6]; + optional uint32 lost_packet_wait_timeout_ms = 12 [default = 10]; + optional OnixSLogMode onixs_log_mode = 13 [default = Disabled]; + optional OnixSLogAdvanced onixs_log_advanced = 14 [default = LogNothing]; + optional bool prefer_backup_feeds = 15 [default = false]; + + optional bool use_tcp_replay = 20 [default = true]; + optional string tcp_replay_username = 21 [default = "MDPFFBLUE"]; + optional string tcp_replay_password = 22 [default = "password01"]; + optional int32 tcp_replay_connect_attempts = 23 [default = 3]; + optional int32 tcp_replay_reconnect_interval_ms = 24 [default = 30000]; + optional int32 tcp_replay_timeout_s = 25 [default = 10]; + + // New settings + } + optional string onixs_license = 1 [default = "{default}"]; + optional string onixs_log_path = 2 [default = "/var/log/bts/onixs"]; + repeated CMEChannel channel = 3; + repeated FeedEngineSettings feed_engine = 5; + + // Prevents parsing trades/rfqs/doing anything not required for quoting + optional bool quoter_mode = 6; +} + +message RecordOptions { + optional string output_path = 2 [default = "/home/bts/md/today"]; + optional bool enable_depth = 3 [default = false]; + optional bool enable_future_depth = 4 [default = true]; +} + +// +// Replay options +// +message ReplayOptions { + optional string host = 1 [default = "mdreplay"]; + optional string base_path = 2 [default = "/md"]; + optional string file_name = 3 [default = "options.coconut"]; + + // 2.0 = double speed, 0.5 = half speed + optional double speed = 11 [default = 1.0]; + + optional bool interactive = 12 [default = false]; +} + + +// +// Activ options +// +message ActivOptions { + optional string username = 1; + optional string password = 2; + optional string gateway = 3; + optional string serviceid = 4 [default = "Service.ContentGateway"]; + optional string network = 5; + optional string protocol = 6 [default="ams"]; + optional bool enable_intra_day_security_add = 7 [default = true]; + + optional int32 entitlement_check_interval_s = 10 [default = 300]; + repeated string child_user = 11; + + optional bool enable_depth = 12 [default = true]; + optional int32 tz_offset = 13 [default = 0]; +} + +message ActivSpreadMap { + message Entry { + optional string symbol = 1; + optional bts.security_model.Exchange.type exchange = 2 [default = TESTEXCHANGE]; + optional string pattern = 3; + } + repeated Entry entry = 1; +} + +message RelayOptions { + optional uint32 naggle_ms = 1 [default = 100]; + optional uint32 security_scan_interval = 2 [default = 500]; + optional uint32 recovery_interval = 3 [default = 30000]; +} + +// +// Some strike thing +// +message NewExpStrikeInfo +{ + required string symbol = 1; + required double low_strike = 2; + required double hi_strike = 3; + required double increment = 4; +} + +message NewExpStrikes +{ + repeated NewExpStrikeInfo add_strike_info = 1; +} + diff --git a/src/lib/config/protos/Ports.proto b/src/lib/config/protos/Ports.proto new file mode 100644 index 000000000..31d9f7f6e --- /dev/null +++ b/src/lib/config/protos/Ports.proto @@ -0,0 +1,14 @@ +syntax='proto2'; +package bts.config; + +message Ports { + // quickbus (this is a tcp port) + optional int32 qbus_port = 245 [default=8000]; + + // REDUNDANT with executor.cfg + optional int32 fix_simulator_port = 290 [default=9999]; + optional int32 fix_simulator_port_2 = 300 [default=9998]; + + // Port 80 will be default for Apache config server + optional int32 config_port = 430 [default = 80]; +} diff --git a/src/lib/config/protos/Securities.proto b/src/lib/config/protos/Securities.proto new file mode 100644 index 000000000..3dee7c882 --- /dev/null +++ b/src/lib/config/protos/Securities.proto @@ -0,0 +1,13 @@ +syntax='proto2'; +package bts.config; +import "security_model/security.proto"; + +message Security { + required string symbol = 1; + required security_model.Exchange.type exchange = 10; + required security_model.SecurityType.type type = 20; +} + +message Securities { + repeated Security security = 10; +} \ No newline at end of file diff --git a/src/lib/config/protos/Symbols.proto b/src/lib/config/protos/Symbols.proto new file mode 100644 index 000000000..a978722d7 --- /dev/null +++ b/src/lib/config/protos/Symbols.proto @@ -0,0 +1,13 @@ +syntax='proto2'; +package bts.config; +import "lib/security_model/security.proto"; + +message Symbol { + required string symbol = 1; + required security_model.Exchange.type exchange = 10; + required security_model.SecurityType.type type = 20; +} + +message Symbols { + repeated Symbol symbol = 10; +} \ No newline at end of file diff --git a/src/lib/config/protos/jetfix.proto b/src/lib/config/protos/jetfix.proto new file mode 100644 index 000000000..41f50fbba --- /dev/null +++ b/src/lib/config/protos/jetfix.proto @@ -0,0 +1,89 @@ +syntax='proto2'; +package bts.config; + +message JetFIX { + message Connection { + // a friendly name used to represent this session in the UI + required string name = 10; + + enum JetFIXPlatform { + GLBX = 1; + CBOE = 6; + } + optional JetFIXPlatform platform = 15 [default = GLBX]; + + // the ip address to connect to + required string socket_connect_host_ip = 30; + + // the port to connect to + required uint32 socket_connect_port = 40; + + // time in seconds before reconnect attempt + optional int32 reconnect_interval = 20 [default = 60]; + + // time of day the FIX session becomes activated (UTC) + optional string start_time = 50 [default = "00:30:00"]; + + // time of day the FIX session become deactiviated (UTC) + optional string end_time = 60 [ default = "23:30:00"]; + + // time in seconds between heartbeat messages (typically 30) + optional int32 seconds_between_hbs = 70 [default = 30]; + + // maximum number of incoming hbs we can miss before we close + // the connection + optional int32 max_missed_hbs = 75 [default = 2]; + + // time of day the FIX session becomes activated (UTC) + optional string start_day = 80; + + // time of day the FIX session become deactiviated (UTC) + optional string end_day = 90; + + optional string sender_compid = 100 [default = ""]; + optional string target_compid = 110 [default = ""]; + optional string sender_subid = 120 [default = ""]; + optional string target_subid = 130 [default = ""]; + optional string sender_locationid = 135 [default = "US,NC"]; + optional string raw_data = 136 [default =""]; + optional string self_match_prevention_id = 140 [default="bluestream"]; + optional string cti_code = 145 [default="1"]; + optional string mm_account = 150 [default = ""]; + + optional int32 quote_entries_per_message = 160 [default = 15]; + optional int32 registered_instrument_group_count = 170 [default = 100]; + optional int32 seconds_to_pause = 180 [default = 35]; + optional int32 seconds_to_logon = 185 [default = 5]; + optional int32 seconds_to_logout = 190 [default = 5]; + optional int32 seconds_between_syncs = 200 [default = 3]; + + optional uint32 max_qps = 210 [default = 750]; + + optional uint32 max_spt = 214 [default = 100]; + optional uint32 max_spt_window_secs = 217 [default = 3]; + + optional uint32 max_msgs_in_resend_req = 220 [default = 2499]; + + optional uint32 msg_buffer_sz = 230 [default = 10000000]; + optional uint32 msg_high_water_mark = 240 [default = 2048]; + optional uint32 seconds_to_replay = 250 [default = 30]; + optional bool log_fix_stream = 260 [default=false]; + optional bool use_persisted_quote_ids = 270 [default=true]; + optional uint32 quote_id_reservation_size = 290 [default=100000]; + optional uint32 poll_period_ms = 300 [default=100]; + optional bool use_dedicated_core = 310 [default=false]; + optional uint32 cme_market_segment_id = 320 [default=0]; + + optional string trading_session_id = 330 [default=""]; + optional int32 udf_support_indicator = 340 [default=1]; + } + + repeated Connection connection = 10; +} + +message JetFIXSimConnections { + required JetFIX.Connection glbx_quoter = 10; + required JetFIX.Connection glbx_eye = 30; + required JetFIX.Connection cboe_eye = 40; + required JetFIX.Connection glbx_hedger = 50; +} diff --git a/src/lib/config/protos/logging.proto b/src/lib/config/protos/logging.proto new file mode 100644 index 000000000..0d91eb29f --- /dev/null +++ b/src/lib/config/protos/logging.proto @@ -0,0 +1,19 @@ +syntax='proto2'; + +package bts.config; + +message Logging { + message SubsystemConfig { + required string name = 1; + required string level = 2; + } + message SinkConfig { + optional string default_level = 2 [default="Info"]; + repeated SubsystemConfig subsystem = 3; + } + required SinkConfig console = 1; + required SinkConfig syslog = 2; + + optional uint32 initial_queue_size = 3 [default=12000000]; + optional uint32 max_queue_growth = 4 [default=4]; +} diff --git a/src/lib/config/protos/quickbus_cfg.proto b/src/lib/config/protos/quickbus_cfg.proto new file mode 100644 index 000000000..c779a2f77 --- /dev/null +++ b/src/lib/config/protos/quickbus_cfg.proto @@ -0,0 +1,20 @@ +syntax='proto2'; +package bts.config; + +message QuickbusNodeCfg { + optional uint32 reconnect_period_secs = 1 [default=5]; + optional string parent_host = 2 [default=""]; + optional uint32 parent_port = 3 [default=0]; + optional uint32 my_port = 4 [default=0]; + optional uint32 hb_period_msecs = 5 [default=15000]; + optional uint32 max_missed_hbs = 6 [default=3]; +} + +message QuickbusCfg { + optional uint32 reconnect_period_secs = 1 [default=5]; + optional string qbus_host = 2 [default=""]; + optional uint32 qbus_port = 3 [default=0]; + optional uint32 hb_period_msecs = 5 [default=15000]; + optional uint32 max_missed_hbs = 6 [default=3]; +} + diff --git a/src/lib/config/protos/system.proto b/src/lib/config/protos/system.proto new file mode 100755 index 000000000..807931ccc --- /dev/null +++ b/src/lib/config/protos/system.proto @@ -0,0 +1,38 @@ +syntax='proto2'; + +package bts.config; + +message System +{ + required string host_name = 1; + required string process_name = 3; + required string process_path = 4; + required string platform = 5; + required string qualifier = 6; + required bool unittest_environment = 7; + required string data_path = 8; + optional string version = 9 [default = "private"]; + repeated string cmd_args = 10; + required string installation = 11; + required string installation_specific_data_path = 12; + required int32 build_number = 13 [default = 0]; + required string build_type = 14 [default = "release"]; + required string config_host = 15; + required string config_ip = 16 [default = "127.0.0.1"]; + optional bool is_remote_app = 17 [default = false]; + required string os_user = 18; + optional int32 dpi = 19 [default = 96]; + optional double dpi_ratio = 20 [default = 1.0]; + optional int32 num_cores = 21; + optional int32 config_port = 22 [default = 80]; +} + +message ConfigIntrospection { + message ConfigProto { + required string name = 1; + repeated string type = 2; + } + repeated ConfigProto config = 1; +} + + diff --git a/src/lib/config/protos/testkeys.proto b/src/lib/config/protos/testkeys.proto new file mode 100755 index 000000000..7742ff653 --- /dev/null +++ b/src/lib/config/protos/testkeys.proto @@ -0,0 +1,84 @@ +syntax='proto2'; + +import "lib/config/extensions/extensions.proto"; + +package bts.config.test; + +// This is a comment about Foo +message Foo { + option(doc_string) = "This is a test key. Don't use it for real data"; + + optional string name = 1 [default = "Blue Foo"]; + optional int32 id = 2 [default = 1234]; + repeated string email = 3; + optional string url = 4; +} + +message FooGroup { + required string group = 1; + required Foo foo = 2; +} + +message NamedFoo { + optional string key = 1; + optional Foo foo = 2; + repeated Foo foox = 3; + optional string rule = 4; +} + +message FooList { + repeated NamedFoo item = 1; + repeated Foo foo = 2; +} + +message StringTable { + repeated string entries = 1; +} + +message RuleRef { + option(doc_string) = "This contains properties of various types which we can use as test cases for self-referenential rules"; + + optional string a_string = 1; + optional int32 a_int = 2; + repeated string list_string = 3; + repeated int32 list_int = 4; + repeated Foo foox = 6; + + optional RuleRef nested = 5; + +} + +// For Config demo +message ConsoleColor { + enum Colors { + DarkBlue = 1; + DarkGreen = 2; + DarkTeel = 3; + DarkRed = 4; + DarkPink = 5; + DarkYellow = 6; + Gray = 7; + DarkGray = 8; + Blue = 9; + Green = 10; + Teal = 11; + Red = 12; + Pink = 13; + Yellow = 14; + White = 15; + } + required Colors color = 1; +} + +message ConsoleColors { + optional ConsoleColor.Colors color = 1; + repeated ConsoleColor.Colors colors = 2; +} + +message TestSite { + required string name = 1 [default="NO NAME!"]; +} + +message Ports { + optional int32 config_port = 1; +} diff --git a/src/lib/config/tests/HTTPServer.cpp b/src/lib/config/tests/HTTPServer.cpp new file mode 100755 index 000000000..3c1e189f8 --- /dev/null +++ b/src/lib/config/tests/HTTPServer.cpp @@ -0,0 +1,104 @@ + +#include "HTTPServer.h" + +using boost::asio::ip::tcp; + +namespace bts { +namespace config { +namespace internal { + +HTTPServer::HTTPServer() + : _io_service(), + _acceptor(_io_service, tcp::endpoint(boost::asio::ip::address_v4::loopback(), 0)), + _cur_socket(), + _request_response_pairs(), + _thread(), + _done(false) +{ + +} + +unsigned int HTTPServer::port() const +{ + return _acceptor.local_endpoint().port(); +} + +void HTTPServer::add_response(const std::string& path, const std::string& response, const std::string& content) +{ + _request_response_pairs[path] = response + content; +} + +void HTTPServer::start() +{ + _thread.reset(new boost::thread(boost::bind(&HTTPServer::worker, this))); +} + +void HTTPServer::stop() +{ + _done = true; + _io_service.stop(); + + if (_thread) { + _thread->join(); + _thread.reset(); + } +} + + +void HTTPServer::worker() +{ + start_accept(); + + _io_service.run(); +} + +void HTTPServer::start_accept() +{ + _cur_socket.reset(new boost::asio::ip::tcp::socket(_io_service)); + _acceptor.async_accept(*_cur_socket.get(), boost::bind(&HTTPServer::handle_accept, this, _cur_socket.get(), boost::asio::placeholders::error)); +} + +void HTTPServer::handle_accept(boost::asio::ip::tcp::socket* socket, const boost::system::error_code& error) +{ + if (!error) + { + handle_connect(socket); + } + start_accept(); +} + +void HTTPServer::handle_connect(boost::asio::ip::tcp::socket* socket) +{ + boost::asio::streambuf buffer; + size_t size = boost::asio::read_until(*socket, buffer, "\r\n\r\n"); + std::string request(size, 0); + buffer.sgetn(&request[0], size); + + size_t start = request.find("GET ") + 4; + size_t end = request.find(" HTTP/1.1\r\n"); + size_t url_args = request.find("?"); + if (url_args != std::string::npos) + end = std::min(end, url_args); + std::string request_path = request.substr(start, end-start); + + std::map::iterator request_match = + _request_response_pairs.find(request_path); + + if (request_match == _request_response_pairs.end()) + { + std::string response = "HTTP/1.0 404 NOT FOUND\r\n" + "Content-Type: text/plain\r\n\r\n" + "not found\r\n"; + boost::asio::write(*socket, boost::asio::buffer(response)); + } + else + { + boost::asio::write(*socket, boost::asio::buffer(request_match->second)); + } + + boost::system::error_code ec; + socket->shutdown(tcp::socket::shutdown_both, ec); + socket->close(ec); +} + +}}} diff --git a/src/lib/config/tests/HTTPServer.h b/src/lib/config/tests/HTTPServer.h new file mode 100755 index 000000000..a52aae5d3 --- /dev/null +++ b/src/lib/config/tests/HTTPServer.h @@ -0,0 +1,53 @@ +#ifndef HTTP_TEST_SERVER_H_ +#define HTTP_TEST_SERVER_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include + +namespace bts { +namespace config { +namespace internal { + +class HTTPServer +{ +public: + HTTPServer(); + unsigned int port() const; + + void start(); + void add_response(const std::string& path, const std::string& response, const std::string& content); + void stop(); + +private: + void worker(); + void start_accept(); + void handle_accept(boost::asio::ip::tcp::socket*, const boost::system::error_code& error); + void handle_connect(boost::asio::ip::tcp::socket*); + + boost::asio::io_service _io_service; + boost::asio::ip::tcp::acceptor _acceptor; + boost::scoped_ptr _cur_socket; + std::map _request_response_pairs; + boost::scoped_ptr _thread; + bool _done; + +}; + +}}} // end namespace +#endif diff --git a/src/lib/config/tests/config_ref_tests.cpp b/src/lib/config/tests/config_ref_tests.cpp new file mode 100755 index 000000000..89cacd262 --- /dev/null +++ b/src/lib/config/tests/config_ref_tests.cpp @@ -0,0 +1,932 @@ +#include "lib/utils/TempFile.h" +#include "lib/utils/TempDir.h" + +#include "../config.h" +#include "../ConfigurationState.h" +#include "../ConfigurationException.h" +#include "lib/expression/ParserExceptions.h" +#include "lib/config/protos/testkeys.pb.h" +#include "lib/config/invalid/invalid.pb.h" + +#include "HTTPServer.h" +#include "catch/catch.h" +#include +#include + +using namespace bts::config; +using namespace bts::config::internal; +namespace proto = google::protobuf; + +using bts::config::test::Foo; +using bts::config::test::FooGroup; +using bts::config::test::FooList; + +namespace bts { +TEST_CASE ("Configuration Reference Tests", "[config]") { + + SECTION("basic_value") + { + initialize_for_test( + "TestSite {" + " value {" + " name: 'ref value'" + " }" + "}" + "Foo { \n" + " value {\n" + " name: @TestSite.name" + " }\n" + "}\n" + ); + + const Foo& foo = get(); + REQUIRE(foo.name() == "ref value"); + } + + SECTION("scalar_to_repeat") { + initialize_for_test( + "TestSite {" + " value {" + " name: 'a@example.com' " + " }" + "}" + "Foo { \n" + " value {\n" + " email: \"before@example.com\" \n" + " email: @TestSite.name \n" + " email: \"after@example.com\" \n" + " }\n" + "}\n" + ); + + const Foo& foo = get(); + + REQUIRE(foo.email_size() == 3); + REQUIRE(foo.email(0) == "before@example.com"); + REQUIRE(foo.email(1) == "a@example.com"); + REQUIRE(foo.email(2) == "after@example.com"); + } + + SECTION("repeat_to_repeat ") { + initialize_for_test( + "StringTable {" + " value {" + " entries: 'a@example.com' " + " entries: 'b@example.com' " + " }" + "}" + "TestSite {" + " value {" + " name: 'c@example.com' " + " }" + "}" + "Foo { \n" + " value {\n" + " email: \"before@example.com\" \n" + " email: @StringTable.entries \n" + " email: @TestSite.name \n" + " email: \"after@example.com\" \n" + " }\n" + "}\n" + ); + + const Foo& foo = get(); + + REQUIRE(foo.email_size() == 5); + REQUIRE(foo.email(0) == "before@example.com"); + REQUIRE(foo.email(1) == "a@example.com"); + REQUIRE(foo.email(2) == "b@example.com"); + REQUIRE(foo.email(3) == "c@example.com"); + REQUIRE(foo.email(4) == "after@example.com"); + } + + SECTION("message_value") { + initialize_for_test( + "Foo { \n" + " rule: 'false'" + " value {\n" + " name: \"bad\"\n" + " }\n" + "}\n" + "Foo { \n" + " value {\n" + " name: \"good\"\n" + " }\n" + "}\n" + "FooGroup {" + " value { " + " group: @Foo.name " + " foo: @Foo " + " }" + "}" + ); + + const FooGroup& g = get(); + REQUIRE(g.group() == "good"); + REQUIRE(g.foo().name() == "good"); + } + + SECTION("nested_message_value") { + initialize_for_test( + "NamedFoo { \n" + " value {\n" + " key: 'good foo' " + " foo { " + " name: \"good\"\n" + " }" + " }\n" + "}\n" + "FooGroup {" + " value { " + " group: @NamedFoo.key " + " foo: @NamedFoo.foo " + " }" + "}" + ); + + const FooGroup& g = get(); + REQUIRE(g.group() == "good foo"); + REQUIRE(g.foo().name() == "good"); + } + + SECTION("macro_value") { + initialize_for_test( + "macro {" + " name: 'macro1' " + " expression: \"'m1'\" " + "}" + "Foo { \n" + " value {\n" + " name: $macro1" + " }\n" + "}\n" + ); + + const Foo& foo = get(); + REQUIRE(foo.name() == "m1"); + } + + SECTION("enum_value") { + initialize_for_test( + "ConsoleColor {" + " value {" + " color: Blue " + " }" + "}" + "ConsoleColors {" + " value {" + " color: @ConsoleColor.color" + " colors: Green" + " colors: @ConsoleColor.color" + " colors: DarkRed" + " colors: @ConsoleColor.color" + " }" + "}" + ); + + const test::ConsoleColor& c1 = get(); + const test::ConsoleColors& c2 = get(); + REQUIRE(c2.color() == c1.color()); + REQUIRE(c2.colors_size() == 4); + REQUIRE(c2.colors(0) == test::ConsoleColor::Green); + REQUIRE(c2.colors(1) == c1.color()); + REQUIRE(c2.colors(2) == test::ConsoleColor::DarkRed); + REQUIRE(c2.colors(3) == c1.color()); + } + + SECTION("named_key") { + initialize_for_test( + "Foo { \n" + " value {\n" + " name: @FooList.item[foo2].foo.name " + " }\n" + "}\n" + "FooList { \n" + " value {\n" + " item { " + " key: 'foo1' " + " foo { " + " name: 'foo1' " + " }" + " }" + " item { " + " key: 'foo2' " + " foo { " + " name: 'foo2' " + " }" + " }" + " item { " + " key: 'foo3' " + " foo { " + " name: 'foo3' " + " }" + " }" + " }\n" + "}\n" + ); + + REQUIRE(get().name() == "foo2"); + } + + SECTION("named_key_rule") { + initialize_for_test( + "Foo { \n" + " value {\n" + " name: @FooList.item[foo2].foo.name " + " }\n" + "}\n" + "FooList { \n" + " value {\n" + " item { " + " key: 'foo1' " + " foo { " + " name: 'foo1' " + " }" + " }" + " item { " + " key: 'foo2' " + " rule: '1=0' " + " foo { " + " name: 'foo2' " + " }" + " }" + " item { " + " key: 'foo3' " + " foo { " + " name: 'foo3' " + " }" + " }" + " }\n" + "}\n" + ); + + //NOTE that a false rule will still create a message with defaults, so the list contains the same number of elements + REQUIRE(get().name() == "Blue Foo"); + REQUIRE(get().item_size() == 3); + } + + SECTION("named_key2") { + initialize_for_test( + "Foo { \n" + " value: @FooList.foo[foo2] " + "}\n" + "FooList { \n" + " value {\n" + " foo { " + " name: 'foo1' " + " id: 1 " + " }" + " foo { " + " name: 'foo2' " + " id: 2 " + " }" + " foo { " + " name: 'foo3' " + " id: 3 " + " }" + " }\n" + "}\n" + ); + + REQUIRE(get().name() == "foo2"); + REQUIRE(get().id() == 2); + } + + SECTION("named_key_syntax_error") { + bool error = false; + try { + initialize_for_test( + "Foo { \n" + " value: @FooList[foo2] " // FooList doesn't have a key or a property named foo2 + "}\n" + "FooList { \n" + " value {\n" + " foo { " + " name: 'foo1' " + " id: 1 " + " }" + " foo { " + " name: 'foo2' " + " id: 2 " + " }" + " foo { " + " name: 'foo3' " + " id: 3 " + " }" + " }\n" + "}\n" + ); + } + catch(...) { + error = true; + } + REQUIRE(error); + } + + SECTION("named_struture") { + initialize_for_test( + "FooGroup { \n" + " value {\n" + " foo: @FooList.item[foo3].foo " + " }\n" + "}\n" + "FooList { \n" + " value {\n" + " item { " + " key: 'foo1' " + " foo { " + " name: 'foo1' " + " id: 1 " + " }" + " }" + " item { " + " key: 'foo2' " + " foo { " + " name: 'foo2' " + " id: 2 " + " }" + " }" + " item { " + " key: 'foo3' " + " foo { " + " name: 'foo3' " + " id: 3 " + " }" + " }" + " }\n" + "}\n" + ); + REQUIRE(get().foo().name() == "foo3"); + REQUIRE(get().foo().id() == 3); + } + + SECTION("named_structure_rule") { + initialize_for_test( + "FooGroup { \n" + " value {\n" + " foo: @FooList.item[foo3].foo " + " }\n" + "}\n" + "FooList { \n" + " value {\n" + " item { " + " key: 'foo1' " + " foo { " + " name: 'foo1' " + " id: 1 " + " }" + " }" + " item { " + " key: 'foo2' " + " foo { " + " name: 'foo2' " + " id: 2 " + " }" + " }" + " item { " + " key: 'foo3' " + " rule: 'false'" + " foo { " + " name: 'foo3' " + " id: 3 " + " }" + " }" + " }\n" + "}\n" + ); + //NOTE that a false rule will still create a message with defaults, so the list contains the same number of elements + REQUIRE(get().foo().name() == "Blue Foo"); + REQUIRE(get().foo().id() == 1234); + REQUIRE(get().item_size() == 3); + } + + SECTION("named_value_rule") { + initialize_for_test( + "FooList { \n" + " value {\n" + " item { " + " key: 'foo1' " + " foo { " + " name: 'foo1' " + " id: 1 " + " }" + " }" + " item { " + " key: 'foo2' " + " foo { " + " name: 'foo2' " + " id: 2 " + " }" + " }" + " item { " + " key: 'foo3' " + " foo { " + " name: 'foo3' " + " id: 3 " + " }" + " }" + " }\n" + "}\n" + "Foo { \n" + // Adding [] to BooleanGrammer.h is very difficult because boost::spirit is crap + // also it risks making list syntax [1,2,3] ambiguous so this is a workaround syntax + " rule: 'FooList.item.foo1.foo.id = 2' " + " value {\n" + " name: 'bad' " + " }\n" + "}\n" + "Foo { \n" + " rule: 'FooList.item.foo2.foo.id = 2' " + " value {\n" + " name: 'good' " + " }\n" + "}\n" + ); + + const Foo& foo = get(); + REQUIRE(foo.name() == "good"); + } + + SECTION("recursive_reference_good") { + initialize_for_test( + "TestSite { \n" + " value {\n" + " name: \"good\"\n" + " }\n" + "}\n" + "Foo { \n" + " value {\n" + " name: @TestSite.name " + " }\n" + "}\n" + "FooGroup {" + " value { " + " group: @Foo.name " + " foo: @Foo " + " }" + "}" + ); + + REQUIRE(get().group() == "good"); + REQUIRE(get().foo().name() == "good"); + } + + SECTION("recursive_reference_bad") { + bool error = false; + try { + + initialize_for_test( + "TestSite { \n" + " value {\n" + " name: @Foo.name " + " }\n" + "}\n" + "Foo { \n" + " value {\n" + " name: @TestSite.name " + " }\n" + "}\n" + "FooGroup {" + " value { " + " group: @Foo.name " + " foo: @Foo " + " }" + "}" + ); + + + } catch(...) { + error = true; + } + REQUIRE(error); + } + + SECTION("recursive_order_independent") { + initialize_for_test( + "FooGroup {" + " value { " + " group: @Foo.name " + " foo: @Foo " + " }" + "}" + "TestSite { \n" + " value {\n" + " name: \"good\"\n" + " }\n" + "}\n" + "Foo { \n" + " value {\n" + " name: @TestSite.name " + " }\n" + "}\n" + ); + + // In this test we mighe expect @Foo.name or @Foo to be evaluated first + // But Foo is incomplete until @TestSite.name is evaluated. + REQUIRE(get().group() == "good"); + REQUIRE(get().foo().name() == "good"); + } + + SECTION("loop_test") { + bool error = false; + try { + initialize_for_test( + "Foo { \n" + " value {\n" + " name: @Foo.name " + " }\n" + "}\n"); + } catch(...) { + error = true; + } + REQUIRE(error); + } + + SECTION("merge_test") { + // Config used to go through a merge step for each file in a directory. + // This would invalidate the memory held by the deferred value queue and cause + // crashes and instability. This is to verify that we don't regress the new behavior. + + HTTPServer server; + std::string port = boost::lexical_cast(server.port()); + std::string config_url = "http://localhost:" + port + "/"; + std::string response = + "HTTP/1.0 200 OK\r\n" + "Content-Type: text/plain\r\n\r\n"; + + server.add_response("/edge.conf", response, + "Foo {\n" + " value: {\n" + " name: \"included_value\"\n" + " id: 4096\n" + " }}\n" + "TestSite {" + " value { " + " name: @FooGroup.group" // reference to testkeys.conf + " }" + "}" + ); + server.add_response("/testkeys.conf", response, + "FooGroup {\n" + " value {\n" + " group: \"http_group\"\n" + " foo {\n" + " name: @Foo.name " // reference to edge.conf + " id: 80\n" + "}}}\n" + ); + + server.start(); + initialize_for_test("include: \"" + config_url + "\""); + server.stop(); + + const FooGroup& group = get(); + const test::TestSite& site = get(); + + REQUIRE(group.foo().name() == "included_value"); + REQUIRE(site.name() == "http_group"); + } + + SECTION("test_conversion") { + initialize_for_test( + "Foo {\n" + " value: {\n" + " name: @RuleRef.a_int \n" + " id: @RuleRef.a_string " + " }}\n" + "Foo {\n" + " rule: 'false' " + " value: {\n" + " name: @RuleRef.a_string \n" + " id: @RuleRef.a_int " + " }}\n" + "RuleRef { " + " value { " + " a_string: '42' " + " a_int: 99 " + " }" + "}" + ); + + const Foo& foo = get(); + REQUIRE(foo.name() == "99"); + REQUIRE(foo.id() == 42); + } + + SECTION("test_int") { + initialize_for_test( + "Foo {\n" + " value: {\n" + " id: @RuleRef.a_int " + " }}\n" + "RuleRef { " + " value { " + " a_string: 'good' " + " a_int: 99 " + " }" + "}" + ); + + const Foo& foo = get(); + REQUIRE(foo.id() == 99); + } + + SECTION("test_repeated_to_repeated") { + initialize_for_test( + "FooList {\n" + " value: {\n" + " foo {\n" + " name: 'good1' \n" + " }\n" + " foo {\n" + " name: 'good2' \n" + " }\n" + " }}\n" + "RuleRef { " + " value { " + " foox { " + " name: 'original' " + " }" + " foox: @FooList.foo " + " }" + "}" + ); + + const test::RuleRef& result = get(); + + REQUIRE(result.foox_size() == 3); + REQUIRE(result.foox(0).name() == "original"); + REQUIRE(result.foox(1).name() == "good1"); + REQUIRE(result.foox(2).name() == "good2"); + } + + SECTION("test_repeated_after_key_rule") { + initialize_for_test( + "StringTable { \n" + " value { \n" + " entries: 'a' \n" + " entries: 'b' \n" + " } \n" + "} \n" + "FooList {\n" + " value: {\n" + " item {\n" + " key: 'one' \n" + " rule: \" 'a' in StringTable.entries\" \n" + " foox { \n" + " name: 'good1' \n" + " }\n" + " foox { \n" + " name: 'good2' \n" + " }\n" + " }\n" + " item {\n" + " key: 'two' \n" + " foox { \n" + " name: 'bad1' \n" + " }\n" + " }\n" + " }}\n" + "RuleRef { " + " value { " + " foox { " + " name: 'original' " + " }" + " foox: @FooList.item[one].foox " + " }" + "}" + ); + + const test::RuleRef& result = get(); + REQUIRE(result.foox_size() == 3); + REQUIRE(result.foox(0).name() == "original"); + REQUIRE(result.foox(1).name() == "good1"); + REQUIRE(result.foox(2).name() == "good2"); + } + + SECTION("test_repeated_after_key_rule2") { + //This test fails on purpose to illustrate the frailty of the rule evaluation feature in 'keyed' list keys + initialize_for_test( + "TestSite {" + " value {" + " name: 'a' " + " }" + "}" + "StringTable {" + " value {" + " entries: @TestSite.name " + " entries: 'b' " + " }" + "}" + "FooList {\n" + " value: {\n" + " item {\n" + " key: 'one' \n" + " rule: \"'a' in StringTable.entries\" \n" + " foox { \n" + " name: 'good1' \n" + " }\n" + " foox { \n" + " name: 'good2' \n" + " }\n" + " }\n" + " item {\n" + " key: 'two' \n" + " foox { \n" + " name: 'bad1' \n" + " }\n" + " }\n" + " }}\n" + "RuleRef { " + " value { " + " foox { " + " name: 'original' " + " }" + " foox: @FooList.item[one].foox " + " }" + "}" + ); + + const test::RuleRef& result = get(); + REQUIRE(result.foox_size() == 3); + REQUIRE(result.foox(0).name() == "original"); + REQUIRE(result.foox(1).name() == "good1"); + REQUIRE(result.foox(2).name() == "good2"); + } + + SECTION("test_repeated_after_key_rule3") { + //This test fails on purpose to illustrate the frailty of the rule evaluation feature in 'keyed' list keys + initialize_for_test( + "RuleRef { " + " value { " + " foox { " + " name: 'original' " + " }" + " foox: @FooList.item[one].foox " + " }" + "}" + "TestSite {" + " value {" + " name: 'a' " + " }" + "}" + "StringTable {" + " value {" + " entries: @TestSite.name " + " entries: 'b' " + " }" + "}" + "FooList {\n" + " value: {\n" + " item {\n" + " key: 'one' \n" + " rule: \"'a' in StringTable.entries\" \n" + " foox { \n" + " name: 'good1' \n" + " }\n" + " foox { \n" + " name: 'good2' \n" + " }\n" + " }\n" + " item {\n" + " key: 'two' \n" + " foox { \n" + " name: 'bad1' \n" + " }\n" + " }\n" + " }}\n" + + ); + + const test::RuleRef& result = get(); + //the order matters here, but seems to be deterministic + REQUIRE(result.foox_size() == 1); + } + + // SECTION("test_repeated_after_key_rule_should_fail") { + // //This test fails on purpose to illustrate the frailty of the rule evaluation feature in 'keyed' list keys + // initialize_for_test( + // "RuleRef { " + // " value { " + // " foox { " + // " name: 'original' " + // " }" + // " foox: @FooList.item[one].foox " + // " }" + // "}" + // "TestSite {" + // " value {" + // " name: 'a' " + // " }" + // "}" + // "StringTable {" + // " value {" + // " entries: @TestSite.name " + // " entries: 'b' " + // " }" + // "}" + // "FooList {\n" + // " value: {\n" + // " item {\n" + // " key: 'one' \n" + // " rule: \"'a' in StringTable.entries\" \n" + // " foox { \n" + // " name: 'good1' \n" + // " }\n" + // " foox { \n" + // " name: 'good2' \n" + // " }\n" + // " }\n" + // " item {\n" + // " key: 'two' \n" + // " foox { \n" + // " name: 'bad1' \n" + // " }\n" + // " }\n" + // " }}\n" + + // ); + + // const test::RuleRef& result = get(); + // //this should fail + // REQUIRE(result.foox_size() == 3); + // } + + SECTION("test_irrelevant_lookup") { + // If a key references another key which is not present it should only be an error if the resulting key will ever be used. + initialize_for_test( + "FooList {\n" + " value: {\n" + " item {\n" + " key: 'one' \n" + " foox { \n" + " name: 'good1' \n" + " }\n" + " foox { \n" + " name: 'good2' \n" + " }\n" + " }\n" + " item {\n" + " key: 'two' \n" + " foox { \n" + " name: 'bad1' \n" + " }\n" + " }\n" + " }}\n" + "RuleRef { " + " rule: 'false' " + " value { " + " foox { " + " name: 'never used' " + " }" + " foox: @FooList.item[XXXXX].foox " // Invalid lookup + " }" + "}" + "RuleRef { " + " value { " + " foox { " + " name: 'original' " + " }" + " foox: @FooList.item[one].foox " + " }" + "}" + ); + const test::RuleRef& result = get(); + REQUIRE(result.foox_size() == 3); + } + + SECTION("macro_definition_order_test_hard") { + // The $is_crusible test hard version + // $macro1 tests Value2.x with rule about $macro2 before $macro2 is defined + // Value2.x is an @reference of a 4th location + initialize_for_test( + "TestSite {" + " value { name: 'cme_1u7_cert' }" + "}" + "Foo {" + " rule: '$is_crucible'" + " value { name: @TestSite.name }" + "}" + "" + "Foo {" + " value { name: '' }" + "}" + "" + "macro {" + " name: 'o2_cme_enabled'" + " expression: \"Foo.name != ''\" " + "}" + "" + "macro {" + " name: 'is_crucible'" + " expression: 'true'" + "}" + ); + + REQUIRE(get().name() == "cme_1u7_cert"); + } + + reset(); +} + +} diff --git a/src/lib/config/tests/config_tests.cpp b/src/lib/config/tests/config_tests.cpp new file mode 100755 index 000000000..f8febbb74 --- /dev/null +++ b/src/lib/config/tests/config_tests.cpp @@ -0,0 +1,578 @@ +#include "lib/utils/TempFile.h" + +#include "../config.h" +#include "../ConfigurationState.h" +#include "../ConfigurationException.h" +#include "lib/expression/ParserExceptions.h" +#include "lib/utils/bts_assert.h" +#include "lib/utils/is_unittest_environment.h" + +#include "HTTPServer.h" + +#include "lib/config/invalid/invalid.pb.h" +#include "lib/config/protos/system.pb.h" +#include "lib/config/protos/testkeys.pb.h" + +#include +#include +#include + +#define URDL_HEADER_ONLY 1 +#define URDL_DISALE_SSL 1 +#include +#include +#include + +using namespace bts::config; +using namespace bts::config::internal; +namespace proto = google::protobuf; + +using bts::config::test::Foo; +using bts::config::test::FooGroup; +using bts::config::test::StringTable; + +namespace bts { +TEST_CASE("Configuration Tests","[config]") { + + bts::utils::internal::force_unittest_environment(); + + SECTION("basic_get_value_test") + { + initialize_for_test( + "Foo { \n" + " rule: \"true\"\n" + " value {\n" + " name: \"Name A\"\n" + " id: 65\n" + " email: \"test@example.com\"\n" + " }\n" + "}\n"); + + const Foo& foo = get(); + + REQUIRE(foo.name() == "Name A"); + REQUIRE(foo.id() == 65); + REQUIRE(foo.email().Get(0) == "test@example.com"); + } + + SECTION("default_value_test") + { + initialize_for_test(""); // Only loads defaults.conf + + // Should load a Foo from testkeys.conf + const Foo& foo = get(); + REQUIRE(foo.name() == "Blue Foo"); // from testkeys.proto + REQUIRE(foo.id() == 1234); // from testkeys.proto + REQUIRE(foo.email().size() > 0); + REQUIRE(foo.email().Get(0) == "default@example.com"); // from testkeys.conf + } + + SECTION("default_include_test") + { + // test defaults.conf with an include + } + + SECTION("invalid_key_type_test") + { + // Load an initialize with no files - OK all values default + + const Foo& foo = get(); // OK + REQUIRE(foo.name() == "Blue Foo"); + + bool error = false; + try + { + get(); // BAD + } + catch(...) + { + error = true; + } + REQUIRE(error == true); + } + + SECTION("invalid_config_test2") + { + utils::TempFile config_b; + + *config_b.out() << + "Foo {\n" + " value { \n" + " name: \"bob\"\n" + " id: 7319 \n" + " }"; + + bool error = false; + try + { + initialize_for_test("include: \"" + config_b.path() + "\"", false); + } + catch(const ConfigurationException& err) + { + std::string what = err.what(); + bts_assert(what.find("Expected identifier.") != std::string::npos); + error = true; + } + REQUIRE(error == true); + + } + + // Tests that we can load using a File:/// url + // Tests complex value load + SECTION("file_url_test") + { + utils::TempFile config_a; + utils::TempFile config_b; + + *config_a.out() << "include: \"" << config_b.path() << "\"" << std::endl << + "Foo { \n" + " rule: \"false\"\n" + " value {\n" + " name: \"Name A\"\n" + " id: 65\n" + " email: \"A@bluecg.com\"\n" + " }\n" + "}\n"; + + *config_b.out() << + "Foo {\n" + " rule: \"true\"\n" + " value {\n" + " name: \"Name B\"\n" + " id: 66 " + " email: \"A@bluecg.com\" \n" + " }}"; + + + // Get key Foo without loading any config files - this will show the default value + Foo foo = bts::config::get(); + + REQUIRE(foo.name() == "Blue Foo"); + REQUIRE(foo.id() == 1234); + + // Parse the config file and file:/// include + initialize_for_test("include: \"" + config_a.path() + "\""); + + // Get the new value of Foo - this time it will come from config_b + foo = bts::config::get(); + REQUIRE(foo.name() == "Name B"); + REQUIRE(foo.id() == 66); + } + + SECTION("http_test") + { + HTTPServer server; + std::string port = boost::lexical_cast(server.port()); + std::string path = "/edge.conf"; + std::string response = + "HTTP/1.0 200 OK\r\n" + "Content-Type: text/plain\r\n\r\n"; + std::string content = + "FooGroup {\n" + " rule: \"true\"\n" + " value {\n" + " group: \"http_group\"\n" + " foo {\n" + " name: \"http\"\n" + " id: 80\n" + "}}}\n"; + + server.add_response(path, response, content); + server.start(); + std::string config_url = "http://127.0.0.1:" + port; + + // Initialize with a configuration which references the http url + initialize_for_test("include: \"" + config_url + "\""); + + server.stop(); + + const FooGroup& foogrp = get(); + + REQUIRE(foogrp.group() == "http_group"); + REQUIRE(foogrp.foo().name() == "http"); + REQUIRE(foogrp.foo().id() == 80); + REQUIRE(get().config_host() == "127.0.0.1"); + } + + SECTION("http_folder_test") + { + HTTPServer server; + std::string port = boost::lexical_cast(server.port()); + std::string config_url = "http://localhost:" + port + "/"; + std::string response = + "HTTP/1.0 200 OK\r\n" + "Content-Type: text/plain\r\n\r\n"; + + server.add_response("/edge.conf", response, "include: \"" + config_url + "other/\""); + server.add_response("/testkeys.conf", response, + "FooGroup {\n" + " rule: \"true\"\n" + " value {\n" + " group: \"http_group\"\n" + " foo {\n" + " name: \"http\"\n" + " id: 80\n" + "}}}\n"); + + server.add_response("/other/edge.conf", response, + "Foo {\n" + " rule: \"true\"\n" + " value: {\n" + " name: \"included_value\"\n" + " id: 4096\n" + " }}\n"); + + + server.start(); + initialize_for_test("include: \"" + config_url + "\""); + + server.stop(); + + const FooGroup& foogrp = get(); + + REQUIRE(foogrp.group() == "http_group"); + REQUIRE(foogrp.foo().name() == "http"); + REQUIRE(foogrp.foo().id() == 80); + + REQUIRE(get().name() == "included_value"); + REQUIRE(get().id() == 4096); + } + + + SECTION("test_rules") + { + initialize_for_test( + "StringTable {\n" + " value { \n" + " entries: [\"foo\", \"bar\", \"baz\"] \n" + " }\n" + " }\n" + "RuleRef { \n" + " # use this rule of StringTable has a cow \n" + " rule: \"'cow' in StringTable.entries\" \n" + " value { \n" + " a_string: \"AAA\"\n" + " a_int: -1\n" + " }\n" + "}\n" + "RuleRef { \n" + " rule: \"'baz' in StringTable.entries\" \n" + " value { \n" + " a_string: \"BBB\"\n" + " a_int: 99\n" + " }\n" + "}\n" + "Foo { \n" + " rule: \"RuleRef.a_int == 789\"\n" + " value {\n" + " name: \"A\"\n" + " }\n" + "}\n" + "Foo { \n" + " rule: \"RuleRef.a_string == 'BBB'\"\n" + " value {\n" + " name: \"B\"\n" + " }\n" + "}\n" + ); + + + // What should happen: + // Foo A + // eval rule + // find RuleRef AAA + // eval rule + // fail - StringTable.entries does not have 'cow' + // find RuleRef BBB + // eval rule + // success + // Fail - RuleRef.a_int != 789 + // Foo B + // eval rule + // - same as above + // success a_string == 'BBB' + // return Foo B + + const Foo& foo = get(); + REQUIRE(foo.name() == "B"); + } + + + + SECTION("test_recursive_rules") + { + initialize_for_test( + "StringTable {\n" + " rule: \"Foo.name == 'foo'\"\n" + " value { \n" + " entries: [\"foo\", \"bar\", \"baz\"] \n" + " }\n" + " }\n" + "RuleRef { \n" + " rule: \"'foo' in StringTable.entries\" \n" + " value { \n" + " a_string: \"AAA\"\n" + " a_int: -1\n" + " }\n" + "}\n" + "Foo { \n" + " rule: \"RuleRef.a_string == 'BBB'\"\n" + " value {\n" + " name: \"foo\"\n" + " }\n" + "}\n" + ); + + // Foo cannot be queried because RuleRef's rule requests StringTable which requests RuleRef + bool error = false; + try + { + get(); + } + catch(const expression::RecursivePropertyGetException&) + { + error = true; + } + REQUIRE(error); + } + + SECTION("test_nested_properties") + { + initialize_for_test( + "RuleRef { \n" + " value { \n" + " nested {\n" + " list_int: [7,8,9] \n" + " nested {\n" + " a_string: 'ABC' \n" + " }\n" + " }\n" + " }\n" + "}\n" + "Foo { \n" + " rule: \"RuleRef.nested.nested.a_string == 'ABC' and 9 in RuleRef.nested.list_int\"\n" + " value {\n" + " name: \"foo\"\n" + " }\n" + "}\n" + ); + + REQUIRE(get().name() == "foo"); + } + + SECTION("built_in_properties_test") + { + initialize_for_test( + "Foo { \n" + " rule: \"System.unittest_environment == true \"\n" + " value {\n" + " name: \"unit tests\"\n" + " }\n" + "}\n" + ); + + REQUIRE(get().name() == "unit tests"); + } + + SECTION("nonbreaking_space_test") + { + bool parse_error = false; + try + { + initialize_for_test( + "Foo { " + " value {\xA0\xA0 " + " name: \"valid name\"" + " \xA0 }" + "} ", false); + } + catch(const bts::config::ConfigurationException) + { + parse_error = true; + } + REQUIRE(parse_error); + } + + SECTION("macro_bool_test") + { + initialize_for_test( + "macro" + "{" + " name: \"bool\"" + " expression: \"'c' in ['a', 'b', 'c', 'd']\"" + "}" + "" + "Foo" + "{" + " rule: \"$bool\"" + " value {" + " name: 'good'" + " }" + "}" + "" + "Foo" + "{" + " value {" + " name: 'bad'" + " }" + "}" + ); + + REQUIRE(config::get().name() == "good"); + + } + + SECTION("macro_list_test") + { + initialize_for_test( + "macro" + "{" + " name: \"list\"" + " expression: \"['a', 'b', 'c', 'd']\"" + "}" + "" + "Foo" + "{" + " rule: \"'a' in $list\"" + " value {" + " name: 'good'" + " }" + "}" + "" + "Foo" + "{" + " value {" + " name: 'bad'" + " }" + "}" + ); + + REQUIRE(config::get().name() == "good"); + } + + SECTION("fallback_test") { + //HTTPServer main; + HTTPServer fallback; + //std::string main_url = "http://localhost:" + boost::lexical_cast(main.port()) + "/"; + std::string main_url = "http://does_not_exist:12345/"; + //main.stop(); + std::string fallback_url = "http://127.0.0.1:" + boost::lexical_cast(fallback.port()) + "/"; + std::string response = + "HTTP/1.0 200 OK\r\n" + "Content-Type: text/plain\r\n\r\n"; + + std::string content = + "FooGroup {\n" + " value {\n" + " group: \"http_group\"\n" + " foo {\n" + " name: \"http\"\n" + " id: 80\n" + "}}}\n"; + + fallback.add_response("/edge.conf", response, content); + + fallback.start(); + initialize_for_test( + "include: '" + main_url + "'\n" + "fallback: '" + fallback_url + "'\n",false + ); + fallback.stop(); + + REQUIRE(get().foo().name() == "http"); + REQUIRE(get().foo().id() == 80); + + } + + SECTION("macro_definition_order_test") { + // The $is_crusible test + // o2_cme_enabled depends on Foo which has a rule which depends on is_crucible. + // Eager evaluation caused a bug which would cause loading o2_cme_enabled to set is_crucible false. Case 3993 + initialize_for_test( + "Foo {" + " rule: '$is_crucible'" + " value { name: 'cme_1u7_cert' }" + "}" + "" + "Foo {" + " value { name: '' }" + "}" + "" + "macro {" + " name: 'o2_cme_enabled'" + " expression: \"Foo.name != ''\" " + "}" + "" + "macro {" + " name: 'is_crucible'" + " expression: 'true'" + "}" + ); + + REQUIRE(get().name() == "cme_1u7_cert"); + } + + SECTION("repeated_value_order_preserved") { + initialize_for_test( + "Foo {" + " value { name: 'edge-eharding' id: 80 }" + "}" + "StringTable {" + " value {" + " entries: 'http://'" + " entries: @Foo.name" + " entries: ':'" + " entries: @Foo.id" + " entries: '/risk'" + " }" + "}"); + std::string result; + for (const auto& part : get().entries()) { + result += part; + } + REQUIRE(result == "http://edge-eharding:80/risk"); + } + + SECTION("string_concat_test") { + initialize_for_test( + "Ports {\n" + " value { \n" + " config_port: 80 \n" + " }\n" + "}\n" + "RuleRef { \n" + " value { \n" + " a_string: \"localhost\"\n" + " }\n" + "}\n" + "macro{\n" + " name: 'list'\n" + " expression: \"['http://', RuleRef.a_string, ':', Ports.config_port, '/risk']\"\n" + "}" + "StringTable {\n" + "value {\n" + " entries: 'http://'\n" + " entries: @RuleRef.a_string\n" + " entries: ':'\n" + " entries: @Ports.config_port\n" + " entries: '/risk'\n" + "}}\n" + "Foo { \n" + " value {\n" + " name: $list\n" + " url: @StringTable.entries \n" + " }\n" + "}" + ); + + auto foo = config::get(); + auto expected = "http://localhost:80/risk"; + REQUIRE(foo.name() == expected); + REQUIRE(foo.url() == expected); + } + + reset(); +} + +} + diff --git a/src/lib/config/text_format/strutil.h b/src/lib/config/text_format/strutil.h new file mode 100644 index 000000000..df28c94da --- /dev/null +++ b/src/lib/config/text_format/strutil.h @@ -0,0 +1,878 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// from google3/strings/strutil.h + +#ifndef GOOGLE_PROTOBUF_STUBS_STRUTIL_H__ +#define GOOGLE_PROTOBUF_STUBS_STRUTIL_H__ + +#include +#include +#include +#include + +namespace google { +namespace protobuf { + +#ifdef _MSC_VER +#define strtoll _strtoi64 +#define strtoull _strtoui64 +#elif defined(__DECCXX) && defined(__osf__) +// HP C++ on Tru64 does not have strtoll, but strtol is already 64-bit. +#define strtoll strtol +#define strtoull strtoul +#endif + +// ---------------------------------------------------------------------- +// ascii_isalnum() +// Check if an ASCII character is alphanumeric. We can't use ctype's +// isalnum() because it is affected by locale. This function is applied +// to identifiers in the protocol buffer language, not to natural-language +// strings, so locale should not be taken into account. +// ascii_isdigit() +// Like above, but only accepts digits. +// ascii_isspace() +// Check if the character is a space character. +// ---------------------------------------------------------------------- + +inline bool ascii_isalnum(char c) { + return ('a' <= c && c <= 'z') || + ('A' <= c && c <= 'Z') || + ('0' <= c && c <= '9'); +} + +inline bool ascii_isdigit(char c) { + return ('0' <= c && c <= '9'); +} + +inline bool ascii_isspace(char c) { + return c == ' ' || c == '\t' || c == '\n' || c == '\v' || c == '\f' || + c == '\r'; +} + +inline bool ascii_isupper(char c) { + return c >= 'A' && c <= 'Z'; +} + +inline bool ascii_islower(char c) { + return c >= 'a' && c <= 'z'; +} + +inline char ascii_toupper(char c) { + return ascii_islower(c) ? c - ('a' - 'A') : c; +} + +inline char ascii_tolower(char c) { + return ascii_isupper(c) ? c + ('a' - 'A') : c; +} + +inline int hex_digit_to_int(char c) { + /* Assume ASCII. */ + int x = static_cast(c); + if (x > '9') { + x += 9; + } + return x & 0xf; +} + +// ---------------------------------------------------------------------- +// HasPrefixString() +// Check if a string begins with a given prefix. +// StripPrefixString() +// Given a string and a putative prefix, returns the string minus the +// prefix string if the prefix matches, otherwise the original +// string. +// ---------------------------------------------------------------------- +inline bool HasPrefixString(const string& str, + const string& prefix) { + return str.size() >= prefix.size() && + str.compare(0, prefix.size(), prefix) == 0; +} + +inline string StripPrefixString(const string& str, const string& prefix) { + if (HasPrefixString(str, prefix)) { + return str.substr(prefix.size()); + } else { + return str; + } +} + +// ---------------------------------------------------------------------- +// HasSuffixString() +// Return true if str ends in suffix. +// StripSuffixString() +// Given a string and a putative suffix, returns the string minus the +// suffix string if the suffix matches, otherwise the original +// string. +// ---------------------------------------------------------------------- +inline bool HasSuffixString(const string& str, + const string& suffix) { + return str.size() >= suffix.size() && + str.compare(str.size() - suffix.size(), suffix.size(), suffix) == 0; +} + +inline string StripSuffixString(const string& str, const string& suffix) { + if (HasSuffixString(str, suffix)) { + return str.substr(0, str.size() - suffix.size()); + } else { + return str; + } +} + +// ---------------------------------------------------------------------- +// ReplaceCharacters +// Replaces any occurrence of the character 'remove' (or the characters +// in 'remove') with the character 'replacewith'. +// Good for keeping html characters or protocol characters (\t) out +// of places where they might cause a problem. +// StripWhitespace +// Removes whitespaces from both ends of the given string. +// ---------------------------------------------------------------------- +LIBPROTOBUF_EXPORT void ReplaceCharacters(string* s, const char* remove, + char replacewith); +LIBPROTOBUF_EXPORT void StripString(string* s, const char* remove, + char replacewith); + +LIBPROTOBUF_EXPORT void StripWhitespace(string* s); + + +// ---------------------------------------------------------------------- +// LowerString() +// UpperString() +// ToUpper() +// Convert the characters in "s" to lowercase or uppercase. ASCII-only: +// these functions intentionally ignore locale because they are applied to +// identifiers used in the Protocol Buffer language, not to natural-language +// strings. +// ---------------------------------------------------------------------- + +inline void LowerString(string * s) { + string::iterator end = s->end(); + for (string::iterator i = s->begin(); i != end; ++i) { + // tolower() changes based on locale. We don't want this! + if ('A' <= *i && *i <= 'Z') *i += 'a' - 'A'; + } +} + +inline void UpperString(string * s) { + string::iterator end = s->end(); + for (string::iterator i = s->begin(); i != end; ++i) { + // toupper() changes based on locale. We don't want this! + if ('a' <= *i && *i <= 'z') *i += 'A' - 'a'; + } +} + +inline string ToUpper(const string& s) { + string out = s; + UpperString(&out); + return out; +} + +// ---------------------------------------------------------------------- +// StringReplace() +// Give me a string and two patterns "old" and "new", and I replace +// the first instance of "old" in the string with "new", if it +// exists. RETURN a new string, regardless of whether the replacement +// happened or not. +// ---------------------------------------------------------------------- + +LIBPROTOBUF_EXPORT string StringReplace(const string& s, const string& oldsub, + const string& newsub, bool replace_all); + +// ---------------------------------------------------------------------- +// SplitStringUsing() +// Split a string using a character delimiter. Append the components +// to 'result'. If there are consecutive delimiters, this function skips +// over all of them. +// ---------------------------------------------------------------------- +LIBPROTOBUF_EXPORT void SplitStringUsing(const string& full, const char* delim, + vector* res); + +// Split a string using one or more byte delimiters, presented +// as a nul-terminated c string. Append the components to 'result'. +// If there are consecutive delimiters, this function will return +// corresponding empty strings. If you want to drop the empty +// strings, try SplitStringUsing(). +// +// If "full" is the empty string, yields an empty string as the only value. +// ---------------------------------------------------------------------- +LIBPROTOBUF_EXPORT void SplitStringAllowEmpty(const string& full, + const char* delim, + vector* result); + +// ---------------------------------------------------------------------- +// Split() +// Split a string using a character delimiter. +// ---------------------------------------------------------------------- +inline vector Split( + const string& full, const char* delim, bool skip_empty = true) { + vector result; + if (skip_empty) { + SplitStringUsing(full, delim, &result); + } else { + SplitStringAllowEmpty(full, delim, &result); + } + return result; +} + +// ---------------------------------------------------------------------- +// JoinStrings() +// These methods concatenate a vector of strings into a C++ string, using +// the C-string "delim" as a separator between components. There are two +// flavors of the function, one flavor returns the concatenated string, +// another takes a pointer to the target string. In the latter case the +// target string is cleared and overwritten. +// ---------------------------------------------------------------------- +LIBPROTOBUF_EXPORT void JoinStrings(const vector& components, + const char* delim, string* result); + +inline string JoinStrings(const vector& components, + const char* delim) { + string result; + JoinStrings(components, delim, &result); + return result; +} + +// ---------------------------------------------------------------------- +// UnescapeCEscapeSequences() +// Copies "source" to "dest", rewriting C-style escape sequences +// -- '\n', '\r', '\\', '\ooo', etc -- to their ASCII +// equivalents. "dest" must be sufficiently large to hold all +// the characters in the rewritten string (i.e. at least as large +// as strlen(source) + 1 should be safe, since the replacements +// are always shorter than the original escaped sequences). It's +// safe for source and dest to be the same. RETURNS the length +// of dest. +// +// It allows hex sequences \xhh, or generally \xhhhhh with an +// arbitrary number of hex digits, but all of them together must +// specify a value of a single byte (e.g. \x0045 is equivalent +// to \x45, and \x1234 is erroneous). +// +// It also allows escape sequences of the form \uhhhh (exactly four +// hex digits, upper or lower case) or \Uhhhhhhhh (exactly eight +// hex digits, upper or lower case) to specify a Unicode code +// point. The dest array will contain the UTF8-encoded version of +// that code-point (e.g., if source contains \u2019, then dest will +// contain the three bytes 0xE2, 0x80, and 0x99). +// +// Errors: In the first form of the call, errors are reported with +// LOG(ERROR). The same is true for the second form of the call if +// the pointer to the string vector is NULL; otherwise, error +// messages are stored in the vector. In either case, the effect on +// the dest array is not defined, but rest of the source will be +// processed. +// ---------------------------------------------------------------------- + +LIBPROTOBUF_EXPORT int UnescapeCEscapeSequences(const char* source, char* dest); +LIBPROTOBUF_EXPORT int UnescapeCEscapeSequences(const char* source, char* dest, + vector *errors); + +// ---------------------------------------------------------------------- +// UnescapeCEscapeString() +// This does the same thing as UnescapeCEscapeSequences, but creates +// a new string. The caller does not need to worry about allocating +// a dest buffer. This should be used for non performance critical +// tasks such as printing debug messages. It is safe for src and dest +// to be the same. +// +// The second call stores its errors in a supplied string vector. +// If the string vector pointer is NULL, it reports the errors with LOG(). +// +// In the first and second calls, the length of dest is returned. In the +// the third call, the new string is returned. +// ---------------------------------------------------------------------- + +LIBPROTOBUF_EXPORT int UnescapeCEscapeString(const string& src, string* dest); +LIBPROTOBUF_EXPORT int UnescapeCEscapeString(const string& src, string* dest, + vector *errors); +LIBPROTOBUF_EXPORT string UnescapeCEscapeString(const string& src); + +// ---------------------------------------------------------------------- +// CEscape() +// Escapes 'src' using C-style escape sequences and returns the resulting +// string. +// +// Escaped chars: \n, \r, \t, ", ', \, and !isprint(). +// ---------------------------------------------------------------------- +LIBPROTOBUF_EXPORT string CEscape(const string& src); + +// ---------------------------------------------------------------------- +// CEscapeAndAppend() +// Escapes 'src' using C-style escape sequences, and appends the escaped +// string to 'dest'. +// ---------------------------------------------------------------------- +LIBPROTOBUF_EXPORT void CEscapeAndAppend(StringPiece src, string* dest); + +namespace strings { +// Like CEscape() but does not escape bytes with the upper bit set. +LIBPROTOBUF_EXPORT string Utf8SafeCEscape(const string& src); + +// Like CEscape() but uses hex (\x) escapes instead of octals. +LIBPROTOBUF_EXPORT string CHexEscape(const string& src); +} // namespace strings + +// ---------------------------------------------------------------------- +// strto32() +// strtou32() +// strto64() +// strtou64() +// Architecture-neutral plug compatible replacements for strtol() and +// strtoul(). Long's have different lengths on ILP-32 and LP-64 +// platforms, so using these is safer, from the point of view of +// overflow behavior, than using the standard libc functions. +// ---------------------------------------------------------------------- +LIBPROTOBUF_EXPORT int32 strto32_adaptor(const char *nptr, char **endptr, + int base); +LIBPROTOBUF_EXPORT uint32 strtou32_adaptor(const char *nptr, char **endptr, + int base); + +inline int32 strto32(const char *nptr, char **endptr, int base) { + if (sizeof(int32) == sizeof(long)) + return strtol(nptr, endptr, base); + else + return strto32_adaptor(nptr, endptr, base); +} + +inline uint32 strtou32(const char *nptr, char **endptr, int base) { + if (sizeof(uint32) == sizeof(unsigned long)) + return strtoul(nptr, endptr, base); + else + return strtou32_adaptor(nptr, endptr, base); +} + +// For now, long long is 64-bit on all the platforms we care about, so these +// functions can simply pass the call to strto[u]ll. +inline int64 strto64(const char *nptr, char **endptr, int base) { + GOOGLE_COMPILE_ASSERT(sizeof(int64) == sizeof(long long), + sizeof_int64_is_not_sizeof_long_long); + return strtoll(nptr, endptr, base); +} + +inline uint64 strtou64(const char *nptr, char **endptr, int base) { + GOOGLE_COMPILE_ASSERT(sizeof(uint64) == sizeof(unsigned long long), + sizeof_uint64_is_not_sizeof_long_long); + return strtoull(nptr, endptr, base); +} + +// ---------------------------------------------------------------------- +// safe_strtob() +// safe_strto32() +// safe_strtou32() +// safe_strto64() +// safe_strtou64() +// safe_strtof() +// safe_strtod() +// ---------------------------------------------------------------------- +LIBPROTOBUF_EXPORT bool safe_strtob(StringPiece str, bool* value); + +LIBPROTOBUF_EXPORT bool safe_strto32(const string& str, int32* value); +LIBPROTOBUF_EXPORT bool safe_strtou32(const string& str, uint32* value); +inline bool safe_strto32(const char* str, int32* value) { + return safe_strto32(string(str), value); +} +inline bool safe_strto32(StringPiece str, int32* value) { + return safe_strto32(str.ToString(), value); +} +inline bool safe_strtou32(const char* str, uint32* value) { + return safe_strtou32(string(str), value); +} +inline bool safe_strtou32(StringPiece str, uint32* value) { + return safe_strtou32(str.ToString(), value); +} + +LIBPROTOBUF_EXPORT bool safe_strto64(const string& str, int64* value); +LIBPROTOBUF_EXPORT bool safe_strtou64(const string& str, uint64* value); +inline bool safe_strto64(const char* str, int64* value) { + return safe_strto64(string(str), value); +} +inline bool safe_strto64(StringPiece str, int64* value) { + return safe_strto64(str.ToString(), value); +} +inline bool safe_strtou64(const char* str, uint64* value) { + return safe_strtou64(string(str), value); +} +inline bool safe_strtou64(StringPiece str, uint64* value) { + return safe_strtou64(str.ToString(), value); +} + +LIBPROTOBUF_EXPORT bool safe_strtof(const char* str, float* value); +LIBPROTOBUF_EXPORT bool safe_strtod(const char* str, double* value); +inline bool safe_strtof(const string& str, float* value) { + return safe_strtof(str.c_str(), value); +} +inline bool safe_strtod(const string& str, double* value) { + return safe_strtod(str.c_str(), value); +} +inline bool safe_strtof(StringPiece str, float* value) { + return safe_strtof(str.ToString(), value); +} +inline bool safe_strtod(StringPiece str, double* value) { + return safe_strtod(str.ToString(), value); +} + +// ---------------------------------------------------------------------- +// FastIntToBuffer() +// FastHexToBuffer() +// FastHex64ToBuffer() +// FastHex32ToBuffer() +// FastTimeToBuffer() +// These are intended for speed. FastIntToBuffer() assumes the +// integer is non-negative. FastHexToBuffer() puts output in +// hex rather than decimal. FastTimeToBuffer() puts the output +// into RFC822 format. +// +// FastHex64ToBuffer() puts a 64-bit unsigned value in hex-format, +// padded to exactly 16 bytes (plus one byte for '\0') +// +// FastHex32ToBuffer() puts a 32-bit unsigned value in hex-format, +// padded to exactly 8 bytes (plus one byte for '\0') +// +// All functions take the output buffer as an arg. +// They all return a pointer to the beginning of the output, +// which may not be the beginning of the input buffer. +// ---------------------------------------------------------------------- + +// Suggested buffer size for FastToBuffer functions. Also works with +// DoubleToBuffer() and FloatToBuffer(). +static const int kFastToBufferSize = 32; + +LIBPROTOBUF_EXPORT char* FastInt32ToBuffer(int32 i, char* buffer); +LIBPROTOBUF_EXPORT char* FastInt64ToBuffer(int64 i, char* buffer); +char* FastUInt32ToBuffer(uint32 i, char* buffer); // inline below +char* FastUInt64ToBuffer(uint64 i, char* buffer); // inline below +LIBPROTOBUF_EXPORT char* FastHexToBuffer(int i, char* buffer); +LIBPROTOBUF_EXPORT char* FastHex64ToBuffer(uint64 i, char* buffer); +LIBPROTOBUF_EXPORT char* FastHex32ToBuffer(uint32 i, char* buffer); + +// at least 22 bytes long +inline char* FastIntToBuffer(int i, char* buffer) { + return (sizeof(i) == 4 ? + FastInt32ToBuffer(i, buffer) : FastInt64ToBuffer(i, buffer)); +} +inline char* FastUIntToBuffer(unsigned int i, char* buffer) { + return (sizeof(i) == 4 ? + FastUInt32ToBuffer(i, buffer) : FastUInt64ToBuffer(i, buffer)); +} +inline char* FastLongToBuffer(long i, char* buffer) { + return (sizeof(i) == 4 ? + FastInt32ToBuffer(i, buffer) : FastInt64ToBuffer(i, buffer)); +} +inline char* FastULongToBuffer(unsigned long i, char* buffer) { + return (sizeof(i) == 4 ? + FastUInt32ToBuffer(i, buffer) : FastUInt64ToBuffer(i, buffer)); +} + +// ---------------------------------------------------------------------- +// FastInt32ToBufferLeft() +// FastUInt32ToBufferLeft() +// FastInt64ToBufferLeft() +// FastUInt64ToBufferLeft() +// +// Like the Fast*ToBuffer() functions above, these are intended for speed. +// Unlike the Fast*ToBuffer() functions, however, these functions write +// their output to the beginning of the buffer (hence the name, as the +// output is left-aligned). The caller is responsible for ensuring that +// the buffer has enough space to hold the output. +// +// Returns a pointer to the end of the string (i.e. the null character +// terminating the string). +// ---------------------------------------------------------------------- + +LIBPROTOBUF_EXPORT char* FastInt32ToBufferLeft(int32 i, char* buffer); +LIBPROTOBUF_EXPORT char* FastUInt32ToBufferLeft(uint32 i, char* buffer); +LIBPROTOBUF_EXPORT char* FastInt64ToBufferLeft(int64 i, char* buffer); +LIBPROTOBUF_EXPORT char* FastUInt64ToBufferLeft(uint64 i, char* buffer); + +// Just define these in terms of the above. +inline char* FastUInt32ToBuffer(uint32 i, char* buffer) { + FastUInt32ToBufferLeft(i, buffer); + return buffer; +} +inline char* FastUInt64ToBuffer(uint64 i, char* buffer) { + FastUInt64ToBufferLeft(i, buffer); + return buffer; +} + +inline string SimpleBtoa(bool value) { + return value ? "true" : "false"; +} + +// ---------------------------------------------------------------------- +// SimpleItoa() +// Description: converts an integer to a string. +// +// Return value: string +// ---------------------------------------------------------------------- +LIBPROTOBUF_EXPORT string SimpleItoa(int i); +LIBPROTOBUF_EXPORT string SimpleItoa(unsigned int i); +LIBPROTOBUF_EXPORT string SimpleItoa(long i); +LIBPROTOBUF_EXPORT string SimpleItoa(unsigned long i); +LIBPROTOBUF_EXPORT string SimpleItoa(long long i); +LIBPROTOBUF_EXPORT string SimpleItoa(unsigned long long i); + +// ---------------------------------------------------------------------- +// SimpleDtoa() +// SimpleFtoa() +// DoubleToBuffer() +// FloatToBuffer() +// Description: converts a double or float to a string which, if +// passed to NoLocaleStrtod(), will produce the exact same original double +// (except in case of NaN; all NaNs are considered the same value). +// We try to keep the string short but it's not guaranteed to be as +// short as possible. +// +// DoubleToBuffer() and FloatToBuffer() write the text to the given +// buffer and return it. The buffer must be at least +// kDoubleToBufferSize bytes for doubles and kFloatToBufferSize +// bytes for floats. kFastToBufferSize is also guaranteed to be large +// enough to hold either. +// +// Return value: string +// ---------------------------------------------------------------------- +LIBPROTOBUF_EXPORT string SimpleDtoa(double value); +LIBPROTOBUF_EXPORT string SimpleFtoa(float value); + +LIBPROTOBUF_EXPORT char* DoubleToBuffer(double i, char* buffer); +LIBPROTOBUF_EXPORT char* FloatToBuffer(float i, char* buffer); + +// In practice, doubles should never need more than 24 bytes and floats +// should never need more than 14 (including null terminators), but we +// overestimate to be safe. +static const int kDoubleToBufferSize = 32; +static const int kFloatToBufferSize = 24; + +namespace strings { + +enum PadSpec { + NO_PAD = 1, + ZERO_PAD_2, + ZERO_PAD_3, + ZERO_PAD_4, + ZERO_PAD_5, + ZERO_PAD_6, + ZERO_PAD_7, + ZERO_PAD_8, + ZERO_PAD_9, + ZERO_PAD_10, + ZERO_PAD_11, + ZERO_PAD_12, + ZERO_PAD_13, + ZERO_PAD_14, + ZERO_PAD_15, + ZERO_PAD_16, +}; + +struct Hex { + uint64 value; + enum PadSpec spec; + template + explicit Hex(Int v, PadSpec s = NO_PAD) + : spec(s) { + // Prevent sign-extension by casting integers to + // their unsigned counterparts. +#ifdef LANG_CXX11 + static_assert( + sizeof(v) == 1 || sizeof(v) == 2 || sizeof(v) == 4 || sizeof(v) == 8, + "Unknown integer type"); +#endif + value = sizeof(v) == 1 ? static_cast(v) + : sizeof(v) == 2 ? static_cast(v) + : sizeof(v) == 4 ? static_cast(v) + : static_cast(v); + } +}; + +struct LIBPROTOBUF_EXPORT AlphaNum { + const char *piece_data_; // move these to string_ref eventually + size_t piece_size_; // move these to string_ref eventually + + char digits[kFastToBufferSize]; + + // No bool ctor -- bools convert to an integral type. + // A bool ctor would also convert incoming pointers (bletch). + + AlphaNum(int32 i32) + : piece_data_(digits), + piece_size_(FastInt32ToBufferLeft(i32, digits) - &digits[0]) {} + AlphaNum(uint32 u32) + : piece_data_(digits), + piece_size_(FastUInt32ToBufferLeft(u32, digits) - &digits[0]) {} + AlphaNum(int64 i64) + : piece_data_(digits), + piece_size_(FastInt64ToBufferLeft(i64, digits) - &digits[0]) {} + AlphaNum(uint64 u64) + : piece_data_(digits), + piece_size_(FastUInt64ToBufferLeft(u64, digits) - &digits[0]) {} + + AlphaNum(float f) + : piece_data_(digits), piece_size_(strlen(FloatToBuffer(f, digits))) {} + AlphaNum(double f) + : piece_data_(digits), piece_size_(strlen(DoubleToBuffer(f, digits))) {} + + AlphaNum(Hex hex); + + AlphaNum(const char* c_str) + : piece_data_(c_str), piece_size_(strlen(c_str)) {} + // TODO: Add a string_ref constructor, eventually + // AlphaNum(const StringPiece &pc) : piece(pc) {} + + AlphaNum(const string& str) + : piece_data_(str.data()), piece_size_(str.size()) {} + + AlphaNum(StringPiece str) + : piece_data_(str.data()), piece_size_(str.size()) {} + + AlphaNum(internal::StringPiecePod str) + : piece_data_(str.data()), piece_size_(str.size()) {} + + size_t size() const { return piece_size_; } + const char *data() const { return piece_data_; } + + private: + // Use ":" not ':' + AlphaNum(char c); // NOLINT(runtime/explicit) + + // Disallow copy and assign. + AlphaNum(const AlphaNum&); + void operator=(const AlphaNum&); +}; + +} // namespace strings + +using strings::AlphaNum; + +// ---------------------------------------------------------------------- +// StrCat() +// This merges the given strings or numbers, with no delimiter. This +// is designed to be the fastest possible way to construct a string out +// of a mix of raw C strings, strings, bool values, +// and numeric values. +// +// Don't use this for user-visible strings. The localization process +// works poorly on strings built up out of fragments. +// +// For clarity and performance, don't use StrCat when appending to a +// string. In particular, avoid using any of these (anti-)patterns: +// str.append(StrCat(...) +// str += StrCat(...) +// str = StrCat(str, ...) +// where the last is the worse, with the potential to change a loop +// from a linear time operation with O(1) dynamic allocations into a +// quadratic time operation with O(n) dynamic allocations. StrAppend +// is a better choice than any of the above, subject to the restriction +// of StrAppend(&str, a, b, c, ...) that none of the a, b, c, ... may +// be a reference into str. +// ---------------------------------------------------------------------- + +LIBPROTOBUF_EXPORT string StrCat(const AlphaNum& a, const AlphaNum& b); +LIBPROTOBUF_EXPORT string StrCat(const AlphaNum& a, const AlphaNum& b, + const AlphaNum& c); +LIBPROTOBUF_EXPORT string StrCat(const AlphaNum& a, const AlphaNum& b, + const AlphaNum& c, const AlphaNum& d); +LIBPROTOBUF_EXPORT string StrCat(const AlphaNum& a, const AlphaNum& b, + const AlphaNum& c, const AlphaNum& d, + const AlphaNum& e); +LIBPROTOBUF_EXPORT string StrCat(const AlphaNum& a, const AlphaNum& b, + const AlphaNum& c, const AlphaNum& d, + const AlphaNum& e, const AlphaNum& f); +LIBPROTOBUF_EXPORT string StrCat(const AlphaNum& a, const AlphaNum& b, + const AlphaNum& c, const AlphaNum& d, + const AlphaNum& e, const AlphaNum& f, + const AlphaNum& g); +LIBPROTOBUF_EXPORT string StrCat(const AlphaNum& a, const AlphaNum& b, + const AlphaNum& c, const AlphaNum& d, + const AlphaNum& e, const AlphaNum& f, + const AlphaNum& g, const AlphaNum& h); +LIBPROTOBUF_EXPORT string StrCat(const AlphaNum& a, const AlphaNum& b, + const AlphaNum& c, const AlphaNum& d, + const AlphaNum& e, const AlphaNum& f, + const AlphaNum& g, const AlphaNum& h, + const AlphaNum& i); + +inline string StrCat(const AlphaNum& a) { return string(a.data(), a.size()); } + +// ---------------------------------------------------------------------- +// StrAppend() +// Same as above, but adds the output to the given string. +// WARNING: For speed, StrAppend does not try to check each of its input +// arguments to be sure that they are not a subset of the string being +// appended to. That is, while this will work: +// +// string s = "foo"; +// s += s; +// +// This will not (necessarily) work: +// +// string s = "foo"; +// StrAppend(&s, s); +// +// Note: while StrCat supports appending up to 9 arguments, StrAppend +// is currently limited to 4. That's rarely an issue except when +// automatically transforming StrCat to StrAppend, and can easily be +// worked around as consecutive calls to StrAppend are quite efficient. +// ---------------------------------------------------------------------- + +LIBPROTOBUF_EXPORT void StrAppend(string* dest, const AlphaNum& a); +LIBPROTOBUF_EXPORT void StrAppend(string* dest, const AlphaNum& a, + const AlphaNum& b); +LIBPROTOBUF_EXPORT void StrAppend(string* dest, const AlphaNum& a, + const AlphaNum& b, const AlphaNum& c); +LIBPROTOBUF_EXPORT void StrAppend(string* dest, const AlphaNum& a, + const AlphaNum& b, const AlphaNum& c, + const AlphaNum& d); + +// ---------------------------------------------------------------------- +// Join() +// These methods concatenate a range of components into a C++ string, using +// the C-string "delim" as a separator between components. +// ---------------------------------------------------------------------- +template +void Join(Iterator start, Iterator end, + const char* delim, string* result) { + for (Iterator it = start; it != end; ++it) { + if (it != start) { + result->append(delim); + } + StrAppend(result, *it); + } +} + +template +string Join(const Range& components, + const char* delim) { + string result; + Join(components.begin(), components.end(), delim, &result); + return result; +} + +// ---------------------------------------------------------------------- +// ToHex() +// Return a lower-case hex string representation of the given integer. +// ---------------------------------------------------------------------- +LIBPROTOBUF_EXPORT string ToHex(uint64 num); + +// ---------------------------------------------------------------------- +// GlobalReplaceSubstring() +// Replaces all instances of a substring in a string. Does nothing +// if 'substring' is empty. Returns the number of replacements. +// +// NOTE: The string pieces must not overlap s. +// ---------------------------------------------------------------------- +LIBPROTOBUF_EXPORT int GlobalReplaceSubstring(const string& substring, + const string& replacement, + string* s); + +// ---------------------------------------------------------------------- +// Base64Unescape() +// Converts "src" which is encoded in Base64 to its binary equivalent and +// writes it to "dest". If src contains invalid characters, dest is cleared +// and the function returns false. Returns true on success. +// ---------------------------------------------------------------------- +LIBPROTOBUF_EXPORT bool Base64Unescape(StringPiece src, string* dest); + +// ---------------------------------------------------------------------- +// WebSafeBase64Unescape() +// This is a variation of Base64Unescape which uses '-' instead of '+', and +// '_' instead of '/'. src is not null terminated, instead specify len. I +// recommend that slen +#include +#include +#include +#include +#include + +#include "text_format.h" + +#include +#include +#include +#include +#include +#include +#include +#include "strutil.h" + +using namespace std; +using namespace google::protobuf; + +namespace bts { +namespace google { +namespace protobuf { + + +// =========================================================================== +// Internal class for parsing an ASCII representation of a Protocol Message. +// This class makes use of the Protocol Message compiler's tokenizer found +// in //google/protobuf/io/tokenizer.h. Note that class's Parse +// method is *not* thread-safe and should only be used in a single thread at +// a time. + +// Makes code slightly more readable. The meaning of "DO(foo)" is +// "Execute foo and fail if it fails.", where failure is indicated by +// returning false. Borrowed from parser.cc (Thanks Kenton!). +#define DO(STATEMENT) if (STATEMENT) {} else return false + +class TextFormat::Parser::ParserImpl { + public: + + // Determines if repeated values for a non-repeated field are + // permitted, e.g., the string "foo: 1 foo: 2" for a + // required/optional field named "foo". + enum SingularOverwritePolicy { + ALLOW_SINGULAR_OVERWRITES = 0, // the last value is retained + FORBID_SINGULAR_OVERWRITES = 1, // an error is issued + }; + + ParserImpl(const Descriptor* root_message_type, + io::ZeroCopyInputStream* input_stream, + io::ErrorCollector* error_collector, + IDeferValueLookup* lookup, + TextFormat::Finder* finder, + SingularOverwritePolicy singular_overwrite_policy) + : error_collector_(error_collector), + lookup_(lookup), + finder_(finder), + tokenizer_error_collector_(this), + tokenizer_(input_stream, &tokenizer_error_collector_), + root_message_type_(root_message_type), + singular_overwrite_policy_(singular_overwrite_policy), + had_errors_(false), location_(), current_message_(NULL) { + // For backwards-compatibility with proto1, we need to allow the 'f' suffix + // for floats. + tokenizer_.set_allow_f_after_float(true); + + // '#' starts a comment. + tokenizer_.set_comment_style(io::Tokenizer::SH_COMMENT_STYLE); + + // Consume the starting token. + tokenizer_.Next(); + } + ~ParserImpl() { } + + // Parses the ASCII representation specified in input and saves the + // information into the output pointer (a Message). Returns + // false if an error occurs (an error will also be logged to + // GOOGLE_LOG(ERROR)). + bool Parse(Message* output) { + // Consume fields until we cannot do so anymore. + while(true) { + if (LookingAtType(io::Tokenizer::TYPE_END)) { + return !had_errors_; + } + + DO(ConsumeField(output)); + } + } + + bool ParseField(const FieldDescriptor* field, Message* output) { + bool suc; + if (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { + suc = ConsumeFieldMessage(output, output->GetReflection(), field); + } else { + suc = ConsumeFieldValue(output, output->GetReflection(), field); + } + return suc && LookingAtType(io::Tokenizer::TYPE_END); + } + + void ReportError(int line, int col, const string& message) { + had_errors_ = true; + if (error_collector_ == NULL) { + if (line >= 0) { + GOOGLE_LOG(ERROR) << "Error parsing text-format " + << root_message_type_->full_name() + << ": " << (line + 1) << ":" + << (col + 1) << ": " << message; + } else { + GOOGLE_LOG(ERROR) << "Error parsing text-format " + << root_message_type_->full_name() + << ": " << message; + } + } else { + error_collector_->AddError(line, col, message); + } + } + + void ReportWarning(int line, int col, const string& message) { + if (error_collector_ == NULL) { + if (line >= 0) { + GOOGLE_LOG(WARNING) << "Warning parsing text-format " + << root_message_type_->full_name() + << ": " << (line + 1) << ":" + << (col + 1) << ": " << message; + } else { + GOOGLE_LOG(WARNING) << "Warning parsing text-format " + << root_message_type_->full_name() + << ": " << message; + } + } else { + error_collector_->AddWarning(line, col, message); + } + } + + private: + GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ParserImpl); + + // Reports an error with the given message with information indicating + // the position (as derived from the current token). + void ReportError(const string& message) { + ReportError(tokenizer_.current().line, tokenizer_.current().column, + message); + } + + // Reports a warning with the given message with information indicating + // the position (as derived from the current token). + void ReportWarning(const string& message) { + ReportWarning(tokenizer_.current().line, tokenizer_.current().column, + message); + } + + // Consumes the specified message with the given starting delimeter. + // This method checks to see that the end delimeter at the conclusion of + // the consumption matches the starting delimeter passed in here. + bool ConsumeMessage(Message* message, const string delimeter) { + while (!LookingAt(">") && !LookingAt("}")) { + DO(ConsumeField(message)); + } + + // Confirm that we have a valid ending delimeter. + DO(Consume(delimeter)); + + return true; + } + + // + // BTS: Patched 1.14.2013 to allow skipping of unknown fields in text format + // + void ConsumeUnknownMessage() { + int stack = 0; + std::string token = tokenizer_.current().text; + if (token == ":") { + tokenizer_.Next(); + // Consume A single value + if (LookingAtType(io::Tokenizer::TYPE_STRING)) { + ConsumeString(&token); + } else { + tokenizer_.Next(); + } + } else { + // Consume a message type {...} + do { + if (LookingAt("{")) stack++; + if (LookingAt("}")) stack--; + tokenizer_.Next(); + } while(stack > 0); + } + } + + + // Consumes the current field (as returned by the tokenizer) on the + // passed in message. + bool ConsumeField(Message* message) { + const Reflection* reflection = message->GetReflection(); + const Descriptor* descriptor = message->GetDescriptor(); + + string field_name; + + const FieldDescriptor* field = NULL; + + if (TryConsume("[")) { + // Extension. + DO(ConsumeIdentifier(&field_name)); + while (TryConsume(".")) { + string part; + DO(ConsumeIdentifier(&part)); + field_name += "."; + field_name += part; + } + DO(Consume("]")); + + field = (finder_ != NULL + ? finder_->FindExtension(message, field_name) + : reflection->FindKnownExtensionByName(field_name)); + + if (field == NULL) { + ReportError("Extension \"" + field_name + "\" is not defined or " + "is not an extension of \"" + + descriptor->full_name() + "\"."); + return false; + } + } else { + DO(ConsumeIdentifier(&field_name)); + + field = descriptor->FindFieldByName(field_name); + // Group names are expected to be capitalized as they appear in the + // .proto file, which actually matches their type names, not their field + // names. + if (field == NULL) { + string lower_field_name = field_name; + LowerString(&lower_field_name); + field = descriptor->FindFieldByName(lower_field_name); + // If the case-insensitive match worked but the field is NOT a group, + if (field != NULL && field->type() != FieldDescriptor::TYPE_GROUP) { + field = NULL; + } + } + // Again, special-case group names as described above. + if (field != NULL && field->type() == FieldDescriptor::TYPE_GROUP + && field->message_type()->name() != field_name) { + field = NULL; + } + + if (field == NULL) { + ReportWarning("Message type \"" + descriptor->full_name() + + "\" has no field named \"" + field_name + "\"."); + ConsumeUnknownMessage(); + return true; + } + } + + // Fail if the field is not repeated and it has already been specified. + if ((singular_overwrite_policy_ == FORBID_SINGULAR_OVERWRITES) && + !field->is_repeated() && reflection->HasField(*message, field)) { + ReportError("Non-repeated field \"" + field_name + + "\" is specified multiple times."); + return false; + } + + // Perform special handling for embedded message types. + if (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { + // ':' is optional here. + TryConsume(":"); + DO(ConsumeFieldMessage(message, reflection, field)); + } else { + DO(Consume(":")); + if (field->is_repeated() && TryConsume("[")) { + // Short repeated format, e.g. "foo: [1, 2, 3]" + while (true) { + DO(ConsumeFieldValue(message, reflection, field)); + if (TryConsume("]")) { + break; + } + DO(Consume(",")); + } + } else { + DO(ConsumeFieldValue(message, reflection, field)); + } + } + + // For historical reasons, fields may optionally be separated by commas or + // semicolons. + TryConsume(";") || TryConsume(","); + + if (field->options().deprecated()) { + ReportWarning("text format contains deprecated field \"" + + field_name + "\""); + } + + return true; + } + + string current_location() { + string res; + vector::iterator iter = location_.begin(); + for (; iter != location_.end(); ++iter) { + if (iter - location_.begin() == 1) { + GOOGLE_CHECK(**iter == "value"); + continue; // skip .value + } + if (res.length() > 0) res.append("."); + + res.append(**iter); + } + return res; + } + + // Handle BTS deferred values + bool TryConsumeDeferredValue(Message* message, + const Reflection* reflection, + const FieldDescriptor* field) { + if (lookup_) { + string token; + int line = tokenizer_.current().line + 1; + int col = tokenizer_.current().column; + int size = field->is_repeated() ? reflection->FieldSize(*message, field) : 1; + if (TryConsume("@")) { + ConsumeDottedIdentifier(&token); + lookup_->defer_value(message, reflection, field, size, line, col, token, current_location(), current_message_); + return true; + } + else if (TryConsume("$")) { + ConsumeDottedIdentifier(&token); + lookup_->defer_value(message, reflection, field, size, line, col, '$' + token, current_location(), current_message_); + return true; + } + } + return false; + } + + bool ConsumeFieldMessage(Message* message, + const Reflection* reflection, + const FieldDescriptor* field) { + location_.push_back(&field->name()); + if (TryConsumeDeferredValue(message, reflection, field)) { + location_.pop_back(); + return true; + } + if (location_.size() == 3) { + current_message_ = message; + } + + string delimeter; + if (TryConsume("<")) { + delimeter = ">"; + } else { + DO(Consume("{")); + delimeter = "}"; + } + + if (field->is_repeated()) { + DO(ConsumeMessage(reflection->AddMessage(message, field), delimeter)); + } else { + DO(ConsumeMessage(reflection->MutableMessage(message, field), + delimeter)); + } + + location_.pop_back(); + //if (set_current) { current_message_ = nullptr; } + return true; + } + + bool ConsumeFieldValue(Message* message, + const Reflection* reflection, + const FieldDescriptor* field) { + + location_.push_back(&field->name()); + if (TryConsumeDeferredValue(message, reflection, field)) { + location_.pop_back(); + return true; + } + +// Define an easy to use macro for setting fields. This macro checks +// to see if the field is repeated (in which case we need to use the Add +// methods or not (in which case we need to use the Set methods). +#define SET_FIELD(CPPTYPE, VALUE) \ + if (field->is_repeated()) { \ + reflection->Add##CPPTYPE(message, field, VALUE); \ + } else { \ + reflection->Set##CPPTYPE(message, field, VALUE); \ + } \ + + switch(field->cpp_type()) { + case FieldDescriptor::CPPTYPE_INT32: { + int64 value; + DO(ConsumeSignedInteger(&value, kint32max)); + SET_FIELD(Int32, static_cast(value)); + break; + } + + case FieldDescriptor::CPPTYPE_UINT32: { + uint64 value; + DO(ConsumeUnsignedInteger(&value, kuint32max)); + SET_FIELD(UInt32, static_cast(value)); + break; + } + + case FieldDescriptor::CPPTYPE_INT64: { + int64 value; + DO(ConsumeSignedInteger(&value, kint64max)); + SET_FIELD(Int64, value); + break; + } + + case FieldDescriptor::CPPTYPE_UINT64: { + uint64 value; + DO(ConsumeUnsignedInteger(&value, kuint64max)); + SET_FIELD(UInt64, value); + break; + } + + case FieldDescriptor::CPPTYPE_FLOAT: { + double value; + DO(ConsumeDouble(&value)); + SET_FIELD(Float, static_cast(value)); + break; + } + + case FieldDescriptor::CPPTYPE_DOUBLE: { + double value; + DO(ConsumeDouble(&value)); + SET_FIELD(Double, value); + break; + } + + case FieldDescriptor::CPPTYPE_STRING: { + string value; + DO(ConsumeString(&value)); + SET_FIELD(String, value); + break; + } + + case FieldDescriptor::CPPTYPE_BOOL: { + if (LookingAtType(io::Tokenizer::TYPE_INTEGER)) { + uint64 value; + DO(ConsumeUnsignedInteger(&value, 1)); + SET_FIELD(Bool, value); + } else { + string value; + DO(ConsumeIdentifier(&value)); + if (value == "true" || value == "t") { + SET_FIELD(Bool, true); + } else if (value == "false" || value == "f") { + SET_FIELD(Bool, false); + } else { + ReportError("Invalid value for boolean field \"" + field->name() + + "\". Value: \"" + value + "\"."); + return false; + } + } + break; + } + + case FieldDescriptor::CPPTYPE_ENUM: { + string value; + const EnumDescriptor* enum_type = field->enum_type(); + const EnumValueDescriptor* enum_value = NULL; + + if (LookingAtType(io::Tokenizer::TYPE_IDENTIFIER)) { + DO(ConsumeIdentifier(&value)); + // Find the enumeration value. + enum_value = enum_type->FindValueByName(value); + + } else if (LookingAt("-") || + LookingAtType(io::Tokenizer::TYPE_INTEGER)) { + int64 int_value; + DO(ConsumeSignedInteger(&int_value, kint32max)); + value = SimpleItoa(int_value); // for error reporting + enum_value = enum_type->FindValueByNumber(int_value); + } else { + ReportError("Expected integer or identifier."); + return false; + } + + if (enum_value == NULL) { + ReportError("Unknown enumeration value of \"" + value + "\" for " + "field \"" + field->name() + "\"."); + return false; + } + + SET_FIELD(Enum, enum_value); + break; + } + + case FieldDescriptor::CPPTYPE_MESSAGE: { + // We should never get here. Put here instead of a default + // so that if new types are added, we get a nice compiler warning. + GOOGLE_LOG(FATAL) << "Reached an unintended state: CPPTYPE_MESSAGE"; + break; + } + } +#undef SET_FIELD + location_.pop_back(); + return true; + } + + // Returns true if the current token's text is equal to that specified. + bool LookingAt(const string& text) { + return tokenizer_.current().text == text; + } + + // Returns true if the current token's type is equal to that specified. + bool LookingAtType(io::Tokenizer::TokenType token_type) { + return tokenizer_.current().type == token_type; + } + + // Consumes an identifier and saves its value in the identifier parameter. + // Returns false if the token is not of type IDENTFIER. + bool ConsumeIdentifier(string* identifier) { + if (!LookingAtType(io::Tokenizer::TYPE_IDENTIFIER)) { + ReportError("Expected identifier."); + return false; + } + + *identifier = tokenizer_.current().text; + + tokenizer_.Next(); + return true; + } + + bool ConsumeDottedIdentifier(string* identifier) { + if (!LookingAtType(io::Tokenizer::TYPE_IDENTIFIER)) { + ReportError("Expected identifier."); + return false; + } + + *identifier = tokenizer_.current().text; + tokenizer_.Next(); + + while(true) { + if (TryConsume(".")) { + *identifier += "."; + *identifier += tokenizer_.current().text; + tokenizer_.Next(); + continue; + } + if (TryConsume("[")) { + *identifier += "["; + *identifier += tokenizer_.current().text; + tokenizer_.Next(); + if (Consume("]")) { + *identifier += "]"; + continue; + } + else { + ReportError("Expected ']'"); + } + } + break; + } + return true; + } + + // Consumes a string and saves its value in the text parameter. + // Returns false if the token is not of type STRING. + bool ConsumeString(string* text) { + if (!LookingAtType(io::Tokenizer::TYPE_STRING)) { + ReportError("Expected string."); + return false; + } + + text->clear(); + while (LookingAtType(io::Tokenizer::TYPE_STRING)) { + io::Tokenizer::ParseStringAppend(tokenizer_.current().text, text); + + tokenizer_.Next(); + } + + return true; + } + + // Consumes a uint64 and saves its value in the value parameter. + // Returns false if the token is not of type INTEGER. + bool ConsumeUnsignedInteger(uint64* value, uint64 max_value) { + if (!LookingAtType(io::Tokenizer::TYPE_INTEGER)) { + ReportError("Expected integer."); + return false; + } + + if (!io::Tokenizer::ParseInteger(tokenizer_.current().text, + max_value, value)) { + ReportError("Integer out of range."); + return false; + } + + tokenizer_.Next(); + return true; + } + + // Consumes an int64 and saves its value in the value parameter. + // Note that since the tokenizer does not support negative numbers, + // we actually may consume an additional token (for the minus sign) in this + // method. Returns false if the token is not an integer + // (signed or otherwise). + bool ConsumeSignedInteger(int64* value, uint64 max_value) { + bool negative = false; + + if (TryConsume("-")) { + negative = true; + // Two's complement always allows one more negative integer than + // positive. + ++max_value; + } + + uint64 unsigned_value; + + DO(ConsumeUnsignedInteger(&unsigned_value, max_value)); + + *value = static_cast(unsigned_value); + + if (negative) { + *value = -*value; + } + + return true; + } + + // Consumes a double and saves its value in the value parameter. + // Note that since the tokenizer does not support negative numbers, + // we actually may consume an additional token (for the minus sign) in this + // method. Returns false if the token is not a double + // (signed or otherwise). + bool ConsumeDouble(double* value) { + bool negative = false; + + if (TryConsume("-")) { + negative = true; + } + + // A double can actually be an integer, according to the tokenizer. + // Therefore, we must check both cases here. + if (LookingAtType(io::Tokenizer::TYPE_INTEGER)) { + // We have found an integer value for the double. + uint64 integer_value; + DO(ConsumeUnsignedInteger(&integer_value, kuint64max)); + + *value = static_cast(integer_value); + } else if (LookingAtType(io::Tokenizer::TYPE_FLOAT)) { + // We have found a float value for the double. + *value = io::Tokenizer::ParseFloat(tokenizer_.current().text); + + // Mark the current token as consumed. + tokenizer_.Next(); + } else if (LookingAtType(io::Tokenizer::TYPE_IDENTIFIER)) { + string text = tokenizer_.current().text; + LowerString(&text); + if (text == "inf" || text == "infinity") { + *value = std::numeric_limits::infinity(); + tokenizer_.Next(); + } else if (text == "nan") { + *value = std::numeric_limits::quiet_NaN(); + tokenizer_.Next(); + } else { + ReportError("Expected double."); + return false; + } + } else { + ReportError("Expected double."); + return false; + } + + if (negative) { + *value = -*value; + } + + return true; + } + + // Consumes a token and confirms that it matches that specified in the + // value parameter. Returns false if the token found does not match that + // which was specified. + bool Consume(const string& value) { + const string& current_value = tokenizer_.current().text; + + if (current_value != value) { + ReportError("Expected \"" + value + "\", found \"" + current_value + + "\"."); + return false; + } + + tokenizer_.Next(); + + return true; + } + + // Attempts to consume the supplied value. Returns false if a the + // token found does not match the value specified. + bool TryConsume(const string& value) { + if (tokenizer_.current().text == value) { + tokenizer_.Next(); + return true; + } else { + return false; + } + } + + // An internal instance of the Tokenizer's error collector, used to + // collect any base-level parse errors and feed them to the ParserImpl. + class ParserErrorCollector : public io::ErrorCollector { + public: + explicit ParserErrorCollector(TextFormat::Parser::ParserImpl* parser) : + parser_(parser) { } + + virtual ~ParserErrorCollector() { }; + + virtual void AddError(int line, int column, const string& message) { + parser_->ReportError(line, column, message); + } + + virtual void AddWarning(int line, int column, const string& message) { + parser_->ReportWarning(line, column, message); + } + + private: + GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ParserErrorCollector); + TextFormat::Parser::ParserImpl* parser_; + }; + + io::ErrorCollector* error_collector_; + IDeferValueLookup* lookup_; + TextFormat::Finder* finder_; + ParserErrorCollector tokenizer_error_collector_; + io::Tokenizer tokenizer_; + const Descriptor* root_message_type_; + SingularOverwritePolicy singular_overwrite_policy_; + bool had_errors_; + vector location_; + Message* current_message_; +}; + +#undef DO + +// =========================================================================== +// Internal class for writing text to the io::ZeroCopyOutputStream. Adapted +// from the Printer found in //google/protobuf/io/printer.h +class TextFormat::Printer::TextGenerator { + public: + explicit TextGenerator(io::ZeroCopyOutputStream* output, + int initial_indent_level) + : output_(output), + buffer_(NULL), + buffer_size_(0), + at_start_of_line_(true), + failed_(false), + indent_(""), + initial_indent_level_(initial_indent_level) { + indent_.resize(initial_indent_level_ * 2, ' '); + } + + ~TextGenerator() { + // Only BackUp() if we're sure we've successfully called Next() at least + // once. + if (buffer_size_ > 0) { + output_->BackUp(buffer_size_); + } + } + + // Indent text by two spaces. After calling Indent(), two spaces will be + // inserted at the beginning of each line of text. Indent() may be called + // multiple times to produce deeper indents. + void Indent() { + indent_ += " "; + } + + // Reduces the current indent level by two spaces, or crashes if the indent + // level is zero. + void Outdent() { + if (indent_.empty() || + indent_.size() < (size_t) initial_indent_level_ * 2) { + GOOGLE_LOG(DFATAL) << " Outdent() without matching Indent()."; + return; + } + + indent_.resize(indent_.size() - 2); + } + + // Print text to the output stream. + void Print(const string& str) { + Print(str.data(), str.size()); + } + + // Print text to the output stream. + void Print(const char* text) { + Print(text, strlen(text)); + } + + // Print text to the output stream. + void Print(const char* text, int size) { + int pos = 0; // The number of bytes we've written so far. + + for (int i = 0; i < size; i++) { + if (text[i] == '\n') { + // Saw newline. If there is more text, we may need to insert an indent + // here. So, write what we have so far, including the '\n'. + Write(text + pos, i - pos + 1); + pos = i + 1; + + // Setting this true will cause the next Write() to insert an indent + // first. + at_start_of_line_ = true; + } + } + + // Write the rest. + Write(text + pos, size - pos); + } + + // True if any write to the underlying stream failed. (We don't just + // crash in this case because this is an I/O failure, not a programming + // error.) + bool failed() const { return failed_; } + + private: + GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(TextGenerator); + + void Write(const char* data, int size) { + if (failed_) return; + if (size == 0) return; + + if (at_start_of_line_) { + // Insert an indent. + at_start_of_line_ = false; + Write(indent_.data(), indent_.size()); + if (failed_) return; + } + + while (size > buffer_size_) { + // Data exceeds space in the buffer. Copy what we can and request a + // new buffer. + memcpy(buffer_, data, buffer_size_); + data += buffer_size_; + size -= buffer_size_; + void* void_buffer; + failed_ = !output_->Next(&void_buffer, &buffer_size_); + if (failed_) return; + buffer_ = reinterpret_cast(void_buffer); + } + + // Buffer is big enough to receive the data; copy it. + memcpy(buffer_, data, size); + buffer_ += size; + buffer_size_ -= size; + } + + io::ZeroCopyOutputStream* const output_; + char* buffer_; + int buffer_size_; + bool at_start_of_line_; + bool failed_; + + string indent_; + int initial_indent_level_; +}; + +// =========================================================================== + +TextFormat::Finder::~Finder() { +} + +TextFormat::Parser::Parser() + : error_collector_(NULL), + lookup_(NULL), + finder_(NULL), + allow_partial_(false) { +} + +TextFormat::Parser::~Parser() {} + +bool TextFormat::Parser::Parse(io::ZeroCopyInputStream* input, + Message* output) { + //output->Clear(); + ParserImpl parser(output->GetDescriptor(), input, error_collector_, lookup_, + finder_, ParserImpl::FORBID_SINGULAR_OVERWRITES); + return MergeUsingImpl(input, output, &parser); +} + +bool TextFormat::Parser::ParseFromString(const string& input, + Message* output) { + io::ArrayInputStream input_stream(input.data(), input.size()); + return Parse(&input_stream, output); +} + +bool TextFormat::Parser::Merge(io::ZeroCopyInputStream* input, + Message* output) { + ParserImpl parser(output->GetDescriptor(), input, error_collector_, lookup_, + finder_, ParserImpl::ALLOW_SINGULAR_OVERWRITES); + return MergeUsingImpl(input, output, &parser); +} + +bool TextFormat::Parser::MergeFromString(const string& input, + Message* output) { + io::ArrayInputStream input_stream(input.data(), input.size()); + return Merge(&input_stream, output); +} + +bool TextFormat::Parser::MergeUsingImpl(io::ZeroCopyInputStream*, + Message* output, + ParserImpl* parser_impl) { + if (!parser_impl->Parse(output)) return false; + if (!allow_partial_ && !output->IsInitialized()) { + vector missing_fields; + output->FindInitializationErrors(&missing_fields); + parser_impl->ReportError(-1, 0, "Message missing required fields: " + + JoinStrings(missing_fields, ", ")); + return false; + } + return true; +} + +bool TextFormat::Parser::ParseFieldValueFromString( + const string& input, + const FieldDescriptor* field, + Message* output) { + io::ArrayInputStream input_stream(input.data(), input.size()); + ParserImpl parser(output->GetDescriptor(), &input_stream, error_collector_, lookup_, + finder_, ParserImpl::ALLOW_SINGULAR_OVERWRITES); + return parser.ParseField(field, output); +} + +/* static */ bool TextFormat::Parse(io::ZeroCopyInputStream* input, + Message* output) { + return Parser().Parse(input, output); +} + +/* static */ bool TextFormat::Merge(io::ZeroCopyInputStream* input, + Message* output) { + return Parser().Merge(input, output); +} + +/* static */ bool TextFormat::ParseFromString(const string& input, + Message* output) { + return Parser().ParseFromString(input, output); +} + +/* static */ bool TextFormat::MergeFromString(const string& input, + Message* output) { + return Parser().MergeFromString(input, output); +} + +// =========================================================================== + +TextFormat::Printer::Printer() + : initial_indent_level_(0), + single_line_mode_(false), + use_short_repeated_primitives_(false), + utf8_string_escaping_(false) {} + +TextFormat::Printer::~Printer() {} + +bool TextFormat::Printer::PrintToString(const Message& message, + string* output) const { + GOOGLE_DCHECK(output) << "output specified is NULL"; + + output->clear(); + io::StringOutputStream output_stream(output); + + bool result = Print(message, &output_stream); + + return result; +} + +bool TextFormat::Printer::PrintUnknownFieldsToString( + const UnknownFieldSet& unknown_fields, + string* output) const { + GOOGLE_DCHECK(output) << "output specified is NULL"; + + output->clear(); + io::StringOutputStream output_stream(output); + return PrintUnknownFields(unknown_fields, &output_stream); +} + +bool TextFormat::Printer::Print(const Message& message, + io::ZeroCopyOutputStream* output) const { + TextGenerator generator(output, initial_indent_level_); + + Print(message, generator); + + // Output false if the generator failed internally. + return !generator.failed(); +} + +bool TextFormat::Printer::PrintUnknownFields( + const UnknownFieldSet& unknown_fields, + io::ZeroCopyOutputStream* output) const { + TextGenerator generator(output, initial_indent_level_); + + PrintUnknownFields(unknown_fields, generator); + + // Output false if the generator failed internally. + return !generator.failed(); +} + +void TextFormat::Printer::Print(const Message& message, + TextGenerator& generator) const { + const Reflection* reflection = message.GetReflection(); + vector fields; + reflection->ListFields(message, &fields); + for (size_t i = 0; i < fields.size(); i++) { + PrintField(message, reflection, fields[i], generator); + } + PrintUnknownFields(reflection->GetUnknownFields(message), generator); +} + +void TextFormat::Printer::PrintFieldValueToString( + const Message& message, + const FieldDescriptor* field, + int index, + string* output) const { + + GOOGLE_DCHECK(output) << "output specified is NULL"; + + output->clear(); + io::StringOutputStream output_stream(output); + TextGenerator generator(&output_stream, initial_indent_level_); + + PrintFieldValue(message, message.GetReflection(), field, index, generator); +} + +void TextFormat::Printer::PrintField(const Message& message, + const Reflection* reflection, + const FieldDescriptor* field, + TextGenerator& generator) const { + if (use_short_repeated_primitives_ && + field->is_repeated() && + field->cpp_type() != FieldDescriptor::CPPTYPE_STRING && + field->cpp_type() != FieldDescriptor::CPPTYPE_MESSAGE) { + PrintShortRepeatedField(message, reflection, field, generator); + return; + } + + int count = 0; + + if (field->is_repeated()) { + count = reflection->FieldSize(message, field); + } else if (reflection->HasField(message, field)) { + count = 1; + } + + for (int j = 0; j < count; ++j) { + PrintFieldName(message, reflection, field, generator); + + if (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { + if (single_line_mode_) { + generator.Print(" { "); + } else { + generator.Print(" {\n"); + generator.Indent(); + } + } else { + generator.Print(": "); + } + + // Write the field value. + int field_index = j; + if (!field->is_repeated()) { + field_index = -1; + } + + PrintFieldValue(message, reflection, field, field_index, generator); + + if (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { + if (single_line_mode_) { + generator.Print("} "); + } else { + generator.Outdent(); + generator.Print("}\n"); + } + } else { + if (single_line_mode_) { + generator.Print(" "); + } else { + generator.Print("\n"); + } + } + } +} + +void TextFormat::Printer::PrintShortRepeatedField( + const Message& message, + const Reflection* reflection, + const FieldDescriptor* field, + TextGenerator& generator) const { + // Print primitive repeated field in short form. + PrintFieldName(message, reflection, field, generator); + + int size = reflection->FieldSize(message, field); + generator.Print(": ["); + for (int i = 0; i < size; i++) { + if (i > 0) generator.Print(", "); + PrintFieldValue(message, reflection, field, i, generator); + } + if (single_line_mode_) { + generator.Print("] "); + } else { + generator.Print("]\n"); + } +} + +void TextFormat::Printer::PrintFieldName(const Message&, + const Reflection*, + const FieldDescriptor* field, + TextGenerator& generator) const { + if (field->is_extension()) { + generator.Print("["); + // We special-case MessageSet elements for compatibility with proto1. + if (field->containing_type()->options().message_set_wire_format() + && field->type() == FieldDescriptor::TYPE_MESSAGE + && field->is_optional() + && field->extension_scope() == field->message_type()) { + generator.Print(field->message_type()->full_name()); + } else { + generator.Print(field->full_name()); + } + generator.Print("]"); + } else { + if (field->type() == FieldDescriptor::TYPE_GROUP) { + // Groups must be serialized with their original capitalization. + generator.Print(field->message_type()->name()); + } else { + generator.Print(field->name()); + } + } +} + +void TextFormat::Printer::PrintFieldValue( + const Message& message, + const Reflection* reflection, + const FieldDescriptor* field, + int index, + TextGenerator& generator) const { + GOOGLE_DCHECK(field->is_repeated() || (index == -1)) + << "Index must be -1 for non-repeated fields"; + + switch (field->cpp_type()) { +#define OUTPUT_FIELD(CPPTYPE, METHOD, TO_STRING) \ + case FieldDescriptor::CPPTYPE_##CPPTYPE: \ + generator.Print(TO_STRING(field->is_repeated() ? \ + reflection->GetRepeated##METHOD(message, field, index) : \ + reflection->Get##METHOD(message, field))); \ + break; \ + + OUTPUT_FIELD( INT32, Int32, SimpleItoa); + OUTPUT_FIELD( INT64, Int64, SimpleItoa); + OUTPUT_FIELD(UINT32, UInt32, SimpleItoa); + OUTPUT_FIELD(UINT64, UInt64, SimpleItoa); + OUTPUT_FIELD( FLOAT, Float, SimpleFtoa); + OUTPUT_FIELD(DOUBLE, Double, SimpleDtoa); +#undef OUTPUT_FIELD + + case FieldDescriptor::CPPTYPE_STRING: { + string scratch; + const string& value = field->is_repeated() ? + reflection->GetRepeatedStringReference( + message, field, index, &scratch) : + reflection->GetStringReference(message, field, &scratch); + + generator.Print("\""); + if (utf8_string_escaping_) { + generator.Print(strings::Utf8SafeCEscape(value)); + } else { + generator.Print(CEscape(value)); + } + generator.Print("\""); + + break; + } + + case FieldDescriptor::CPPTYPE_BOOL: + if (field->is_repeated()) { + generator.Print(reflection->GetRepeatedBool(message, field, index) + ? "true" : "false"); + } else { + generator.Print(reflection->GetBool(message, field) + ? "true" : "false"); + } + break; + + case FieldDescriptor::CPPTYPE_ENUM: + generator.Print(field->is_repeated() ? + reflection->GetRepeatedEnum(message, field, index)->name() : + reflection->GetEnum(message, field)->name()); + break; + + case FieldDescriptor::CPPTYPE_MESSAGE: + Print(field->is_repeated() ? + reflection->GetRepeatedMessage(message, field, index) : + reflection->GetMessage(message, field), + generator); + break; + } +} + +/* static */ bool TextFormat::Print(const Message& message, + io::ZeroCopyOutputStream* output) { + return Printer().Print(message, output); +} + +/* static */ bool TextFormat::PrintUnknownFields( + const UnknownFieldSet& unknown_fields, + io::ZeroCopyOutputStream* output) { + return Printer().PrintUnknownFields(unknown_fields, output); +} + +/* static */ bool TextFormat::PrintToString( + const Message& message, string* output) { + return Printer().PrintToString(message, output); +} + +/* static */ bool TextFormat::PrintUnknownFieldsToString( + const UnknownFieldSet& unknown_fields, string* output) { + return Printer().PrintUnknownFieldsToString(unknown_fields, output); +} + +/* static */ void TextFormat::PrintFieldValueToString( + const Message& message, + const FieldDescriptor* field, + int index, + string* output) { + return Printer().PrintFieldValueToString(message, field, index, output); +} + +/* static */ bool TextFormat::ParseFieldValueFromString( + const string& input, + const FieldDescriptor* field, + Message* message) { + return Parser().ParseFieldValueFromString(input, field, message); +} + +// Prints an integer as hex with a fixed number of digits dependent on the +// integer type. +template +static string PaddedHex(IntType value) { + string result; + result.reserve(sizeof(value) * 2); + for (int i = sizeof(value) * 2 - 1; i >= 0; i--) { + result.push_back(int_to_hex_digit(value >> (i*4) & 0x0F)); + } + return result; +} + +void TextFormat::Printer::PrintUnknownFields( + const UnknownFieldSet& unknown_fields, TextGenerator& generator) const { + for (int i = 0; i < unknown_fields.field_count(); i++) { + const UnknownField& field = unknown_fields.field(i); + string field_number = SimpleItoa(field.number()); + + switch (field.type()) { + case UnknownField::TYPE_VARINT: + generator.Print(field_number); + generator.Print(": "); + generator.Print(SimpleItoa(field.varint())); + if (single_line_mode_) { + generator.Print(" "); + } else { + generator.Print("\n"); + } + break; + case UnknownField::TYPE_FIXED32: { + generator.Print(field_number); + generator.Print(": 0x"); + char buffer[kFastToBufferSize]; + generator.Print(FastHex32ToBuffer(field.fixed32(), buffer)); + if (single_line_mode_) { + generator.Print(" "); + } else { + generator.Print("\n"); + } + break; + } + case UnknownField::TYPE_FIXED64: { + generator.Print(field_number); + generator.Print(": 0x"); + char buffer[kFastToBufferSize]; + generator.Print(FastHex64ToBuffer(field.fixed64(), buffer)); + if (single_line_mode_) { + generator.Print(" "); + } else { + generator.Print("\n"); + } + break; + } + case UnknownField::TYPE_LENGTH_DELIMITED: { + generator.Print(field_number); + const string& value = field.length_delimited(); + UnknownFieldSet embedded_unknown_fields; + if (!value.empty() && embedded_unknown_fields.ParseFromString(value)) { + // This field is parseable as a Message. + // So it is probably an embedded message. + if (single_line_mode_) { + generator.Print(" { "); + } else { + generator.Print(" {\n"); + generator.Indent(); + } + PrintUnknownFields(embedded_unknown_fields, generator); + if (single_line_mode_) { + generator.Print("} "); + } else { + generator.Outdent(); + generator.Print("}\n"); + } + } else { + // This field is not parseable as a Message. + // So it is probably just a plain string. + generator.Print(": \""); + generator.Print(CEscape(value)); + generator.Print("\""); + if (single_line_mode_) { + generator.Print(" "); + } else { + generator.Print("\n"); + } + } + break; + } + case UnknownField::TYPE_GROUP: + generator.Print(field_number); + if (single_line_mode_) { + generator.Print(" { "); + } else { + generator.Print(" {\n"); + generator.Indent(); + } + PrintUnknownFields(field.group(), generator); + if (single_line_mode_) { + generator.Print("} "); + } else { + generator.Outdent(); + generator.Print("}\n"); + } + break; + } + } +} + +} // namespace protobuf +} // namespace google +} + diff --git a/src/lib/config/text_format/text_format.h b/src/lib/config/text_format/text_format.h new file mode 100755 index 000000000..01ae25d6e --- /dev/null +++ b/src/lib/config/text_format/text_format.h @@ -0,0 +1,320 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// http://code.google.com/p/protobuf/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Author: jschorr@google.com (Joseph Schorr) +// Based on original Protocol Buffers design by +// Sanjay Ghemawat, Jeff Dean, and others. +// +// Utilities for printing and parsing protocol messages in a human-readable, +// text-based format. +// + +// Forked from LibProtobuf for BTS Config by Eric Harding on 5/29/2013 +// + + +#ifndef GOOGLE_PROTOBUF_TEXT_FORMAT_H__ +#define GOOGLE_PROTOBUF_TEXT_FORMAT_H__ +#pragma once + +#include +#include +#include + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Weffc++" +#endif + +namespace google { + namespace protobuf { + namespace io { + class ErrorCollector; // tokenizer.h + } + } +} + +namespace bts { +namespace google { +namespace protobuf { + +using namespace ::google::protobuf; + + +class IDeferValueLookup { +public: + virtual ~IDeferValueLookup() {}; + virtual void defer_value(Message*, const Reflection* reflection, const FieldDescriptor* field, int index, int line, int col, const std::string& value, const std::string& key_name, const Message* root) = 0; +}; + + +// This class implements protocol buffer text format. Printing and parsing +// protocol messages in text format is useful for debugging and human editing +// of messages. +// +// This class is really a namespace that contains only static methods. +class TextFormat { + public: + // Outputs a textual representation of the given message to the given + // output stream. + static bool Print(const Message& message, io::ZeroCopyOutputStream* output); + + // Print the fields in an UnknownFieldSet. They are printed by tag number + // only. Embedded messages are heuristically identified by attempting to + // parse them. + static bool PrintUnknownFields(const UnknownFieldSet& unknown_fields, + io::ZeroCopyOutputStream* output); + + // Like Print(), but outputs directly to a string. + static bool PrintToString(const Message& message, string* output); + + // Like PrintUnknownFields(), but outputs directly to a string. + static bool PrintUnknownFieldsToString(const UnknownFieldSet& unknown_fields, + string* output); + + // Outputs a textual representation of the value of the field supplied on + // the message supplied. For non-repeated fields, an index of -1 must + // be supplied. Note that this method will print the default value for a + // field if it is not set. + static void PrintFieldValueToString(const Message& message, + const FieldDescriptor* field, + int index, + string* output); + + // Class for those users which require more fine-grained control over how + // a protobuffer message is printed out. + class Printer { + public: + Printer(); + ~Printer(); + + // Like TextFormat::Print + bool Print(const Message& message, io::ZeroCopyOutputStream* output) const; + // Like TextFormat::PrintUnknownFields + bool PrintUnknownFields(const UnknownFieldSet& unknown_fields, + io::ZeroCopyOutputStream* output) const; + // Like TextFormat::PrintToString + bool PrintToString(const Message& message, string* output) const; + // Like TextFormat::PrintUnknownFieldsToString + bool PrintUnknownFieldsToString(const UnknownFieldSet& unknown_fields, + string* output) const; + // Like TextFormat::PrintFieldValueToString + void PrintFieldValueToString(const Message& message, + const FieldDescriptor* field, + int index, + string* output) const; + + // Adjust the initial indent level of all output. Each indent level is + // equal to two spaces. + void SetInitialIndentLevel(int indent_level) { + initial_indent_level_ = indent_level; + } + + // If printing in single line mode, then the entire message will be output + // on a single line with no line breaks. + void SetSingleLineMode(bool single_line_mode) { + single_line_mode_ = single_line_mode; + } + + // Set true to print repeated primitives in a format like: + // field_name: [1, 2, 3, 4] + // instead of printing each value on its own line. Short format applies + // only to primitive values -- i.e. everything except strings and + // sub-messages/groups. + void SetUseShortRepeatedPrimitives(bool use_short_repeated_primitives) { + use_short_repeated_primitives_ = use_short_repeated_primitives; + } + + // Set true to output UTF-8 instead of ASCII. The only difference + // is that bytes >= 0x80 in string fields will not be escaped, + // because they are assumed to be part of UTF-8 multi-byte + // sequences. + void SetUseUtf8StringEscaping(bool as_utf8) { + utf8_string_escaping_ = as_utf8; + } + + private: + // Forward declaration of an internal class used to print the text + // output to the OutputStream (see text_format.cc for implementation). + class TextGenerator; + + // Internal Print method, used for writing to the OutputStream via + // the TextGenerator class. + void Print(const Message& message, + TextGenerator& generator) const; + + // Print a single field. + void PrintField(const Message& message, + const Reflection* reflection, + const FieldDescriptor* field, + TextGenerator& generator) const; + + // Print a repeated primitive field in short form. + void PrintShortRepeatedField(const Message& message, + const Reflection* reflection, + const FieldDescriptor* field, + TextGenerator& generator) const; + + // Print the name of a field -- i.e. everything that comes before the + // ':' for a single name/value pair. + void PrintFieldName(const Message& message, + const Reflection* reflection, + const FieldDescriptor* field, + TextGenerator& generator) const; + + // Outputs a textual representation of the value of the field supplied on + // the message supplied or the default value if not set. + void PrintFieldValue(const Message& message, + const Reflection* reflection, + const FieldDescriptor* field, + int index, + TextGenerator& generator) const; + + // Print the fields in an UnknownFieldSet. They are printed by tag number + // only. Embedded messages are heuristically identified by attempting to + // parse them. + void PrintUnknownFields(const UnknownFieldSet& unknown_fields, + TextGenerator& generator) const; + + int initial_indent_level_; + + bool single_line_mode_; + + bool use_short_repeated_primitives_; + + bool utf8_string_escaping_; + }; + + // Parses a text-format protocol message from the given input stream to + // the given message object. This function parses the format written + // by Print(). + static bool Parse(io::ZeroCopyInputStream* input, Message* output); + // Like Parse(), but reads directly from a string. + static bool ParseFromString(const string& input, Message* output); + + // Like Parse(), but the data is merged into the given message, as if + // using Message::MergeFrom(). + static bool Merge(io::ZeroCopyInputStream* input, Message* output); + // Like Merge(), but reads directly from a string. + static bool MergeFromString(const string& input, Message* output); + + // Parse the given text as a single field value and store it into the + // given field of the given message. If the field is a repeated field, + // the new value will be added to the end + static bool ParseFieldValueFromString(const string& input, + const FieldDescriptor* field, + Message* message); + + // Interface that TextFormat::Parser can use to find extensions. + // This class may be extended in the future to find more information + // like fields, etc. + class Finder { + public: + virtual ~Finder(); + + // Try to find an extension of *message by fully-qualified field + // name. Returns NULL if no extension is known for this name or number. + virtual const FieldDescriptor* FindExtension( + Message* message, + const string& name) const = 0; + }; + + // For more control over parsing, use this class. + class Parser { + public: + Parser(); + ~Parser(); + + // Like TextFormat::Parse(). + bool Parse(io::ZeroCopyInputStream* input, Message* output); + // Like TextFormat::ParseFromString(). + bool ParseFromString(const string& input, Message* output); + // Like TextFormat::Merge(). + bool Merge(io::ZeroCopyInputStream* input, Message* output); + // Like TextFormat::MergeFromString(). + bool MergeFromString(const string& input, Message* output); + + void SetValueLookup(IDeferValueLookup* lookup) { + lookup_ = lookup; + } + + // Set where to report parse errors. If NULL (the default), errors will + // be printed to stderr. + void RecordErrorsTo(io::ErrorCollector* error_collector) { + error_collector_ = error_collector; + } + + // Set how parser finds extensions. If NULL (the default), the + // parser will use the standard Reflection object associated with + // the message being parsed. + void SetFinder(Finder* finder) { + finder_ = finder; + } + + // Normally parsing fails if, after parsing, output->IsInitialized() + // returns false. Call AllowPartialMessage(true) to skip this check. + void AllowPartialMessage(bool allow) { + allow_partial_ = allow; + } + + // Like TextFormat::ParseFieldValueFromString + bool ParseFieldValueFromString(const string& input, + const FieldDescriptor* field, + Message* output); + + private: + // Forward declaration of an internal class used to parse text + // representations (see text_format.cc for implementation). + class ParserImpl; + + // Like TextFormat::Merge(). The provided implementation is used + // to do the parsing. + bool MergeUsingImpl(io::ZeroCopyInputStream* input, + Message* output, + ParserImpl* parser_impl); + + io::ErrorCollector* error_collector_; + IDeferValueLookup* lookup_; + Finder* finder_; + bool allow_partial_; + }; + + private: + GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(TextFormat); +}; + +} // namespace protobuf +} // namespace google +} +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif + +#endif // GOOGLE_PROTOBUF_TEXT_FORMAT_H__ diff --git a/src/lib/expression/BooleanGrammar.cpp b/src/lib/expression/BooleanGrammar.cpp new file mode 100755 index 000000000..33441b972 --- /dev/null +++ b/src/lib/expression/BooleanGrammar.cpp @@ -0,0 +1,28 @@ +#include "BooleanGrammar.h" + +namespace bts { +namespace expression { + +ph::function const and_expr = binary_operator("and"); +ph::function const or_expr = binary_operator("or"); +ph::function const in_expr = binary_operator("in"); +ph::function const gt_expr = binary_operator(">"); +ph::function const gte_expr = binary_operator(">="); +ph::function const lt_expr = binary_operator("<"); +ph::function const lte_expr = binary_operator("<="); +ph::function const eq_expr = binary_operator("=="); +ph::function const neq_expr = binary_operator("!="); +ph::function const match_expr = binary_operator("~="); + +ph::function not_expr = unary_operator("!"); + +ph::function const property_expr; + +ph::function list_expr; + +// For the sake of build performance this is instantiated here +// and referenced by ExpressionEvaluator so that minor changes +// to ExpressionEvaluator don't take 30 seconds to build. +BooleanGrammar boolean_string_grammar; + +}}//namespace diff --git a/src/lib/expression/BooleanGrammar.h b/src/lib/expression/BooleanGrammar.h new file mode 100755 index 000000000..81fb30a5d --- /dev/null +++ b/src/lib/expression/BooleanGrammar.h @@ -0,0 +1,276 @@ +#ifndef BOOLEANGRAMMAR_H_ +#define BOOLEANGRAMMAR_H_ + +#include "lib/utils/bts_no_copy.h" + +#include +#include +#include +#include +#include + +#include + +/* + * Using and modifying this grammar: + * + * Boost::spirit is a fantastically complex bit of technology which could take years to master. + * I'll try to describe the conventsions followed in this grammar which form a more comprehensible + * sub-set. + * + * spirit::utree is a generic syntax type which can act as the result (aka "property") of an expression or grammar. + * boost::spirit can act on any arbitrary type using template based duck typing and a set of rules to transform the result + * of your expressions to those types. However, this is incredibly tricky to get right and the error messages aren't very + * useful. Whenever possible try to use utree as the result type of your grammar elements. It should support any type needed + * for a typical parse grammar. To learn more about utree see the slides and talk + * here: http://boost-spirit.com/home/2011/06/12/ast-construction-with-the-universal-tree/ + * and docs here http://www.boost.org/doc/libs/1_47_0/libs/spirit/doc/html/spirit/support/utree.html + * + * Note: + * In utree there are 2 string type nodes. The standard std::string represents a quoted string value and utf8_symbol_range_type represents + * an operator or key word in your grammar. + * + * Note 2: + * When you modify your grammar keep an eye on the raw build output. There is a lot of debug information that gets printed but not included in the Visual Studio + * error log. + * + * This is not meant ot be a complete tutorial so you should also read though the spirit docs. + * + * Basics: + * + * ** Example 0 + * Most spirit parse expressions will look something like this: + * a = b [X] >> ";" + * this means a will be a parser which matches b followed by a literal semicolon. The statement inside the [ ] is called a "semantic action" + * When b is matched it will evaluate the semantic action X passing in the result (or property) of b as _1. The final result is always available + * as _val. + * If X takes the form _val = _1 it assigns the result of b to a assuming their type is compatible (nearly all types are compatible with utree) + * If X takes the form [myoperator(_val, _1)] then _val and _1 are passed in to your user defined operator and you can modify the utree AST directly. + * + * ** Example 1 + * rule term = + * bool_ [_val = _1] + * | '!' >> bool_ [not_expr(_val, _1)] + * + * If this encounters the phrase "true" the first part matches and we evaluate [_val = _1] the boolean is automatically converted to a utree (which contains a bool) + * If we have "!true" the second part matches and we evaluate [not_expr(_val, _1)] which calls our unary_operator class passing in expr which is the result property + * on term and rhs which is the bool value automatically converted to a utree. The operator is then free to modify the result (expr) directly. In this + * case it pushes 2 child nodes into expr a not operator and the original utree bool node. + * + * + * ** Example 2 + * rule quoted_string = + * char_("'\"") + * >> (lexeme[*(alnum [_val += _1] | space [_val += _1])]) + * >> char_("'\""); + * + * This is currently the only rule in this grammar which does not use a utree. Here we use a regular std::string rule because in each semantic action we want to append + * the new character to the result. If this semantic action was not there at all the resulting property would be a vector which included the quotes. This would + * be OK beause boost::spirit can automatically convert a vector to a std::string but we prefer to remove the quotes now. + * lexeme[] - is a spirit transform which tells the parser to ignore space skipping for the expression. In this case [ ] do not indicate a semantic action. + * *( ) - a kleene star indicateing 0 or more matches of the contained grammar. + * char_('") - this syntax matches either of the contained string literals. + * + * + * ** Example 3 + * This is an excerpt from the term expression that might help you understand how the types flow though boost::spirit (and why utree is so useful). + * + * ("in" >> ((list | property) [in_expr(_val, _1)] + * | _internal_string [in_expr(_val, _1)])) + * + * Q: Why did we separate quoted_string from (list | property) even though they both appear to have the same semantic action? + * In this case list and property both have the result type utree but quoted_string has the result type std::string. Since in_expr() has overloads for both + * a right hand utree and a right hand std::string so shoudln't it do the right thing? + * A: If we wrote (list | property | quoted_string) [in_expr(_val, _1] the type of _1 would not be either utree or string but a union of both types. Specifically + * it would be a boost::variant; Since we don't provide an in_expr::operator() overload for that compilation would fail. + * + */ + +#ifdef __GNUC__ +#pragma GCC diagnostic push +// Disble effc++ for this file because a lot of the spirit base classes don't include a virtual destructor +#pragma GCC diagnostic ignored "-Weffc++" +#endif + + +namespace bts { +namespace expression { + +namespace qi = boost::spirit::qi; +namespace ph = boost::phoenix; +namespace sp = boost::spirit; + +// Base operator holds the result type +struct base_operator +{ + template + struct result { typedef void type; }; + + base_operator(const std::string& name) + : _name(name) + {} + +protected: + std::string _name; +}; + +// Class of operator that acts on a left and right side +struct binary_operator : public base_operator +{ + binary_operator(const std::string& name) + : base_operator(name) + {} + + void operator()(sp::utree& expr, const sp::utree& rhs) const + { + sp::utree lhs; + lhs.swap(expr); + expr.push_back(sp::utf8_symbol_type(_name)); + expr.push_back(lhs); + expr.push_back(rhs); + } + +}; +extern ph::function const and_expr; +extern ph::function const or_expr; +extern ph::function const in_expr; +extern ph::function const gt_expr; +extern ph::function const gte_expr; +extern ph::function const lt_expr; +extern ph::function const lte_expr; +extern ph::function const eq_expr; +extern ph::function const neq_expr; +extern ph::function const match_expr; + +// "!" +struct unary_operator : public base_operator +{ + unary_operator(const std::string& name) + : base_operator(name) + {} + + void operator()(sp::utree& expr, const sp::utree & rhs) const + { + expr.push_back(sp::utf8_symbol_type(_name)); + expr.push_back(rhs); + } +}; +extern ph::function not_expr; + +// +// The property expression needs to accept a raw vector of characters. +// +struct property_expr_type +{ + template + struct result { typedef void type; }; + + void operator() (sp::utree& expr, const std::vector& rhs) const + { + std::string prop_name(rhs.begin (), rhs.end ()); + expr.clear(); + expr.push_back(sp::utf8_symbol_type("property")); + expr.push_back(sp::utf8_symbol_type(prop_name)); + } +}; +extern ph::function const property_expr; + +// +// The list expression simply concatenates values in to one utree node +// +struct list_expr_type +{ + template + struct result { typedef void type; }; + + void operator()(sp::utree& expr, const sp::utree& rhs) const + { + if (expr.empty()) + { + expr.push_back(sp::utf8_symbol_type("list")); + } + expr.push_back(rhs); + } +}; +extern ph::function list_expr; + +template +class BooleanGrammar : public qi::grammar +{ +public: + BooleanGrammar() + : BooleanGrammar::base_type(_expression) + { + using namespace qi; + using qi::_1; + + // + // Basic types + // + + // quoted string's property is std::string instead of utree b/c we need to keep the type compatible with the += operator to build up the string contents + _internal_string = + char_("'\"") + >> (lexeme[*(alnum | space | (punct - char_("'\""))) [_val += _1]]) + >> char_("'\""); + _quoted_string = _internal_string; + + // [1,2,3,4] - sorry no lists of lists + _list = '[' >> (_primitive [list_expr(_val, _1)] % ',') >> ']'; + + // A property is an unquoted token which will be passed to an IPropertyEvaluator + // to be turned into a primitive value or a list. + _property = lexeme[(+(alnum | char_("_.$")))] [property_expr(_val, _1)]; + + _primitive = + qi::no_case[qi::bool_] [_val = _1] + | qi::double_ [_val = _1] + | _quoted_string [_val = _1] + | _property [_val = _1] + | _list [_val = _1] + | '!' >> _primitive [not_expr(_val, _1)] + | '(' >> _expression [_val = _1] >> ')' + ; + + // + // operators + // + + _expression = + _term [_val = _1] + >> *( + ("and" >> _term [and_expr(_val, _1)]) + | ("&&" >> _term [and_expr(_val, _1)]) + | ("or" >> _term [or_expr(_val, _1)]) + | ("||" >> _term [or_expr(_val, _1)]) + ); + + _term = + _primitive [_val = _1] + >> *( ("in" >> ((_list | _property | _quoted_string) [in_expr(_val, _1)])) + | ("~=" >> (( _property | _quoted_string) [match_expr(_val, _1)])) + | (">" >> _primitive [gt_expr(_val, _1)]) + | (">=" >> _primitive [gte_expr(_val, _1)]) + | ("<" >> _primitive [lt_expr(_val, _1)]) + | ("<=" >> _primitive [lte_expr(_val, _1)]) + | ("==" >> _primitive [eq_expr(_val, _1)]) + | ("=" >> _primitive [eq_expr(_val, _1)]) + | ("!=" >> _primitive [neq_expr(_val, _1)]) + ); + + } + +private: + BTS_NO_COPY(BooleanGrammar); + qi::rule _internal_string; + qi::rule _expression, _primitive, _term, _list, _property, _quoted_string; +}; + +extern BooleanGrammar boolean_string_grammar; + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif + +}}//namespace +#endif diff --git a/src/lib/expression/ExpressionEvaluator.cpp b/src/lib/expression/ExpressionEvaluator.cpp new file mode 100755 index 000000000..8c4ecfce3 --- /dev/null +++ b/src/lib/expression/ExpressionEvaluator.cpp @@ -0,0 +1,478 @@ +#include "ExpressionEvaluator.h" +#include "BooleanGrammar.h" +#include "ParserExceptions.h" +#include "IPropertyEvaluator.h" + +#include +#include +#include +#include // iequals +#include + +namespace qi = boost::spirit::qi; +namespace sp = boost::spirit; +using namespace std; + +namespace bts { +namespace expression { + +class ExpressionEvaluatorImpl +{ +public: + ExpressionEvaluatorImpl(std::shared_ptr property_eval); + ~ExpressionEvaluatorImpl(); + + bool eval(const std::string& expression) const; + bool check_valid(const std::string& expression, std::string* failed_property) const; + void pre_parse(const string& expression, sp::utree& parse_result) const; + void pre_parse(const std::vector& expressions, std::vector& asts) const; + bool eval_ast(const boost::spirit::utree& ast) const; + Value eval_to_value(const std::string& expression) const; + +private: + BTS_NO_COPY(ExpressionEvaluatorImpl); + + bool parse(std::string::const_iterator& begin, const std::string::const_iterator& end, boost::spirit::utree& result) const; + bool check_properties(const boost::spirit::utree& node, std::string* failed_property) const; + Value eval_node(const boost::spirit::utree& node) const; + + inline bool op_and(boost::spirit::utree::const_iterator& args) const; + inline bool op_or(boost::spirit::utree::const_iterator& args) const; + inline bool op_in(boost::spirit::utree::const_iterator& args) const; + inline bool op_gt(boost::spirit::utree::const_iterator& args) const; + inline bool op_gte(boost::spirit::utree::const_iterator& args) const; + inline bool op_lt(boost::spirit::utree::const_iterator& args) const; + inline bool op_lte(boost::spirit::utree::const_iterator& args) const; + inline bool op_eq(boost::spirit::utree::const_iterator args) const; + inline bool op_neq(boost::spirit::utree::const_iterator& args) const; + inline bool op_match(boost::spirit::utree::const_iterator& args) const; + + + inline bool op_not(const boost::spirit::utree& operand) const; + inline Value op_list(const boost::spirit::utree& operands) const; + inline Value op_prop(const boost::spirit::utree& operand) const; + + const std::shared_ptr _property_eval; + + mutable RecursionGuard _recursion_guard; +}; + + +ExpressionEvaluatorImpl::ExpressionEvaluatorImpl(std::shared_ptr property_eval) + : + _property_eval(property_eval), + _recursion_guard() +{ +} + +ExpressionEvaluatorImpl::~ExpressionEvaluatorImpl() +{ +} + +bool ExpressionEvaluatorImpl::eval(const std::string& expression) const +{ + boost::spirit::utree parse_result; + pre_parse(expression, parse_result); + return eval_ast(parse_result); +} + +Value ExpressionEvaluatorImpl::eval_to_value(const std::string& expression) const +{ + boost::spirit::utree parse_result; + pre_parse(expression, parse_result); + return eval_node(parse_result); +} + +void ExpressionEvaluatorImpl::pre_parse(const std::string& expression, boost::spirit::utree& parse_result) const +{ + std::string::const_iterator begin = expression.begin (); + std::string::const_iterator end = expression.end (); + bool result = parse(begin, end, parse_result); + if (!result) + { + size_t pos = begin - expression.begin (); + throw ParseFailedException(expression, pos); + } + // std::cout << std::endl << parse_result; +} + +void ExpressionEvaluatorImpl::pre_parse(const std::vector& expressions, std::vector& asts) const +{ + std::vector::const_iterator i; + asts.reserve(expressions.size()); + for(i = expressions.begin(); i != expressions.end(); ++i) + { + boost::spirit::utree parse_result; + pre_parse(*i, parse_result); + asts.push_back(parse_result); + } +} + +bool ExpressionEvaluatorImpl::eval_ast(const boost::spirit::utree& ast) const +{ + return value_to_bool(eval_node(ast)); +} + +bool ExpressionEvaluatorImpl::check_valid(const std::string& expression, std::string* failed_property) const +{ + sp::utree parse_result; + std::string::const_iterator begin = expression.begin (); + std::string::const_iterator end = expression.end (); + bool result = parse(begin, end, parse_result); + if (result) + { + result = check_properties(parse_result, failed_property); + } + return result; +} + +bool ExpressionEvaluatorImpl::parse(std::string::const_iterator& begin, const std::string::const_iterator& end, sp::utree& ast) const +{ + // We reference a grammar defined elsewhere for compile time performance + bool result = qi::phrase_parse(begin, end, boolean_string_grammar, qi::ascii::space, ast); + return result && (begin==end); +} + +bool ExpressionEvaluatorImpl::check_properties(const boost::spirit::utree& node, std::string* failed_property) const +{ + bool result = true; + + // a list is either a list [a,b,c] or (operator, arg1, arg2,...) + if (node.which () == sp::utree_type::list_type) + { + boost::spirit::utree::const_iterator iter = node.begin (); + boost::spirit::utree::const_iterator end = node.end (); + + const sp::utree& operator_node = node.front(); + + if (operator_node.which() == sp::utree_type::symbol_type) + { + std::string op = get_symbol(operator_node); + ++iter; // advanced past operator + + if (op == "property") + { + std::string symbol = get_symbol(*iter); + if(!_property_eval->get_value(symbol, nullptr)) + { + if (failed_property != nullptr) + { + *failed_property = symbol; + } + return false; + } + } + } + + for(; result && iter != end; ++iter) + { + result = check_properties(*iter, failed_property); + } + } + else if (node.which () == sp::utree_type::invalid_type) + { + result = false; + } + + return result; +} + +void check_args(const std::string& op, const sp::utree& node, size_t got, size_t expected) +{ + if (got != expected) + { + std::ostringstream s_exp; + s_exp << node; + throw InsuficientArgumentsException(op, s_exp.str(), expected); + } +} + +Value ExpressionEvaluatorImpl::eval_node(const sp::utree& node) const +{ + if (node.which() != sp::utree_type::list_type || node.size() == 1) + { + return value_from_utree(node); + } + + sp::utree::const_iterator iter = node.begin (); + const sp::utree& operator_node = *iter; + ++iter; // advance past operator + + if (operator_node.which() != sp::utree_type::symbol_type) + throw std::invalid_argument("Operator expression must begin with operator symbol"); + + std::string op = get_symbol(operator_node); + + if (op == "property") + { + check_args(op, node, node.size(), 2); + return op_prop(*iter); + } + else if (op == "==") + { + check_args(op, node, node.size(), 3); + return op_eq(iter); + } + else if (op == "~=") + { + check_args(op, node, node.size(), 3); + return op_match(iter); + } + else if (op == "!") + { + check_args(op, node, node.size(), 2); + return op_not(*iter); + } + else if (op == "list") + { + return op_list(node); + } + else if (op == "and") + { + check_args(op, node, node.size(), 3); + return op_and(iter); + } + else if (op == "or") + { + check_args(op, node, node.size(), 3); + return op_or(iter); + } + else if (op == "in") + { + check_args(op, node, node.size(), 3); + return op_in(iter); + } + else if (op == ">") + { + check_args(op, node, node.size(), 3); + return op_gt(iter); + } + else if (op == ">=") + { + check_args(op, node, node.size(), 3); + return op_gte(iter); + } + else if (op == "<") + { + check_args(op, node, node.size(), 3); + return op_lt(iter); + } + else if (op == "<=") + { + check_args(op, node, node.size(), 3); + return op_lte(iter); + } + else if (op == "!=") + { + check_args(op, node, node.size(), 3); + return op_neq(iter); + } + + std::string exception_msg = "Unknown operator: " + op; + throw ParserException(exception_msg); +} + +bool ExpressionEvaluatorImpl::op_and(boost::spirit::utree::const_iterator& args) const +{ + // short circuit rhs + bool left = value_to_bool(eval_node(*args)); + args++; + return left && value_to_bool(eval_node(*args)); +} + +bool ExpressionEvaluatorImpl::op_or(boost::spirit::utree::const_iterator& args) const +{ + // short circuit rhs + bool left = value_to_bool(eval_node(*args)); + return left || value_to_bool(eval_node(*++args)); +} + +struct InsensitiveStringCompare +{ + InsensitiveStringCompare(const std::string& val) : value(val) { } + const std::string& value; + + bool operator() (const std::string& val) const + { + return boost::iequals(value, val); + } + + bool operator() (const Value val) const { + if (val.which() != ValueType::anything) { + return boost::iequals(value, expression::value_to_string(val)); + } + return false; + } +}; + +bool ExpressionEvaluatorImpl::op_in(boost::spirit::utree::const_iterator& args) const +{ + Value left = eval_node(*args); + Value right = eval_node(*++args); + + std::vector vector; + if (right.which() == ValueType::list) + { + vector = boost::get >(right); + if (left.which() == ValueType::string) + { + return std::find_if(vector.begin(), vector.end(), InsensitiveStringCompare(boost::get(left))) != vector.end(); + } + else + { + return std::find(vector.begin (), vector.end (), left) != vector.end (); + } + } + else if (right.which() == ValueType::string) + { + string r = boost::get(right); + string l = boost::get(left); + boost::algorithm::to_lower(r); + boost::algorithm::to_lower(l); + return r.find(l) != std::string::npos; + } + else + { + throw std::invalid_argument("Right hand argument to 'in' operator must be a list"); + } +} + +bool ExpressionEvaluatorImpl::op_gt(boost::spirit::utree::const_iterator& args) const +{ + Value left = eval_node(*args); + Value right = eval_node(*++args); + return right < left; +} + +bool ExpressionEvaluatorImpl::op_gte(boost::spirit::utree::const_iterator& args) const +{ + Value left = eval_node(*args); + Value right = eval_node(*++args); + return right == left || right < left; +} + +bool ExpressionEvaluatorImpl::op_lt(boost::spirit::utree::const_iterator& args) const +{ + Value left = eval_node(*args); + Value right = eval_node(*++args); + return left < right; +} + +bool ExpressionEvaluatorImpl::op_lte(boost::spirit::utree::const_iterator& args) const +{ + Value left = eval_node(*args); + Value right = eval_node(*++args); + return left == right || left < right; +} + +bool ExpressionEvaluatorImpl::op_eq(boost::spirit::utree::const_iterator args) const +{ + Value left = eval_node(*args); + args++; + Value right = eval_node(*args); + if (left.which() == ValueType::string && right.which() == ValueType::string) + { + return boost::iequals( + boost::get(left), + boost::get(right)); + } + return left == right; +} + +bool ExpressionEvaluatorImpl::op_neq(boost::spirit::utree::const_iterator& args) const +{ + return !op_eq(args); +} + +inline bool ExpressionEvaluatorImpl::op_match(boost::spirit::utree::const_iterator& args) const +{ + Value left = eval_node(*args); + args++; + Value right = eval_node(*args); + + return ::boost::regex_search(boost::get(left), boost::regex(boost::get(right))); +} + +bool ExpressionEvaluatorImpl::op_not(const boost::spirit::utree& operand) const +{ + return !value_to_bool(eval_node(operand)); +} + +Value ExpressionEvaluatorImpl::op_list(const boost::spirit::utree& operands) const +{ + boost::spirit::utree::const_iterator iter = operands.begin (), end = operands.end (); + ++iter; // advance over "list" operator + + std::vector list_values; + list_values.reserve(operands.size() - 1); + + for (; iter != end; ++iter) + { + list_values.push_back(eval_node(*iter)); + } + + return Value(list_values); +} + +Value ExpressionEvaluatorImpl::op_prop(const sp::utree& node) const +{ + Value result; + std::string symbol = get_symbol(node); + + if (_recursion_guard.check(symbol)) + { + throw RecursivePropertyGetException(symbol, _recursion_guard.get()); + } + GuardFrame frame; + _recursion_guard.push(symbol, &frame); + + if (!_property_eval->get_value(symbol, &result)) + { + throw InvalidPropertyException(symbol); + } + + return result; +} + +// +// Outter class method definitions +// + +ExpressionEvaluator::ExpressionEvaluator(std::shared_ptr property_eval) + : _impl(new ExpressionEvaluatorImpl(property_eval)) +{ +} + +ExpressionEvaluator::~ExpressionEvaluator() +{ +} + +bool ExpressionEvaluator::eval(const std::string& expression) const +{ + return _impl->eval(expression); +} + +Value ExpressionEvaluator::eval_to_value(const std::string& expression) const +{ + return _impl->eval_to_value(expression); +} + +bool ExpressionEvaluator::check_valid(const std::string& expression, std::string* failed_property) const +{ + return _impl->check_valid(expression, failed_property); +} + +void ExpressionEvaluator::pre_parse(const std::string& expression, boost::spirit::utree& ast) const { + return _impl->pre_parse(expression, ast); +} +void ExpressionEvaluator::pre_parse(const std::vector& expressions, std::vector& asts) const +{ + return _impl->pre_parse(expressions, asts); +} +bool ExpressionEvaluator::eval_ast(const boost::spirit::utree& ast) const +{ + return _impl->eval_ast(ast); +} + + +}}//namespace diff --git a/src/lib/expression/ExpressionEvaluator.h b/src/lib/expression/ExpressionEvaluator.h new file mode 100755 index 000000000..f2a56f823 --- /dev/null +++ b/src/lib/expression/ExpressionEvaluator.h @@ -0,0 +1,39 @@ +#ifndef EXPRESSIONEVALUATOR_H_ +#define EXPRESSIONEVALUATOR_H_ + +#include "Value.h" +#include "RecursionGuard.h" + +#include +#include +#include +#include + +namespace bts { +namespace expression { + +class IPropertyEvaluator; +class ExpressionEvaluatorImpl; + +class ExpressionEvaluator +{ +public: + ExpressionEvaluator(std::shared_ptr property_eval); + ~ExpressionEvaluator(); + + bool eval(const std::string& expression) const; + Value eval_to_value(const std::string& expression) const; + + bool check_valid(const std::string& expression, std::string* failed_property) const; + + void pre_parse(const std::string& expression, boost::spirit::utree& ast) const; + void pre_parse(const std::vector& expressions, std::vector& asts) const; + bool eval_ast(const boost::spirit::utree& ast) const; + +private: + BTS_NO_COPY(ExpressionEvaluator); + std::unique_ptr _impl; +}; + +}}//namespace +#endif diff --git a/src/lib/expression/IPropertyEvaluator.h b/src/lib/expression/IPropertyEvaluator.h new file mode 100755 index 000000000..cf13148ff --- /dev/null +++ b/src/lib/expression/IPropertyEvaluator.h @@ -0,0 +1,23 @@ +#ifndef IPROPERTY_EVAULATOR_H_ +#define IPROPERTY_EVAULATOR_H_ + +#include "ExpressionEvaluator.h" +#include "Value.h" + +#include +#include + +namespace bts { +namespace expression { + +class IPropertyEvaluator +{ +public: + virtual bool get_value(const std::string& property, Value* out_value) = 0; + + virtual ~IPropertyEvaluator() { } +}; + + +}}//namespace +#endif diff --git a/src/lib/expression/ParserExceptions.cpp b/src/lib/expression/ParserExceptions.cpp new file mode 100755 index 000000000..f19b0fc74 --- /dev/null +++ b/src/lib/expression/ParserExceptions.cpp @@ -0,0 +1,87 @@ +#include "ParserExceptions.h" + +#include +#include + +namespace bts { +namespace expression { + + // + // ParserException + // + ParserException::ParserException(const char* what) + : std::logic_error(""), _what(what) + { + } + + ParserException::ParserException(const std::string& what) + : std::logic_error(""), _what(what) + { + } + + ParserException::ParserException(const ParserException& other) + : std::logic_error(""), _what(other._what) + { + } + + ParserException& ParserException::operator=(const ParserException& other) + { + _what = other._what; + return *this; + } + + const char* ParserException::what() const throw() + { + return _what.c_str(); + } + + ParserException::~ParserException() throw() { } + + + // + // ParseFailedException + // + ParseFailedException::ParseFailedException(const std::string& context, size_t at) + : ParserException("Parse failed at position: " + boost::lexical_cast(at) + " in expression: " + context), + context(context) + { + } + + // + // InvalidPropertyException + // + InvalidPropertyException::InvalidPropertyException(const std::string& prop_name) + : ParserException("Invalid property name: " + prop_name) + {} + + // + // InsuficientArgumentsException + // + InsuficientArgumentsException::InsuficientArgumentsException(const std::string& op, const std::string& got, size_t expected) + : ParserException(""), + expected_argument_count(expected), + expression(got) + { + std::ostringstream what; + what << "Bad argument count for operator " << op << std::endl << "expected " << expected_argument_count << " arguments. Got '" << expression << "'"; + _what = what.str(); + } + + // + // RecursivePropertyGetException + // + RecursivePropertyGetException::RecursivePropertyGetException(const std::string& property, const std::vector& stack) + : ParserException("") + { + std::ostringstream what; + what << "Error evaluating property '" << property << "': recursive property request at:" << std::endl; + std::vector::const_reverse_iterator iter = stack.rbegin (); + std::vector::const_reverse_iterator end = stack.rend (); + for(; iter != end; iter++) + { + what << "> " << *iter << std::endl; + } + _what = what.str(); + } + +}}//namespace diff --git a/src/lib/expression/ParserExceptions.h b/src/lib/expression/ParserExceptions.h new file mode 100755 index 000000000..6069d1207 --- /dev/null +++ b/src/lib/expression/ParserExceptions.h @@ -0,0 +1,55 @@ +#ifndef PARSEREXCEPTIONS_H_ +#define PARSEREXCEPTIONS_H_ + +//#include +#include +#include +#include + +namespace bts { +namespace expression { + +struct ParserException : public std::logic_error +{ + ParserException(const char* what); + ParserException(const std::string& what); + ParserException(const ParserException& other); + ParserException& operator=(const ParserException& other); + virtual ~ParserException() throw(); + + const char* what() const throw(); + +protected: + std::string _what; +}; + +struct ParseFailedException : public ParserException +{ + ParseFailedException(const std::string& context, size_t at); + ~ParseFailedException() throw() {} + std::string context; +}; + +struct InvalidPropertyException : public ParserException +{ + InvalidPropertyException(const std::string& prop_name); + ~InvalidPropertyException() throw() {} +}; + +struct InsuficientArgumentsException : public ParserException +{ + InsuficientArgumentsException(const std::string& oper, const std::string& expression, size_t expected); + ~InsuficientArgumentsException() throw() {} + + size_t expected_argument_count; + std::string expression; +}; + +struct RecursivePropertyGetException : public ParserException +{ + RecursivePropertyGetException(const std::string& property, const std::vector& stack); + ~RecursivePropertyGetException() throw() {} +}; + +}}//namespace +#endif diff --git a/src/lib/expression/RecursionGuard.h b/src/lib/expression/RecursionGuard.h new file mode 100755 index 000000000..fa9d5360c --- /dev/null +++ b/src/lib/expression/RecursionGuard.h @@ -0,0 +1,91 @@ +#ifndef RECURSIONGUARD_H_ +#define RECURSIONGUARD_H_ + +#include "lib/utils/bts_no_copy.h" +#include +#include + +namespace bts { +namespace expression { + + template + class RecursionGuard; + + // Safely pops the stack when destroyed + template + struct GuardFrame + { + GuardFrame () + : _owner (0) + {} + + GuardFrame(RecursionGuard& owner) + : _owner(&owner) + {} + + ~GuardFrame() + { + if (_owner) + { + std::vector& guard = _owner->get(); + guard.pop_back(); + } + } + + void set_owner (RecursionGuard *owner) + { + _owner = owner; + } + + private: + BTS_NO_COPY(GuardFrame); + RecursionGuard* _owner; + }; + + + template + class RecursionGuard + { + public: + RecursionGuard() + : _recursion_guard() + {} + ~RecursionGuard() + {} + + std::vector& get() + { + std::vector* guard = _recursion_guard.get(); + if (guard == 0) + { + guard = new std::vector(); + _recursion_guard.reset(guard); + } + return *guard; + } + + bool check(const T& value) + { + std::vector& guard = get(); + typename std::vector::iterator begin = guard.begin (), end = guard.end (); + bool is_recursive = std::find(begin, end, value) != end; + return is_recursive; + } + + void push(const T& value, GuardFrame *frame) + { + std::vector& guard = get(); + guard.push_back(value); + frame->set_owner (this); + } + + template + friend struct GuardFrame; + private: + BTS_NO_COPY(RecursionGuard); + boost::thread_specific_ptr > _recursion_guard; + }; + + +}}//namespace +#endif diff --git a/src/lib/expression/Value.cpp b/src/lib/expression/Value.cpp new file mode 100755 index 000000000..4c8d1e5ca --- /dev/null +++ b/src/lib/expression/Value.cpp @@ -0,0 +1,94 @@ +#include "Value.h" + +#include + +#include +#include + +namespace bts { +namespace expression { + +namespace sp = boost::spirit; + +bool value_to_bool(const Value& val) +{ + switch(val.which()) + { + case ValueType::boolean: + return boost::get(val); + case ValueType::integer: + return boost::get(val) != 0; + case ValueType::real: + return boost::get(val) != 0; + case ValueType::string: + return !boost::get(val).empty(); + case ValueType::list: + return boost::get >(val).size() > 0; + default: + throw std::logic_error("Invalid type in variant. Please update this switch statement"); + } +} +std::string value_to_string(const Value& val) +{ + switch(val.which()) + { + case ValueType::boolean: + return boost::get(val) ? "true" : "false"; + case ValueType::integer: + return boost::lexical_cast( + boost::get(val)); + case ValueType::real: + return boost::lexical_cast( + boost::get(val)); + case ValueType::string: + return boost::get(val); + case ValueType::list: { + auto vals = boost::get>(val); + std::string result; + for (const auto& v : vals) { + result += value_to_string(v); + } + return result; + } + default: + throw std::logic_error("Invalid type in variant. Please update this switch statement"); + } +} + +const std::string get_symbol(const sp::utree& node) +{ + sp::utf8_symbol_range_type rt = node.get(); + return std::string(rt.begin (),rt.end ()); +} + +const std::string get_string(const boost::spirit::utree& node) +{ + sp::utf8_symbol_range_type string_range = node.get(); + return std::string(string_range.begin (), string_range.end ()); +} + +Value value_from_utree(const boost::spirit::utree& node) +{ + switch(node.which()) + { + case sp::utree_type::bool_type: + return Value(node.get()); + case sp::utree_type::int_type: + return Value(node.get()); + case sp::utree_type::double_type: + return Value(node.get()); + case sp::utree_type::string_type: + return Value(get_string(node)); + case sp::utree_type::list_type: + if (node.size() == 1) + { + return value_from_utree(node.front()); + } + // else fall through + default: + // Switch may need updating + throw std::logic_error("Invalid type node primitive type"); + } +} + +}}//namespace diff --git a/src/lib/expression/Value.h b/src/lib/expression/Value.h new file mode 100755 index 000000000..8d13dcebd --- /dev/null +++ b/src/lib/expression/Value.h @@ -0,0 +1,44 @@ +#ifndef VALUE_H_ +#define VALUE_H_ + +#include +#include + +namespace boost { +namespace spirit { + class utree; +}} + + +namespace bts { +namespace expression { + +struct ValueType +{ + enum type { + boolean, + integer, + real, + string, + list, + anything + }; +}; + +typedef boost::make_recursive_variant< + bool, + int, + double, + std::string, + std::vector, + void*>::type Value; + +bool value_to_bool(const Value& val); +std::string value_to_string(const Value& val); +Value value_from_utree(const boost::spirit::utree& node); + +const std::string get_symbol(const boost::spirit::utree& node); +const std::string get_string(const boost::spirit::utree& node); + +}}//namespace +#endif diff --git a/src/lib/expression/tests/expression_parser_tests.cpp b/src/lib/expression/tests/expression_parser_tests.cpp new file mode 100755 index 000000000..3d9f08ded --- /dev/null +++ b/src/lib/expression/tests/expression_parser_tests.cpp @@ -0,0 +1,179 @@ +#include "core/bts_core/utest/utest.h" + +#include "../IPropertyEvaluator.h" +#include "../ExpressionEvaluator.h" +#include "../ParserExceptions.h" + +#include "core/bts_core/utils/BTS_SharedPtr.h" +#include + +using namespace bts::expression; + +namespace bts { + + // Simple property evaluator for testing + class EvaluatesToFoo : public IPropertyEvaluator + { + public: + EvaluatesToFoo() + { } + ~EvaluatesToFoo() + { } + BTS_NO_COPY(EvaluatesToFoo); + + bool get_value(const std::string& prop, Value* out_value) OVERRIDE + { + bool valid = prop.substr(0,3) == "Foo"; + if (valid && out_value != nullptr) + { + if (prop == "Foo.number") + { + *out_value = Value(42.0); + } + else if (prop == "Foo.list") + { + std::vector values; + values.push_back(Value(std::string("foo"))); + values.push_back(Value(std::string("bar"))); + values.push_back(Value(std::string("baz"))); + *out_value = Value(values); + } + else if (prop == "Foo.bool") + { + *out_value = Value(true); + } + else + { + // Foo.* + *out_value = Value(std::string("Foo")); + } + } + + return valid; + } + }; + + class EvaluatesRecursively : public IPropertyEvaluator + { + public: + EvaluatesRecursively() + : _e(nullptr) + { } + ~EvaluatesRecursively() + { } + BTS_NO_COPY(EvaluatesRecursively); + + void set_evaluator(ExpressionEvaluator* e) + { + _e = e; + } + + bool get_value(const std::string& prop, Value* out_value) OVERRIDE + { + std::string recur = "42 == " + prop; + if (out_value) + { + *out_value = Value(_e->eval(recur)); + } + return true; + } + + ExpressionEvaluator* _e; + }; + + struct expression_tests : public TestSuite + { + expression_tests() { } + ~expression_tests() { } + BTS_NO_COPY(expression_tests); + + void setup() + { + } + + void teardown() + { + } + + void simple_operator_test() + { + ExpressionEvaluator e(BTS_SharedPtr (new EvaluatesToFoo())); + test_assert(e.eval("!!true")); + test_assert(e.eval("3 == 3")); + test_assert(e.eval("4 > 3")); + test_assert(e.eval("3 < 6")); + test_assert(e.eval("3 <= 3.5")); + test_assert(e.eval("3 in [1,2,3]")); + test_assert(!e.eval("3 in [1,2,4]")); + test_assert(e.eval("'oo' in Foo.foo")); + test_assert(e.eval("'oo' == 'oo'")); + test_assert(e.eval("'afoobar' ~= 'foo.*'")); + test_assert(!e.eval("'afoobar' ~= '^foo.*'")); + } + + void test_bool_conversion() + { + ExpressionEvaluator e(BTS_SharedPtr (new EvaluatesToFoo())); + test_assert(e.eval("true")); + test_assert(e.eval("1")); + test_assert(e.eval("1.5")); + test_assert(e.eval("'true string'")); + //test_assert(e.eval("[1,2,3]")); // list not valid here - only after "in" + test_assert(e.eval("Foo.x")); + test_assert(e.eval("Foo.list")); + test_assert(!e.eval("''")); + } + + void property_validator_test() + { + ExpressionEvaluator e(BTS_SharedPtr (new EvaluatesToFoo())); + test_assert(e.check_valid("Foo.bar == 'Foo' or 'x' in [Foo.x, Foo.y, 'z']", nullptr)); + std::string bad_property; + test_assert(!e.check_valid("Foo.bar == 'Foo' or 'x' in [Fuu.x, Foo.y, 'z']", &bad_property)); + test_assert(bad_property == "Fuu.x"); + } + + void more_complex_test() + { + ExpressionEvaluator e(BTS_SharedPtr (new EvaluatesToFoo())); + + test_assert(e.eval("Foo.bar == 'Foo' and 'b' in ['a', 'b', 'c', Foo.bar]")); + test_assert(e.eval("Foo.number > 2 or (3.5 <= 7 and Foo.number == 3) or 'x' in ['a', 'b', 'c']")); + test_assert(!e.eval("(3.5 <= 7 and Foo.number == 3) or 'x' in ['a', 'b', 'c']")); + test_assert(e.eval("'foo' in Foo.list and Foo.number == 42.0")); + test_assert(e.eval("!!('oo' in Foo.foo)")); + test_assert(e.eval("!('ooo' in Foo.foo)")); + test_assert(e.eval("!false in [false, false, false, false, true, Foo.bool]")); + } + + void recursive_test() + { + BTS_SharedPtr badprops (new EvaluatesRecursively()); + ExpressionEvaluator e(badprops); + badprops->set_evaluator(&e); + + // Try to get the 'fuzzy'property recursively + bool success = false; + try + { + test_assert(e.eval("fuzzy == 42")); + } + catch(RecursivePropertyGetException) + { + success = true; + } + test_assert(success); + } + + }; + + + BEGIN_TEST_SUITE(expression_tests) + ADD_TEST(simple_operator_test); + ADD_TEST(test_bool_conversion); + ADD_TEST(property_validator_test); + ADD_TEST(more_complex_test); + ADD_TEST(recursive_test); + END_TEST_SUITE + +} diff --git a/src/lib/logging/Common.h b/src/lib/logging/Common.h new file mode 100644 index 000000000..1e9488534 --- /dev/null +++ b/src/lib/logging/Common.h @@ -0,0 +1,29 @@ +#pragma once + +#include +#include +#include +#include + +namespace bts { +namespace log { + +struct Level { + using Rep = uint8_t; + constexpr static auto Max = std::numeric_limits::max(); + Rep ordinal; + bool operator<(const Level &rhs) const {return ordinal < rhs.ordinal;} +}; + +struct Subsystem { + using Rep = uint8_t; + constexpr static auto Max = std::numeric_limits::max(); + Rep ordinal; + bool operator<(const Subsystem &rhs) const {return ordinal < rhs.ordinal;} +}; + +using ArgBuffer = std::array; +using MsgFormatter = std::string(*) (ArgBuffer const&,const char*); +using Filter = std::array; + +}} diff --git a/src/lib/logging/LogRec.h b/src/lib/logging/LogRec.h new file mode 100644 index 000000000..c2e496b64 --- /dev/null +++ b/src/lib/logging/LogRec.h @@ -0,0 +1,42 @@ +#pragma once + +#include "lib/utils/warn_ctrl.h" +#include "Common.h" +#include "MsgPacking.h" + +namespace bts { +namespace log { + +using ArgUnpacker = std::string (*) (ArgBuffer const&b, const char*msg_template); + +GCC_DIAG_OFF("-Weffc++") +struct LogRec { + const char *file; + const char *msg_template; + ArgUnpacker unpacker; + uint16_t line; + Level level; + Subsystem subsystem; + ArgBuffer args; + + template + void pack_msg (const char *msg_t, Args&&... msg_args) { + msg_template = msg_t; + + unpacker = unpack_msg_args::type...>; + pack_msg_args(args,std::forward(msg_args)...); + } + + std::string unpack_msg() + { + return unpacker(args,msg_template); + } +}; +GCC_DIAG_RESTORE() + +#ifndef WIN32 +static_assert(sizeof(LogRec) == 64, "LogRec should be 64 bytes"); +#endif + +}} + diff --git a/src/lib/logging/Logger.h b/src/lib/logging/Logger.h new file mode 100644 index 000000000..1dd5bee74 --- /dev/null +++ b/src/lib/logging/Logger.h @@ -0,0 +1,134 @@ +#pragma once + +#include "MsgPacking.h" +#include "LogRec.h" +#include "lib/utils/bts_assert.h" + +#include +#include + +namespace bts { +namespace log { + +class Logger { +public: + Logger(const Logger&) = delete; + Logger&operator=(const Logger&) = delete; + + Logger(std::thread::id tid, + Filter const&f, + size_t initial_queue_size, + size_t max_queue_growth): + _mutex () , + _read_q (&_queue_b), + _write_q(&_queue_a), + _wcursor(0), + _filter(f), + _drop_count(0), + _queue_a(initial_queue_size), + _queue_b(initial_queue_size), + _tid(tid), + _active(true), + _max_queue_size(initial_queue_size*max_queue_growth) + { + } + + template + void enqueue (Subsystem subsystem,Level lv, const char*file, int line, + const char *msg_template, Args&&... msg_args) + { + bts_assert(_active); + lock(); + if (lv.ordinal >= _filter[subsystem.ordinal].ordinal) { + if (_wcursor == _write_q->size()) + _drop_count++; + else { + auto &lrec = (*_write_q) [_wcursor++]; + lrec.file = file; + lrec.line = line; + lrec.level = lv; + lrec.subsystem = subsystem; + lrec.pack_msg(msg_template, std::forward(msg_args)...); + } + } + unlock(); + } + + template + void unqueue(S const&s) { + size_t logged_msg_count = 0; + size_t dropped_msg_count = 0; + bool active = true; + + lock(); + std::swap(_read_q,_write_q); + logged_msg_count = _wcursor; + dropped_msg_count = _drop_count; + active = _active; + _wcursor = 0; + _drop_count = 0; + unlock(); + + s(_tid,active,dropped_msg_count,logged_msg_count,*_read_q); + + if ((logged_msg_count > _read_q->size() / 2) && + _read_q->size() < _max_queue_size) + _read_q->resize(_read_q->size()*2); + } + + void update_filter(Filter const&new_filter) + { + lock(); + _filter = new_filter; + unlock(); + } + + void update_queue_limits(size_t initial_queue_size, + size_t max_queue_growth) { + lock(); + if (_wcursor == 0) { + // we can only do this if the queue is empty because each log + // record contains a tuple with possibly non trivially + // constructuable objects and cannot be moved + _write_q->resize(initial_queue_size); + } + _read_q->resize(initial_queue_size); + _max_queue_size = initial_queue_size*max_queue_growth; + unlock(); + } + + void mark_inactive() + { + lock(); + bts_assert(_active); + _active=false; + unlock(); + } + + ~Logger() + { + bts_assert(!_active); + bts_assert(_wcursor == 0); + } + + +private: + void lock () + {while (_mutex.test_and_set(std::memory_order_acquire));} + + void unlock () + {_mutex.clear(std::memory_order_release);} + + std::atomic_flag _mutex = ATOMIC_FLAG_INIT; + std::vector* _read_q; + std::vector* _write_q; + size_t _wcursor; + Filter _filter; + size_t _drop_count; + std::vector _queue_a; + std::vector _queue_b; + std::thread::id _tid; + bool _active; + size_t _max_queue_size; +}; +}} diff --git a/src/lib/logging/LoggerHub.cpp b/src/lib/logging/LoggerHub.cpp new file mode 100644 index 000000000..574a66afb --- /dev/null +++ b/src/lib/logging/LoggerHub.cpp @@ -0,0 +1,201 @@ +#include "LoggerHub.h" + +#include "lib/utils/bts_assert.h" +#include "Logger.h" +#include "lib/os/os.h" +#include +#include +#include +#include + +namespace bts { +namespace log { + +struct LoggerHub::SinkRec { + Sink sink; + Filter filter; + std::string name; +}; + +struct LoggerHub::PImpl { + PImpl(std::string const&logger_name) : + lock(), + sinks(), + loggers(), + level_names(), + subsystem_names(), + worker(), + done(false), + logger_initial_queue_size(), + logger_max_queue_growth(), + logger_name(logger_name) + {} + + std::mutex lock; + std::vector sinks; + std::vector> loggers; + std::array level_names; + std::array subsystem_names; + std::thread worker; + std::atomic done; + size_t logger_initial_queue_size; + size_t logger_max_queue_growth; + std::string logger_name; +}; + +LoggerHub::LoggerHub(std::string const&logger_name) + : me (new PImpl(logger_name)) +{ + LoggingConfig cfg ; + config(cfg); + me->worker = std::thread(std::bind(&LoggerHub::do_work,this)); +} + +void LoggerHub::config(const LoggingConfig &cfg) +{ + std::lock_guard l (me->lock); + + me->logger_initial_queue_size = cfg.logger_initial_queue_size; + me->logger_max_queue_growth = cfg.logger_max_queue_growth; + + for (size_t i = 0; i <= Level::Max ; i++) + me->level_names[i] = std::to_string(i); + + for (size_t i = 0; i <= Subsystem::Max ; i++) + me->subsystem_names[i] = std::to_string(i); + + for (auto const &kvp : cfg.level_names) + me->level_names[kvp.first.ordinal] = kvp.second; + + for (auto const &kvp : cfg.subsystem_names) + me->subsystem_names[kvp.first.ordinal ] = kvp.second; + + me->sinks.clear(); + for (auto const &kvp : cfg.sinks) { + auto const &name = kvp.first; + auto const &sink = kvp.second.first; + auto const &filter = kvp.second.second; + me->sinks.push_back(SinkRec{sink,filter,name}); + } + rebuild_logger_filters(); + update_logger_queue_limits(); +} + +LoggerHub::~LoggerHub() +{ + me->done=true; + me->worker.join(); +} + +Logger &LoggerHub::make_logger() +{ + std::lock_guard l (me->lock); + me->loggers.push_back(std::make_unique (std::this_thread::get_id(), + get_base_filter(), + me->logger_initial_queue_size, + me->logger_max_queue_growth)); + return *me->loggers.back(); +} + +void LoggerHub::do_work() +{ + os::set_thread_name(me->logger_name.c_str()); + while (!me->done) { + size_t consumed = 0; + { + std::lock_guard l (me->lock); + consumed = consume_log_recs(); + } + if (consumed == 0) + std::this_thread::yield(); + } + + for (;;) { + auto all_shutdown = false; + { + std::lock_guard l (me->lock); + consume_log_recs(); + all_shutdown = me->loggers.empty(); + } + if (all_shutdown) + return ; + else + std::this_thread::yield(); + } +} + +size_t LoggerHub::consume_log_recs() +{ + std::vector> to_keep; + to_keep.reserve(me->loggers.size()); + + size_t consumed = 0; + for (size_t i = 0; i < me->loggers.size();i++) { + auto &l = me->loggers[i]; + l->unqueue([&l,&to_keep,i,&consumed,this] (std::thread::id tid, bool active, + size_t dropped_msg_count, + size_t logged_msg_count, + std::vector &msgs) { + consumed+= (dropped_msg_count + logged_msg_count); + if (dropped_msg_count) { + LogRec lrec; + lrec.file = __FILE__; + lrec.line = __LINE__; + lrec.level = Level {Level::Max}; + lrec.subsystem = Subsystem {Subsystem::Max}; + lrec.pack_msg ("Dropped {} messages. Increase logging buffer size.", + dropped_msg_count); + dispatch_to_sinks (tid,lrec); + } + + for (size_t i = 0; i < logged_msg_count; i++) + dispatch_to_sinks(tid,msgs[i]); + + if (active) + to_keep.emplace_back(std::move(l)); + }); + } + me->loggers = std::move(to_keep); + return consumed; +} + +void LoggerHub::dispatch_to_sinks(std::thread::id tid,LogRec&lr) +{ + auto msg = lr.unpack_msg(); + auto lm = LogMsg { msg, lr.file, lr.line, + lr.level, me->level_names[lr.level.ordinal], + lr.subsystem, me->subsystem_names[lr.subsystem.ordinal], + tid }; + + for (auto &s : me->sinks) + if (lr.level.ordinal >= s.filter[lr.subsystem.ordinal].ordinal) + s.sink(lm); +} + + +void LoggerHub::rebuild_logger_filters() +{ + auto f = get_base_filter(); + for (auto &l : me->loggers) + l->update_filter(f); +} + +Filter LoggerHub::get_base_filter() const +{ + Filter f; + for (auto &l : f) + l.ordinal = 255; + + for (auto &s : me->sinks) + for(size_t i = 0; i < Subsystem::Max+1; i++) + f[i].ordinal = std::min(f[i].ordinal,s.filter[i].ordinal); + return f; +} + +void LoggerHub::update_logger_queue_limits() +{ + for (auto &l : me->loggers) + l->update_queue_limits(me->logger_initial_queue_size,me->logger_max_queue_growth); +} + +}} diff --git a/src/lib/logging/LoggerHub.h b/src/lib/logging/LoggerHub.h new file mode 100644 index 000000000..0ca757010 --- /dev/null +++ b/src/lib/logging/LoggerHub.h @@ -0,0 +1,35 @@ +#pragma once + +#include "Common.h" +#include "Sink.h" +#include "LoggingConfig.h" + +#include + +namespace bts { +namespace log { + +class Logger; +struct LogRec; + +class LoggerHub { +public: + LoggerHub(std::string const& ); + ~LoggerHub(); + void config(const LoggingConfig &cfg); + Logger &make_logger(); + +private: + struct SinkRec; + struct PImpl; + std::unique_ptr me; + + void do_work(); + void update_logger_queue_limits(); + size_t consume_log_recs(); + void rebuild_logger_filters(); + void dispatch_to_sinks(std::thread::id,LogRec&); + Filter get_base_filter() const; +}; + +}} diff --git a/src/lib/logging/LoggingConfig.h b/src/lib/logging/LoggingConfig.h new file mode 100644 index 000000000..1d50646bf --- /dev/null +++ b/src/lib/logging/LoggingConfig.h @@ -0,0 +1,26 @@ +#pragma once + +#include "Sink.h" +#include "Common.h" +#include +#include + +namespace bts { +namespace log { + +struct LoggingConfig { + LoggingConfig() : logger_initial_queue_size(1200000), + logger_max_queue_growth(4), + sinks(), + level_names(), + subsystem_names() + {} + + size_t logger_initial_queue_size = 0; + size_t logger_max_queue_growth = 0; + std::map> sinks; + std::map level_names; + std::map subsystem_names; +}; + +}} diff --git a/src/lib/logging/MsgPacking.h b/src/lib/logging/MsgPacking.h new file mode 100644 index 000000000..405bf09e6 --- /dev/null +++ b/src/lib/logging/MsgPacking.h @@ -0,0 +1,45 @@ +#pragma once + +#include +#include +#include +#include + +namespace bts { +namespace log { + +template +std::string format_tuple(const char*msg_template, Tuple const &t,std::index_sequence) +{return fmt::format(msg_template,std::get(t)...);} + +template +std::string unpack_msg_args(std::array const&buffer, const char*msg_template) +{ + std::string result; + using Store = std::tuple; + if (sizeof(Store) <= SZ) { + auto args = (Store*) &buffer[0]; + result = format_tuple(msg_template,*args,std::index_sequence_for{}); + args->~Store(); + } + else { + auto indirect_store = (Store**) &buffer[0]; + auto args = *indirect_store; + result = format_tuple(msg_template,*args,std::index_sequence_for{}); + delete args; + } + return result; +} + +template +void pack_msg_args(std::array &buffer,Args&&...args) { + using Store = std::tuple::type...>; + if(sizeof(Store) <= SZ) + new (&buffer[0]) Store(std::forward(args)...); + else { + auto indirect_store = new Store(std::forward(args)...); + *((Store**) &buffer[0]) = indirect_store; + } +} + +}} diff --git a/src/lib/logging/Sink.cpp b/src/lib/logging/Sink.cpp new file mode 100644 index 000000000..c584f755c --- /dev/null +++ b/src/lib/logging/Sink.cpp @@ -0,0 +1,86 @@ +#include "Sink.h" + +#include "lib/logging/logging.h" +#include "lib/os/os.h" +#include "lib/utils/datetime.h" + +#include +#include +#include +#include + +#ifdef WIN32 +#include +#else +#include +#endif + + +namespace bts { +namespace log { + +#ifdef WIN32 +HANDLE win_event_log; +#endif + +struct InitSysLog { + InitSysLog() { +#ifdef WIN32 + win_event_log = RegisterEventSource(NULL,"spark"); +#else + openlog("spark",0,LOG_USER); +#endif + + } +} init; + + +auto formatter() { + return [] (LogMsg const&m) { + return fmt::format("{} | {} {} {}:{} tid:{} pid:{} {}", + m.msg, + m.subsystem_name, + m.level_name, + boost::filesystem::path(m.file).filename().string() , + m.line, + m.tid, + os::process_id(), + datetime::time_string(datetime::now())); + }; +} + +Sink console_sink() { + auto f = formatter(); + return [f] (LogMsg const &m) {std::cout << f(m) << std::endl;}; +} + +Sink syslog_sink() { + auto f = formatter(); +#ifdef WIN32 + return [f] (LogMsg const &m) { + int win_level; + switch (m.level.ordinal) { + case 0: + win_level = EVENTLOG_SUCCESS; + break; + case 1: + win_level = EVENTLOG_INFORMATION_TYPE; + break; + case 2: + win_level = EVENTLOG_WARNING_TYPE; + break; + default: + win_level = EVENTLOG_ERROR_TYPE; + break; + + }; + auto str = f(m).c_str(); + ReportEvent(win_event_log,win_level,0,0,NULL,1,0,&str,0); + }; +#else + return [f] (LogMsg const &m) {syslog (LOG_INFO | LOG_USER,"%s",f(m).c_str ());}; +#endif + +} + +}} diff --git a/src/lib/logging/Sink.h b/src/lib/logging/Sink.h new file mode 100644 index 000000000..dd3333c9c --- /dev/null +++ b/src/lib/logging/Sink.h @@ -0,0 +1,31 @@ +#pragma once + +#include "lib/utils/warn_ctrl.h" +#include "Common.h" + +#include +#include +#include + +namespace bts { +namespace log { +GCC_DIAG_OFF("-Weffc++") +struct LogMsg { + const std::string &msg; + + const char *file; + const uint16_t line; + + const Level level; + const std::string &level_name; + + const Subsystem subsystem; + const std::string &subsystem_name; + const std::thread::id tid; +}; +GCC_DIAG_RESTORE() + +using Sink = std::function ; +Sink syslog_sink(); +Sink console_sink(); +}} diff --git a/src/lib/logging/logging.cpp b/src/lib/logging/logging.cpp new file mode 100644 index 000000000..e5313d4d1 --- /dev/null +++ b/src/lib/logging/logging.cpp @@ -0,0 +1,140 @@ +#include "logging.h" + +#include "LoggerHub.h" +#include "lib/config/config.h" +#include "lib/os/os.h" +#include "lib/utils/bts_assert.h" +#include "lib/utils/bts_chrono.h" +#include "lib/config/protos/logging.pb.h" + +#include +#include +#include +#include +#include +#include + +namespace bts { +namespace log { + +static LoggerHub _default_logger_hub("logger"); +thread_local LoggerHandle _thread_logger(_default_logger_hub.make_logger()); + + +template +typename M::const_iterator find_key(M const&m, typename M::mapped_type const &v) +{ + for (auto it = m.begin(); it != m.end();it++) + if (it->second == v) + return it; + return m.end(); +} + +Filter get_filter (LoggingConfig &cfg, + config::Logging::SinkConfig const&scfg) +{ + Filter result; + + auto it = find_key(cfg.level_names,scfg.default_level()); + bts_assert(it != cfg.level_names.end()); + auto default_level = it->first; + for (auto &l : result) + l = default_level; + + for (auto const &sub_system : scfg.subsystem()) { + + auto ss_it = find_key(cfg.subsystem_names,sub_system.name()); + bts_assert(ss_it != cfg.subsystem_names.end()); + auto ss = ss_it->first; + + auto level_it = find_key(cfg.level_names,sub_system.level()); + bts_assert(level_it != cfg.level_names.end()); + auto level = level_it->first; + + result[ss.ordinal] = level; + } + return result; +} + +void configure_logging(LoggingConfig const&cfg) {_default_logger_hub.config(cfg);} + +void configure_logging(bts::config::Logging const&cfg) +{ + LoggingConfig real_cfg; + real_cfg.logger_initial_queue_size = cfg.initial_queue_size(); + real_cfg.logger_max_queue_growth = cfg.max_queue_growth(); + + real_cfg.level_names = { {Diag,"Diag"},{Info,"Info"}, + {Warn,"Warn"},{Error,"Error"}, + {Fatal,"Fatal"},{Sys,"Sys"} }; + + real_cfg.subsystem_names = { {Gen,"Gen"}, + {Cfg,"Cfg"}, + {QBus,"QBus"}, + {Log,"Log"} }; + + real_cfg.sinks["console"] = std::make_pair (console_sink(),get_filter(real_cfg,cfg.console())); + real_cfg.sinks["syslog"] = std::make_pair (syslog_sink(),get_filter(real_cfg,cfg.syslog())); + configure_logging(real_cfg); +} + +LoggingConfigPoller::LoggingConfigPoller (bts::config::Logging const&cfg) : _poller_thread(), + _done(false) +{ + init(cfg); +} + +LoggingConfigPoller::LoggingConfigPoller () : _poller_thread(), _done(false) +{ + // retrieve cache from config + init(config::get ()); +} + +void LoggingConfigPoller::init (bts::config::Logging const&cfg) +{ + // cache config in a file + auto log_cache_file = os::bts_data_path(); + log_cache_file += "/logging_control"; + std::ofstream fs (log_cache_file); + google::protobuf::io::OstreamOutputStream gs (&fs); + google::protobuf::TextFormat::Print (cfg,&gs); + + + // initialize logging + configure_logging(cfg); + + _poller_thread = std::thread([this,log_cache_file] { + os::set_thread_name("log config poll"); + auto last_ts = std::time(nullptr); + + while (!_done) { + + auto current_ts = + boost::filesystem::last_write_time(boost::filesystem::path(log_cache_file)) ; + + if (current_ts != last_ts) { + last_ts = current_ts; + bts::config::Logging cfg; + std::ifstream fs (log_cache_file); + google::protobuf::io::IstreamInputStream gs (&fs); + if (google::protobuf::TextFormat::Parse (&gs,&cfg)) { + LOG(Log,Warn,"Detected logging configuration change. Reconfiguring logging."); + configure_logging(cfg); + } + else + LOG(Log,Error,"Detected invalid logging configuration. Ignoring."); + } + else + std::this_thread::sleep_for(1s); + } + }); +} + +LoggingConfigPoller::~LoggingConfigPoller () +{ + _done = true; + _poller_thread.join(); +} + +}} + diff --git a/src/lib/logging/logging.h b/src/lib/logging/logging.h new file mode 100644 index 000000000..2d393051d --- /dev/null +++ b/src/lib/logging/logging.h @@ -0,0 +1,57 @@ +#pragma once + +#include +#include +#include "Common.h" +#include "Logger.h" +#include "LoggerHub.h" + +namespace bts { +namespace config {class Logging;} + +constexpr log::Level Diag {0}; +constexpr log::Level Info {Diag.ordinal+1}; +constexpr log::Level Warn {Info.ordinal+1}; +constexpr log::Level Error {Warn.ordinal+1}; +constexpr log::Level Fatal {Error.ordinal+1}; +constexpr log::Level Sys {log::Level::Max-1}; + +constexpr log::Subsystem Gen {0}; +constexpr log::Subsystem Cfg {Gen.ordinal+1}; +constexpr log::Subsystem QBus {Cfg.ordinal+1}; +constexpr log::Subsystem Log {QBus.ordinal+1}; +constexpr log::Subsystem Jetfix {Log.ordinal+1}; +constexpr log::Subsystem MarketData {Jetfix.ordinal+1}; +constexpr log::Subsystem Api {MarketData.ordinal+1}; + +namespace log { + +struct LoggerHandle { + LoggerHandle(LoggerHandle const&) = delete; + LoggerHandle& operator=(LoggerHandle const&) = delete; + + LoggerHandle() : logger(nullptr) {} + LoggerHandle (Logger &l) : logger(&l) {} + ~LoggerHandle() { if (logger) logger->mark_inactive();} + Logger *logger; +}; + +extern thread_local LoggerHandle _thread_logger; +void configure_logging(LoggingConfig const&cfg); +void configure_logging(bts::config::Logging const&cfg); + +class LoggingConfigPoller { +public: + LoggingConfigPoller (); + LoggingConfigPoller (bts::config::Logging const&cfg); + ~LoggingConfigPoller (); + +private: + void init(bts::config::Logging const&cfg); + std::thread _poller_thread; + std::atomic _done; +}; + +#define LOG(subsystem,level,...) bts::log::_thread_logger.logger->enqueue(subsystem,level,__FILE__,__LINE__,__VA_ARGS__) + +}} diff --git a/src/lib/logging/tests/logging_tests.cpp b/src/lib/logging/tests/logging_tests.cpp new file mode 100644 index 000000000..48b32ea0f --- /dev/null +++ b/src/lib/logging/tests/logging_tests.cpp @@ -0,0 +1,78 @@ +#include "catch/catch.h" +#include "lib/utils/bts_chrono.h" +#include "lib/logging/logging.h" + +namespace bts { + +TEST_CASE("Logging Tests","[logging] [benchmark]") { + + auto sz = 1000000; + log::Filter f; + for (auto &l : f) + l = Diag; + log::LoggingConfig cfg; + cfg.logger_initial_queue_size = sz*10; + cfg.sinks["null_sink"] = std::make_pair ( [](log::LogMsg const&) {}, f ); + log::configure_logging(cfg); + LOG(Gen,Info,"Kicking things off"); + + SECTION("timestamp_interpolation") + { + auto start = bts::Timer::now(); + for (int i = 0; i < sz ;i++) + LOG(Gen,Info,"Hello World. It is now: {0}",bts::Timer::now()); + auto stop = bts::Timer::now(); + std::cout << "\tLogging: ns/msg for log with timestamps: " + << bts::ns_diff(stop,start) / sz + << " tt: " << bts::ns_diff(stop,start) + << std::endl; + } + + SECTION("int_interpolation") + { + auto start = bts::Timer::now(); + for (int i = 0; i < sz;i++) + LOG(Gen,Info,"Hello World. This is interpolating a number {0}",12); + auto stop = bts::Timer::now(); + std::cout << "\tLogging ns/msg for log with simple number interpolation: " + << bts::ns_diff(stop,start) / sz + << " tt: " << bts::ns_diff(stop,start) + << std::endl; + } + + SECTION("large_string_interpolation") + { + std::array strs; + int i = 0; + strs[i++] = "aaaaaaaaaaaaaaaaa"; + strs[i++] = "aaaaaaaaaaaaaaaaa"; + strs[i++] = "aaaaaaaaaaaaaaaaa"; + strs[i++] = "aaaaaaaaaaaaaaaaa"; + strs[i++] = "aaaaaaaaaaaaaaaaa"; + strs[i++] = "aaaaaaaaaaaaaaaaa"; + strs[i++] = "aaaaaaaaaaaaaaaaa"; + strs[i++] = "aaaaaaaaaaaaaaaaa"; + + auto start = bts::Timer::now(); + for (int i = 0; i < sz;i++) + LOG(Gen,Info,"Hello World. This is interpolating a string ",strs[i & 0x00000007] ); + auto stop = bts::Timer::now(); + std::cout << "\tLogging ns/msg for log with large string interpolation: " << bts::ns_diff(stop,start) / sz + << " tt: " << bts::ns_diff(stop,start) + << std::endl; + } + + SECTION("log_skipping") + { + auto start = bts::Timer::now(); + for (int i = 0; i < sz;i++) + LOG(Gen,Diag,"Hello World. I am {} years old!",i); + auto stop = bts::Timer::now(); + std::cout << "\tLogging ns/msg for log skipping: " + << bts::ns_diff(stop,start) / sz + << " tt: " << bts::ns_diff(stop,start) + << std::endl; + } +} + +} diff --git a/src/lib/os/Win32Exception.cpp b/src/lib/os/Win32Exception.cpp new file mode 100755 index 000000000..7507a602f --- /dev/null +++ b/src/lib/os/Win32Exception.cpp @@ -0,0 +1,52 @@ +#ifdef _WIN32 + +#include "core/bts_core/utils/defer.h" +#include "Win32Exception.h" +#include + +namespace bts { +namespace os { + +Win32Exception::Win32Exception(DWORD error_code) : + _code(error_code), + _what(get_error_string(error_code)), + std::runtime_error(_what) +{ +} + +Win32Exception::~Win32Exception() +{ +} + +std::string Win32Exception::get_error_string(DWORD error_code) +{ + LPSTR buffer = NULL; + DEFER(if (buffer) ::LocalFree(buffer)); + + ::FormatMessageA( + FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_ALLOCATE_BUFFER, + NULL, + error_code, + 0, + (LPSTR)&buffer, + 0, NULL); + + + std::ostringstream os; + os << "HRESULT: 0x" << std::hex << error_code; + os << " - " << buffer; + return os.str(); +} + +std::string Win32Exception::get_last_error_string() +{ + return get_error_string(::GetLastError()); +} + +const char* Win32Exception::what() const +{ + return _what.c_str(); +} + +}}//namespace +#endif diff --git a/src/lib/os/Win32Exception.h b/src/lib/os/Win32Exception.h new file mode 100755 index 000000000..3cf47360a --- /dev/null +++ b/src/lib/os/Win32Exception.h @@ -0,0 +1,32 @@ +#pragma once + +#ifdef _WIN32 +#include "../utils/windows_wrapper.h" +#include "../bts_core_exports.h" +#include +#include + +namespace bts { +namespace os { + +class BTS_CORE_API Win32Exception + : public std::runtime_error +{ +public: + Win32Exception(DWORD error_code); + virtual ~Win32Exception(); + virtual const char* what() const override; + + DWORD error_code() { return _code; } + + static std::string get_error_string(DWORD error_code); + static std::string get_last_error_string(); + +private: + DWORD _code; + std::string _what; +}; + +}}//namespace + +#endif diff --git a/src/lib/os/os.cpp b/src/lib/os/os.cpp new file mode 100755 index 000000000..965a4e2b4 --- /dev/null +++ b/src/lib/os/os.cpp @@ -0,0 +1,1095 @@ +#include "os.h" +#include "lib/utils/bts_assert.h" +#include "lib/utils/bts_debugger.h" +#include "lib/config/config.h" +#include "lib/utils/bts_array_size.h" + +#include +#include + +#ifdef _WIN32 +#include +#include +#ifdef GetMessage +#undef GetMessage +#endif +#include +#include +#include +#include +#include +#include +#else +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#endif + +#include + +#include +#include +#include +#include // strlcpy +#include +#include +#include +#include + +#ifndef NO_CPP0X +#include +#endif + +#include + +#ifdef WIN32 +# pragma comment(lib, "wbemuuid.lib") +#endif + +namespace bts { +namespace os { + +unsigned int process_id () +{ +#ifdef _WIN32 + return GetCurrentProcessId (); +#else + return getpid (); +#endif +} + +bool is_running(unsigned int pid) { +#ifdef _WIN32 + HANDLE handle = OpenProcess(SYNCHRONIZE, false, pid); + auto err = GetLastError(); + // ERROR_INVALID_PARAMETER is returned when the pid is not valid + // If we're out of memory a different error code is returned but I'm having a hard time figuring out what it is... + if (!handle && err == ERROR_INVALID_PARAMETER) return false; + CloseHandle(handle); + return true; +#else + return kill(pid,0) != -1; +#endif +} + +bool parse_int(const std::string& str, int *result) { + std::stringstream conv; + conv << str; + conv >> *result; + return conv.eof(); +} + +std::vector parse_cmd_line(std::string const &cmd_line) { + const int ws = 0; + const int qt = 1; + const int t = 2; + + std::vector result; + size_t n = 0; + int state = ws; + while (n < cmd_line.size()) { + switch (state) { + case ws: + if (isspace (cmd_line[n])) + n++; + else if (cmd_line[n] == '"') + { + n++; + result.push_back(""); + state = qt; + } + else { + result.push_back(""); + result.back().push_back(cmd_line[n]); + n++; + state = t; + } + break; + case qt: + if (cmd_line[n] == '"') { + n++; + state = ws; + } + else { + result.back().push_back(cmd_line[n]); + n++; + } + break; + case t: + if (isspace (cmd_line[n])) { + n++; + state = ws; + } + else { + result.back().push_back(cmd_line[n]); + n++; + } + break; + } + } + return result; +} + +#ifdef _WIN32 +std::string ConvertWCSToMBS(const wchar_t* pstr, long wslen) +{ + int len = ::WideCharToMultiByte(CP_ACP, 0, pstr, wslen, NULL, 0, NULL, NULL); + + std::string dblstr(len, '\0'); + len = ::WideCharToMultiByte(CP_ACP, 0 /* no flags */, + pstr, wslen /* not necessary NULL-terminated */, + &dblstr[0], len, + NULL, NULL /* no default char */); + + return dblstr; +} + +std::string bstr_to_str (BSTR const &bstr) +{ + int wslen = ::SysStringLen(bstr); + return ConvertWCSToMBS((wchar_t*)bstr, wslen); +} +#endif + +#ifdef _WIN32 +struct RunningProcessesQuerier::PImpl +{ + PImpl() : pLoc (NULL),pSvc (NULL), where () + {} + IWbemLocator *pLoc; + IWbemServices *pSvc; + std::string where; +}; + +RunningProcessesQuerier::RunningProcessesQuerier (std::vector const&exes) + : me (new PImpl()) { + + for (size_t i = 0; i < exes.size ();i++) { + if (me->where.size ()) + me->where += " or " ; + me->where += "Name='" + exes[i] +".exe'"; + } + + HRESULT hres; + CoInitializeEx(0, COINIT_MULTITHREADED); + CoInitializeSecurity( + NULL, + -1, // COM authentication + NULL, // Authentication services + NULL, // Reserved + RPC_C_AUTHN_LEVEL_DEFAULT, // Default authentication + RPC_C_IMP_LEVEL_IMPERSONATE, // Default Impersonation + NULL, // Authentication info + EOAC_NONE, // Additional capabilities + NULL // Reserved + ); + hres = CoCreateInstance( + CLSID_WbemLocator, + 0, + CLSCTX_INPROC_SERVER, + IID_IWbemLocator, (LPVOID *) &me->pLoc); + + assert (!FAILED(hres)); + + hres = me->pLoc->ConnectServer( + _bstr_t(L"ROOT\\CIMV2"), + NULL, + NULL, + 0, + 0, + 0, + 0, + &me->pSvc + ); + + assert (!FAILED(hres)); + + + hres = CoSetProxyBlanket( + me->pSvc, // Indicates the proxy to set + RPC_C_AUTHN_WINNT, // RPC_C_AUTHN_xxx + RPC_C_AUTHZ_NONE, // RPC_C_AUTHZ_xxx + NULL, // Server principal name + RPC_C_AUTHN_LEVEL_CALL, // RPC_C_AUTHN_LEVEL_xxx + RPC_C_IMP_LEVEL_IMPERSONATE, // RPC_C_IMP_LEVEL_xxx + NULL, // client identity + EOAC_NONE // proxy capabilities + ); + + assert (!FAILED(hres)); +} + +std::vector RunningProcessesQuerier::query () { + std::vector result; + IEnumWbemClassObject* pEnumerator = NULL; + std::string query = "Select Name,ProcessId,CommandLine from Win32_Process"; + if (me->where.size ()) + query += " Where " + me->where; + + HRESULT hres = me->pSvc->ExecQuery( + bstr_t("WQL"), + bstr_t(query.c_str ()), + WBEM_FLAG_FORWARD_ONLY | WBEM_FLAG_RETURN_IMMEDIATELY, + NULL, + &pEnumerator); + + assert (!FAILED(hres)); + + IWbemClassObject *pclsObj = 0; + ULONG uReturn = 0; + + while (pEnumerator) + { + HRESULT hr = pEnumerator->Next(WBEM_INFINITE, 1, &pclsObj, &uReturn); + if(0 == uReturn) + break; + + ProcInfo i; + + VARIANT vtProp; + // Get the value of the Name property + pclsObj->Get(L"Name", 0, &vtProp, 0, 0); + i.exe = bstr_to_str (vtProp.bstrVal); + boost::replace_all(i.exe, ".exe", ""); + VariantClear(&vtProp); + + pclsObj->Get(L"CommandLine", 0, &vtProp, 0, 0); + i.args = parse_cmd_line(bstr_to_str (vtProp.bstrVal)); + VariantClear(&vtProp); + + pclsObj->Get(L"ProcessId", 0, &vtProp, 0, 0); + i.pid = vtProp.intVal; + VariantClear(&vtProp); + + pclsObj->Release(); + result.push_back(i); + } + pEnumerator->Release(); + return result; +} + +RunningProcessesQuerier::~RunningProcessesQuerier () +{ + me->pSvc->Release(); + me->pLoc->Release(); +} + +#else +struct RunningProcessesQuerier::PImpl {}; +RunningProcessesQuerier::RunningProcessesQuerier (std::vector const&) : me () {} +RunningProcessesQuerier::~RunningProcessesQuerier () {} +std::vector RunningProcessesQuerier::query () {return os::running_processes ();} +#endif + +std::vector running_processes () +{ + std::vector result; +#ifdef _WIN32 + HRESULT hres; + CoInitializeEx(0, COINIT_MULTITHREADED); + CoInitializeSecurity( + NULL, + -1, // COM authentication + NULL, // Authentication services + NULL, // Reserved + RPC_C_AUTHN_LEVEL_DEFAULT, // Default authentication + RPC_C_IMP_LEVEL_IMPERSONATE, // Default Impersonation + NULL, // Authentication info + EOAC_NONE, // Additional capabilities + NULL // Reserved + ); + IWbemLocator *pLoc = NULL; + + hres = CoCreateInstance( + CLSID_WbemLocator, + 0, + CLSCTX_INPROC_SERVER, + IID_IWbemLocator, (LPVOID *) &pLoc); + + assert (!FAILED(hres)); + + IWbemServices *pSvc = NULL; + + hres = pLoc->ConnectServer( + _bstr_t(L"ROOT\\CIMV2"), + NULL, + NULL, + 0, + 0, + 0, + 0, + &pSvc + ); + + assert (!FAILED(hres)); + + + hres = CoSetProxyBlanket( + pSvc, // Indicates the proxy to set + RPC_C_AUTHN_WINNT, // RPC_C_AUTHN_xxx + RPC_C_AUTHZ_NONE, // RPC_C_AUTHZ_xxx + NULL, // Server principal name + RPC_C_AUTHN_LEVEL_CALL, // RPC_C_AUTHN_LEVEL_xxx + RPC_C_IMP_LEVEL_IMPERSONATE, // RPC_C_IMP_LEVEL_xxx + NULL, // client identity + EOAC_NONE // proxy capabilities + ); + + assert (!FAILED(hres)); + + IEnumWbemClassObject* pEnumerator = NULL; + hres = pSvc->ExecQuery( + bstr_t("WQL"), + bstr_t("Select * from Win32_Process"), + WBEM_FLAG_FORWARD_ONLY | WBEM_FLAG_RETURN_IMMEDIATELY, + NULL, + &pEnumerator); + + assert (!FAILED(hres)); + + IWbemClassObject *pclsObj = 0; + ULONG uReturn = 0; + + while (pEnumerator) + { + HRESULT hr = pEnumerator->Next(WBEM_INFINITE, 1, &pclsObj, &uReturn); + if(0 == uReturn) + break; + + ProcInfo i; + + VARIANT vtProp; + // Get the value of the Name property + pclsObj->Get(L"Name", 0, &vtProp, 0, 0); + i.exe = bstr_to_str (vtProp.bstrVal); + boost::replace_all(i.exe, ".exe", ""); + VariantClear(&vtProp); + + pclsObj->Get(L"CommandLine", 0, &vtProp, 0, 0); + i.args = parse_cmd_line(bstr_to_str (vtProp.bstrVal)); + VariantClear(&vtProp); + + pclsObj->Get(L"ProcessId", 0, &vtProp, 0, 0); + i.pid = vtProp.intVal; + VariantClear(&vtProp); + + pclsObj->Release(); + result.push_back(i); + } + + pSvc->Release(); + pLoc->Release(); + pEnumerator->Release(); + +#else + boost::filesystem::directory_iterator p ("/proc"); + for (;p != boost::filesystem::directory_iterator (); p++) { + ProcInfo pi; + int pid; + if (parse_int (p->path ().filename ().string (),&pid)) { + pi.pid =pid; + std::string file_name (p->path ().string ()+"/cmdline"); + std::ifstream fs (file_name.c_str ()); + std::stringstream sstr; + sstr << fs.rdbuf(); + if (!fs.bad ()) { + std::string cmd_line = sstr.str(); + bool first_char = true; + std::string *current = 0; + for (size_t i = 0; i < cmd_line.size ();i++) + if (cmd_line[i] != 0) { + if (first_char) { + pi.args.push_back (""); + current = &(pi.args.back ()); + } + first_char = false; + current->push_back (cmd_line[i]); + } + else + first_char = true; + // shamelessly copied from get_application_path + char buffer[1024]; + char temp[32]; + sprintf(temp, "/proc/%d/exe", pi.pid); + int bytes = readlink(temp, buffer, sizeof(buffer)); + if (bytes>0) { + buffer[bytes] = 0; + pi.exe =boost::filesystem::path (buffer).filename ().string (); + } + result.push_back (pi); + } + } + } +#endif + + return result; +} + +char const* platform_name () +{ +#ifdef _WIN32 + return "Windows"; +#elif __linux__ + return "Linux"; +#else +#error This function needs to be implemented for your platform +#endif +} + +std::string host_name () +{ + char buffer[128]; + bts_assert(gethostname(buffer,BTS_ARRAYSIZE(buffer)) == 0); + return std::string (buffer); +} + +std::string get_install_path() +{ + auto result = get_application_path(); + // Trim the filename + size_t last = result.find_last_of("/\\"); + result.erase(last); + return result; +} + +std::string get_application_path() +{ + char buffer[1024]; +#ifdef _WIN32 + int bytes = GetModuleFileName(NULL, buffer, sizeof(buffer)); +#elif __linux__ + // I'm not completely happy with this solution but there doesn't seem to be anything + // more standard. + char temp[32]; + sprintf(temp, "/proc/%d/exe", getpid()); + int bytes = readlink(temp, buffer, sizeof(buffer)); +#else + // Most unixes don't have /proc +#error This function needs to be implemented for your platform +#endif + if (bytes > 0) + { + buffer[bytes] = 0; + return std::string(buffer); + } + else + { + throw std::runtime_error("Cannot read application directory"); + } +} + +std::string get_application_name() +{ + std::string app_path = get_application_path(); + size_t last = app_path.find_last_of("/\\"); + if (last + 1 >= app_path.size()) + return ""; + return app_path.substr(last + 1, app_path.size() - last); +} + +std::string get_cwd() +{ +#ifdef _WIN32 +#define GET_CURRENT_DIR _getcwd +#else +#define GET_CURRENT_DIR getcwd +#endif + + char current_path[1024]; + if (!GET_CURRENT_DIR(current_path, sizeof(current_path))) + { + throw std::runtime_error("current directory unreadable"); + } + return std::string(current_path); +#undef GET_CURRENT_DIR +} + + +std::string& normalize_path(std::string& path, char separator) +{ + size_t len = path.length(); + for (size_t x = 0; x < len; x++) + { + if (path[x] == '/' || path[x] == '\\') + { + path[x] = separator; + } + } + return path; +} + +std::string sanitize_file_name(std::string const& file_name) +{ +#ifndef NO_CPP0X + const std::string forbidden = "\\/:?\"<>|"; + std::string result = file_name; + std::transform (result.begin (), result.end (),result.begin (), + [&forbidden](char c) -> char { + if(forbidden.find(c) != std::string::npos) + return '_'; + else + return c; + }); + return result; +#else + return file_name; +#endif +} + +void gen_uuid (unsigned char out [16]) +{ +#ifdef _WIN32 + GUID out_guid; + /* + typedef struct _GUID { + unsigned long Data1; + unsigned short Data2; + unsigned short Data3; + unsigned char Data4[ 8 ]; + } GUID; + + I don't think C++ guarantees the above structure is packed, so I + do the safe thing below and copy each member in turn. -pp + */ + CoCreateGuid(&out_guid); + int idx = 0; + memcpy (out + idx,&out_guid.Data1,sizeof(out_guid.Data1)); + idx += sizeof(out_guid.Data1); + + memcpy (out + idx,&out_guid.Data2,sizeof(out_guid.Data2)); + idx += sizeof(out_guid.Data2); + + memcpy (out + idx,&out_guid.Data3,sizeof(out_guid.Data3)); + idx += sizeof(out_guid.Data3); + + memcpy (out + idx,&out_guid.Data4,sizeof(out_guid.Data4)); + idx += sizeof(out_guid.Data4); + bts_assert (idx == 16); +#else + uuid_generate(out); +#endif +} + +uint64_t tick_count () +{ +#ifdef _WIN32 + LARGE_INTEGER qpcnt; + QueryPerformanceCounter(&qpcnt); + return qpcnt.QuadPart; +#else + struct timespec ts; + clock_gettime( CLOCK_REALTIME, &ts ); + return + static_cast(1000000000UL)* + static_cast(ts.tv_sec) + + static_cast(ts.tv_nsec); +#endif +} + +#ifdef _WIN32 +std::vector g_cmd_args; +bool g_cmd_args_set; + +// For compat with langues that don't follow normal CRT init or use a different CRT +extern "C" BTS_EXPORT void provide_command_line(const char** argv, int argc) +{ + if (!g_cmd_args_set) { + g_cmd_args.reserve(argc); + for (int x = 0; x < argc; x++) + { + g_cmd_args.push_back(argv[x]); + } + g_cmd_args_set = true; + } +} +#endif + +std::vector get_command_line() +{ +#ifdef _WIN32 + if (g_cmd_args_set) + { + // return cached copy + return g_cmd_args; + } + for (int index = 0; index < __argc; index++) + { + if (__argv != nullptr) + { + g_cmd_args.push_back(__argv[index]); + } + else if (__wargv != nullptr) + { + char temp[1024]; + WideCharToMultiByte(CP_ACP, 0, __wargv[index], -1, temp, BTS_ARRAYSIZE(temp), nullptr, nullptr); + g_cmd_args.push_back(temp); + } + else + { + bts_assert(__argv != nullptr || __wargv != nullptr); + } + } + g_cmd_args_set = true; + return g_cmd_args; +#elif __linux__ + std::vector args; + std::ifstream in("/proc/self/cmdline", std::ios::in); + if (!in.fail()) + { + std::string current; + do + { + char ch = in.get(); + if (ch == '\0') + { + args.push_back(current); + current.clear(); + } + else + { + current += ch; + } + } while(!in.eof()); + } + return args; +#else +#error This function needs to be implemented for your platform +#endif +} + +std::string get_qualifier() +{ + std::vector cmd_line = get_command_line(); + if (cmd_line.size() > 0) + return cmd_line.back(); + return ""; +} + +int get_random_port () +{ + srand ( os::tick_count () ); + return rand() % 10000 + 20000; +} + +int num_cores() { +#ifdef WIN32 + SYSTEM_INFO sysinfo; + GetSystemInfo(&sysinfo); + return sysinfo.dwNumberOfProcessors; +#else + return sysconf(_SC_NPROCESSORS_ONLN); +#endif +} + +std::string bts_user_path() +{ +#ifdef _WIN32 + const char* path = getenv("APPDATA"); + bts_assert(path); + return std::string(path) + "/bts/"; +#else + const char* path = getenv("HOME"); + bts_assert_msg(path, "Environment variable $HOME must be defined"); + return std::string(path) + "/.bts/"; +#endif +} + +std::string bts_system_path() +{ +#ifdef _WIN32 + const char* path = getenv("PROGRAMDATA"); + bts_assert(path); + return std::string(path) + "/bts/"; +#else + return "/etc/bts/"; +#endif +} + +std::string bts_data_path() +{ + std::string path = + bts_user_path() + "data/"; + + if (!boost::filesystem::is_directory(path)) + { + boost::filesystem::create_directories(path); + } + return path; +} + +std::string username() { +#ifdef _WIN32 + char buffer[1024]; + DWORD buffer_size = BTS_ARRAYSIZE(buffer); + if (GetUserNameA((LPSTR)&buffer, &buffer_size)) { + return buffer; + } +#else + uid_t uid = geteuid(); + struct passwd* pw = getpwuid(uid); + if (pw) { return pw->pw_name; } +#endif + return ""; +} + +char* strncpy(char* dest, const char* src, size_t count) +{ +#ifdef _WIN32 + ::strncpy_s(dest, count, src, count); +#else + // could use strlcpy but that isn't supported on orc + ::strncpy(dest, src, count); + dest[count-1] = '\0'; +#endif + return dest; +} + +void setenv(const char* name, const char* value) +{ +#ifdef _WIN32 + SetEnvironmentVariable(name, value); +#else + ::setenv(name, value, true); +#endif +} + +#ifdef _WIN32 + +// Link with Iphlpapi.lib +#pragma comment(lib, "IPHLPAPI.lib") + +#define WORKING_BUFFER_SIZE 15000 +#define MAX_TRIES 3 + +#define MALLOC(x) HeapAlloc(GetProcessHeap(), 0, (x)) +#define FREE(x) HeapFree(GetProcessHeap(), 0, (x)) + +// inline std::wstring convert( const std::string& as ) +// { +// wchar_t* buf = new wchar_t[as.size() * 2 + 2]; +// swprintf( buf, L"%S", as.c_str() ); +// std::wstring rval = buf; +// delete[] buf; +// return rval; +// } + +std::map get_windows_ips () +{ + std::map result; + + char address_buffer[256]; + bool success = false; + + DWORD dwSize = 0; + DWORD dwRetVal = 0; + + unsigned int i = 0; + ULONG flags = GAA_FLAG_SKIP_ANYCAST | GAA_FLAG_SKIP_MULTICAST | GAA_FLAG_SKIP_DNS_SERVER; + ULONG family = AF_UNSPEC; + LPVOID lpMsgBuf = NULL; + PIP_ADAPTER_ADDRESSES pAddresses = NULL; + ULONG outBufLen = 0; + ULONG Iterations = 0; + PIP_ADAPTER_ADDRESSES pCurrAddresses = NULL; + PIP_ADAPTER_UNICAST_ADDRESS pUnicast = NULL; + + family = AF_INET; + outBufLen = WORKING_BUFFER_SIZE; + + do { + pAddresses = (IP_ADAPTER_ADDRESSES *) MALLOC(outBufLen); + if (pAddresses == NULL) { + bts_assert(false); + } + + dwRetVal = + GetAdaptersAddresses(family, flags, NULL, pAddresses, &outBufLen); + + if (dwRetVal == ERROR_BUFFER_OVERFLOW) { + FREE(pAddresses); + pAddresses = NULL; + } else { + break; + } + + Iterations++; + + } while ((dwRetVal == ERROR_BUFFER_OVERFLOW) && (Iterations < MAX_TRIES)); + + if (dwRetVal == NO_ERROR) { + pCurrAddresses = pAddresses; + while (pCurrAddresses) { + if ((pCurrAddresses->OperStatus == IfOperStatusUp) && + (pCurrAddresses->FirstUnicastAddress)) { + DWORD length = 256; + WSAAddressToStringA ( + pCurrAddresses->FirstUnicastAddress->Address.lpSockaddr, + pCurrAddresses->FirstUnicastAddress->Address.iSockaddrLength, + NULL, + address_buffer, + &length); + + char friendly_name_buffer[256]; + wcstombs (friendly_name_buffer, + pCurrAddresses->FriendlyName, + 256); + result[std::string (friendly_name_buffer)] = std::string (address_buffer); + } + pCurrAddresses = pCurrAddresses->Next; + } + } + + if (pAddresses) + FREE(pAddresses); + + return result; +} + +#endif + +BTS_EXPORT std::string default_host_ip () +{ + return host_ip (""); +} + +BTS_EXPORT std::map host_ips () +{ + static std::mutex cache_lock; + static std::map active_interfaces; + static bool first_time = true; + + std::lock_guard l (cache_lock); + if (!first_time) + return active_interfaces; + else + first_time = false; + +#ifdef _WIN32 + active_interfaces = get_windows_ips (); +#else + std::string result; + struct ifaddrs * ifAddrStruct=NULL; + struct ifaddrs * ifa=NULL; + void * tmpAddrPtr=NULL; + getifaddrs(&ifAddrStruct); + + for (ifa = ifAddrStruct; ifa != NULL; ifa = ifa->ifa_next) { + if (ifa->ifa_addr != NULL && ifa->ifa_addr->sa_family==AF_INET) { // check it is IP4 + // is a valid IP4 Address + tmpAddrPtr=&((struct sockaddr_in *)ifa->ifa_addr)->sin_addr; + char addressBuffer[INET_ADDRSTRLEN]; + inet_ntop(AF_INET, tmpAddrPtr, addressBuffer, INET_ADDRSTRLEN); + + if (ifa->ifa_flags & IFF_UP) { + std::string interface (ifa->ifa_name); + std::string address (addressBuffer); + + // LOG (Info,gen) << "Interface: " + // << interface << " is active and has ip " + // << address; + active_interfaces[interface] = address; + } + } + } + if (ifAddrStruct!=NULL) + freeifaddrs(ifAddrStruct); +#endif + return active_interfaces; +} + +void remove_loopback (std::map &interfaces) +{ + std::map::iterator i ; + for (i = interfaces.begin () ; i != interfaces.end ();i++) + if (i->second == "127.0.0.1") { + interfaces.erase (i->first); + return; + } +} + +BTS_EXPORT std::string host_ip (std::string const &src_interface_name) +{ + static std::mutex cache_lock; + static std::map interface_ips; + std::lock_guard l (cache_lock); + + std::map::const_iterator it = interface_ips.find (src_interface_name); + if (it != interface_ips.end ()) + return it->second; + + auto interface_name = src_interface_name; + + std::string default_interface; + std::string default_ip; + std::map ips = host_ips (); + std::vector interface_order; + +#ifdef _WIN32 + interface_order.push_back ("Local Area Connection 1"); +#else + interface_order.push_back ("eth0"); + interface_order.push_back ("br0"); + interface_order.push_back ("wlan0"); +#endif + + if (interface_name.size ()) { + default_interface = interface_name; + default_ip = ips[interface_name]; + } + else { + if (ips.empty ()) { + default_interface= ""; + default_ip = ""; + } + else if (ips.size () == 1) { + default_interface = ips.begin ()->first; + default_ip = ips.begin ()->second; + } else { + remove_loopback (ips); + if (ips.size () == 1) { + default_interface = ips.begin ()->first; + default_ip = ips.begin ()->second; + } else { + for (std::string &in : interface_order) + if (ips[in].size ()) { + default_interface = in; + default_ip = ips[in]; + break; + } + } + } + } + + if (default_ip.empty ()) { + + if (src_interface_name.empty ()) { + std::ostringstream msg; + std::pair p; + for (auto p : ips) + msg << p.first << " "; + // LOG (Fatal,gen) << "Cannot pick default interface. Too many options: " + // << msg.str (); + } + // else + // LOG (Fatal,gen) << "Could not find interface '" << interface_name; + bts_assert (false); + } + // else + // LOG (Info,gen) << "Selecting default interface: " << default_interface + // << " ip is " << default_ip; + + interface_ips[src_interface_name] = default_ip; + return default_ip; +} + +BTS_EXPORT void debug_break() +{ + DEBUG_BREAK(); +} + +BTS_EXPORT void set_thread_name(const char* name) +{ +#ifdef _WIN32 + if (!IsDebuggerPresent()) return; + + const DWORD MS_VC_EXCEPTION=0x406D1388; +#pragma pack(push,8) + struct tagTHREADNAME_INFO + { + DWORD dwType; // Must be 0x1000. + LPCSTR szName; // Pointer to name (in user addr space). + DWORD dwThreadID; // Thread ID (-1=caller thread). + DWORD dwFlags; // Reserved for future use, must be zero. + } info; +#pragma pack(pop) + + info.dwType = 0x1000; + info.szName = name; + info.dwThreadID = -1; + info.dwFlags = 0; + +#ifdef _MSC_VER + __try + { +#endif + RaiseException(MS_VC_EXCEPTION, 0, sizeof(info)/sizeof(ULONG_PTR), (ULONG_PTR*)&info); +#ifdef _MSC_VER + } + __except(EXCEPTION_EXECUTE_HANDLER) + { + } +#endif +#else + prctl(PR_SET_NAME, name, 0,0,0); +#endif +} + +BTS_EXPORT bool is_nan(double d) +{ +#ifdef _WIN32 + return (::_isnan(d) != 0); +#else + return (std::isnan(d) != 0); +#endif +} + +BTS_EXPORT bool is_nan(float f) +{ +#ifdef _WIN32 + return (::_isnan(f) != 0); +#else + return (std::isnan(f)); +#endif +} + +BTS_EXPORT double nan () +{ + return std::numeric_limits::quiet_NaN(); +} + +BTS_EXPORT float fnan () +{ + return std::numeric_limits::quiet_NaN(); +} + +BTS_EXPORT bool is_finite(double d) +{ +#ifdef _WIN32 + return (::_finite(d) != 0); +#else + return (::finite(d) != 0); +#endif +} + + + +// +// Interop functions +// +namespace interop +{ + +extern "C" BTS_EXPORT char* bts_data_path () +{ + std::string p = os::bts_data_path (); + char * result = new char [p.size ()+1]; + result[p.size ()] = '\0'; + std::copy (p.begin (),p.end (),result); + return result; +} + +extern "C" BTS_EXPORT void free_bts_data_path (char *dp) +{ + delete [] dp; +} + +} + +}} // end namespaces diff --git a/src/lib/os/os.h b/src/lib/os/os.h new file mode 100755 index 000000000..83850174c --- /dev/null +++ b/src/lib/os/os.h @@ -0,0 +1,86 @@ +#ifndef OS_H_ +#define OS_H_ + +#include "lib/utils/bts_export.h" +#include +#include +#include +#include +#include +namespace bts { +namespace os { + +#ifdef _WIN32 +#define ALIGNED_(x) __declspec(align(x)) +#else +#define ALIGNED_(x) __attribute__((aligned(x))) +#endif +#define CACHE_LINE 64 + +inline bool is_windows() { +#ifdef _WIN32 + return true; +#else + return false; +#endif +} +BTS_EXPORT char const* platform_name (); +BTS_EXPORT unsigned int process_id (); +BTS_EXPORT bool is_running(unsigned int pid); +struct ProcInfo +{ + ProcInfo ():pid (0), exe (), args () + {} + + int pid; + std::string exe; + std::vector args; +}; + +class BTS_EXPORT RunningProcessesQuerier +{ +public: + RunningProcessesQuerier(std::vector const&exe_name_hints); + std::vector query (); + ~RunningProcessesQuerier(); +private: + struct PImpl; + std::unique_ptr me; +}; +BTS_EXPORT std::vector running_processes (); +BTS_EXPORT std::string host_name (); +BTS_EXPORT std::string default_host_ip (); +BTS_EXPORT std::string host_ip (std::string const &interface_name); +BTS_EXPORT std::map host_ips (); +BTS_EXPORT uint64_t tick_count (); +BTS_EXPORT std::string get_application_path(); +BTS_EXPORT std::string get_install_path(); +BTS_EXPORT std::string get_application_name(); +BTS_EXPORT std::string get_cwd(); +BTS_EXPORT void gen_uuid (unsigned char out[16]); +BTS_EXPORT std::string& normalize_path(std::string& path, char separator='/'); +BTS_EXPORT std::string sanitize_file_name(std::string const& file_name); +BTS_EXPORT std::vector get_command_line(); +BTS_EXPORT std::string get_qualifier(); +extern "C" BTS_EXPORT int get_random_port(); +extern "C" BTS_EXPORT int num_cores(); + +BTS_EXPORT std::string bts_user_path(); +BTS_EXPORT std::string bts_system_path(); +BTS_EXPORT std::string bts_data_path(); +BTS_EXPORT std::string username(); + +BTS_EXPORT char* strncpy(char* dest, const char* src, size_t count); +BTS_EXPORT void setenv(const char* name, const char* value); + +BTS_EXPORT void debug_break(); +BTS_EXPORT void set_thread_name(const char* name); + +BTS_EXPORT bool is_nan(double d); +BTS_EXPORT bool is_nan(float f); +BTS_EXPORT double nan (); +BTS_EXPORT float fnan (); +BTS_EXPORT bool is_finite(double d); +}} + +#endif diff --git a/src/lib/os/tests/os_tests.cpp b/src/lib/os/tests/os_tests.cpp new file mode 100755 index 000000000..423abef7e --- /dev/null +++ b/src/lib/os/tests/os_tests.cpp @@ -0,0 +1,37 @@ +#include "core/bts_core/utest/utest.h" +#include "core/bts_core/utest/TempFile.h" + +#include "../os.h" + + +namespace bts { +namespace os { + + class FileNameSanitization : public TestSuite + { + public: + void empty_file_name_works () + { + test_equal (sanitize_file_name (""),""); + } + + void legal_file_name_passes_through () + { + test_equal (sanitize_file_name ("bts_shell"),"bts_shell"); + } + + void bad_file_name_is_fixed () + { + test_equal (sanitize_file_name ("bts_shell -configserver http://edge-qa/config/"), + "bts_shell -configserver http___edge-qa_config_"); + } + }; + +BEGIN_TEST_SUITE(FileNameSanitization) +ADD_TEST (bad_file_name_is_fixed) +ADD_TEST (legal_file_name_passes_through) +ADD_TEST (bad_file_name_is_fixed) +END_TEST_SUITE +}} + + diff --git a/src/lib/quickbus/core/Advertisement.h b/src/lib/quickbus/core/Advertisement.h new file mode 100644 index 000000000..071c1ae11 --- /dev/null +++ b/src/lib/quickbus/core/Advertisement.h @@ -0,0 +1,18 @@ +#pragma once + +#include "lib/quickbus/core/Topic.h" +#include + +namespace bts { +namespace qbus { + +class Advertisement +{ +public: + Advertisement() : topic(), maintain_synopsis() {} + Advertisement(Topic const&t,bool syn=true) : topic(t), maintain_synopsis(syn) {} + Topic topic; + bool maintain_synopsis; +}; + +}} diff --git a/src/lib/quickbus/core/LocalNode.cpp b/src/lib/quickbus/core/LocalNode.cpp new file mode 100644 index 000000000..0d95dd721 --- /dev/null +++ b/src/lib/quickbus/core/LocalNode.cpp @@ -0,0 +1,121 @@ +#include "LocalNode.h" + +#include "lib/quickbus/core/Quickbus.h" +#include "lib/quickbus/routing/Router.h" +#include "lib/quickbus/routing/NodeID.h" +#include "lib/logging/logging.h" + +namespace bts { +namespace qbus { + +struct LocalNode::PImpl +{ + PImpl(Quickbus &qb) : + my_id(make_node_id()),qb (qb), registered(false) + {} + NodeID my_id; + Quickbus &qb; + bool registered; +}; + +LocalNode::LocalNode (Quickbus &qb) : me (new PImpl(qb)) +{ + LOG(QBus,Diag,"LocalNode {} connected to router {} created", + me->my_id.value(), me->qb.router().id().value() ); +} + +LocalNode::~LocalNode () {} + +bool LocalNode::on_qb_thread() const{return me->qb.on_qb_thread();} + +void LocalNode::async_invoke(std::function const&action) + +{ + auto self = shared_from_this(); + me->qb.post([self,action] () { + LOG(QBus,Diag, "LocalNode {} async_invoke",self->me->my_id.value()) ; + action(); + }); +} + +void LocalNode::async_register() +{ + auto self = shared_from_this(); + me->qb.post([self] () { + LOG(QBus,Diag, "LocalNode {} registering " ,self->me->my_id.value() ); + bts_assert(!self->me->registered); + self->me->registered = true; + self->me->qb.router().add_neighbour(self->me->my_id,self); + }); +} + +std::future LocalNode::async_unregister() +{ + auto self = shared_from_this(); + auto p = std::make_shared> (); + me->qb.post([self,p] () { + + bts_assert(self->me->registered); + self->me->registered = false; + self->me->qb.router().remove_neighbour(self->me->my_id); + p->set_value(); + }); + return p->get_future(); +} + +void LocalNode::sync_advertise(TableID const&id,Advertisement const&adv) +{me->qb.router().advertise(me->my_id,id,adv);} + +void LocalNode::sync_unadvertise(TableID const&tid) +{me->qb.router().unadvertise(me->my_id,tid);} + +void LocalNode::sync_subscribe (TableID const&tid,SubscriptionID const&sid) +{me->qb.router().subscribe(me->my_id,tid,sid);} + +void LocalNode::sync_notify(TableID const&tid,SubscriptionID const&sid, Update const&u) +{me->qb.router().notify(me->my_id,tid,sid,u);} + +void LocalNode::sync_unsubscribe (TableID const&tid) +{me->qb.router().unsubscribe(me->my_id,tid);} + +void LocalNode::advertise(NodeID const& nid,TableID const&tid,Advertisement const&a) +{ + LOG(QBus,Diag,"LocalNode {} received advertisement from {} for {} (topic: {})", + me->my_id.value() , nid.value(), tid.value(), a.topic.value()); + + on_advertise(tid,a); +} + +void LocalNode::unadvertise(NodeID const& nid,TableID const&tid) +{ + LOG(QBus,Diag, "LocalNode {} received advertisement from {} for {}", + me->my_id.value(),nid.value(),tid.value()); + + on_unadvertise(tid); +} + +void LocalNode::subscribe (NodeID const&nid, TableID const&tid,SubscriptionID const&sid) +{ + LOG(QBus,Diag,"LocalNode {} received subscribe from {} for {} (sid: {})", + me->my_id.value() ,nid.value(), tid.value(),sid.value()); + + on_subscribe(tid,sid); +} + +void LocalNode::notify(NodeID const&nid,TableID const&tid,SubscriptionID const&sid, Update const&u) +{ + LOG(QBus,Diag,"Localnode {} received notify {} from {} for {} (sid: {})", + me->my_id.value() ,nid.value(), tid.value(),sid.value()); + + on_notify(tid,sid,u); +} + +void LocalNode::unsubscribe (NodeID const&nid, TableID const&tid) +{ + LOG(QBus,Diag,"LocalNode {} received unsubscribe from {} for {} ", + me->my_id.value(),nid.value(), tid.value()) ; + on_unsubscribe(tid); +} + +}} + diff --git a/src/lib/quickbus/core/LocalNode.h b/src/lib/quickbus/core/LocalNode.h new file mode 100644 index 000000000..c9c7b1395 --- /dev/null +++ b/src/lib/quickbus/core/LocalNode.h @@ -0,0 +1,58 @@ +#pragma once + +#include +#include + +#include "lib/quickbus/routing/INode.h" +#include "lib/utils/warn_ctrl.h" + +namespace bts {namespace qbus {class Quickbus;}} + +namespace bts { +namespace qbus { + +GCC_DIAG_OFF("-Wnon-virtual-dtor") +class LocalNode : public std::enable_shared_from_this, public INode +{ +public: + LocalNode(Quickbus &qb); + virtual ~LocalNode(); + + // wheter the current thread is the QB thread + bool on_qb_thread() const; + + // async + void async_invoke(std::function const&action); + void async_register(); + std::future async_unregister(); + + // sync + void sync_advertise(TableID const&id,Advertisement const&adv) ; + void sync_unadvertise(TableID const&tid) ; + void sync_subscribe (TableID const&tid,SubscriptionID const&sid) ; + void sync_notify(TableID const&,SubscriptionID const&, Update const&) ; + void sync_unsubscribe (TableID const&tid) ; + + // callbacks (invoked from qb thtread) + virtual void on_advertise(TableID const&,Advertisement const&) {}; + virtual void on_unadvertise(TableID const&) {}; + virtual void on_subscribe (TableID const&,SubscriptionID const&) {}; + virtual void on_notify(TableID const&,SubscriptionID const&, Update const&) {}; + virtual void on_unsubscribe (TableID const&) {}; + // end callbacks + +private: + // INode callbacks, invoked from the QB thread + void advertise(NodeID const& ,TableID const&,Advertisement const&) override ; + void unadvertise(NodeID const& ,TableID const&) override ; + void subscribe (NodeID const&, TableID const&,SubscriptionID const&) override ; + void notify(NodeID const&,TableID const&,SubscriptionID const&, Update const&) override ; + void unsubscribe (NodeID const&, TableID const&) override ; + // end INode + + struct PImpl; + std::unique_ptr me; +}; + +GCC_DIAG_RESTORE() +}} diff --git a/src/lib/quickbus/core/Quickbus.cpp b/src/lib/quickbus/core/Quickbus.cpp new file mode 100644 index 000000000..bdaf48049 --- /dev/null +++ b/src/lib/quickbus/core/Quickbus.cpp @@ -0,0 +1,55 @@ +#include "Quickbus.h" + +#include "lib/quickbus/routing/Router.h" +#include "lib/quickbus/network/QBConnectionManager.h" +#include "lib/config/protos/quickbus_cfg.pb.h" + +#include +#include + +namespace bts { +namespace qbus { + +struct Quickbus::PImpl +{ + PImpl(config::QuickbusNodeCfg const&cfg) : + io_service(), + router(make_node_id()), + connection_manager(io_service,router,cfg), + worker() + {} + + boost::asio::io_service io_service; + Router router; + QBConnectionManager connection_manager; + std::thread worker; +}; + +Quickbus::Quickbus () : + me (new PImpl(config::QuickbusNodeCfg())) +{init();} + +Quickbus::Quickbus (config::QuickbusNodeCfg const&cfg) : + me (new PImpl(cfg)) +{init();} + +Quickbus::~Quickbus () +{ + me->io_service.stop(); + me->worker.join(); +} + +bool Quickbus::on_qb_thread() const +{return std::this_thread::get_id() == me->worker.get_id();} + +void Quickbus::post (std::function const &action) +{me->io_service.post(action);} + +Router& Quickbus::router() const +{return me->router;} + +void Quickbus::init() +{me->worker = std::thread ([this] () {me->io_service.run();});} + +}} + diff --git a/src/lib/quickbus/core/Quickbus.h b/src/lib/quickbus/core/Quickbus.h new file mode 100644 index 000000000..3bca21858 --- /dev/null +++ b/src/lib/quickbus/core/Quickbus.h @@ -0,0 +1,51 @@ +#pragma once + +#include +#include "lib/quickbus/pubsub/TableFilter.h" +#include "lib/quickbus/core/Topic.h" +#include "lib/quickbus/pubsub/TableEvent.h" +#include "lib/quickbus/message_model/Update.h" + +#include "lib/utils/bts_export.h" + +// fwds +namespace bts { +namespace config {class QuickbusNodeCfg;} +namespace qbus {class TablePublisher; class PublisherImp;} +namespace qbus {class TableSubscriber; class SubscriberImp;} +namespace qbus {class Change;} +namespace qbus {class LocalNode;} +namespace qbus {class Router;} +} +// end fwds + +namespace bts { +namespace qbus { + +// This class is not intended for direct use, use Subscriber, +// Publisher, RPCClient and RPCServer instead. + +// The point of this class is to provide a facade for the core +// components of the Quickbus system. + +class BTS_EXPORT Quickbus +{ +public: + Quickbus(); + Quickbus(config::QuickbusNodeCfg const&cfg); + ~Quickbus(); + + bool on_qb_thread() const; + +protected: + friend class LocalNode; + void post (std::function const &); + Router& router() const; + +private: + struct PImpl; + std::unique_ptr me; + void init(); +}; + +}} diff --git a/src/lib/quickbus/core/SubscriptionID.cpp b/src/lib/quickbus/core/SubscriptionID.cpp new file mode 100644 index 000000000..33d7ad2c1 --- /dev/null +++ b/src/lib/quickbus/core/SubscriptionID.cpp @@ -0,0 +1,6 @@ +#include "SubscriptionID.h" + +namespace bts { +namespace qbus { + +}} diff --git a/src/lib/quickbus/core/SubscriptionID.h b/src/lib/quickbus/core/SubscriptionID.h new file mode 100644 index 000000000..edeedb837 --- /dev/null +++ b/src/lib/quickbus/core/SubscriptionID.h @@ -0,0 +1,12 @@ +#pragma once + +#include "lib/utils/ValueWrapper.h" +#include + +namespace bts { +namespace qbus { +class SubscriptionIDTag {}; +typedef utils::ValueWrapper SubscriptionID; +inline SubscriptionID null_subscription_id() {return SubscriptionID(0);} +inline SubscriptionID next_subscription_id(SubscriptionID value) {return SubscriptionID(value.value()+1);} +}} diff --git a/src/lib/quickbus/core/TableID.cpp b/src/lib/quickbus/core/TableID.cpp new file mode 100644 index 000000000..33826c69f --- /dev/null +++ b/src/lib/quickbus/core/TableID.cpp @@ -0,0 +1,11 @@ +#include "TableID.h" +#include "lib/utils/Uuid.h" + +namespace bts { +namespace qbus { + +TableID make_table_id() { + return TableID(utils::uuid()); +} + +}} diff --git a/src/lib/quickbus/core/TableID.h b/src/lib/quickbus/core/TableID.h new file mode 100644 index 000000000..555e5ebf9 --- /dev/null +++ b/src/lib/quickbus/core/TableID.h @@ -0,0 +1,22 @@ +#pragma once + +#include "lib/utils/ValueWrapper.h" +#include + +namespace bts { +namespace qbus { +class TableTag {}; +typedef utils::ValueWrapper TableID; +TableID make_table_id(); +}} + +// yeah, this is crazy, but that's how you do it. +namespace std { +template <> +struct hash +{ + std::size_t operator()(const bts::qbus::TableID& k) const { + return std::hash () (k.value()); + } +}; +} diff --git a/src/lib/quickbus/core/Topic.h b/src/lib/quickbus/core/Topic.h new file mode 100644 index 000000000..4ac12c179 --- /dev/null +++ b/src/lib/quickbus/core/Topic.h @@ -0,0 +1,10 @@ +#pragma once + +#include "lib/utils/ValueWrapper.h" +#include + +namespace bts { +namespace qbus { +class TopicTag {}; +typedef utils::ValueWrapper Topic; +}} diff --git a/src/lib/quickbus/message_model/Change.h b/src/lib/quickbus/message_model/Change.h new file mode 100644 index 000000000..b9de5a99b --- /dev/null +++ b/src/lib/quickbus/message_model/Change.h @@ -0,0 +1,30 @@ +#pragma once + +#include +#include "lib/quickbus/message_model/RecordKey.h" +#include "lib/quickbus/message_model/Record.h" + +namespace bts { +namespace qbus { + +class Change +{ +public: + enum Type { + REMOVE, + REPLACE, + CREATE + }; + + Type type; + RecordKey key; + RecordPtr value; + + Change(Change::Type t, RecordKey key, const RecordPtr &value) : + type(t),key(key),value(value) + {} + + Change() : type(),key(),value(){} +}; + +}} diff --git a/src/lib/quickbus/message_model/Record.h b/src/lib/quickbus/message_model/Record.h new file mode 100644 index 000000000..d047859d7 --- /dev/null +++ b/src/lib/quickbus/message_model/Record.h @@ -0,0 +1,10 @@ +#pragma once + +#include + +namespace bts { +namespace qbus { +class RecordTag {}; +typedef utils::ValueWrapper Record; +typedef std::shared_ptr RecordPtr; +}} diff --git a/src/lib/quickbus/message_model/RecordKey.h b/src/lib/quickbus/message_model/RecordKey.h new file mode 100644 index 000000000..79edca465 --- /dev/null +++ b/src/lib/quickbus/message_model/RecordKey.h @@ -0,0 +1,21 @@ +#pragma once + +#include "lib/utils/ValueWrapper.h" + +namespace bts { +namespace qbus { +class RecordKeyTag {}; +typedef utils::ValueWrapper RecordKey; + +}} + +// yeah, this is crazy, but that's how you do it. +namespace std { +template <> +struct hash +{ + std::size_t operator()(const bts::qbus::RecordKey& k) const { + return std::hash () (k.value()); + } +}; +} diff --git a/src/lib/quickbus/message_model/RecordValue.h b/src/lib/quickbus/message_model/RecordValue.h new file mode 100644 index 000000000..e69de29bb diff --git a/src/lib/quickbus/message_model/Update.cpp b/src/lib/quickbus/message_model/Update.cpp new file mode 100644 index 000000000..7c8d11547 --- /dev/null +++ b/src/lib/quickbus/message_model/Update.cpp @@ -0,0 +1,6 @@ +#include "Update.h" + +namespace bts { +namespace qbus { + +}} diff --git a/src/lib/quickbus/message_model/Update.h b/src/lib/quickbus/message_model/Update.h new file mode 100644 index 000000000..62b6b9b8d --- /dev/null +++ b/src/lib/quickbus/message_model/Update.h @@ -0,0 +1,10 @@ +#pragma once + +#include +#include +#include "lib/quickbus/message_model/Change.h" + +namespace bts { +namespace qbus { +typedef std::vector Update; +}} diff --git a/src/lib/quickbus/message_model/merge_updates.cpp b/src/lib/quickbus/message_model/merge_updates.cpp new file mode 100644 index 000000000..a2d010abc --- /dev/null +++ b/src/lib/quickbus/message_model/merge_updates.cpp @@ -0,0 +1,42 @@ +#include "merge_updates.h" +#include "lib/quickbus/network/quickbus_msgs.pb.h" + +namespace bts { +namespace qbus { + +bool update_synopsis(Update const&update, std::unordered_map *base) +{ + for (auto const &ch : update) + switch (ch.type) { + case Change::CREATE: + if (base->find(ch.key) != base->end()) + return false; + else + (*base)[ch.key] = ch.value; + break; + case Change::REPLACE: + if (base->find(ch.key) == base->end()) + return false; + else + (*base)[ch.key] = ch.value; + break; + case Change::REMOVE: + if (!base->erase(ch.key)) + return false; + break; + } + return true; +} + +void get_update_from_synopsis(std::unordered_map &synopsis, Update *out) +{ + out->clear(); + out->reserve(synopsis.size()); + + for (auto &kvp : synopsis) { + Change ch (Change::CREATE,kvp.first,kvp.second); + out->push_back(ch); + } +} + +}} diff --git a/src/lib/quickbus/message_model/merge_updates.h b/src/lib/quickbus/message_model/merge_updates.h new file mode 100644 index 000000000..e57c0b41f --- /dev/null +++ b/src/lib/quickbus/message_model/merge_updates.h @@ -0,0 +1,16 @@ +#pragma once + +#include "lib/quickbus/message_model/Update.h" +#include "lib/quickbus/message_model/RecordKey.h" +#include "lib/quickbus/message_model/Change.h" +#include + +namespace bts {namespace qbus {class Change;}} + +namespace bts { +namespace qbus { + +bool update_synopsis(Update const&update, std::unordered_map *synopsis); +void get_update_from_synopsis(std::unordered_map &synopsis, Update *out); + +}} diff --git a/src/lib/quickbus/network/QBConnection.cpp b/src/lib/quickbus/network/QBConnection.cpp new file mode 100644 index 000000000..3a780fe6d --- /dev/null +++ b/src/lib/quickbus/network/QBConnection.cpp @@ -0,0 +1,446 @@ +#include "QBConnection.h" + +#include "lib/os/os.h" +#include "lib/utils/PBUtils.h" +#include "lib/utils/datetime.h" +#include "lib/logging/logging.h" + +#include "lib/quickbus/routing/Router.h" +#include "lib/quickbus/network/endpoint_description.h" +#include "lib/quickbus/core/Advertisement.h" + +#include "lib/quickbus/network/quickbus_msgs.pb.h" +#include "lib/config/protos/quickbus_cfg.pb.h" + +#include +#include + +namespace bts { +namespace qbus { + +using boost::asio::ip::tcp; +using namespace boost; + +struct QBConnection::PImpl +{ + PImpl(tcp::socket &s,Router &router,config::QuickbusNodeCfg const&cfg) + : socket(std::move(s)), + router(router), + pending_send(), + write_buffer(), + write_buffer_size(), + read_buffer(), + read_buffer_size(), + has_intro(false), + intro(), + my_id(), + my_desc(local_endpoint_description(socket)), + remote_desc(remote_endpoint_description(socket)), + cfg(cfg), + last_received_ts(datetime::now()), + last_sent_ts(datetime::now()), + hb_timer(socket.get_io_service()) + { + write_buffer.reserve(100*1024); + read_buffer.reserve(100*1024); + } + + tcp::socket socket; + Router &router; + + msgs::QBMsg pending_send; + std::string write_buffer; + int32_t write_buffer_size; + + std::string read_buffer; + int32_t read_buffer_size; + + bool has_intro; + msgs::Intro intro; + NodeID my_id; + + const std::string my_desc; + const std::string remote_desc; + const config::QuickbusNodeCfg cfg; + + datetime::PTime last_received_ts; + datetime::PTime last_sent_ts; + boost::asio::deadline_timer hb_timer; +}; + +QBConnection::QBConnection (tcp::socket s,Router &router, config::QuickbusNodeCfg const&cfg) + : + me(new PImpl(s,router,cfg)) +{} + +QBConnection::~QBConnection() +{} + +void QBConnection::init() +{ + send_intro(); + recv_intro(); + + check_hbs(); +} + +bool QBConnection::connected() const +{return me->socket.is_open();} + +void QBConnection::send_intro() +{ + LOG(QBus,Info,"Sending intro from {} to {}", me->my_desc , me->remote_desc ); + + bts_assert(!is_send_in_flight()); + + auto &intro = *me->pending_send.add_op()->mutable_intro(); + intro.set_host_name(os::host_name()); + intro.set_app_cmd_line(boost::algorithm::join(os::get_command_line()," ")); + intro.set_node_id(me->router.id().value()); + intro.set_pid(os::process_id()); + + try_send_pending(); +} + +void QBConnection::recv_intro() +{ + recv_msg_header(); +} + +void QBConnection::check_hbs() +{ + auto self = shared_from_this(); + + me->hb_timer.expires_from_now(boost::posix_time::milliseconds(me->cfg.hb_period_msecs())); + me->hb_timer.async_wait([self,this](const boost::system::error_code& ec) { + + if (!ec && connected()) { + if (datetime::is_at_least_msecs_old (me->last_received_ts, + me->cfg.hb_period_msecs() * + me->cfg.max_missed_hbs())) + close_connection("No heartbeats received"); + else { + if (!is_send_in_flight() && + datetime::is_at_least_msecs_old (me->last_sent_ts, me->cfg.hb_period_msecs())) { + + LOG(QBus,Diag,"{0} sending HB to {}", me->my_desc ,me->remote_desc ); + me->pending_send.add_op()->mutable_heartbeat(); + try_send_pending(); + } + check_hbs(); + } + } + }); + +} + +void QBConnection::recv_msg_header() +{ + auto self = shared_from_this(); + + asio::async_read (me->socket, + asio::buffer (&me->read_buffer_size, sizeof(me->read_buffer_size)), + [this, self](system::error_code error, std::size_t ) { + if (!error) + recv_msg_body(); + else + close_connection(error.message()); + }); +} + +void QBConnection::recv_msg_body() +{ + auto self = shared_from_this(); + + me->read_buffer.resize(me->read_buffer_size); + asio::async_read (me->socket, + asio::buffer (&me->read_buffer[0],me->read_buffer_size), + [this, self](system::error_code error, std::size_t ) { + if (!error) { + msgs::QBMsg m; + auto parse_ok = m.ParseFromArray (&(me->read_buffer[0]), + me->read_buffer.size ()); + if (!parse_ok) + close_connection("Corrupt message"); + else + handle_msg(&m); + } + else + recv_msg_body(); + + }); +} + +void QBConnection::handle_msg(msgs::QBMsg *msgs) +{ + me->last_received_ts = datetime::now(); + for (auto const&op : msgs->op()) { + + if (op.op_case() == msgs::QBOp::kIntro) { + if (me->has_intro) { + close_connection("Received two intro messages"); + return; + } else { + LOG(QBus,Info, + "Received intro from {} for {} intro: {} connection is owned by node: {}" + " tcp connection is local {} to remote {}", + me->remote_desc ,me->my_desc , + utils::pb::to_string(op.intro()), me->router.id().value(), + me->my_desc, me->remote_desc ); + + me->has_intro = true; + me->intro=op.intro(); + me->my_id = NodeID(me->intro.node_id()); + + if (!me->router.add_neighbour(me->my_id,shared_from_this())) { + close_connection("Node is already connected"); + return; + } + } + break; + } + + if (!me->has_intro) { + close_connection("Expected intro"); + return; + } + + switch (op.op_case()) { + + case msgs::QBOp::kIntro: + bts_assert(false); + break; + + case msgs::QBOp::kNotification: { + LOG(QBus,Diag, + "{} received notification from {} for table {} sid: {} size: {}", + me->my_desc , me->remote_desc , op.notification().table_id(), + op.notification().subscription_id(), + op.notification().update().size()); + + + auto &source_update = op.notification().update(); + Update target_update; + target_update.resize(source_update.size()); + + for (int i = 0; i < source_update.size(); i++) { + auto &target_change = target_update[i]; + auto &source_change = op.notification().update(i); + + target_change.key = RecordKey(source_change.key()); + target_change.value = std::make_shared(std::move(source_change.value())); + + switch (source_change.type()) { + case msgs::Change::REMOVE: + target_change.type = Change::REMOVE; + break; + case msgs::Change::REPLACE: + target_change.type = Change::REPLACE; + break; + case msgs::Change::CREATE: + target_change.type = Change::CREATE; + break; + default: + bts_assert(false); + break; + } + } + me->router.notify(me->my_id, + TableID(op.notification().table_id()), + SubscriptionID(op.notification().subscription_id()),target_update); + break; + } + + case msgs::QBOp::kAdvertisement: { + LOG(QBus,Info, "{} received advertisement from {} adv: {}" , + me->my_desc , me->remote_desc , utils::pb::to_string(op.advertisement())); + + Advertisement ad(Topic(op.advertisement().topic()),op.advertisement().maintain_synopsis()); + me->router.advertise(me->my_id, TableID(op.advertisement().table_id()),ad); + break; + } + case msgs::QBOp::kUnadvertisement: + LOG(QBus,Info, "{} received unadvertisement from {} unadv: {}" , + me->my_desc , me->remote_desc , utils::pb::to_string(op.unadvertisement())); + + me->router.unadvertise(me->my_id, TableID(op.unadvertisement().table_id())); + break; + case msgs::QBOp::kSubscription: + LOG(QBus,Info, "{} received subscription from {} sub: {}" , + me->my_desc , me->remote_desc , utils::pb::to_string(op.subscription())); + + me->router.subscribe(me->my_id, TableID(op.subscription().table_id()), + SubscriptionID(op.subscription().subscription_id())); + break; + case msgs::QBOp::kUnsubscription: + LOG(QBus,Info, "{} received unsubscription from {} unsub: {}" , + me->my_desc , me->remote_desc , utils::pb::to_string(op.unsubscription())); + + me->router.unsubscribe(me->my_id, TableID(op.unsubscription().table_id())); + break; + case msgs::QBOp::kHeartbeat: + LOG(QBus,Diag,"{} received heartbeat from {}", me->my_desc ,me->remote_desc) ; + break; + case msgs::QBOp::OP_NOT_SET: + LOG(QBus,Error,"{} received corrupted message from {}", me->my_desc ,me->remote_desc) ; + close_connection("Corrupted message"); + return; + } // switch + } // for + recv_msg_header(); +} + +void QBConnection::advertise(NodeID const& sender, TableID const&id,Advertisement const&ad) +{ + bts_assert(me->router.id() == sender); + bts_assert(me->has_intro); + + auto &target_ad = *me->pending_send.add_op()->mutable_advertisement(); + target_ad.set_topic(ad.topic.value()); + target_ad.set_table_id(id.value()); + target_ad.set_maintain_synopsis(ad.maintain_synopsis); + + LOG(QBus,Info, "{} is sending an advertisement to {} ad: {}" , + me->my_desc , me->remote_desc,utils::pb::to_string(target_ad)); + + try_send_pending(); +} + +void QBConnection::notify(NodeID const&sender,TableID const&id,SubscriptionID const&sid, Update const&u) +{ + bts_assert(me->router.id() == sender); + bts_assert(me->has_intro); + + auto &target_notification = *me->pending_send.add_op()->mutable_notification(); + + target_notification.set_table_id(id.value()); + target_notification.set_subscription_id(sid.value()); + + auto &target_update = *target_notification.mutable_update(); + target_update.Reserve(u.size()); + + for (auto &ch : u) { + + auto *target_change = target_update.Add(); + target_change->set_key(ch.key.value()); + + switch(ch.type) { + case Change::REMOVE: + target_change->set_type(msgs::Change::REMOVE); + break; + case Change::REPLACE: + *target_change->mutable_value() = ch.value->value(); + target_change->set_type(msgs::Change::REPLACE); + break; + case Change::CREATE: + *target_change->mutable_value() = ch.value->value(); + target_change->set_type(msgs::Change::CREATE); + break; + } + } + + LOG(QBus,Diag,"{} is sending a notification to {} table id: {}", + me->my_desc , me->remote_desc , id.value() ) ; + try_send_pending(); +} + +void QBConnection::unadvertise(NodeID const&sender,TableID const&id) +{ + bts_assert(me->router.id() == sender); + bts_assert(me->has_intro); + + auto &target_unadvertise = *me->pending_send.add_op()->mutable_unadvertisement(); + target_unadvertise.set_table_id(id.value()); + + LOG(QBus,Info,"{} is sending a unadvertisement to {} table id: {}", + me->my_desc , me->remote_desc , id.value()) ; + + try_send_pending(); +} + +void QBConnection::subscribe(NodeID const&sender, TableID const&id,SubscriptionID const&sid) +{ + bts_assert(me->router.id() == sender); + bts_assert(me->has_intro); + + auto &target_subscribe = *me->pending_send.add_op()->mutable_subscription(); + + target_subscribe.set_table_id(id.value()); + target_subscribe.set_subscription_id(sid.value()); + + LOG(QBus,Info,"{} is sending a subscription to {} sub: {}", + me->my_desc , me->remote_desc , utils::pb::to_string(target_subscribe)); + + try_send_pending(); +} + +void QBConnection::unsubscribe (NodeID const&sender, TableID const&id) +{ + bts_assert(me->router.id() == sender); + bts_assert(me->has_intro); + + auto &target_unsubscribe = *me->pending_send.add_op()->mutable_unsubscription(); + target_unsubscribe.set_table_id(id.value()); + + LOG(QBus,Info,"{} is sending a unsubscription to {} table id: {}", + me->my_desc , me->remote_desc , id.value()); + + try_send_pending(); +} + +void QBConnection::close_connection(std::string const&reason) +{ + if (me->socket.is_open()) { + LOG(QBus,Warn, "{} is closing its connection to {} reason: {}" , + me->my_desc , me->remote_desc , reason); + me->socket.close(); + if (me->has_intro) + me->router.remove_neighbour(me->my_id); + } +} + +bool QBConnection::is_send_in_flight() const +{return me->write_buffer.size() != 0;} + +bool QBConnection::can_send_now() const +{return !is_send_in_flight() && me->pending_send.op_size();} + +void QBConnection::try_send_pending() +{ + if (can_send_now()) { + send(me->pending_send); + me->pending_send.Clear(); + } +} + +void QBConnection::send(msgs::QBMsg const&msg) +{ + auto self = shared_from_this(); + me->last_sent_ts = datetime::now(); + + msg.SerializeToString(&me->write_buffer); + bts_assert(me->write_buffer.size()); + me->write_buffer_size = me->write_buffer.size(); + + std::array to_send; + to_send[0] = asio::buffer (&(me->write_buffer_size), sizeof(me->write_buffer_size)); + to_send[1] = asio::buffer (&(me->write_buffer[0]), me->write_buffer.size()); + + LOG(QBus,Diag, + "Sending message from {} to {} size: {}", + me->my_desc , me->remote_desc , me->write_buffer_size); + + asio::async_write (me->socket, to_send, + [this, self](system::error_code error, std::size_t) { + if (!error) { + me->write_buffer.clear(); + try_send_pending(); + } + else + close_connection(error.message()); + }); +} + +}} + + diff --git a/src/lib/quickbus/network/QBConnection.h b/src/lib/quickbus/network/QBConnection.h new file mode 100644 index 000000000..52ca3d641 --- /dev/null +++ b/src/lib/quickbus/network/QBConnection.h @@ -0,0 +1,58 @@ +#pragma once + +#include "lib/quickbus/routing/INode.h" +#include "lib/utils/warn_ctrl.h" + +#include +#include +#include + +// fwds +namespace bts {namespace qbus {namespace msgs {class QBMsg;}}} +namespace bts {namespace qbus {class Router;}} +namespace bts {namespace config {class QuickbusNodeCfg;}} +// end fwds + +namespace bts { +namespace qbus { + +GCC_DIAG_OFF("-Wnon-virtual-dtor") + +class QBConnection : public std::enable_shared_from_this, public INode +{ +public: + QBConnection (boost::asio::ip::tcp::socket socket,Router &router,config::QuickbusNodeCfg const&); + void init(); + bool connected() const; + ~QBConnection(); + + // from INode + void advertise(NodeID const& sender, TableID const&id,Advertisement const&adv) override; + void unadvertise(NodeID const& sender, TableID const&tid) override; + void subscribe (NodeID const&sender, TableID const&tid,SubscriptionID const&sid) override; + void notify(NodeID const&nid,TableID const&tid,SubscriptionID const&sid,Update const&u) override; + void unsubscribe (NodeID const&sender, TableID const&tid) override; + // end INode + +private: + struct PImpl; + std::unique_ptr me; + + bool is_send_in_flight() const; + bool can_send_now() const; + + void try_send_pending(); + + void send_intro(); + void recv_intro(); + void check_hbs(); + void send(msgs::QBMsg const&m); + void recv_msg_header(); + void recv_msg_body(); + void handle_msg(msgs::QBMsg *m); + void close_connection(std::string const&reason); +}; + +GCC_DIAG_RESTORE() +}} + diff --git a/src/lib/quickbus/network/QBConnectionManager.cpp b/src/lib/quickbus/network/QBConnectionManager.cpp new file mode 100644 index 000000000..21c7a07bd --- /dev/null +++ b/src/lib/quickbus/network/QBConnectionManager.cpp @@ -0,0 +1,126 @@ +#include "QBConnectionManager.h" + +#include "lib/quickbus/routing/Router.h" +#include "lib/quickbus/network/QBConnection.h" +#include "lib/logging/logging.h" +#include "lib/quickbus/network/endpoint_description.h" + +#include "lib/config/protos/quickbus_cfg.pb.h" + +#include +#include +#include + +using boost::asio::ip::tcp; + +namespace bts { +namespace qbus { + +struct QBConnectionManager::PImpl +{ + PImpl(boost::asio::io_service &service, + Router &router, config::QuickbusNodeCfg const&cfg) : + router(router), + cfg(cfg), + io_service(service), + acceptor(io_service,tcp::endpoint(tcp::v4(),cfg.my_port())), + acceptor_socket(io_service), + connector_socket(io_service), + reconnect_timer(io_service), + parent_connection() + {} + + Router &router; + const config::QuickbusNodeCfg cfg; + boost::asio::io_service &io_service; + tcp::acceptor acceptor; + tcp::socket acceptor_socket; + tcp::socket connector_socket; + boost::asio::deadline_timer reconnect_timer; + std::shared_ptr parent_connection; +}; + +QBConnectionManager::QBConnectionManager (boost::asio::io_service &service, + Router &router, + config::QuickbusNodeCfg const&cfg): + me (new PImpl(service,router,cfg)) +{ + check_parent_connection(); + if (me->cfg.my_port()) + accept(); +} + +QBConnectionManager::~QBConnectionManager () +{ + LOG(QBus,Info, "Destroyed QBConnectionManager"); +} + +void QBConnectionManager::accept() +{ + LOG(QBus,Info,"Accepting new connections on port {}", me->cfg.my_port()); + + me->acceptor.async_accept( + me->acceptor_socket, + [this](const boost::system::error_code ec) { + if (!ec) { + LOG(QBus,Info,"{} accepted connection from {}", + local_endpoint_description(me->acceptor_socket), + remote_endpoint_description(me->acceptor_socket)) ; + + auto c = std::make_shared(std::move(me->acceptor_socket), me->router,me->cfg); + c->init(); + } + else + LOG(QBus,Info,"Accept failed on port {} msg: {} code: {}", + me->cfg.my_port(), ec.message(), ec.value()); + accept(); + }); +} + +void QBConnectionManager::connect() +{ + auto addr = boost::asio::ip::address_v4::from_string(me->cfg.parent_host()); + tcp::endpoint ep (addr,me->cfg.parent_port()); + auto desc = endpoint_description(ep); + + LOG(QBus,Info,"Initiating connection to {}", endpoint_description(ep)); + + auto &socket = me->connector_socket; + socket.async_connect(ep, + [this,desc] (const boost::system::error_code ec) { + if (!ec) { + LOG(QBus,Info,"{} connected to {}", + local_endpoint_description(me->connector_socket), + remote_endpoint_description(me->connector_socket)) ; + bts_assert (!me->parent_connection); + me->parent_connection = + std::make_shared (std::move(me->connector_socket),me->router,me->cfg); + me->parent_connection->init(); + } + else + LOG(QBus,Info,"Failed to connect to {} error: {} code:{}", + desc ,ec.message(),ec.value()); + }); +} + +void QBConnectionManager::check_parent_connection() +{ + if (!me->parent_connection || !me->parent_connection->connected()) + if (me->cfg.parent_host().size()) { + me->parent_connection.reset(); + connect(); + } + + me->reconnect_timer.expires_from_now(boost::posix_time::seconds(me->cfg.reconnect_period_secs())); + me->reconnect_timer.async_wait([this](const boost::system::error_code& ec) { + if (!ec) + check_parent_connection(); + else + LOG(QBus,Error,"Reconnect timer failure. error: {} code: {}", + ec.message(), ec.value()); + }); +} + + + +}} diff --git a/src/lib/quickbus/network/QBConnectionManager.h b/src/lib/quickbus/network/QBConnectionManager.h new file mode 100644 index 000000000..a158ebe4a --- /dev/null +++ b/src/lib/quickbus/network/QBConnectionManager.h @@ -0,0 +1,33 @@ +#pragma once + +#include +#include +#include + +// fwds +namespace bts { +namespace qbus {class Router;} +namespace config {class QuickbusNodeCfg;}} +namespace boost {namespace asio {class io_service;}} +// end fwds + +namespace bts { +namespace qbus { + +class INode; + +class QBConnectionManager +{ +public: + QBConnectionManager (boost::asio::io_service &, bts::qbus::Router &,config::QuickbusNodeCfg const&); + ~QBConnectionManager(); +private: + struct PImpl; + std::unique_ptr me; + void accept(); + void connect(); + void check_parent_connection(); + +}; + +}} diff --git a/src/lib/quickbus/network/endpoint_description.cpp b/src/lib/quickbus/network/endpoint_description.cpp new file mode 100644 index 000000000..dcb7e82fc --- /dev/null +++ b/src/lib/quickbus/network/endpoint_description.cpp @@ -0,0 +1,36 @@ +#include "endpoint_description.h" + +#include +#include + +namespace bts { +namespace qbus { + +std::string endpoint_description(boost::asio::ip::tcp::endpoint const&ep) +{ + try { + return (boost::format ("%s:%d") % ep.address ().to_string () % ep.port()).str(); + } catch (boost::system::system_error &) { + return ""; + } +} + +std::string local_endpoint_description(boost::asio::ip::tcp::socket const&s) +{ + try { + return endpoint_description(s.local_endpoint()); + } catch (boost::system::system_error &) { + return ""; + } +} + +std::string remote_endpoint_description(boost::asio::ip::tcp::socket const&s) +{ + try { + return endpoint_description(s.remote_endpoint()); + } catch (boost::system::system_error &) { + return ""; + } +} + +}} diff --git a/src/lib/quickbus/network/endpoint_description.h b/src/lib/quickbus/network/endpoint_description.h new file mode 100644 index 000000000..2ca176c94 --- /dev/null +++ b/src/lib/quickbus/network/endpoint_description.h @@ -0,0 +1,10 @@ +#pragma once + +#include + +namespace bts { +namespace qbus { +std::string endpoint_description(boost::asio::ip::tcp::endpoint const&ep); +std::string local_endpoint_description(boost::asio::ip::tcp::socket const&s); +std::string remote_endpoint_description(boost::asio::ip::tcp::socket const&s); +}} diff --git a/src/lib/quickbus/network/quickbus_msgs.proto b/src/lib/quickbus/network/quickbus_msgs.proto new file mode 100644 index 000000000..d3d84cffd --- /dev/null +++ b/src/lib/quickbus/network/quickbus_msgs.proto @@ -0,0 +1,89 @@ +syntax='proto3'; +package bts.qbus.msgs; + +message Intro { + string host_name =1; + uint32 pid =2; + string app_cmd_line = 3; + string node_id = 4; +} + +message Change { + enum Type { + REMOVE = 0; + REPLACE= 1; + CREATE = 2; + } + + Change.Type type = 1; + uint32 key = 2; + bytes value = 3; +} + +message Notification { + string table_id = 1; + uint32 subscription_id = 2; + repeated Change update = 4; +} + +message Advertisement +{ + string topic = 1; + string table_id = 2; + bool maintain_synopsis = 3; +} + +message Unadvertisement +{ + string table_id = 1; +} + +message Subscription +{ + string table_id = 1; + uint32 subscription_id = 2; +} + +message Unsubscription +{ + string table_id = 1; +} + +message Heartbeat {} + +message QBOp +{ + oneof op { + Intro intro = 1; + Notification notification = 2; + Advertisement advertisement = 3; + Unadvertisement unadvertisement = 4; + Subscription subscription = 5; + Unsubscription unsubscription = 6; + Heartbeat heartbeat = 7; + } +} + +message QBMsg { + repeated QBOp op = 1; +} + +// used for testint Table{Publisher|Subscriber} +message TestRecord { + int32 key = 1; + string value = 2; +} + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/lib/quickbus/pubsub/TableEvent.h b/src/lib/quickbus/pubsub/TableEvent.h new file mode 100644 index 000000000..0de84cc16 --- /dev/null +++ b/src/lib/quickbus/pubsub/TableEvent.h @@ -0,0 +1,27 @@ +#pragma once +#include + +#include "lib/quickbus/core/TableID.h" +#include "lib/quickbus/core/Topic.h" +#include "lib/quickbus/message_model/Update.h" + +namespace bts {namespace qbus {class Advertisement;}} + +namespace bts { +namespace qbus { + +struct TableEvent { + enum EventType { + TABLE_CREATED, // advertisement != null, update == null + TABLE_UPDATED, // advertisement == null, update != null + TABLE_DESTROYED // advertisement == null, update == null + }; + + const EventType event_type; + const TableID &tid; + const Advertisement * const advertisement; + const Update *const update; +}; + +typedef std::function OnTableEvent; +}} diff --git a/src/lib/quickbus/pubsub/TableFilter.h b/src/lib/quickbus/pubsub/TableFilter.h new file mode 100644 index 000000000..869f9046f --- /dev/null +++ b/src/lib/quickbus/pubsub/TableFilter.h @@ -0,0 +1,18 @@ +#pragma once +#include "lib/quickbus/core/Advertisement.h" + +#include +#include + + +namespace bts { +namespace qbus { + +typedef std::function TableFilter; + +inline TableFilter equal_topic(Topic const&t) { + auto p = [t] (Advertisement const&ad) {return ad.topic == t;}; + return TableFilter(p); +} + +}} diff --git a/src/lib/quickbus/pubsub/TablePublisher.cpp b/src/lib/quickbus/pubsub/TablePublisher.cpp new file mode 100644 index 000000000..c78b12e81 --- /dev/null +++ b/src/lib/quickbus/pubsub/TablePublisher.cpp @@ -0,0 +1,138 @@ +#include "TablePublisher.h" + +#include "lib/quickbus/core/Advertisement.h" +#include "lib/quickbus/core/LocalNode.h" +#include "lib/quickbus/core/TableID.h" +#include "lib/quickbus/core/SubscriptionID.h" +#include "lib/quickbus/message_model/merge_updates.h" +#include "lib/utils/bts_assert.h" + +#include + +namespace bts { +namespace qbus { + +struct TablePublisher::PublisherImp : LocalNode +{ + PublisherImp(Quickbus &qb,Topic const &t) + : + LocalNode(qb), + my_table_id(make_table_id()), + topic(t), + sub_id(null_subscription_id()), + synopsis(), + has_synopsis(false) + {} + void start() + { + async_register(); + + async_invoke([this] () { + Advertisement ad; + ad.topic = topic; + ad.maintain_synopsis = true; + sync_advertise(my_table_id,ad); + }); + } + + void stop () + {async_unregister();} + + void publish(std::shared_ptr const &changes) + { + async_invoke([this,changes] () { + has_synopsis = true; + update_synopsis(*changes,&synopsis); + if (sub_id != null_subscription_id()) + sync_notify(my_table_id,sub_id,*changes); + }); + } + + void on_subscribe (TableID const&tid, SubscriptionID const&sid) override + { + bts_assert (on_qb_thread()); + + bts_assert(tid == my_table_id); + bts_assert(sid != null_subscription_id()); + + // we do this asynchrously because we want to avoid reentry into the router + async_invoke([this,sid] () { + bts_assert(sub_id == null_subscription_id()); + sub_id = sid; + + if (has_synopsis) { + Update update; + get_update_from_synopsis(synopsis,&update); + sync_notify(my_table_id,sub_id,update); + } + }); + } + + void on_unsubscribe (TableID const&tid) override + { + bts_assert (on_qb_thread()); + bts_assert(tid == my_table_id); + + // since subscribe is asynchronous this must be as + // well. Otherwise the subscribe call could be handled out of + // order with unsubscribe. + async_invoke([this] () { + bts_assert(sub_id != null_subscription_id()); + sub_id = null_subscription_id(); + }); + } + + const TableID my_table_id; + const Topic topic; + SubscriptionID sub_id; + std::unordered_map synopsis; + bool has_synopsis; +}; + +TablePublisher::TablePublisher () : pub_imp() {} +TablePublisher::TablePublisher (TablePublisher &&src) : pub_imp (std::move(src.pub_imp)) {} +TablePublisher::TablePublisher(Quickbus &qb,Topic const &t) : + pub_imp (new PublisherImp(qb,t)) +{pub_imp->start();} + +TablePublisher::~TablePublisher () {disconnect();} + +void TablePublisher::disconnect() +{ + if (!is_null()) { + pub_imp->stop(); + pub_imp.reset(); + } +} + +#ifndef WIN32 +void TablePublisher::publish (std::initializer_list const&changes) +{publish(std::vector (changes));} +#endif + +void TablePublisher::publish (std::vector const &changes) +{ + bts_assert (!is_null()); + auto v = std::make_shared> (changes); + pub_imp->publish(v); +} + +void TablePublisher::publish (std::vector &&changes) +{ + bts_assert (!is_null()); + auto v = std::make_shared> (std::move(changes)); + pub_imp->publish(v); +} + +TablePublisher&TablePublisher::operator=(TablePublisher &&src) +{ + disconnect(); + pub_imp = std::move(src.pub_imp); + return *this; +} + +bool TablePublisher::is_null() const +{return !pub_imp;} + +}} + diff --git a/src/lib/quickbus/pubsub/TablePublisher.h b/src/lib/quickbus/pubsub/TablePublisher.h new file mode 100644 index 000000000..4f51fc542 --- /dev/null +++ b/src/lib/quickbus/pubsub/TablePublisher.h @@ -0,0 +1,41 @@ +#pragma once + +#include +#include + +#include "lib/quickbus/core/Topic.h" +#include "lib/utils/bts_export.h" + +namespace bts {namespace qbus {class Quickbus; class Change;}} + +namespace bts { +namespace qbus { + +// Allows publishing tables to the Quickbus middleware. + +// This class is not thread safe. + +class BTS_EXPORT TablePublisher +{ +public: + TablePublisher (); + TablePublisher (TablePublisher &&src); + TablePublisher(Quickbus &qb,Topic const&topic); + ~TablePublisher(); + TablePublisher&operator=(TablePublisher &&src); + bool is_null() const; + +#ifndef WIN32 + void publish (std::initializer_list const&changes); +#endif + + void publish (std::vector const &changes); + void publish (std::vector &&changes); + + void disconnect(); +private: + struct PublisherImp; + std::shared_ptr pub_imp; +}; + +}} diff --git a/src/lib/quickbus/pubsub/TableSubscriber.cpp b/src/lib/quickbus/pubsub/TableSubscriber.cpp new file mode 100644 index 000000000..e9170fbe3 --- /dev/null +++ b/src/lib/quickbus/pubsub/TableSubscriber.cpp @@ -0,0 +1,147 @@ +#include "TableSubscriber.h" + +#include "lib/quickbus/core/Advertisement.h" +#include "lib/quickbus/core/LocalNode.h" +#include "lib/quickbus/routing/NodeID.h" +#include "lib/quickbus/core/SubscriptionID.h" +#include "lib/utils/bts_assert.h" + +#include +#include +#include + +namespace bts { +namespace qbus { + +struct TableSubscriber::SubscriberImp : LocalNode +{ + SubscriberImp(Quickbus &qb,TableFilter const&f,OnTableEvent const&cb,bool single_t) : + LocalNode(qb), + my_node_id(make_node_id()), + filter(f), + single_table(single_t), + on_event(cb), + tables(), + subscriptions(), + enabled(true), + sub_id(next_subscription_id(null_subscription_id())) + {} + + void start() + { + async_register(); + } + + void stop() + { + enabled = false; + + if (on_qb_thread()) + async_unregister(); + else + async_unregister().wait(); + } + + void on_advertise(TableID const&tid,Advertisement const&ad) override + { + bts_assert (on_qb_thread()); + + if (enabled && table_relevant(ad)) { + + bts_assert(tables.count(tid) == 0); + tables[tid] = ad; + + if (!single_table || subscriptions.empty()) { + subscriptions.insert(tid); + + async_invoke([this,tid] {sync_subscribe(tid,sub_id);}); + + TableEvent e = {TableEvent::TABLE_CREATED,tid,&ad,nullptr}; + on_event(e); + } + } + } + + void on_unadvertise(TableID const&tid) override + { + bts_assert (on_qb_thread()); + + if (enabled && tables.count(tid)) { + + tables.erase(tid); + + if (subscriptions.erase(tid)) { + + TableEvent e = {TableEvent::TABLE_DESTROYED,tid,nullptr,nullptr}; + on_event(e); + + // the _on_event() callback above might have resulted in + // stop_notifications(), so we need to check _enabled + // again + if (enabled && single_table && tables.size()) { + auto const& replacement = tables.begin(); + auto replacement_tid = replacement->first; + subscriptions.insert(replacement_tid); + + async_invoke([this,replacement_tid] {sync_subscribe(replacement_tid,sub_id);}); + + TableEvent e = {TableEvent::TABLE_CREATED,replacement_tid,&replacement->second, nullptr}; + on_event(e); + } + } + } + } + + void notify(NodeID const&,TableID const&tid,SubscriptionID const&,Update const&update) override + { + bts_assert (on_qb_thread()); + + if (enabled) { + bts_assert_d(subscriptions.count(tid)); + TableEvent e = {TableEvent::TABLE_UPDATED,tid,nullptr,&update}; + on_event(e); + } + } + + bool table_relevant(Advertisement const&ad) const {return filter(ad);} + + const NodeID my_node_id; + const TableFilter filter; + const bool single_table; + const OnTableEvent on_event; + + std::map tables; + std::set subscriptions; + std::atomic enabled; + SubscriptionID sub_id; +}; + +TableSubscriber::TableSubscriber () : sub_imp () {} +TableSubscriber::TableSubscriber(TableSubscriber &&src) : sub_imp (std::move(src.sub_imp)) {} + +TableSubscriber::TableSubscriber(Quickbus &qb,TableFilter const&f,OnTableEvent const&cb,bool single_table) : + sub_imp (new SubscriberImp(qb,f,cb,single_table)) +{sub_imp->start();} + +TableSubscriber::~TableSubscriber () {disconnect();} + +void TableSubscriber::disconnect() +{ + if (!is_null()) { + sub_imp->stop(); + sub_imp.reset(); + } +} + +TableSubscriber& TableSubscriber::operator=(TableSubscriber&&src) +{ + disconnect(); + sub_imp = std::move(src.sub_imp); + return *this; +} + +bool TableSubscriber::is_null() const +{return !sub_imp;} + +}} + diff --git a/src/lib/quickbus/pubsub/TableSubscriber.h b/src/lib/quickbus/pubsub/TableSubscriber.h new file mode 100644 index 000000000..1adb842fe --- /dev/null +++ b/src/lib/quickbus/pubsub/TableSubscriber.h @@ -0,0 +1,39 @@ +#pragma once + +#include +#include +#include "lib/quickbus/pubsub/TableEvent.h" +#include "lib/quickbus/pubsub/TableFilter.h" + +#include "lib/utils/bts_export.h" + +// fwds +namespace bts {namespace qbus {class Quickbus;}} +// fwds + +namespace bts { +namespace qbus { + +// Supports subscribing to Quickbus tables. + +// This class is not thread safe. + +class BTS_EXPORT TableSubscriber +{ +public: + TableSubscriber(); + TableSubscriber(TableSubscriber &&src); + TableSubscriber(Quickbus &qb,TableFilter const&filter, + OnTableEvent const&cb,bool single_table=false); + ~TableSubscriber(); + TableSubscriber &operator=(TableSubscriber &&src); + bool is_null() const; + + void disconnect(); + +private: + struct SubscriberImp; + std::shared_ptr sub_imp; +}; + +}} diff --git a/src/lib/quickbus/pubsub/tests/qb_pubsub_tests.cpp b/src/lib/quickbus/pubsub/tests/qb_pubsub_tests.cpp new file mode 100644 index 000000000..7a742bb3c --- /dev/null +++ b/src/lib/quickbus/pubsub/tests/qb_pubsub_tests.cpp @@ -0,0 +1,231 @@ +#include "catch/catch.h" +#include "lib/quickbus/quickbus.h" +#include "lib/os/os.h" +#include "lib/utils/bts_chrono.h" + +#include +#include + +namespace bts { +namespace qbus { + + +TEST_CASE("Quickbus Pubsub Tests","[quickbus]") +{ + auto parent_port = os::get_random_port(); + + SECTION("smoke") + { + config::QuickbusNodeCfg pcfg; + pcfg.set_my_port (parent_port); + Quickbus parent(pcfg); + + config::QuickbusNodeCfg ccfg; + ccfg.set_parent_host("127.0.0.1"); + ccfg.set_reconnect_period_secs(1); + ccfg.set_parent_port (parent_port); + Quickbus child(ccfg); + + TablePublisher pub (parent,Topic("A")); + std::promise p; + auto f = p.get_future(); + TableSubscriber sub (child,equal_topic(Topic("A")),[&p] (TableEvent const&ev) { + if (ev.event_type == TableEvent::TABLE_UPDATED) { + REQUIRE(ev.update); + REQUIRE(ev.update->size() == 1); + auto change = *ev.update->begin(); + REQUIRE(change.type == Change::CREATE); + REQUIRE(change.key == RecordKey(1)); + REQUIRE(*change.value == Record("B")); + p.set_value(); + }}); + + pub.publish({{Change::CREATE,RecordKey(1),std::make_shared ("B")}}); + REQUIRE(f.wait_for(std::chrono::seconds(15)) == std::future_status::ready); + } + + SECTION("quickbus_late_join_works") + { + config::QuickbusNodeCfg pcfg; + pcfg.set_my_port (parent_port); + Quickbus parent(pcfg); + + config::QuickbusNodeCfg ccfg; + ccfg.set_parent_host("127.0.0.1"); + ccfg.set_reconnect_period_secs(1); + ccfg.set_parent_port (parent_port); + Quickbus child(ccfg); + + TablePublisher pub (parent,Topic("A")); + std::promise p; + auto f = p.get_future(); + + TableSubscriber sub (child,equal_topic(Topic("A")),[&p] (TableEvent const&ev) { + if (ev.event_type == TableEvent::TABLE_UPDATED) { + p.set_value(); + }}); + + pub.publish({{Change::CREATE,RecordKey(1),std::make_shared ("B")}}); + REQUIRE(f.wait_for(std::chrono::seconds(15)) == std::future_status::ready); + + Quickbus child2(ccfg); + std::promise p2; + auto f2 = p2.get_future(); + TableSubscriber sub2 (child2,equal_topic(Topic("A")),[&p2] (TableEvent const&ev) { + if (ev.event_type == TableEvent::TABLE_UPDATED) { + p2.set_value(); + }}); + REQUIRE(f2.wait_for(std::chrono::seconds(15)) == std::future_status::ready); + } + + SECTION("subscriber_late_join_work") + { + config::QuickbusNodeCfg pcfg; + pcfg.set_my_port (parent_port); + Quickbus parent(pcfg); + + config::QuickbusNodeCfg ccfg; + ccfg.set_parent_host("127.0.0.1"); + ccfg.set_reconnect_period_secs(1); + ccfg.set_parent_port (parent_port); + Quickbus child(ccfg); + + TablePublisher pub (parent,Topic("A")); + std::promise p; + auto f = p.get_future(); + TableSubscriber sub (child,equal_topic(Topic("A")),[&p] (TableEvent const&ev) { + if (ev.event_type == TableEvent::TABLE_UPDATED) { + p.set_value(); + }}); + + pub.publish({{Change::CREATE,RecordKey(1),std::make_shared ("B")}}); + REQUIRE(f.wait_for(std::chrono::seconds(15)) == std::future_status::ready); + + std::promise p2; + auto f2 = p2.get_future(); + TableSubscriber sub2 (child,equal_topic(Topic("A")),[&p2] (TableEvent const&ev) { + if (ev.event_type == TableEvent::TABLE_UPDATED) { + p2.set_value(); + }}); + REQUIRE(f2.wait_for(std::chrono::seconds(15)) == std::future_status::ready); + } + + SECTION("filtering_work") + { + config::QuickbusNodeCfg pcfg; + pcfg.set_my_port (parent_port); + Quickbus parent(pcfg); + + config::QuickbusNodeCfg ccfg; + ccfg.set_parent_host("127.0.0.1"); + ccfg.set_reconnect_period_secs(1); + ccfg.set_parent_port (parent_port); + Quickbus child(ccfg); + + TablePublisher pub (parent,Topic("A")); + TableSubscriber sub1 (child,equal_topic(Topic("B")),[](TableEvent const&) {REQUIRE(false);}); + + pub.publish({{Change::CREATE,RecordKey(1),std::make_shared ("B")}}); + std::promise p; + auto f = p.get_future(); + TableSubscriber sub2 (child,equal_topic(Topic("A")),[&p] (TableEvent const&ev) { + if (ev.event_type == TableEvent::TABLE_UPDATED) + p.set_value(); + }); + + std::this_thread::sleep_for(2s); + REQUIRE(f.wait_for(std::chrono::seconds(15)) == std::future_status::ready); + } + + SECTION("unsubscribe_work") + { + config::QuickbusNodeCfg pcfg; + pcfg.set_my_port (parent_port); + Quickbus parent(pcfg); + + config::QuickbusNodeCfg ccfg; + ccfg.set_parent_host("127.0.0.1"); + ccfg.set_reconnect_period_secs(1); + ccfg.set_parent_port (parent_port); + Quickbus child(ccfg); + + TablePublisher pub (parent,Topic("A")); + std::promise p; + auto f = p.get_future(); + TableSubscriber sub (child,equal_topic(Topic("A")),[&p] (TableEvent const&ev) { + if (ev.event_type == TableEvent::TABLE_UPDATED) { + REQUIRE(ev.update); + REQUIRE(ev.update->size() == 1); + auto change = *ev.update->begin(); + REQUIRE(change.type == Change::CREATE); + REQUIRE(change.key == RecordKey(1)); + REQUIRE(*change.value == Record("B")); + p.set_value(); + }}); + + pub.publish({{Change::CREATE,RecordKey(1),std::make_shared ("B")}}); + + REQUIRE(f.wait_for(std::chrono::seconds(15)) == std::future_status::ready); + sub.disconnect(); + + pub.publish({{Change::REPLACE,RecordKey(1),std::make_shared ("C")}}); + std::this_thread::sleep_for(2s); + } + + SECTION("force_singleton_topics_work") + { + Quickbus qb{config::QuickbusNodeCfg()}; + + TablePublisher pub1 (qb,Topic("A")); + TablePublisher pub2 (qb,Topic("A")); + + std::promise p1; + bool sub1_got_a = false; + bool sub1_got_b = false; + auto f1 = p1.get_future(); + TableSubscriber sub1 (qb,equal_topic(Topic("A")),[&p1,&sub1_got_b,&sub1_got_a] (TableEvent const&ev) { + if (ev.event_type == TableEvent::TABLE_UPDATED) { + REQUIRE(ev.update); + REQUIRE(ev.update->size() == 1); + auto change = *ev.update->begin(); + REQUIRE(change.type == Change::CREATE); + REQUIRE(change.key == RecordKey(1)); + if(*change.value == Record("B")) + sub1_got_b = true; + if(*change.value == Record("A")) + sub1_got_a = true; + if(sub1_got_a && sub1_got_b) + p1.set_value(); + }}); + + std::promise p2; + bool sub2_got_a = false; + bool sub2_got_b = false; + auto f2 = p2.get_future(); + TableSubscriber sub2 (qb,equal_topic(Topic("A")),[&p2,&sub2_got_b,&sub2_got_a] (TableEvent const&ev) { + if (ev.event_type == TableEvent::TABLE_UPDATED) { + REQUIRE(ev.update); + REQUIRE(ev.update->size() == 1); + auto change = *ev.update->begin(); + REQUIRE(change.type == Change::CREATE); + REQUIRE(change.key == RecordKey(1)); + if(*change.value == Record("B")) + sub2_got_b = true; + if(*change.value == Record("A")) + sub2_got_a = true; + }},true); + + pub1.publish({{Change::CREATE,RecordKey(1),std::make_shared ("B")}}); + pub2.publish({{Change::CREATE,RecordKey(1),std::make_shared ("A")}}); + + REQUIRE(f1.wait_for(std::chrono::seconds(15)) == std::future_status::ready); + std::this_thread::sleep_for(2s); + + REQUIRE(sub1_got_a); + REQUIRE(sub1_got_b); + REQUIRE(sub2_got_a != sub2_got_b); + } +} + +}} + diff --git a/src/lib/quickbus/quickbus.h b/src/lib/quickbus/quickbus.h new file mode 100644 index 000000000..344417866 --- /dev/null +++ b/src/lib/quickbus/quickbus.h @@ -0,0 +1,11 @@ +#pragma once + +#include "lib/quickbus/core/Quickbus.h" +#include "lib/quickbus/pubsub/TableEvent.h" +#include "lib/quickbus/core/Topic.h" +#include "lib/quickbus/pubsub/TablePublisher.h" +#include "lib/quickbus/pubsub/TableSubscriber.h" +#include "lib/quickbus/rpc/RPCClient.h" +#include "lib/quickbus/rpc/RPCServer.h" + +#include "lib/config/protos/quickbus_cfg.pb.h" diff --git a/src/lib/quickbus/routing/INode.cpp b/src/lib/quickbus/routing/INode.cpp new file mode 100644 index 000000000..2c35a280b --- /dev/null +++ b/src/lib/quickbus/routing/INode.cpp @@ -0,0 +1,8 @@ +#include "lib/quickbus/routing/INode.h" + +namespace bts { +namespace qbus { + +INode::~INode() {} + +}} diff --git a/src/lib/quickbus/routing/INode.h b/src/lib/quickbus/routing/INode.h new file mode 100644 index 000000000..8b1c2fa35 --- /dev/null +++ b/src/lib/quickbus/routing/INode.h @@ -0,0 +1,27 @@ +#pragma once + +#include "lib/quickbus/routing/NodeID.h" +#include "lib/quickbus/core/TableID.h" +#include "lib/quickbus/core/Topic.h" +#include "lib/quickbus/core/SubscriptionID.h" +#include "lib/quickbus/message_model/Update.h" + +namespace bts {namespace qbus {class Advertisement;}} + +namespace bts { +namespace qbus { + +class INode +{ +public: + virtual void advertise(NodeID const& sender,TableID const&id,Advertisement const&adv) = 0; + virtual void unadvertise(NodeID const& sender,TableID const&tid) = 0; + + virtual void subscribe (NodeID const&sender, TableID const&tid,SubscriptionID const&sid) = 0; + virtual void notify(NodeID const&,TableID const&,SubscriptionID const&, Update const&) = 0; + virtual void unsubscribe (NodeID const&sender, TableID const&tid) = 0; + + virtual ~INode(); +}; + +}} diff --git a/src/lib/quickbus/routing/INodePtr.h b/src/lib/quickbus/routing/INodePtr.h new file mode 100644 index 000000000..6428c95b4 --- /dev/null +++ b/src/lib/quickbus/routing/INodePtr.h @@ -0,0 +1,9 @@ +#pragma once + +#include + +namespace bts { +namespace qbus { +class INode; +typedef std::shared_ptr INodePtr; +}} diff --git a/src/lib/quickbus/routing/NodeID.cpp b/src/lib/quickbus/routing/NodeID.cpp new file mode 100644 index 000000000..fc83fe2b0 --- /dev/null +++ b/src/lib/quickbus/routing/NodeID.cpp @@ -0,0 +1,11 @@ +#include "NodeID.h" +#include "lib/utils/Uuid.h" + +namespace bts { +namespace qbus { + +NodeID make_node_id() { + return NodeID(utils::uuid()); +} + +}} diff --git a/src/lib/quickbus/routing/NodeID.h b/src/lib/quickbus/routing/NodeID.h new file mode 100644 index 000000000..de2e55922 --- /dev/null +++ b/src/lib/quickbus/routing/NodeID.h @@ -0,0 +1,11 @@ +#pragma once + +#include "lib/utils/ValueWrapper.h" +#include + +namespace bts { +namespace qbus { +class NodeTag {}; +typedef utils::ValueWrapper NodeID; +NodeID make_node_id(); +}} diff --git a/src/lib/quickbus/routing/Router.cpp b/src/lib/quickbus/routing/Router.cpp new file mode 100644 index 000000000..8e6f865cb --- /dev/null +++ b/src/lib/quickbus/routing/Router.cpp @@ -0,0 +1,367 @@ +#include "Router.h" + +#include "lib/utils/bts_assert.h" +#include "lib/quickbus/message_model/merge_updates.h" +#include "lib/quickbus/core/Advertisement.h" +#include "lib/logging/logging.h" +#include "lib/utils/PBUtils.h" + +#include "lib/quickbus/network/quickbus_msgs.pb.h" + +#include +#include +#include +#include +#include +#include + +namespace bts { +namespace qbus { + +// Router is not reentrant, we use this class to enforce this at +// runtime +struct Router::NoReenter { + NoReenter(bool &inside) : inside(inside) + { + bts_assert(!inside); + inside = true; + } + ~NoReenter() {inside = false;} + bool &inside; +}; + +// Describes a single subscriber for a table +struct Router::SubscriberRec { + + SubscriberRec(NodeID const&node_id, INodePtr const&node, const SubscriptionID &sub_id) : + node_id(node_id), node(node), sub_id(sub_id) + {} + + // the node id for the subscribing node + NodeID node_id; + // the subscribing node + INodePtr node; + // a token used by the subscribing to node to ensure subscriptions + // and notifications are paired correctly. + SubscriptionID sub_id; +}; + +// Describes a table currently published by a node. We may or may not +// be subscribed to this table (if subscribers is empty we are not, +// otherwise we are) +struct Router::TableRec { + TableRec (NodeID const&owner_id, + INodePtr const& owner, + const Advertisement &ad) : + owner_id(owner_id), + owner_node(owner), + sub_id(null_subscription_id()), + advertisement(ad), + has_synopsis(false), + synopsis(), + subscribers() + {} + + // the id of the node publishing this table + NodeID owner_id; + + // a pointer to the node publishing this table + INodePtr owner_node; + + // a token used to ensure notifications are paired correctly with + // subscriptions + SubscriptionID sub_id; + + // the advertisement describing this table + Advertisement advertisement; + + // wheter this subscription has a synopsis that can be forwarded to + // new subscribers + bool has_synopsis; + + // the synopsis data + std::unordered_map synopsis; + + // the subscribers to this table + std::vector subscribers; +}; + +struct Router::PImpl { + PImpl(NodeID const&my_id) : + my_id(my_id), + neighbours(), + tables(), + last_sub_id(null_subscription_id()), + inside(false) + {} + + NodeID my_id; + std::map neighbours; + std::map> tables; + SubscriptionID last_sub_id; + bool inside; +}; + +Router::Router(NodeID const&id) : me (new PImpl(id)) +{ + LOG(QBus,Diag, "Router {} created ",id.value()); +} + +Router::~Router(){} + +NodeID Router::id() const {return me->my_id;} + +bool Router::add_neighbour (NodeID const& neighbour_id, INodePtr const &new_neighbour) +{ + LOG(QBus,Diag,"{} add_neighbour nid: {}",id().value(),neighbour_id.value()); + + NoReenter nr(me->inside); + + if (me->neighbours.find(neighbour_id) == me->neighbours.end()) { + // index new neighbour + me->neighbours[neighbour_id] = new_neighbour; + + // fwd adv + for(auto &t_pair : me->tables) { + LOG(QBus,Diag, + "{} sending existing advertisement for tid: {} topic: {} owned by: {} to: {} " , + id().value() , t_pair.first.value() , t_pair.second->advertisement.topic.value () , + t_pair.second->owner_id.value(), neighbour_id.value() ); + + new_neighbour->advertise(me->my_id,t_pair.first,t_pair.second->advertisement); + } + return true; + } + else + return false; +} + +void Router::remove_neighbour(NodeID const&removed_neighbour_id) +{ + LOG(QBus,Diag,"{} remove_neighbour nid: {} ", + id().value() , removed_neighbour_id.value()); + + NoReenter nr(me->inside); + + auto &n_rec = me->neighbours[removed_neighbour_id]; + bts_assert(n_rec); + + // collect removed neigbhours's tables and subscribers + std::vector its_tables; + std::vector its_subscriptions; + for (auto &kvp : me->tables) { + auto &tid = kvp.first; + auto &tr = kvp.second; + + // does the removed node own this table? + if (tr->owner_id == removed_neighbour_id) + its_tables.push_back(tid); + else + // the removed node does not own this table, but it might be + // subscribed to it. + for (auto &s : tr->subscribers) + if (s.node_id == removed_neighbour_id) { + its_subscriptions.push_back(tid); + break; + } + } + + // unadvertise the orhpan tables + for (auto tid : its_tables) { + LOG(QBus,Diag,"{} sending unadvertisement for table owned by dead node: {} tid: {}", + id().value() , removed_neighbour_id.value() , tid.value()); + _unadvertise(removed_neighbour_id,tid); + } + + // cancel the orphan subscriptions + for (auto &tid : its_subscriptions) { + LOG(QBus,Diag, "{} canceling subscription for table requested by dead node: {} tid: {}", + id().value(), removed_neighbour_id.value() , tid.value()); + _unsubscribe(removed_neighbour_id,tid); + } + + // finally remove the node from our index + me->neighbours.erase(removed_neighbour_id); +} + +void Router::advertise(NodeID const& sender,TableID const&tid,Advertisement const&ad) +{ + LOG(QBus,Diag, "{} received advertisement from nid: { tid: {} ad topic: {}", + id().value() , sender.value() , tid.value() , ad.topic.value()); + + NoReenter nr(me->inside); + + auto it = me->neighbours.find(sender); + if (it != me->neighbours.end()) { + auto const&node = it->second; + bts_assert (node); + + // create an entry for this table + auto &t_rec = me->tables[tid]; + bts_assert (!t_rec); + t_rec = std::make_unique (sender,node,ad); + + // forward to neighbours + for (auto &kvp : me->neighbours) + if (kvp.first != sender) { + LOG(QBus,Diag, "{} forwarding advertisement for table owned by nid: {} to: {} tid: {} ", + id().value() , sender.value () , kvp.first.value() , tid.value()); + + kvp.second->advertise(me->my_id,tid,ad); + } + } +} + +void Router::unadvertise(NodeID const&sender,TableID const&tid) +{ + LOG(QBus,Diag, "{} received unadvertisement from nid: {} tid: {}", + id().value() , sender.value() , tid.value() ); + NoReenter nr(me->inside); + _unadvertise(sender,tid); +} + +void Router::_unadvertise(NodeID const&sender,TableID const&tid) { + auto it = me->tables.find(tid); + if (it != me->tables.end()) { + auto &tr = it->second; + bts_assert(tr); + bts_assert(tr->owner_id == sender); + + // remove table + me->tables.erase(tid); + + // fwd to others + for (auto &kvp : me->neighbours) { + auto &nid = kvp.first; + auto &node = kvp.second; + if (nid != sender) { + LOG(QBus,Diag, "{} forwarding unadvertisement for tid: {} to nid: {}", + id().value() , tid.value() , nid.value()); + node->unadvertise(me->my_id,tid); + } + } + } +} + +void Router::subscribe (NodeID const&sender, TableID const&tid,SubscriptionID const&sid) +{ + LOG(QBus,Diag, + "{} received subscription from nid: {} tid: {} sid: {} " , + id().value() , sender.value() , tid.value() , sid.value()); + + NoReenter nr(me->inside); + + auto it = me->neighbours.find(sender); + if (it != me->neighbours.end()) { + + auto const &node = it->second; + bts_assert(node); + + auto it = me->tables.find(tid); + if (it != me->tables.end()) { + auto &tr = it->second; + bts_assert(tr); + bts_assert(sender != tr->owner_id); + auto &subscribers = tr->subscribers; + + // no duplicate subscriptions + bts_assert(std::find_if(subscribers.begin(),subscribers.end(), + [&sender] (SubscriberRec const&s) { + return s.node_id == sender; + }) == subscribers.end()); + + // add subscriber to this table's collection + SubscriberRec sub (sender,node,sid); + subscribers.push_back(sub); + + // if first subscription then issue our own subscription to source + if (subscribers.size() == 1) { + if (tr->advertisement.maintain_synopsis) { + me->last_sub_id = next_subscription_id(me->last_sub_id); + tr->sub_id = me->last_sub_id; + } + else + tr->sub_id = null_subscription_id(); + + LOG(QBus,Diag, "{} forwarding subscription for tid: {} sid: {} to nid: {}" , + id().value(), tid.value(), tr->sub_id.value() , tr->owner_id.value()); + tr->owner_node->subscribe(me->my_id,tid,tr->sub_id); + } + // otherwise if we have a synospsis, fwd it to subscriber + else if (tr->has_synopsis) { + Update update; + get_update_from_synopsis(tr->synopsis,&update); + LOG(QBus,Diag, "{} sending synopsis for tid: {} sid: {} to nid: {} ", + id().value() , tid.value() , tr->sub_id.value() , sender.value()); + node->notify(me->my_id,tid,tr->sub_id,update); + } + } + } +} + +void Router::notify(NodeID const&sender,TableID const&tid,SubscriptionID const&sid,Update const&u) +{ + LOG(QBus,Diag, "{} received notify from nid: {} tid: {} sid: {} size: {}" , + id().value(), sender.value() , tid.value() , sid.value() , u.size()); + + NoReenter nr(me->inside); + + auto it = me->tables.find(tid); + if (it != me->tables.end()) { + auto &tr = it->second; + if( (!tr->advertisement.maintain_synopsis || tr->sub_id == sid)) { + bts_assert(tr->owner_id == sender); + + // fwd to subscribers + for (auto &s : tr->subscribers) { + LOG(QBus,Diag,"{} forwarding notification for tid: {} sid: {} to nid {} ", + id().value(),tid.value(),s.sub_id.value(),s.node_id.value()); + s.node->notify(me->my_id,tid,s.sub_id,u); + } + + if (tr->advertisement.maintain_synopsis) { + // update synopsis + update_synopsis(u,&tr->synopsis); + tr->has_synopsis = true; + } + } + } +} + +void Router::unsubscribe (NodeID const&sender,TableID const&tid) +{ + LOG(QBus,Diag," {} received unsubscribe from nid: {} tid: {}", + id().value(), sender.value(), tid.value()); + + NoReenter nr(me->inside); + _unsubscribe (sender,tid); +} + +void Router::_unsubscribe (NodeID const&sender,TableID const&tid) +{ + auto it = me->tables.find(tid); + if (it != me->tables.end()) { + auto &tr = it->second; + auto &subscribers = tr->subscribers; + auto it = std::find_if (subscribers.begin(),subscribers.end(), + [&sender] (SubscriberRec const&s) { + return s.node_id == sender; + }); + bts_assert(it!=subscribers.end()); + subscribers.erase(it); + + // if this was the last subscription, fwd the unsubscribe and clean up + if (subscribers.empty()) { + LOG(QBus,Diag, "{} forwarding unsubscribe for tid {} sid: {} to: {}", + id().value(), tid.value() , tr->sub_id.value() , tr->owner_id.value()); + + tr->owner_node->unsubscribe(me->my_id,tid); + tr->sub_id = null_subscription_id(); + tr->synopsis.clear(); + tr->has_synopsis = false; + } + } +} + +}} + diff --git a/src/lib/quickbus/routing/Router.h b/src/lib/quickbus/routing/Router.h new file mode 100644 index 000000000..dc8012201 --- /dev/null +++ b/src/lib/quickbus/routing/Router.h @@ -0,0 +1,38 @@ +#pragma once + +#include "lib/quickbus/routing/INodePtr.h" +#include "lib/quickbus/routing/INode.h" + +namespace bts { +namespace qbus { + +class Router : public INode +{ +public: + Router(NodeID const&bid) ; + ~Router(); + + NodeID id() const ; + bool add_neighbour (NodeID const& neighbour_id, INodePtr const& neighbour); + void remove_neighbour (NodeID const& neighbour_id); + + // from INode + void advertise(NodeID const& sender,TableID const&id,Advertisement const&adv) override; + void unadvertise(NodeID const& sender,TableID const&tid) override; + void subscribe (NodeID const&sender, TableID const&tid,SubscriptionID const&sid) override; + void notify(NodeID const&,TableID const&,SubscriptionID const&, Update const&) override; + void unsubscribe (NodeID const&sender, TableID const&tid) override; + // end INode + +private: + struct NoReenter; + struct SubscriberRec; + struct TableRec; + struct PImpl; + + void _unadvertise(NodeID const& sender,TableID const&tid) ; + void _unsubscribe (NodeID const&sender, TableID const&tid) ; + std::unique_ptr me; +}; + +}} diff --git a/src/lib/quickbus/rpc/OnRPCReply.h b/src/lib/quickbus/rpc/OnRPCReply.h new file mode 100644 index 000000000..1eb108840 --- /dev/null +++ b/src/lib/quickbus/rpc/OnRPCReply.h @@ -0,0 +1,11 @@ +#pragma once + +#include +#include +#include "lib/quickbus/rpc/RPCError.h" +#include "lib/quickbus/rpc/RPCReply.h" + +namespace bts { +namespace qbus { +typedef std::function OnRPCReply; +}} diff --git a/src/lib/quickbus/rpc/OnRPCRequest.h b/src/lib/quickbus/rpc/OnRPCRequest.h new file mode 100644 index 000000000..13550fe90 --- /dev/null +++ b/src/lib/quickbus/rpc/OnRPCRequest.h @@ -0,0 +1,14 @@ +#pragma once + +#include "lib/quickbus/rpc/RPCReply.h" + +#include +#include + +namespace bts { +namespace qbus { + +typedef std::function ReplyHandler; +typedef std::function OnRPCRequest; + +}} diff --git a/src/lib/quickbus/rpc/RPCClient.cpp b/src/lib/quickbus/rpc/RPCClient.cpp new file mode 100644 index 000000000..f6d5ad9eb --- /dev/null +++ b/src/lib/quickbus/rpc/RPCClient.cpp @@ -0,0 +1,319 @@ +#include "RPCClient.h" + +#include "lib/quickbus/rpc/RPCSessionID.h" +#include "lib/quickbus/core/Topic.h" +#include "lib/quickbus/core/LocalNode.h" +#include "lib/quickbus/core/Advertisement.h" +#include "lib/quickbus/core/SubscriptionID.h" +#include "lib/quickbus/rpc/rpc_utils.h" +#include "lib/logging/logging.h" +#include "lib/utils/datetime.h" +#include "lib/utils/bts_assert.h" +#include "lib/utils/bts_chrono.h" + +#include "lib/quickbus/network/quickbus_msgs.pb.h" + +#include +#include +#include +#include + +namespace bts { +namespace qbus { + +class RPCClient::RPCClientImp : public LocalNode +{ +public: + RPCClientImp(Quickbus &qb, RPCServiceName const&sn) : + LocalNode(qb), + service_name(sn), + state (IDLE), + session(sn), + last_req_id(0), + enabled(true) + {} + + void start() + { + bts_assert(enabled); + + async_register(); + async_invoke ([this] () {start_session();}); + } + + void stop () + { + bts_assert(enabled); + enabled = false; + + if (on_qb_thread()) + async_unregister(); + else + async_unregister().wait(); + } + + void send(RPCRequestPtr const &req,OnRPCReply const&on_reply) + { + bts_assert(enabled); + async_invoke([this, req, on_reply] () {sync_send(req,on_reply);}); + } + + bool connected() const + {return state == READY;} + +private: + void sync_send(RPCRequestPtr const &req,OnRPCReply const&on_reply) + { + LOG(QBus,Diag, "Trying to sync_send {} bytes {} to {}", + req->value().size() , service_name.value()); + + bts_assert (on_qb_thread()); + + if (enabled) + switch (state.load()) { + case CONNECTING: + case IDLE: + LOG(QBus,Warn, "Unable to find {}. sync_send failed", service_name.value()) ; + on_reply(RPCError::NO_SERVER,nullptr); + break; + case READY: { + // index request + auto req_id = next_request_id(); + session.in_flight[req_id] = on_reply; + + // create update; + Update u; + u.emplace_back(Change::CREATE,req_id,req); + + // send it out + sync_notify(session.request_table,null_subscription_id(),u); + + LOG(QBus,Diag, "sync_send of {} bytes to {} completed. Req_id: {}", + req->value().size() , service_name.value () , req_id.value()); + + break; + } + } + } + + void start_session() + { + bts_assert (on_qb_thread()); + bts_assert(state == IDLE); + bts_assert(session.in_flight.empty()); + + state = CONNECTING; + session = Session(service_name); + + async_invoke([this] () { + LOG(QBus,Info, "Advertising {} on tid: {}", + session.request_topic.value() , session.request_table.value()); + Advertisement ad (session.request_topic,false); + sync_advertise(session.request_table,ad); + }); + } + + void stop_session() + { + bts_assert (on_qb_thread()); + bts_assert(state == READY); + + LOG(QBus,Info, "Stoping client session for {} tid: {}" , + session.request_topic.value(), session.request_table.value()); + + for (auto &cb : session.in_flight) + cb.second(RPCError::SERVER_DISCONNECT,nullptr); + session.in_flight.clear(); + async_invoke ([this] () {sync_unadvertise(session.request_table);}); + } + + void on_advertise(TableID const&id,Advertisement const&ad) override + { + bts_assert (on_qb_thread()); + + switch (state.load()) { + case READY: + case IDLE: + break; + case CONNECTING: + if (ad.topic == session.reply_topic) { + LOG(QBus,Info, + "Received advertisement for {} carried by tid: {}. Starting session", + session.reply_topic.value() , id.value() ); + + session.reply_table = id; + state = READY; + async_invoke ([this] () { + sync_subscribe(session.reply_table,null_subscription_id()); + }); + } + break; + } + } + + void on_unadvertise (TableID const&tid) override + { + bts_assert (on_qb_thread()); + + switch (state.load()) { + case CONNECTING: + case IDLE: + break; + case READY: + if (session.reply_table == tid) { + LOG(QBus,Info, "Received unadvertisement for {} carried by tid: {}. Ending session" , + session.reply_topic.value() , tid.value()); + + stop_session(); + state = IDLE; + start_session(); + } + break; + } + } + + void on_notify(TableID const&tid,SubscriptionID const&sid, Update const&u) override + { + bts_assert (on_qb_thread()); + + switch (state.load()) { + case CONNECTING: + case IDLE: + break; + case READY: + if (session.reply_table == tid && enabled) { + bts_assert(sid == null_subscription_id()); + bts_assert(u.size() == 1); + auto const&r = u[0]; + bts_assert(r.type == Change::CREATE); + + auto req_id = r.key; + auto const&reply = r.value; + + auto it = session.in_flight.find(req_id); + if (it != session.in_flight.end()) { + LOG(QBus,Diag, "Received {} bytes in response to req_id: {} from: {}" , + reply->value().size() , req_id.value() , service_name.value () ); + it->second(RPCError::RPC_OK,reply); + session.in_flight.erase(it); + } else + LOG(QBus,Error, + " Received {} bytes in response to " + " unknown request req_id: {} from: {}", + reply->value().size() , req_id.value() , + service_name.value ()) ; + + } + break; + } + } + + RecordKey next_request_id() { + last_req_id = RecordKey(last_req_id.value()+1); + return last_req_id; + } + + enum State {IDLE,CONNECTING,READY}; + + struct Session { + + Session(RPCServiceName const&sn) + : session_id(make_rpc_session_id()), + request_topic(make_rpc_request_topic(sn,session_id)), + reply_topic(make_rpc_reply_topic(sn,session_id)), + request_table(make_table_id()), + reply_table(), + in_flight() + {} + + RPCSessionID session_id; + Topic request_topic; + Topic reply_topic; + TableID request_table; + TableID reply_table; + std::unordered_map in_flight; + }; + + const RPCServiceName service_name; + std::atomic state; + Session session; + RecordKey last_req_id; + std::atomic enabled; +}; + +RPCClient::RPCClient() : rpc_imp() {} + +RPCClient::RPCClient(RPCClient &&src) : rpc_imp(std::move(src.rpc_imp)) {} + +RPCClient::RPCClient(Quickbus &qb,RPCServiceName const&name) + : rpc_imp(new RPCClientImp(qb,name)) +{ + rpc_imp->start(); +} + +RPCClient::~RPCClient() +{disconnect();} + +void RPCClient::disconnect() +{ + if (rpc_imp) { + rpc_imp->stop(); + rpc_imp.reset(); + } +} + +RPCClient&RPCClient::operator=(RPCClient &&src) +{ + disconnect(); + rpc_imp = std::move(src.rpc_imp); + return *this; +} + +void RPCClient::send(RPCRequest const&req,OnRPCReply const&on_reply) +{ + send(std::make_shared (req),on_reply); +} + +void RPCClient::send(RPCRequest &&req,OnRPCReply const&on_reply) +{ + send(std::make_shared (std::move(req)), on_reply); +} + +void RPCClient::send(RPCRequestPtr const&req,OnRPCReply const&on_reply) +{ + bts_assert(rpc_imp); + rpc_imp->send(req,on_reply); +} + +bool RPCClient::wait_until_connected(uint32_t max_secs) { + bts_assert(rpc_imp); + auto start = datetime::now(); + + for (;;) { + if (rpc_imp->connected()) + return true; + + if (datetime::is_at_least_secs_old(start,max_secs)) + return false; + + std::this_thread::sleep_for(100ms); + } +} + +bool RPCClient::wait_until_disconnected(uint32_t max_secs) { + bts_assert(rpc_imp); + auto start = datetime::now(); + + for (;;) { + if (!rpc_imp->connected()) + return true; + + if (datetime::is_at_least_secs_old(start,max_secs)) + return false; + + std::this_thread::sleep_for(100ms); + } +} + +}} + diff --git a/src/lib/quickbus/rpc/RPCClient.h b/src/lib/quickbus/rpc/RPCClient.h new file mode 100644 index 000000000..372a44d0b --- /dev/null +++ b/src/lib/quickbus/rpc/RPCClient.h @@ -0,0 +1,40 @@ +#pragma once + +#include "lib/quickbus/rpc/RPCServiceName.h" +#include "lib/quickbus/rpc/RPCRequest.h" +#include "lib/quickbus/rpc/OnRPCReply.h" +#include "lib/utils/bts_export.h" +#include + +namespace bts {namespace qbus {class Quickbus;}} +namespace bts {namespace qbus {class QBRPC_Tests;}} + +namespace bts { +namespace qbus { + +class BTS_EXPORT RPCClient +{ +public: + RPCClient (); + RPCClient (RPCClient &&src); + RPCClient(Quickbus&qb,RPCServiceName const&name); + ~RPCClient(); + RPCClient&operator=(RPCClient &&src); + + void send(RPCRequest const&req,OnRPCReply const&on_reply); + void send(RPCRequest &&req,OnRPCReply const&on_reply); + void send(RPCRequestPtr const&req,OnRPCReply const&on_reply); + + void disconnect(); + + // only meant to simplify testing + friend class QBRPC_Tests; + bool wait_until_connected(uint32_t max_secs); + bool wait_until_disconnected(uint32_t max_secs); +private: + + class RPCClientImp; + std::shared_ptr rpc_imp; +}; + +}} diff --git a/src/lib/quickbus/rpc/RPCError.h b/src/lib/quickbus/rpc/RPCError.h new file mode 100644 index 000000000..8d17341b1 --- /dev/null +++ b/src/lib/quickbus/rpc/RPCError.h @@ -0,0 +1,10 @@ +#pragma once + +#include + +namespace bts { +namespace qbus { + +enum class RPCError {RPC_OK,NO_SERVER,SERVER_DISCONNECT}; + +}} diff --git a/src/lib/quickbus/rpc/RPCReply.h b/src/lib/quickbus/rpc/RPCReply.h new file mode 100644 index 000000000..40bea8d7d --- /dev/null +++ b/src/lib/quickbus/rpc/RPCReply.h @@ -0,0 +1,10 @@ +#pragma once + +#include +#include "lib/quickbus/message_model/Record.h" + +namespace bts { +namespace qbus { +typedef Record RPCReply; +typedef RecordPtr RPCReplyPtr; +}} diff --git a/src/lib/quickbus/rpc/RPCRequest.h b/src/lib/quickbus/rpc/RPCRequest.h new file mode 100644 index 000000000..77a520505 --- /dev/null +++ b/src/lib/quickbus/rpc/RPCRequest.h @@ -0,0 +1,10 @@ +#pragma once + +#include +#include "lib/quickbus/message_model/Record.h" + +namespace bts { +namespace qbus { +typedef Record RPCRequest; +typedef RecordPtr RPCRequestPtr; +}} diff --git a/src/lib/quickbus/rpc/RPCServer.cpp b/src/lib/quickbus/rpc/RPCServer.cpp new file mode 100644 index 000000000..4fddac8d5 --- /dev/null +++ b/src/lib/quickbus/rpc/RPCServer.cpp @@ -0,0 +1,175 @@ +#include "RPCServer.h" + +#include "lib/quickbus/core/LocalNode.h" +#include "lib/quickbus/rpc/RPCSessionID.h" +#include "lib/quickbus/rpc/rpc_utils.h" +#include "lib/quickbus/core/Advertisement.h" +#include "lib/logging/logging.h" + +#include +#include + +namespace bts { +namespace qbus { + +class RPCServer::RPCServerImp : public LocalNode +{ +public: + RPCServerImp(Quickbus&qb, RPCServiceName const&sn, OnRPCRequest const&handler) : + LocalNode(qb), + service_name(sn), + handler(handler), + sessions(), + enabled(true) + {} + + void start() + { + bts_assert(enabled); + async_register(); + } + + void stop () + { + bts_assert(enabled); + enabled = false; + + if (on_qb_thread()) + async_unregister(); + else + async_unregister().wait(); + } + + void reply(TableID const&request_table, RecordKey req_id, RPCReplyPtr const&r) + { + bts_assert(enabled); + + async_invoke([this,request_table,req_id,r] () {sync_reply(request_table,req_id,r);}); + } + +private: + void sync_reply(TableID const&request_table,RecordKey req_id, RPCReplyPtr const&reply) + { + bts_assert(on_qb_thread()); + + if (enabled) { + auto sit = sessions.find(request_table); + if (sit != sessions.end()) { + auto const &reply_table = sit->second; + Update u; + u.emplace_back(Change::CREATE,req_id,reply); + sync_notify(reply_table,null_subscription_id(),u); + LOG(QBus,Diag, "Sent {} bytes in response to req_id: {} from: {} carried by tid: {}", + reply->value().size(),req_id.value(),service_name.value (), + reply_table.value()); + } + } + } + + void on_advertise(TableID const&request_tid,Advertisement const&ad) override + { + + bts_assert(on_qb_thread()); + + if (enabled) { + if (is_rpc_request_topic(service_name,ad.topic)) { + bts_assert(sessions.find(request_tid) == sessions.end()); + auto reply_tid = make_table_id(); + + // index new session + sessions[request_tid] = reply_tid; + + // subscribe to request table + async_invoke([this,request_tid] () {sync_subscribe(request_tid,null_subscription_id());}); + + // advertise reply table + Advertisement reply_ad; + reply_ad.topic = make_rpc_reply_topic(service_name,get_session_id(ad.topic)); + async_invoke([this,reply_tid,reply_ad] {sync_advertise(reply_tid,reply_ad);}); + + LOG(QBus,Info,"Received advertisement for {} tid: {} replying with {} on {}", + ad.topic.value(), request_tid.value() , reply_ad.topic.value() , reply_tid.value()); + + } + } + } + + void on_unadvertise(TableID const&request_tid) override + { + bts_assert(on_qb_thread()); + + if (enabled) { + auto it = sessions.find(request_tid); + if (it != sessions.end()) { + auto &reply_tid = it->second; + async_invoke([this,reply_tid] () {sync_unadvertise(reply_tid);}); + sessions.erase(reply_tid); + + LOG(QBus,Info,"Received unadvertisement for tid: {} Shutting down session {}" , + request_tid.value() , request_tid.value()); + } + } + } + + void on_notify(TableID const&request_tid,SubscriptionID const&sid, Update const&u) override + { + bts_assert(on_qb_thread()); + + if (enabled) { + auto it = sessions.find(request_tid); + if (it != sessions.end()) { + + bts_assert(sid == null_subscription_id()); + bts_assert(u.size() == 1); + auto const&r = u[0]; + bts_assert(r.type == Change::CREATE); + + auto req_id = r.key; + auto const&request = r.value; + + LOG(QBus,Diag, "Service {} received rpc. {} bytes with req_id: {} on tid: {}", + service_name.value() , request->value().size() , req_id.value() , request_tid.value()); + + handler(request,[this,req_id,request_tid] (RPCReplyPtr const&rep) + {reply(request_tid,req_id,rep);}); + } + } + } + + const RPCServiceName service_name; + const OnRPCRequest handler; + + // request -> reply mapping + std::unordered_map sessions; + std::atomic enabled; +}; + +RPCServer::RPCServer() : rpc_imp() {} + +RPCServer::RPCServer(RPCServer &&src) : rpc_imp(std::move(src.rpc_imp)) {} + +RPCServer::RPCServer(Quickbus&qb,RPCServiceName const&name,OnRPCRequest const&handler) + : rpc_imp(new RPCServerImp(qb,name,handler)) +{ + rpc_imp->start(); +} + +RPCServer::~RPCServer() {disconnect();} + +void RPCServer::disconnect() +{ + if (rpc_imp) { + rpc_imp->stop(); + rpc_imp.reset(); + } +} + +RPCServer&RPCServer::operator=(RPCServer &&src) +{ + disconnect(); + rpc_imp = std::move(src.rpc_imp); + return *this; +} + +}} + diff --git a/src/lib/quickbus/rpc/RPCServer.h b/src/lib/quickbus/rpc/RPCServer.h new file mode 100644 index 000000000..0446ff5e8 --- /dev/null +++ b/src/lib/quickbus/rpc/RPCServer.h @@ -0,0 +1,31 @@ +#pragma once + +#include "lib/quickbus/rpc/RPCServiceName.h" +#include "lib/quickbus/rpc/RPCRequest.h" +#include "lib/quickbus/rpc/OnRPCRequest.h" +#include "lib/quickbus/rpc/OnRPCReply.h" +#include "lib/utils/bts_export.h" +#include + +namespace bts {namespace qbus {class Quickbus;}} + +namespace bts { +namespace qbus { + +class BTS_EXPORT RPCServer +{ +public: + RPCServer(); + RPCServer (RPCServer &&src); + RPCServer(Quickbus&qb, RPCServiceName const&name, OnRPCRequest const&handler); + ~RPCServer(); + RPCServer&operator=(RPCServer &&src); + + void disconnect(); + +private: + class RPCServerImp; + std::shared_ptr rpc_imp; +}; + +}} diff --git a/src/lib/quickbus/rpc/RPCServiceName.h b/src/lib/quickbus/rpc/RPCServiceName.h new file mode 100644 index 000000000..a3a49cdf5 --- /dev/null +++ b/src/lib/quickbus/rpc/RPCServiceName.h @@ -0,0 +1,13 @@ +#pragma once + +#include "lib/utils/ValueWrapper.h" +#include +#include + +namespace bts { +namespace qbus { + +class RPCServiceNameTag {}; +typedef utils::ValueWrapper RPCServiceName; + +}} diff --git a/src/lib/quickbus/rpc/RPCSessionID.cpp b/src/lib/quickbus/rpc/RPCSessionID.cpp new file mode 100644 index 000000000..e19d800df --- /dev/null +++ b/src/lib/quickbus/rpc/RPCSessionID.cpp @@ -0,0 +1,9 @@ +#include "RPCSessionID.h" +#include "lib/utils/Uuid.h" + +namespace bts { +namespace qbus { + +RPCSessionID make_rpc_session_id() {return RPCSessionID(utils::uuid());} + +}} diff --git a/src/lib/quickbus/rpc/RPCSessionID.h b/src/lib/quickbus/rpc/RPCSessionID.h new file mode 100644 index 000000000..f1ea25096 --- /dev/null +++ b/src/lib/quickbus/rpc/RPCSessionID.h @@ -0,0 +1,23 @@ +#pragma once + +#include +#include +#include "lib/utils/ValueWrapper.h" + +namespace bts { +namespace qbus { +class RPCSessionIDTag {}; +typedef utils::ValueWrapper RPCSessionID; +RPCSessionID make_rpc_session_id(); +}} + +// yeah, this is crazy, but that's how you do it. +namespace std { +template <> +struct hash +{ + std::size_t operator()(const bts::qbus::RPCSessionID& k) const { + return std::hash () (k.value()); + } +}; +} diff --git a/src/lib/quickbus/rpc/rpc_utils.cpp b/src/lib/quickbus/rpc/rpc_utils.cpp new file mode 100644 index 000000000..614304d64 --- /dev/null +++ b/src/lib/quickbus/rpc/rpc_utils.cpp @@ -0,0 +1,34 @@ +#include "rpc_utils.h" + +#include "lib/quickbus/rpc/RPCSessionID.h" +#include "lib/utils/bts_assert.h" + +#include +#include +#include + +namespace bts { +namespace qbus { + +Topic make_rpc_request_topic(RPCServiceName const &sn,RPCSessionID const&sid) +{return Topic((boost::format ("%s/rpc/request/%s") % sn.value() % sid.value()).str());} + +Topic make_rpc_reply_topic(RPCServiceName const &sn,RPCSessionID const&sid) +{return Topic((boost::format ("%s/rpc/reply/%s") % sn.value() % sid.value()).str());} + +bool is_rpc_request_topic(RPCServiceName const&sn,Topic const&topic) +{ + boost::regex req_regex ((boost::format("%s/rpc/request/(.*)") % sn.value()).str()); + return boost::regex_match (topic.value(), req_regex); +} + +RPCSessionID get_session_id(Topic const&from_request_topic) +{ + std::vector parts; + boost::split( parts,from_request_topic.value(), boost::is_any_of("/")); + bts_assert(parts.size() >= 4); + return RPCSessionID(parts[3]); +} + +}} + diff --git a/src/lib/quickbus/rpc/rpc_utils.h b/src/lib/quickbus/rpc/rpc_utils.h new file mode 100644 index 000000000..a14a6618f --- /dev/null +++ b/src/lib/quickbus/rpc/rpc_utils.h @@ -0,0 +1,18 @@ +#pragma once + +#include "lib/quickbus/rpc/RPCServiceName.h" +#include "lib/quickbus/rpc/RPCSessionID.h" +#include "lib/quickbus/core/SubscriptionID.h" +#include "lib/quickbus/core/Topic.h" + +#include + +namespace bts { +namespace qbus { + +Topic make_rpc_request_topic(RPCServiceName const &sn,RPCSessionID const&sid); +Topic make_rpc_reply_topic(RPCServiceName const &sn,RPCSessionID const&sid); +bool is_rpc_request_topic(RPCServiceName const&sn,Topic const&topic); +RPCSessionID get_session_id(Topic const&from_request_topic); + +}} diff --git a/src/lib/quickbus/rpc/tests/qb_rpc_tests.cpp b/src/lib/quickbus/rpc/tests/qb_rpc_tests.cpp new file mode 100644 index 000000000..c9b510e2c --- /dev/null +++ b/src/lib/quickbus/rpc/tests/qb_rpc_tests.cpp @@ -0,0 +1,272 @@ +#include "catch/catch.h" +#include "lib/quickbus/quickbus.h" +#include "lib/os/os.h" +#include "lib/utils/TimeStamp.h" +#include "lib/utils/benchmarking_utils.h" + +#include +#include +#include + +namespace bts { +namespace qbus { + +double measure_latency(RPCClient &cli) +{ + REQUIRE(cli.wait_until_connected(5)); + + std::promise p; + auto f = p.get_future(); + + std::vector times; + times.reserve(10000); + utils::TimeStamp rpc_begin; + auto msg = std::make_shared ("hello"); + + OnRPCReply on_reply; + + on_reply = [&] (RPCError er, RPCReplyPtr const&) { + REQUIRE(er==RPCError::RPC_OK); + times.push_back((utils::TimeStamp::now()-rpc_begin).nanoseconds()); + if (times.size() == 1000) + p.set_value(); + else { + rpc_begin = utils::TimeStamp::now(); + cli.send(msg,on_reply); + } + }; + rpc_begin = utils::TimeStamp::now(); + cli.send(msg,on_reply); + REQUIRE(f.wait_for(std::chrono::seconds(5)) == std::future_status::ready); + return utils::median(times) / 1000; +} + + +TEST_CASE("Quickbus RPC Tests","[quickbus]") +{ + auto qb_port = os::get_random_port(); + + SECTION("smoke") + { + config::QuickbusNodeCfg cfg; + cfg.set_my_port (qb_port); + Quickbus qb(cfg); + auto sn = RPCServiceName("test"); + RPCClient cli(qb,sn); + + RPCServer server(qb,sn,[] (RPCRequestPtr const&req,const ReplyHandler &on_reply) { + on_reply(req); + }); + + REQUIRE(cli.wait_until_connected(5)); + + std::promise p; + auto f = p.get_future(); + cli.send(std::make_shared ("hello"), + [&p] (RPCError er, RPCReplyPtr const&ptr) { + REQUIRE(er == RPCError::RPC_OK); + REQUIRE(ptr); + REQUIRE(ptr->value()=="hello"); + p.set_value(); + }); + REQUIRE(f.wait_for(std::chrono::seconds(5)) == std::future_status::ready); + } + + SECTION("no_server_is_reported") + { + config::QuickbusNodeCfg cfg; + cfg.set_my_port (qb_port); + Quickbus qb(cfg); + auto sn = RPCServiceName("test"); + RPCClient cli(qb,sn); + + std::promise p; + auto f = p.get_future(); + + cli.send(std::make_shared ("hello"), + [&p] (RPCError er, RPCReplyPtr const&ptr) { + REQUIRE(er==RPCError::NO_SERVER); + REQUIRE(!ptr); + p.set_value(); + }); + REQUIRE(f.wait_for(std::chrono::seconds(15)) == std::future_status::ready); + } + + SECTION("failure_with_in_flight_is_reported") + { + config::QuickbusNodeCfg cfg; + cfg.set_my_port (qb_port); + Quickbus qb(cfg); + auto sn = RPCServiceName("test"); + RPCClient cli(qb,sn); + + RPCServer server(qb,sn,[] (RPCRequestPtr const&,const ReplyHandler &) {}); + + REQUIRE(cli.wait_until_connected(5)); + + std::promise p; + auto f = p.get_future(); + cli.send(std::make_shared ("hello"), + [&p] (RPCError er, RPCReplyPtr const&ptr) { + REQUIRE(er == RPCError::SERVER_DISCONNECT); + REQUIRE(!ptr); + p.set_value(); + }); + server.disconnect(); + REQUIRE(f.wait_for(std::chrono::seconds(5)) == std::future_status::ready); + } + + SECTION("failure_with_no_in_flight_is_hidden") + { + config::QuickbusNodeCfg cfg; + cfg.set_my_port (qb_port); + Quickbus qb(cfg); + auto sn = RPCServiceName("test"); + RPCClient cli(qb,sn); + + RPCServer server(qb,sn,[] (RPCRequestPtr const&req,const ReplyHandler &on_reply) { + on_reply(req); + }); + + REQUIRE(cli.wait_until_connected(5)); + + std::promise p; + auto f = p.get_future(); + cli.send(std::make_shared ("hello"), + [&p] (RPCError er, RPCReplyPtr const&ptr) { + REQUIRE(er == RPCError::RPC_OK); + REQUIRE(ptr); + REQUIRE(ptr->value() == "hello"); + p.set_value(); + }); + REQUIRE(f.wait_for(std::chrono::seconds(5)) == std::future_status::ready); + + // got first reply, now kill the server and replace it + server.disconnect(); + REQUIRE(cli.wait_until_disconnected(5)); + server = RPCServer (qb,sn,[] (RPCRequestPtr const&req,const ReplyHandler &on_reply) { + on_reply(std::make_shared(req->value() + "_2")); + }); + + // second call should be replied by the second server + REQUIRE(cli.wait_until_connected(5)); + std::promise p2; + auto f2 = p2.get_future(); + cli.send(std::make_shared ("hello"), + [&p2] (RPCError er, RPCReplyPtr const&ptr) { + REQUIRE(er == RPCError::RPC_OK); + REQUIRE(ptr); + REQUIRE(ptr->value() == "hello_2"); + p2.set_value(); + }); + REQUIRE(f2.wait_for(std::chrono::seconds(5)) == std::future_status::ready); + } + + SECTION("replies_are_matched_to_requests") + { + config::QuickbusNodeCfg cfg; + cfg.set_my_port (qb_port); + Quickbus qb(cfg); + auto sn = RPCServiceName("test"); + + RPCClient cli(qb,sn); + + RPCServer server(qb,sn,[] (RPCRequestPtr const&req,const ReplyHandler &on_reply) {on_reply(req);}); + + REQUIRE(cli.wait_until_connected(5)); + std::vector> pis(100); + + + for (size_t i = 0; i (token), + [&p,token] (RPCError er, RPCReplyPtr const&ptr) { + REQUIRE(er==RPCError::RPC_OK); + REQUIRE(ptr); + REQUIRE(ptr->value()==token); + p.set_value(); + }); + } + + for (auto &p : pis) { + auto f= p.get_future(); + REQUIRE(f.wait_for(std::chrono::seconds(5)) == std::future_status::ready); + } + } + + SECTION("replies_are_matched_to_clients") + { + config::QuickbusNodeCfg cfg; + cfg.set_my_port (qb_port); + Quickbus qb(cfg); + auto sn = RPCServiceName("test"); + + std::vector clis(100); + for (auto &cli :clis) + cli = RPCClient (qb,sn); + + RPCServer server(qb,sn,[] (RPCRequestPtr const&req,const ReplyHandler &on_reply) {on_reply(req);}); + + for (auto &cli :clis) + REQUIRE(cli.wait_until_connected(5)); + + std::vector> pis(clis.size()); + + for (size_t i = 0; i (token), + [&p,token] (RPCError er, RPCReplyPtr const&ptr) { + REQUIRE(er==RPCError::RPC_OK); + REQUIRE(ptr); + REQUIRE(ptr->value()==token); + p.set_value(); + }); + } + + for (auto &p : pis) { + auto f= p.get_future(); + REQUIRE(f.wait_for(std::chrono::seconds(5)) == std::future_status::ready); + } + } + + SECTION("rpc_latency_same_qb") + { + config::QuickbusNodeCfg cfg; + cfg.set_my_port (qb_port); + Quickbus qb(cfg); + auto sn = RPCServiceName("test"); + + RPCClient cli(qb,sn); + RPCServer server(qb,sn,[] (RPCRequestPtr const&req,const ReplyHandler &on_reply) {on_reply(req);}); + + std::cout << "\tQuickbus RPC intra bus one way time: " << measure_latency(cli) << "us\n"; + } + + SECTION("rpc_latency_between_qb") + { + config::QuickbusNodeCfg cfg; + cfg.set_my_port (qb_port); + Quickbus parent_qb(cfg); + + cfg.set_my_port (qb_port+1); + cfg.set_parent_port (qb_port); + cfg.set_parent_host ("127.0.0.1"); + Quickbus child_qb(cfg); + + auto sn = RPCServiceName("test"); + + RPCClient cli(child_qb,sn); + RPCServer server(parent_qb,sn,[] (RPCRequestPtr const&req,const ReplyHandler &on_reply) {on_reply(req);}); + + std::cout << "\tQuickbus RPC inter bus one way time: " << measure_latency(cli) << "us\n"; + } +} + + +}} + diff --git a/src/lib/quickbus/tests/quickbus_tests.cpp b/src/lib/quickbus/tests/quickbus_tests.cpp new file mode 100644 index 000000000..eb07fa976 --- /dev/null +++ b/src/lib/quickbus/tests/quickbus_tests.cpp @@ -0,0 +1,244 @@ +#include "catch/catch.h" + +#include "lib/quickbus/quickbus.h" +#include "lib/quickbus/routing/Router.h" + +#include + +namespace bts { +namespace qbus { + +struct FakeNode : public INode { + + FakeNode() + : on_advertise(), + on_unadvertise(), + on_subscribe (), + on_notify(), + on_unsubscribe () + {} + + void advertise(NodeID const& sender, TableID const&id, Advertisement const &ad) override + {on_advertise(sender,id,ad);} + + void unadvertise(NodeID const&sender,TableID const&cid) override + {on_unadvertise(sender,cid);} + + void subscribe (NodeID const&sender, TableID const&tid,SubscriptionID const&sid) override + {on_subscribe(sender,tid,sid);} + + void notify(NodeID const&sender,TableID const&tid,SubscriptionID const&sid,Update const&u) override + {on_notify(sender,tid,sid,u);} + + void unsubscribe (NodeID const&sender, TableID const&tid) override + {on_unsubscribe(sender, tid);} + + boost::signals2::signal on_advertise; + boost::signals2::signal on_unadvertise; + boost::signals2::signal on_subscribe ; + boost::signals2::signal on_notify; + boost::signals2::signal on_unsubscribe ; +}; + +TEST_CASE ("Quickbus QBRouter_Tests" , "[quickbus]") +{ + SECTION("advertisments_are_forwarded") + { + NodeID r_id("src"); + Router r(r_id); + + NodeID adv_source_id ("adv_source"); + auto adv_source = std::make_shared(); + adv_source->on_advertise.connect([] (NodeID const&, TableID const&, Advertisement const &) + {REQUIRE(false);}); + + NodeID adv_sink1_id ("adv_sink1"); + auto adv_sink1 = std::make_shared(); + bool s1_got_it = false; + + adv_sink1->on_advertise.connect( + [r_id,&s1_got_it] (NodeID const&sender, TableID const&,Advertisement const&) { + REQUIRE(sender == r_id); + s1_got_it = true; + }); + + NodeID adv_sink2_id ("adv_sink2"); + auto adv_sink2 = std::make_shared(); + bool s2_got_it = false; + + adv_sink2->on_advertise.connect( + [r_id,&s2_got_it] (NodeID const&sender, TableID const&, const Advertisement &) { + REQUIRE(sender == r_id); + s2_got_it = true; + }); + + + r.add_neighbour(adv_source_id,adv_source); + r.add_neighbour(adv_sink1_id,adv_sink1); + r.add_neighbour(adv_sink2_id,adv_sink2); + + auto tid = make_table_id(); + Advertisement ad(Topic("hello")); + r.advertise(adv_source_id,tid, ad); + REQUIRE(s1_got_it); + REQUIRE(s2_got_it); + } + + SECTION("late_joiners_receive_advertisments") + { + NodeID r_id("src"); + Router r(r_id); + + NodeID adv_source_id ("adv_source"); + auto adv_source=std::make_shared(); + adv_source->on_advertise.connect( + [] (NodeID const&, TableID const&, Advertisement const &) + {REQUIRE(false);}); + + NodeID adv_sink1_id ("adv_sink1"); + auto adv_sink1 = std::make_shared(); + bool s1_got_it = false; + adv_sink1->on_advertise.connect( + [r_id,&s1_got_it] (NodeID const&sender, TableID const&, Advertisement const &) { + REQUIRE(sender == r_id); + s1_got_it = true; + }); + + auto tid = make_table_id(); + Advertisement ad(Topic("hello")); + + r.add_neighbour(adv_source_id,adv_source); + r.advertise(adv_source_id,tid, ad); + r.add_neighbour(adv_sink1_id,adv_sink1); + REQUIRE(s1_got_it); + } + + SECTION("removed_nodes_clean_advertisdements") + { + NodeID r_id("src"); + Router r(r_id); + + NodeID adv_source_id ("adv_source"); + auto adv_source = std::make_shared(); + adv_source->on_advertise.connect( + [] (NodeID const&, TableID const&, Advertisement const &) + {REQUIRE(false);}); + + NodeID adv_sink1_id ("adv_sink1"); + auto adv_sink1 = std::make_shared(); + bool s1_got_it = false; + adv_sink1->on_advertise.connect( + [r_id,&s1_got_it] (NodeID const&sender, TableID const&, Advertisement const &) { + REQUIRE(sender == r_id); + s1_got_it = true; + }); + + adv_sink1->on_unadvertise.connect( + [r_id,&s1_got_it] (NodeID const&sender, TableID const&) { + REQUIRE(sender == r_id); + s1_got_it = false; + }); + + auto tid = make_table_id(); + Advertisement ad(Topic("hello")); + + r.add_neighbour(adv_source_id,adv_source); + r.advertise(adv_source_id,tid, ad); + r.add_neighbour(adv_sink1_id,adv_sink1); + REQUIRE(s1_got_it); + + r.remove_neighbour(adv_source_id); + REQUIRE(!s1_got_it); + } + + SECTION("subs_are_forwarded") + { + NodeID r_id("src"); + Router r(r_id); + + auto pid = make_node_id(); + Advertisement ad(Topic("hello")); + auto tid = make_table_id(); + auto producer = std::make_shared(); + bool p1_got_sub=false; + producer->on_subscribe.connect([tid,&p1_got_sub] (NodeID const&,TableID const&stid,SubscriptionID const&sid) { + REQUIRE(tid == stid); + REQUIRE(sid.value() == 1); + p1_got_sub =true; + }); + + auto pid2 = make_node_id(); + auto tid2 = make_table_id(); + auto producer2 = std::make_shared(); + producer2->on_subscribe.connect([] (NodeID const&, TableID const&,SubscriptionID const&) { + REQUIRE(false); + }); + + auto sid = make_node_id(); + auto consumer = std::make_shared(); + + + r.add_neighbour(pid,producer); + r.advertise(pid,tid, ad); + + r.add_neighbour(pid2,producer2); + r.advertise(pid2,tid2,ad); + + r.add_neighbour(sid,consumer); + r.subscribe(sid,tid,SubscriptionID(1)); + REQUIRE(p1_got_sub); + } + + SECTION("unsubs_are_forwarded") + { + NodeID r_id("src"); + Router r(r_id); + + auto pid = make_node_id(); + auto tid = make_table_id(); + auto producer = std::make_shared(); + bool p1_got_sub=false; + + producer->on_subscribe.connect([tid,&p1_got_sub] (NodeID const&,TableID const&stid,const SubscriptionID&sid) { + REQUIRE(tid == stid); + REQUIRE(sid.value() == 1); + p1_got_sub =true; + }); + + producer->on_unsubscribe.connect([tid,&p1_got_sub] (NodeID const&,TableID const&stid) { + REQUIRE(tid == stid); + p1_got_sub =false; + }); + + auto pid2 = make_node_id(); + auto tid2 = make_table_id(); + auto producer2 = std::make_shared(); + producer2->on_subscribe.connect([] (NodeID const&, TableID const&,const SubscriptionID&) { + REQUIRE(false); + }); + + producer2->on_unsubscribe.connect([] (NodeID const&, TableID const&) { + REQUIRE(false); + }); + + auto sid = make_node_id(); + auto consumer = std::make_shared(); + + + r.add_neighbour(pid,producer); + Advertisement ad(Topic("hello")); + r.advertise(pid,tid, ad); + + r.add_neighbour(pid2,producer2); + r.advertise(pid2,tid2,ad); + + r.add_neighbour(sid,consumer); + r.subscribe(sid,tid,SubscriptionID(1)); + REQUIRE(p1_got_sub); + r.unsubscribe(sid,tid); + REQUIRE(!p1_got_sub); + } +} + + +}} diff --git a/src/lib/security_model/SecurityHelper.cpp b/src/lib/security_model/SecurityHelper.cpp new file mode 100755 index 000000000..fd148be06 --- /dev/null +++ b/src/lib/security_model/SecurityHelper.cpp @@ -0,0 +1,435 @@ + +#ifndef NO_CPP0X +#include "SecurityHelper.h" + +#include "lib/logging/logging.h" +#include "lib/utils/bts_assert.h" +#include "lib/utils/datetime.h" +#include "lib/utils/is_unittest_environment.h" +#include "lib/os/os.h" + +#include +#include +#include +#include + +namespace bts { +namespace md { + +const double NO_PRICE = std::numeric_limits().quiet_NaN(); + +namespace SecurityHelper { + +const std::string& symbol(const Security& sec) +{ + switch(sec.security_type()) + { + case SecurityType::OPTION: + return sec.option().symbol(); + case SecurityType::FUTURE: + return sec.future().symbol(); + case SecurityType::STOCK: + return sec.stock().symbol(); + case SecurityType::SPREAD: + return sec.spread().symbol(); + default: + bts_assert(false); + } +} + + +const std::string usymbol(const Security& sec) { + return boost::to_upper_copy(symbol(sec)); +} + +template std::string date_string (T const &contract) +{ + std::ostringstream result; + result << std::setw( 2 ) << std::setfill( '0' ) << contract.expiration_month () << "/" + << std::setw( 2 ) << std::setfill( '0' ) << contract.expiration_day () << "/" + << contract.expiration_year ()-2000; + return result.str (); +} + +// This should generate the same descriptor string as FixedSecurity.h's descriptor method +const std::string descriptor(const Security& security) +{ + bts_assert(security.security_type()); + const char separator = ' '; + std::ostringstream result; + result << symbol(security) << separator << Exchange::type_Name(security.exchange()); + switch(security.security_type()) + { + case SecurityType::OPTION: { + auto const &option = security.option (); + result << separator + << date_string(option) << separator + << option.strike() + << (option.call_put() == security_model::PutOrCall::CALL? "C":"P"); + break; + } + case SecurityType::FUTURE: { + auto const &future = security.future(); + result << separator << date_string(future); + break; + } + case SecurityType::SPREAD: { + const auto& spread = security.spread(); + for (int x = 0; x < spread.legs_size(); ++x) { + auto& leg = spread.legs(x); + security_key leg_key = leg.security().security_key(); + if (leg_key == invalid_security_key) return ""; // Cannot generate descriptor + result << separator << leg_key << "-q" << spread.legs(x).leg_quantity() << "p" << leg.leg_price() << "d" << leg.leg_delta() << Side::type_Name(leg.leg_side()); + } + break; + } + case SecurityType::STOCK: + break; + default: + LOG(MarketData,Error, "Unknown security type: {} ", + SecurityType::type_Name(security.security_type())); + bts_assert(false); + } + return result.str (); +} + + +void set_symbol(Security& security, const std::string& symbol) +{ + bts_assert(security.security_type()); + switch(security.security_type()) + { + case SecurityType::OPTION: + security.mutable_option()->set_symbol(symbol); break; + case SecurityType::FUTURE: + security.mutable_future()->set_symbol(symbol); break; + case SecurityType::STOCK: + security.mutable_stock()->set_symbol(symbol); break; + case SecurityType::SPREAD: + security.mutable_spread()->set_symbol(symbol); break; + default: + LOG(MarketData,Error, "Unknown security type: {} ", + SecurityType::type_Name(security.security_type())); + bts_assert(false); + } +} + +security_key key(const Security& key) { + if (key.has_security_key()) { return key.security_key(); } + return invalid_security_key; +} + +security_model::Exchange::type from_b2_exchange (const std::string &exchange) +{ + if (exchange == "CME") + return security_model::Exchange::GLBX; + else { + security_model::Exchange::type result; + bts_assert (security_model::Exchange::type_Parse (exchange, &result)); + return result; + } +} + +security_model::SecurityType::type from_b2_security_type (const std::string &security_type) +{ + if (security_type == "ETF" || security_type == "Equity" || security_type == "Index") + return security_model::SecurityType::STOCK; + else if (security_type == "InterestRateFuture") + return security_model::SecurityType::FUTURE; + else { + auto upper = boost::to_upper_copy (security_type); + security_model::SecurityType::type result; + bts_assert (security_model::SecurityType::type_Parse (upper, &result)); + return result; + } +} + +boost::posix_time::ptime expiration (const Security&security) +{ + if (security.has_option ()) { + auto &option = security.option (); + return datetime::make_ptime (option.expiration_year (), + option.expiration_month (), + option.expiration_day ()); + } + else if (security.has_future ()) { + auto &future = security.future (); + return datetime::make_ptime (future.expiration_year (), + future.expiration_month (), + future.has_expiration_day () ? + future.expiration_day () : 1); + } else { + bts_assert (false); + return boost::posix_time::ptime (); + } +} + +bool has_expiration (const Security&security) +{ + return security.has_option () || security.has_future (); +} + +double get_min_tick_size (Security const&security) +{ + bts_assert (security.exchange () == security_model::Exchange::GLBX || + (security.exchange () == security_model::Exchange::TESTEXCHANGE && + utils::internal::is_unittest_environment ())); + + double tick_size = 0; + if (security.has_cme_min_price_increment ()) + tick_size = security.cme_min_price_increment (); + else { + // lookup VTT see + // http://www.cmegroup.com/confluence/display/EPICSANDBOX/Variable+Tick+Table + bts_assert (security.has_cme_vtt ()); + auto vtt = security.cme_vtt (); + + bts_assert (security.has_display_factor ()); + bts_assert (security.display_factor () != 0);; + + switch (vtt) + { + case 1: + tick_size = 5; + break; + case 2: + tick_size = 0.5; + break; + case 3: + tick_size = 1; + break; + case 4: + tick_size = 5; + break; + case 10: + tick_size = 5; + break; + case 11: + tick_size = 5; + break; + case 12: + tick_size = 0.25; + break; + default: + break; + } + } + + bts_assert (tick_size >0); + return tick_size * security.display_factor (); +} + +extern "C" double raw_get_tick_size (double price, double min_tick_size, double display_factor, int vtt) +{ + if (vtt == 0) + return min_tick_size * display_factor; + else { + double P = price / display_factor; + double tick_size = 0; + // decoding the VTT. See: + // http://www.cmegroup.com/confluence/display/EPICSANDBOX/Variable+Tick+Table + switch (vtt) + { + case 1: + if (P < -500) + tick_size = 10; + else if (-500 <= P && P <= 500) + tick_size = 5; + else if (P > 500) + tick_size = 10; + break; + case 2: + if (-5 <= P && P <= 5) + tick_size = 0.5; + else if (P > 5) + tick_size = 1; + break; + case 3: + if (-10 <= P && P <= 10) + tick_size = 1; + else if (P < 10) + tick_size = 2; + break; + case 4: + if (P < -500) + tick_size = 25; + else if (-500 <= P && P <= 500) + tick_size = 5; + else if (P > 500) + tick_size = 25; + break; + case 10: + if (P < -300) + tick_size= 25; + else if (-300 <= P && P <= 300) + tick_size = 5; + else if (P > 300) + tick_size = 25; + break; + case 11: + if (P < -300) + tick_size = 10; + else if (-300 <= P && P <=300) + tick_size = 5; + else if (P > 300) + tick_size = 10; + break; + case 12: + if (P < -5) + tick_size = 0.50; + else if(-5 <= P && P <= 5) + tick_size = 0.25; + else if (P > 5) + tick_size = 0.5; + break; + default: + break; + } + return tick_size * display_factor; + } +} + +double get_tick_size (Security const&security, + double raw_price) +{ + bts_assert (security.exchange () == security_model::Exchange::GLBX || + (security.exchange () == security_model::Exchange::TESTEXCHANGE && + utils::internal::is_unittest_environment ())); + bts_assert (!os::is_nan (raw_price)); + bts_assert (security.has_display_factor ()); + bts_assert (security.display_factor () != 0);; + + if (security.has_cme_min_price_increment ()) + return security.cme_min_price_increment () * security.display_factor (); + else { + // lookup VTT see + // http://www.cmegroup.com/confluence/display/EPICSANDBOX/Variable+Tick+Table + bts_assert (security.has_cme_vtt ()); + auto vtt = security.cme_vtt (); + return raw_get_tick_size (raw_price,0,security.display_factor (),vtt); + } +} + +#ifdef WIN32 +// Fucking Microsoft +long llround(double d) +{ + return floor(d + 0.5); +} +#endif + +// a suficient factor that we can multiply a floating price for to +// obtain an integer with no loss of information. +int get_to_integer_shift (double) +{ + return 1000000; +} + +extern "C" double raw_round_to_quote_price (double raw_price, double tick_size, security_model::Side::type side) +{ + bts_assert (!os::is_nan (raw_price)); + bts_assert (!os::is_nan (tick_size)); + + double factor = get_to_integer_shift (tick_size); + auto i_price = llround (factor * raw_price); + auto i_tick_size = llround (tick_size * factor); + bts_assert (i_tick_size >0); + + long new_i_price = 0; + long mod = i_price % i_tick_size; + + if (mod == 0) + // price is already a valid tick + new_i_price = i_price; + else if (side == security_model::Side::BUY) { + // buy, round down + if (i_price > 0) + new_i_price = i_price - mod; + else + new_i_price = i_price - mod - i_tick_size; + } + else if (side == security_model::Side::SELL) { + // sell, round up + if (i_price > 0) + new_i_price = i_price - mod + i_tick_size; + else + new_i_price = i_price - mod; + } + + return new_i_price / factor; +} + +double round_to_quote_price (Security const& security, double raw_price, security_model::Side::type side) +{ + auto raw_result = raw_round_to_quote_price (raw_price, + get_tick_size (security, raw_price), + side); + if (security.security_type () != security_model::SecurityType::SPREAD) + return std::max (raw_result,0.0); + else + return raw_result; +} + +Side::type reduce_sides_ (Side::type s1, Side::type s2) +{ + if (s1 == s2) + return Side::BUY; + else + return Side::SELL; +} + +void get_flat_spread_legs_ (const Security& security, Side::type parent_side, + double parent_leg_quantity,std::vector *out) +{ + bts_assert (security.has_spread ()); + + for (auto const &original_leg : security.spread().legs()) { + + auto new_side = reduce_sides_(parent_side, original_leg.leg_side()); + auto new_quantity = parent_leg_quantity * original_leg.leg_quantity(); + + if (original_leg.security ().has_spread ()) + get_flat_spread_legs_ (original_leg.security (),new_side,new_quantity,out); + else { + out->emplace_back(original_leg); + auto &new_leg = out->back(); + new_leg.set_leg_side(new_side); + new_leg.set_leg_quantity(new_quantity); + } + } +} + +std::vector get_flat_spread_legs (const Security& security) +{ + bts_assert (security.has_spread ()); + + std::vector legs; + get_flat_spread_legs_(security, Side::BUY, 1, &legs); + return legs; +} + +void flatten_spread (const Security& in, Security* out) +{ + bts_assert (in.has_spread ()); + + *out = in; + auto out_legs = out->mutable_spread()->mutable_legs(); + out_legs->Clear(); + auto flat_legs = get_flat_spread_legs(in); + for (auto const &leg : flat_legs) + *out_legs->Add() = leg; +} + +bool is_covered_spread(const Security&in) { + if (!in.has_spread()) + return false; + auto const&spread = in.spread(); + for (auto const &l : spread.legs()) + if (!l.security().has_option()) + return true; + return false; +} + +}}}//namespace + +#endif diff --git a/src/lib/security_model/SecurityHelper.h b/src/lib/security_model/SecurityHelper.h new file mode 100755 index 000000000..b93f3fc6a --- /dev/null +++ b/src/lib/security_model/SecurityHelper.h @@ -0,0 +1,63 @@ +#pragma once + +#include "lib/utils/bts_export.h" +#include "lib/security_model/security.pb.h" + +#include +#include + + +namespace boost { +namespace posix_time { +class ptime; +}} + +namespace bts { +namespace md { + +typedef security_model::SecurityType SecurityType; +typedef security_model::Security Security; +typedef security_model::Exchange Exchange; +typedef security_model::Leg Leg; +using security_model::PutOrCall; +using security_model::Side; + +const int32_t COCONUT_MAX_DEPTH = 10; +BTS_EXPORT const extern double NO_PRICE; + +typedef uint32_t security_key; +const security_key invalid_security_key = 0; + +namespace SecurityHelper +{ +BTS_EXPORT const std::string& symbol(const Security&); +BTS_EXPORT boost::posix_time::ptime expiration (const Security&); +BTS_EXPORT bool has_expiration (const Security&); +BTS_EXPORT bool is_covered_spread(const Security&); +BTS_EXPORT const std::string usymbol(const Security&); +BTS_EXPORT const std::string descriptor(const Security&); +BTS_EXPORT void set_symbol(Security&, const std::string&); +BTS_EXPORT security_key key(const Security& key); +BTS_EXPORT security_model::Exchange::type from_b2_exchange (const std::string &exchange); +BTS_EXPORT security_model::SecurityType::type from_b2_security_type (const std::string &exchange); +BTS_EXPORT double to_b2_expiration (Security const&security); +BTS_EXPORT double get_tick_size (Security const& security,double raw_price); +BTS_EXPORT double get_min_tick_size (Security const& security); + +// this function will take the security type into account and convert +// negative prices to 0 prices unless the security is a spread. +BTS_EXPORT double round_to_quote_price (Security const& security, + double raw_price, + security_model::Side::type side); + +// this function will return negative prices if given negatives prices +// as an input, regardless of security type. call the function above +// if this is not what you want. +extern "C" BTS_EXPORT double raw_round_to_quote_price (double raw_price, double tick_size, security_model::Side::type side); +extern "C" BTS_EXPORT double raw_get_tick_size (double price, double min_tick_size, double display_factor, int vtt); + +// Flatten out recursive spreads +BTS_EXPORT void flatten_spread(const Security& in,Security* out); + +}}}//namespace + diff --git a/src/lib/security_model/security.proto b/src/lib/security_model/security.proto new file mode 100755 index 000000000..04fceb2a3 --- /dev/null +++ b/src/lib/security_model/security.proto @@ -0,0 +1,172 @@ +syntax='proto2'; +package bts.security_model; + +message Exchange +{ + enum type + { + TESTEXCHANGE = 0 ; + ARCA = 100 ; + CBOE = 205 ; + CBOEC2 = 206 ; + CBOECFE = 207 ; + CBOESX = 208 ; + GLBX = 301 ; + COMP = 305 ; + ICE = 306 ; + REPLAY = 999 ; + UNSET = 1200; + } +} + +message Side +{ + enum type + { + BUY = 49 ; + SELL = 50 ; + SHORT = 53 ; + SHORT_EXEMPT = 54 ; + } +} + +message PutOrCall +{ + enum type + { + CALL = 0 ; + PUT = 1 ; + BOTH = 2 ; + } +} + +message SecurityType +{ + enum type + { + UNSET = 0; + STOCK = 1 ; + OPTION = 2 ; + FUTURE = 3 ; + SPREAD = 4 ; + } +} + +message MultiLegType +{ + enum type + { + OUTRIGHT = 49 ; + LEG = 50 ; + SPREAD = 51 ; + } +} + + +message SecuritySubType +{ + enum type + { + COMBO = 0 ; + COVERED = 1 ; + } +} + +message OptionContract +{ + optional uint32 expiration_year = 1; + optional uint32 expiration_month = 2; + optional uint32 expiration_day = 3; + optional double strike = 4; + required string symbol = 5; + required PutOrCall.type call_put = 6; +} + +message FutureContract +{ + optional uint32 expiration_year = 1; + optional uint32 expiration_month = 2; + optional uint32 expiration_day = 3; + required string symbol = 4; +} + +message StockContract +{ + required string symbol = 1; + optional string suffix = 2; +} + +message Leg +{ + required Security security = 1; + required Side.type leg_side = 4; + optional double leg_quantity = 5; + optional double leg_price = 6; + //A Note about leg_delta at the CME: when a user requests the + //creation of a 'covered' spread at cme (either an outright covered + //by one future or an options spread covered by 1 or 2 futures (2 + //is allowed only if there are at least two options expirations)) + //the user creates the spread by submitting the options leg and + //then asking CME to size the futures leg according to the value + //specified in leg_delta. Leg_delta is the delta of the FUTURES + //LEG, so a bought call paired with futures leg of side = sell with + //leg_delta = say .3, would mean for each spread done the trader + //would do .3 futures. Option Legs NEED: leg_side, leg_quantity + //FUTURES Legs need: leg_side, leg_price (the tie price), leg_delta + //(really, it's still a quantity) + optional double leg_delta = 7; +} + +message MultiLegSecurity +{ + optional string symbol = 20; + optional bool normalized = 30 [default = false]; + repeated Leg legs = 40; +} + +message Security +{ + optional string security_id = 1; + optional string security_desc = 2; + required SecurityType.type security_type = 3; + required Exchange.type exchange = 10; + + optional OptionContract option = 4; + optional FutureContract future = 5; + optional StockContract stock = 6; + optional MultiLegSecurity spread = 7; + + optional double display_factor = 8 [default = 1.0]; + optional double strike_display_factor = 20 [default = 1.0]; + optional string platform_id = 9; // An id for internal use by the platform + optional uint64 numeric_id = 11; + optional uint32 class_key = 12; + optional uint32 contract_size = 13; + optional string cfi_code = 14; + optional int32 underlying_product = 15; + + optional double settlement_price = 16; + optional uint32 settlement_date = 17; + optional uint32 security_key = 18; + + // CME specific fields + optional string cme_symbol = 30; + optional string cme_security_group = 31; + optional int32 cme_vtt = 32 ; + optional int32 cme_display_decimals = 33; + optional int32 cme_denominator_sub_fraction = 34; + optional double cme_min_price_increment = 35; + optional uint32 cme_market_segment_id = 36; + optional double cme_unit_of_measure_qty = 37; + + // CBOE specific fields + optional double cboe_premium_break = 50; + optional double cboe_minimum_above_premium_fraction = 51; + optional double cboe_minimum_below_premium_fraction = 52; +} + +message SecurityList { + repeated Security securities = 1; +} + + diff --git a/src/lib/utils/CallTimer.h b/src/lib/utils/CallTimer.h new file mode 100755 index 000000000..81928f4fb --- /dev/null +++ b/src/lib/utils/CallTimer.h @@ -0,0 +1,78 @@ +#pragma once + +#include "lib/utils/bts_no_copy.h" + +#include "datetime.h" +#include "lib/logging/logging.h" +#include +#include + + +#if 0 // Change to 1 to enable tracing +#define TIME_FUNCTION(MAX_US) static utils::LogAtExit __lae__(__FUNCTION__); \ + utils::CBTimer timer(__lae__.sum, __lae__.max, __lae__.count, __lae__.slow_count, __FUNCTION__, __LINE__); +#define LOG_IF_SLOW TIME_FUNCTION(1000) // 1 ms +#else +#define TIME_FUNCTION(MAX_MS) +#define LOG_IF_SLOW +#endif + +namespace bts { +namespace utils { + +template +class CBTimer +{ +public: + CBTimer(double& sum, double& max, double& count, double& slow_count, const char* fn, uint32_t line) : + _time(tbb::tick_count::now()), + _fn(fn), + _line(line), + _max(max), + _sum(sum), + _count(count), + _slow_count(slow_count) + {} + + ~CBTimer() { + double us = (tbb::tick_count::now() - _time).seconds() * 1000 * 1000; + _count++; + _sum += us; + _max = std::max(us, _max); + if (us > T) { + _slow_count++; + LOG(Gen,Warn,"Slow callback at {}:{} {} us",_fn ,_line ,us ); + } + } + +private: + BTS_NO_COPY(CBTimer); + tbb::tick_count _time; + const char* _fn; + uint32_t _line; + double& _max; + double& _sum; + double& _count; + double& _slow_count; +}; + +struct LogAtExit { + LogAtExit(const char* fn) : fn(fn), sum(0), max(0), count(0), slow_count(0) { + } + const char* fn; + double sum; + double max; + double count; + double slow_count; + + ~LogAtExit() { + LOG(Gen, Info, "{} -- average: {} max: {} count: {} slow_count: {}", + (sum/count),max,count,slow_count); + } +private: + BTS_NO_COPY(LogAtExit); +}; + +}}//namespace + + diff --git a/src/lib/utils/Dispatcher.cpp b/src/lib/utils/Dispatcher.cpp new file mode 100755 index 000000000..bb9c30f5f --- /dev/null +++ b/src/lib/utils/Dispatcher.cpp @@ -0,0 +1,299 @@ +#ifndef NO_CPP0X + +#include "Dispatcher.h" +#include +#include "lib/utils/bts_assert.h" +#include "../os/os.h" +#include "defer.h" +#include "lib/utils/bts_chrono.h" +#include "lib/logging/logging.h" + +#include + +#include +#include +#include +#include +#include + +using namespace std; +using namespace boost::asio; + +namespace bts { +namespace utils { + +struct DispatcherImpl +{ +public: + DispatcherImpl() : + thread(std::this_thread::get_id()), + service(), + work(), + is_pool(false) + { + } + + std::thread::id thread; + io_service service; + unique_ptr work; + bool is_pool; +}; + + +Dispatcher::Dispatcher() : + _me(new DispatcherImpl()) +{ +#ifdef _WIN32 + _mm_mfence(); +#else + __sync_synchronize(); +#endif +} + +void wait_for_stop(io_service& service) +{ + while(!service.stopped()) + { + std::this_thread::sleep_for(1ms); + } +} + +Dispatcher::~Dispatcher() +{ + if (!_me->service.stopped()) + { + _me->work.reset(); + _me->service.stop(); + wait_for_stop(_me->service); + } +} + +void Dispatcher::run(size_t threads) +{ + bts_assert_msg(_me && _me->work == nullptr, "Error dispatcher is already running."); + _me->service.reset(); + _me->work.reset(new io_service::work(_me->service)); + DEFER(_me->work.reset()); + // Start N-1 threads in addition to the current one + for (size_t x = 1; x < threads; ++x) { + add_thread(); + } + _me->service.run(); +} + +void Dispatcher::add_thread() { + _me->is_pool = true; + boost::thread t([=]() { + os::set_thread_name("DispatcherPool"); + _me->service.run(); + }); + t.detach(); +} + +bool Dispatcher::is_threadpool() { + return _me->is_pool; +} + +bool Dispatcher::is_running() +{ + return _me->work != nullptr && !_me->service.stopped(); +} + +void Dispatcher::terminate() +{ + // allows run() to exit normally + _me->work.reset(); + // At this point this Dispatcher instance may be asyncronously deleted and is no longer safe to use. +} + +bool Dispatcher::check_access() +{ + return std::this_thread::get_id() == _me->thread; +} + + +boost::thread_specific_ptr _tl_dispatchers; +Dispatcher& Dispatcher::current(bool set_name) { + if (_tl_dispatchers.get() == nullptr) { + if(set_name) os::set_thread_name("Dispatcher"); + _tl_dispatchers.reset(new Dispatcher()); + } + return *(_tl_dispatchers.get()); +} + +Dispatcher& Dispatcher::get_named(const char* name) { + static std::mutex mutex; + static std::map s_dispatchers; + std::lock_guard lock(mutex); + auto result = s_dispatchers.find(name); + if (result != s_dispatchers.end()) { + return *result->second; + } + + Dispatcher& d = create_new(name); + s_dispatchers[name] = &d; + return d; +} + +Dispatcher& Dispatcher::background() +{ + static Dispatcher* s_dispatcher = nullptr; + Dispatcher** p_s_dispatcher = &s_dispatcher; + if (s_dispatcher == nullptr) + { + *p_s_dispatcher = &get_named("DispatcherBG"); + } + + return *s_dispatcher; +} + +Dispatcher& Dispatcher::create_new(const char* name) { + Dispatcher* new_dispatcher = nullptr; + mutex m; + condition_variable c; + // Take the lock before starting the thread to ensure c.notify_all() cannot happen before we wait on the condition + unique_lock lock(m); + boost::thread t([&]() { + Dispatcher& temp = current(false); + os::set_thread_name(name); + try { + temp.post([&]() { + unique_lock lock(m); + new_dispatcher = ¤t(); + c.notify_all(); + }); + temp.run(); + } + catch(const std::exception& ex) { + LOG(Gen,Error ,"Exception on dispatcher thread: {}" ,ex.what()); + bts_assert_msg(false, ex.what()); + } + }); + t.detach(); + auto result = c.wait_for(lock, chrono::seconds(5));// releases the lock temporarily + bts_assert_msg(result == cv_status::no_timeout, "Dispatcher failed to start."); + bts_assert(new_dispatcher); // Could happen to to spurious wake + return *new_dispatcher; +} + +void Dispatcher::invoke(Action action) +{ + bts_assert(!_me->service.stopped()); + + // If we're calling from the thread that owns this dispatcher + // We can just call the function - to do otherwise would result in deadlock + if (check_access()) + { + action(); + return; + } + + mutex m; + condition_variable c; + unique_lock lock(m); + bool complete = false; + _me->service.dispatch([&]() + { + action(); + unique_lock lock(m); + complete = true; + c.notify_all(); + }); + while(!complete) { + c.wait(lock); + } +} + +void Dispatcher::post(Action action, int64_t delay_ms) +{ + if (delay_ms > 0) + { + shared_ptr timer = make_shared(_me->service); + timer->expires_from_now(boost::posix_time::milliseconds(delay_ms)); + timer->async_wait([timer, action] (const boost::system::error_code& err) { if (!err) action(); }); + } + else + { + _me->service.post(action); + } +} + +// +// DispatcherTimer +// + +struct DispatcherTimerImpl +{ + DispatcherTimerImpl(Dispatcher& dispatcher, int64_t ms, Action& cb) : + running(false), + callback(cb), + delay(ms), + timer(new deadline_timer(dispatcher._me->service)) + { + } + + void schedule() + { + if (running) + { + timer->expires_from_now(boost::posix_time::milliseconds(delay)); + timer->async_wait(std::bind(&DispatcherTimerImpl::execute, this, std::placeholders::_1)); + } + } + + void execute(const boost::system::error_code& error) + { + if (!error) + { + callback(); + schedule(); + } + } + + bool running; + Action callback; + int64_t delay; + unique_ptr timer; +}; + + +DispatcherTimer::DispatcherTimer(Dispatcher& dispatcher, int64_t time_ms, Action cb) + : _me(new DispatcherTimerImpl(dispatcher, time_ms, cb)) +{ +} + +DispatcherTimer::~DispatcherTimer() { } + +void DispatcherTimer::start() +{ + _me->running = true; + _me->schedule(); +} + +void DispatcherTimer::stop() +{ + _me->running = false; + _me->timer->cancel(); +} + +bool DispatcherTimer::running() +{ + return _me->running; +} + +void DispatcherTimer::set_interval(int64_t time_ms) +{ + // This invalidates existing waits so restart + _me->timer->expires_from_now(boost::posix_time::milliseconds(time_ms)); + _me->schedule(); +} + +void DispatcherTimer::set_callback(Action cb) +{ + _me->callback = cb; +} + + +}}//namespace + +#endif diff --git a/src/lib/utils/Dispatcher.h b/src/lib/utils/Dispatcher.h new file mode 100755 index 000000000..cf0f74fc5 --- /dev/null +++ b/src/lib/utils/Dispatcher.h @@ -0,0 +1,116 @@ +#ifndef NO_CPP0X +#pragma once + +#include "lib/utils/bts_no_copy.h" +#include "lib/utils/bts_assert.h" +#include "lib/utils/bts_export.h" + +#include +#include +#include + +/* + * Dispatcher is a simple delegate based event queue built on boost::asio. + * + * Each dispatcher is owned by a single thread which is responsible for calling run() to dispatch events. + * The run call will not return until terminate() is called. It is safe to call terminate() from the same thread + * + * Sample use: + * Create a thread and get Dispatcher::current() for that thread. Pass that Dispatcher& to some other code. + * If that other code makes calls through the dispatcher post() (asyncronous) or invoke() (syncronous) the callback + * is guarunteed to happen on the Dispatcher's thread. + * + * + * Some use cases: + * 1) Short asyncronous work + * You can use Dispatcher::background() to get a handle to a common background thread (created on demand). + * This is useful for short tasks like asyncronous logging or timer events where you do not need to monopolize a thread + * of your own. You can also use post() with a delay to set up one time callbacks after a short time. + * Ex: Simple async logging + * #define ALOG(x) Dispatcher::background().post([=]() { LOG(Info, Something) << x; }); + * + * 2) Use a DispatcherTimer to drive a WorkScheduler without tying up a thread during the idle times. + * Ex: + * Dispatcher& d = Dispatcher::background(); + * WorkScheduler* s = new WorkScheduler(...); + * DispatcherTimer t(d, 100, [=]() { s->dispatch(); }); + * t.start(); + * + * Now the WorkScheduler will be invoked every 100ms but you are still free to use + * d.post([](){ ... }); to do asyncronous work between timer ticks. If your async task takes longer + * than the timer interval the timer callback will fire immediately when it is done. Simlilarly if + * the timer callback is still working when you call post() your call will be queued and fire when the + * timer routine finishes. + * + * 3) Controlling access to non-thread safe resources. + * If you have an object which is not thread safe you can give it a Dispatcher "owner". + * A thread can test object.owner().check_access() to see if it is the owning thread and if not + * serialize access by using object.owner().invoke([&](){ object.unsafe_method(); }; + * The unsafe_method might assert that access is only allowed from its owning thread. + * + */ + +namespace bts { +namespace utils { + +struct DispatcherImpl; +struct DispatcherTimerImpl; +class DispatcherTimer; + +typedef std::function Action; + +class BTS_EXPORT Dispatcher +{ +public: + Dispatcher(); + ~Dispatcher(); + + void run(size_t threads=1); + void add_thread(); + bool check_access(); + void terminate(); + bool is_running(); + bool is_threadpool(); + + // Get/create the dispatcher for the current thread + static Dispatcher& current(bool set_name=true); + // Get a default background thread to dispatch work to + static Dispatcher& background(); + // Create a new background dispatcher thread and return the dispatcher for it + static Dispatcher& get_named(const char* name); + static Dispatcher& create_new(const char* name); + + // Perform an action on the dispatcher thread. Block until the action is complete. + void invoke(Action action); + // Queue an action on the dispatcher thread and return immediately. + void post(Action action, int64_t delay_ms=0); + +private: + BTS_NO_COPY(Dispatcher); + friend struct DispatcherTimerImpl; + std::unique_ptr _me; +}; + + + +class BTS_EXPORT DispatcherTimer +{ +public: + DispatcherTimer(Dispatcher& dispatcher, int64_t time_ms, Action cb); + ~DispatcherTimer(); + void start(); + void stop(); + + bool running(); + void set_interval(int64_t time_ms); + void set_callback(Action cb); + +private: + BTS_NO_COPY(DispatcherTimer); + std::unique_ptr _me; +}; + + +}}//namespace + +#endif diff --git a/src/lib/utils/HTTPStream.cpp b/src/lib/utils/HTTPStream.cpp new file mode 100755 index 000000000..6b66345e0 --- /dev/null +++ b/src/lib/utils/HTTPStream.cpp @@ -0,0 +1,100 @@ +#include "HTTPStream.h" +#include "lib/logging/logging.h" +#include "lib/utils/defer.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace std; +using namespace boost; +using namespace boost::asio; + +namespace bts { +namespace utils { + +int read_headers(istream& input, map& result) { + string line; + getline(input, line); + + auto code_start = line.find_first_of(" "); + auto code_end = line.find_first_of(" ", code_start+1); + string http_code = line.substr(code_start+1, (code_end - code_start - 1)); + boost::algorithm::trim(http_code); + while(true) { + std::getline(input, line); + auto index = line.find_first_of(" "); + if (index == string::npos) break; + result.insert(std::make_pair(line.substr(0, index-1), boost::algorithm::trim_copy(line.substr(index+1)))); + } + return boost::lexical_cast(http_code); +} + +HTTPStream::HTTPStream(const std::string& host, const std::string& path, const std::string& verb) : + ip::tcp::iostream(host, "http"), + _response(-1), + _headers() +{ + // Send the request + *this << verb << " " << path << " HTTP/1.0\r\n"; + *this << "HOST: " << host << "\r\n"; + *this << "Accept: text/*" << "\r\n"; + *this << "\r\n"; + + if (bad()) { + LOG(MarketData,Error,"Failed to connect to host {}. Error: {}", + host,error().message() ); + bts_assert_msg(!bad(), error().message()); + } + + // parse the headers + _response = read_headers(*this, _headers); + + // Leave stream ready for reading body +} + + +HTTPStream::~HTTPStream() { } + + +bool download_file_if_newer(const string& host, const string& path, const string& destination) { + struct stat st = boost::initialized_value; + stat(destination.c_str(), &st); + // Begin download and check http headers + HTTPStream download(host, path, "GET"); + tm modtm = boost::initialized_value; + const char* rfc_1123_fmt = "%a, %d %b %Y %H:%M:%S %Z"; + stringstream ss(download.headers().at("Last-Modified")); + ss >> std::get_time(&modtm, rfc_1123_fmt); + time_t modified = mktime(&modtm); + + // If our file mtime is newer do not download + if (difftime(modified, st.st_mtime) > 0) { + LOG(MarketData,Info,"Downloading from: {}{}",host,path); + ofstream writer(destination.c_str(), ios::out | ios::binary); + if (writer) { + // This removes all '\n' characters + //copy(istream_iterator(download), istream_iterator(), os treambuf_iterator(writer)); + std::vector buffer(1024); + while (!download.eof()) { + download.read(&buffer[0], buffer.size()); + writer.write(&buffer[0], download.gcount()); + } + return true; + } + } + else { + LOG(MarketData,Info,"Skipping download as cache is newer"); + } + + return false; +} + +}}//namespace + diff --git a/src/lib/utils/HTTPStream.h b/src/lib/utils/HTTPStream.h new file mode 100755 index 000000000..d382b18f7 --- /dev/null +++ b/src/lib/utils/HTTPStream.h @@ -0,0 +1,30 @@ +#pragma once +#include "bts_export.h" +#include "lib/utils/windows_wrapper.h" +#include "lib/utils/bts_no_copy.h" +#include + +#include + +namespace bts { +namespace utils { + +class BTS_EXPORT HTTPStream : public boost::asio::ip::tcp::iostream +{ +public: + HTTPStream(const std::string& host, const std::string& path, const std::string& verb="GET"); + ~HTTPStream(); + static const int OK = 200; + + int response() { return _response; } + const std::map& headers() const { return _headers; } + +private: + int _response; + std::map _headers; + BTS_NO_COPY(HTTPStream); +}; +bool download_file_if_newer(const std::string& host, const std::string& path, const std::string& destination); + +}}//namespace + diff --git a/src/lib/utils/PBUtils.cpp b/src/lib/utils/PBUtils.cpp new file mode 100755 index 000000000..f48664b8e --- /dev/null +++ b/src/lib/utils/PBUtils.cpp @@ -0,0 +1,17 @@ +#include "PBUtils.h" + +#include + +namespace bts { +namespace utils { +namespace pb { + +std::string to_string (google::protobuf::Message const &msg) +{ + std::string result; + google::protobuf::TextFormat::PrintToString (msg, &result); + return result; +} + + +}}} diff --git a/src/lib/utils/PBUtils.h b/src/lib/utils/PBUtils.h new file mode 100755 index 000000000..4acd1858f --- /dev/null +++ b/src/lib/utils/PBUtils.h @@ -0,0 +1,17 @@ +#pragma once + +#include +#include "bts_export.h" + +namespace google { +namespace protobuf { +class Message; +}} + +namespace bts { +namespace utils { +namespace pb { +BTS_EXPORT std::string to_string (google::protobuf::Message const &msg); +}}} + + diff --git a/src/lib/utils/SharedList.h b/src/lib/utils/SharedList.h new file mode 100644 index 000000000..d93faab9d --- /dev/null +++ b/src/lib/utils/SharedList.h @@ -0,0 +1,309 @@ +#pragma once + +#include "lib/utils/bts_no_copy.h" +#include "lib/utils/defer.h" +#include "lib/utils/SharedMemory.h" +#include "lib/utils/bts_array_size.h" +#include "lib/logging/logging.h" + +#include "lib/os/os.h" +#include "lib/config/config.h" +#include "lib/config/protos/system.pb.h" + +#include +#include +#include +#include + +#include + +namespace bts { +namespace utils { + + +/* + * SharedList - Write only, single thread owner, list in shared memory + * SharedListReader - Multi thread / multi reader safe contains the same items as SharedList of the same name + * + * SharedListEntryBase, SharedListHeaderBase + * - example structures for use in the SharedList showing what fields are required in the template + * + * Notes: + * initial_size and block_size should be multiples of the page size (4k) + * Not doing this won't break but will waste space. + */ + +struct ALIGNED_(CACHE_LINE) SegmentHeader { + SegmentHeader() : count(0) {} + size_t count; +}; + +inline bool should_share(bool desire_shared) { + using namespace bts::config; + return desire_shared && !get().unittest_environment(); +} + +// Single writer, single thread! +template +class SharedList +{ + +public: + SharedList(const std::string& name, size_t initial_size_kb = 4096, size_t block_size_kb = 1024 /*1mb*/, bool shared=true) : + _name(name), + _use_count(0), + _block_size(block_size_kb *1024), + _current_offset(0), + _current_region_size(initial_size_kb * 1024), + _segments_mapped(0), + _current_region(nullptr), + _current_segment_header(nullptr), + _header(nullptr), + _shm(create_low_level_shared_memory(should_share(shared))) + { + if (shared) delete_segments(_name); + + // Start with 1 segment + expand(_current_region_size); + + // Create the header in the first segment + _header = new (allocate(sizeof(THeader))) THeader(); + _header->segment_count = 1; + + bts_assert_d(_current_offset > 0); + } + + ~SharedList() { + LOG(MarketData,Info, "~SharedList {}" ,_name); + delete_segments(_name); + } + + THeader& header() { + bts_assert_d(_header != nullptr); + return *_header; + } + + TEntry* add(TEntry& entry) { + // We pass in an entry to copy isntead of forwarding parameters and using + // an in place construction to allow entry to vary in size + enter(); + DEFER(leave()); + void* buffer = allocate(entry.size()); + memcpy(buffer, &entry, entry.size()); + // After the entry is copied increment the segment counter and total counter + // So anyone walking will see the values + _current_segment_header->count++; + _header->count++; + return (TEntry*)buffer; + } + + // Allow the client to pass in a size instead of a fully constructed object + // Assumption is that the result will be cast to the right type. + // Returned memory will be zeroed. + byte* add(size_t entry_size) { + enter(); + DEFER(leave()); + byte* buffer = allocate(entry_size); + memset(buffer, 0, entry_size); + _current_segment_header->count++; + _header->count++; + return buffer; + } + + static void get_segment_name(char* buffer, size_t buffer_len, const std::string& basename, int index) { + bts_assert((size_t) sprintf(buffer, "%s_%i", basename.c_str(), index) < buffer_len); + } + + static int delete_segments(const std::string& name) { + using namespace bts::config; + if (!should_share(true)) return 0; + std::unique_ptr pshm = create_low_level_shared_memory(true); + + int x; + int count = 0; + for (x = 0; x < 1024; x++) { + char segment_name[512]; + get_segment_name(segment_name, BTS_ARRAYSIZE(segment_name), name, x); + bool removed = pshm->release(segment_name); + if (removed) ++count; + if (x - count > 10) { break; } + } + return count; + } + +private: + void expand(size_t size) { + //using namespace boost::interprocess; + char segment_name[512]; + get_segment_name(segment_name, BTS_ARRAYSIZE(segment_name), _name, _segments_mapped); + + LOG(MarketData,Info,"Creating shared segment '{}'", std::string(segment_name)); + + _current_region = _shm->create(segment_name, size).address; + bts_assert(_current_region); + _current_region_size = size; + _current_offset = 0; + // Start each segment with a SegmentHeader + _current_segment_header = new (allocate(sizeof(SegmentHeader))) SegmentHeader(); + if (_header) { _header->segment_count++; } + bts_assert(_current_segment_header->count == 0); + _segments_mapped++; + } + + byte* allocate(uint32_t size) { + bts_assert(_current_region); + bts_assert(size < _block_size); + // Always leave 1 blank TEntry at the end of the buffer + if (_current_offset + size + sizeof(TEntry) >= _current_region_size) { + expand(_block_size); + } + bts_assert(_current_offset + size < _current_region_size); + byte* result = _current_region + _current_offset; + _current_offset += size; + return result; + } + + void enter() { bts_assert(++_use_count == 1); } + void leave() { --_use_count; } + +private: + BTS_NO_COPY(SharedList); + const std::string _name; + std::atomic _use_count; + + size_t _block_size; + size_t _current_offset; + size_t _current_region_size; + size_t _segments_mapped; + byte* _current_region; + SegmentHeader* _current_segment_header; + THeader* _header; + std::unique_ptr _shm; +}; + +// Thread safe to read +template +class SharedListReader +{ +public: + SharedListReader(const std::string& name, bool shared=true) : + _name(name), + _header(nullptr), + _mutex(), + _shm(create_low_level_shared_memory(should_share(shared))), + _segments() + { + init(); + } + + bool is_initialized() const { return _header != nullptr; } + + const THeader& header() { + init(); + if (!is_initialized()) return header_static_holder(); + return *_header; + } + + template + void all_entries(CB callback) { + init(); + if (!is_initialized()) return; + + // Copy the region list to avoid holding the lock + // Once a region is allocated it is never freed or moved so this is safe + tbb::queuing_mutex::scoped_lock lock(_mutex); + std::vector segments(_segments.begin(), _segments.end()); + lock.release(); + + // For each segment + // Start at 0 and walk until the end or until size=0 + // Also stop when we reach the header count + size_t offset = sizeof(THeader) + sizeof(SegmentHeader); // The first segment has both a SegmentHeader and a THeader + size_t count = 0; + + for (const auto& segment : segments) { + size_t segment_count = 0; + const byte* region_base = segment.address; + size_t region_size = segment.size; + SegmentHeader* seg_header = (SegmentHeader*)region_base; + while (offset < region_size && segment_count < seg_header->count) { + const TEntry* entry = (TEntry*)(region_base + offset); + ++count; ++segment_count; + callback(*entry); + offset += entry->size(); + if (count == _header->count) return; + } + // Each segment starts with a SegmentHeader + offset = sizeof(SegmentHeader); + } + } + +private: + BTS_NO_COPY(SharedListReader); + + void init() { + tbb::queuing_mutex::scoped_lock lock(_mutex); + if (is_initialized()) { check_segments(); return; } + // Find the first segment + char segment_name[512]; + SharedList::get_segment_name(segment_name, BTS_ARRAYSIZE(segment_name), _name, 0); + auto segment = _shm->open(segment_name, !os::is_windows()); + if (segment) { + LOG(MarketData,Info, "Opening shared segment '{}' - SUCCEEDED",std::string(segment_name)); + _header = (THeader*)(segment.address + sizeof(SegmentHeader)); + _segments.push_back(segment); + // Map all segments + check_segments(); + } + else { + LOG(MarketData,Info, "Opening shared segment '{}' - FAILED",std::string(segment_name)); + return; + } + } + + void check_segments() { + if (_segments.size() != _header->segment_count) { + for (uint32_t x = _segments.size(); x < _header->segment_count; ++x) { + char segment_name[512]; + SharedList::get_segment_name(segment_name, BTS_ARRAYSIZE(segment_name), _name, x); + auto segment = _shm->open(segment_name, !os::is_windows()); + bts_assert(segment); + _segments.push_back(segment); + } + } + } + + const THeader& header_static_holder() const { + static THeader header; + return header; + } + + const std::string _name; + THeader* _header; // null value indicates no memory segment + tbb::queuing_mutex _mutex; + std::unique_ptr _shm; + std::vector _segments; +}; + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Weffc++" +#endif + +struct SharedListHeaderBase { + size_t count; + size_t segment_count; +}; + +struct SharedListEntryBase { + // Using a method allows us to compute a fixed size with no space cost + uint32_t size() const { return sizeof(SharedListEntryBase); } + bool is_public; +}; + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif + +}}//namespace + diff --git a/src/lib/utils/SharedMemory.cpp b/src/lib/utils/SharedMemory.cpp new file mode 100644 index 000000000..cab42d06c --- /dev/null +++ b/src/lib/utils/SharedMemory.cpp @@ -0,0 +1,201 @@ +#ifndef NO_CPP0X + +#include "SharedMemory.h" +#include "lib/utils/bts_assert.h" +#include "lib/logging/logging.h" + +#include +#include +#include +#include +#include + +#include +#include + + +using namespace std; +using namespace boost::interprocess; + +namespace bts { +namespace utils { + +struct BoostLowLevelSharedMemoryImpl; +class BoostLowLevelSharedMemory : public LowLevelSharedMemory +{ + std::unique_ptr _me; +public: + BoostLowLevelSharedMemory(); + + // Inherited via LowLevelSharedMemory + virtual Segment create(const std::string & name, size_t size) override; + virtual const Segment open(const std::string& name, bool read_only) override; + virtual bool release(const std::string & name) override; +}; + +struct PrivateSharedMemoryImpl; +struct RcSegment +{ + RcSegment() : name(""), size(0), buffer() {} + RcSegment(string n, size_t s) : + name(n), size(s), buffer(make_shared>(s)) + {} + string name; + size_t size; + std::shared_ptr> buffer; + byte* data() const { return buffer ? &(*buffer)[0] : nullptr; } + + operator Segment() const { + Segment s = { size, data() }; + return s; + } +}; +class PrivateSharedMemory : public LowLevelSharedMemory +{ + vector> _segments; +public: + PrivateSharedMemory(); + ~PrivateSharedMemory(); + + // Inherited via LowLevelSharedMemory + virtual Segment create(const std::string & name, size_t size) override; + virtual const Segment open(const std::string& name, bool read_only) override; + virtual bool release(const std::string & name) override; +}; + +/* + * Boot shared memory impl + */ + +Segment make_segment(size_t size, void* ptr) { + Segment s = {size, (byte*)ptr }; + return s; +} + +struct BoostLowLevelSharedMemoryImpl { + BoostLowLevelSharedMemoryImpl() : + _mappings() + { } + + void add_region(shared_memory_object block) { + mapped_region region(block, block.get_mode()); + _mappings.push_back(make_pair(move(block), move(region))); + } + + Segment create(const string & name, size_t size) { + try { + shared_memory_object block(create_only, name.c_str(), read_write); + block.truncate(size); + add_region(move(block)); + return make_segment(size, _mappings.back().second.get_address()); + } + catch (...) { + return make_segment(0, nullptr); + } + } + Segment open(const string& name, bool ro) { + try { + add_region(shared_memory_object(open_only, name.c_str(), ro ? read_only : read_write)); + return make_segment(_mappings.back().second.get_size(), _mappings.back().second.get_address()); + } + catch (...) { + return make_segment(0, nullptr); + } + } + + bool release(const string& segment_name) { + _mappings.erase( + remove_if(_mappings.begin(), _mappings.end(), [&](const pair& p) { + return p.first.get_name() == segment_name; + }), _mappings.end()); + LOG(MarketData,Diag, "Remove shared memory segment: {}" , std::string(segment_name)); + return boost::interprocess::shared_memory_object::remove(segment_name.c_str()); + } + vector> _mappings; +}; + +BoostLowLevelSharedMemory::BoostLowLevelSharedMemory() : + _me(std::make_unique()) +{} +Segment utils::BoostLowLevelSharedMemory::create(const string & name, size_t size) { + return _me->create(name, size); +} +const Segment BoostLowLevelSharedMemory::open(const std::string & name, bool read_only) { + return _me->open(name, read_only); +} +bool utils::BoostLowLevelSharedMemory::release(const string & name) { + return _me->release(name); +} + +/* + * Non-shared memory implementation + */ +struct PrivateSharedMemoryImpl { + PrivateSharedMemoryImpl() : + _mutex(), _mallocs() + {} + std::mutex _mutex; + map _mallocs; + + RcSegment create(const string & name, size_t size) { + std::lock_guard lock(_mutex); + RcSegment segment(name, size); + // If an old segment existed it is discarded + _mallocs[name] = segment; + return segment; + } + RcSegment open(const string& name, bool) { + std::lock_guard lock(_mutex); + auto find_result = _mallocs.find(name); + if (find_result != _mallocs.end()) { + return find_result->second; + } + return RcSegment(); + } + bool release(const string& name) { + std::lock_guard lock(_mutex); + auto find_result = _mallocs.find(name); + if (find_result != _mallocs.end()) { + _mallocs.erase(name); + return true; + } + return false; + } + static PrivateSharedMemoryImpl& instance() { + static PrivateSharedMemoryImpl impl; + return impl; + } +}; + + +PrivateSharedMemory::PrivateSharedMemory() : _segments() {} +PrivateSharedMemory::~PrivateSharedMemory() { + for (auto& pair : _segments) { + bool owned = pair.first; + if (owned) { + PrivateSharedMemoryImpl::instance().release(pair.second.name); + } + } +} +Segment PrivateSharedMemory::create(const string & name, size_t size) { + auto result = PrivateSharedMemoryImpl::instance().create(name, size); + if (result.buffer) _segments.push_back(make_pair(true, result)); + return (Segment)result; +} +const Segment PrivateSharedMemory::open(const std::string & name, bool read_only) { + auto result = PrivateSharedMemoryImpl::instance().open(name, read_only); + if (result.buffer) _segments.push_back(make_pair(false, result)); + return (Segment)result; +} +bool PrivateSharedMemory::release(const string & name) { + return PrivateSharedMemoryImpl::instance().release(name); +} + +std::unique_ptr create_low_level_shared_memory(bool shared) { + return unique_ptr(shared ? + static_cast(new BoostLowLevelSharedMemory()) : + static_cast(new PrivateSharedMemory())); +} + +}}//namespace +#endif diff --git a/src/lib/utils/SharedMemory.h b/src/lib/utils/SharedMemory.h new file mode 100644 index 000000000..e44bc56fe --- /dev/null +++ b/src/lib/utils/SharedMemory.h @@ -0,0 +1,30 @@ +#pragma once + +#include "lib/utils/bts_no_copy.h" +#include "lib/utils/bts_export.h" + +#include +#include +#include /*pair*/ + +namespace bts { +namespace utils { +typedef uint8_t byte; + +struct Segment { + size_t size; + byte* address; + operator bool() const { return size != 0 && address != nullptr; } +}; + +struct LowLevelSharedMemory { + virtual ~LowLevelSharedMemory() {}; + virtual Segment create(const std::string& name, size_t size) = 0; + virtual const Segment open(const std::string& name, bool read_only) = 0; + virtual bool release(const std::string& name) = 0; +}; + +BTS_EXPORT std::unique_ptr create_low_level_shared_memory(bool shared); + +}}//namespace + diff --git a/src/lib/utils/Sqlite_utils.cpp b/src/lib/utils/Sqlite_utils.cpp new file mode 100644 index 000000000..4d31926a3 --- /dev/null +++ b/src/lib/utils/Sqlite_utils.cpp @@ -0,0 +1,35 @@ +#ifndef WIN32 +#include "Sqlite_utils.h" +#include "bts_assert.h" +#include +namespace bts { +namespace utils { + +void sqlite3_exec_nofail(sqlite3* db, const char* sql) { + char* error_message = nullptr; + int result = sqlite3_exec(db, sql, nullptr, nullptr, &error_message); + bts_assert_msg(result == SQLITE_OK, error_message); +} + +void sqlite3_exec_nofail(sqlite3_ptr & db, const char * sql) { + return sqlite3_exec_nofail(db.get(), sql); +} + +sqlite3_ptr open_db(const char* path) { + sqlite3* temp; + bts_assert(sqlite3_open(path, &temp) == SQLITE_OK); + return sqlite3_ptr(temp, sqlite3_close); +} + +sqlite3_stmt_ptr prepare_query(sqlite3 * db, const char * query) { + sqlite3_stmt* pquery; + const char* tail = nullptr; + sqlite3_prepare_v2(db, query, -1, &pquery, &tail); + bts_assert(tail == nullptr || strlen(tail) == 0);// A tail indicates multiple statements in prepared statement string + return sqlite3_stmt_ptr(pquery, sqlite3_finalize); +} +sqlite3_stmt_ptr prepare_query(sqlite3_ptr& db, const char* query) { return prepare_query(db.get(), query); } + + +}}//namespace +#endif diff --git a/src/lib/utils/Sqlite_utils.h b/src/lib/utils/Sqlite_utils.h new file mode 100644 index 000000000..8f5837dba --- /dev/null +++ b/src/lib/utils/Sqlite_utils.h @@ -0,0 +1,24 @@ +#pragma once +#ifndef WIN32 +#include +#include +#include "lib/utils/bts_export.h" + +typedef struct sqlite3 sqlite3; +typedef struct sqlite3_stmt sqlite3_stmt; +typedef std::unique_ptr sqlite3_ptr; +typedef std::unique_ptr sqlite3_stmt_ptr; + +namespace bts { +namespace utils { + +BTS_EXPORT void sqlite3_exec_nofail(sqlite3* db, const char* sql); +BTS_EXPORT void sqlite3_exec_nofail(sqlite3_ptr& db, const char* sql); + +BTS_EXPORT sqlite3_ptr open_db(const char* path); +BTS_EXPORT sqlite3_stmt_ptr prepare_query(sqlite3* db, const char* query); +BTS_EXPORT sqlite3_stmt_ptr prepare_query(sqlite3_ptr& db, const char* query); +#endif + +}}//namespace + diff --git a/src/lib/utils/TempDir.h b/src/lib/utils/TempDir.h new file mode 100755 index 000000000..c1bff1fcd --- /dev/null +++ b/src/lib/utils/TempDir.h @@ -0,0 +1,60 @@ +#pragma once + +#include "lib/os/os.h" + +#include +#include +#include + +namespace bts { +namespace utils { +class TempDir +{ +public: + TempDir () : path_ () + { + // stolen from TempFile + std::string temp; + char* temp_cstr = getenv("TEMP"); + if (temp_cstr != NULL) temp = temp_cstr; + bts::os::normalize_path(temp); + + if (!bts::os::is_windows() && temp.empty()) + temp = "/tmp"; + path_ = boost::filesystem::path (temp); + try + { + path_ /= boost::filesystem::unique_path ("bts-utest-%%%%-%%%%-%%%%-%%%%"); + } + catch(std::exception) + { + std::ostringstream digits; + digits << "bts-utest"; + for(int i = 0; i < 21; i++) + { + if(i % 5 == 0) + digits << "-"; + else + digits << rand() % 9; + } + path_ /= boost::filesystem::path (digits.str()); + } + + boost::filesystem::create_directory (path_); + } + + std::string name () + { + return path_.string (); + } + + ~TempDir () + { + boost::filesystem::remove_all (path_); + } +private: + boost::filesystem::path path_; +}; + +}} + diff --git a/src/lib/utils/TempFile.h b/src/lib/utils/TempFile.h new file mode 100755 index 000000000..762bfd4a0 --- /dev/null +++ b/src/lib/utils/TempFile.h @@ -0,0 +1,84 @@ +#ifndef TEMPFILE_H_ +#define TEMPFILE_H_ + +#include "lib/os/os.h" + +#ifdef _WIN32 +#include +#define mkstemps +#else +#include +#endif +#include "bts_assert.h" + +#include +#include +#include +#include +#include +#include + +namespace bts { +namespace utils { + +class TempFile +{ +private: + std::string _name; + +public: + TempFile() : _name() + { + std::string temp; + char* temp_cstr = getenv("TEMP"); + if (temp_cstr != NULL) temp = temp_cstr; + bts::os::normalize_path(temp); + + if (!bts::os::is_windows() && temp.empty()) + temp = "/tmp"; + temp += "/bts_tmp_XXXXXX"; + + char name_buf[1024] = {0}; + bts::os::strncpy(name_buf, temp.c_str(), sizeof(name_buf)); + +#ifdef _WIN32 + int result; + int count = 0; + FILE* fp; + do + { + count++; + bts_assert(count < 10); + result = _mktemp_s(name_buf, sizeof(name_buf)); + bts_assert(!result); + result = fopen_s(&fp, name_buf, "w"); + } while(result); + fclose(fp); +#else + int result = mkstemp(name_buf); + assert(result != -1); + close(result); +#endif + _name = name_buf; + } + + std::shared_ptr out() + { + std::shared_ptr o(new std::ofstream(_name.c_str (), std::ios::out)); + assert(!o->fail()); + return o; + } + + const std::string& path() + { + return _name; + } + + ~TempFile() + { + std::remove(_name.c_str()); + } +}; +}} +#endif + diff --git a/src/lib/utils/TimeStamp.h b/src/lib/utils/TimeStamp.h new file mode 100644 index 000000000..af56a5523 --- /dev/null +++ b/src/lib/utils/TimeStamp.h @@ -0,0 +1,170 @@ +#pragma once + +#ifdef _WIN32 +#include "windows_wrapper.h" +#elif __linux__ +#include +#else +#error Unsupported platform +#endif + +#include "../os/os.h" +#include "bts_assert.h" +#include +#include + +namespace bts { +namespace utils { + +class TimeStamp +{ +public: + TimeStamp() : _time(0) {} + explicit TimeStamp (uint64_t time) : _time (time) {} + + static TimeStamp now(); + static TimeStamp invalid() { return TimeStamp(std::numeric_limits().max()); } + + class Interval { + public: + Interval() : _val(0) {} + double seconds() const; + int64_t microseconds() const; + int64_t milliseconds() const; + int64_t nanoseconds() const; + + friend class bts::utils::TimeStamp; + friend Interval operator-( const TimeStamp& t1, const TimeStamp& t2); + friend Interval operator+( const Interval& a, const Interval& b) { + return Interval(a._val + b._val); + } + friend Interval operator-( const Interval& a, const Interval& b) { + return Interval(a._val - b._val); + } + Interval& operator+=(const Interval& other) { _val += other._val; return *this; } + Interval& operator-=(const Interval& other) { _val -= other._val; return *this; } + static Interval from_microseconds(uint64_t microseconds); + + private: + explicit Interval(uint64_t val) : _val(val) {} + uint64_t _val; + }; + + friend Interval operator - (const TimeStamp& t1, const TimeStamp& t2); + friend bool operator == (const TimeStamp& t1, const TimeStamp& t2); + friend bool operator != (const TimeStamp& t1, const TimeStamp& t2); + friend bool operator > (const TimeStamp& t1, const TimeStamp& t2); + friend bool operator < (const TimeStamp& t1, const TimeStamp& t2); + friend bool operator >= (const TimeStamp& t1, const TimeStamp& t2); + friend bool operator <= (const TimeStamp& t1, const TimeStamp& t2); + + TimeStamp operator-(const TimeStamp::Interval& delta) { + bts_assert_d(*this != TimeStamp::invalid()); + return TimeStamp(to_uint64() - delta._val); + } + + // implicit conversion to bool + operator bool() const { return _time != 0; } + + uint64_t to_uint64 () const + {return _time;} + + TimeStamp& shift_ns (int64_t ns) {_time += ns;return *this;} + TimeStamp& shift_us (int64_t us) {return shift_ns(us*1000);}; + TimeStamp& shift_ms (int64_t ms) {return shift_us(ms*1000);}; + TimeStamp& shift_secs (int64_t secs) {return shift_ms(secs*1000);}; + TimeStamp& shift_mins (int64_t mins) {return shift_secs(mins*60);}; + TimeStamp& shift_hours (int64_t hours) {return shift_mins(hours*60);}; + TimeStamp& shift_days (int64_t days) {return shift_hours(days*24);}; + +private: + uint64_t _time; +}; + +inline TimeStamp TimeStamp::now() { + TimeStamp result; +#ifdef _WIN32 + LARGE_INTEGER qpcnt; + QueryPerformanceCounter(&qpcnt); + result._time = qpcnt.QuadPart; +#else + struct timespec ts; + clock_gettime( CLOCK_MONOTONIC, &ts ); + result._time = + static_cast(1000000000UL)* + static_cast(ts.tv_sec) + + static_cast(ts.tv_nsec); +#endif + return result; +} + +inline TimeStamp::Interval TimeStamp::Interval::from_microseconds(uint64_t microseconds) { + TimeStamp::Interval result; +#ifdef _WIN32 + LARGE_INTEGER qpfreq; + QueryPerformanceFrequency(&qpfreq); + result._val = static_cast(microseconds * qpfreq.QuadPart / 1000000); +#else + result._val = microseconds * 1E3; +#endif + return result; +} + +inline double TimeStamp::Interval::seconds() const { + #ifdef _WIN32 + LARGE_INTEGER qpfreq; + QueryPerformanceFrequency(&qpfreq); + return _val / (double)qpfreq.QuadPart; + #else + return _val * 1E-9; + #endif +} +inline int64_t TimeStamp::Interval::microseconds() const { + #ifdef _WIN32 + LARGE_INTEGER qpfreq; + QueryPerformanceFrequency(&qpfreq); + return (_val * 1E6 )/ qpfreq.QuadPart; + #else + return _val / 1000; + #endif +} +inline int64_t TimeStamp::Interval::nanoseconds() const { + #ifdef _WIN32 + LARGE_INTEGER qpfreq; + QueryPerformanceFrequency(&qpfreq); + return (_val * 1000000000 )/ qpfreq.QuadPart; + #else + return _val ; + #endif +} + +inline int64_t TimeStamp::Interval::milliseconds() const { + return microseconds() / 1000; +} + +inline TimeStamp::Interval operator-(const TimeStamp& t1, const TimeStamp& t2) { + bts_assert_d(t1 != TimeStamp::invalid()); + bts_assert_d(t2 != TimeStamp::invalid()); + return TimeStamp::Interval(t1._time - t2._time); +} + +inline bool operator == ( const TimeStamp& t1, const TimeStamp& t2) { + return t1._time == t2._time; +} +inline bool operator != ( const TimeStamp& t1, const TimeStamp& t2) { + return t1._time != t2._time; +} +inline bool operator > ( const TimeStamp& t1, const TimeStamp& t2) { + return t1._time > t2._time; +} +inline bool operator < ( const TimeStamp& t1, const TimeStamp& t2) { + return t1._time < t2._time; +} +inline bool operator >= ( const TimeStamp& t1, const TimeStamp& t2) { + return t1._time >= t2._time; +} +inline bool operator <= ( const TimeStamp& t1, const TimeStamp& t2) { + return t1._time <= t2._time; +} + +}}//namespace diff --git a/src/lib/utils/Uuid.cpp b/src/lib/utils/Uuid.cpp new file mode 100755 index 000000000..f1aacbe0a --- /dev/null +++ b/src/lib/utils/Uuid.cpp @@ -0,0 +1,98 @@ +#include "Uuid.h" + +#include "lib/os/os.h" +#include "bts_array_size.h" +#include "bts_assert.h" +#include +#include +#include + +namespace bts { +namespace utils { + +std::string uuid () +{ + return Uuid ().to_text_string (); +} + +Uuid::Uuid (std::string const &uuid, bool is_byte_string) +{ + if (is_byte_string) { + bts_assert (uuid.size () == sizeof(_id)); + std::copy(uuid.begin(),uuid.end(),_id); + } else { + std::istringstream in (uuid); + in >> *this; + } +} + +Uuid::~Uuid (){} + +Uuid::Uuid () +{ + os::gen_uuid (_id); +} + +bool Uuid::operator==(Uuid const&rhs) const +{return memcmp (_id,rhs._id,BTS_ARRAYSIZE (_id)) == 0;} + +bool Uuid::operator!=(Uuid const&rhs) const +{return !operator==(rhs);} + +bool Uuid::operator<(Uuid const&rhs) const +{return std::lexicographical_compare (_id, _id + BTS_ARRAYSIZE (_id), rhs._id,rhs._id + BTS_ARRAYSIZE (rhs._id));} + +std::string Uuid::to_byte_string () const +{ + std::string publisher_id_string(BTS_ARRAYSIZE (_id),0); + std::copy(_id, _id + BTS_ARRAYSIZE (_id), publisher_id_string.begin()); + return publisher_id_string; +} + +std::string Uuid::to_text_string () const +{ + std::ostringstream out; + out << *this; + return out.str (); +} + +std::istream &operator>>(std::istream &stream,Uuid &sid) +{ + for (size_t i = 0; i < sizeof(sid._id); i++) { + char high_nibble, low_nibble; + stream >> high_nibble >> low_nibble; + if (high_nibble >= 'a') + high_nibble = high_nibble - 'a' + 10; + else + high_nibble -= '0'; + + if (low_nibble >= 'a') + low_nibble = low_nibble - 'a' + 10; + else + low_nibble -= '0'; + sid._id[i] = (high_nibble << 4) + low_nibble; + } + return stream; +} + +std::ostream &operator<<(std::ostream &stream,Uuid const&sid) +{ + for (size_t i = 0; i < sizeof(sid._id); i++) { + char high_nibble = (sid._id[i] & 0xf0) >> 4; + char low_nibble = sid._id[i] & 0x0f; + if (high_nibble >= 10) + high_nibble = (high_nibble - 10) + 'a'; + else + high_nibble += '0'; + if (low_nibble >= 10) + low_nibble = (low_nibble - 10) + 'a'; + else + low_nibble += '0'; + stream << high_nibble << low_nibble; + } + + return stream; +} + +}} + diff --git a/src/lib/utils/Uuid.h b/src/lib/utils/Uuid.h new file mode 100755 index 000000000..74c6293c2 --- /dev/null +++ b/src/lib/utils/Uuid.h @@ -0,0 +1,29 @@ +#pragma once + +#include +#include +#include "bts_export.h" + +namespace bts { +namespace utils { + +class BTS_EXPORT Uuid +{ +public: + Uuid (std::string const &sid, bool is_byte_string); + Uuid (); + virtual ~Uuid (); + bool operator==(Uuid const&rhs) const; + bool operator!=(Uuid const&rhs) const; + bool operator<(Uuid const&rhs) const ; + std::string to_byte_string () const; + std::string to_text_string () const; + friend BTS_EXPORT std::ostream &operator<<(std::ostream &stream,Uuid const&sid); + friend BTS_EXPORT std::istream &operator>>(std::istream &stream,Uuid &sid); +private: + unsigned char _id[16]; +}; + +BTS_EXPORT std::string uuid (); + +}} diff --git a/src/lib/utils/ValueWrapper.h b/src/lib/utils/ValueWrapper.h new file mode 100644 index 000000000..c9100e99a --- /dev/null +++ b/src/lib/utils/ValueWrapper.h @@ -0,0 +1,31 @@ +#pragma once +#include + +namespace bts { +namespace utils { +template +class ValueWrapper { +public: + typedef ValueT Value; + + ValueWrapper() : _v() {} + explicit ValueWrapper(ValueT const&v) : _v(v) {} + explicit ValueWrapper (ValueT &&v) : _v (std::move(v)) {} + + ValueWrapper(ValueWrapper const&v) : _v (v._v) {} + ValueWrapper(ValueWrapper &&v) : _v (std::move(v._v)) {} + + ValueWrapper &operator=(ValueWrapper const&v) {_v = v._v;return *this;} + ValueWrapper &operator=(ValueWrapper &&v) {_v = std::move(v._v);return *this;} + + bool operator==(ValueWrapper const&v) const {return _v == v._v;} + bool operator!=(ValueWrapper const&v) const {return _v != v._v;} + bool operator<(ValueWrapper const&v) const {return _v < v._v;} + + ValueT const& value() const {return _v;} + +private: + ValueT _v; +}; + +}} diff --git a/src/lib/utils/WaitFreeQueue.h b/src/lib/utils/WaitFreeQueue.h new file mode 100644 index 000000000..470dec5c4 --- /dev/null +++ b/src/lib/utils/WaitFreeQueue.h @@ -0,0 +1,178 @@ +#pragma once + +#include "lib/utils/bts_no_copy.h" +#include +#include +#include +#include +#include + +namespace bts { +namespace utils { + +/* + * WaitFreeQueue defines a circular buffer in the given space (which may be shared memory and should probably be at least 1mb) + * There may only be *one* writer thread which can write without *any* back pressure + * There may be any number of readers which must poll to read new values. + + * If the reader does not poll quickly enough it can be overrun by the writer in which case it does not see + * the lost values. In this case the overrun is detected and the read cursor re-syncronizes with the queue + * and returns the full current state along with the reset flag. + + * This class is optimized first for never blocking the writer. Latency and throughput are seconday but will probably be pretty good. + */ + +template +class WaitFreeQueue +{ + typedef int64_t position; + const static position invalid_id = -1; + // Store up to 4 per cache line + struct ALIGNED_(16) Entry { + Entry() : id(invalid_id), value() {} + std::atomic id; + T value; + + Entry& operator=(const Entry& other) { + set(other.value, other.id.load(std::memory_order_acquire)); + return *this; + } + + void set(const T& other, position new_id) { + id.store(invalid_id, std::memory_order_release); + value = other; + id.store(new_id, std::memory_order_release); + } + + bool try_read(T& result, position& out_id) const { + out_id = invalid_id; + auto curid = id.load(std::memory_order_acquire); + if (curid == invalid_id) return false; + result = value; + out_id = curid; + // The read is only valid if the id is unchanged + return (curid == id.load(std::memory_order_acquire)); + } + + }; + +public: + WaitFreeQueue(void* buffer, size_t size_bytes, bool writer=false) : + _pos(0), + _size(size_bytes / sizeof(Entry)), + _writer(writer), + _buffer((Entry*)buffer) { + // Initialize the buffer using the entry constructor + if (writer) { + for (size_t x = 0; x < _size; ++x) { + _buffer[x] = *( new (_buffer+x) Entry() ); + } + } + } + + ~WaitFreeQueue() { + // Mark the whole buffer as invalid + if (_writer) { + for (size_t x = 0; x < _size; ++x) { + _buffer[x].id = invalid_id; + } + } + } + + void push(const T& t) { + entry(_pos).set(t, _pos); + ++_pos; + } + + class Cursor + { + const WaitFreeQueue& _owner; + position _pos; + public: + Cursor(const WaitFreeQueue& owner) : + _owner(owner), _pos(0) + {} + Cursor(const WaitFreeQueue& owner, position pos) : + _owner(owner), _pos(pos) {} + public: + size_t read(std::vector& result, bool* p_overrun=nullptr) { + T temp; + size_t count = 0; + bool overrun; + while(pop(temp, overrun)) { + result.push_back(temp); + ++count; + } + if (overrun) { + // Attempt to re-sync with the queue + result.clear(); + _pos = invalid_id; + _pos = _owner.entry(0).id.load(); + if (_pos == invalid_id) { _pos = 0; } // If we attempted to jump to the head somehow + if (p_overrun) { *p_overrun = true; } + return read(result); + } + return count; + } + size_t read(google::protobuf::RepeatedPtrField& result, bool* p_overrun = nullptr) { + T temp; + size_t count = 0; + bool overrun; + while(pop(temp, overrun)) { + result.Add()->CopyFrom(temp); + ++count; + } + if (overrun) { + // Attempt to re-sync with the queue + result.Clear(); + _pos = invalid_id; + _pos = _owner.entry(0).id.load(); + if (_pos == invalid_id) { _pos = 0; } // If we attempted to jump to the head somehow + if (p_overrun) { *p_overrun = true; } + return read(result); + } + return count; + } + // Attempt to read the next value and detect writer overrun + bool pop(T& result, bool& overrun) { + overrun = false; + const Entry& entry = _owner.entry(_pos); + // If the next entry is what we expect we can read it + position id; + if (entry.try_read(result, id)) { + if (id == _pos) { + _pos++; + return true; + } + if (id > _pos) { + overrun = true; + } + } + return false; + } + + position current_position() { return _pos; } + }; + + const Cursor get_cursor() const { + return Cursor(*this); + } + +private: + position _pos; + size_t _size; + bool _writer; + Entry* _buffer; + + Entry& entry(position index) { + return _buffer[index % _size]; + } + const Entry& entry(position index) const { + return _buffer[index % _size]; + } + + BTS_NO_COPY(WaitFreeQueue); +}; + +}}//namespace + diff --git a/src/lib/utils/benchmarking_utils.h b/src/lib/utils/benchmarking_utils.h new file mode 100644 index 000000000..1dedc4aa4 --- /dev/null +++ b/src/lib/utils/benchmarking_utils.h @@ -0,0 +1,23 @@ +#pragma once + +#include +#include +#include "bts_assert.h" + +namespace bts { +namespace utils { + +template +double median(C const&c) +{ + bts_assert(c.size()); + C c2(c); + std::sort(c2.begin(),c2.end()); + if (c2.size() % 2 == 0) { + auto midx = c2.size() / 2; + return (c2[midx-1]+c2[midx]) / 2; + } else + return c2[c2.size()/2]; +} + +}} diff --git a/src/lib/utils/bts_array_size.h b/src/lib/utils/bts_array_size.h new file mode 100644 index 000000000..9daca0c90 --- /dev/null +++ b/src/lib/utils/bts_array_size.h @@ -0,0 +1,6 @@ +#pragma once + +#undef BTS_ARRAYSIZE +#define BTS_ARRAYSIZE(a) \ + ((sizeof(a) / sizeof(*(a))) / \ + static_cast(!(sizeof(a) % sizeof(*(a))))) diff --git a/src/lib/utils/bts_assert.h b/src/lib/utils/bts_assert.h new file mode 100644 index 000000000..d07542965 --- /dev/null +++ b/src/lib/utils/bts_assert.h @@ -0,0 +1,23 @@ +#pragma once +#include + +#ifdef _WIN32 +#define TERMINATE ::_exit(-1); +#else +#define TERMINATE abort (); +#endif + +#define bts_assert_msg(EXP, msg) do { \ + if (!(EXP)) { \ + std::cerr << "Assert failure: " << __FILE__ << ":" << __LINE__ << " Exp: " << #EXP; \ + TERMINATE; \ + }} while(0) + +#define bts_assert(EXP) bts_assert_msg(EXP, "") + +#ifndef NDEBUG +#define bts_assert_d bts_assert +#else +#define bts_assert_d(EXP) +#endif + diff --git a/src/lib/utils/bts_chrono.h b/src/lib/utils/bts_chrono.h new file mode 100644 index 000000000..052215a7f --- /dev/null +++ b/src/lib/utils/bts_chrono.h @@ -0,0 +1,37 @@ +#pragma once +#include +#include + +namespace bts { + +using Timer = std::chrono::system_clock; +using TimeStamp = Timer::time_point; +using Duration = Timer::duration; + +inline int64_t ns_diff (TimeStamp t1, TimeStamp t2) { + return std::chrono::duration_cast(t1 - t2).count() ; +} + +inline int64_t us_diff (TimeStamp t1, TimeStamp t2) { + return std::chrono::duration_cast(t1 - t2).count() ; +} + +inline int64_t ms_diff (TimeStamp t1, TimeStamp t2) { + return std::chrono::duration_cast(t1 - t2).count() ; +} + +inline int64_t sec_diff (TimeStamp t1, TimeStamp t2) { + return std::chrono::duration_cast(t1 - t2).count() ; +} + +} + +inline std::ostream &operator<<(std::ostream &o,const bts::TimeStamp ts) +{ + using namespace std::chrono; + auto now_c = bts::Timer::system_clock::to_time_t(ts); + o << std::string(std::ctime(&now_c)); + return o; +} + +using namespace std::chrono_literals; diff --git a/src/lib/utils/bts_debugger.h b/src/lib/utils/bts_debugger.h new file mode 100755 index 000000000..04e0f0a00 --- /dev/null +++ b/src/lib/utils/bts_debugger.h @@ -0,0 +1,16 @@ +#ifndef BTS_DEBUGGER_H_ +#define BTS_DEBUGGER_H_ + +#include "../os/os.h" +#include "windows_wrapper.h" + +#ifdef _WIN32 + #define DEBUG_BREAK() do { if (IsDebuggerPresent()) __debugbreak(); } while(0) +#else +//#include +//#include +// #define DEBUG_BREAK() do { if (ptrace(PTRACE_TRACEME, 0, NULL, 0) == -1) raise(SIGTRAP); } while(0) +#define DEBUG_BREAK() +#endif + +#endif diff --git a/src/lib/utils/bts_export.h b/src/lib/utils/bts_export.h new file mode 100644 index 000000000..1a9957639 --- /dev/null +++ b/src/lib/utils/bts_export.h @@ -0,0 +1,5 @@ +#ifdef _WIN32 +#define BTS_EXPORT __declspec(dllexport) +#else +#define BTS_EXPORT +#endif diff --git a/src/lib/utils/bts_no_copy.h b/src/lib/utils/bts_no_copy.h new file mode 100755 index 000000000..ad0fd60d4 --- /dev/null +++ b/src/lib/utils/bts_no_copy.h @@ -0,0 +1,10 @@ +#ifndef bts_no_copy_H +#define bts_no_copy_H + +#undef BTS_NO_COPY +#define BTS_NO_COPY(TYPE) \ + TYPE (const TYPE&); \ + TYPE &operator=(const TYPE&) + + +#endif diff --git a/src/lib/utils/catch_test_runner.cpp b/src/lib/utils/catch_test_runner.cpp new file mode 100644 index 000000000..1ae7ba70f --- /dev/null +++ b/src/lib/utils/catch_test_runner.cpp @@ -0,0 +1,20 @@ +#include "catch_test_runner.h" + +#define CATCH_CONFIG_RUNNER +#include "catch/catch.hpp" + +#include "lib/logging/logging.h" +#include "lib/config/config.h" +#include "lib/config/protos/logging.pb.h" + +namespace bts { +int run_tests (int argc, char **argv) +{ + auto cfg = config::get (); + cfg.mutable_console()->set_default_level("Fatal"); + log::configure_logging(cfg); + + int result = Catch::Session().run( argc-1, argv+1 ); + return ( result < 0xff ? result : 0xff ); +} +} diff --git a/src/lib/utils/catch_test_runner.h b/src/lib/utils/catch_test_runner.h new file mode 100644 index 000000000..5fc9d907a --- /dev/null +++ b/src/lib/utils/catch_test_runner.h @@ -0,0 +1,5 @@ +#pragma once + +namespace bts { +int run_tests (int argc, char **argv); +} diff --git a/src/lib/utils/checked_cast.h b/src/lib/utils/checked_cast.h new file mode 100644 index 000000000..79028e783 --- /dev/null +++ b/src/lib/utils/checked_cast.h @@ -0,0 +1,14 @@ +#pragma once +namespace bts { +namespace utils { + +template +inline To checked_cast(From& f) { +#ifdef DEBUG + return dynamic_cast(f); +#else + return static_cast(f); +#endif +} + +}} diff --git a/src/lib/utils/datetime.cpp b/src/lib/utils/datetime.cpp new file mode 100755 index 000000000..55a0f007e --- /dev/null +++ b/src/lib/utils/datetime.cpp @@ -0,0 +1,427 @@ +#include "datetime.h" + +#include "bts_assert.h" +#include "TimeStamp.h" + +#include +#include +#include + +namespace bts { +namespace datetime { + +const boost::gregorian::date anchor (1970, boost::gregorian::Jan, 1); + +unsigned long long ptime_to_ll (PTime const &t) +{ + long long days = (t.date () - anchor).days (); + long long usecs = t.time_of_day ().total_microseconds (); + return (days << 40) + usecs; +} + +LLDateTime ll_now () +{ + return ptime_to_ll (boost::posix_time::microsec_clock::universal_time ()); +} + +LLDateTime ll_local_now () +{ + return ptime_to_ll (boost::posix_time::microsec_clock::local_time ()); +} + +long unix_now() { + return (boost::posix_time::microsec_clock::universal_time() - PTime(anchor)).total_microseconds() / (long)1E6; +} + +long long ll_diff_in_us (unsigned long long ll1, unsigned long long ll2) +{ +#ifndef NO_CPP0X + auto p1 = ll_to_ptime (ll1); + auto p2 = ll_to_ptime (ll2); + return (p2 - p1).total_microseconds (); +#else + return 0; +#endif +} + + +long long ptime_diff_in_us (PTime const &t1, PTime const &t2) +{ + return (t1 - t2).total_microseconds (); +} + +long long ptime_diff_in_ms (PTime const &t1, PTime const &t2) +{ + return (t1 - t2).total_milliseconds (); +} + +long long ptime_diff_in_secs (PTime const &t1, PTime const &t2) +{ + return (t1 - t2).total_seconds (); +} + +PTime timestamp_to_ptime(utils::TimeStamp const&t) +{ + auto now_ts = utils::TimeStamp::now(); + auto now = datetime::now(); + + auto delta = (now_ts - t).milliseconds(); + return now - boost::posix_time::milliseconds(delta); +} + +utils::TimeStamp ptime_to_timestamp(PTime const &t) +{ + auto now_ts = utils::TimeStamp::now(); + auto now = datetime::now(); + + auto delta = (now - t).total_milliseconds(); + return now_ts.shift_ms (delta); +} + +long long usecs_since (PTime const &t1) +{ + return (boost::posix_time::microsec_clock::universal_time () - t1).total_microseconds (); +} + +long long msecs_since (PTime const &t1) +{ + return (boost::posix_time::microsec_clock::universal_time () - t1).total_milliseconds (); +} + +PTime now () +{ + return boost::posix_time::microsec_clock::universal_time (); +} + +PTime local_now () +{ + return boost::posix_time::microsec_clock::local_time (); +} + +PTime make_ptime (int year, int month, int day) +{ + boost::gregorian::date dt (year, boost::gregorian::greg_month (month),day); + return PTime (dt); +} + +PTime make_ptime (int year, int month, int day, int hour, int minute, int second, int millisecond) +{ + boost::gregorian::date dt (year, boost::gregorian::greg_month (month),day); + return PTime (dt, boost::posix_time::time_duration(hour, minute, second) + boost::posix_time::milliseconds(millisecond)); +} + +PTime ll_to_ptime (LLDateTime ll) +{ + boost::gregorian::date_duration days (ll >> 40); + boost::posix_time::time_duration usecs (boost::posix_time::microseconds (0x000000ffffffffff & ll)); + return PTime (anchor + days,usecs); +} + +extern "C" BTS_EXPORT int ll_to_cstr (LLDateTime ll, char*buffer, int buffer_sz) +{ + const int str_sz = sizeof ("20121204-14:44:02.000"); + bts_assert (buffer_sz >= str_sz); + auto str = FIX_timestamp (ll_to_ptime (ll)); + bts_assert (str.size () == str_sz-1); + memcpy (buffer,&str[0],str.size ()); + buffer[str.size ()] = '\0'; + return str.size (); +} + +std::string time_string (PTime const &t) +{ + int hours = t.time_of_day ().hours (); + int minutes = t.time_of_day ().minutes (); + int seconds = t.time_of_day ().seconds (); + int whole_ms = seconds * 1000 + minutes * 1000 * 60 + hours * 1000 * 60 * 60; + int frac_ms = t.time_of_day ().total_milliseconds () - whole_ms; + std::ostringstream out; + out << std::setw( 2 ) << std::setfill( '0' ) << hours << ":" + << std::setw( 2 ) << std::setfill( '0' ) << minutes << ":" + << std::setw( 2 ) << std::setfill( '0' ) << seconds << "." + << std::setw( 3 ) << std::setfill( '0' ) << frac_ms; + return out.str (); +} + +std::string date_string (PTime const &t) +{ + std::string result (10,' '); + sprintf (&result[0], "%02d/%02d/%04d", + t.date ().month ().as_number (), + t.date ().day ().as_number (), + (unsigned short) t.date ().year ()); + return result; +} + +std::string iso_date_time_string (PTime const &t) +{ + return boost::posix_time::to_iso_extended_string(t); +} + +std::string FIX_date_time_string (PTime const &t) +{ + std::string result (strlen ("20121204-14:44:02"),' '); + sprintf (&result[0],"%d%02d%02d-%02d:%02d:%02d", + (unsigned short) t.date ().year (), + t.date ().month ().as_number (), + t.date ().day ().as_number (), + t.time_of_day ().hours (), + t.time_of_day ().minutes (), + t.time_of_day ().seconds ()); + return result; +} + +void FIX_timestamp (PTime const &t, std::string *out) +{ + int hours = t.time_of_day ().hours (); + int minutes = t.time_of_day ().minutes (); + int seconds = t.time_of_day ().seconds (); + int whole_ms = seconds * 1000 + minutes * 1000 * 60 + hours * 1000 * 60 * 60; + int frac_ms = t.time_of_day ().total_milliseconds () - whole_ms; + out->resize(sizeof ("20121204-14:44:02.000")); + +#ifdef WIN32 + _snprintf (&(*out)[0],out->size (), + "%04d%02d%02d-%02d:%02d:%02d.%03d", + (int) t.date ().year (), + t.date ().month ().as_number (), + t.date ().day ().as_number (), + hours, + minutes, + seconds, + frac_ms); +#else + snprintf (&(*out)[0],out->size (), + "%04d%02d%02d-%02d:%02d:%02d.%03d", + (int) t.date ().year (), + t.date ().month ().as_number (), + t.date ().day ().as_number (), + hours, + minutes, + seconds, + frac_ms); +#endif + out->resize (out->size ()-1); +} + +std::string FIX_timestamp (PTime const &t) +{ + std::string result; + FIX_timestamp(t,&result); + return result; +} + +bool is_from_today(PTime const &time) +{ + return (time.date() == boost::posix_time::microsec_clock::universal_time().date()); +} + +bool is_at_least_msecs_old (PTime const &time, uint64_t ms) +{ + PTime now = boost::posix_time::microsec_clock::universal_time (); + return (now - time).total_milliseconds () >= (int64_t) ms; +} + +bool is_at_least_secs_old (PTime const &time, uint64_t sec) +{ + PTime now = boost::posix_time::microsec_clock::universal_time (); + return (now - time).total_seconds () >= (int64_t) sec; +} + +bool is_at_least_mins_old (PTime const &time, uint64_t mins) +{ + PTime now = boost::posix_time::microsec_clock::universal_time (); + return ((now - time).total_seconds ()/60) >= (int64_t) mins; +} + +BTS_EXPORT bool is_at_least_hours_old (PTime const &time, uint64_t hours) +{ + PTime now = boost::posix_time::microsec_clock::universal_time (); + int64_t secs = (int64_t) ((now - time).total_seconds ()); + return (secs/60/60) >= (int64_t) hours; +} + +BTS_EXPORT bool is_at_least_days_old (PTime const &time, uint64_t days) +{ + return is_at_least_hours_old (time, days *24); +} + +BTS_EXPORT LLDateTime ll_from_unix_epoch(long seconds) +{ + PTime result(anchor, boost::posix_time::seconds(seconds)); + return ptime_to_ll(result); +} + +BTS_EXPORT LLDateTime ll_from_unix_epoch_ms(long long milliseconds) +{ + PTime result(anchor, boost::posix_time::millisec(milliseconds)); + return ptime_to_ll(result); +} + +BTS_EXPORT void ll_to_parts(LLDateTime ll_time, int& year, int& month, int& day, int& hour, int& minute, int& second, int& millisecond) +{ + PTime t = ll_to_ptime(ll_time); + year = t.date().year(); + month = t.date().month(); + day = t.date().day(); + hour = t.time_of_day().hours(); + minute = t.time_of_day().minutes(); + second = t.time_of_day().seconds(); + int whole_ms = second * 1000 + minute * 1000 * 60 + hour * 1000 * 60 * 60; + millisecond = t.time_of_day().total_milliseconds() - whole_ms; +} + +BTS_EXPORT LLDateTime ll_from_parts(int year, int month, int day, int hour, int minute, int second, int millisecond) +{ + return ptime_to_ll(make_ptime(year, month, day, hour, minute, second, millisecond)); +} + +BTS_EXPORT LLDateTime ll_from_date(const Date& date) { + if (date.year == 0 && date.day == 0 && date.month == 0) { + return 0; + } + return ll_from_parts(date.year, date.month, date.day, 0, 0, 0, 0); +} + +Date ll_to_date(LLDateTime time) { + Date date; + int year, month, day, unused; + ll_to_parts(time, year, month, day, unused, unused, unused, unused); + date.year = year; + date.month = month; + date.day = day; + return date; +} + +struct UtcTimeStamp { + int hour; + int min; + int sec; + int msec; + int day; + int month; + int year; +}; + +struct UtcTimeStampConvertor +{ + static UtcTimeStamp convert( const std::string& value) + { + bool haveMilliseconds = false; + + switch( value.size() ) + { + case 21: haveMilliseconds = true; + case 17: break; + default: bts_assert(false); + } + + int i = 0; + int c = 0; + for( c = 0; c < 8; ++c ) + if( !isdigit(value[i++]) ) bts_assert(false); + if (value[i++] != '-') bts_assert(false); + for( c = 0; c < 2; ++c ) + if( !isdigit(value[i++]) ) bts_assert(false); + if( value[i++] != ':' ) bts_assert(false); + for( c = 0; c < 2; ++c ) + if( !isdigit(value[i++]) ) bts_assert(false); + if( value[i++] != ':' ) bts_assert(false); + for( c = 0; c < 2; ++c ) + if( !isdigit(value[i++]) ) bts_assert(false); + + if( haveMilliseconds ) + { + if( value[i++] != '.' ) bts_assert(false); + for( c = 0; c < 3; ++c ) + if( !isdigit(value[i++]) ) bts_assert(false); + } + + int year, mon, mday, hour, min, sec, millis; + + i = 0; + + year = value[i++] - '0'; + year = 10 * year + value[i++] - '0'; + year = 10 * year + value[i++] - '0'; + year = 10 * year + value[i++] - '0'; + + mon = value[i++] - '0'; + mon = 10 * mon + value[i++] - '0'; + if( mon < 1 || 12 < mon ) bts_assert(false); + + mday = value[i++] - '0'; + mday = 10 * mday + value[i++] - '0'; + if( mday < 1 || 31 < mday ) bts_assert(false); + + ++i; // skip '-' + + hour = value[i++] - '0'; + hour = 10 * hour + value[i++] - '0'; + // No check for >= 0 as no '-' are converted here + if( 23 < hour ) bts_assert(false); + + ++i; // skip ':' + + min = value[i++] - '0'; + min = 10 * min + value[i++] - '0'; + // No check for >= 0 as no '-' are converted here + if( 59 < min ) bts_assert(false); + + ++i; // skip ':' + + sec = value[i++] - '0'; + sec = 10 * sec + value[i++] - '0'; + + // No check for >= 0 as no '-' are converted here + if( 60 < sec ) bts_assert(false); + + if( haveMilliseconds ) + { + millis = (100 * (value[i+1] - '0') + + 10 * (value[i+2] - '0') + + (value[i+3] - '0')); + } + else + millis = 0; + + return UtcTimeStamp {hour, min, sec, millis, mday, mon, year}; + } +}; + +PTime parse_FIX_timestamp (const std::string &ts_str) +{ + auto ts = UtcTimeStampConvertor::convert (ts_str); + boost::gregorian::date dt(ts.year, ts.month, ts.day); + // fucking boost. unbelieavable. + auto ticks_per_second = boost::posix_time::time_duration::rep_type::res_adjust(); + boost::posix_time::time_duration td (ts.hour, ts.min, ts.sec, ts.msec * (ticks_per_second / 1000)); + return PTime(dt, td); +} + +#ifndef NO_CPP0X +extern "C" BTS_EXPORT LLDateTime ll_today() { + return datetime::ll_from_date(today()); +} + +BTS_EXPORT Date today() { + auto now = datetime::now(); + Date date; + date.year = now.date().year(); + date.month = now.date().month(); + date.day = now.date().day(); + return date; +} +BTS_EXPORT Date add_days(Date indate, int days) { + boost::gregorian::date dt(indate.year, indate.month, indate.day); + auto adjusted_dt = dt + boost::gregorian::days(days); + Date date(adjusted_dt.year(), adjusted_dt.month(), adjusted_dt.day()); + return date; +} + +extern "C" BTS_EXPORT LLDateTime ll_yesterday() { + auto now = datetime::now() - boost::gregorian::days(1); + return datetime::ll_from_parts(now.date().year(), now.date().month(), now.date().day(), 0, 0, 0, 0); +} +#endif +}} diff --git a/src/lib/utils/datetime.h b/src/lib/utils/datetime.h new file mode 100755 index 000000000..31abefe44 --- /dev/null +++ b/src/lib/utils/datetime.h @@ -0,0 +1,84 @@ +#pragma once + +#include "bts_export.h" +#include +#include "windows_wrapper.h" // tick_count.h includes windows.h + +namespace boost {namespace posix_time {class ptime;}} +namespace bts {namespace utils {class TimeStamp;}} + +namespace bts { +namespace datetime { +typedef boost::posix_time::ptime PTime; + +typedef unsigned long long LLDateTime; +// Simple Date structure +struct Date { + Date() : year(0), month(0), day(0) {} + Date(uint16_t y, uint8_t m, uint8_t d) : year(y), month(m), day(d) {} + Date(uint16_t date) : year((date >> 9) + 1970), month(date >> 5 & 15), day(date&31) {} + uint16_t year; + uint8_t month; + uint8_t day; + bool operator<(const Date& other) const { return !(*this == other) && !(year > other.year) && !(month > other.month) && !(day > other.day); } + bool operator>(const Date& other) const { return !(*this == other) && !(*this < other); } + bool operator==(const Date& other) const { + return day == other.day && month == other.month && year == other.year; + } + bool operator!=(const Date& other) const { return !(*this == other); } + uint16_t to_short() { return ((year - 1970) << 9) + (month << 5) + day; } +}; + +extern "C" BTS_EXPORT LLDateTime ll_now (); +extern "C" BTS_EXPORT LLDateTime ll_local_now (); +extern "C" BTS_EXPORT double OA_date_from_ll (LLDateTime ll); +extern "C" BTS_EXPORT LLDateTime ll_from_OA_date (double oa_date); +extern "C" BTS_EXPORT long unix_now(); + +BTS_EXPORT PTime now (); +BTS_EXPORT PTime local_now (); +BTS_EXPORT PTime make_ptime (int year, int month, int day); +BTS_EXPORT PTime make_ptime (int year, int month, int day, int hour, int minute, int second, int millisecond); +BTS_EXPORT LLDateTime ptime_to_ll (PTime const &t); +BTS_EXPORT PTime ll_to_ptime (LLDateTime ll); +extern "C" BTS_EXPORT int ll_to_cstr (LLDateTime ll, char*buffer, int buffer_sz); +BTS_EXPORT long long ll_diff_in_us (unsigned long long start, unsigned long long stop); +BTS_EXPORT long long ptime_diff_in_us (PTime const &t1, PTime const &t2); +BTS_EXPORT long long ptime_diff_in_ms (PTime const &t1, PTime const &t2); +BTS_EXPORT long long ptime_diff_in_secs (PTime const &t1, PTime const &t2); +BTS_EXPORT PTime timestamp_to_ptime(utils::TimeStamp const&t); +utils::TimeStamp ptime_to_timestamp(PTime const &t); +BTS_EXPORT long long usecs_since (PTime const &t1); +BTS_EXPORT long long msecs_since (PTime const &t1); +BTS_EXPORT std::string time_string (PTime const &t); +BTS_EXPORT std::string date_string (PTime const &t); +BTS_EXPORT std::string iso_date_time_string (PTime const &t); +BTS_EXPORT std::string FIX_date_time_string (PTime const &t); +BTS_EXPORT PTime parse_FIX_timestamp (const std::string &ts); +BTS_EXPORT std::string FIX_timestamp (PTime const &t); +BTS_EXPORT void FIX_timestamp (PTime const &t,std::string *out); +BTS_EXPORT PTime parse_FIX_timestamp (const std::string &ts); +BTS_EXPORT bool is_from_today(PTime const &time); +BTS_EXPORT bool is_at_least_msecs_old (PTime const &time, uint64_t ms); +BTS_EXPORT bool is_at_least_secs_old (PTime const &time, uint64_t sec); +BTS_EXPORT bool is_at_least_mins_old (PTime const &time, uint64_t min); +BTS_EXPORT bool is_at_least_hours_old (PTime const &time, uint64_t hour); +BTS_EXPORT bool is_at_least_hours_old (PTime const &time, uint64_t hour); +BTS_EXPORT bool is_at_least_days_old (PTime const &time, uint64_t days); + +BTS_EXPORT LLDateTime ll_from_unix_epoch(long); +BTS_EXPORT LLDateTime ll_from_unix_epoch_ms(long long); +extern "C" BTS_EXPORT void ll_to_parts(LLDateTime, int& year, int& month, int& day, int& hour, int& minute, int& second, int& millisecond); +BTS_EXPORT Date ll_to_date(LLDateTime); +extern "C" BTS_EXPORT LLDateTime ll_from_parts(int year, int month, int day, int hour, int minute, int second, int millisecond); +BTS_EXPORT LLDateTime ll_from_date(const Date& date); + +#ifndef NO_CPP0X +extern "C" BTS_EXPORT LLDateTime ll_today(); +BTS_EXPORT Date today(); +BTS_EXPORT Date add_days(Date date, int days); +extern "C" BTS_EXPORT LLDateTime ll_yesterday(); +#endif + +}} + diff --git a/src/lib/utils/defer.h b/src/lib/utils/defer.h new file mode 100755 index 000000000..7268d1ddf --- /dev/null +++ b/src/lib/utils/defer.h @@ -0,0 +1,38 @@ +#pragma once + +#include "bts_no_copy.h" +#include + +namespace bts { +namespace utils { + +template +class DeferredAction +{ +public: + explicit DeferredAction(T& t) : + _action(std::addressof(t)) + { + } + + ~DeferredAction() + { + if (_action) + (*_action)(); + } + +private: + BTS_NO_COPY(DeferredAction); + T * _action; +}; + +#define DEFER_CONCAT_IMPL(x, y) x##y +#define DEFER_CONCAT(x, y) DEFER_CONCAT_IMPL(x, y) +#define DEFER_UNIQUE(prefix) DEFER_CONCAT(prefix, __LINE__ ) + +#define DEFER(...) \ + auto DEFER_UNIQUE(__DEFER_PROC_) = [&]() { __VA_ARGS__;}; \ + bts::utils::DeferredAction DEFER_UNIQUE(__DEFER_CONTAINER_)(DEFER_UNIQUE(__DEFER_PROC_)); + + +}} // namespace diff --git a/src/lib/utils/file_utils.cpp b/src/lib/utils/file_utils.cpp new file mode 100644 index 000000000..487f19ebf --- /dev/null +++ b/src/lib/utils/file_utils.cpp @@ -0,0 +1,25 @@ +#include "file_utils.h" +#include "lib/os/os.h" +#include "bts_assert.h" + +#include +#include +#include + +namespace bts { +namespace utils { + +void create_if_not_there (std::string const &path) +{ + if (!boost::filesystem::exists (path)) + bts_assert_msg (boost::filesystem::create_directories (path), "Unable to create directory"); +} + +std::string normalize_data_path (std::string const &input_path) +{ + std::string result (input_path); + boost::algorithm::replace_first (result,"$data_path", os::bts_data_path ()); + return result; +} + +}} diff --git a/src/lib/utils/file_utils.h b/src/lib/utils/file_utils.h new file mode 100644 index 000000000..56555ba5b --- /dev/null +++ b/src/lib/utils/file_utils.h @@ -0,0 +1,9 @@ +#pragma once + +#include + +namespace bts { +namespace utils { +std::string normalize_data_path (std::string const &input_path); +void create_if_not_there (std::string const &path); +}} diff --git a/src/lib/utils/helper_macros.h b/src/lib/utils/helper_macros.h new file mode 100644 index 000000000..bb249958f --- /dev/null +++ b/src/lib/utils/helper_macros.h @@ -0,0 +1,4 @@ +#pragma once + +#define UNUSED(x) ((void)x) +#define HASFLAG(x,y) ((x & y) != 0) diff --git a/src/lib/utils/is_unittest_environment.cpp b/src/lib/utils/is_unittest_environment.cpp new file mode 100644 index 000000000..a1370cd1b --- /dev/null +++ b/src/lib/utils/is_unittest_environment.cpp @@ -0,0 +1,23 @@ +#include "is_unittest_environment.h" +#include + +namespace bts { +namespace utils { +namespace internal { + +static bool forced_unittest_environment = false; + +void force_unittest_environment () +{ + forced_unittest_environment = true; +} + + +bool is_unittest_environment() +{ + return + forced_unittest_environment || + getenv("BTS_TEST_ENVIRONMENT") != nullptr; +} + +}}} diff --git a/src/lib/utils/is_unittest_environment.h b/src/lib/utils/is_unittest_environment.h new file mode 100644 index 000000000..ed6c60926 --- /dev/null +++ b/src/lib/utils/is_unittest_environment.h @@ -0,0 +1,8 @@ +#pragma once +#include "lib/utils/bts_export.h" +namespace bts { +namespace utils { +namespace internal { + extern "C" void BTS_EXPORT force_unittest_environment (); + bool BTS_EXPORT is_unittest_environment(); +}}} diff --git a/src/lib/utils/meta_programming.h b/src/lib/utils/meta_programming.h new file mode 100755 index 000000000..7ee5d5794 --- /dev/null +++ b/src/lib/utils/meta_programming.h @@ -0,0 +1,163 @@ +#pragma once + +#include +#include +#include + +namespace bts { +namespace meta { + +template +struct member_signature +{ + typedef R result; + typedef C functor; + typedef A1 arg1; + typedef A2 arg2; + typedef A3 arg3; + typedef A4 arg4; + typedef A5 arg5; + + // needed to keep GCC warnings at bay, even though no instances of + // this class are ever created. + virtual ~member_signature (); +}; + +template +member_signature deduce_member_signature (R (C::*) ()); + +template +member_signature deduce_member_signature (R (C::*) (A1)); + +template +member_signature deduce_member_signature (R (C::*) (A1,A2)) ; + +template +member_signature deduce_member_signature (R (C::*) (A1,A2,A3)) ; + +template +member_signature deduce_member_signature (R (C::*) (A1,A2,A3,A4)) ; + +template +member_signature deduce_member_signature (R (C::*) (A1,A2,A3,A4,A5)) ; + +template +member_signature deduce_member_signature (R (C::*) () const); + +template +member_signature deduce_member_signature (R (C::*) (A1) const); + +template +member_signature deduce_member_signature (R (C::*) (A1,A2) const) ; + +template +member_signature deduce_member_signature (R (C::*) (A1,A2,A3) const) ; + +template +member_signature deduce_member_signature (R (C::*) (A1,A2,A3,A4) const) ; + +template +member_signature deduce_member_signature (R (C::*) (A1,A2,A3,A4,A5) const) ; + +template +struct functor_signature +{ + typedef decltype (deduce_member_signature(&F::operator())) type; +}; + + +// INT TO STRING + +// Recursive case +template +struct int_to_string2 +{ + typedef typename boost::mpl::push_back< + typename int_to_string2< N < 10, N/10>::type + , boost::mpl::char_<'0' + N%10> + >::type type; +}; + +// Base case +template <> +struct int_to_string2 +{ + typedef boost::mpl::string<> type; +}; + + +template +struct int_to_string +{ + typedef typename boost::mpl::c_str::type>::type type; + const static auto size = boost::mpl::size::type>::value; +}; + + +// FIND THE FIRST OCCURRENCE OF A TYPE IN A TYPE LIST +template struct GetTypeIdxImp {}; +template struct GetTypeIdxImp {}; +template struct GetTypeIdxImp +{ + static constexpr int value = + std::conditional::value, + std::integral_constant, + GetTypeIdxImp>::type::value; +}; + +template struct GetTypeIdx +{static constexpr int value = GetTypeIdxImp<0,Target,TypeList...>::value;}; + +static_assert (GetTypeIdx::value == 1,"Should be one"); + +// FIND THE FIRST OCCURRENCE OF A TYPE IN A TUPLE TYPE + +template < typename Target, class... Types> struct GetTypeIdxInTuple {}; +template < typename Target, class... TypeList> struct GetTypeIdxInTuple> +{static constexpr int value = GetTypeIdxImp<0,Target,TypeList...>::value;}; + +static_assert (GetTypeIdxInTuple>::value == 1,"Should be one"); + +template < int N, int SZ, typename Collection, typename Tuple> +struct CollectTupleItemsImp : CollectTupleItemsImp +{ + CollectTupleItemsImp (Tuple &t, Collection *out_col) + : CollectTupleItemsImp (t,out_col) + { + out_col->push_back (&std::get(t)); + } +}; + + +template < int N, typename Collection, typename Tuple> +struct CollectTupleItemsImp +{ + CollectTupleItemsImp (Tuple &, Collection *) {} + virtual ~CollectTupleItemsImp () {} +}; + +template +void collect_tuple_items (Tuple &tuple, Collection *out_col) +{ + CollectTupleItemsImp<0,std::tuple_size::value,Collection,Tuple> (tuple,out_col); + std::reverse (out_col->begin (),out_col->end ()); +} + +}} + + + + + + + + + + + + + + + + + diff --git a/src/lib/utils/mpmc_bounded_queue.h b/src/lib/utils/mpmc_bounded_queue.h new file mode 100644 index 000000000..263d61cec --- /dev/null +++ b/src/lib/utils/mpmc_bounded_queue.h @@ -0,0 +1,166 @@ + +/* +A modified version of Bounded MPMC queue by Dmitry Vyukov. +Original code from: +http://www.1024cores.net/home/lock-free-algorithms/queues/bounded-mpmc-queue +licensed by Dmitry Vyukov under the terms below: +Simplified BSD license +Copyright (c) 2010-2011 Dmitry Vyukov. All rights reserved. +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright notice, this list of +conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, this list +of conditions and the following disclaimer in the documentation and/or other materials +provided with the distribution. +THIS SOFTWARE IS PROVIDED BY DMITRY VYUKOV "AS IS" AND ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT +SHALL DMITRY VYUKOV OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +The views and conclusions contained in the software and documentation are those of the authors and +should not be interpreted as representing official policies, either expressed or implied, of Dmitry Vyukov. +*/ + +/* +The code in its current form adds the license below: +Copyright(c) 2015 Gabi Melman. +Distributed under the MIT License (http://opensource.org/licenses/MIT) +*/ + +#pragma once + +#include "lib/utils/warn_ctrl.h" + +#include +#include + +namespace spdlog +{ +namespace details +{ + +GCC_DIAG_OFF("-Weffc++") +template +class mpmc_bounded_queue +{ +public: + + using item_type = T; + mpmc_bounded_queue(size_t buffer_size) + :max_size_(buffer_size), + buffer_(new cell_t [buffer_size]), + buffer_mask_(buffer_size - 1) + { + //queue size must be power of two + if(!((buffer_size >= 2) && ((buffer_size & (buffer_size - 1)) == 0))) + assert(false); + + for (size_t i = 0; i != buffer_size; i += 1) + buffer_[i].sequence_.store(i, std::memory_order_relaxed); + enqueue_pos_.store(0, std::memory_order_relaxed); + dequeue_pos_.store(0, std::memory_order_relaxed); + } + + ~mpmc_bounded_queue() + { + delete [] buffer_; + } + + + bool enqueue(const T& data) + { + cell_t* cell; + size_t pos = enqueue_pos_.load(std::memory_order_relaxed); + for (;;) + { + cell = &buffer_[pos & buffer_mask_]; + size_t seq = cell->sequence_.load(std::memory_order_acquire); + intptr_t dif = (intptr_t)seq - (intptr_t)pos; + if (dif == 0) + { + if (enqueue_pos_.compare_exchange_weak(pos, pos + 1, std::memory_order_relaxed)) + break; + } + else if (dif < 0) + { + return false; + } + else + { + pos = enqueue_pos_.load(std::memory_order_relaxed); + } + } + cell->data_ = data; + cell->sequence_.store(pos + 1, std::memory_order_release); + return true; + } + + bool dequeue(T& data) + { + cell_t* cell; + size_t pos = dequeue_pos_.load(std::memory_order_relaxed); + for (;;) + { + cell = &buffer_[pos & buffer_mask_]; + size_t seq = + cell->sequence_.load(std::memory_order_acquire); + intptr_t dif = (intptr_t)seq - (intptr_t)(pos + 1); + if (dif == 0) + { + if (dequeue_pos_.compare_exchange_weak(pos, pos + 1, std::memory_order_relaxed)) + break; + } + else if (dif < 0) + return false; + else + pos = dequeue_pos_.load(std::memory_order_relaxed); + } + data = cell->data_; + cell->sequence_.store(pos + buffer_mask_ + 1, std::memory_order_release); + return true; + } + + size_t approx_size() + { + size_t first_pos = dequeue_pos_.load(std::memory_order_relaxed); + size_t last_pos = enqueue_pos_.load(std::memory_order_relaxed); + if (last_pos <= first_pos) + return 0; + auto size = last_pos - first_pos; + return size < max_size_ ? size : max_size_; + } + +private: + struct cell_t + { + std::atomic sequence_; + T data_; + }; + + size_t const max_size_; + + static size_t const cacheline_size = 64; + typedef char cacheline_pad_t [cacheline_size]; + + cacheline_pad_t pad0_; + cell_t* const buffer_; + size_t const buffer_mask_; + cacheline_pad_t pad1_; + std::atomic enqueue_pos_; + cacheline_pad_t pad2_; + std::atomic dequeue_pos_; + cacheline_pad_t pad3_; + + mpmc_bounded_queue(mpmc_bounded_queue const&) = delete; + void operator= (mpmc_bounded_queue const&) = delete; +}; + +GCC_DIAG_RESTORE() +} // ns details +} // ns spdlog diff --git a/src/lib/utils/proto_helper.h b/src/lib/utils/proto_helper.h new file mode 100755 index 000000000..ff509b8a0 --- /dev/null +++ b/src/lib/utils/proto_helper.h @@ -0,0 +1,250 @@ +#pragma once + +#include +#include +#include +#include "lib/utils/bts_assert.h" + +#include + +namespace bts { +namespace util { +namespace proto { + +// +// set() + +template +void set( + google::protobuf::Message* , + const google::protobuf::Reflection* , + const google::protobuf::FieldDescriptor* , + T ) { + bts_assert(false); +#ifdef _MSC_VER + BOOST_STATIC_ASSERT(false); +#endif +} + +template <> +inline void set( + google::protobuf::Message* message, + const google::protobuf::Reflection* reflection, + const google::protobuf::FieldDescriptor* field, + double value) { + reflection->SetDouble(message, field, value); +} + +template <> +inline void set( + google::protobuf::Message* message, + const google::protobuf::Reflection* reflection, + const google::protobuf::FieldDescriptor* field, + float value) { + reflection->SetFloat(message, field, value); +} + +template <> +inline void set( + google::protobuf::Message* message, + const google::protobuf::Reflection* reflection, + const google::protobuf::FieldDescriptor* field, + int value) { + reflection->SetInt32(message, field, value); +} + +template <> +inline void set( + google::protobuf::Message* message, + const google::protobuf::Reflection* reflection, + const google::protobuf::FieldDescriptor* field, + int64_t value) { + reflection->SetInt64(message, field, value); +} + +template <> +inline void set( + google::protobuf::Message* message, + const google::protobuf::Reflection* reflection, + const google::protobuf::FieldDescriptor* field, + uint64_t value) { + reflection->SetUInt64(message, field, value); +} + +template <> +inline void set( + google::protobuf::Message* message, + const google::protobuf::Reflection* reflection, + const google::protobuf::FieldDescriptor* field, + uint32_t value) { + reflection->SetUInt32(message, field, value); +} + +template <> +inline void set( + google::protobuf::Message* message, + const google::protobuf::Reflection* reflection, + const google::protobuf::FieldDescriptor* field, + bool value) { + reflection->SetBool(message, field, value); +} + +template <> +inline void set( + google::protobuf::Message* message, + const google::protobuf::Reflection* reflection, + const google::protobuf::FieldDescriptor* field, + std::string value) { + reflection->SetString(message, field, value); +} + +template <> +inline void set( + google::protobuf::Message* message, + const google::protobuf::Reflection* reflection, + const google::protobuf::FieldDescriptor* field, + const google::protobuf::EnumValueDescriptor* value) { + reflection->SetEnum(message, field, value); +} + +template <> +inline void set( + google::protobuf::Message* message, + const google::protobuf::Reflection* reflection, + const google::protobuf::FieldDescriptor* field, + const google::protobuf::Message* msg) { + bts_assert(msg); + reflection->MutableMessage(message, field)->CopyFrom(*msg); +} + +// +// Add + +template +void add( + google::protobuf::Message* , + const google::protobuf::Reflection* , + const google::protobuf::FieldDescriptor* , + T ) { + bts_assert(false); +#ifdef _MSC_VER + BOOST_STATIC_ASSERT(false); +#endif +} + +template <> +inline void add( + google::protobuf::Message* message, + const google::protobuf::Reflection* reflection, + const google::protobuf::FieldDescriptor* field, + double value) { + reflection->AddDouble(message, field, value); +} + +template <> +inline void add( + google::protobuf::Message* message, + const google::protobuf::Reflection* reflection, + const google::protobuf::FieldDescriptor* field, + float value) { + reflection->AddFloat(message, field, value); +} + +template <> +inline void add( + google::protobuf::Message* message, + const google::protobuf::Reflection* reflection, + const google::protobuf::FieldDescriptor* field, + int value) { + reflection->AddInt32(message, field, value); +} + +template <> +inline void add( + google::protobuf::Message* message, + const google::protobuf::Reflection* reflection, + const google::protobuf::FieldDescriptor* field, + int64_t value) { + reflection->AddInt64(message, field, value); +} + +template <> +inline void add( + google::protobuf::Message* message, + const google::protobuf::Reflection* reflection, + const google::protobuf::FieldDescriptor* field, + uint64_t value) { + reflection->AddUInt64(message, field, value); +} + +template <> +inline void add( + google::protobuf::Message* message, + const google::protobuf::Reflection* reflection, + const google::protobuf::FieldDescriptor* field, + uint32_t value) { + reflection->AddUInt32(message, field, value); +} + +template <> +inline void add( + google::protobuf::Message* message, + const google::protobuf::Reflection* reflection, + const google::protobuf::FieldDescriptor* field, + bool value) { + reflection->AddBool(message, field, value); +} + +template <> +inline void add( + google::protobuf::Message* message, + const google::protobuf::Reflection* reflection, + const google::protobuf::FieldDescriptor* field, + std::string value) { + reflection->AddString(message, field, value); +} + +template <> +inline void add( + google::protobuf::Message* message, + const google::protobuf::Reflection* reflection, + const google::protobuf::FieldDescriptor* field, + const google::protobuf::EnumValueDescriptor* value) { + reflection->AddEnum(message, field, value); +} + +template <> +inline void add( + google::protobuf::Message* message, + const google::protobuf::Reflection* reflection, + const google::protobuf::FieldDescriptor* field, + const google::protobuf::Message* msg) { + bts_assert(msg); + reflection->AddMessage(message, field)->CopyFrom(*msg); +} + +// +// Assignment + +template +void set_field( + google::protobuf::Message* message, + const google::protobuf::Reflection* reflection, + const google::protobuf::FieldDescriptor* field, + T value, + int& index) { + if (field->is_repeated()) { + int size = reflection->FieldSize(*message, field); + add(message, reflection, field, value); + if (index != size) { + for (int i = index; i < size; ++i) { + reflection->SwapElements(message, field, i, size); + } + } + index++; + } else { + set(message, reflection, field, value); + } +} + +}}} diff --git a/src/lib/utils/type_name.h b/src/lib/utils/type_name.h new file mode 100644 index 000000000..8117882b5 --- /dev/null +++ b/src/lib/utils/type_name.h @@ -0,0 +1,17 @@ +#pragma once + +#ifdef __GNUC__ +#include +#endif + +namespace bts { +namespace utils { +template +const char* type_name() +{ + T* t(nullptr); + int status; + return abi::__cxa_demangle(typeid (*t).name (), 0, 0, &status); +} + +}} diff --git a/src/lib/utils/warn_ctrl.h b/src/lib/utils/warn_ctrl.h new file mode 100644 index 000000000..efb772857 --- /dev/null +++ b/src/lib/utils/warn_ctrl.h @@ -0,0 +1,19 @@ +#pragma once + +#ifdef __GNUC__ +#define GCC_DIAG_STR(s) #s +#define GCC_DIAG_JOINSTR(x,y) GCC_DIAG_STR(x ## y) +# define GCC_DIAG_DO_PRAGMA(x) _Pragma (#x) +# define GCC_DIAG_PRAGMA(x) GCC_DIAG_DO_PRAGMA(GCC diagnostic x) +# define GCC_DIAG_OFF(x) GCC_DIAG_PRAGMA(push) \ + GCC_DIAG_PRAGMA(ignored x) +# define GCC_DIAG_RESTORE() GCC_DIAG_PRAGMA(pop) +# define WIN32_DIAG_OFF(x) +# define WIN32_DIAG_RESTORE() +#else +# define GCC_DIAG_OFF(x) +# define GCC_DIAG_RESTORE() +# define WIN32_DIAG_OFF(x) __pragma (warning( push )) __pragma ( warning(disable: x) ) +# define WIN32_DIAG_RESTORE() __pragma ( warning( pop ) ) +#endif + diff --git a/src/lib/utils/windows_wrapper.h b/src/lib/utils/windows_wrapper.h new file mode 100755 index 000000000..8ebbd973f --- /dev/null +++ b/src/lib/utils/windows_wrapper.h @@ -0,0 +1,11 @@ +#pragma once +#ifdef _WIN32 +#define WIN32_LEAN_AND_MEAN +#ifndef NOMINMAX +#define NOMINMAX +#endif +#pragma warning(push,0) +#include +#pragma warning(pop) +#undef GetMessage +#endif diff --git a/src/server/sparkd/CMakeLists.txt b/src/server/sparkd/CMakeLists.txt new file mode 100644 index 000000000..2bafb9f4b --- /dev/null +++ b/src/server/sparkd/CMakeLists.txt @@ -0,0 +1,92 @@ +cmake_minimum_required(VERSION 3.5) +project(sparkd) + +# proto sources +set (Protos + md/protos/co2.proto + md/protos/coconut.proto + ) +include(${CMAKE_SOURCE_DIR}/tools/Protobuf.cmake) +protobuf_generate_cpp( Proto_Srcs PROTO_HDRS ${Protos} ) + +# cpp sources +set (Sparkd_Sources + main.cpp + + # interconnect + interconnect/Actor.cpp + interconnect/Executor.cpp + interconnect/Domain.cpp + interconnect/IActor.cpp + interconnect/IDomain.cpp + interconnect/TopicDirectory.cpp + interconnect/tests/interconnect_test.cpp + interconnect/tests/baseline_perf_interconnect_test.cpp + + # jetfix + jetfix/codec/FIXArray.cpp + jetfix/common/OutFIX.cpp + jetfix/connection/AsyncReader.cpp + jetfix/connection/IConnection.cpp + jetfix/connection/SocketConnection.cpp + jetfix/gateway/CoreGateway.cpp + jetfix/gateway/JetFIXGateway.cpp + jetfix/gateway/Session.cpp + jetfix/store/FileStore.cpp + jetfix/store/session_name.cpp + jetfix/tests/async_reader_tests.cpp + jetfix/tests/decoding_tests.cpp + jetfix/tests/fix_encoding_stream_tests.cpp + jetfix/tests/fix_fragment_tests.cpp + jetfix/tests/jetfix_performance_tests.cpp + jetfix/tests/store_tests.cpp + + # md + md/MDConsumer.cpp + md/MDConsumerHandle.cpp + md/MDConsumerInterop.cpp + md/MDProducer.cpp + md/MDProducerHandle.cpp + md/MDRecoverableConsumer.cpp + md/MemorySegmentUtils.cpp + md/PlatformLoader.cpp + md/SecurityIDRecorder.cpp + md/SettlementRecorder.cpp + md/common.cpp + md/relay/QueryServer.cpp + md/relay/RelayClient.cpp + md/relay/RelayRecorder.cpp + md/Structure/FixedSecurity.cpp + md/Structure/PriceSize.cpp + md/Diag/MDPriceLantencyTester.cpp + md/Diag/ShmDumper.cpp + md/test/Co2TestInstrument.cpp + md/test/md2_tests.cpp + + # md, mdp3 onixs platform + md/platforms/mdp3_onixs/MDP3Listener.cpp + md/platforms/mdp3_onixs/MDP3Main.cpp + md/platforms/mdp3_onixs/MDP3Utils.cpp + md/platforms/mdp3_onixs/tests/mdp3_tests.cpp + + ) + +# sparkd +add_executable(sparkd ${Sparkd_Sources} ${Proto_Srcs}) + +#include paths +target_include_directories( sparkd PUBLIC ${CMAKE_SOURCE_DIR}/src ${CMAKE_BINARY_DIR}/src) + +# linking +target_link_libraries( sparkd + spark_lib + libtbb.a + double-conversion.a + pugixml.a + -lOnixS.CmeMdp3HandlerCpp + -static-libgcc + -static-libstdc++ + -pthread) + + + diff --git a/src/server/sparkd/interconnect/Actor.cpp b/src/server/sparkd/interconnect/Actor.cpp new file mode 100644 index 000000000..553f0b909 --- /dev/null +++ b/src/server/sparkd/interconnect/Actor.cpp @@ -0,0 +1,10 @@ +#include "Actor.h" + +namespace bts { + + +} + + + + diff --git a/src/server/sparkd/interconnect/Actor.h b/src/server/sparkd/interconnect/Actor.h new file mode 100644 index 000000000..da3ee3784 --- /dev/null +++ b/src/server/sparkd/interconnect/Actor.h @@ -0,0 +1,282 @@ +#pragma once + +#include "IActor.h" +#include "Msg.h" +#include "Executor.h" +#include "RPC.h" +#include "TopicDirectory.h" + +#include "lib/utils/bts_assert.h" + +#include + +namespace bts { + +template +constexpr int method_idx() {return ActorT::idx((RPCT const*) nullptr);} + +constexpr int max_methods() {return std::numeric_limits::max();} + +using RequestHandler = std::function; + +template +struct Ref { + using Actor = A; + Ref(const Ref&) = default; + Ref() : address () {} + Ref(Address address) : address(address) {} + Ref& operator=(Ref const&) = default; + Address address; +}; + +template +class ActorHelper : public IActor +{ + static_assert(N < max_methods(), "Too many methods for Actor"); + + ActorHelper(ActorHelper const&) = delete; + ActorHelper& operator=(ActorHelper const&) = delete; + +public: + ActorHelper() : _my_address(), + _executor(), + _request_handlers(), + _notification_handlers(), + _topic_dir() + {} + + void set_ctx(Address my_address,Executor& d,TopicDirectory &tdir ) override { + bts_assert(!_executor); + _my_address = my_address; + _executor= &d; + _topic_dir = &tdir; + } + + void on_request(MethodID mid, MethodArg const&arg, Replier const&r) override { + bts_assert(mid.value() < N); + _request_handlers[mid.value()](arg,r); + } + + NotificationHandler get_notification_handler(TopicID for_topic) override { + auto it = _notification_handlers.find(for_topic); + if (it != _notification_handlers.end()) + return it->second; + else + return [] (auto const&) {}; + } + + +protected: + Address _my_address; + Executor* _executor; + std::array _request_handlers; + boost::container::flat_map _notification_handlers; + TopicDirectory *_topic_dir; + + void set_handler(int n,RequestHandler const &h) { + bts_assert(n < N); + _request_handlers[n] = h; + } +}; + +template +class ActorHelper : public ActorHelper { +public: + using In = typename FirstRPC::In; + using Out = typename FirstRPC::Out; + using OnReply = typename FirstRPC::OnReply; + using RPC = FirstRPC; + using Base = ActorHelper; + + ActorHelper() + { + auto make_handler = HandlerMaker(); + this->set_handler(N,make_handler(this)); + } + + static constexpr int idx(FirstRPC const*) {return N;} + virtual void on(RPC const&) = 0; + +private: + // non void arg, non void return + template + struct HandlerMaker { + RequestHandler operator()(ActorHelper*self) { + + RequestHandler handler = [self] (MethodArg const&arg,Replier const&r) { + // wrap raw replier in a type safe way + OnReply on_reply = [self, &r] (Out const &out) { + r(RequestError::no_error,[&out] (MethodArg &dst) {safe_placement_new (&dst,out);}); + }; + self->on (RPC { *(const In*) &arg[0], on_reply}); + }; + + return handler; + } + }; + + // void arg, non void return + template + struct HandlerMaker { + RequestHandler operator()(ActorHelper*self) { + + RequestHandler handler = [self] (MethodArg const&,Replier const&r) { + OnReply on_reply = [self, &r] (Out const &out) { + r(RequestError::no_error,[&out] (MethodArg &dst) {safe_placement_new (&dst,out);}); + }; + self->on (RPC {on_reply}); + }; + + return handler; + } + }; + + // non void arg, void return + template + struct HandlerMaker { + RequestHandler operator()(ActorHelper*self) { + + RequestHandler handler = [self] (MethodArg const&arg,Replier const&r) { + OnReply on_reply = [self,&r] { + r(RequestError::no_error,[] (auto) {}); + }; + + self->on (RPC { *(const In*) &arg[0],on_reply }); + }; + + return handler; + } + }; + + // void arg, void return + template + struct HandlerMaker { + RequestHandler operator()(ActorHelper*self) { + + RequestHandler handler = [self] (MethodArg const&,Replier const&r) { + OnReply on_reply = [self,&r] { + r(RequestError::no_error,[] (auto) {}); + }; + + self->on (RPC { on_reply }); + }; + + return handler; + } + }; +}; + +template +class Actor : public ActorHelper<0,RPCs...> +{ + using Base = ActorHelper; + +public: + // non void arg + template + void call (RefT const&ref,typename RPCT::In const&in, OnReply const &r) { + bts_assert(Base::_executor); + + auto mid = MethodID(method_idx()); + + MethodArgCopier mcopier = [&in] (MethodArg&arg) { + safe_placement_new(&arg,in); + }; + + ReplyHandler on_reply = [r] (RequestError e, ReplyArg const&arg) { + if (e == RequestError::no_error) + r (e,(typename RPCT::Out*) &arg[0]); + else + r (e,nullptr); + }; + + Base::_executor->send_request(Base::_my_address,ref.address,mid,on_reply, mcopier); + } + + // void arg + template + void call (RefT const&ref, OnReply const &r) + { + bts_assert(Base::_executor); + + std::enable_if_t::value,Msg> m; + + auto mid = MethodID(method_idx()); + + ReplyHandler on_reply = [r] (RequestError e, ReplyArg const&arg) { + if (e == RequestError::no_error) + r (e,(typename RPCT::Out*) &arg[0]); + else + r (e,nullptr); + }; + + Base::_executor->send_request(Base::_my_address,ref.address,mid,on_reply,[](auto){}); + } + + template + void spawn(ExecutorID target_exe, Callable on_address, ConsArgs&&... args) + { + bts_assert(Base::_executor); + + auto ptr = std::make_unique (std::forward(args)...); + Base::_executor->spawn(Base::_my_address,target_exe, + std::move(ptr), + [on_address] (RequestError e,Address *addr) { + if (e == RequestError::no_error) { + auto ref = Ref (*addr); + on_address(e,&ref); + } + else + on_address(e,nullptr); + }); + } + + template + void kill(RefT const&ref,Callable on_kill) + { + bts_assert(Base::_executor); + Base::_executor->kill(Base::_my_address,ref.address(),on_kill); + } + + template + void subscribe (TopicT const&topic, Callable const&cb) { + bts_assert(Base::_executor); + + bts_assert(Base::_notification_handlers.find(topic.id) == Base::_notification_handlers.end()); + + Base::_notification_handlers[topic.id] = [cb] (NotificationArg const&arg) { + using ValueType = typename TopicT::Type; + cb(*(ValueType*) &arg[0]); + }; + Base::_executor->subscribe( Base::_my_address.actor(), topic.id); + } + + template + void unsubscribe (TopicT const&topic) { + bts_assert(Base::_executor); + + bts_assert(Base::_notification_handlers.find(topic.id) != Base::_notification_handlers.end()); + Base::_notification_handlers[topic.id] = [] (auto &) {}; + Base::_executor->unsubscribe( Base::_my_address.actor(), topic.id); + } + + template + Topic lookup (std::string const&uniq="") + { + bts_assert(Base::_executor); + TopicDirectory *dir = Base::_topic_dir; + return Topic {dir->lookup(uniq)}; + } + + template + void publish(TopicT const&t,typename TopicT::Type const&v) + { + bts_assert(Base::_executor); + Base::_executor->publish(Base::_my_address.actor(),t.id,[&v] (NotificationArg &arg) { + safe_placement_new (&arg,v); + }); + } +}; + + +} diff --git a/src/server/sparkd/interconnect/Domain.cpp b/src/server/sparkd/interconnect/Domain.cpp new file mode 100644 index 000000000..9c3173670 --- /dev/null +++ b/src/server/sparkd/interconnect/Domain.cpp @@ -0,0 +1,192 @@ +#include "Domain.h" + +#include "Executor.h" +#include "Msg.h" +#include + +#include + + +namespace bts { + +struct Domain::ExecutorRecord { + ExecutorRecord (DomainID const&domain_id, + ExecutorID const&exe_id, + IDomain &domain, + TopicDirectory &td) : + executor(Address {executor_aid(),domain_id, exe_id}, domain,td), + queue(12000), + worker([this] () {run();}) + {} + + void run() { + cpu_set_t cpuset; + CPU_ZERO(&cpuset); + CPU_SET(executor.address().executor().value(), &cpuset); + pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset); + + Msg m; + while (true ) { + executor.on_poll(); + if (queue.try_dequeue(m)) + switch (m.type) { + case Msg::REQUEST: + executor.on_request(m.request); + break; + case Msg::REPLY: + executor.on_reply(m.reply); + break; + case Msg::SUBSCRIPTION: + executor.on_subscription(m.subscription); + break; + case Msg::UNSUBSCRIPTION: + executor.on_unsubscription(m.unsubscription); + break; + case Msg::NOTIFICATION: + executor.on_notification(m.notification); + break; + case Msg::SHUTDOWN: + return; + }; + } + } + + ~ExecutorRecord() { + Msg m; + m.type = Msg::SHUTDOWN; + queue.enqueue(m); + worker.join(); + } + + Executor executor; + moodycamel::ReaderWriterQueue queue; + std::thread worker; +}; + +struct Domain::PImpl { + PImpl(DomainID id) : + my_id(id), + executor_record(), + topic_directory() + {} + + DomainID my_id; + std::vector> executor_record; + TopicDirectory topic_directory; +}; + +Domain::Domain(DomainID my_id, + size_t n_executors, + ActorPtr &&bootstrap_actor) : me (new PImpl(my_id)) +{ + bts_assert(n_executors); + for (size_t i = 0; i < n_executors; i++) + me->executor_record.emplace_back(new ExecutorRecord(my_id,ExecutorID(i),*this,me->topic_directory)); + me->executor_record[0]->executor.bootstrap(std::move(bootstrap_actor)); +} + +Domain::~Domain() {} + +void Domain::send_request(Address from, Address to, MethodID mid,RequestID rid,MethodArgCopier const&cp) +{ + bts_assert_d(to.domain() == me->my_id && + to.executor().value() < me->executor_record.size()); + Msg m; + m.type = Msg::REQUEST; + m.request.sender = from; + m.request.receiver = to; + m.request.set_request_id(rid); + m.request.set_method_id(mid); + cp(m.request.method_arg); + post(to,m); +} + +void Domain::send_reply(Address from, Address to, RequestID rid,RequestError e,ReplyArgCopier const &cp) +{ + bts_assert_d(to.domain() == me->my_id && + to.executor().value() < me->executor_record.size()); + Msg m; + m.type = Msg::REPLY; + m.reply.sender = from; + m.reply.receiver = to; + m.reply.set_request_id(rid); + m.reply.error = e; + cp(m.reply.reply_arg); + post(to,m); + +} + +void Domain::send_subscribe(Address from, Address to, TopicID tid) +{ + bts_assert_d(to.domain() == me->my_id && + to.executor().value() < me->executor_record.size()); + Msg m; + m.type = Msg::SUBSCRIPTION; + m.subscription.subscriber = from; + m.subscription.set_topic(tid); + post(to,m); +} + +void Domain::send_unsubscribe(Address from, Address to, TopicID tid) +{ + bts_assert_d(to.domain() == me->my_id && + to.executor().value() < me->executor_record.size()); + Msg m; + m.type = Msg::UNSUBSCRIPTION; + m.unsubscription.subscriber = from; + m.unsubscription.set_topic(tid); + post(to,m); +} + +void Domain::forward_subscribe(Address from, TopicID tid) { + + Msg m; + m.type = Msg::SUBSCRIPTION; + m.subscription.subscriber = from; + m.subscription.set_topic(tid); + + Address to {executor_aid(), me->my_id, ExecutorID(0)}; + auto src_exe = from.executor().value(); + for (size_t i = 0 ; i < me->executor_record.size();i++) + if (src_exe != i) { + to.set_executor(ExecutorID(i)); + post(to,m); + } +} + +void Domain::forward_unsubscribe(Address from, TopicID tid) { + + Msg m; + m.type = Msg::UNSUBSCRIPTION; + m.unsubscription.subscriber = from; + m.unsubscription.set_topic(tid); + + Address to {executor_aid(), me->my_id, ExecutorID(0)}; + auto src_exe = from.executor().value(); + for (size_t i = 0 ; i < me->executor_record.size();i++) + if (src_exe != i) { + to.set_executor(ExecutorID(i)); + post(to,m); + } +} + +void Domain::send_notify(Address from,TopicID tid,std::vector
const& to,NotificationCopier const&cp) +{ + Msg m; + m.type = Msg::NOTIFICATION; + m.notification.publisher = from; + m.notification.set_topic(tid); + + for (auto &a : to ) { + cp(m.notification.notification_arg); + post(a,m); + } +} + +void Domain::post(Address to, const Msg &m) +{ + me->executor_record[to.executor().value()]->queue.enqueue(m); +} + +} + diff --git a/src/server/sparkd/interconnect/Domain.h b/src/server/sparkd/interconnect/Domain.h new file mode 100644 index 000000000..547ae6fca --- /dev/null +++ b/src/server/sparkd/interconnect/Domain.h @@ -0,0 +1,30 @@ +#pragma once + +#include "IDomain.h" +#include "IActor.h" + +namespace bts { + +class Domain : IDomain { +public: + Domain (DomainID my_id,size_t n_executors, ActorPtr &&bootstrap); + ~Domain() ; +private: + struct PImpl; + struct ExecutorRecord; + std::unique_ptr me; + + // from IDomain + void send_request(Address from, Address to, MethodID,RequestID,MethodArgCopier const&) override; + void send_reply(Address from, Address to, RequestID,RequestError,ReplyArgCopier const &) override; + void send_subscribe(Address from, Address to, TopicID) override; + void send_unsubscribe(Address from, Address to, TopicID) override; + void forward_subscribe(Address from, TopicID) override; + void forward_unsubscribe(Address from, TopicID) override; + void send_notify(Address from,TopicID,std::vector
const& to,NotificationCopier const&) override; + // end IDomain + + void post(Address to, Msg const&m); +}; + +} diff --git a/src/server/sparkd/interconnect/Executor.cpp b/src/server/sparkd/interconnect/Executor.cpp new file mode 100644 index 000000000..0a36ff841 --- /dev/null +++ b/src/server/sparkd/interconnect/Executor.cpp @@ -0,0 +1,326 @@ +#include "Executor.h" +#include "IDomain.h" +#include "IPollable.h" + +#include + +namespace bts { + +struct Executor::TopicRecord { + TopicRecord() : + local_subscribers(), + external_subscribers() + {} + + TopicRecord (TopicRecord&&src) : + local_subscribers(), + external_subscribers() + {operator=(std::move(src));} + + TopicRecord (TopicRecord const&src) : + local_subscribers(), + external_subscribers() + {operator=(src);} + + + TopicRecord& operator=(TopicRecord const &src) + { + local_subscribers = src.local_subscribers; + external_subscribers = src.external_subscribers; + return *this; + } + + TopicRecord& operator=(TopicRecord&&src) + { + local_subscribers = std::move(src.local_subscribers); + external_subscribers = std::move(src.external_subscribers); + return *this; + } + + std::vector> local_subscribers; + std::vector
external_subscribers; +}; + +struct Executor::LifeCycleManager : public IActor { + LifeCycleManager(Executor&owner) :owner(owner) {} + + Executor &owner; + + void set_ctx(Address, Executor&,TopicDirectory&) override {} + + void on_request(MethodID mid, MethodArg const&arg, Replier const&replier) override + { + if (mid == spawn_actor_mid()) { + ActorPtr actor_ptr (*(IActor**)&arg[0] ); + auto p = owner.on_spawn_actor(std::move(actor_ptr)); + auto error = std::get<0>(p); + auto address = std::get<1>(p); + auto copier = [address] (ReplyArg&arg) {safe_placement_new
(&arg,address);}; + replier(error,copier); + } + else if (mid == kill_actor_mid()) { + ActorID aid = *(ActorID*) &arg[0]; + auto error = owner.on_kill_actor(aid); + replier (error,[] (auto) {} ); + } + else + replier(RequestError::unknown_method,[](auto){}); + } + + NotificationHandler get_notification_handler(TopicID) override + {return [](const auto&) {};} +}; + +struct Executor::PImpl +{ + PImpl(Address my_address,IDomain &domain,TopicDirectory &topic_directory) : + domain(domain), + actors(), + pollable_actors(), + pending_requests(), + subscriptions(), + next_actor_id(executor_aid().value()+1), + next_request_id(1), + my_address(my_address), + topic_directory(topic_directory) + {} + + IDomain& domain; + boost::container::flat_map actors; + std::vector> pollable_actors; + boost::container::flat_map> pending_requests; + boost::container::flat_map subscriptions; + ActorID::Value next_actor_id; + RequestID next_request_id; + Address my_address; + TopicDirectory &topic_directory; +}; + +Executor::Executor(Address my_address, + IDomain &domain, + TopicDirectory &directory) : me (new PImpl(my_address,domain,directory)) +{ + me->actors[executor_aid()].reset (new LifeCycleManager {*this} ); +} + +Executor::~Executor() {} + +void Executor::on_poll() +{ + for (auto &ipoll : me->pollable_actors) + ipoll.second->on_poll(); +} + +void Executor::on_request(Request const&r) { + auto it = me->actors.find(r.receiver.actor()); + if (it != me->actors.end()) { + + auto replier = + [this, + sender=r.sender, + receiver=r.receiver, + rid = r.request_id()] + (RequestError error,ReplyArgCopier const &cp) + { + me->domain.send_reply(receiver,sender,rid,error,cp); + }; + + it->second->on_request(r.method_id(),r.method_arg,replier); + } + else + me->domain.send_reply(r.receiver,r.sender,r.request_id(), + RequestError::unknown_actor,[](auto) {}); +} + +void Executor::on_reply(Reply const&r) +{ + auto it = me->pending_requests.find(r.request_id()); + if (it != me->pending_requests.end()) + it->second.second(r.error,r.reply_arg); +} + +void Executor::send_request(Address from, + Address to, + MethodID method, + ReplyHandler const&on_reply, + MethodArgCopier const&copier) +{ + if (me->next_request_id.value()) { + auto& pr = me->pending_requests[me->next_request_id]; + pr.first = from.actor(); + pr.second = on_reply; + me->domain.send_request(from,to,method,me->next_request_id,copier); + me->next_request_id = RequestID(me->next_request_id.value()+1); + } + else + on_reply(RequestError::too_many_requests,ReplyArg {}); +} + +std::tuple Executor::on_spawn_actor(ActorPtr &&actor) +{ + if (me->next_actor_id != 0) { + auto new_actor_id = ActorID(me->next_actor_id++); + auto ptr = actor.get(); + me->actors[new_actor_id] = std::move(actor); + Address new_actor_address = me->my_address; + new_actor_address.set_actor (new_actor_id); + ptr->set_ctx(new_actor_address,*this,me->topic_directory); + ptr->init(); + auto ipoll = dynamic_cast (ptr); + if (ipoll) + me->pollable_actors.push_back(std::make_pair (new_actor_id,ipoll)); + + return std::make_tuple (RequestError::no_error,new_actor_address); + } + else + return std::make_tuple (RequestError::too_many_actors,Address{}); +} + +RequestError Executor::on_kill_actor(ActorID aid) +{ + auto it = me->actors.find(aid); + if (it != me->actors.end()) { + it->second->finit(); + me->actors.erase(aid); + + // get rid of any pending calls by this actor + std::vector discarded; + for (auto &kvp : me->pending_requests) { + auto & k = kvp.first; + auto & v = kvp.second; + if (v.first == aid) + discarded.push_back(k); + } + for (auto rid : discarded) + me->pending_requests.erase(rid); + + // remove the actor from the list of pollable actors + auto &pollable = me->pollable_actors; + pollable.erase(std::remove_if(pollable.begin(),pollable.end(), + [aid] (auto p) {return p.first == aid;}),pollable.end()); + + // remove any subscriptions by this actor + for (auto &kvp : me->subscriptions) { + auto &topic = kvp.first; + auto &tr = kvp.second; + auto &ls = tr.local_subscribers; + if (ls.size()) { + ls.erase(std::remove_if(ls.begin(),ls.end(), + [aid] (auto p) {return p.first == aid;}),ls.end()); + if (ls.empty()) // last sub, send out unsub + me->domain.forward_unsubscribe(me->my_address,topic); + } + } + + return RequestError::no_error; + } + else + return RequestError::unknown_actor; +} + +void Executor::on_subscription(Subscription const&sub) +{ + if (local_address(sub.subscriber)) { + auto aid = sub.subscriber.actor(); + auto it = me->actors.find(aid); + if (it != me->actors.end()) { + auto handler = it->second->get_notification_handler(sub.topic()); + auto &sr = me->subscriptions[sub.topic()]; + sr.local_subscribers.push_back(std::make_pair (aid,handler)); + + if (sr.local_subscribers.size() == 1) { + sr.external_subscribers.push_back(me->my_address); + me->domain.forward_subscribe(me->my_address,sub.topic()); + } + } + } + else { + auto &sr = me->subscriptions[sub.topic()]; + sr.external_subscribers.push_back(sub.subscriber); + } +} + +void Executor::on_unsubscription(Unsubscription const&unsub) +{ + if (local_address(unsub.subscriber)) { + auto aid = unsub.subscriber.actor(); + auto &s = me->subscriptions[unsub.topic()]; + auto &ls = s.local_subscribers; + auto &ex = s.external_subscribers; + ls.erase(std::remove_if(ls.begin(),ls.end(), [aid] (auto p) {return p.first == aid;}),ls.end()); + if (ls.empty()) { + ex.erase(std::remove_if(ex.begin(),ex.end(), [this] (auto a) {return me->my_address == a;}),ex.end()); + me->domain.forward_unsubscribe(me->my_address,unsub.topic()); + } + } + else { + auto &es = me->subscriptions[unsub.topic()].external_subscribers; + es.erase(std::remove_if(es.begin(),es.end(), + [s=unsub.subscriber,this] + (auto a) {return a == s;}), + es.end()); + } +} + +void Executor::on_notification(Notification const& n) +{ + auto &ls = me->subscriptions[n.topic()].local_subscribers; + for (auto &kvp : ls) + kvp.second(n.notification_arg); +} + +void Executor::subscribe(ActorID subscriber, TopicID tid) +{ + auto from = me->my_address; + from.set_actor (subscriber); + me->domain.send_subscribe(from,me->my_address, tid); +} + +void Executor::unsubscribe(ActorID subscriber, TopicID tid) +{ + auto from = me->my_address; + from.set_actor(subscriber); + me->domain.send_unsubscribe(from,me->my_address,tid); +} + +void Executor::publish(ActorID aid, TopicID tid, NotificationCopier const&cp) +{ + auto addr = me->my_address; + addr.set_actor(aid); + me->domain.send_notify(addr,tid,me->subscriptions[tid].external_subscribers, cp); +} + +void Executor::bootstrap(ActorPtr &&with_actor) +{on_spawn_actor(std::move(with_actor));} + +bool Executor::local_address(Address a) const +{return a.domain() == me->my_address.domain() && a.executor() == me->my_address.executor();} + +Address Executor::address() const {return me->my_address;} + +void Executor::spawn (Address from, ExecutorID to, ActorPtr &&a, OnSpawn const&on_reply) +{ + send_request(from, + Address {executor_aid(),me->my_address.domain(),to}, + spawn_actor_mid(), + [on_reply] (RequestError error, ReplyArg const&reply_arg) { + if (error == RequestError::no_error) + on_reply(error,(Address*) &reply_arg[0]); + else + on_reply(error,nullptr); + }, + [aptr = a.release() ] (MethodArg &marg) { + safe_placement_new (&marg,aptr); + }); +} + +void Executor::kill (Address from, Address to, std::function const&on_reply) +{ + send_request(from, + Address {executor_aid(),to.domain(),to.executor()}, + kill_actor_mid(), + [on_reply] (RequestError error, ReplyArg const&) {on_reply(error);}, + [aid = to.actor()] (MethodArg &marg) {safe_placement_new (&marg,aid);}); +} + +} diff --git a/src/server/sparkd/interconnect/Executor.h b/src/server/sparkd/interconnect/Executor.h new file mode 100644 index 000000000..32a864a5e --- /dev/null +++ b/src/server/sparkd/interconnect/Executor.h @@ -0,0 +1,50 @@ +#pragma once + +#include "Msg.h" +#include "Actor.h" + +#include +#include + +namespace bts { + +class IDomain; +class TopicDirectory; + +class Executor { +public: + Executor(Address my_address,IDomain &,TopicDirectory &td); + ~Executor(); + void bootstrap(ActorPtr &&with_actor); + + void on_poll(); + void on_request(Request const&r) ; + void on_reply(Reply const&r) ; + + void on_subscription(Subscription const&s); + void on_unsubscription(Unsubscription const&s); + void on_notification(Notification const& notification); + + void send_request(Address from, Address to,MethodID,ReplyHandler const&,MethodArgCopier const&); + void subscribe(ActorID subscriber, TopicID); + void unsubscribe(ActorID subscriber, TopicID); + void publish(ActorID, TopicID, NotificationCopier const&); + + void spawn (Address from, ExecutorID to, ActorPtr &&, OnSpawn const&on_reply); + void kill (Address from, Address to, OnKill const&on_reply); + Address address() const ; + +private: + struct LifeCycleManager; + struct TopicRecord; + struct PImpl; + std::unique_ptr me; + + std::tuple on_spawn_actor(ActorPtr &&actor); + RequestError on_kill_actor(ActorID aid); + bool local_address(Address a) const; +}; + + +} + diff --git a/src/server/sparkd/interconnect/IActor.cpp b/src/server/sparkd/interconnect/IActor.cpp new file mode 100644 index 000000000..b1ead8089 --- /dev/null +++ b/src/server/sparkd/interconnect/IActor.cpp @@ -0,0 +1,5 @@ +#include "IActor.h" + +namespace bts { +IActor::~IActor () {} +} diff --git a/src/server/sparkd/interconnect/IActor.h b/src/server/sparkd/interconnect/IActor.h new file mode 100644 index 000000000..ca973577b --- /dev/null +++ b/src/server/sparkd/interconnect/IActor.h @@ -0,0 +1,25 @@ +#pragma once + +#include "Msg.h" +#include + +namespace bts { + +class Executor; +class TopicDirectory; + +class IActor { +public: + virtual ~IActor() ; + virtual void set_ctx(Address my_address, Executor&,TopicDirectory&) = 0; + virtual void on_request(MethodID mid, MethodArg const&arg, Replier const&r) = 0; + virtual NotificationHandler get_notification_handler(TopicID for_topic) = 0; + + virtual void init() {}; + virtual void finit() {}; +}; + +using ActorPtr = std::unique_ptr; + +} + diff --git a/src/server/sparkd/interconnect/IDomain.cpp b/src/server/sparkd/interconnect/IDomain.cpp new file mode 100644 index 000000000..0d7eed316 --- /dev/null +++ b/src/server/sparkd/interconnect/IDomain.cpp @@ -0,0 +1,5 @@ +#include "IDomain.h" + +namespace bts { +IDomain::~IDomain() {} +} diff --git a/src/server/sparkd/interconnect/IDomain.h b/src/server/sparkd/interconnect/IDomain.h new file mode 100644 index 000000000..f357d3650 --- /dev/null +++ b/src/server/sparkd/interconnect/IDomain.h @@ -0,0 +1,23 @@ +#pragma once + +#include "Msg.h" +#include + +namespace bts { +class IDomain { +public: + virtual ~IDomain() ; + virtual void send_request(Address from, Address to, + MethodID, RequestID, MethodArgCopier const&) = 0; + virtual void send_reply(Address from, Address to, + RequestID, RequestError, ReplyArgCopier const &) = 0; + + virtual void send_subscribe(Address from, Address to, TopicID topic) = 0; + virtual void send_unsubscribe(Address from, Address to, TopicID topic) = 0; + + virtual void forward_subscribe(Address from, TopicID topic) = 0; + virtual void forward_unsubscribe(Address from, TopicID topic) = 0; + + virtual void send_notify(Address from,TopicID,std::vector
const& to,NotificationCopier const&) = 0; +}; +} diff --git a/src/server/sparkd/interconnect/IPollable.h b/src/server/sparkd/interconnect/IPollable.h new file mode 100644 index 000000000..6e802afcb --- /dev/null +++ b/src/server/sparkd/interconnect/IPollable.h @@ -0,0 +1,7 @@ +#pragma once + +class IPollable { +public: + virtual ~IPollable() {}; + virtual void on_poll() = 0; +}; diff --git a/src/server/sparkd/interconnect/Msg.h b/src/server/sparkd/interconnect/Msg.h new file mode 100644 index 000000000..0292f1a64 --- /dev/null +++ b/src/server/sparkd/interconnect/Msg.h @@ -0,0 +1,151 @@ +#pragma once + +#include "lib/utils/ValueWrapper.h" +#include "lib/utils/bts_assert.h" +#include "lib/utils/warn_ctrl.h" + +#include +#include + +namespace bts { + +GCC_DIAG_OFF("-Weffc++") +template +void safe_placement_new (Target *t,Args&&... args) { + static_assert(sizeof(Target) >= sizeof(Source),"Destination is not large enough to construct source type"); + new (t) Source(std::forward(args)...); +} + +enum class RequestError { + no_error, + timeout, + unknown_method, + unknown_actor, + unknown_domain, + too_many_actors, + too_many_requests +}; + +using ExecutorID = utils::ValueWrapper; +using DomainID = utils::ValueWrapper; +using ActorID = utils::ValueWrapper; +using TopicID = utils::ValueWrapper; + +using MethodID = utils::ValueWrapper; +using MethodArg = std::array ; +using MethodArgCopier = std::function; + +using RequestID = utils::ValueWrapper; +using ReplyArg = std::array ; +using ReplyArgCopier = std::function; +using Replier = std::function; +using ReplyHandler = std::function; +using Method = std::function; +using ActorMethodMap = std::vector; + +using NotificationArg = std::array ; +using NotificationHandler = std::function; +using NotificationCopier = std::function; + +inline ActorID executor_aid() {return ActorID(0);} +inline MethodID spawn_actor_mid() {return MethodID(1);} +inline MethodID kill_actor_mid() {return MethodID(0);} + +struct Address { + ActorID::Value _actor; + DomainID::Value _domain; + ExecutorID::Value _executor; + + Address() = default; + Address& operator=(const Address&)= default; + + Address(ActorID aid, DomainID did, ExecutorID eid) + : _actor(aid.value()), + _domain(did.value()), + _executor(eid.value()) + {} + + ActorID actor() const {return ActorID(_actor);} + DomainID domain() const {return DomainID(_domain);} + ExecutorID executor() const {return ExecutorID(_executor);} + + void set_actor(ActorID id) {_actor = id.value();} + void set_domain(DomainID did) {_domain = did.value();} + void set_executor(ExecutorID eid) {_executor = eid.value();} + + bool operator==(Address rhs) const { + return + _actor==rhs._actor && + _domain==rhs._domain && + _executor==rhs._executor; + } +}; + +using OnSpawn = std::function ; +using OnKill = std::function ; + +struct Request { + Address sender; + Address receiver; + + RequestID request_id() const {return RequestID(_request_id);} + MethodID method_id() const {return MethodID(_method_id);} + void set_request_id(RequestID rid) {_request_id = rid.value();} + void set_method_id(MethodID mid) {_method_id = mid.value();} + + RequestID::Value _request_id; + MethodID::Value _method_id; + MethodArg method_arg; +}; + +struct Reply { + Address sender; + Address receiver; + RequestID request_id() const {return RequestID(_request_id);} + void set_request_id(RequestID rid) {_request_id = rid.value();} + RequestID::Value _request_id; + RequestError error; + ReplyArg reply_arg; +}; + +struct Subscription { + Address subscriber; + TopicID topic() const {return TopicID(_topic);} + void set_topic(TopicID tid) {_topic = tid.value();} + TopicID::Value _topic; +}; + +struct Unsubscription { + Address subscriber ; + TopicID topic() const {return TopicID(_topic);} + void set_topic(TopicID tid) {_topic = tid.value();} + TopicID::Value _topic; +}; + +struct Notification { + Address publisher; + TopicID topic() const {return TopicID(_topic);} + void set_topic(TopicID tid) {_topic = tid.value();} + TopicID::Value _topic; + NotificationArg notification_arg; +}; + +struct Msg { + union { + Request request; + Reply reply; + Subscription subscription; + Unsubscription unsubscription; + Notification notification; + }; + + enum Type { + REQUEST,REPLY,SUBSCRIPTION, UNSUBSCRIPTION, NOTIFICATION, SHUTDOWN + } type ; +}; + +GCC_DIAG_RESTORE() + +//static_assert(sizeof(Msg) == 64,"Msg should be 64 bytes (a cache line)"); + +} diff --git a/src/server/sparkd/interconnect/RPC.h b/src/server/sparkd/interconnect/RPC.h new file mode 100644 index 000000000..dadd00db7 --- /dev/null +++ b/src/server/sparkd/interconnect/RPC.h @@ -0,0 +1,53 @@ +#pragma once + +#include +#include "Msg.h" + +namespace bts { +template +struct RPC { + using In = I; + using Out = O; + using OnReply = std::function; + + In const ∈ + OnReply const &on_reply; + + static_assert(sizeof(In) <= sizeof(Request::method_arg),"Invalid RPC. Input type is too large"); + static_assert(sizeof(Out) <= sizeof(Reply::reply_arg),"Invalid RPC. Output type is too large"); +}; + +template +struct RPC { + using In = void; + using Out = O; + using OnReply = std::function; + + OnReply const &on_reply; + + static_assert(sizeof(Out) <= sizeof(Reply::reply_arg),"Invalid RPC. Output type is too large"); +}; + +template +struct RPC { + using In = I; + using Out = void; + using OnReply = std::function; + + OnReply const &on_reply; + + static_assert(sizeof(In) <= sizeof(Request::method_arg),"Invalid RPC. Input type is too large"); +}; + +template +struct RPC { + using In = void; + using Out = void; + using OnReply = std::function; + + OnReply const &on_reply; +}; + +} + + diff --git a/src/server/sparkd/interconnect/Topic.h b/src/server/sparkd/interconnect/Topic.h new file mode 100644 index 000000000..feba1c4e9 --- /dev/null +++ b/src/server/sparkd/interconnect/Topic.h @@ -0,0 +1,18 @@ +#pragma once + +#include "lib/utils/type_name.h" +#include "lib/utils/warn_ctrl.h" + +namespace bts { + +GCC_DIAG_OFF("-Weffc++") +template +struct Topic +{ + using Type = V; + TopicID id; +}; +GCC_DIAG_RESTORE() + +} + diff --git a/src/server/sparkd/interconnect/TopicDirectory.cpp b/src/server/sparkd/interconnect/TopicDirectory.cpp new file mode 100644 index 000000000..47d7dee03 --- /dev/null +++ b/src/server/sparkd/interconnect/TopicDirectory.cpp @@ -0,0 +1,5 @@ +#include "TopicDirectory.h" + +namespace bts { + +} diff --git a/src/server/sparkd/interconnect/TopicDirectory.h b/src/server/sparkd/interconnect/TopicDirectory.h new file mode 100644 index 000000000..534e350ec --- /dev/null +++ b/src/server/sparkd/interconnect/TopicDirectory.h @@ -0,0 +1,35 @@ +#pragma once + +#include +#include +#include + +#include "lib/utils/type_name.h" +#include "Msg.h" +#include "Topic.h" + +namespace bts { + +class TopicDirectory +{ +public: + TopicDirectory() : _mtx(), _map(), _next_topic(1) {} + + template + Topic lookup (std::string const&uniq) { + std::lock_guard l(_mtx); + auto &t = _map[uniq+":"+utils::type_name()]; + if (t.value() == 0) { + t = TopicID(_next_topic++); + bts_assert(_next_topic); + } + return Topic {t}; + } + +private: + std::mutex _mtx; + std::unordered_map _map; + TopicID::Value _next_topic; +}; + +} diff --git a/src/server/sparkd/interconnect/tests/baseline_perf_interconnect_test.cpp b/src/server/sparkd/interconnect/tests/baseline_perf_interconnect_test.cpp new file mode 100644 index 000000000..46e634234 --- /dev/null +++ b/src/server/sparkd/interconnect/tests/baseline_perf_interconnect_test.cpp @@ -0,0 +1,432 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace bts { + +auto n_iterations = 10000; +using TimeStamp = std::chrono::high_resolution_clock::time_point; +using Duration = std::chrono::high_resolution_clock::duration; +using Timer = std::chrono::high_resolution_clock; +std::vector durations; + +struct Payload { + Payload() : ts(),padding() {} + TimeStamp ts; + char padding[1]; +}; + +void receive(Payload const &src) {durations.push_back(Timer::now() - src.ts);} + +struct Functor { + Functor () : ts () {} + Functor (Payload const &pl) : ts (pl) {} + Payload ts; + void operator()() + {receive (ts);} +}; + +void spin_us(int64_t us) { + auto start = Timer::now(); + + while (std::chrono::duration_cast(Timer::now()-start).count() < us) + ; +} + +template +size_t loop() { + durations.clear(); + durations.reserve(n_iterations); + { + S s(n_iterations); + for (int i = 0; i < n_iterations; i++) { + spin_us(1); + Payload p; + p.ts = Timer::now(); + s(p); + } + } + + std::sort(durations.begin(),durations.end()); + return std::chrono::duration_cast(durations[durations.size()/2]).count(); +} + + +struct Baseline +{ + Baseline (size_t) {} + void operator()(Payload const&ts) {receive(ts);} +}; + + +struct SPSC_POD { + template using SpscQ = boost::lockfree::spsc_queue; + + SPSC_POD (size_t n_iterations) + : n_iterations(n_iterations), + promise(), + queue(n_iterations), + consumer(&SPSC_POD::consume,this) + { + promise.get_future().wait(); + } + + void operator()(const Payload &t) {queue.push(t);} + + void consume() { + cpu_set_t cpuset; + CPU_ZERO(&cpuset); + CPU_SET(1, &cpuset); + pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset); + + promise.set_value(); + while(n_iterations) { + Payload t; + if (queue.pop(t)) { + receive(t); + --n_iterations; + } + } + } + + ~SPSC_POD() {consumer.join();} + + size_t n_iterations; + std::promise promise; + SpscQ queue; + std::thread consumer; +}; + +struct MoodyCamel_Local_POD { + template using Q = moodycamel::ReaderWriterQueue; + + MoodyCamel_Local_POD (size_t n_iterations) + : n_iterations(n_iterations), + queue(n_iterations) + { + } + + void operator()(const Payload &t) + { + queue.enqueue(t); + Payload p2; + queue.try_dequeue(p2); + receive(p2); + } + + size_t n_iterations; + Q queue; +}; + +struct MoodyCamel_POD { + template using Q = moodycamel::ReaderWriterQueue; + + MoodyCamel_POD (size_t n_iterations) + : n_iterations(n_iterations), + promise(), + queue(n_iterations), + consumer(&MoodyCamel_POD::consume,this) + { + promise.get_future().wait(); + } + + void operator()(const Payload &t) {queue.enqueue(t);} + + void consume() { + cpu_set_t cpuset; + CPU_ZERO(&cpuset); + CPU_SET(1, &cpuset); + pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset); + + promise.set_value(); + while(n_iterations) { + Payload t; + if (queue.try_dequeue(t)) { + receive(t); + --n_iterations; + } + } + } + + ~MoodyCamel_POD() {consumer.join();} + + size_t n_iterations; + std::promise promise; + Q queue; + std::thread consumer; +}; + +struct MPMC_POD { + template using Q = boost::lockfree::queue; + + MPMC_POD (size_t n_iterations) + : n_iterations(n_iterations), + promise(), + queue(n_iterations), + consumer(&MPMC_POD::consume,this) + { + promise.get_future().wait(); + } + + void operator()(const Payload &t) {queue.push(t);} + + void consume() { + cpu_set_t cpuset; + CPU_ZERO(&cpuset); + CPU_SET(1, &cpuset); + pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset); + + promise.set_value(); + while(n_iterations) { + Payload t; + if (queue.pop(t)) { + receive(t); + --n_iterations; + } + } + } + + ~MPMC_POD() {consumer.join();} + + size_t n_iterations; + std::promise promise; + Q queue; + std::thread consumer; +}; + +struct Buffered_POD { + + Buffered_POD (size_t n_iterations) + : buffer() + { + buffer.reserve(n_iterations); + } + + void operator()(const Payload &t) + { + buffer.push_back(t); + receive(buffer.back()); + } + + std::vector buffer; +}; + +struct Buffered_Functor { + + Buffered_Functor (size_t n_iterations) + : buffer() + { + buffer.reserve(n_iterations); + } + + void operator()(const Payload &t) + { + buffer.push_back(Functor(t)); + buffer.back()(); + } + + std::vector buffer; +}; +struct Buffered_Lambda { + using Action = std::function; + + Buffered_Lambda (size_t n_iterations) + : buffer() + {buffer.reserve(n_iterations);} + + void operator()(const Payload &t) + { + Action a= [t] () {receive(t);}; + buffer.push_back(a); + Action a2= buffer.back(); + a2(); + } + + std::vector buffer; +}; + +struct Signals { + using Signal = + boost::signals2::signal_type >::type ; + + Signals(size_t) + : signal () + { + signal.connect(receive); + } + + void operator() (const Payload &ts) { + signal(ts); + } + Signal signal; +}; + +struct STDFunction { + using Signal = std::function; + + STDFunction(size_t) + : signals () + { + signals.push_back([] (Payload const &ts) {receive (ts);}); + } + + void operator() (const Payload &ts) { + for (auto &s : signals) + s(ts); + } + std::vector signals; +}; + + +struct SPSC_Functor { + template using SpscQ = boost::lockfree::spsc_queue; + + SPSC_Functor (size_t n_iterations) + : n_iterations(n_iterations), + promise(), + queue(n_iterations), + consumer(&SPSC_Functor::consume,this) + { + promise.get_future().wait(); + } + + void operator()(const Payload &t) { + queue.push(Functor(t)); + } + + void consume() { + cpu_set_t cpuset; + CPU_ZERO(&cpuset); + CPU_SET(1, &cpuset); + pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset); + + promise.set_value(); + while(n_iterations) { + Functor t; + if (queue.pop(t)) { + t(); + --n_iterations; + } + } + } + + ~SPSC_Functor() {consumer.join();} + + size_t n_iterations; + std::promise promise; + SpscQ queue; + std::thread consumer; +}; + +struct SPSC_Lambda { + template using SpscQ = boost::lockfree::spsc_queue; + using Action = std::function; + + SPSC_Lambda (size_t n_iterations) + : n_iterations(n_iterations), + promise(), + queue(n_iterations), + consumer(&SPSC_Lambda::consume,this) + { + promise.get_future().wait(); + } + + void operator()(const Payload &t) { + queue.push([t] () {receive(t);}); + } + + void consume() { + cpu_set_t cpuset; + CPU_ZERO(&cpuset); + CPU_SET(1, &cpuset); + pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset); + + promise.set_value(); + while(n_iterations) { + Action t; + if (queue.pop(t)) { + t(); + --n_iterations; + } + } + } + + ~SPSC_Lambda() {consumer.join();} + + size_t n_iterations; + std::promise promise; + SpscQ queue; + std::thread consumer; +}; + +struct SPSC_Local_Lambda { + template using SpscQ = boost::lockfree::spsc_queue; + using Action = std::function; + + SPSC_Local_Lambda (size_t n_iterations) + : queue(n_iterations) + {} + + void operator()(const Payload &t) { + queue.push([t] () {receive(t);}); + Action a; + queue.pop(a); + a(); + } + SpscQ queue; +}; + +int run_benchmarks() +{ + using namespace bts; + cpu_set_t cpuset; + CPU_ZERO(&cpuset); + CPU_SET(0, &cpuset); + pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset); + + std::cout << "\tdirect calls baseline: " << loop() << "ns\n"; + std::cout << "\tdirect calls buffered functor: " << loop() << "ns\n"; + std::cout << "\tdirect calls signals2: " << loop() << "ns\n"; + std::cout << "\tdirect calls std_function: " << loop() << "ns\n"; + + std::cout << std::endl; + std::cout << "\tbuffered pod: " << loop() << "ns\n"; + std::cout << "\tbuffered lambda: " << loop() << "ns\n"; // + + std::cout << std::endl; + std::cout << "\tspsc functor: " << loop() << "ns\n"; + std::cout << "\tspsc pod: " << loop() << "ns\n"; + std::cout << "\tspsc lambda: " << loop() << "ns\n"; + std::cout << "\tspsc local_lambda: " << loop() << "ns\n"; + + std::cout << std::endl; + std::cout << "\tmpmc pod: " << loop() << "ns\n"; + std::cout << "\tmoody pod: " << loop() << "ns\n"; + std::cout << "\tmoody pod local: " << loop() << "ns\n"; + return 0; +} + +} + +TEST_CASE( "Performance baseline", "[interconnect]") { + REQUIRE (bts::run_benchmarks() == 0); +} + + + diff --git a/src/server/sparkd/interconnect/tests/interconnect_test.cpp b/src/server/sparkd/interconnect/tests/interconnect_test.cpp new file mode 100644 index 000000000..130314704 --- /dev/null +++ b/src/server/sparkd/interconnect/tests/interconnect_test.cpp @@ -0,0 +1,168 @@ +#include "server/sparkd/interconnect/RPC.h" +#include "server/sparkd/interconnect/Actor.h" +#include "server/sparkd/interconnect/Domain.h" +#include "server/sparkd/interconnect/TopicDirectory.h" +#include "server/sparkd/interconnect/IPollable.h" + +#include "lib/utils/bts_chrono.h" + +#include "catch/catch.h" + +#include + +namespace bts { + +static constexpr auto n_iterations = 10000; +static std::vector durations; + +using PingRPC = RPC ; + +struct PongActor : public Actor { + + void on (PingRPC const&ping) override + { + ping.on_reply(ns_diff(Timer::now(),ping.in)); + } +}; + +struct PingActor : public Actor<> { + std::promise &promise; + Ref pong_ref; + ExecutorID pong_host; + + PingActor(ExecutorID pong_host,std::promise &promise) + : promise(promise), + pong_ref(), + pong_host(pong_host) + {} + + void init () override { + spawn (pong_host, + [this] (RequestError , Ref const *addr) { + pong_ref = *addr; + on_pong(0); + }); + } + + void on_pong (int64_t delta) + { + if (delta != 0) + durations.push_back(delta); + if (durations.size() < n_iterations) + call (pong_ref,Timer::now(), + [this] (RequestError,int64_t *delta) {on_pong (*delta);}); + else + promise.set_value(); + } +}; + +void ping_pong_test(ExecutorID pong_exe) +{ + std::promise promise; + auto ping_done = promise.get_future(); + auto ping_actor = std::make_unique (pong_exe,promise); + Domain d {DomainID{0},2,std::move(ping_actor)}; + ping_done.get(); +} + +int ping_pong_test() +{ + durations.clear(); + durations.reserve(n_iterations); + ping_pong_test(ExecutorID(0)); + std::sort(durations.begin(),durations.end()); + std::cout << "\nactor intra_core rpc\n" + << "\tmedian ns: " << durations[durations.size()/2] << std::endl; + + durations.clear(); + durations.reserve(n_iterations); + ping_pong_test(ExecutorID(1)); + std::sort(durations.begin(),durations.end()); + std::cout << "\nactor inter_core rpc\n" + << "\tmedian ns: " << durations[durations.size()/2] << std::endl; + return 0; +} + + +struct Pub : public Actor<>, public IPollable { + Topic ts_topic; + TimeStamp last_pub; + + Pub() : ts_topic(),last_pub(Timer::now()) {} + + void init() override { + ts_topic = lookup(); + } + + void on_poll() override { + auto now = Timer::now(); + if (ns_diff(now, last_pub) > 2000) { + publish(ts_topic,now); + last_pub = Timer::now(); + } + } + +}; + +struct Sub : public Actor<> { + ExecutorID pub_host; + Topic ts_topic; + std::promise &promise; + + Sub(ExecutorID pub_host,std::promise &promise) + : pub_host(pub_host), + ts_topic(), + promise(promise) + {} + + void init() { + ts_topic = lookup (); + subscribe(ts_topic,[this] (TimeStamp const&ts) { + if (durations.size() < n_iterations) + durations.push_back(ns_diff(Timer::now(),ts)); + else { + unsubscribe(ts_topic); + promise.set_value(); + } + }); + + spawn(pub_host,[] (auto const&,auto const&) {}); + } +}; + +void pub_sub_test(ExecutorID pub_exe) +{ + std::promise promise; + auto done = promise.get_future(); + auto sub_actor = std::make_unique (pub_exe,promise); + Domain d {DomainID{0},2,std::move(sub_actor)}; + done.get(); +} + +int pub_sub_test() +{ + durations.clear(); + durations.reserve(n_iterations); + pub_sub_test(ExecutorID(0)); + std::sort(durations.begin(),durations.end()); + std::cout << "\nactor intra_core pubsub\n" + << "\tmedian ns: " << durations[durations.size()/2] << std::endl; + + durations.clear(); + durations.reserve(n_iterations); + pub_sub_test(ExecutorID(1)); + std::sort(durations.begin(),durations.end()); + std::cout << "\nactor inter_core pubsub\n" + << "\tmedian ns: " << durations[durations.size()/2] << std::endl; + return 0; +} + +} + +// TEST_CASE( "RPC Works", "[interconnect]") { +// REQUIRE (bts::ping_pong_test() == 0); +// } + +// TEST_CASE( "PubSub Works", "[interconnect]") { +// REQUIRE (bts::pub_sub_test() == 0); +// } diff --git a/src/server/sparkd/jetfix/codec/FIXArray.cpp b/src/server/sparkd/jetfix/codec/FIXArray.cpp new file mode 100644 index 000000000..640ea7ddb --- /dev/null +++ b/src/server/sparkd/jetfix/codec/FIXArray.cpp @@ -0,0 +1 @@ +#include "FIXArray.h" diff --git a/src/server/sparkd/jetfix/codec/FIXArray.h b/src/server/sparkd/jetfix/codec/FIXArray.h new file mode 100644 index 000000000..421647148 --- /dev/null +++ b/src/server/sparkd/jetfix/codec/FIXArray.h @@ -0,0 +1,31 @@ +#pragma once + +#include +#include "server/sparkd/jetfix/codec/FIXFragment.h" + +namespace bts { +namespace jetfix { + +template +class FIXArray +{ +public: + inline char*data () {return &_data[0];} + inline const char*data () const {return &_data[0];} + inline uint32_t capacity () const {return _data.size ();} + inline uint32_t size () const {return _size;} + inline uint32_t checksum () const{return _checksum;} + inline void update (jetfix::FIXFragment const&frag) + { + bts_assert (frag.data () == &_data[0]); + _size = frag.size (); + _checksum = frag.checksum (); + } + +private: + std::array _data; + uint32_t _size; + uint32_t _checksum; +}; + +}} diff --git a/src/server/sparkd/jetfix/codec/FIXBuffer.h b/src/server/sparkd/jetfix/codec/FIXBuffer.h new file mode 100644 index 000000000..0dc30f599 --- /dev/null +++ b/src/server/sparkd/jetfix/codec/FIXBuffer.h @@ -0,0 +1,145 @@ +#pragma once + +#include +#include +#include "lib/utils/bts_assert.h" + +namespace bts { +namespace jetfix { + +class FIXBuffer +{ +public: + // memory is owned by someone else + inline FIXBuffer(char *buffer_to_use, uint32_t buffer_capacity) : + _buffer (buffer_to_use), + _size (0), + _capacity (buffer_capacity), + _owns_buffer (false), + _initial_size () + {} + // a buffer that already has some content + inline FIXBuffer(char *buffer_to_use, uint32_t size, uint32_t capacity): + _buffer (buffer_to_use), + _size (size), + _capacity (capacity), + _owns_buffer (false), + _initial_size (size) + {} + + // memory owned by this buffer + explicit inline FIXBuffer (uint32_t initial_capacity) : + _buffer (new char[initial_capacity]), + _size (0), + _capacity (initial_capacity), + _owns_buffer (true), + _initial_size () + {} + + inline FIXBuffer (FIXBuffer const&src) : + _buffer (), + _size (), + _capacity (), + _owns_buffer (false), + _initial_size () + { + operator=(src); + } + + inline FIXBuffer (FIXBuffer &&src) + : + _buffer (src._buffer), + _size (src._size), + _capacity (src._capacity), + _owns_buffer (src._owns_buffer), + _initial_size (src._initial_size) + { + src._buffer = nullptr; + src._size = 0; + src._capacity = 0; + src._owns_buffer = false; + src._initial_size = 0; + } + + inline ~FIXBuffer () + { + if (_owns_buffer) + delete [] _buffer; + } + + inline FIXBuffer&operator=(FIXBuffer const&src) + { + this->~FIXBuffer (); + if (src._owns_buffer) { + _buffer = new char[src._capacity]; + memcpy (_buffer,src._buffer,src._size); + } + else + _buffer = src._buffer; + _size = src._size; + _capacity = src._capacity; + _owns_buffer = src._owns_buffer; + _initial_size = src._initial_size; + return *this; + } + + inline FIXBuffer&operator=(FIXBuffer &&src) + { + this->~FIXBuffer (); + _buffer = src._buffer; + _size = src._size; + _capacity= src._capacity; + _owns_buffer = src._owns_buffer; + _initial_size = src._initial_size; + + src._buffer = nullptr; + src._size = 0; + src._capacity = 0; + src._owns_buffer = false; + src._initial_size = 0; + return *this; + } + + inline char* reserve (uint32_t need_bytes) + { + if (_size + need_bytes > _capacity) { + bts_assert_d (_owns_buffer); + _capacity = (_capacity+need_bytes) * 2; + auto old_buffer = _buffer; + _buffer = new char[_capacity]; + memcpy (_buffer,old_buffer,_size); + delete [] old_buffer; + } + return _buffer + _size; + } + + inline char* get_offset (uint32_t offset) + { + bts_assert (offset <= _size); + return _buffer+offset; + } + + inline char* advance (uint32_t bytes) + { + _size += bytes; + bts_assert_d (_size<=_capacity); + return _buffer+_size; + } + + void reset () + {_size = _initial_size;} + + uint32_t size () const {return _size;} + uint32_t capacity () const {return _capacity;} + const char*data () const {return _buffer;} + char*data () {return _buffer;} + +private: + char *_buffer; + uint32_t _size; + uint32_t _capacity; + bool _owns_buffer; + uint32_t _initial_size; +}; + +}} diff --git a/src/server/sparkd/jetfix/codec/FIXEncodingStream.h b/src/server/sparkd/jetfix/codec/FIXEncodingStream.h new file mode 100644 index 000000000..f9a880525 --- /dev/null +++ b/src/server/sparkd/jetfix/codec/FIXEncodingStream.h @@ -0,0 +1,224 @@ +#pragma once + +#include +#include "server/sparkd/jetfix/codec/FIXBuffer.h" +#include "server/sparkd/jetfix/codec/MessageTypes.h" +#include "server/sparkd/jetfix/codec/FIXFragment.h" +#include "server/sparkd/jetfix/codec/Fields.h" +#include "server/sparkd/jetfix/codec/FieldDefinitions.h" + +#include "lib/config/protos/jetfix.pb.h" + +namespace bts { +namespace jetfix { + +class FIXEncodingStream +{ +public: + FIXEncodingStream(config::JetFIX::Connection const &cfg, + uint32_t initial_capacity = 100*1024*10) + : + _msg_header_template (), + _header_template_size (), + _header_template_checksum (), + _buffer (initial_capacity), + _message_map (), + _current_msg (_buffer), + _state (State::OPEN), + _body_length_bm (), + _msg_type_bm (), + _seq_num_bm (), + _last_seq_num_bm (), + _sending_time_bm () + { + init(cfg,cfg.sender_subid()); + } + + FIXEncodingStream(config::JetFIX::Connection const &cfg, + std::string const&sender_sub_id, + uint32_t initial_capacity = 100*1024*10) + : + _msg_header_template (), + _header_template_size (), + _header_template_checksum (), + _buffer (initial_capacity), + _message_map (), + _current_msg (_buffer), + _state (State::OPEN), + _body_length_bm (), + _msg_type_bm (), + _seq_num_bm (), + _last_seq_num_bm (), + _sending_time_bm () + { + init(cfg,sender_sub_id); + } + + inline FIXFragment &new_msg (MessageTypes type) + { + bts_assert (_state == State::OPEN); + tidy_last_msg (); + + auto msg_offset = _buffer.size (); + _message_map.emplace_back (msg_offset,0u,0u,0u,FIXFragment::Bookmark>()); + _current_msg = FIXFragment (_buffer); + _current_msg.append (&_msg_header_template[0], + _header_template_size, + _header_template_checksum); + char msg_type_str[1] = {(char) type}; + _current_msg.overwrite (_msg_type_bm.shift (msg_offset),msg_type_str,1); + return _current_msg; + } + + inline void close () + { + bts_assert (_state == State::OPEN); + tidy_last_msg (); + _state = State::CLOSED; + } + + inline void reopen_empty () + { + bts_assert (_state == State::CLOSED); + bts_assert (_message_map.empty ()); + _state = State::OPEN; + } + + inline uint32_t size () const + {return _buffer.size ();} + + inline const char* data () const + {return _buffer.data ();} + + struct MessageDesc + { + MessageDesc (uint32_t offset, + uint32_t size, + uint32_t checksum, + uint32_t seq_num, + FIXFragment::Bookmark checksum_bm) + : offset (offset), + size (size), + checksum (checksum), + seq_num (seq_num), + checksum_bm (checksum_bm) + {} + + + uint32_t offset; + uint32_t size; + uint32_t checksum; + uint32_t seq_num; + FIXFragment::Bookmark checksum_bm; + }; + + std::string to_str () const + { + std::string result; + + if (size ()) { + // return std::string(data(),size() is simpler but valgrind + // reports a warning in that call + result.resize (size ()); + std::copy (data (), data ()+size(),result.begin ()); + } + return result; + } + + std::string to_human_str () const + { + auto s = to_str (); + std::replace( s.begin(), s.end(), '\001', '|' ); + return s; + } + + inline uint32_t stamp (uint32_t seq_num,uint32_t last_processed, std::string const&time_str) + { + bts_assert (_state == State::CLOSED); + _state = State::STAMPED; + + for (auto &m : _message_map) { + ++seq_num; + m.seq_num = seq_num; + + FIXFragment frag (_buffer,m.checksum); + frag.overwrite (_seq_num_bm.shift (m.offset),m.seq_num,9); + frag.overwrite (_last_seq_num_bm.shift (m.offset),last_processed,9); + frag.overwrite (_sending_time_bm.shift (m.offset),time_str); + frag.overwrite_checksum (m.checksum_bm,frag.checksum () % 256); + } + return seq_num; + } + + inline void reset () + { + _buffer.reset (); + _message_map.clear (); + _state = State::OPEN; + } + + inline bool is_closed () const + {return _state == State::CLOSED;} + + size_t n_msgs () const {return _message_map.size ();} + + std::vector const &msg_map () const {return _message_map;} +private: + void init(config::JetFIX::Connection const &cfg, + std::string const&sender_sub_id) { + + _message_map.reserve (10*1024); + + FIXBuffer header_buffer (&_msg_header_template[0],_msg_header_template.size ()); + FIXFragment header (header_buffer); + header.add("FIX.4.2"); + + _body_length_bm = header.add(0,6); + _msg_type_bm = header.add(" ",1); + _seq_num_bm = header.add(0,9); + _last_seq_num_bm = header.add(0,9); + _sending_time_bm = header.add( + "20150427-19:05:18.923",sizeof("20150427-19:05:18.923")-1); + + header.add(cfg.sender_compid ()); + header.add(sender_sub_id); + header.add(cfg.target_compid ()); + header.add(cfg.target_subid ()); + header.add(cfg.sender_locationid ()); + _header_template_size = header.size (); + _header_template_checksum = header.checksum (); + } + + std::array _msg_header_template; + uint32_t _header_template_size; + uint32_t _header_template_checksum; + FIXBuffer _buffer; + std::vector _message_map; + FIXFragment _current_msg; + enum class State { OPEN,CLOSED,STAMPED}; + State _state; + + FIXFragment::Bookmark _body_length_bm; + FIXFragment::Bookmark _msg_type_bm; + FIXFragment::Bookmark _seq_num_bm; + FIXFragment::Bookmark _last_seq_num_bm; + typedef StringField SendingTimeStr; + FIXFragment::Bookmark _sending_time_bm; + + inline void tidy_last_msg () + { + if (_message_map.size ()) { + auto &this_msg_map = _message_map.back (); + auto start = + this_msg_map.offset + _msg_type_bm.offset - fix::MsgType::tag_sz-1; + auto end = _buffer.size (); + _current_msg.overwrite (_body_length_bm.shift (this_msg_map.offset), + end-start,6); + this_msg_map.checksum = _current_msg.checksum (); + this_msg_map.checksum_bm = _current_msg.add_checksum (0); + this_msg_map.size = _buffer.size () - this_msg_map.offset; + } + } +}; + +}} diff --git a/src/server/sparkd/jetfix/codec/FIXFragment.h b/src/server/sparkd/jetfix/codec/FIXFragment.h new file mode 100644 index 000000000..276d54f75 --- /dev/null +++ b/src/server/sparkd/jetfix/codec/FIXFragment.h @@ -0,0 +1,201 @@ +#pragma once + +#include +#include + +#include "lib/utils/bts_assert.h" +#include "server/sparkd/jetfix/codec/FIXBuffer.h" +#include "server/sparkd/jetfix/codec/Fields.h" + +namespace bts { +namespace jetfix { + +class FIXFragment +{ +public: + FIXFragment(FIXBuffer &buffer,uint32_t checksum) + : + _buffer (&buffer), + _checksum (checksum), + _initial_offset (buffer.size ()), + _owns_buffer (false) + {} + + FIXFragment(FIXBuffer &buffer) + : + _buffer (&buffer), + _checksum (0), + _initial_offset (buffer.size ()), + _owns_buffer (false) + {} + + explicit FIXFragment(uint32_t initial_capacity=100*1024) + : + _buffer (new FIXBuffer (initial_capacity)), + _checksum (0), + _initial_offset (0), + _owns_buffer (true) + {} + + ~FIXFragment () + { + if (_owns_buffer) + delete _buffer; + } + + FIXFragment (FIXFragment const&src) : + _buffer (), + _checksum (0), + _initial_offset (0), + _owns_buffer () + { + operator=(src); + } + + FIXFragment (FIXFragment &&src) : + _buffer (std::move (src._buffer)), + _checksum (src._checksum), + _initial_offset (src._initial_offset), + _owns_buffer (src._owns_buffer) + { + src._buffer = nullptr; + src._checksum = 0; + src._initial_offset = 0; + src._owns_buffer = false; + } + + void reset () + { + bts_assert (_owns_buffer); + _buffer->reset (); + } + + FIXFragment&operator=(FIXFragment const&src) + { + this->~FIXFragment (); + + if (src._owns_buffer) + _buffer = new FIXBuffer (*src._buffer); + else + _buffer = src._buffer; + _checksum = src._checksum; + _initial_offset = src._initial_offset; + _owns_buffer = src._owns_buffer; + return *this; + } + + FIXFragment&operator=(FIXFragment &&src) + { + this->~FIXFragment (); + _buffer = std::move (src._buffer); + _checksum= src._checksum; + _initial_offset = src._initial_offset; + src._buffer = nullptr; + src._checksum = 0; + src._initial_offset = 0; + src._owns_buffer = false; + return *this; + } + + template + struct Bookmark + { + Bookmark (uint32_t idx,uint32_t sz,uint32_t checksum) : + offset (idx), + sz (sz), + val_checksum (checksum) + {} + Bookmark () : + offset (), + sz (), + val_checksum () + {} + + Bookmark shift (uint32_t by) const + {return Bookmark (offset+by,sz,val_checksum);} + + typedef FT Field; + uint32_t offset; + uint32_t sz; + uint32_t val_checksum; + }; + + template + inline Bookmark add (Args&&... args) + { + auto non_value_data_sz = Field::tag_sz + 2; // =,SOH + auto val_max_size = Field::max_value_size (std::forward(args)...); + + auto ptr = _buffer->reserve (val_max_size + non_value_data_sz); + std::memcpy (ptr,Field::tag_s,Field::tag_sz); + ptr = _buffer->advance (Field::tag_sz); + *ptr = '='; + ptr = _buffer->advance (1); + auto start_of_value = _buffer->size (); + auto write_result = Field::write (ptr,std::forward(args)...); + ptr = _buffer->advance (write_result.first); + *ptr = 1; + ptr = _buffer->advance (1); + _checksum += (write_result.second + Field::tag_checksum + '=' + 1); + return Bookmark (start_of_value,write_result.first, write_result.second); + } + + template + inline Bookmark overwrite (Bookmark const &bm,Args &&... args) + { + _checksum -= bm.val_checksum; + auto ptr = _buffer->get_offset (bm.offset); + auto write_result = Field::write (ptr,std::forward(args)...); + ptr[bm.sz] = 1; // i32_toa puts a 0 at end of the number + bts_assert (write_result.first == bm.sz); + _checksum += write_result.second; + return Bookmark (bm.offset,bm.sz, write_result.second); + } + + inline Bookmark> add_checksum (uint8_t chk) + { + auto checksum = _checksum; + auto bm = add> (chk,3); + _checksum = checksum; + bm.val_checksum = 0; + return bm; + } + + inline Bookmark> overwrite_checksum (Bookmark> const&bm,uint8_t chk) + { + auto checksum = _checksum; + overwrite> (bm,chk,3); + _checksum = checksum; + return bm; + } + + template inline void append (FIX_DATA const &src) + {append (src.data (),src.size (),src.checksum ());} + + inline void append (const char*data, uint32_t data_size, uint32_t data_checksum) + { + auto ptr = _buffer->reserve (data_size); + _checksum += data_checksum; + memcpy (ptr,data,data_size); + _buffer->advance (data_size); + } + + std::string to_str () const + { + return std::string (_buffer->data (), + _initial_offset, + _buffer->size ()); + } + + uint32_t size () const {return _buffer->size () - _initial_offset;} + uint32_t checksum () const {return _checksum; } + const char* data () const {return _buffer->get_offset (_initial_offset);} + +private: + FIXBuffer *_buffer; + size_t _checksum; + uint32_t _initial_offset; + bool _owns_buffer; +}; + +}} diff --git a/src/server/sparkd/jetfix/codec/FIXMsg.h b/src/server/sparkd/jetfix/codec/FIXMsg.h new file mode 100644 index 000000000..7c940ea3b --- /dev/null +++ b/src/server/sparkd/jetfix/codec/FIXMsg.h @@ -0,0 +1,104 @@ +#pragma once + +#include +#include +#include + +#include "server/sparkd/jetfix/codec/MessageTypes.h" +#include "server/sparkd/jetfix/codec/WriterCursor.h" +#include "server/sparkd/jetfix/codec/ReaderCursor.h" +#include "server/sparkd/jetfix/codec/IFieldReader.h" +#include "server/sparkd/jetfix/codec/IFIXMsg.h" +#include "server/sparkd/jetfix/common/InMsgBuffer.h" +#include "lib/utils/meta_programming.h" + +namespace bts { +namespace jetfix { + +template +class FIXMsg : public IFIXMsg +{ +public: + static const auto tag = fix_tag; + + FIXMsg () : + _fields (), _has_value (), _field_idx () + { + index_fields (); + } + + FIXMsg (InMsgBuffer const&buffer) : + _fields (), _has_value (), _field_idx () + { + index_fields (); + read (buffer.data (),buffer.size ()); + } + + FIXMsg (std::string const&fix) : _fields (), _has_value (),_field_idx () + { + index_fields (); + read (fix); + } + + MessageTypes msg_type () const override + {return tag;} + + template bool has () const + { + auto field_idx = meta::GetTypeIdxInTuple::value; + return _has_value[field_idx]; + } + + template typename Field::Value const &get () const + { + bts_assert (has ()); + return std::get::value> (_fields).value (); + } + + void read (std::string const&str) + { + read (str.data (),str.size ()); + } + + void read (const char*data, size_t sz) + { + _has_value.reset (); + ReaderCursor cursor (data,sz); + while (!cursor.eof ()) + { + if (!try_read (cursor.read_tag (),cursor)) + cursor.read_field (); + } + } + +private: + typedef std::tuple FieldTuple; + + void index_fields () + { + std::vector rws; + meta::collect_tuple_items (_fields,&rws); + for (auto rw : rws) + _field_idx.push_back (std::make_pair (rw->get_tag (),rw)); + } + + bool try_read (uint32_t tag, ReaderCursor &cursor) + { + size_t i = 0; + for (auto &rwp : _field_idx) + if (rwp.first == tag) { + rwp.second->read (cursor); + _has_value[i] = true; + return true; + } + else i++; + return false; + } + + FieldTuple _fields; + std::bitset::value> _has_value; + std::vector> _field_idx; +}; + + +}} diff --git a/src/server/sparkd/jetfix/codec/FIXParser.h b/src/server/sparkd/jetfix/codec/FIXParser.h new file mode 100644 index 000000000..5babb4d6e --- /dev/null +++ b/src/server/sparkd/jetfix/codec/FIXParser.h @@ -0,0 +1,63 @@ +#pragma once + +#include +#include "server/sparkd/jetfix/codec/IFIXParser.h" + +namespace bts { +namespace jetfix { + + +template +class FIXParser : public IFIXParser +{ +public: + FIXParser() : _parsers (), _parser_idx () + { + std::vector parser_vec; + meta::collect_tuple_items (_parsers,&parser_vec); + for (auto p : parser_vec) + _parser_idx.emplace_back (p->tag (),p); + typedef std::pair Pair; + + std::sort (_parser_idx.begin (),_parser_idx.end (), + [] (Pair const&l, Pair const&r) { + return (int) l.first < (int) r.first; + }); + } + + IFIXMsgPtr parse (InMsgBuffer const&msg) const override + { + auto type = msg.type (); + for (auto &p : _parser_idx) + if (p.first == type) + return p.second->parse (msg); + else if (p.first > type) + break; + + return IFIXMsgPtr (); + } + +private: + struct IParser + { + virtual ~IParser() {} + virtual MessageTypes tag () const = 0; + virtual IFIXMsgPtr parse (InMsgBuffer const&msg) const = 0; + }; + + template struct Parser : public IParser + { + + MessageTypes tag () const override {return MT::tag;} + IFIXMsgPtr parse (InMsgBuffer const&msg) const override + { + auto ptr = std::make_shared (msg); + return std::dynamic_pointer_cast (ptr); + } + }; + + std::tuple...> _parsers; + std::vector> _parser_idx; +}; + +}} diff --git a/src/server/sparkd/jetfix/codec/FIXWriter.h b/src/server/sparkd/jetfix/codec/FIXWriter.h new file mode 100644 index 000000000..da2c44d93 --- /dev/null +++ b/src/server/sparkd/jetfix/codec/FIXWriter.h @@ -0,0 +1,20 @@ +#pragma once + +#include + +namespace bts { +namespace jetfix { + +class FIXWriter +{ +public: + FIXWriter(); + ~FIXWriter(); + +private: + char *_buffer; + size_t _buffer_size; + +}; + +}} diff --git a/src/server/sparkd/jetfix/codec/FieldDefinitions.h b/src/server/sparkd/jetfix/codec/FieldDefinitions.h new file mode 100644 index 000000000..d08382207 --- /dev/null +++ b/src/server/sparkd/jetfix/codec/FieldDefinitions.h @@ -0,0 +1,129 @@ +#pragma once + +#include +#include "Fields.h" +#include "Group.h" + +namespace bts { +namespace jetfix { +namespace fix { + +typedef StringField<8> BeginString; +typedef IntField<9> BodyLength; +typedef StringField<35> MsgType; +typedef IntField<34> MsgSeqNum; +typedef IntField<369> LastSeqNumProcessed; +typedef StringField<49> SenderCompID; +typedef StringField<50> SenderSubID ; +typedef TimeField<52> SendingTime; +typedef TimeField<122> OrigSendingTime; +typedef StringField<56> TargetCompId; +typedef StringField<57> TargetSubID; +typedef StringField<142> SenderLocationID; +typedef StringField<117> QuoteID; +typedef IntField<117> QuoteIDInt; +typedef IntField<204> CustomerOrFirm; +typedef IntField<296> NoQuoteSets; +typedef StringField<302> QuoteSetID; +typedef StringField<307> UnderlyingSecurityDesc; +typedef IntField<304> TotQuoteEntries; +typedef IntField<295> NoQuoteEntries; +typedef StringField<304> TotQuoteEntriesStr; +typedef StringField<295> NoQuoteEntriesStr; + +typedef BoolField<1028> ManualOrderIndicator; +typedef StringField<7928> SelfMatchPreventionID; +typedef CharField<9702> CtiCode; +typedef StringField<9771> MMAccount; +typedef CharField<21> HandlInst; +typedef CharField<40> OrdType; +typedef CharField<59> TimeInForce; + + +typedef FloatField<132> BidPx; +typedef IntField<134> BidSize; +typedef StringField<132> BidPxStr; +typedef StringField<134> BidSizeStr; + +typedef FloatField<133> OfferPx; +typedef IntField<135> OfferSize; +typedef StringField<133> OfferPxStr; +typedef StringField<135> OfferSizeStr; + +typedef StringField<299> QuoteEntryID; +typedef IntField<299> QuoteEntryIDInt; +typedef StringField<55> Symbol; +typedef StringField<167> SecurityType; +typedef StringField<107> SecurityDesc; +typedef StringField<48> SecurityID; +typedef CharField<22> SecurityIDSource; +typedef TimeField<60> TransactTime; +typedef StringField<60> TransactTimeStr; +typedef StringField<143> TargetLocationID; +typedef StringField<142> SenderLocationID; +typedef StringField<142> SenderLocationID; +typedef IntField<10> Checksum; +typedef IntField<98> EncryptMethod; +typedef IntField<108> HeartBtInt; +typedef StringField<1603> ApplicationSystemName; +typedef StringField<1604> ApplicationSystemVersion; +typedef StringField<1605> ApplicationSystemVendor; +typedef IntField<95> RawDataLength; +typedef StringField<96> RawData; +typedef StringField<109> ClientID; +typedef IntField<7> BeginSeqNo; +typedef IntField<16> EndSeqNo; +typedef StringField<112> TestReqId; +typedef BoolField<123> GapFillFlag; +typedef IntField<36> NewSeqNo; +typedef BoolField<43> PossDupFlag; +typedef IntField<297> QuoteAckStatus; +typedef CharField<9775> UnsolicitedCancelType; +typedef IntField<300> QuoteRejectReason; +typedef StringField<58> Text; +typedef IntField<368> QuoteEntryRejectReason; +typedef IntField<298> QuoteCancelType; +typedef StringField<1> Account; +typedef CharField<150> ExecType; +typedef StringField<11> ClOrdID; +typedef CharField<54> Side; +typedef FloatField<31> LastPx; +typedef IntField<32> LastShares; +typedef IntField<14> CumQty; +typedef StringField<527> SecondaryExecID; +typedef StringField<76> ExecBroker; +typedef StringField<17> ExecID; +typedef FloatField<151> LeavesQty; +typedef IntField<442> MultiLegReportingType; +typedef IntField<386> NoTradingSessions; +typedef StringField<336> TradingSessionID; +typedef StringField<440> ClearingAccount; +typedef CharField<47> Rule80a; +typedef StringField<9324> OrderOptionalData; + +typedef Group +QuoteEntries; + +typedef Group +QuoteSets; + +typedef IntField<38> OrderQty; +typedef FloatField<44> Price; +typedef IntField<9003> UDFSupportIndicator; +}}} diff --git a/src/server/sparkd/jetfix/codec/Fields.h b/src/server/sparkd/jetfix/codec/Fields.h new file mode 100644 index 000000000..c368cd7e0 --- /dev/null +++ b/src/server/sparkd/jetfix/codec/Fields.h @@ -0,0 +1,209 @@ +#pragma once + +#include +#include +#include "server/sparkd/jetfix/codec/ReaderCursor.h" +#include "lib/utils/meta_programming.h" +#include "lib/utils/datetime.h" +#include "server/sparkd/jetfix/codec/sum_bytes.h" +#include "server/sparkd/jetfix/codec/i32_toa.h" +#include "server/sparkd/jetfix/codec/f_toa.h" +#include "server/sparkd/jetfix/codec/IFieldReader.h" + +namespace bts { +namespace jetfix { + +template +class BaseField : public IFieldReader +{ + // compile time calculation of tag checksum + template struct calc_checksum; + + template + struct calc_checksum + {static const uint32_t value = N + '0';}; + + template + struct calc_checksum + {static const uint32_t value = (N%10 + '0') + calc_checksum::value;}; + + +public: + static uint16_t const tag; + static const char* tag_s; + static size_t const tag_sz; + static uint32_t const tag_checksum ; + + typedef ValueType Value; + + BaseField () : _value () {} + const ValueType & value () const {return _value;} + + virtual uint32_t get_tag () + {return tag;} +protected: + ValueType _value; +}; + +template +uint16_t const BaseField::tag = t_tag; + +template +const char* BaseField::tag_s = meta::int_to_string::type::value; + +template +size_t const BaseField::tag_sz = meta::int_to_string::size; + +template +uint32_t const BaseField::tag_checksum = calc_checksum::value; + +template +class CharField : public BaseField +{ +public: + typedef BaseField Parent; + + void read (ReaderCursor &cursor) + { + auto loc = cursor.read_field (); + bts_assert (loc.second == 1); + Parent::_value = cursor.data ()[loc.first]; + } + + constexpr static uint32_t max_value_size (char ) {return 1;} + static std::pair write (char* buffer,char c) + { + *buffer = c; + return std::make_pair (1,c); + } +}; + +template +class BoolField : public BaseField +{ +public: + typedef BaseField Parent; + + void read(ReaderCursor &cursor) + { + auto loc = cursor.read_field (); + bts_assert (loc.second == 1); + Parent::_value = cursor.data ()[loc.first] == 'Y'; + } + + constexpr static uint32_t max_value_size ( bool) {return 1;} + static std::pair write (char *buffer,bool b) + { + if (b) + *buffer = 'Y'; + else + *buffer = 'N'; + return std::make_pair (1,*buffer); + } +}; + +template +class FloatField : public BaseField +{ +public: + typedef BaseField Parent; + + void read (ReaderCursor &cursor) + { + auto loc = cursor.read_field (); + Parent::_value = atof (&(cursor.data ()[loc.first])); + } + + static uint32_t max_value_size ( float,int pad = 0) + { + if (pad == 0) + return 15; + else + return pad; + } + + static std::pair write (char*buffer,float f,int pad = 0) + { + auto written = f_toa (f,buffer,max_value_size (f,pad),pad); + return std::make_pair (written,sum_bytes (buffer,written)); + } +}; + +template +class IntField : public BaseField +{ +public: + typedef BaseField Parent; + + void read (ReaderCursor &cursor) + { + auto loc = cursor.read_field (); + Parent::_value = atoi (&(cursor.data ()[loc.first])); + } + + static uint32_t max_value_size (int32_t,int pad = 0) + { + if (pad == 0) + return 11; + else + return pad; + } + + static std::pair write (char*buffer,int32_t i,int pad = 0) + { + auto written = i32_toa (i,buffer,pad); + return std::make_pair (written,sum_bytes (buffer,written)); + } +}; + +template +class StringField : public BaseField +{ +public: + typedef BaseField Parent; + + void read (ReaderCursor &cursor) + { + auto loc = cursor.read_field (); + BaseField::_value = + std::string (cursor.data ()+loc.first,loc.second); + } + + + static uint32_t max_value_size (std::string const&str) {return str.size ();} + static uint32_t max_value_size (const char*,uint32_t sz) {return sz;} + + static std::pair write (char*buffer,std::string const &str) + {return write (buffer,str.data (),str.size ());} + + static std::pair write (char*buffer,const char*str,uint32_t sz) + { + memcpy (buffer,str,sz); + return std::make_pair (sz,sum_bytes (buffer,sz)); + } +}; + +template +class TimeField : public BaseField +{ +public: + typedef BaseField Parent; + + void read (ReaderCursor &cursor) + { + auto loc = cursor.read_field (); + BaseField::_value = + datetime::parse_FIX_timestamp (std::string (cursor.data ()+loc.first,loc.second)); + } + + constexpr static uint32_t max_value_size (boost::posix_time::ptime const&) + {return sizeof("20121204-14:44:02.000")-1;} + + static std::pair write (char*buffer,boost::posix_time::ptime const &t) + { + memcpy (buffer,datetime::FIX_timestamp (t).data (),max_value_size (t)); + return std::make_pair (max_value_size (t),sum_bytes (buffer,max_value_size (t))); + } +}; + +}} diff --git a/src/server/sparkd/jetfix/codec/Group.h b/src/server/sparkd/jetfix/codec/Group.h new file mode 100644 index 000000000..a703b9128 --- /dev/null +++ b/src/server/sparkd/jetfix/codec/Group.h @@ -0,0 +1,135 @@ +#pragma once + +#include +#include +#include + +#include "ReaderCursor.h" +#include "WriterCursor.h" +#include "IFieldReader.h" + +#include "lib/utils/meta_programming.h" +#include "lib/utils/bts_assert.h" + +namespace bts { +namespace jetfix { + +template +class GroupElement +{ +public: + GroupElement () : _fields (),_has_value (),_field_idx () {index_fields ();} + + template bool has () const + { + auto field_idx = meta::GetTypeIdxInTuple::value; + return _has_value[field_idx]; + } + + template typename Field::Value const &get () const + { + bts_assert (has ()); + return std::get::value> (_fields).value (); + } + + void read (ReaderCursor &cursor) + { + _has_value.reset (); + auto &separator = std::get<0> (_fields); + auto tag = cursor.read_tag (); + bts_assert (tag == separator.tag); + separator.read (cursor); + _has_value[0] = true; + + do { + if (!cursor.eof ()) + tag = cursor.read_tag (); + else + break; + } while (tag != separator.tag && try_read (tag,cursor)); + + if (!cursor.eof ()) + cursor.unread_tag (); + } + +private: + typedef std::tuple FieldTuple; + + void index_fields () + { + std::vector rws; + meta::collect_tuple_items (_fields,&rws); + for (auto rw : rws) + _field_idx.push_back (std::make_pair (rw->get_tag (),rw)); + } + + bool try_read (uint32_t tag, ReaderCursor &cursor) + { + size_t i = 0; + for (auto &rwp : _field_idx) + if (rwp.first == tag) { + rwp.second->read (cursor); + _has_value[i] = true; + return true; + } + else i++; + return false; + } + + FieldTuple _fields; + std::bitset::value> _has_value; + std::vector> _field_idx; +}; + +template +class Group : public IFieldReader +{ +public: + typedef Group Value; + typedef GroupElement Element; + + Group () : _elements () {} + + Value const &value () const + {return *this;} + + size_t size () const + {return _elements.size ();} + + Element const& operator[](size_t idx) const + { + bts_assert (idx < size ()); + return _elements[idx]; + } + + typename std::vector::const_iterator begin () const + {return _elements.begin ();} + + typename std::vector::const_iterator end () const + {return _elements.end ();} + + void write (WriterCursor &cursor) + { + WriterCursor::Loc dummy_result; + CountField::write (cursor,(int) size ()); + for (auto &el : _elements) + el.write (cursor); + } + + void read (ReaderCursor &cursor) + { + CountField count; + count.read (cursor); + _elements.resize (count.value ()); + for (auto &element : _elements) + element.read (cursor); + } + + uint32_t get_tag () + {return CountField::tag;} + +private: + std::vector _elements; +}; + +}} diff --git a/src/server/sparkd/jetfix/codec/IFIXMsg.h b/src/server/sparkd/jetfix/codec/IFIXMsg.h new file mode 100644 index 000000000..34b4d7450 --- /dev/null +++ b/src/server/sparkd/jetfix/codec/IFIXMsg.h @@ -0,0 +1,16 @@ +#pragma once + +#include +#include "server/sparkd/jetfix/codec/MessageTypes.h" + +namespace bts { +namespace jetfix { + +class IFIXMsg +{ +public: + virtual MessageTypes msg_type () const = 0; + virtual ~IFIXMsg () {} +}; +typedef std::shared_ptr IFIXMsgPtr; +}} diff --git a/src/server/sparkd/jetfix/codec/IFIXParser.h b/src/server/sparkd/jetfix/codec/IFIXParser.h new file mode 100644 index 000000000..37024e6ca --- /dev/null +++ b/src/server/sparkd/jetfix/codec/IFIXParser.h @@ -0,0 +1,18 @@ +#pragma once + +#include +#include "server/sparkd/jetfix/codec/IFIXMsg.h" + +namespace bts { +namespace jetfix { + +class InMsgBuffer; + +class IFIXParser +{ +public: + virtual IFIXMsgPtr parse (InMsgBuffer const&msg) const = 0; + virtual ~IFIXParser () {} +}; + +}} diff --git a/src/server/sparkd/jetfix/codec/IFieldReader.h b/src/server/sparkd/jetfix/codec/IFieldReader.h new file mode 100644 index 000000000..06cebdd7b --- /dev/null +++ b/src/server/sparkd/jetfix/codec/IFieldReader.h @@ -0,0 +1,18 @@ +#pragma once + +#include "ReaderCursor.h" +#include + +namespace bts { +namespace jetfix { + +class IFieldReader +{ +public: + virtual void read (ReaderCursor &cursor) = 0; + virtual uint32_t get_tag () = 0; + virtual ~IFieldReader () {} +}; + + +}} diff --git a/src/server/sparkd/jetfix/codec/MessageDefinitions.h b/src/server/sparkd/jetfix/codec/MessageDefinitions.h new file mode 100644 index 000000000..f2a411207 --- /dev/null +++ b/src/server/sparkd/jetfix/codec/MessageDefinitions.h @@ -0,0 +1,46 @@ +#pragma once + +#include +#include "conn/bts_conn/jetfix/codec/FIXMsg.h" +#include "conn/bts_conn/jetfix/codec/MessageTypes.h" +#include "conn/bts_conn/jetfix/codec/FieldDefinitions.h" + +namespace bts { +namespace jetfix { +namespace fix { + +typedef FIXMsg +QuoteAcknowledgementMsg; + +typedef FIXMsg +ExecutionReportMsg; + +typedef FIXMsg +BusinessMessageRejectMsg; + +typedef FIXMsg +RejectMsg; + +}}} + diff --git a/src/server/sparkd/jetfix/codec/MessageTypes.h b/src/server/sparkd/jetfix/codec/MessageTypes.h new file mode 100644 index 000000000..39dfd7b34 --- /dev/null +++ b/src/server/sparkd/jetfix/codec/MessageTypes.h @@ -0,0 +1,54 @@ +#pragma once + +#include + +namespace bts { +namespace jetfix { + +enum class MessageTypes +{ + Advertisement = '7', + Allocation = 'J', + AllocationACK = 'P', + BidRequest = 'k', + BidResponse = 'l', + BusinessMessageReject = 'j', + Cancel = 'F', + CancelReject = '9', + CancelReplace = 'G', + DontKnowTrade = 'Q', + Email = 'C', + ExecutionReport = '8', + HB = '0', + IndicationOfInterest = '6', + ListCancelRequest = 'K', + ListExecute = 'L', + ListStatus = 'N', + ListStatusRequest = 'M', + ListStrikePrice = 'm', + Logon = 'A', + Logout = '5', + MassQuote = 'i', + NewOrderList = 'E', + NewOrderSingle = 'D', + News = 'B', + OrderStatusRequest = 'H', + Quote = 'S', + QuoteAck = 'b', + QuoteCancel = 'Z', + QuoteRequest = 'R', + QuoteStatusRequest = 'a', + Reject = '3', + ResendRequest = '2', + SecurityDefinition = 'd', + SecurityDefinitionRequest = 'c', + SecurityStatus = 'f', + SecurityStatusRequest = 'e', + SeqReset = '4', + SettlementInstructions = 'T', + TestRequest = '1', + TradingSessionStatus = 'h', + TradingSessionStatusRequest = 'g' +}; + +}} diff --git a/src/server/sparkd/jetfix/codec/ReaderCursor.h b/src/server/sparkd/jetfix/codec/ReaderCursor.h new file mode 100644 index 000000000..7a56e650d --- /dev/null +++ b/src/server/sparkd/jetfix/codec/ReaderCursor.h @@ -0,0 +1,72 @@ +#pragma once + +#include +#include "lib/utils/bts_assert.h" + +namespace bts { +namespace jetfix { + +class ReaderCursor +{ +public: + ReaderCursor (std::string const &data) : _in_tag (true), + _data (data.data ()), + _idx (0), + _size (data.size ()), + _previous_tag (0) + {} + + ReaderCursor (const char*data, size_t sz) : _in_tag (true), + _data (data), + _idx (0), + _size (sz), + _previous_tag (0) + {} + + uint32_t read_tag () + { + bts_assert (_in_tag); + _previous_tag = _idx; + size_t tag = 0; + while (_data[_idx] != '=') { + tag = (tag*10) + _data[_idx]-'0'; + _idx++; + } + _idx++; + _in_tag = false; + return tag; + } + + std::pair read_field () + { + bts_assert (!_in_tag); + std::pair result (_idx,0); + while (_data[_idx] != 1) + _idx++; + result.second = _idx - result.first; + _idx++; + _in_tag = true; + return result; + } + + const char* data () const {return _data;} + + void unread_tag () + { + bts_assert (_idx != 0); + _idx = _previous_tag; + _in_tag = true; + } + + bool eof () + {return _idx == _size;} + +private: + bool _in_tag; + const char* _data; + size_t _idx; + size_t _size; + size_t _previous_tag; +}; + +}} diff --git a/src/server/sparkd/jetfix/codec/WriterCursor.h b/src/server/sparkd/jetfix/codec/WriterCursor.h new file mode 100644 index 000000000..edd982901 --- /dev/null +++ b/src/server/sparkd/jetfix/codec/WriterCursor.h @@ -0,0 +1,163 @@ +#pragma once + +#include +#include "lib/utils/bts_assert.h" +#include "server/sparkd/jetfix/codec/i32_toa.h" +#include "server/sparkd/jetfix/codec/f_toa.h" + +namespace bts { +namespace jetfix { + +template +class WriterCursorT +{ +public: + typedef std::pair Loc; + + WriterCursorT (const WriterCursorT &) = delete; + WriterCursorT (std::string &data, uint32_t cursor=0) : _data (&data),_cursor (cursor) {} + ~WriterCursorT () {reset ();} + WriterCursorT& operator=(const WriterCursorT &) = delete; + + void reassign (std::string &data, uint32_t cursor) + { + _data = &data; + _cursor = cursor; + } + + void reset () + { + if (_cursor > 0) { + _data->resize (_cursor); + _cursor = 0; + } + } + + void tidy () + { + if (_cursor > 0) { + _data->resize (_cursor); + } + } + + void need (size_t n) const + { + if (_cursor + n > _data->size ()) + _data->resize (2*(_cursor + n)); + } + + uint32_t pos () const + {return _cursor;} + + size_t free_bytes () const + { return _data->size () - _cursor;} + + void write_tag (const char *tag, size_t len) + { + write_bytes (tag,len); + *data (_cursor-1) ='='; + } + + void append (WriterCursorT const &src) + { + need (src.pos ()); + memcpy (data (_cursor), + src._data->data (), + src.pos ()); + _cursor+=src.pos (); + } + + template + inline void append (A const&a) + { + need (a.size ()); + memcpy (data (_cursor), + &a[0], + a.size ()); + _cursor += a.size (); + } + + std::pair write_bytes (const char *from, size_t count) + { + bts_assert_d (free_bytes () >= count + 1); + memcpy (data (_cursor),from,count); + Loc result (_cursor,count); + _cursor += count; + *data (_cursor)=1; + _cursor++; + return result; + } + + void overwrite_bytes (Loc l, char const*from, size_t count) + { + bts_assert (l.second == count); + bts_assert (l.first + count < _cursor); + memcpy (data (l.first),from,count); + } + + std::pair write_float (float f, int pad) + { + Loc loc; + loc.first = _cursor; + auto written = f_toa (f,data (_cursor),_data->size ()-_cursor,pad); + _cursor += written; + bts_assert (free_bytes ()>0); + *data (_cursor) = 1; + _cursor++; + loc.second = _cursor - loc.first - 1; + return loc; + } + + void overwrite_float (Loc l,float f, int pad) + { + bts_assert (l.first + l.second < _cursor); + auto written = f_toa (f,data (l.first),l.second+1,pad); + *data (l.first+l.second) = 1; + bts_assert (written == (int) l.second); + } + + std::pair write_byte (char byte) + { + bts_assert (_cursor < _data->size ()); + *data (_cursor++)=byte; + *data (_cursor++)=1; + return Loc (_cursor-2,1); + } + + void overwrite_byte (Loc l,char byte) + { + bts_assert (l.first < _cursor); + *data (l.first)=byte; + } + + std::pair write_int (int32_t i,int pad) + { + Loc loc; + loc.first = _cursor; + auto written = i32_toa (i,data (_cursor),pad); + _cursor += written; + bts_assert (free_bytes ()>0); + *data (_cursor)=1; + _cursor++; + loc.second = _cursor - loc.first - 1; + return loc; + } + + void overwrite_int (Loc l,int32_t i,int pad) + { + bts_assert (l.first + l.second < _cursor); + auto written = i32_toa (i,data (l.first),pad); + *data (l.first+l.second) = 1; + bts_assert (written == (int) l.second); + } + +private: + inline char *data (size_t i) + {return ((char*) (_data->data ()) + i);} + + Storage *_data; + uint32_t _cursor; +}; + +typedef WriterCursorT WriterCursor; +}} diff --git a/src/server/sparkd/jetfix/codec/f_toa.h b/src/server/sparkd/jetfix/codec/f_toa.h new file mode 100644 index 000000000..e5d2a77e9 --- /dev/null +++ b/src/server/sparkd/jetfix/codec/f_toa.h @@ -0,0 +1,42 @@ +#pragma once + +#include "lib/utils/bts_assert.h" + +#include +#include + +inline int f_toa (float n,char*buffer, size_t buffer_size,int pad=0) +{ + static double_conversion::DoubleToStringConverter const &c = + double_conversion::DoubleToStringConverter::EcmaScriptConverter (); + + if (pad == 0) { + double_conversion::StringBuilder builder(buffer,buffer_size); + bts_assert (c.ToShortestSingle (n,&builder)); + return builder.position (); + } else { + bts_assert ((int) buffer_size >= pad); + + char temp_buffer[100]; + double_conversion::StringBuilder builder(temp_buffer,100); + bts_assert (c.ToShortestSingle (n,&builder)); + auto written = builder.position (); + + if (n < 0) { + *buffer++='-'; + auto n_zeros = pad - written; + while (n_zeros--) { + *buffer++='0'; + } + memcpy (buffer,temp_buffer+1,written-1); + } else { + auto n_zeros = pad - written; + while (n_zeros--) { + *buffer++='0'; + } + memcpy (buffer,temp_buffer,written); + } + return pad; + } +} + diff --git a/src/server/sparkd/jetfix/codec/i32_toa.h b/src/server/sparkd/jetfix/codec/i32_toa.h new file mode 100644 index 000000000..c6df0fbd0 --- /dev/null +++ b/src/server/sparkd/jetfix/codec/i32_toa.h @@ -0,0 +1,218 @@ +// SSE2 implementation according to http://0x80.pl/articles/sse-itoa.html +// Modifications: (1) fix incorrect digits (2) accept all ranges (3) write to user provided buffer. + +#pragma once + +#include +#include +#include + +#ifdef _MSC_VER +#include "intrin.h" +#endif + +#ifdef _MSC_VERss +#define ALIGN_PRE __declspec(align(16)) +#define ALIGN_SUF +#else +#define ALIGN_PRE +#define ALIGN_SUF __attribute__ ((aligned(16))) +#endif + +const char gDigitsLut[200] = { + '0','0','0','1','0','2','0','3','0','4','0','5','0','6','0','7','0','8','0','9', + '1','0','1','1','1','2','1','3','1','4','1','5','1','6','1','7','1','8','1','9', + '2','0','2','1','2','2','2','3','2','4','2','5','2','6','2','7','2','8','2','9', + '3','0','3','1','3','2','3','3','3','4','3','5','3','6','3','7','3','8','3','9', + '4','0','4','1','4','2','4','3','4','4','4','5','4','6','4','7','4','8','4','9', + '5','0','5','1','5','2','5','3','5','4','5','5','5','6','5','7','5','8','5','9', + '6','0','6','1','6','2','6','3','6','4','6','5','6','6','6','7','6','8','6','9', + '7','0','7','1','7','2','7','3','7','4','7','5','7','6','7','7','7','8','7','9', + '8','0','8','1','8','2','8','3','8','4','8','5','8','6','8','7','8','8','8','9', + '9','0','9','1','9','2','9','3','9','4','9','5','9','6','9','7','9','8','9','9' +}; + +static const uint32_t kDiv10000 = 0xd1b71759; +ALIGN_PRE static const uint32_t kDiv10000Vector[4] ALIGN_SUF = { kDiv10000, kDiv10000, kDiv10000, kDiv10000 }; +ALIGN_PRE static const uint32_t k10000Vector[4] ALIGN_SUF = { 10000, 10000, 10000, 10000 }; +ALIGN_PRE static const uint16_t kDivPowersVector[8] ALIGN_SUF = { 8389, 5243, 13108, 32768, 8389, 5243, 13108, 32768 }; // 10^3, 10^2, 10^1, 10^0 +ALIGN_PRE static const uint16_t kShiftPowersVector[8] ALIGN_SUF = { + 1 << (16 - (23 + 2 - 16)), + 1 << (16 - (19 + 2 - 16)), + 1 << (16 - 1 - 2), + 1 << (15), + 1 << (16 - (23 + 2 - 16)), + 1 << (16 - (19 + 2 - 16)), + 1 << (16 - 1 - 2), + 1 << (15) +}; +ALIGN_PRE static const uint16_t k10Vector[8] ALIGN_SUF = { 10, 10, 10, 10, 10, 10, 10, 10 }; +ALIGN_PRE static const char kAsciiZero[16] ALIGN_SUF = { '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0' }; + +inline __m128i Convert8DigitsSSE2(uint32_t value) { + assert(value <= 99999999); + + // abcd, efgh = abcdefgh divmod 10000 + const __m128i abcdefgh = _mm_cvtsi32_si128(value); + const __m128i abcd = _mm_srli_epi64(_mm_mul_epu32(abcdefgh, reinterpret_cast(kDiv10000Vector)[0]), 45); + const __m128i efgh = _mm_sub_epi32(abcdefgh, _mm_mul_epu32(abcd, reinterpret_cast(k10000Vector)[0])); + + // v1 = [ abcd, efgh, 0, 0, 0, 0, 0, 0 ] + const __m128i v1 = _mm_unpacklo_epi16(abcd, efgh); + + // v1a = v1 * 4 = [ abcd * 4, efgh * 4, 0, 0, 0, 0, 0, 0 ] + const __m128i v1a = _mm_slli_epi64(v1, 2); + + // v2 = [ abcd * 4, abcd * 4, abcd * 4, abcd * 4, efgh * 4, efgh * 4, efgh * 4, efgh * 4 ] + const __m128i v2a = _mm_unpacklo_epi16(v1a, v1a); + const __m128i v2 = _mm_unpacklo_epi32(v2a, v2a); + + // v4 = v2 div 10^3, 10^2, 10^1, 10^0 = [ a, ab, abc, abcd, e, ef, efg, efgh ] + const __m128i v3 = _mm_mulhi_epu16(v2, reinterpret_cast(kDivPowersVector)[0]); + const __m128i v4 = _mm_mulhi_epu16(v3, reinterpret_cast(kShiftPowersVector)[0]); + + // v5 = v4 * 10 = [ a0, ab0, abc0, abcd0, e0, ef0, efg0, efgh0 ] + const __m128i v5 = _mm_mullo_epi16(v4, reinterpret_cast(k10Vector)[0]); + + // v6 = v5 << 16 = [ 0, a0, ab0, abc0, 0, e0, ef0, efg0 ] + const __m128i v6 = _mm_slli_epi64(v5, 16); + + // v7 = v4 - v6 = { a, b, c, d, e, f, g, h } + const __m128i v7 = _mm_sub_epi16(v4, v6); + + return v7; +} + +inline __m128i ShiftDigits_SSE2(__m128i a, unsigned digit) { + assert(digit <= 8); + switch (digit) { + case 0: return a; + case 1: return _mm_srli_si128(a, 1); + case 2: return _mm_srli_si128(a, 2); + case 3: return _mm_srli_si128(a, 3); + case 4: return _mm_srli_si128(a, 4); + case 5: return _mm_srli_si128(a, 5); + case 6: return _mm_srli_si128(a, 6); + case 7: return _mm_srli_si128(a, 7); + case 8: return _mm_srli_si128(a, 8); + } + return a; // should not execute here. +} + +inline int u32_toa(uint32_t value, char* buffer) { + char *start = buffer; + if (value < 10000) { + const uint32_t d1 = (value / 100) << 1; + const uint32_t d2 = (value % 100) << 1; + + if (value >= 1000) + *buffer++ = gDigitsLut[d1]; + if (value >= 100) + *buffer++ = gDigitsLut[d1 + 1]; + if (value >= 10) + *buffer++ = gDigitsLut[d2]; + *buffer++ = gDigitsLut[d2 + 1]; + *buffer++ = '\0'; + } + else if (value < 100000000) { + // value = bbbbcccc + const uint32_t b = value / 10000; + const uint32_t c = value % 10000; + + const uint32_t d1 = (b / 100) << 1; + const uint32_t d2 = (b % 100) << 1; + + const uint32_t d3 = (c / 100) << 1; + const uint32_t d4 = (c % 100) << 1; + + if (value >= 10000000) + *buffer++ = gDigitsLut[d1]; + if (value >= 1000000) + *buffer++ = gDigitsLut[d1 + 1]; + if (value >= 100000) + *buffer++ = gDigitsLut[d2]; + *buffer++ = gDigitsLut[d2 + 1]; + + *buffer++ = gDigitsLut[d3]; + *buffer++ = gDigitsLut[d3 + 1]; + *buffer++ = gDigitsLut[d4]; + *buffer++ = gDigitsLut[d4 + 1]; + *buffer++ = '\0'; + } + else { + // value = aabbbbbbbb in decimal + + const uint32_t a = value / 100000000; // 1 to 42 + value %= 100000000; + + if (a >= 10) { + const unsigned i = a << 1; + *buffer++ = gDigitsLut[i]; + *buffer++ = gDigitsLut[i + 1]; + } + else + *buffer++ = '0' + static_cast(a); + + const __m128i b = Convert8DigitsSSE2(value); + const __m128i ba = _mm_add_epi8(_mm_packus_epi16(_mm_setzero_si128(), b), reinterpret_cast(kAsciiZero)[0]); + const __m128i result = _mm_srli_si128(ba, 8); + _mm_storel_epi64(reinterpret_cast<__m128i*>(buffer), result); + buffer[8] = '\0'; + return buffer+8-start; + } + return buffer-start-1; +} + +inline int count_digits (int32_t value) +{ + static auto P01 = 10u; + static auto P02 = P01*10; + static auto P03 = P02*10; + static auto P04 = P03*10; + static auto P05 = P04*10; + static auto P06 = P05*10; + static auto P07 = P06*10; + static auto P08 = P07*10; + static auto P09 = P08*10; + + uint32_t u = static_cast(value); + if (value < 0) + u = ~u + 1; + + if (u < P01) return 1; + if (u < P02) return 2; + if (u < P03) return 3; + if (u < P04) return 4; + if (u < P05) return 5; + if (u < P06) return 6; + if (u < P07) return 7; + if (u < P08) return 8; + if (u < P09) return 9; + return 10; +} + +inline int i32_toa(int32_t value, char* buffer, int pad) { + auto start = buffer; + uint32_t u = static_cast(value); + if (value < 0) { + *buffer++ = '-'; + u = ~u + 1; + } + if (pad) { + auto digits = count_digits (value); + auto offset = pad - digits; + if (offset > 0) { + auto start = buffer + offset; + while (buffer != start) + { + *buffer='0'; + buffer++; + } + } + else if (offset < 0) + return 0; + } + return (buffer-start) + u32_toa(u, buffer); +} + + diff --git a/src/server/sparkd/jetfix/codec/sum_bytes.h b/src/server/sparkd/jetfix/codec/sum_bytes.h new file mode 100644 index 000000000..4b4947c59 --- /dev/null +++ b/src/server/sparkd/jetfix/codec/sum_bytes.h @@ -0,0 +1,16 @@ +#pragma once + +#include + +namespace bts { +namespace jetfix { + +inline uint32_t sum_bytes (const char*in_buffer,uint32_t buffer_sz) +{ + uint32_t result =0; + for (size_t i =0; i< buffer_sz;i++) + result+=in_buffer[i]; + return result; +} + +}} diff --git a/src/server/sparkd/jetfix/common/InMsgBuffer.h b/src/server/sparkd/jetfix/common/InMsgBuffer.h new file mode 100644 index 000000000..faa0746b4 --- /dev/null +++ b/src/server/sparkd/jetfix/common/InMsgBuffer.h @@ -0,0 +1,172 @@ +#pragma once + +#include +#include +#include +#include +#include "lib/utils/bts_assert.h" +#include "server/sparkd/jetfix/codec/MessageTypes.h" + +namespace bts { +namespace jetfix { + +class InMsgBuffer +{ +public: + InMsgBuffer () : _data (),_size (),_type (),_buffer () {} + InMsgBuffer (InMsgBuffer const&src) : _data (src._data), + _size (src._size), + _type (src._type), + _buffer (src._buffer) + {} + + InMsgBuffer (std::string const &data) : _data (),_size (),_type (),_buffer () + { + _buffer = std::make_shared (data); + _data = _buffer->data (); + _size = _buffer->size (); + _type = get_type (); + } + + InMsgBuffer (const char*data, size_t sz, std::shared_ptr const &buffer) + : _data (data), + _size (sz), + _type (get_type ()), + _buffer (buffer) + {} + + InMsgBuffer &operator=(InMsgBuffer const&msg) + { + _data = msg._data; + _size = msg._size; + _type = msg._type; + _buffer = msg._buffer; + return *this; + } + + const char*data () const + {return _data;} + + size_t size () const + {return _size;} + + std::string to_str () const + { + return std::string (_data,_size); + } + + MessageTypes type () const + {return _type;} + + uint32_t get_seq_num () const + {return get_uint_field ("\00134=",sizeof("\00134=")-1);} + + uint32_t get_last_seq_num_processed () const + {return get_uint_field ("\001369=",sizeof("\001369=")-1);} + + uint32_t get_new_seq_no () const + {return get_uint_field ("\00136=",sizeof("\00136=")-1);} + + uint32_t get_begin_seq_no () const + {return get_uint_field ("\0017=",sizeof("\0017=")-1);} + + uint32_t get_end_seq_no () const + {return get_uint_field ("\00116=",sizeof("\00116=")-1);} + + bool get_poss_dup_flag () const + {return get_uint_field ("\00143=",sizeof("\00143=")-1)=='Y';} + bool has_poss_dup_flag_flag () const + { + auto loc = + find_field ("\00143=",sizeof("\00143=")-1); + return loc.first != nullptr; + } + + bool has_test_req_id () const + { + auto loc = + find_field ("\001112=",sizeof("\001112=")-1); + return loc.first != nullptr; + } + + bool has_text () const + { + auto loc = + find_field ("\00158=",sizeof("\00158=")-1); + return loc.first != nullptr; + } + + std::string get_text () const + { + if (has_text ()) + return get_string_field ("\00158=",sizeof("\00158=")-1); + else + return ""; + } + + std::string get_test_req_id () const + {return get_string_field ("\001112=",sizeof("\001112=")-1);} + + +private: + const char *_data; + size_t _size; + MessageTypes _type; + std::shared_ptr _buffer; + + MessageTypes get_type () const + { + auto loc = find_field ("\00135=",sizeof("\00135=")-1); + bts_assert (loc.first); + uint32_t result = 0; + for (auto i = 0ul; i < loc.second; i++) { + result *= 256; + result += *(loc.first+i); + } + return (MessageTypes) result; + } + + uint32_t get_uint_field (const char *field, size_t field_sz) const + { + auto loc = find_field (field,field_sz); + bts_assert (loc.first); + uint32_t result = 0; + for (auto i = 0ul; i < loc.second; i++) { + result *= 10; + result += *(loc.first+i) - '0'; + } + return result; + } + + std::string get_string_field (const char *field, size_t field_sz) const + { + auto loc = find_field (field,field_sz); + bts_assert (loc.first); + return std::string (loc.first,loc.second); + } + + std::pair find_field (const char*fn,size_t field_sz) const + { + auto begin = data (); + auto end = begin + size (); + + while (begin != end) { + auto it = std::find (begin,end,*fn); + if (it == end || (end - it) < (int) field_sz) + break; + else if (memcmp (it,fn,field_sz) == 0) { + // found it + it += field_sz; + auto field_start = it; + auto field_stop = std::find (field_start,end,1); + bts_assert (field_stop != end); + return std::make_pair (field_start, field_stop - field_start); + } + else + begin++; + } + return std::make_pair (nullptr,0); + } +}; + +}} diff --git a/src/server/sparkd/jetfix/common/JetFIXError.h b/src/server/sparkd/jetfix/common/JetFIXError.h new file mode 100644 index 000000000..dc955a163 --- /dev/null +++ b/src/server/sparkd/jetfix/common/JetFIXError.h @@ -0,0 +1,98 @@ +#pragma once + +#include + +namespace bts { +namespace jetfix { + +class JetFIXError +{ +public: + enum Code { + ERROR_NONE, + LOGIN_TIMEOUT, + HB_TIMEOUT, + LOGOUT_TIMEOUT, + CONNECTION_ERROR, + INCOMING_SN_TOO_SMALL, + OUTGOING_SN_TOO_SMALL, + UNEXPECTED_LOGON, + LOGGED_OUT_BY_EXCHANGE, + EXPECTED_LOGON, + UNEXPECTED_RESEND_REQUEST, + UNEXPECTED_SEQ_RESET, + USER_DISCONNECTED + }; + + JetFIXError () : + _error_code (JetFIXError::ERROR_NONE), + _msg () + {} + + JetFIXError (const JetFIXError &src) : + _error_code (src._error_code), + _msg (src._msg) + {} + + JetFIXError (JetFIXError::Code error_code, + std::string const&msg) : + _error_code (error_code), + _msg (msg) + {} + + JetFIXError &operator=(const JetFIXError &rhs) + { + _error_code = rhs._error_code; + _msg = rhs._msg; + return *this; + } + + Code error () const + {return _error_code;} + + std::string const &msg () const + {return _msg;} + + static const char* code_msg (JetFIXError::Code error_code) + { + switch (error_code) + { + case ERROR_NONE: + return "No error"; + case LOGIN_TIMEOUT: + return "Timed out while waiting for logon reply"; + case HB_TIMEOUT: + return "Timed out while waiting for exchange heartbeats"; + case LOGOUT_TIMEOUT: + return "Timed out while waiting for logout to complete"; + case CONNECTION_ERROR: + return "Network error"; + case INCOMING_SN_TOO_SMALL: + return "Incoming sequence number too small"; + case OUTGOING_SN_TOO_SMALL: + return "Outgoing sequence number too small"; + case UNEXPECTED_LOGON: + return "Received unexpected logon message"; + case LOGGED_OUT_BY_EXCHANGE: + return "Logged out by the exchange"; + case EXPECTED_LOGON: + return "Expected a logon reply"; + case UNEXPECTED_RESEND_REQUEST: + return "Received unexpected resend request"; + case UNEXPECTED_SEQ_RESET: + return "Received unexpected sequence reset"; + case USER_DISCONNECTED: + return "Disconnected by user request"; + default: + bts_assert (false); + } + } + + +private: + Code _error_code; + std::string _msg; +}; + + +}} diff --git a/src/server/sparkd/jetfix/common/OutFIX.cpp b/src/server/sparkd/jetfix/common/OutFIX.cpp new file mode 100644 index 000000000..3bf96b68e --- /dev/null +++ b/src/server/sparkd/jetfix/common/OutFIX.cpp @@ -0,0 +1,8 @@ +#ifndef WIN32 +#include "OutFIX.h" + +namespace bts { +namespace bluestream { + +}} +#endif diff --git a/src/server/sparkd/jetfix/common/OutFIX.h b/src/server/sparkd/jetfix/common/OutFIX.h new file mode 100644 index 000000000..32e0e940a --- /dev/null +++ b/src/server/sparkd/jetfix/common/OutFIX.h @@ -0,0 +1,69 @@ +#pragma once + +#include +#include "lib/utils/TimeStamp.h" +#include "server/sparkd/jetfix/codec/FIXEncodingStream.h" + +#include "lib/config/protos/jetfix.pb.h" + +// fwds +namespace bts { +namespace jetfix { +class FIXEncodingStream; +}} + +namespace bts { +namespace jetfix { + +class OutFIX +{ +public: + OutFIX(config::JetFIX::Connection const &cfg, + std::string const&sender_sub_id, + size_t n_streams = 1) + : timestamps (), + fix () + { + init(cfg,sender_sub_id,n_streams); + } + + OutFIX(config::JetFIX::Connection const &cfg, + size_t n_streams = 1) + : timestamps (), + fix () + { + init(cfg,cfg.sender_subid(),n_streams); + } + + enum TimePoint { + MD_WRITE_TS, + JF_ARRIVE, + CONN_WRITE_TS, + PRE_STAMP, + POST_STAMP, + POST_STORE, + POST_SEND, + LAST + }; + + utils::TimeStamp get_ts (TimePoint tp) + {return timestamps[(int)tp];} + + void set_ts (TimePoint ts, utils::TimeStamp t) + {timestamps[(int)ts] = t;} + + std::array timestamps; + std::vector fix; +private: + void init(config::JetFIX::Connection const &cfg, + std::string const&sender_sub_id, + size_t n_streams) + { + fix.reserve (n_streams); + for (size_t i = 0; i < n_streams;i++) + fix.emplace_back (cfg,sender_sub_id); + } +}; + +}} + diff --git a/src/server/sparkd/jetfix/common/OutFIXPtr.h b/src/server/sparkd/jetfix/common/OutFIXPtr.h new file mode 100644 index 000000000..621621a68 --- /dev/null +++ b/src/server/sparkd/jetfix/common/OutFIXPtr.h @@ -0,0 +1,9 @@ +#pragma once + +#include + +namespace bts { +namespace jetfix { +class OutFIX; +typedef std::shared_ptr OutFIXPtr; +}} diff --git a/src/server/sparkd/jetfix/common/OutMsgVecPtr.h b/src/server/sparkd/jetfix/common/OutMsgVecPtr.h new file mode 100644 index 000000000..41355deec --- /dev/null +++ b/src/server/sparkd/jetfix/common/OutMsgVecPtr.h @@ -0,0 +1,13 @@ +#pragma once + +#include +#include +namespace bts { +namespace jetfix { +class FIXEncodingStream; +}} + +namespace bts { +namespace jetfix { +typedef std::shared_ptr> OutMsgVecPtr; +}} diff --git a/src/server/sparkd/jetfix/connection/AsyncReader.cpp b/src/server/sparkd/jetfix/connection/AsyncReader.cpp new file mode 100644 index 000000000..ccebb1103 --- /dev/null +++ b/src/server/sparkd/jetfix/connection/AsyncReader.cpp @@ -0,0 +1,162 @@ +#ifndef WIN32 +#include "AsyncReader.h" + +#include +#include "IConnection.h" +#include "server/sparkd/jetfix/gateway/ICoreGatewayEventSink.h" +#include "lib/utils/bts_assert.h" + +#include + +namespace bts { +namespace jetfix { + +size_t extract (char *start, char *end, + std::shared_ptr const &buffer, + std::vector *msgs) +{ + auto first_unprocessed_byte = 0; + auto current = start; + + for (;;) { + // consume prefix + auto this_msg_begin= current; + constexpr const char *prefix = "8=FIX.4.2\0019="; + for (int i = 0; i < 12;i++) + if (current == end) + return first_unprocessed_byte; + else if (*current != prefix[i]) + return first_unprocessed_byte; + else + current++; + + // consume body_length; + auto body_length = 0; + for (;;) { + if (current == end) + return first_unprocessed_byte; + else if (*current == 1) + { + current ++; + break; + } else { + body_length *= 10; + body_length += *current - '0'; + current ++; + } + } + bts_assert (body_length); + + // consume body + current += body_length + sizeof("10=000"); + if ( current > end) + return first_unprocessed_byte; + else { + msgs->emplace_back (this_msg_begin, + current - this_msg_begin, + buffer); + first_unprocessed_byte = current - start; + } + } +} + +struct AsyncReader::PImpl +{ + PImpl(IConnection &connection, + ICoreGatewayEventSink &sink, + size_t buffer_sz, + size_t high_water_mark) + : + connection (connection), + working_buffer (std::make_shared (buffer_sz,0)), + first_pending_msg (0), + write_cursor (0), + reader_thread (), + sink (sink), + high_water_mark (high_water_mark) + {} + + IConnection &connection; + std::shared_ptr working_buffer; + size_t first_pending_msg; + size_t write_cursor; + std::thread reader_thread; + ICoreGatewayEventSink &sink; + uint32_t high_water_mark; +}; + +AsyncReader::AsyncReader (IConnection &connection, + ICoreGatewayEventSink &sink, + size_t buffer_sz, size_t high_water_mark): + me (new PImpl(connection,sink,buffer_sz,high_water_mark)) +{} + +void AsyncReader::start () +{ + me->reader_thread = std::thread ([this] () {dispatch();}); +} + +char* AsyncReader::cursor () +{ + return &((*me->working_buffer)[me->write_cursor]); +} + +size_t AsyncReader::free_bytes () +{ + return me->working_buffer->size () - me->write_cursor; +} + +void AsyncReader::dispatch () +{ + std::vector framed_msgs; + framed_msgs.reserve (100); + + for (;;) { + auto read_count = me->connection.read (cursor (),free_bytes ()); + + if (read_count <= 0) { + me->sink.reader_disconnected (); + return; + } else { + me->write_cursor += read_count; + + framed_msgs.clear (); + + auto first_unprocessed_offset = extract (&((*me->working_buffer)[me->first_pending_msg]), + &((*me->working_buffer)[me->write_cursor]), + me->working_buffer, + &framed_msgs); + me->first_pending_msg += first_unprocessed_offset; + + + for (auto &msg : framed_msgs) { + me->sink.in_msg_raw (msg); + } + + + // do we need a new buffer? + if (me->write_cursor >= me->working_buffer->size () - me->high_water_mark) { + auto old_buffer = me->working_buffer; + auto pending_size = me->write_cursor - me->first_pending_msg; + + me->working_buffer = std::make_shared (std::max (2*pending_size, + me->working_buffer->size ()),0); + memcpy (&((*me->working_buffer)[0]), + &((*old_buffer)[me->first_pending_msg]), + pending_size); + me->first_pending_msg = 0; + me->write_cursor = pending_size; + } + } + } +} + +AsyncReader::~AsyncReader () +{ + if (me->reader_thread.joinable ()) + me->reader_thread.join (); +} + +}} +#endif + diff --git a/src/server/sparkd/jetfix/connection/AsyncReader.h b/src/server/sparkd/jetfix/connection/AsyncReader.h new file mode 100644 index 000000000..7d4bdc4bb --- /dev/null +++ b/src/server/sparkd/jetfix/connection/AsyncReader.h @@ -0,0 +1,30 @@ +#pragma once + +#include + +namespace bts { +namespace jetfix { + +class IConnection; +class ICoreGatewayEventSink; + +class AsyncReader +{ +public: + AsyncReader(IConnection &connection, + ICoreGatewayEventSink &sink, + size_t buffer_sz, + size_t high_water_mark); + void start (); + ~AsyncReader(); + +private: + friend struct AsyncReaderTests; + char *cursor (); + size_t free_bytes (); + void dispatch (); + struct PImpl; + std::unique_ptr me; +}; + +}} diff --git a/src/server/sparkd/jetfix/connection/IConnection.cpp b/src/server/sparkd/jetfix/connection/IConnection.cpp new file mode 100644 index 000000000..3b16fd124 --- /dev/null +++ b/src/server/sparkd/jetfix/connection/IConnection.cpp @@ -0,0 +1,3 @@ +#ifndef WIN32 +#include "IConnection.h" +#endif diff --git a/src/server/sparkd/jetfix/connection/IConnection.h b/src/server/sparkd/jetfix/connection/IConnection.h new file mode 100644 index 000000000..bfc028f83 --- /dev/null +++ b/src/server/sparkd/jetfix/connection/IConnection.h @@ -0,0 +1,20 @@ +#pragma once + +#include + +namespace bts { +namespace jetfix { + +class IConnection +{ +public: + typedef std::array,64> WriteVec; + static auto constexpr MaxWriteVecSz = std::tuple_size::value; + virtual bool connect () = 0; + virtual void shutdown () = 0; + virtual int read (char* to_buffer,size_t read_count) = 0; + virtual int write (WriteVec const&vec,size_t vec_size,size_t correlation_id) = 0; + virtual ~IConnection() {}; +}; + +}} diff --git a/src/server/sparkd/jetfix/connection/IConnectionObserver.h b/src/server/sparkd/jetfix/connection/IConnectionObserver.h new file mode 100644 index 000000000..0179d02e7 --- /dev/null +++ b/src/server/sparkd/jetfix/connection/IConnectionObserver.h @@ -0,0 +1,22 @@ +#pragma once + +#include + +namespace bts { +namespace jetfix { +class IncomingMsg; +class RawMsg; +typedef std::shared_ptr> MsgVecPtr; + +class IConnectionObserver +{ +public: + virtual ~IConnectionObserver() {}; + virtual void on_connection_error () = 0; + virtual void on_connection_sync_ack () = 0; + virtual void on_connection_close_ack () = 0; + virtual void on_rcvd (IncomingMsg const &msg) = 0; + virtual void on_sent (MsgVecPtr const &msg) = 0; +}; + +}} diff --git a/src/server/sparkd/jetfix/connection/SocketConnection.cpp b/src/server/sparkd/jetfix/connection/SocketConnection.cpp new file mode 100644 index 000000000..b4b00d05e --- /dev/null +++ b/src/server/sparkd/jetfix/connection/SocketConnection.cpp @@ -0,0 +1,127 @@ +#include "SocketConnection.h" +#include "lib/utils/bts_assert.h" +#include "server/sparkd/jetfix/store/session_name.h" +#include "lib/logging/logging.h" + +#include +#include +#include +#include +#include +#include + +namespace bts { +namespace jetfix { + +SocketConnection::SocketConnection (bts::config::JetFIX::Connection const &cfg) + : _fd (-1), _cfg (cfg),_log (cfg.log_fix_stream ()),_in_log (),_out_log () +{ + if (_log) { + _in_log.open (std::string ("/tmp/fix_raw_read_log_") + session_name (cfg),std::ios_base::app); + _out_log.open (std::string ("/tmp/fix_raw_write_log_") + session_name (cfg),std::ios_base::app); + } +} + +SocketConnection::~SocketConnection () { + if (_fd > 0) + close (_fd); +} + +bool SocketConnection::connect () +{ + bts_assert (_fd == -1); + _fd = socket(AF_INET, SOCK_STREAM, 0); + if (_fd < 0) { + auto er = errno; + char msg [256]; + LOG (Jetfix,Error, + "Failed to allocate socket when connecting to {}:{} errno:{} msg: {}", + _cfg.socket_connect_host_ip (), _cfg.socket_connect_port (), + er, strerror_r(er,msg,sizeof(msg))); + _fd = -1; + return false; + } + struct sockaddr_in serv_addr; + memset(&serv_addr, '0', sizeof(serv_addr)); + + serv_addr.sin_family = AF_INET; + serv_addr.sin_port = htons(_cfg.socket_connect_port ()); + auto result = inet_pton(AF_INET, _cfg.socket_connect_host_ip ().c_str (), + &serv_addr.sin_addr); + bts_assert (result >0); + + + result = ::connect (_fd,(struct sockaddr *)&serv_addr, sizeof(serv_addr)); + + if (result < 0) { + LOG (Jetfix,Info, "Failed to connect to {}:{}", + _cfg.socket_connect_host_ip (), + _cfg.socket_connect_port ()); + return false; + } + else { + LOG (Jetfix,Info, "Connected to {}:{}", + _cfg.socket_connect_host_ip (), + _cfg.socket_connect_port ()); + int flag = 1; + result = ::setsockopt (_fd,IPPROTO_TCP, + TCP_NODELAY, + (char*) &flag, + sizeof(int)); + bts_assert (result == 0); + return true; + } +} + +void SocketConnection::shutdown () +{ + bts_assert (_fd >= 0); + ::shutdown (_fd, SHUT_RDWR); +} + +int SocketConnection::read (char* to_buffer,size_t read_count) +{ + + bts_assert (_fd >= 0); + auto result= ::read (_fd,to_buffer,read_count); + if (_log && result>0) { + _in_log.write (to_buffer,result); + _in_log.flush (); + } + if (result < 0) { + auto er = errno; + char msg [256]; + LOG (Jetfix,Warn,"Could not read from jetfix connection. {}", + strerror_r(er,msg,sizeof(msg))); + } + return result; +} + +int SocketConnection::write (WriteVec const&vec,size_t vec_size,size_t ) +{ + bts_assert (_fd>=0); + + struct iovec iv [MaxWriteVecSz]; + for (size_t i = 0; i < vec_size;i++) { + iv[i].iov_base = (void*) vec[i].first; + iv[i].iov_len = vec[i].second; + } + + auto result = ::writev (_fd,iv,(int) vec_size); + if (_log && result >0) { + for (size_t i = 0; i < vec_size;i++) + _out_log.write (vec[i].first, vec[i].second); + _out_log.flush (); + } + if (result < 0) { + auto er = errno; + char msg [256]; + LOG (Jetfix,Warn,"Could not write to jetfix connection. ", + strerror_r(er,msg,sizeof(msg))); + } + return result; +} + +}} + + diff --git a/src/server/sparkd/jetfix/connection/SocketConnection.h b/src/server/sparkd/jetfix/connection/SocketConnection.h new file mode 100644 index 000000000..fc3dadf61 --- /dev/null +++ b/src/server/sparkd/jetfix/connection/SocketConnection.h @@ -0,0 +1,29 @@ +#pragma once + +#include +#include +#include "IConnection.h" +#include "lib/config/protos/jetfix.pb.h" + +namespace bts { +namespace jetfix { + +class SocketConnection : public IConnection +{ +public: + SocketConnection(bts::config::JetFIX::Connection const &cfg); + ~SocketConnection(); + bool connect () override; + void shutdown () override; + int read (char* to_buffer,size_t read_count) override; + int write (WriteVec const&vec,size_t vec_size, size_t correlation_id) override; + +private: + int _fd; + bts::config::JetFIX::Connection const _cfg; + bool _log; + std::ofstream _in_log; + std::ofstream _out_log; +}; + +}} diff --git a/src/server/sparkd/jetfix/gateway/CoreGateway.cpp b/src/server/sparkd/jetfix/gateway/CoreGateway.cpp new file mode 100644 index 000000000..b21e60e39 --- /dev/null +++ b/src/server/sparkd/jetfix/gateway/CoreGateway.cpp @@ -0,0 +1,537 @@ +#include "CoreGateway.h" + +#include "server/sparkd/jetfix/common/OutFIX.h" +#include "server/sparkd/jetfix/connection/AsyncReader.h" +#include "server/sparkd/jetfix/connection/IConnection.h" +#include "server/sparkd/jetfix/gateway/ICoreGatewayObserver.h" +#include "server/sparkd/jetfix/gateway/Session.h" +#include "server/sparkd/jetfix/store/IStore.h" +#include "lib/logging/logging.h" +#include "lib/utils/TimeStamp.h" +#include "lib/utils/datetime.h" +#include "lib/utils/bts_chrono.h" + +#include +#include +#include +#include +#include + +namespace bts { +namespace jetfix { + +enum class CoreGateway::State {OFF,PAUSED,CONNECTING,READY,WREADER_DISCONNECT,WSESSION_TERMINATE}; + +struct CoreGateway::SessionInfo +{ + SessionInfo () : connection (), store (), reader (), session () + {} + + std::unique_ptr connection; + std::unique_ptr store; + std::unique_ptr reader; + std::unique_ptr session; +}; + +struct CoreGateway::EventData +{ + typedef void (CoreGateway::*Handler)(EventData const&ev); + + EventData (const EventData &src) : handler (src.handler), + out_msgs (src.out_msgs), + in_msg (src.in_msg), + ts (src.ts) + {} + + EventData () : handler (), out_msgs (),in_msg (), ts () {} + EventData (Handler handler) : handler (handler), out_msgs (),in_msg (),ts () {} + EventData (Handler handler,OutFIXPtr const &out_msgs) : + handler (handler), + out_msgs (out_msgs), + in_msg (), + ts () + {} + EventData (Handler handler,InMsgBuffer const&in_msg) : handler (handler), out_msgs (), + in_msg (in_msg),ts () {} + + EventData (Handler handler,std::shared_ptr const&ts) + : handler (handler), + out_msgs (),in_msg (),ts (ts) + {} + + EventData& operator=(EventData&src) + { + handler = src.handler; + out_msgs = src.out_msgs; + in_msg = src.in_msg; + ts = src.ts; + return *this; + } + + EventData& operator=(EventData&&src) + { + handler = std::move(src.handler); + out_msgs = std::move(src.out_msgs); + in_msg = std::move(src.in_msg); + ts = std::move(src.ts); + return *this; + } + Handler handler; + OutFIXPtr out_msgs; + InMsgBuffer in_msg; + std::shared_ptr ts; +}; + +struct CoreGateway::PImpl +{ + PImpl(config::JetFIX::Connection const &cfg, + ICoreGatewayObserver &observer, + ConnectionFactory const &conn_factory, + StoreFactory const &store_factory) : + cfg (cfg), + observer (observer), + make_connection (conn_factory), + make_store (store_factory), + state (State::OFF), + start_of_pause (), + session_info (), + event_queue (), + timer_thread (), + destroy (false), + is_on (false), + has_replayed (false), + send_batch (), + recv_batch () + { + send_batch.reserve (1024); + recv_batch.reserve (1024); + } + + const config::JetFIX::Connection cfg; + ICoreGatewayObserver &observer; + ConnectionFactory const make_connection; + StoreFactory const make_store; + + State state; + datetime::PTime start_of_pause; + std::unique_ptr session_info; + moodycamel::ReaderWriterQueue event_queue; + std::thread timer_thread; + std::atomic destroy; + bool is_on; + bool has_replayed; + std::vector send_batch; + std::vector recv_batch; +}; + +CoreGateway::CoreGateway(config::JetFIX::Connection const &cfg, + ICoreGatewayObserver &observer, + ConnectionFactory const &conn_factory, + StoreFactory const &store_factory) + : me (new PImpl (cfg,observer,conn_factory,store_factory)) +{ + me->timer_thread = std::thread (std::bind (&CoreGateway::call_time,this)); +} + +CoreGateway::~CoreGateway () +{ + stop (); + while (me->state != State::OFF) { + dispatch_events (); + std::this_thread::sleep_for(100ms); + } + me->destroy = true; + me->timer_thread.join (); +} + +void CoreGateway::send (OutFIXPtr const &fix) +{ + fix->set_ts (OutFIX::JF_ARRIVE,utils::TimeStamp::now ()); + me->event_queue.enqueue (EventData (&CoreGateway::sync_send,fix)); +} + +void CoreGateway::start () +{me->event_queue.enqueue (EventData (&CoreGateway::sync_start));} + +void CoreGateway::stop () +{me->event_queue.enqueue (EventData (&CoreGateway::sync_stop));} + +void CoreGateway::session_ready () +{me->event_queue.enqueue (EventData (&CoreGateway::sync_session_ready)); } + +void CoreGateway::session_terminated () +{me->event_queue.enqueue (EventData (&CoreGateway::sync_session_terminated));} + +void CoreGateway::reader_disconnected () +{me->event_queue.enqueue (EventData (&CoreGateway::sync_reader_disconnected));} + +void CoreGateway::in_msg_raw (InMsgBuffer const&msg) +{me->event_queue.enqueue (EventData (&CoreGateway::sync_in_msg_raw,msg));} + +void CoreGateway::in_msg_sequenced (InMsgBuffer const&msg) +{me->event_queue.enqueue (EventData (&CoreGateway::sync_in_msg_sequenced,msg));} + +void CoreGateway::dispatch_events () +{ + EventData ev; + utils::TimeStamp cid; + // we prioratize sends over receives + while (me->event_queue.try_dequeue (ev)) { + if (ev.out_msgs) { + cid = ev.out_msgs->get_ts (jetfix::OutFIX::MD_WRITE_TS); + me->send_batch.push_back (ev); + } + else + me->recv_batch.push_back (ev); + } + + if (me->send_batch.size () || me->recv_batch.size ()) { + for (auto &ev : me->send_batch) + (this->*ev.handler) (ev); + for (auto &ev : me->recv_batch) + (this->*ev.handler) (ev); + me->send_batch.clear (); + me->recv_batch.clear (); + } +} + +void CoreGateway::call_time () +{ + while (!me->destroy) { + std::this_thread::sleep_for(500ms); + auto ts = std::make_shared ( + datetime::FIX_timestamp (datetime::now ())); + me->event_queue.enqueue (EventData (&CoreGateway::sync_time,ts)); + } +} + +void CoreGateway::sync_start (EventData const &) +{ + switch (me->state) + { + case State::OFF: + { + me->is_on = true; + pause_or_shutdown (); + break; + } + case State::PAUSED: + case State::CONNECTING: + case State::READY: + bts_assert (false); + break; + case State::WREADER_DISCONNECT: + case State::WSESSION_TERMINATE: + bts_assert (!me->is_on); + me->is_on = true; + break; + default: + bts_assert (false); + } +} + +void CoreGateway::sync_stop (EventData const&) +{ + switch (me->state) + { + case State::OFF: + break; + case State::PAUSED: + me->is_on = false; + transition (State::OFF); + break; + case State::CONNECTING: + case State::READY: + me->is_on = false; + me->session_info->session->shutdown (); + break; + case State::WREADER_DISCONNECT: + case State::WSESSION_TERMINATE: + me->is_on = false; + break; + default: + bts_assert (false); + } +} + +void CoreGateway::sync_send (EventData const &ev) +{ + switch (me->state) + { + case State::OFF: + bts_assert (false); + break; + case State::PAUSED: + case State::CONNECTING: + case State::WREADER_DISCONNECT: + case State::WSESSION_TERMINATE: + break; + case State::READY: + me->session_info->session->send (ev.out_msgs); + break; + default: + bts_assert (false); + } +} + +void CoreGateway::sync_session_ready (EventData const&) +{ + switch (me->state) + { + case State::OFF: + case State::PAUSED: + case State::READY: + case State::WREADER_DISCONNECT: + bts_assert (false); + break; + case State::WSESSION_TERMINATE: + break; + case State::CONNECTING: + transition (State::READY); + me->observer.ready (); + break; + default: + bts_assert (false); + } +} + +void CoreGateway::sync_session_terminated (EventData const&) +{ + switch (me->state) + { + case State::OFF: + case State::PAUSED: + case State::WREADER_DISCONNECT: + bts_assert (false); + break; + + case State::WSESSION_TERMINATE: + pause_or_shutdown (); + break; + + case State::READY: + me->session_info->session.reset (); + me->session_info->connection->shutdown (); + transition (State::WREADER_DISCONNECT); + me->observer.disconnect (); + break; + + case State::CONNECTING: + me->session_info->session.reset (); + me->session_info->connection->shutdown (); + transition (State::WREADER_DISCONNECT); + break; + + default: + bts_assert (false); + } +} + +void CoreGateway::sync_reader_disconnected (EventData const&) +{ + switch (me->state) + { + case State::OFF: + case State::PAUSED: + case State::WSESSION_TERMINATE: + bts_assert (false); + break; + + case State::READY: + me->session_info->session.reset (); + me->observer.disconnect (); + transition (State::WSESSION_TERMINATE); + break; + + case State::CONNECTING: + me->session_info->session.reset (); + transition (State::WSESSION_TERMINATE); + break; + + case State::WREADER_DISCONNECT: + pause_or_shutdown (); + break; + + default: + bts_assert (false); + } +} + + +void CoreGateway::sync_in_msg_raw (EventData const&ev) +{ + switch (me->state) + { + case State::OFF: + case State::PAUSED: + bts_assert (false); + break; + case State::CONNECTING: + case State::READY: + me->session_info->session->receive (ev.in_msg); + break; + case State::WREADER_DISCONNECT: + case State::WSESSION_TERMINATE: + break; + default: + bts_assert (false); + } +} + +void CoreGateway::sync_in_msg_sequenced (EventData const&ev) +{ + switch (me->state) + { + case State::OFF: + case State::PAUSED: + case State::WREADER_DISCONNECT: + bts_assert (false); + break; + case State::CONNECTING: + case State::READY: + me->observer.msg (ev.in_msg); + break; + case State::WSESSION_TERMINATE: + break; + default: + bts_assert (false); + } +} + +void CoreGateway::sync_time (EventData const &ev) +{ + switch (me->state) + { + case State::OFF: + break; + case State::PAUSED: + if (datetime::is_at_least_secs_old (me->start_of_pause, + me->cfg.seconds_to_pause ())) { + LOG (Jetfix,Info,"Pause period ended for jetfix connection {}", me->cfg.name ()); + me->session_info = std::make_unique(); + + me->session_info->store.reset (me->make_store (me->cfg)); + + if (me->session_info->store->session_active ()) { + if (!me->has_replayed) { + me->observer.replay (me->session_info->store->replay ()); + me->has_replayed = true; + } + + me->session_info->connection.reset (me->make_connection (me->cfg)); + LOG (Jetfix,Warn, "Trying to connect to exchange for {}",me->cfg.name ()); + if (me->session_info->connection->connect ()) { + LOG (Jetfix,Warn,"Connected to exchange gateway for {}",me->cfg.name ()); + + me->session_info->session.reset (new Session (me->cfg, + *me->session_info->connection, + *me->session_info->store, + *this, + [] () {return utils::TimeStamp::now ();})); + me->session_info->reader.reset (new AsyncReader (*me->session_info->connection, + *this, + me->cfg.msg_buffer_sz (), + me->cfg.msg_high_water_mark ())); + me->session_info->reader->start (); + me->session_info->session->start (); + transition (State::CONNECTING); + } + else { + me->session_info.reset (); + me->start_of_pause = datetime::now (); + LOG (Jetfix,Warn,"Failed to connect to exchange for {}", me->cfg.name ()); + } + } else { + me->session_info.reset (); + me->start_of_pause = datetime::now (); + } + } + break; + case State::CONNECTING: + case State::READY: + me->session_info->session->time (ev.ts); + break; + case State::WREADER_DISCONNECT: + case State::WSESSION_TERMINATE: + break; + default: + bts_assert (false); + } +} + +void CoreGateway::pause_or_shutdown () +{ + me->session_info.reset (); + + if (me->is_on) { + me->start_of_pause = datetime::now (); + transition (State::PAUSED); + } else + transition (State::OFF); +} + +void CoreGateway::transition (State new_state) +{ + LOG (Jetfix,Warn,"Transitioning Jetfix gateway {} from {} to {}", + me->cfg.name (), + state_name (me->state), + state_name (new_state)); + me->state = new_state; + check_state_invariants (); +} + +void CoreGateway::check_state_invariants () +{ + switch (me->state) + { + case State::OFF: + bts_assert (!me->session_info); + bts_assert (!me->is_on); + return; + case State::PAUSED: + bts_assert (!me->session_info); + bts_assert (me->is_on); + return; + case State::CONNECTING: + case State::READY: + bts_assert (me->session_info); + bts_assert (me->session_info->session); + bts_assert (me->is_on); + return; + case State::WREADER_DISCONNECT: + bts_assert (me->session_info); + bts_assert (!me->session_info->session); + return; + case State::WSESSION_TERMINATE: + bts_assert (me->session_info); + bts_assert (!me->session_info->session); + break; + default: + bts_assert (false); + return; + } +} + +const char* CoreGateway::state_name (State st) +{ + switch (st) + { + case State::OFF: + return "OFF"; + case State::PAUSED: + return "PAUSED"; + case State::CONNECTING: + return "CONNECTING"; + case State::READY: + return "READY"; + case State::WREADER_DISCONNECT: + return "WREADER_DISCONNECT"; + case State::WSESSION_TERMINATE: + return "WSESSION_TERMINATE"; + default: + bts_assert (false); + } +} + +}} + diff --git a/src/server/sparkd/jetfix/gateway/CoreGateway.h b/src/server/sparkd/jetfix/gateway/CoreGateway.h new file mode 100644 index 000000000..83c28e7b9 --- /dev/null +++ b/src/server/sparkd/jetfix/gateway/CoreGateway.h @@ -0,0 +1,67 @@ +#pragma once + +#include +#include + +#include "server/sparkd/jetfix/gateway/ICoreGatewayEventSink.h" +#include "server/sparkd/jetfix/common/OutFIXPtr.h" + +#include "lib/config/protos/jetfix.pb.h" + +namespace bts { +namespace jetfix { +class ICoreGatewayObserver; +class IConnection; +class IStore; +class OutMsgsBuffer; + +class CoreGateway : ICoreGatewayEventSink +{ +public: + typedef std::function ConnectionFactory; + typedef std::function StoreFactory; + + CoreGateway(config::JetFIX::Connection const &cfg, + ICoreGatewayObserver &observer, + ConnectionFactory const &conn_factory, + StoreFactory const &store_factory); + + ~CoreGateway(); + void start (); + void stop (); + void send (OutFIXPtr const &msgs); + void dispatch_events (); + +private: + enum class State; + struct SessionInfo; + struct PImpl; + struct EventData; + + void session_ready () override; + void session_terminated () override; + void reader_disconnected () override; + void in_msg_raw (InMsgBuffer const &msg) override; + void in_msg_sequenced (InMsgBuffer const &msg) override; + + void call_time (); + + void sync_start (EventData const &); + void sync_stop (EventData const &); + void sync_send (EventData const &); + void sync_session_ready (EventData const&) ; + void sync_session_terminated (EventData const&); + void sync_reader_disconnected (EventData const&) ; + void sync_in_msg_raw (EventData const&) ; + void sync_in_msg_sequenced (EventData const&) ; + void sync_time (EventData const&); + + void pause_or_shutdown (); + void transition (State new_state); + void check_state_invariants (); + const char* state_name (State st); + + std::unique_ptr me; +}; + +}} diff --git a/src/server/sparkd/jetfix/gateway/ICoreGatewayEventSink.h b/src/server/sparkd/jetfix/gateway/ICoreGatewayEventSink.h new file mode 100644 index 000000000..899a10679 --- /dev/null +++ b/src/server/sparkd/jetfix/gateway/ICoreGatewayEventSink.h @@ -0,0 +1,21 @@ +#pragma once + +#include +#include "server/sparkd/jetfix/common/InMsgBuffer.h" + +namespace bts { +namespace jetfix { +class JetFIXError; + +class ICoreGatewayEventSink +{ +public: + virtual void session_ready () = 0; + virtual void session_terminated () = 0; + virtual void reader_disconnected () = 0; + virtual void in_msg_raw (InMsgBuffer const &msg) = 0; + virtual void in_msg_sequenced (InMsgBuffer const &msg) = 0; + virtual ~ICoreGatewayEventSink() {}; +}; + +}} diff --git a/src/server/sparkd/jetfix/gateway/ICoreGatewayObserver.h b/src/server/sparkd/jetfix/gateway/ICoreGatewayObserver.h new file mode 100644 index 000000000..6e501d18a --- /dev/null +++ b/src/server/sparkd/jetfix/gateway/ICoreGatewayObserver.h @@ -0,0 +1,19 @@ +#pragma once + +#include +#include "server/sparkd/jetfix/common/InMsgBuffer.h" + +namespace bts { +namespace jetfix { + +class ICoreGatewayObserver +{ +public: + virtual ~ICoreGatewayObserver() {}; + virtual void ready () = 0; + virtual void disconnect () = 0; + virtual void msg (InMsgBuffer const &msg) = 0; + virtual void replay (std::vector const &msgs) = 0; +}; + +}} diff --git a/src/server/sparkd/jetfix/gateway/JetFIXGateway.cpp b/src/server/sparkd/jetfix/gateway/JetFIXGateway.cpp new file mode 100644 index 000000000..4cfc5efb3 --- /dev/null +++ b/src/server/sparkd/jetfix/gateway/JetFIXGateway.cpp @@ -0,0 +1,9 @@ +#ifndef WIN32 +#include "JetFIXGateway.h" + +namespace bts { +namespace jetfix { + +}} +#endif + diff --git a/src/server/sparkd/jetfix/gateway/JetFIXGateway.h b/src/server/sparkd/jetfix/gateway/JetFIXGateway.h new file mode 100644 index 000000000..9f4197de5 --- /dev/null +++ b/src/server/sparkd/jetfix/gateway/JetFIXGateway.h @@ -0,0 +1,116 @@ +#pragma once + +#include +#include "server/sparkd/jetfix/gateway/CoreGateway.h" +#include "server/sparkd/jetfix/gateway/ICoreGatewayObserver.h" +#include "server/sparkd/jetfix/connection/SocketConnection.h" +#include "server/sparkd/jetfix/store/FileStore.h" +#include "lib/utils/meta_programming.h" + +#include "lib/config/protos/jetfix.pb.h" + +namespace bts { +namespace jetfix { + +template +class JetFIXGateway : ICoreGatewayObserver +{ +public: + JetFIXGateway(config::JetFIX::Connection const &cfg) + : + _core (cfg,*this, + + [] (config::JetFIX::Connection const &cfg) + {return new SocketConnection (cfg);}, + + [] (config::JetFIX::Connection const &cfg) + { return new FileStore (cfg);}), + _dispatchers (), + _dispatcher_idx (), + _on_ready ([](){}), + _on_disconnect ([](){}), + _on_begin_replay ([](){}), + _on_end_replay ([](){}) + { + std::vector disp_vec; + meta::collect_tuple_items (_dispatchers,&disp_vec); + for (auto d : disp_vec) + _dispatcher_idx.emplace_back (d->tag (),d); + } + + + void start () {_core.start ();} + void stop () {_core.stop ();} + void send (OutFIXPtr const &msgs) {_core.send (msgs);} + void dispatch_events () {_core.dispatch_events ();} + + std::function &on_ready () {return _on_ready;} + std::function &on_disconnect () {return _on_disconnect;} + std::function &on_begin_replay () {return _on_begin_replay;} + std::function &on_end_replay () {return _on_end_replay;} + + template + std::function &on () + { + return std::get, + DispatcherTuple>::value> (_dispatchers).handler; + } + +private: + struct IDispatcher + { + virtual MessageTypes tag () const = 0; + virtual void dispatch (InMsgBuffer const&msg) = 0; + }; + + template + struct Dispatcher : public IDispatcher + { + typedef std::function Handler; + + Dispatcher () : msg (), handler ([](MT const&){}) {} + + MessageTypes tag () const override {return MT::tag;} + + void dispatch (InMsgBuffer const&in_msg) override + { + msg.read (in_msg.data (),in_msg.size ()); + handler (msg); + } + + MT msg; + Handler handler; + }; + + typedef std::tuple...> DispatcherTuple; + + void ready () override {_on_ready ();} + + void disconnect () override {_on_disconnect ();} + + void msg (InMsgBuffer const&msg) override + { + auto t = msg.type (); + for (auto &d : _dispatcher_idx) + if (d.first == t) + d.second->dispatch (msg); + } + + void replay (std::vector const &msgs) + { + _on_begin_replay (); + for (auto const &m :msgs) + msg (m); + _on_end_replay (); + } + + CoreGateway _core; + std::tuple...> _dispatchers; + std::vector> _dispatcher_idx; + std::function _on_ready; + std::function _on_disconnect; + std::function _on_begin_replay; + std::function _on_end_replay; +}; + +}} diff --git a/src/server/sparkd/jetfix/gateway/Session.cpp b/src/server/sparkd/jetfix/gateway/Session.cpp new file mode 100644 index 000000000..c5fb09663 --- /dev/null +++ b/src/server/sparkd/jetfix/gateway/Session.cpp @@ -0,0 +1,802 @@ +#include "Session.h" + + +#include "lib/config/config.h" +#include "server/sparkd/jetfix/codec/FIXEncodingStream.h" +#include "server/sparkd/jetfix/codec/FieldDefinitions.h" +#include "server/sparkd/jetfix/codec/MessageTypes.h" +#include "server/sparkd/jetfix/common/OutFIX.h" +#include "server/sparkd/jetfix/connection/IConnection.h" +#include "server/sparkd/jetfix/gateway/ICoreGatewayEventSink.h" +#include "server/sparkd/jetfix/store/IStore.h" +#include "lib/utils/TimeStamp.h" +#include "lib/utils/datetime.h" +#include "lib/logging/logging.h" + +#include "lib/config/protos/jetfix.pb.h" + +#include +#include + +namespace bts { +namespace jetfix { + +OutFIXPtr make_logon (config::JetFIX::Connection const &session_cfg); +OutFIXPtr make_logout (config::JetFIX::Connection const &session_cfg); +OutFIXPtr make_resend_request (config::JetFIX::Connection const &session_cfg, uint32_t min, uint32_t max); +OutFIXPtr make_hb (config::JetFIX::Connection const &session_cfg, std::string const&req_id); +OutFIXPtr make_hb (config::JetFIX::Connection const &session_cfg); +OutFIXPtr make_test_request (config::JetFIX::Connection const &session_cfg,std::string const&tr_id); +OutFIXPtr make_seq_reset (config::JetFIX::Connection const &session_cfg, uint32_t end); + +enum class Session::State {WLOGON,RECOVER,WTR_REPLY,READY,TERMINATED,WLOGOUT}; + +struct Session::RecoveryState +{ + RecoveryState () + : buffered_msgs (), + resend_request (0,0) + {} + + std::map buffered_msgs; + std::pair resend_request; +}; + +struct Session::PImpl +{ + PImpl(config::JetFIX::Connection const &cfg, + IConnection &connection, + IStore &store, + ICoreGatewayEventSink &sink, + Clock const &clock) + : + cfg (cfg), + state (State::TERMINATED), + connection (connection), + store (store), + event_sink (sink), + now (clock), + begin_of_this_session (), + last_seq_num (), + last_seq_num_processed (), + last_sent_msg_timestamp (), + last_sent_sync_timestamp (), + last_rcvd_msg_timestamp (), + start_of_logon (), + start_of_logout (), + current_time (std::make_shared ( + datetime::FIX_timestamp (datetime::now ()))), + recovery_state () + { + } + + config::JetFIX::Connection cfg; + State state; + IConnection &connection; + IStore &store; + ICoreGatewayEventSink &event_sink; + Clock now; + uint32_t begin_of_this_session; + uint32_t last_seq_num; + uint32_t last_seq_num_processed; + utils::TimeStamp last_sent_msg_timestamp; + utils::TimeStamp last_sent_sync_timestamp; + utils::TimeStamp last_rcvd_msg_timestamp; + utils::TimeStamp start_of_logon; + utils::TimeStamp start_of_logout; + std::shared_ptr current_time; + std::unique_ptr recovery_state; +}; + +Session::Session(config::JetFIX::Connection const &cfg, + IConnection &connection, + IStore &store, + ICoreGatewayEventSink &sink, + Clock const &clock) + : me (new PImpl(cfg,connection,store,sink,clock)) +{} + +Session::~Session () +{ + if (me->state != State::TERMINATED) + terminate_session (JetFIXError::ERROR_NONE); +} + +void Session::start () +{ + bts_assert (me->state == State::TERMINATED); + me->begin_of_this_session = me->store.last_seq_num (); + me->start_of_logon = me->now (); + me->last_seq_num = me->begin_of_this_session; + me->last_seq_num_processed = me->store.last_seq_num_processed (); + transition (State::WLOGON); + internal_send (make_logon (me->cfg)); +} + +void Session::send (OutFIXPtr const&msgs) +{ + switch (me->state) + { + case State::READY: + internal_send (msgs); + break; + case State::TERMINATED: + case State::WLOGOUT: + break; + case State::WLOGON: + case State::RECOVER: + case State::WTR_REPLY: + default: + bts_assert (false); + } +} + +void Session::receive (InMsgBuffer const&msg) +{ + me->store.store (msg); + me->last_rcvd_msg_timestamp = me->now (); + + if (me->state != State::WLOGON && + msg.get_seq_num () <= me->last_seq_num_processed) + { + LOG (Jetfix,Error, "Discarding repeated message: {} sn: {} state: {} conn: {}", + (char) msg.type (), msg.get_seq_num (), + state_name (me->state), me->cfg.name ()); + return; + } + + switch (msg.type ()) { + case MessageTypes::Logon: + on_logon (msg); + break; + case MessageTypes::Logout: + on_logout (msg); + break; + case MessageTypes::TestRequest: + on_test_request (msg); + break; + case MessageTypes::ResendRequest: + on_resend_request (msg); + break; + case MessageTypes::SeqReset: + on_seq_reset (msg); + break; + case MessageTypes::HB: + on_hb (msg); + break; + default: + on_app (msg); + break; + } +} + +void Session::shutdown () +{ + switch (me->state) + { + case State::WLOGON: + case State::RECOVER: + case State::WTR_REPLY: + case State::READY: + internal_send (make_logout (me->cfg)); + transition (State::WLOGOUT); + break; + case State::WLOGOUT: + break; + case State::TERMINATED: + default: + bts_assert (false); + } +} + +void Session::time (std::shared_ptr const&time_str) +{ + me->current_time = time_str; + + switch (me->state) + { + case State::WLOGON: + if (seconds_elapsed (me->start_of_logon) > me->cfg.seconds_to_logon ()) + logout (JetFIXError::LOGIN_TIMEOUT); + break; + case State::RECOVER: + update_resend_request_progress (); + if (!resend_request_active ()) + send_resend_request (); + break; + case State::WTR_REPLY: + if (seconds_elapsed (me->last_sent_sync_timestamp) > me->cfg.seconds_between_syncs ()) + send_sync_test_request (); + break; + case State::READY: + if (seconds_elapsed (me->last_sent_msg_timestamp) > me->cfg.seconds_between_hbs ()) + internal_send (make_hb (me->cfg)); + else if (seconds_elapsed (me->last_rcvd_msg_timestamp) > + me->cfg.seconds_between_hbs () * me->cfg.max_missed_hbs ()) + logout (JetFIXError::HB_TIMEOUT); + break; + case State::WLOGOUT: + if (seconds_elapsed (me->start_of_logout) > me->cfg.seconds_to_logout ()) + terminate_session (JetFIXError::LOGOUT_TIMEOUT); + break; + case State::TERMINATED: + break; + default: + bts_assert (false); + } +} + +void Session::on_logon (InMsgBuffer const&msg) +{ + switch (me->state) + { + case State::WLOGON: + { + auto their_last = msg.get_seq_num (); + auto our_last_processed = me->last_seq_num_processed; + + if (their_last <= our_last_processed) + terminate_session (JetFIXError::INCOMING_SN_TOO_SMALL, + (boost::format ("Incoming sequence number was %d " + "but the last one we processed was %d. " + "Most likely the FIX sequencer numbers file " + "(.bts/data/jetfix/connection_name/current/seqnum) " + "was deleted. Update this file with the correct " + "incoming sequence number (second column)") + % their_last % our_last_processed).str ()); + + else if (their_last == (our_last_processed+1)) + { + transition (State::WTR_REPLY); + send_sync_test_request (); + deliver (msg); + } + else { + me->recovery_state.reset (new RecoveryState ()); + transition (State::RECOVER); + buffer (msg); + send_resend_request (); + } + break; + } + case State::RECOVER: + case State::WTR_REPLY: + case State::READY: + terminate_session (JetFIXError::UNEXPECTED_LOGON); + break; + case State::WLOGOUT: + deliver (msg); + break; + case State::TERMINATED: + break; + default: + bts_assert (false); + break; + } +} + +void Session::on_logout (InMsgBuffer const&msg) +{ + switch (me->state) + { + case State::WLOGON: + terminate_session (JetFIXError::LOGGED_OUT_BY_EXCHANGE, + (boost::format (". %s") + % msg.get_text ()).str ()); + break; + + case State::RECOVER: + internal_send (make_logout (me->cfg)); + buffer (msg); + terminate_session (JetFIXError::LOGGED_OUT_BY_EXCHANGE, + (boost::format (". %s") + % msg.get_text ()).str ()); + break; + + case State::WTR_REPLY: + case State::READY: + internal_send (make_logout (me->cfg)); + deliver (msg); + terminate_session (JetFIXError::LOGGED_OUT_BY_EXCHANGE, + (boost::format (". %s") + % msg.get_text ()).str ()); + break; + case State::WLOGOUT: + deliver (msg); + terminate_session (JetFIXError::ERROR_NONE); + break; + case State::TERMINATED: + break; + default: + bts_assert (false); + break; + } +} + +void Session::on_test_request (InMsgBuffer const&msg) +{ + switch (me->state) + { + case State::WLOGON: + terminate_session (JetFIXError::EXPECTED_LOGON, + (boost::format ("Received message type %c ") + % (char) msg.type ()).str ()); + break; + case State::RECOVER: + internal_send (make_hb (me->cfg,msg.get_test_req_id ())); + buffer (msg); + break; + + case State::WTR_REPLY: + case State::READY: + deliver (msg); + internal_send (make_hb (me->cfg,msg.get_test_req_id ())); + break; + case State::WLOGOUT: + deliver (msg); + break; + case State::TERMINATED: + break; + default: + bts_assert (false); + } +} + +void Session::on_resend_request(InMsgBuffer const&msg) +{ + switch (me->state) + { + case State::WLOGON: + terminate_session (JetFIXError::EXPECTED_LOGON, + (boost::format ("Received message type %c ") + % (int) msg.type ()).str ()); + break; + case State::RECOVER: + case State::WTR_REPLY: + case State::READY: + { + if (me->state == State::RECOVER) + buffer (msg); + else + deliver (msg); + + auto last_msg_in_request = msg.get_end_seq_no (); + if (last_msg_in_request <= me->begin_of_this_session) + send_seq_reset (msg.get_begin_seq_no (), msg.get_end_seq_no ()); + else + logout (JetFIXError::UNEXPECTED_RESEND_REQUEST, + (boost::format ("Received resend request for messages " + "%d to %d but my initial sequence number is %d ") + % msg.get_begin_seq_no () + % msg.get_end_seq_no () + % me->begin_of_this_session).str ()); + break; + } + case State::WLOGOUT: + deliver (msg); + break; + case State::TERMINATED: + break; + default: + bts_assert (false); + } +} + +void Session::on_seq_reset (InMsgBuffer const&msg) +{ + switch (me->state) + { + case State::WLOGON: + terminate_session (JetFIXError::EXPECTED_LOGON, + (boost::format ("Received message type %c ") + % (int) msg.type ()).str ()); + break; + case State::RECOVER: + buffer_or_deliver (msg); + break; + case State::WTR_REPLY: + case State::READY: + deliver (msg); + logout (JetFIXError::UNEXPECTED_SEQ_RESET); + break; + case State::WLOGOUT: + deliver (msg); + break; + case State::TERMINATED: + break; + default: + bts_assert (false); + } +} + +void Session::on_hb (InMsgBuffer const &msg) +{ + switch (me->state) + { + case State::WLOGON: + terminate_session (JetFIXError::EXPECTED_LOGON, + (boost::format ("Received message type %c ") + % (int) msg.type ()).str ()); + break; + case State::RECOVER: + buffer (msg); + break; + case State::WTR_REPLY: + deliver (msg); + if (msg.has_test_req_id () && + msg.get_test_req_id () == "sync") + { + me->event_sink.session_ready (); + transition (State::READY); + } + break; + case State::READY: + case State::WLOGOUT: + deliver (msg); + break; + case State::TERMINATED: + break; + default: + bts_assert (false); + } +} + +void Session::on_app (InMsgBuffer const &msg) +{ + switch (me->state) + { + case State::WLOGON: + terminate_session (JetFIXError::EXPECTED_LOGON, + (boost::format ("Received message type %c ") + % (int) msg.type ()).str ()); + break; + case State::RECOVER: + buffer_or_deliver (msg); + break; + case State::WTR_REPLY: + case State::READY: + case State::WLOGOUT: + deliver (msg); + break; + case State::TERMINATED: + break; + default: + bts_assert (false); + } +} + +void Session::deliver (InMsgBuffer const&msg) +{ + auto sn = msg.get_seq_num (); + bts_assert (sn == me->last_seq_num_processed+1); + me->last_seq_num_processed++; + me->store.set_last_msg_processed (me->last_seq_num_processed); + me->event_sink.in_msg_sequenced (msg); +} + +void Session::buffer (InMsgBuffer const &msg) +{ + bts_assert (me->state == State::RECOVER); + bts_assert (msg.type () != MessageTypes::SeqReset); + + auto sn = msg.get_seq_num (); + bts_assert (sn > me->last_seq_num_processed+1); + bts_assert (me->recovery_state); + me->recovery_state->buffered_msgs[sn] = msg; + + bts_assert (me->recovery_state->buffered_msgs.begin ()->first > me->last_seq_num_processed+1); +} + +void Session::buffer_or_deliver (InMsgBuffer const&msg) +{ + bts_assert (me->state == State::RECOVER); + auto sn = msg.get_seq_num (); + + me->recovery_state->buffered_msgs[sn] = msg; + + std::set to_remove; + for (auto &p : me->recovery_state->buffered_msgs) { + auto sn = p.first; + auto msg = p.second; + if (sn == me->last_seq_num_processed+1) { + to_remove.insert (sn); + deliver (msg); + if (msg.type () == MessageTypes::SeqReset) { + me->last_seq_num_processed = msg.get_new_seq_no ()-1; + me->store.set_last_msg_processed (me->last_seq_num_processed); + } + } + else if (sn <= me->last_seq_num_processed) { + // a sequence reset might have advanced us past + // buffered admin messages + to_remove.insert (sn); + } + else + break; + } + + for (auto sn : to_remove) + me->recovery_state->buffered_msgs.erase (sn); + + if (me->recovery_state->buffered_msgs.empty ()) + { + me->recovery_state.reset (); + transition (State::WTR_REPLY); + send_sync_test_request (); + } +} + +void Session::internal_send (OutFIXPtr const&msgs) +{ + IConnection::WriteVec data_vec; + bts_assert (msgs->fix.size () < IConnection::MaxWriteVecSz); + + msgs->set_ts (OutFIX::PRE_STAMP,utils::TimeStamp::now ()); + for (size_t i = 0; i < msgs->fix.size ();i++) { + auto &s = msgs->fix[i]; + me->last_seq_num = + s.stamp (me->last_seq_num, + me->last_seq_num_processed, + *me->current_time); + data_vec[i].first = s.data (); + data_vec[i].second = s.size (); + } + + auto correlation_id = msgs->get_ts (jetfix::OutFIX::MD_WRITE_TS).to_uint64 (); + + msgs->set_ts (OutFIX::POST_STAMP,utils::TimeStamp::now ()); + if (me->connection.write (data_vec,msgs->fix.size (),correlation_id) < 0) { + terminate_session (JetFIXError::CONNECTION_ERROR); + return; + } + msgs->set_ts (OutFIX::POST_SEND,utils::TimeStamp::now ()); + me->last_sent_msg_timestamp = me->now (); + me->store.store (msgs); + msgs->set_ts (OutFIX::POST_STORE,utils::TimeStamp::now ()); +} + +void Session::logout (JetFIXError::Code error) +{ + logout (error,""); +} + +void Session::logout (JetFIXError::Code error,std::string const &extra_info) +{ + auto level = Info; + if (error != JetFIXError::ERROR_NONE && error != JetFIXError::USER_DISCONNECTED) + level = Error; + else + level = Info; + + LOG (Jetfix,level,"Logging out of jetfix session: {} {} conn: {}", + JetFIXError::code_msg (error), extra_info, me->cfg.name ()); + + me->start_of_logout = me->now (); + internal_send (make_logout (me->cfg)); + transition (State::WLOGOUT); +} + +void Session::terminate_session (JetFIXError::Code reason) +{terminate_session (reason,"");} + +void Session::terminate_session (JetFIXError::Code error,std::string const&extra_info) +{ + me->recovery_state.reset (); + + auto level = Info; + if (error != JetFIXError::ERROR_NONE && error != JetFIXError::USER_DISCONNECTED) + level = Error; + else + level = Info; + + LOG (Jetfix,level,"Jetfix session disconnected: {} {} conn: {}", + JetFIXError::code_msg (error), extra_info, me->cfg.name ()); + + transition (State::TERMINATED); + me->event_sink.session_terminated (); +} + +int32_t Session::seconds_elapsed (utils::TimeStamp since) +{return (me->now () - since).seconds ();} + +void Session::send_seq_reset (uint32_t sn, uint32_t new_sn) +{ + int end_seq_n; + if (new_sn == 0) + end_seq_n = me->last_seq_num+1; + else + end_seq_n = new_sn + 1; + + auto msg = make_seq_reset (me->cfg,end_seq_n); + auto &stream = msg->fix[0]; + stream.stamp (sn-1,me->last_seq_num_processed,*me->current_time); + IConnection::WriteVec data_vec; + data_vec[0].first = stream.data (); + data_vec[0].second = stream.size (); + if (me->connection.write (data_vec,1,0) < 0) { + terminate_session (JetFIXError::CONNECTION_ERROR); + return; + } else { + me->last_sent_msg_timestamp = me->now (); + me->store.store (msg); + } +} + +void Session::send_resend_request () +{ + bts_assert (me->state == State::RECOVER); + bts_assert (!resend_request_active ()); + + auto gap = find_first_incoming_msg_gap (); + bts_assert (gap.first); + bts_assert (gap.second); + // adjust to exchange limit on max messages in request + if (gap.second > (gap.first + me->cfg.max_msgs_in_resend_req ())) + gap.second = gap.first + me->cfg.max_msgs_in_resend_req (); + + LOG (Jetfix,Warn, + "Issuing recovery request in connection {} " + " asking for messages with sequence numbers from {} to {}" , + me->cfg.name (), gap.first, gap.second); + + internal_send (make_resend_request (me->cfg,gap.first, gap.second)); + me->recovery_state->resend_request = gap; +} + +void Session::send_sync_test_request () +{ + bts_assert (me->state == State::WTR_REPLY); + me->last_sent_sync_timestamp = me->now (); + internal_send (make_test_request (me->cfg,"sync")); +} + +void Session::update_resend_request_progress () +{ + bts_assert (me->state == State::RECOVER); + + if (resend_request_active ()) { + auto request_last = me->recovery_state->resend_request.second; + if (me->last_seq_num_processed >= request_last) + me->recovery_state->resend_request = std::make_pair (0,0); + } +} + +std::pair Session::find_first_incoming_msg_gap () +{ + if (me->recovery_state->buffered_msgs.empty ()) + return std::make_pair (0u,0u); + else + return std::make_pair (me->last_seq_num_processed + 1, + me->recovery_state->buffered_msgs.begin ()->first-1); +} + +bool Session::resend_request_active () +{return me->recovery_state->resend_request.first != 0;} + + +void Session::transition (State new_state) +{ + LOG (Jetfix,Warn,"Jetfix connection {} transitioning from {} to {}", + me->cfg.name (), state_name (me->state), state_name (new_state)); + me->state = new_state; +} + +const char * Session::state_name (State state) +{ + switch (state) + { + case State::WLOGON: + return "WLOGON"; + case State::RECOVER: + return "RECOVER"; + case State::WTR_REPLY: + return "WTR_REPLY"; + case State::READY: + return "READY"; + case State::TERMINATED: + return "TERMINATED"; + case State::WLOGOUT: + return "WLOGOUT"; + default: + bts_assert (false); + return nullptr; + } +} + +OutFIXPtr make_logon (config::JetFIX::Connection const &session_cfg) +{ + auto result = std::make_shared (session_cfg); + auto &stream = result->fix.back (); + auto &buffer = stream.new_msg (MessageTypes::Logon); + using namespace fix; + + buffer.add(session_cfg.raw_data ().size ()); + buffer.add(session_cfg.raw_data ()); + buffer.add (0); + buffer.add (session_cfg.seconds_between_hbs ()); + buffer.add ("O2"); + buffer.add ("1.0"); + buffer.add ("BTS"); + + if (session_cfg.has_trading_session_id()) { + buffer.add (1); + buffer.add (session_cfg.trading_session_id()); + } + if (session_cfg.has_udf_support_indicator()) + buffer.add (session_cfg.udf_support_indicator()); + stream.close (); + return result; +} + +OutFIXPtr make_logout (config::JetFIX::Connection const &session_cfg) +{ + auto result = std::make_shared (session_cfg); + auto stream = &result->fix.back (); + using namespace fix; + + stream->new_msg (MessageTypes::Logout); + stream->close (); + return result; +} + +OutFIXPtr make_resend_request (config::JetFIX::Connection const &session_cfg, uint32_t min, uint32_t max) +{ + auto result = std::make_shared (session_cfg); + auto &stream = result->fix.back (); + auto &buffer = stream.new_msg (MessageTypes::ResendRequest); + using namespace fix; + + buffer.add (min); + buffer.add (max); + stream.close (); + return result; +} + +OutFIXPtr make_hb (config::JetFIX::Connection const &session_cfg, std::string const&req_id) +{ + auto result = std::make_shared (session_cfg); + auto &stream = result->fix.back (); + auto &buffer = stream.new_msg (MessageTypes::HB); + using namespace fix; + + buffer.add (req_id); + stream.close (); + return result; +} + +OutFIXPtr make_hb (config::JetFIX::Connection const &session_cfg) +{ + auto result = std::make_shared (session_cfg); + auto &stream = result->fix.back (); + stream.new_msg (MessageTypes::HB); + stream.close (); + return result; +} + +OutFIXPtr make_test_request (config::JetFIX::Connection const &session_cfg,std::string const&req_id) +{ + auto result = std::make_shared (session_cfg); + auto &stream = result->fix.back (); + auto &buffer = stream.new_msg (MessageTypes::TestRequest); + using namespace fix; + buffer.add (req_id); + stream.close (); + return result; +} + +OutFIXPtr make_seq_reset (config::JetFIX::Connection const &session_cfg, uint32_t end) +{ + auto result = std::make_shared (session_cfg); + auto &stream = result->fix.back (); + auto &buffer = stream.new_msg (MessageTypes::SeqReset); + using namespace fix; + buffer.add (datetime::now ()); + buffer.add(true); + buffer.add(true); + buffer.add(end); + stream.close (); + return result; +} + +}} + diff --git a/src/server/sparkd/jetfix/gateway/Session.h b/src/server/sparkd/jetfix/gateway/Session.h new file mode 100644 index 000000000..fa27e7697 --- /dev/null +++ b/src/server/sparkd/jetfix/gateway/Session.h @@ -0,0 +1,78 @@ +#pragma once + +#include +#include + +#include "server/sparkd/jetfix/common/InMsgBuffer.h" +#include "server/sparkd/jetfix/common/JetFIXError.h" +#include "lib/utils/datetime.h" +#include "server/sparkd/jetfix/common/OutFIXPtr.h" + +#include "lib/config/protos/jetfix.pb.h" + +namespace bts { +namespace utils {class TimeStamp;} +namespace jetfix { +class IConnection; +class IStore; +class ICoreGatewayEventSink; + +class Session +{ +public: + typedef std::function Clock; + + Session(config::JetFIX::Connection const &cfg, + IConnection &connection, + IStore &store, + ICoreGatewayEventSink &sink, + Clock const &clock); + ~Session(); + + void start (); + void send (OutFIXPtr const&); + void receive (InMsgBuffer const&); + void shutdown (); + void time (std::shared_ptr const&time_str); + +private: + enum class State; + enum class Stats; + struct PImpl; + struct RecoveryState; + + + void on_logon (InMsgBuffer const&msg); + void on_logout (InMsgBuffer const&msg); + void on_test_request (InMsgBuffer const&msg); + void on_resend_request(InMsgBuffer const&msg); + void on_seq_reset (InMsgBuffer const&msg); + void on_hb (InMsgBuffer const &msg); + void on_app (InMsgBuffer const &msg); + + void deliver (InMsgBuffer const&msg); + void buffer (InMsgBuffer const &msg); + void buffer_or_deliver (InMsgBuffer const&msg); + + void internal_send (OutFIXPtr const&); + void logout (JetFIXError::Code reason); + void logout (JetFIXError::Code reason,std::string const &extra_info); + void terminate_session (JetFIXError::Code reason); + void terminate_session (JetFIXError::Code reason,std::string const&extra_info); + + int32_t seconds_elapsed (utils::TimeStamp since); + + void send_seq_reset (uint32_t sn, uint32_t new_sn); + void send_resend_request (); + void send_sync_test_request (); + + void update_resend_request_progress (); + std::pair find_first_incoming_msg_gap (); + bool resend_request_active (); + + void transition (State new_state); + static const char * state_name (State state); + std::unique_ptr me; +}; + +}} diff --git a/src/server/sparkd/jetfix/jetfix.h b/src/server/sparkd/jetfix/jetfix.h new file mode 100644 index 000000000..63f07adcc --- /dev/null +++ b/src/server/sparkd/jetfix/jetfix.h @@ -0,0 +1,5 @@ +#pragma once +#include "conn/bts_conn/jetfix/codec/FieldDefinitions.h" +#include "conn/bts_conn/jetfix/codec/MessageDefinitions.h" +#include "conn/bts_conn/jetfix/gateway/JetFIXGateway.h" +#include "conn/bts_conn/jetfix/codec/FIXFragment.h" diff --git a/src/server/sparkd/jetfix/store/FileStore.cpp b/src/server/sparkd/jetfix/store/FileStore.cpp new file mode 100644 index 000000000..a2c404628 --- /dev/null +++ b/src/server/sparkd/jetfix/store/FileStore.cpp @@ -0,0 +1,326 @@ +#include "FileStore.h" + +#include "lib/config/config.h" +#include "server/sparkd/jetfix/codec/FIXEncodingStream.h" +#include "server/sparkd/jetfix/common/InMsgBuffer.h" +#include "server/sparkd/jetfix/common/OutFIX.h" +#include "server/sparkd/jetfix/store/session_name.h" +#include "lib/utils/datetime.h" +#include "lib/utils/file_utils.h" +#include "lib/utils/bts_chrono.h" +#include "lib/os/os.h" + +#include +#include +#include +#include +#include +#include + +namespace bts { +namespace jetfix { + +struct FileStore::Event +{ + Event () + : out_msgs (), in_msg (), + quit (false), + has_last_msg_processed (false), + last_msg_processed (0) + {} + + Event (Event const &src) + : out_msgs (src.out_msgs), + in_msg (src.in_msg), + quit (src.quit), + has_last_msg_processed (src.has_last_msg_processed), + last_msg_processed (src.last_msg_processed) + {} + + Event (Event &&src) + : out_msgs (std::move(src.out_msgs)), + in_msg (std::move(src.in_msg)), + quit (src.quit), + has_last_msg_processed (src.has_last_msg_processed), + last_msg_processed (src.last_msg_processed) + {} + + Event& operator=(Event const &src) + { + out_msgs = src.out_msgs; + in_msg = src.in_msg; + quit = src.quit; + has_last_msg_processed = src.has_last_msg_processed; + last_msg_processed = src.last_msg_processed; + return *this; + } + + Event& operator=(Event const &&src) + { + out_msgs = std::move(src.out_msgs); + in_msg = std::move(src.in_msg); + quit = src.quit; + has_last_msg_processed = src.has_last_msg_processed; + last_msg_processed = src.last_msg_processed; + return *this; + } + + OutFIXPtr out_msgs; + InMsgBuffer in_msg; + bool quit; + bool has_last_msg_processed; + uint32_t last_msg_processed; +}; + +struct FileStore::PImpl +{ + PImpl() : + active (false), + store_called (false), + replay_messages (), + last_seq_num (0), + last_seq_num_processed (0), + seq_num_path (), + fix_store (), + writer_thread (), + event_queue () + {} + + bool active; + bool store_called; + std::vector replay_messages; + uint32_t last_seq_num; + uint32_t last_seq_num_processed; + std::string seq_num_path; + std::ofstream fix_store; + std::thread writer_thread; + moodycamel::ReaderWriterQueue event_queue; +}; + +std::string fix_log_path (std::string const&base_path) +{ + boost::filesystem::path result = base_path; + result /= "fix"; + return result.string (); +} + +std::string seqnum_path (std::string const&base_path) +{ + boost::filesystem::path result = base_path; + result /= "seqnum"; + return result.string (); +} + +std::string store_path (std::string base_path, + std::string const&connection_name, + std::string const&session_name) +{ + if (base_path.empty ()) + base_path = os::bts_data_path (); + boost::filesystem::path result = utils::normalize_data_path (base_path); + result /= "jetfix"; + result /= connection_name; + result /= session_name; + return result.string (); +} + +void create_current_simlink (std::string const &path) +{ + auto sym_link_path = boost::filesystem::path (path); + sym_link_path /= ".."; + sym_link_path /= "current"; + if (boost::filesystem::exists (sym_link_path)) + boost::filesystem::remove (sym_link_path); + + boost::filesystem::create_symlink (boost::filesystem::path (path),sym_link_path); +} + +std::map +load_replay (config::JetFIX::Connection const &cfg,std::string const&path, uint32_t max_sn) +{ + std::map result; + if (boost::filesystem::exists (path)) { + + std::ifstream f (path); + std::string line_buffer; + line_buffer.reserve (10*1024); + auto header_size = sizeof("20150427-19:05:17.032 < ")-1; + std::string date_buffer ("20150427-19:05:17.032"); + + for (;;) { + std::getline (f,line_buffer); + if (!f.eof ()) { + bts_assert (line_buffer.size () > header_size); + if (line_buffer[22]=='<') { + auto ts = datetime::parse_FIX_timestamp (line_buffer.substr (0,21)); + if (!datetime::is_at_least_secs_old (ts,cfg.seconds_to_replay ())) { + InMsgBuffer msg (line_buffer.substr (24)); + auto sn = msg.get_seq_num (); + if (sn <= max_sn) + result[sn] = msg; + } + } + } + else + break; + } + } + return result; +} + +void save_fix (std::ostream &out, const char*data, size_t size, bool is_outgoing) +{ + out << datetime::FIX_timestamp (datetime::now ()) << " " + << (is_outgoing ? ">" : "<") << " "; + out.write (data,size); + out << std::endl; +} + +void save_seq_nums (std::string const&path,uint32_t last_seq_num, uint32_t last_seq_num_processed) +{ + std::ofstream f (path); + f << last_seq_num << " " << last_seq_num_processed << std::endl; +} + +void load_seq_nums (std::string const&path,uint32_t* last_seq_num, uint32_t *last_seq_num_processed) +{ + if (boost::filesystem::exists (path)) { + std::ifstream f (path); + f >> *last_seq_num >> *last_seq_num_processed; + } else { + *last_seq_num = 0; + *last_seq_num_processed = 0; + save_seq_nums (path, *last_seq_num,*last_seq_num_processed); + } +} + +FileStore::FileStore (config::JetFIX::Connection const&cfg,std::string base_path) : me (new PImpl()) { + auto sn = session_name (cfg); + me->active = sn.size () > 0; + if (me->active) { + + auto path = store_path (base_path,cfg.name (),sn); + if (!boost::filesystem::exists (path)) + bts_assert (boost::filesystem::create_directories (path)); + + // load seq numbers + auto seq_num_p = seqnum_path (path); + load_seq_nums (seq_num_p,&me->last_seq_num,&me->last_seq_num_processed); + me->seq_num_path = seq_num_p; + + // load replay + auto fix_log_p = fix_log_path (path); + auto replay_messages = load_replay (cfg,fix_log_p,me->last_seq_num_processed); + uint32_t previous_sn = 0; + for (auto &p : replay_messages) { + auto sn = p.first; + auto &m = p.second; + if (!(m.has_poss_dup_flag_flag () && m.get_poss_dup_flag ())) { + if (previous_sn != 0) + bts_assert (sn == (previous_sn + 1)); + previous_sn = sn; + me->replay_messages.emplace_back (m); + } + } + me->fix_store.open (fix_log_p,std::ofstream::app); + + create_current_simlink (path); + me->writer_thread = std::thread (std::bind (&FileStore::write,this)); + } +} + +FileStore::~FileStore () +{ + if (me->active) { + Event ev; + ev.quit = true; + me->event_queue.enqueue (ev); + me->writer_thread.join (); + } +} + +bool FileStore::session_active () const +{return me->active;} + +std::vector FileStore::replay () const +{ + bts_assert (me->active && !me->store_called); + return me->replay_messages; +} + +uint32_t FileStore::last_seq_num () const +{ + bts_assert (me->active && !me->store_called); + return me->last_seq_num; +} + +uint32_t FileStore::last_seq_num_processed () const +{ + bts_assert (me->active && !me->store_called); + return me->last_seq_num_processed; +} + +void FileStore::store (OutFIXPtr const&msgs) +{ + bts_assert (me->active); + me->store_called = true; + Event ev; + ev.out_msgs = msgs; + me->event_queue.enqueue (ev); +} + +void FileStore::store (InMsgBuffer const&msg) +{ + bts_assert (me->active); + me->store_called = true; + Event ev; + ev.in_msg = msg; + me->event_queue.enqueue (ev); +} + +void FileStore::set_last_msg_processed (uint32_t last_msg_processed) +{ + bts_assert (me->active); + me->store_called = true; + Event ev; + ev.has_last_msg_processed = true; + ev.last_msg_processed = last_msg_processed; + me->event_queue.enqueue (ev); +} + +void FileStore::write () +{ + Event ev; + + while (true) + if (me->event_queue.try_dequeue (ev)) { + if (ev.quit) { + return; + } + else if (ev.out_msgs) { + for (auto &encoded_stream : ev.out_msgs->fix) + for (auto &mm : encoded_stream.msg_map ()) { + me->last_seq_num = std::max (me->last_seq_num,mm.seq_num); + save_seq_nums (me->seq_num_path,me->last_seq_num,me->last_seq_num_processed); + save_fix (me->fix_store,encoded_stream.data ()+mm.offset,mm.size,true); + } + me->fix_store.flush (); + } + else if (ev.in_msg.size ()) + { + save_fix (me->fix_store,ev.in_msg.data (),ev.in_msg.size (),false); + me->fix_store.flush (); + } + else if (ev.has_last_msg_processed) { + me->last_seq_num_processed = ev.last_msg_processed; + save_seq_nums (me->seq_num_path,me->last_seq_num,me->last_seq_num_processed); + } + else + bts_assert (false); + } + else + std::this_thread::sleep_for (100ms); +} + +}} + diff --git a/src/server/sparkd/jetfix/store/FileStore.h b/src/server/sparkd/jetfix/store/FileStore.h new file mode 100644 index 000000000..9be698474 --- /dev/null +++ b/src/server/sparkd/jetfix/store/FileStore.h @@ -0,0 +1,31 @@ +#pragma once + +#include +#include "server/sparkd/jetfix/store/IStore.h" +#include "lib/config/protos/jetfix.pb.h" + +namespace bts { +namespace jetfix { + +class FileStore : public IStore +{ +public: + FileStore(config::JetFIX::Connection const&cfg,std::string base_path=""); + ~FileStore(); + + bool session_active () const override; + std::vector replay () const override; + uint32_t last_seq_num () const override; + uint32_t last_seq_num_processed () const override; + + void store (InMsgBuffer const&buffer) override; + void store (OutFIXPtr const&msgs) override; + void set_last_msg_processed (uint32_t last_msg_processed) override; +private: + struct PImpl; + struct Event; + void write (); + std::unique_ptr me; +}; + +}} diff --git a/src/server/sparkd/jetfix/store/IStore.h b/src/server/sparkd/jetfix/store/IStore.h new file mode 100644 index 000000000..885795b41 --- /dev/null +++ b/src/server/sparkd/jetfix/store/IStore.h @@ -0,0 +1,27 @@ +#pragma once + +#include +#include +#include "server/sparkd/jetfix/common/OutFIXPtr.h" + + +namespace bts { +namespace jetfix { +class InMsgBuffer; + +class IStore +{ +public: + virtual ~IStore() {}; + + virtual bool session_active () const = 0; + virtual std::vector replay () const = 0; + virtual uint32_t last_seq_num () const = 0; + virtual uint32_t last_seq_num_processed () const = 0; + + virtual void store (OutFIXPtr const&msgs) = 0; + virtual void store (InMsgBuffer const&buffer) = 0; + virtual void set_last_msg_processed (uint32_t last_msg_processed) = 0; +}; + +}} diff --git a/src/server/sparkd/jetfix/store/session_name.cpp b/src/server/sparkd/jetfix/store/session_name.cpp new file mode 100644 index 000000000..774dcc29f --- /dev/null +++ b/src/server/sparkd/jetfix/store/session_name.cpp @@ -0,0 +1,140 @@ +#include "session_name.h" +#include "lib/utils/bts_assert.h" +#include + +namespace bts { +namespace jetfix { + +boost::gregorian::greg_weekday parse_weekday (std::string weekday_str) +{ + boost::to_lower (weekday_str); + for (int i = 0; i < 7; i++) { + auto gw_day = boost::gregorian::greg_weekday (i); + auto prefix = std::string (gw_day.as_short_string ()); + boost::to_lower (prefix); + if (weekday_str.find (prefix) == 0) + return gw_day; + } + bts_assert (false); + return boost::gregorian::greg_weekday (0); +} + +boost::posix_time::time_duration parse_time (std::string const &time_str) +{ + try { + return boost::posix_time::duration_from_string (time_str); + } + catch (boost::bad_lexical_cast &) + { + bts_assert (false); + return boost::posix_time::time_duration (); + } +} + +datetime::PTime closest_before (datetime::PTime const &ref_date, + boost::gregorian::greg_weekday weekday, + boost::posix_time::time_duration time_of_day) +{ + if (ref_date.date ().day_of_week () == weekday) { + if (time_of_day < ref_date.time_of_day ()) + return datetime::PTime (ref_date.date (),time_of_day); + else { + return datetime::PTime (ref_date.date () - boost::gregorian::date_duration (7), + time_of_day); + } + } else { + auto distance = + ref_date.date ().day_of_week ().as_number () - + weekday.as_number (); + if (distance < 0) + distance += 7; + return datetime::PTime (ref_date.date () - boost::gregorian::date_duration (distance), + time_of_day); + } +} + +bool in_session (datetime::PTime const &ref_date, + boost::posix_time::time_duration time_of_day_start, + boost::posix_time::time_duration time_of_day_end) +{ + auto t = ref_date.time_of_day (); + if (time_of_day_start < time_of_day_end) + return t >= time_of_day_start && t <= time_of_day_end; + else + return t >= time_of_day_start || t <= time_of_day_end; +} + +bool in_session (datetime::PTime const &ref_date, + boost::gregorian::greg_weekday weekday_start, + boost::posix_time::time_duration time_of_day_start, + boost::gregorian::greg_weekday weekday_end, + boost::posix_time::time_duration time_of_day_end) +{ + auto closest_earlier_start = closest_before (ref_date,weekday_start,time_of_day_start); + auto closest_earlier_end = closest_before (ref_date,weekday_end,time_of_day_end); + return closest_earlier_end time_of_day_end && + start_of_session.time_of_day () <= time_of_day_end) + start_of_session -= boost::gregorian::date_duration (1); + + sprintf (&result[0], "%04d-%02d-%02d", + (unsigned short) start_of_session.date ().year (), + start_of_session.date ().month ().as_number (), + start_of_session.date ().day ().as_number ()); + return result; +} + +std::string session_name (datetime::PTime const &ref_date, + boost::gregorian::greg_weekday weekday_start, + boost::posix_time::time_duration time_of_day_start, + boost::gregorian::greg_weekday weekday_end, + boost::posix_time::time_duration time_of_day_end) +{ + if (in_session (ref_date,weekday_start,time_of_day_start,weekday_end,time_of_day_end)) + return session_name_from_ptime (closest_before (ref_date,weekday_start,time_of_day_start), + boost::posix_time::time_duration(0, 0, 0), + boost::posix_time::time_duration(0, 0, 1)); + else + return std::string (); +} + +std::string session_name (datetime::PTime const &ref_date, + boost::posix_time::time_duration time_of_day_start, + boost::posix_time::time_duration time_of_day_end) +{ + if (in_session (ref_date,time_of_day_start,time_of_day_end)) + return session_name_from_ptime (ref_date, time_of_day_start, time_of_day_end); + else + return ""; +} + +std::string session_name (config::JetFIX::Connection const &session_cfg) +{ + auto now = datetime::local_now (); + + if (session_cfg.has_start_day ()) { + bts_assert (session_cfg.has_end_day ()); + return session_name (now, + parse_weekday (session_cfg.start_day ()), + parse_time (session_cfg.start_time ()), + parse_weekday (session_cfg.end_day ()), + parse_time (session_cfg.end_time ())); + } else + return session_name (now, + parse_time (session_cfg.start_time ()), + parse_time (session_cfg.end_time ())); +} + + +}} + + diff --git a/src/server/sparkd/jetfix/store/session_name.h b/src/server/sparkd/jetfix/store/session_name.h new file mode 100644 index 000000000..7b0c9a30a --- /dev/null +++ b/src/server/sparkd/jetfix/store/session_name.h @@ -0,0 +1,49 @@ +#pragma once + +#include +#include +#include "lib/utils/datetime.h" +#include "lib/config/protos/jetfix.pb.h" + +namespace boost { +namespace gregorian { +class greg_weekday; +} +namespace posix_time { +class time_duration; +}} + + +namespace bts { +namespace jetfix { + +boost::gregorian::greg_weekday parse_weekday (std::string weekday_str); +bool parse_time (std::string const &time_str,boost::posix_time::time_duration *out); + +datetime::PTime closest_before (datetime::PTime const &ref_date, + boost::gregorian::greg_weekday weekday, + boost::posix_time::time_duration time_of_day); + +bool in_session (datetime::PTime const &ref_date, + boost::gregorian::greg_weekday weekday_start, + boost::posix_time::time_duration time_of_day_start, + boost::gregorian::greg_weekday weekday_end, + boost::posix_time::time_duration time_of_day_end); + +bool in_session (datetime::PTime const &ref_date, + boost::posix_time::time_duration time_of_day_start, + boost::posix_time::time_duration time_of_day_end); + +std::string session_name (datetime::PTime const &ref_date, + boost::gregorian::greg_weekday weekday_start, + boost::posix_time::time_duration time_of_day_start, + boost::gregorian::greg_weekday weekday_end, + boost::posix_time::time_duration time_of_day_end); + +std::string session_name (datetime::PTime const &ref_date, + boost::posix_time::time_duration time_of_day_start, + boost::posix_time::time_duration time_of_day_end); + +std::string session_name (config::JetFIX::Connection const &session_cfg); + +}} diff --git a/src/server/sparkd/jetfix/tests/async_reader_tests.cpp b/src/server/sparkd/jetfix/tests/async_reader_tests.cpp new file mode 100644 index 000000000..5f87549ab --- /dev/null +++ b/src/server/sparkd/jetfix/tests/async_reader_tests.cpp @@ -0,0 +1,158 @@ +#include "catch/catch.h" + +#include "server/sparkd/jetfix/connection/AsyncReader.h" +#include "server/sparkd/jetfix/common/InMsgBuffer.h" +#include "server/sparkd/jetfix/gateway/ICoreGatewayEventSink.h" +#include "server/sparkd/jetfix/connection/IConnection.h" + +#include + +namespace bts { +namespace jetfix { + +const static char *sample_logon = + "8=FIX.4.2" "\001" + "9=127" "\001" + "35=A" "\001" + "34=25" "\001" + "369=1" "\001" + "52=20150427-19:05:18.922" "\001" + "49=CME" "\001" + "50=G" "\001" + "56=1U7001N" "\001" + "57=BLUESTREAM" "\001" + "143=US,NC" "\001" + "98=0" "\001" + "108=30" "\001" + "1603=O2" "\001" + "1604=1.0" "\001" + "1605=BTS" "\001" + "10=046\001"; + +const static char *sample_mass_quote = + "8=FIX.4.2\001" + "9=304\001" + "35=i\001" + "34=61\001" + "49=1U7001N\001" + "50=bluestream\001" + "52=20150427-19:29:40.610\001" + "56=CME\001" + "57=G\001" + "142=US,NC\001" + "117=58\001" + "204=1\001" + + "1028=N\001" + "7928=bluestream\001" + "9702=1\001" + "9771=TESTACC\001" + + "296=1\001" + "302=1\001" + "304=2\001" + "295=2\001" + + "299=10857\001" + "55=EZ\001" + "167=OPT\001" + "107=ESK5 C1855\001" + "132=25200\001" + "133=25350\001" + "134=1\001" + "135=1\001" + + "299=10856\001" + "55=EZ\001" + "167=OPT\001" + "107=ESK5 C1890\001" + "132=25201\001" + "133=25351\001" + "134=2\001" + "135=2\001" + + "10=046\001"; + +struct MyGWObserver : public ICoreGatewayEventSink +{ + bool got_eof; + std::vector msgs; + + MyGWObserver () : got_eof (false), msgs () {} + + void session_ready () + {REQUIRE(false);} + + void session_terminated () + {REQUIRE(false);} + + void reader_disconnected () + {got_eof = true;} + + void in_msg_raw (InMsgBuffer const &msg) + {msgs.emplace_back (msg);} + + void in_msg_sequenced (InMsgBuffer const &) + {REQUIRE(false);} +}; + +struct Connection : public IConnection +{ + Connection (std::istream &str) : str (str) + {} + + bool connect () {return true;} + void shutdown () {} + + int read (char* to_buffer,size_t read_count) + { + str.read (to_buffer,read_count); + return str.gcount (); + } + + int write (WriteVec const&,size_t,size_t ) + {return -1;} + + std::istream &str; +}; + +TEST_CASE ("JetFIX AsyncReaderTests", "[jetfix]") +{ + SECTION ("jeftix_smoke") + { + std::stringstream str; + str << sample_logon << sample_mass_quote; + Connection c (str); + + MyGWObserver ob; + + { + AsyncReader r (c,ob,10,5); + r.start (); + } + + REQUIRE(ob.got_eof); + REQUIRE(ob.msgs.size () == 2); + } + + SECTION ("jeftix_parse_big_file") + { + std::ifstream f ("data/test/sample_fix.dat",std::ios_base::binary); + Connection c (f); + + MyGWObserver ob; + + { + AsyncReader r (c,ob,10,5); + r.start (); + } + REQUIRE (ob.got_eof); + REQUIRE (ob.msgs.size () == 1000); + } +} + + +}} + + + diff --git a/src/server/sparkd/jetfix/tests/decoding_tests.cpp b/src/server/sparkd/jetfix/tests/decoding_tests.cpp new file mode 100644 index 000000000..815442fd4 --- /dev/null +++ b/src/server/sparkd/jetfix/tests/decoding_tests.cpp @@ -0,0 +1,161 @@ +#include "catch/catch.h" + +#include "server/sparkd/jetfix/codec/Fields.h" +#include "server/sparkd/jetfix/codec/Group.h" +#include "server/sparkd/jetfix/codec/FIXMsg.h" +#include "server/sparkd/jetfix/codec/FieldDefinitions.h" +#include "server/sparkd/jetfix/connection/IConnectionObserver.h" +#include "server/sparkd/jetfix/common/InMsgBuffer.h" +#include "server/sparkd/jetfix/codec/FIXParser.h" +#include "server/sparkd/jetfix/codec/FIXFragment.h" +#include "lib/utils/TimeStamp.h" + +namespace bts { +namespace jetfix { + +const static char *sample_logon_header = + "8=FIX.4.2" "\001" + "9=127" "\001" + "35=A" "\001" + "34=25" "\001" + "369=1" "\001" + "52=20150427-19:05:18.922" "\001" + "49=CME" "\001" + "50=G" "\001" + "56=1U7001N" "\001" + "57=BLUESTREAM" "\001" + "143=US,NC" "\001"; + +const static char *sample_mass_quote = + "8=FIX.4.2\001" + "9=234\001" + "35=i\001" + "34=61\001" + "49=1U7001N\001" + "50=bluestream\001" + "52=20150427-19:29:40.610\001" + "56=CME\001" + "57=G\001" + "142=US,NC\001" + "117=58\001" + "204=1\001" + + "1028=N\001" + "7928=bluestream\001" + "9702=1\001" + "9771=TESTACC\001" + + "296=1\001" + "302=1\001" + "304=2\001" + "295=2\001" + + "299=10857\001" + "55=EZ\001" + "167=OPT\001" + "107=ESK5 C1855\001" + "132=25200\001" + "133=25350\001" + "134=1\001" + "135=1\001" + + "299=10856\001" + "55=EZ\001" + "167=OPT\001" + "107=ESK5 C1890\001" + "132=25201\001" + "133=25351\001" + "134=2\001" + "135=2\001"; + +TEST_CASE ("Jetfix InMsgTests" ,"[jetfix]") +{ + SECTION ("basic_parsing_works") + { + InMsgBuffer msg (sample_logon_header,strlen(sample_logon_header)-1,std::shared_ptr()); + REQUIRE (msg.type ()==MessageTypes::Logon); + REQUIRE (msg.get_seq_num ()==25); + REQUIRE (msg.get_last_seq_num_processed ()==1); + } +} + + +TEST_CASE("Jeftix ParserTests","[jetfix]") +{ + SECTION("parse_logon") + { + using namespace fix; + typedef FIXMsg + LogonMsg; + + LogonMsg m; + m.read (sample_logon_header); + REQUIRE (m.get()=="FIX.4.2"); + REQUIRE (m.get()== 127); + REQUIRE (m.get()=="A"); + REQUIRE (m.get()==25); + REQUIRE (m.get()==1); + REQUIRE (m.get()==datetime::make_ptime (2015,4,27,19,5,18,922)); + REQUIRE (m.get()=="CME"); + REQUIRE (m.get()=="G"); + REQUIRE (m.get()=="1U7001N"); + REQUIRE (m.get()=="BLUESTREAM"); + REQUIRE (m.get()=="US,NC"); + } + + SECTION("parse_group") + { + using namespace fix; + typedef FIXMsg MassQuote; + + MassQuote m; + m.read (sample_mass_quote); + REQUIRE (m.get()=="58"); + REQUIRE (m.get()==1); + REQUIRE (m.get()==false); + + auto &qs = m.get (); + REQUIRE (qs.size ()==1); + REQUIRE (qs[0].get()=="1"); + REQUIRE (qs[0].get()==2); + + auto &qe = qs[0].get (); + REQUIRE (qe.size ()==2); + REQUIRE (qe[0].get()=="10857"); + REQUIRE (qe[0].get()=="EZ"); + REQUIRE (qe[0].get()=="OPT"); + REQUIRE (qe[0].get()=="ESK5 C1855"); + REQUIRE (qe[0].get()==25200); + REQUIRE (qe[0].get()==25350); + REQUIRE (qe[0].get()==1); + REQUIRE (qe[0].get()==1); + + REQUIRE (qe[1].get()=="10856"); + REQUIRE (qe[1].get()=="EZ"); + REQUIRE (qe[1].get()=="OPT"); + REQUIRE (qe[1].get()=="ESK5 C1890"); + REQUIRE (qe[1].get()==25201); + REQUIRE (qe[1].get()==25351); + REQUIRE (qe[1].get()==2); + REQUIRE (qe[1].get()==2); + } +} + + +}} + diff --git a/src/server/sparkd/jetfix/tests/fix_encoding_stream_tests.cpp b/src/server/sparkd/jetfix/tests/fix_encoding_stream_tests.cpp new file mode 100644 index 000000000..89fca6bc7 --- /dev/null +++ b/src/server/sparkd/jetfix/tests/fix_encoding_stream_tests.cpp @@ -0,0 +1,105 @@ +#include "catch/catch.h" + +#include "server/sparkd/jetfix/codec/FIXEncodingStream.h" +#include + +namespace bts { +namespace jetfix { + +static config::JetFIX::Connection get_jetfix_cfg () +{ + config::JetFIX::Connection jfcfg; + jfcfg.set_name ("test"); + jfcfg.set_socket_connect_host_ip ("10.96.18.160"); + jfcfg.set_socket_connect_port (9999); + jfcfg.set_sender_compid ("O2TESTER"); + jfcfg.set_target_compid ("BTS"); + jfcfg.set_target_subid ("BTS"); + + jfcfg.set_use_persisted_quote_ids (false); + jfcfg.set_start_time ("00:30:00"); + jfcfg.set_end_time ("23:30:00"); + jfcfg.set_start_day ("sunday"); + jfcfg.set_end_day ("friday"); + jfcfg.set_log_fix_stream (true); + jfcfg.set_seconds_to_pause (10); + jfcfg.set_sender_subid ("BTS"); + jfcfg.set_raw_data ("1U7"); + jfcfg.set_self_match_prevention_id ("bluestream"); + return jfcfg; +} + +TEST_CASE("JetFIX FIX Encoding","[jetfix]") +{ + SECTION("smoke") + { + FIXEncodingStream stream (get_jetfix_cfg ()); + auto &msg = stream.new_msg (MessageTypes::MassQuote); + msg.add ("hello",5); + auto &msg2 = stream.new_msg (MessageTypes::Advertisement); + msg2.add("olleh",5); + stream.close (); + + auto out = stream.to_str (); + REQUIRE (out.size ()); + } + + SECTION("body_lenght_is_correct") + { + using namespace fix; + + FIXEncodingStream stream (get_jetfix_cfg ()); + auto &msg = stream.new_msg (MessageTypes::MassQuote); + msg.add ("hello",5); + stream.close (); + + auto str = stream.to_human_str (); + REQUIRE (str.compare (0,strlen ("8=FIX.4.2|9=000109|"), + "8=FIX.4.2|9=000109|")==0); + } + + SECTION("checksum_is_correct") + { + char target [] = + "8=FIX.4.2\0019=000109\00135=i\00134=000000000\001369=000000000\001" + "52=20150427-19:05:18.923\00149=O2TESTER\00150=BTS\00156=BTS\00157=BTS\001" + "142=US,NC\00158=hello\001"; + + uint32_t byte_sum = 0; + for (auto c : target) + byte_sum += c; + + FIXEncodingStream stream (get_jetfix_cfg ()); + auto &msg = stream.new_msg (MessageTypes::MassQuote); + msg.add ("hello",5); + stream.close (); + auto chk = stream.msg_map ()[0].checksum; + REQUIRE (chk==byte_sum); + } + + SECTION("stamp_works") + { + FIXEncodingStream stream (get_jetfix_cfg ()); + auto &msg = stream.new_msg (MessageTypes::MassQuote); + msg.add ("hello",5); + stream.close (); + stream.stamp (1,1000,"20150427-19:05:18.922"); + + std::string lit = + "8=FIX.4.2\0019=000109\00135=i\00134=000000002\001369=000001000\001" + "52=20150427-19:05:18.922\00149=O2TESTER\00150=BTS\00156=BTS\00157=BTS\001" + "142=US,NC\00158=hello\001"; + uint32_t lit_checksum = 0; + for (auto c : lit) + lit_checksum += c; + std::ostringstream s; + s << lit << "10=" << std::setfill('0') << std::setw(3) << (lit_checksum % 256) << "\001"; + lit = s.str (); + REQUIRE (lit==stream.to_str ()); + } + +} + + +}} + diff --git a/src/server/sparkd/jetfix/tests/fix_fragment_tests.cpp b/src/server/sparkd/jetfix/tests/fix_fragment_tests.cpp new file mode 100644 index 000000000..01b6b358a --- /dev/null +++ b/src/server/sparkd/jetfix/tests/fix_fragment_tests.cpp @@ -0,0 +1,199 @@ +#include "catch/catch.h" + +#include "server/sparkd/jetfix/codec/FIXFragment.h" +#include "server/sparkd/jetfix/codec/Fields.h" +#include "lib/utils/datetime.h" + +namespace bts { +namespace jetfix { + +TEST_CASE ("JetFIX Field2Tests", "[jetfix]") +{ + SECTION("char_tests") + { + char out[] = {0,0}; + typedef CharField<123> Field; + + REQUIRE (Field::max_value_size ('A')==1); + auto result = Field::write (out,'A'); + REQUIRE (Field::tag_s==std::string ("123")); + REQUIRE (Field::tag_sz==3); + REQUIRE (Field::tag_checksum=='1'+'2'+'3'); + REQUIRE (result.first==1); + REQUIRE (result.second=='A'); + REQUIRE (out[0]=='A'); + REQUIRE (out[1]==0); + } + + SECTION("bool_tests") + { + char out[] = {0,0}; + typedef BoolField<9> Field; + + REQUIRE (Field::max_value_size (true)==1); + REQUIRE (Field::tag_s==std::string ("9")); + REQUIRE (Field::tag_sz==1); + REQUIRE (Field::tag_checksum=='9'); + + auto result = Field::write (out,true); + REQUIRE (result.first==1); + REQUIRE (result.second=='Y'); + REQUIRE (out[0]=='Y'); + REQUIRE (out[1]==0); + + result = Field::write (out,false); + REQUIRE (result.first==1); + REQUIRE (result.second=='N'); + REQUIRE (out[0]=='N'); + REQUIRE (out[1]==0); + } + + SECTION("float_tests") + { + std::string out (20, ' '); + typedef FloatField<123> Field; + + REQUIRE (Field::max_value_size (1.2,6)==6); + + auto result = Field::write (&out[0],1.2,6); + REQUIRE (result.first==6); + REQUIRE (result.second=='0'+'0'+'0'+'1'+'.'+'2'); + REQUIRE (out.compare (0,6,"0001.2") == 0); + REQUIRE (out[6]==' '); + } + + SECTION("int_tests") + { + std::string out (20, ' '); + typedef IntField<123> Field; + + REQUIRE(Field::max_value_size (12,3)==3); + + auto result = Field::write (&out[0],12,3); + REQUIRE (result.first==3); + REQUIRE (result.second=='0'+'1'+'2'); + REQUIRE (out.compare (0,3,"012") == 0); + REQUIRE (out[4]==' '); + } + + SECTION("string_tests") + { + std::string out (20, ' '); + typedef StringField<123> Field; + + REQUIRE(Field::max_value_size ("hello")==5); + + auto result = Field::write (&out[0],"hello"); + REQUIRE (result.first==5); + REQUIRE (result.second=='h'+'e'+'l'+'l'+'o'); + REQUIRE (out.compare (0,5,"hello") == 0); + REQUIRE (out[6]==' '); + } + + SECTION("ptime_tests") + { + std::string out (50, ' '); + typedef TimeField<123> Field; + + const char*t_s = "20150427-19:05:18.922"; + auto result = Field::write (&out[0],datetime::parse_FIX_timestamp (t_s)); + REQUIRE (result.first==strlen (t_s)); + REQUIRE (out.compare (0,strlen (t_s),t_s) == 0); + REQUIRE (out[strlen (t_s)]==' '); + } + +} + + +TEST_CASE ("JetFIX FIXFragmentTests" , "[jetfix]") +{ + SECTION("encoding_works") + { + FIXFragment g (0); + g.add> ('a'); + g.add> (1.2,6); + g.add> ("hello"); + g.add> (123,3); + const char*t_s = "20150427-19:05:18.922"; + g.add> (datetime::parse_FIX_timestamp (t_s)); + g.add> (true); + + std::string out = g.to_str (); + REQUIRE (out == + "1=a" "\001" + "2=0001.2" "\001" + "3=hello" "\001" + "4=123" "\001" + "5=20150427-19:05:18.922" "\001" + "6=Y" "\001"); + + uint32_t checksum = 0; + for (auto c : out) + checksum +=c; + REQUIRE (checksum==g.checksum ()); + } + + SECTION("bookmarking_works") + { + const char*t_s1 = "20150427-19:05:18.922"; + const char*t_s2 = "20150427-19:05:18.923"; + + FIXFragment g; + auto c_bm = g.add> ('a'); + auto f_bm = g.add> (1.2,6); + auto s_bm = g.add> ("hello"); + auto i_bm = g.add> (123,3); + auto t_bm = g.add> (datetime::parse_FIX_timestamp (t_s1)); + auto b_bm = g.add> (true); + + g.overwrite (c_bm,'b'); + g.overwrite (f_bm,1.3,6); + g.overwrite (s_bm,"olleh"); + g.overwrite (i_bm,321,3); + g.overwrite (t_bm,(datetime::parse_FIX_timestamp (t_s2))); + g.overwrite (b_bm,false); + + std::string out = g.to_str (); + REQUIRE (out== "1=b" "\001" + "2=0001.3" "\001" + "3=olleh" "\001" + "4=321" "\001" + "5=20150427-19:05:18.923" "\001" + "6=N" "\001"); + + uint32_t checksum = 0; + for (auto c : out) + checksum +=c; + REQUIRE (checksum==g.checksum ()); + } + + SECTION("append_works") + { + FIXFragment g1;; + g1.add> ('a'); + g1.add> (1.2,6); + g1.add> ("hello"); + auto chk1 = g1.checksum (); + + FIXFragment g2; + g2.add> ("goodbye"); + g2.add> (321,3); + auto chk2 = g2.checksum (); + + g1.append (g2); + + std::string out = g1.to_str (); + REQUIRE (out== "1=a" "\001" + "2=0001.2" "\001" + "3=hello" "\001" + "4=goodbye" "\001" + "5=321" "\001"); + + REQUIRE (chk1+chk2==g1.checksum ()); + } + +} + + +}} + diff --git a/src/server/sparkd/jetfix/tests/jetfix_performance_tests.cpp b/src/server/sparkd/jetfix/tests/jetfix_performance_tests.cpp new file mode 100644 index 000000000..b5f5ddb32 --- /dev/null +++ b/src/server/sparkd/jetfix/tests/jetfix_performance_tests.cpp @@ -0,0 +1,261 @@ +#include "catch/catch.h" + +#include "server/sparkd/jetfix/codec/Fields.h" +#include "server/sparkd/jetfix/codec/Group.h" +#include "server/sparkd/jetfix/codec/FIXMsg.h" +#include "server/sparkd/jetfix/codec/FieldDefinitions.h" +#include "server/sparkd/jetfix/connection/IConnectionObserver.h" +#include "server/sparkd/jetfix/common/InMsgBuffer.h" +#include "server/sparkd/jetfix/codec/FIXParser.h" +#include "server/sparkd/jetfix/codec/FIXFragment.h" +#include "lib/utils/TimeStamp.h" + +namespace bts { +namespace jetfix { + + +TEST_CASE ("JetFIX Performance Tests", "[jetfix] [benchmark]") +{ + const char* mq_char = + "8=FIX.4.2\001" + "9=305\001" + "35=i\001" + "34=61\001" + "49=1U7001N\001" + "50=bluestream\001" + "52=20150427-19:29:40.610\001" + "56=CME\001" + "57=G\001" + "142=US,NC\001" + + "117=58\001" + "204=1\001" + "1028=N\001" + "7928=bluestream\001" + "9702=1\001" + "9771=TESTACC\001" + "296=1\001" + "302=1\001" + "304=2\001" + "295=2\001" + + "299=10857\001" + "55=EZ\001" + "167=OPT\001" + "107=ESK5 C1855\001" + "132=25200\001" + "133=25350\001" + "134=1\001" + "135=1\001" + + "299=10856\001" + "55=EZ\001" + "167=OPT\001" + "107=ESK5 C1890\001" + "132=25201\001" + "133=25351\001" + "134=2\001" + "135=2\001" + "10=002\001"; + + SECTION("decode_jetfix") + { + std::string mqs (mq_char); + + using namespace fix; + typedef FIXMsg MassQuote; + MassQuote mq; + + std::vector times; + for (int i = 0; i < 10000;i++) { + auto start = utils::TimeStamp::now (); + mq.read (mqs); + auto stop = utils::TimeStamp::now (); + times.push_back ((stop-start).microseconds ()); + } + + std::sort (times.begin (),times.end ()); + std::cout << "\tdecode jetfix t: " << times[times.size ()/2] << "us\n "; + } + + SECTION("decode_with_parser") + { + auto fix = std::make_shared (mq_char); + + using namespace fix; + typedef FIXMsg MassQuote; + MassQuote mq; + FIXParser parser; + std::vector result; + result.reserve (10000); + + std::vector times; + for (int i = 0; i < 10000;i++) { + auto start = utils::TimeStamp::now (); + result.emplace_back (parser.parse (InMsgBuffer (fix->data (),fix->size (),fix))); + auto stop = utils::TimeStamp::now (); + times.push_back ((stop-start).microseconds ()); + } + + std::sort (times.begin (),times.end ()); + std::cout << "\tdecode_with_parser t: " << times[times.size ()/2] << "us\n "; + } + + SECTION("encode_jetfix") + { + std::vector times; + for (int i = 0; i < 1000;i++) { + + auto start = utils::TimeStamp::now (); + for (int n = 0; n < 100;n++) { + FIXFragment g; + using namespace fix; + g.add("58",sizeof("58")-1); + g.add(1); + g.add(true); + g.add("bluestream",sizeof("bluestream")-1); + g.add('1'); + g.add("TESTACC",sizeof("TESTACC")-1); + + g.add(1); + g.add("302",sizeof("302")-1); + g.add(2); + + g.add (2); + + g.add("10857",sizeof("10857")-1); + g.add("55",sizeof("55")-1); + g.add("ESK5 C1855",sizeof("ESK5 C1855")-1); + g.add("OPT",sizeof("OPT")-1); + g.add(25200.0); + g.add(1); + g.add(25200.0); + g.add(1); + + g.add("10857",sizeof("10857")-1); + g.add("55",sizeof("55")-1); + g.add("ESK5 C1855",sizeof("ESK5 C1855")-1); + g.add("OPT",sizeof("OPT")-1); + g.add(25200.0); + g.add(1); + g.add(25200.0); + g.add(1); + + g.add(2,3); + } + auto stop = utils::TimeStamp::now (); + times.push_back ((stop-start).microseconds ()); + } + std::sort (times.begin (),times.end ()); + std::cout << "\tencode_jetfix t: " << times[times.size ()/2] / 100 << "us\n "; + } + + SECTION("templated_encoding") + { + using namespace fix; + std::vector times; + FIXFragment header; + header.add("58",sizeof("58")-1); + header.add(1); + header.add(true); + header.add("bluestream",sizeof("bluestream")-1); + header.add('1'); + header.add("TESTACC",sizeof("TESTACC")-1); + header.add(1); + header.add("302",sizeof("302")-1); + + FIXFragment qt1; + auto qt1_quote_entry_id = qt1.add("10857",sizeof("10857")-1); + qt1.add("55",sizeof("55")-1); + qt1.add("ESK5 C1855",sizeof("ESK5 C1855")-1); + qt1.add("OPT",sizeof("OPT")-1); + + auto qt1_bid_px = qt1.add(25200.0); + auto qt1_bid_sz = qt1.add(1); + auto qt1_offer_px = qt1.add(25200.0); + auto qt1_offer_sz = qt1.add(1); + + FIXFragment qt2; + auto qt2_quote_entry_id = qt2.add("10857",sizeof("10857")-1); + qt2.add("55",sizeof("55")-1); + qt2.add("ESK5 C1855",sizeof("ESK5 C1855")-1); + qt2.add("OPT",sizeof("OPT")-1); + + auto qt2_bid_px = qt2.add(25200.0); + auto qt2_bid_sz = qt2.add(1); + auto qt2_offer_px = qt2.add(25200.0); + auto qt2_offer_sz = qt2.add(1); + + FIXFragment g; + for (int i = 0; i < 1000;i++) { + auto start = utils::TimeStamp::now (); + for (int n = 0; n < 100;n++) { + g.reset (); + g.append (header); + g.add(2); + g.add (2); + + qt1.overwrite (qt1_quote_entry_id,"10857",sizeof("10857")-1); + qt1.overwrite (qt1_bid_px,25200.0); + qt1.overwrite (qt1_bid_sz,1); + qt1.overwrite (qt1_offer_px,25200.0); + qt1.overwrite (qt1_offer_sz,1); + g.append (qt1); + + qt2.overwrite (qt2_quote_entry_id,"10857",sizeof("10857")-1); + qt2.overwrite (qt2_bid_px,25200.0); + qt2.overwrite (qt2_bid_sz,1); + qt2.overwrite (qt2_offer_px,25200.0); + qt2.overwrite (qt2_offer_sz,1); + g.append (qt2); + + g.add(2,3); + } + auto stop = utils::TimeStamp::now (); + times.push_back ((stop-start).microseconds ()); + } + std::sort (times.begin (),times.end ()); + std::cout << "\ttemplated_encoding t: " << times[times.size ()/2]/100 << "us\n "; + } + +} + + +}} + diff --git a/src/server/sparkd/jetfix/tests/store_tests.cpp b/src/server/sparkd/jetfix/tests/store_tests.cpp new file mode 100644 index 000000000..081152a41 --- /dev/null +++ b/src/server/sparkd/jetfix/tests/store_tests.cpp @@ -0,0 +1,283 @@ +#include "catch/catch.h" + +#include "server/sparkd/jetfix/codec/FIXEncodingStream.h" +#include "server/sparkd/jetfix/common/InMsgBuffer.h" +#include "server/sparkd/jetfix/common/OutFIX.h" +#include "server/sparkd/jetfix/common/OutFIXPtr.h" +#include "server/sparkd/jetfix/store/FileStore.h" +#include "server/sparkd/jetfix/store/session_name.h" +#include "lib/utils/TempDir.h" +#include "lib/utils/TimeStamp.h" +#include "lib/utils/datetime.h" + +#include +#include +#include + +namespace bts { +namespace jetfix { + +TEST_CASE ("JetFIX SessionNameTests", "[jetfix]") +{ + SECTION("greg_weekday_works") + { + const char*days [] = {"sun","Monday","tuesday","Wedn","Thursday","Friday","sat"}; + + int i =0; + for (auto d : days) + REQUIRE (parse_weekday (d)==boost::gregorian::greg_weekday (i++)); + } + + SECTION("closest_before_works") + { + auto same_day_before = closest_before (datetime::make_ptime (2015,7,17,17,0,0,0), + boost::gregorian::Friday, + boost::posix_time::time_duration(16, 59, 59)); + REQUIRE (datetime::FIX_date_time_string (same_day_before)=="20150717-16:59:59"); + + auto same_day_after = closest_before (datetime::make_ptime (2015,7,17,17,0,0,0), + boost::gregorian::Friday, + boost::posix_time::time_duration(17,0,1)); + REQUIRE (datetime::FIX_date_time_string (same_day_after)=="20150710-17:00:01"); + + auto another_day_same_week = closest_before (datetime::make_ptime (2015,7,17,17,0,0,0), + boost::gregorian::Thursday, + boost::posix_time::time_duration(16, 59, 59)); + REQUIRE (datetime::FIX_date_time_string (another_day_same_week)=="20150716-16:59:59"); + + auto another_day_previous_week = closest_before (datetime::make_ptime (2015,7,17,17,0,0,0), + boost::gregorian::Saturday, + boost::posix_time::time_duration(16, 59, 59)); + REQUIRE (datetime::FIX_date_time_string (another_day_previous_week)=="20150711-16:59:59"); + + } + + SECTION("in_session_works") + { + REQUIRE (in_session (datetime::make_ptime (2015,7,17,17,0,0,0), + boost::gregorian::Sunday, + boost::posix_time::time_duration(16, 59, 59), + + boost::gregorian::Friday, + boost::posix_time::time_duration(17, 0, 1))); + + REQUIRE (!in_session (datetime::make_ptime (2015,7,17,17,0,0,0), + boost::gregorian::Sunday, + boost::posix_time::time_duration(16, 59, 59), + + boost::gregorian::Friday, + boost::posix_time::time_duration(16, 59, 59))); + + REQUIRE (in_session (datetime::make_ptime (2015,7,17,17,0,0,0), + boost::posix_time::time_duration(16, 59, 59), + boost::posix_time::time_duration(17, 59, 59))); + + REQUIRE (!in_session (datetime::make_ptime (2015,7,17,17,0,0,0), + boost::posix_time::time_duration(0,0,0), + boost::posix_time::time_duration(16, 0,0))); + + REQUIRE (in_session (datetime::make_ptime (2015,7,17,17,0,0,0), + boost::posix_time::time_duration(17, 50, 59), + boost::posix_time::time_duration(17, 45, 59))); + + REQUIRE (!in_session (datetime::make_ptime (2015,7,17,17,47,0,0), + boost::posix_time::time_duration(17, 50, 59), + boost::posix_time::time_duration(17, 45, 59))); + } + + SECTION("session_name_works") + { + REQUIRE (session_name (datetime::make_ptime (2015,7,17,17,0,0,0), + boost::gregorian::Sunday, + boost::posix_time::time_duration(16, 59, 59), + + boost::gregorian::Friday, + boost::posix_time::time_duration(17, 0, 1)) == + "2015-07-12"); + + + REQUIRE (session_name (datetime::make_ptime (2015,7,17,17,0,0,0), + boost::gregorian::Sunday, + boost::posix_time::time_duration(16, 59, 59), + + boost::gregorian::Friday, + boost::posix_time::time_duration(16, 59, 59)) == ""); + + REQUIRE (session_name (datetime::make_ptime (2015,7,17,17,0,0,0), + boost::posix_time::time_duration(16, 59, 59), + boost::posix_time::time_duration(17, 59, 59)) == + "2015-07-17"); + + REQUIRE (session_name (datetime::make_ptime (2015,7,17,18,0,0,0), + boost::posix_time::time_duration(16, 59, 59), + boost::posix_time::time_duration(17, 59, 59)) == + ""); + + REQUIRE (session_name (datetime::make_ptime (2015,7,17,17,0,0,0), + boost::posix_time::time_duration(17, 50, 59), + boost::posix_time::time_duration(17, 45, 59)) == + "2015-07-16"); + + REQUIRE (session_name (datetime::make_ptime (2015,7,17,17,51,0,0), + boost::posix_time::time_duration(17, 50, 59), + boost::posix_time::time_duration(17, 45, 59)) == + "2015-07-17"); + + REQUIRE (session_name (datetime::make_ptime (2015,7,17,17,47,0,0), + boost::posix_time::time_duration(17, 50, 59), + boost::posix_time::time_duration(17, 45, 59)) == + ""); + + } + +} + + +TEST_CASE ("JetFIX FileStoreTests", "[jetfix]") +{ + SECTION("empty_store_works") + { + config::JetFIX::Connection c; + c.set_name ("test"); + c.set_sender_compid ("CME"); + c.set_sender_subid ("G"); + c.set_target_compid ("1U7001N"); + c.set_target_subid ("BLUESTREAM"); + + c.set_start_time ("00:00:00"); + c.set_end_time ("23:59:59"); + + utils::TempDir td; + + FileStore fs (c,td.name ()); + REQUIRE (fs.session_active ()); + REQUIRE (fs.replay ().size ()==0); + REQUIRE (fs.last_seq_num ()==0); + REQUIRE (fs.last_seq_num_processed ()==0); + } + + SECTION("in_msg_is_stored") + { + auto td = std::make_shared (); + config::JetFIX::Connection c; + c.set_name ("test"); + c.set_sender_compid ("CME"); + c.set_sender_subid ("G"); + c.set_target_compid ("1U7001N"); + c.set_target_subid ("BLUESTREAM"); + + c.set_start_time ("00:00:00"); + c.set_end_time ("23:59:59"); + + FIXEncodingStream stream (c); + auto &buf = stream.new_msg (MessageTypes::Logon); + buf.add ("US,NC"); + buf.add (0); + buf.add(30); + buf.add ("O2"); + buf.add ("1.0"); + buf.add ("BTS"); + stream.close (); + stream.stamp (2,3,datetime::FIX_timestamp (datetime::now ())); + auto fix = stream.to_str (); + + { + FileStore fs (c,td->name ()); + fs.store (InMsgBuffer (fix)); + fs.set_last_msg_processed (3); + } + + FileStore fs (c,td->name ()); + auto replay = fs.replay (); + REQUIRE (replay.size ()==1); + REQUIRE (replay[0].type ()==MessageTypes::Logon); + REQUIRE (replay[0].to_str ()==fix); + REQUIRE (fs.last_seq_num ()==0); + REQUIRE (fs.last_seq_num_processed ()==3); + } + + SECTION("out_msg_is_stored") + { + auto td = std::make_shared (); + config::JetFIX::Connection c; + c.set_name ("test"); + c.set_sender_compid ("CME"); + c.set_sender_subid ("G"); + c.set_target_compid ("1U7001N"); + c.set_target_subid ("BLUESTREAM"); + + c.set_start_time ("00:00:00"); + c.set_end_time ("23:59:59"); + + auto msg_vec = std::make_shared(c); + auto &stream = msg_vec->fix.back (); + auto &buf = stream.new_msg (MessageTypes::Logon); + buf.add ("US,NC"); + buf.add (0); + buf.add(30); + buf.add ("O2"); + buf.add ("1.0"); + buf.add ("BTS"); + stream.close (); + stream.stamp (2,3,datetime::FIX_timestamp (datetime::now ())); + + { + FileStore fs (c,td->name ()); + fs.store (msg_vec); + } + + FileStore fs (c,td->name ()); + auto replay = fs.replay (); + REQUIRE (replay.size ()==0); + REQUIRE (fs.last_seq_num ()==3); + REQUIRE (fs.last_seq_num_processed ()==0); + } + + SECTION("test_performance") + { + auto td = std::make_shared (); + config::JetFIX::Connection c; + c.set_name ("test"); + c.set_sender_compid ("CME"); + c.set_sender_subid ("G"); + c.set_target_compid ("1U7001N"); + c.set_target_subid ("BLUESTREAM"); + + c.set_start_time ("00:00:00"); + c.set_end_time ("23:59:59"); + + auto start = utils::TimeStamp::now (); + { + FileStore fs (c,td->name ()); + for (int i = 0; i < 10000;i++) { + auto msg_vec = std::make_shared>(); + msg_vec->emplace_back (c); + auto &stream = msg_vec->back (); + auto &buf = stream.new_msg (MessageTypes::Logon); + buf.add ("US,NC"); + buf.add (0); + buf.add(30); + buf.add ("O2"); + buf.add ("1.0"); + buf.add ("BTS"); + stream.close (); + stream.stamp (i,3,datetime::FIX_timestamp (datetime::now () - boost::gregorian::days (1))); + fs.store (InMsgBuffer (stream.to_str ())); + } + } + auto stop = utils::TimeStamp::now (); + std::cout << "\tjetfix store gen: " << (stop-start).milliseconds () << "ms\n"; + + start = utils::TimeStamp::now (); + FileStore fs (c,td->name ()); + stop = utils::TimeStamp::now (); + std::cout << "\tjetfix store load: " << (stop-start).milliseconds () << "ms\n"; + } + +} + +}} + + + + diff --git a/src/server/sparkd/main.cpp b/src/server/sparkd/main.cpp new file mode 100644 index 000000000..283464929 --- /dev/null +++ b/src/server/sparkd/main.cpp @@ -0,0 +1,20 @@ +#include "lib/os/os.h" +#include "lib/logging/logging.h" +#include "lib/utils/catch_test_runner.h" + +int main( int argc, char*argv [] ) +{ + using namespace bts; + os::set_thread_name("main"); + + if (argc > 1 && std::string(argv[1]) == "test") + bts::run_tests(--argc, ++argv); + else { + log::LoggingConfigPoller poller; + LOG(Gen,Warn, "Nothing here yet"); + } + return 0; +} + + + diff --git a/src/server/sparkd/md/Diag/MDPriceLantencyTester.cpp b/src/server/sparkd/md/Diag/MDPriceLantencyTester.cpp new file mode 100644 index 000000000..a2dabb22c --- /dev/null +++ b/src/server/sparkd/md/Diag/MDPriceLantencyTester.cpp @@ -0,0 +1,69 @@ +#ifndef NO_CPP0X +#include "MDPriceLantencyTester.h" +#include "../MDConsumer.h" +#include "lib/utils/Dispatcher.h" + +#include +#include +#include + +using namespace std; +using namespace bts::md2; +namespace bts { +namespace md { + +struct HandleData { + MDConsumerHandle handle; + utils::TimeStamp time; + utils::TimeStamp last_price_change; + float bid, ask; +}; + +void run_md_price_latency_test(UserSymbol symbol) { + set symbols; + symbols.insert(symbol); + MDConsumer consumer(symbols); + + vector handles; + + consumer.walk_security_list([&](const Security&, MDConsumerHandle handle) { + HandleData entry = { handle, TimeStamp(), TimeStamp(), 0, 0 }; + if (!entry.handle.is_initialized()) { + cout << "Please start the md publisher before running this command." << endl; + return; + } + handles.push_back(entry); + }); + + cout << "key, sequence, handover, delta, bid price, ask price" << endl; + while (true) { + for(auto& entry : handles) { + PriceSize bid, ask; + if (entry.handle.get_bbo(entry.time, bid, ask)) { + auto bid_price = bid.price(); + auto ask_price = ask.price(); + if (bid_price != entry.bid || ask_price != entry.ask) { + auto now = TimeStamp::now(); + auto handover = (now - entry.time); + auto delta = (now - entry.last_price_change); + entry.last_price_change = entry.time; + entry.bid = bid_price; + entry.ask = ask_price; + utils::Dispatcher::background().post([=]() { + cout << + entry.handle.key() << ',' + << entry.handle.sequence() << ',' + << handover.microseconds() << ',' + << delta.microseconds() << ',' + << bid_price << ',' << ask_price + << endl; + }); + } + } + } + } +} + + +}}//namespace +#endif diff --git a/src/server/sparkd/md/Diag/MDPriceLantencyTester.h b/src/server/sparkd/md/Diag/MDPriceLantencyTester.h new file mode 100644 index 000000000..118562faf --- /dev/null +++ b/src/server/sparkd/md/Diag/MDPriceLantencyTester.h @@ -0,0 +1,15 @@ +#pragma once + +#include "lib/utils/bts_export.h" +#include "../common.h" +#include "../UserSymbol.h" +#include + +namespace bts { +namespace md { + +BTS_EXPORT void run_md_price_latency_test(md2::UserSymbol); + + +}}//namespace + diff --git a/src/server/sparkd/md/Diag/ShmDumper.cpp b/src/server/sparkd/md/Diag/ShmDumper.cpp new file mode 100644 index 000000000..213868917 --- /dev/null +++ b/src/server/sparkd/md/Diag/ShmDumper.cpp @@ -0,0 +1,248 @@ +#ifndef NO_CPP0X + +#include "ShmDumper.h" + +#include "lib/utils/SharedList.h" + +#include "server/sparkd/md/Structure/structure.h" +#include "lib/logging/logging.h" + +#include +#include + +namespace bts { +namespace md { +using namespace utils; +using namespace md2; +using namespace std; + +ShmDumper::ShmDumper(const string& name) : _name(name) +{} + +ShmDumper::~ShmDumper() {} + +double dump(TimeStamp time) { + // stringstream temp; + // temp << (TimeStamp::now() - time).seconds(); + // return temp.str(); + return (TimeStamp::now() - time).seconds(); +} +string dump(PriceSize ps) { + stringstream res; + res << "{"; + res << "'price': " << ps.price() << ", "; + res << "'size': " << ps.size(); + res << "}"; + return res.str(); +} + +string dump(const SingleWriterPriceSizeStore& pss) { + return dump(pss.load()); +} + +string dump_header(const ColdHeader& header) { + stringstream res; + res << "{"; + res << "'count': " << header.count << ", "; + res << "'segment_count': " << header.segment_count << ", "; + res << "'create_time': " << dump(header.create_time) << ", "; + res << "'heartbeat': " << dump(header.heartbeat) << ", "; + res << "'security_count': " << header.security_count << ", "; + res << "'has_depth': " << std::boolalpha << header.has_depth << ", "; + res << "'platform': '" << Platform::type_Name(header.platform) << "', "; + res << "'trade_count': " << header.trade_count << ", "; + res << "'coconut_status_count': " << header.coconut_status_count << ", "; + res << "'exchange_status_count': " << header.exchange_status_count << ", "; + res << "'rfq_count': " << header.rfq_count << ", "; + res << "'customer_bbo_count': " << header.customer_bbo_count; + res << "}"; + return res.str(); +} +void write_date_string(uint32_t year, uint32_t month, uint32_t day, std::ostringstream& result) { + result << std::setw(2) << std::setfill('0') << month << "/" + << std::setw(2) << std::setfill('0') << day << "/" + << year - 2000; +} +string dump_security(const FixedSecurity& security) { + ostringstream res; + res << "{"; + res << "'key': " << security.key << ", "; + res << "'symbol': '" << security.symbol() << "', "; + res << "'type': '" << SecurityType::type_Name(security.type) << "', "; + res << "'exchange': '" << Exchange::type_Name(security.exchange) << "', "; + res << "'alt_symbol': '" << security.alt_symbol() << "', "; + res << "'cfi_code': '" << security.cfi_code() << "', "; + res << "'security_desc': '" << security.security_desc() << "', "; + res << "'platform_id': '" << security.platform_id() << "', "; + res << "'display_factor': " << (int)security.display_factor << ", "; + res << "'strike_display_factor': " << (int)security.strike_display_factor << ", "; + res << "'numeric_id': " << security.numeric_id << ", "; + res << "'contract_size': " << security.contract_size << ", "; + res << "'cboe_class_key': " << security.cboe_class_key << ", "; + res << "'cme_vtt': " << security.cme_vtt << ", "; + res << "'cme_display_decimals': " << security.cme_display_decimals << ", "; + res << "'cme_denominator_sub_fraction': " << security.cme_denominator_sub_fraction << ", "; + res << "'cme_min_price_increment': " << security.cme_min_price_increment << ", "; + + switch(security.type) { + case SecurityType::STOCK: break; + case SecurityType::FUTURE: { + auto* psec = static_cast(&security); + res << "'expiration': '"; + write_date_string(psec->expiration_year, psec->expiration_month, psec->expiration_day, res); + res << "' "; + break; + } + case SecurityType::OPTION: { + auto* psec = static_cast(&security); + res << "'expiration': '"; + write_date_string(psec->expiration_year, psec->expiration_month, psec->expiration_day, res); + res << "', "; + res << "'strike': " <strike << ", "; + res << "'callput': '" << PutOrCall::type_Name(psec->call_put) << "' "; + break; + } + case SecurityType::SPREAD: { + auto* psec = static_cast(&security); + res << "'legs': ["; + for (size_t x = 0; x < psec->leg_count(); ++x) { + auto& leg = psec->legs[x]; + if (x > 0) res << ","; + res << "{ 'key': " << leg.key + << ", 'quantity': " << leg.leg_quantity + << ", 'leg_price': " << leg.leg_price + << ", 'delta': " << leg.leg_delta + << ", 'side': '" << Side::type_Name(leg.leg_side) + << "' }"; + } + res << "]"; + break; + } + default: break; + } + + res << "}"; + return res.str(); +} + +string dump(datetime::Date date) { + stringstream res; + res << "{"; + res << "'year': " << date.year << ", "; + res << "'month': " << (int)date.month << ", "; + res << "'day': " << (int)date.day; + res << "}"; + return res.str(); +} + +bool has_value(PriceSize p) { return p.size() > 0; } +bool has_value(const SingleWriterPriceSizeStore& p) { return has_value(p.load()); } + +string dump_entry(const ColdEntry& entry) { + stringstream res; + res << "{"; + res << "'security': " << dump_security(entry.security) << ", "; + if (has_value(entry.last_trade)) res << "'last_trade': " << dump(entry.last_trade) << ", "; + if (entry.total_volume > 0) res << "'total_volume': " << entry.total_volume << ", "; + if (entry.last_trade_time > 0) res << "'last_trade_time_ll': " << entry.last_trade_time << ", "; + if (has_value(entry.customer_bid)) res << "'customer_bid': " << dump(entry.customer_bid) << ", "; + if (has_value(entry.customer_ask)) res << "'customer_ask': " << dump(entry.customer_ask) << ", "; + if (!os::is_nan(entry.misc.settlement_price)) res << "'settlement_price': " << entry.misc.settlement_price << ", "; + if (entry.misc.settlement_date.year > 0) res << "'settlement_date': " << dump(entry.misc.settlement_date) << ", "; + if (!os::is_nan(entry.misc.previous_settlement_price)) res << "'previous_settlement_price': " << entry.misc.previous_settlement_price << ", "; + if (entry.misc.previous_settlement_date.year > 0) res << "'previous_settlement_date': " << dump(entry.misc.previous_settlement_date) << ", "; + res << "'deleted': " << std::boolalpha << entry.deleted << " "; + res << "}"; + return res.str(); +} + +string dump_entry(const HotDepthEntry& entry) { + stringstream res; + res << "{ "; + res << "'timestamp': " << dump(entry.timestamp) << ", "; + res << "'key': " << entry.key << ", "; + + res << "'bids': ["; + for (uint32_t x = 0; x < COCONUT_MAX_DEPTH; ++x) { + res << dump(entry.bid(x)) << ", "; + } + res << "], "; + + res << "'asks': ["; + for (uint32_t x = 0; x < COCONUT_MAX_DEPTH; ++x) { + res << dump(entry.ask(x)) << ", "; + } + res << "], "; + + res << "'status': '" << CoconutSecurityStatus::type_Name((CoconutSecurityStatus::type)entry.status) << "', "; + res << "'sequence': '" << entry.sequence << "', "; + res << " }"; + return res.str(); +} + +string dump_entry(const HotEntry& entry) { + stringstream res; + res << "{ "; + res << "'timestamp': " << dump(entry.timestamp) << ", "; + res << "'key': " << entry.key << ", "; + res << "'bid': " << dump(entry.bid) << ", "; + res << "'ask': " << dump(entry.ask) << ", "; + res << "'status': '" << CoconutSecurityStatus::type_Name((CoconutSecurityStatus::type)entry.status) << "', "; + res << "'sequence': '" << entry.sequence << "', "; + res << " }"; + return res.str(); +} + +void ShmDumper::dump_cold() { + SharedListReader reader(_name+"_cold"); + if (!reader.is_initialized()) { LOG(MarketData,Info,"Unable to initialize section {}", _name); return; } + const auto& header = reader.header(); + // dump the header + cout << "{ " << endl; + cout << "'header': " << dump_header(header) << "," << endl; + cout << "'entries': [" << endl; + bool first = true; + reader.all_entries([&](const ColdEntry& entry) { + if (!first) { cout << "," << endl; } + cout << dump_entry(entry); + first = false; + }); + cout << endl << "]}" << endl; +} + +template +void dump_hot_t(SharedListReader& reader) { + if (!reader.is_initialized()) { LOG(MarketData,Info,"Unable to initialize section "); return; } + auto& header = reader.header(); + cout << "{ " << endl; + cout << "'header': { 'count': " << header.count << ", 'segment_count': " << header.segment_count << ", 'bbo_count': " << header.bbo_count << ", 'depth_count': " << header.depth_count << "}," << endl; + cout << "'entries': [ " << endl; + reader.all_entries([&](const TEntry& entry) { cout << dump_entry(entry) << "," << endl; }); + cout << "] }" << endl; +} + +void ShmDumper::dump_hot() { + SharedListReader cold_reader(_name + "_cold"); + if (!cold_reader.is_initialized()) + { + LOG(MarketData,Info, + "Unable to initialize section {}_cold - needed to " + "determine metadata.",_name); + return; + } + + const auto& header = cold_reader.header(); + if (header.has_depth) { + SharedListReader reader(_name + "_hot"); + dump_hot_t(reader); + } + else { + SharedListReader reader(_name + "_hot"); + dump_hot_t(reader); + } + +} + + +}}//namespace +#endif diff --git a/src/server/sparkd/md/Diag/ShmDumper.h b/src/server/sparkd/md/Diag/ShmDumper.h new file mode 100644 index 000000000..40d50a6a5 --- /dev/null +++ b/src/server/sparkd/md/Diag/ShmDumper.h @@ -0,0 +1,24 @@ +#pragma once +#include "lib/utils/bts_no_copy.h" +#include "lib/utils/bts_export.h" +#include + +namespace bts { +namespace md { + +class BTS_EXPORT ShmDumper +{ +public: + ShmDumper(const std::string& name); + ~ShmDumper(); + + void dump_hot(); + void dump_cold(); + +private: + BTS_NO_COPY(ShmDumper); + std::string _name; +}; + +}}//namespace + diff --git a/src/server/sparkd/md/MDConsumer.cpp b/src/server/sparkd/md/MDConsumer.cpp new file mode 100644 index 000000000..abaa96290 --- /dev/null +++ b/src/server/sparkd/md/MDConsumer.cpp @@ -0,0 +1,681 @@ +#ifndef NO_CPP0X +#include "MDConsumer.h" +#include "MemorySegmentUtils.h" +#include "PlatformLoader.h" +#include "lib/config/config.h" +#include "lib/utils/SharedList.h" +#include "lib/utils/WaitFreeQueue.h" +#include "lib/utils/bts_chrono.h" + +#include "lib/config/protos/MarketData.pb.h" + +#include +#include +#include +#include +#include +#include + +using namespace std; +namespace bts { +namespace md2 { + +/* + * Implementation notes: + * SlowHeader keeps a security count to know when a list has changed + * When a security is made visible increment count (non-atomic) + * When user hits walk_security_list just walk all of them + * When scanning for new securities we can short circuit the search if the security_count hasn't incremented + * + */ + +bool match_type(SecurityType::type val, SecurityType::type b) { + return (val == b || b == SecurityType::UNSET); +} +bool match_symbol(const string& val, const string& b) { + bts_assert_d(!val.empty()); + return b.empty() || (val == b); +} + +typedef shared_ptr PSecurity; +struct SecurityLess { + bool operator()(const PSecurity& left, const PSecurity& right) { + return left->security_key() < right->security_key(); + } +}; + +PSecurity convert_psecurity(const FixedSecurity& security) { + PSecurity psec = make_shared(); + convert_security(security, psec.get()); + return psec; +} + +/////////////////////////////////////////////////////////////////////////////////////////////////// +// Represents one pair of shared memory segments for a single Symbol+type +/////////////////////////////////////////////////////////////////////////////////////////////////// +struct QueueStruct { + utils::WaitFreeQueue _trade_queue; + utils::WaitFreeQueue _rfq_queue; + QueueStruct(const ColdHeader& header): + _trade_queue((void*)header.trade_buffer, BTS_ARRAYSIZE(header.trade_buffer)), + _rfq_queue((void*)header.rfq_buffer, BTS_ARRAYSIZE(header.rfq_buffer)) + {} +}; + +bool is_heart_beat_ok(const ColdHeader &); + +class MDConsumerInst { +public: + MDConsumerInst(const string& symbol, SecurityType::type type, Exchange::type exchange, bool shared) : + _symbol(symbol), + _type(type), + _exchange(exchange), + _security_count(0), + _found_keys(), + _hot(), + _hot_depth(), + _cold(cold_segment_name(symbol, type, exchange, platform_for_exchange(exchange)), shared), + _queues() + { + if (enable_depth(symbol, type)) { + _hot_depth.reset(new utils::SharedListReader(hot_segment_name(symbol, type, exchange, platform_for_exchange(exchange)), shared)); + } + else { + _hot.reset(new utils::SharedListReader(hot_segment_name(symbol, type, exchange, platform_for_exchange(exchange)),shared)); + } + } + ~MDConsumerInst() { } + + template + size_t walk_sec_list(CB cb, const string& symbol="", SecurityType::type type=SecurityType::UNSET) { + size_t result = 0; + _cold.all_entries([&symbol, type, &cb, &result](const ColdEntry& entry) { + // not sure this double check is needed since the caller also filters + if (entry.security.key != invalid_security_key && + match_type(entry.security.type, type) && + match_symbol(entry.security.symbol(), symbol)) { + cb(convert_psecurity(entry.security)); + ++result; + } + }); + return result; + } + + template + const void* find_hot(utils::SharedListReader* phot, security_key key) { + const void* hot_entry = nullptr; + phot->all_entries([&](const T& entry) { + if (entry.key == key) { + hot_entry = &entry; + } + }); + return hot_entry; + } + + bool find_handle(security_key key, const ColdEntry** ppcold, const void** ppvhot) { + *ppcold = nullptr; + *ppvhot = nullptr; + _cold.all_entries([&](const ColdEntry& entry) { + if (entry.security.key == key) { + *ppcold = &entry; + } + }); + + // Only search the hot list if we found cold + if(*ppcold) { + // It is possible that cold is initialized and hot is not yet due to race between process mapped memory being visible + size_t count = 0; + do { + ++count; + *ppvhot = + cold_header().has_depth ? + find_hot(_hot_depth.get(), key) : + find_hot(_hot.get(), key); + if (!*ppvhot) { + std::this_thread::sleep_for(1ms); + if (count == 4096) LOG(MarketData,Error, "Waiting an awfully long time for key: {}",key); + if (count == 60000) { + LOG(MarketData,Error, + "Waited far too long for key: {} security discovery may be hung." + " symbol: {} type: {}", + key, + (*ppcold)->security.symbol(), + SecurityType::type_Name((*ppcold)->security.type)); + return false; + } + } + } while (!*ppvhot); + + return true; + } + + // Nothing found + return false; + } + + void search_handles(map& handles) { + // Walk hot and cold lists and fill in any found handles + if (!_cold.is_initialized()) return; + _cold.all_entries([&](const ColdEntry& entry) { + auto iter = handles.find(entry.security.key); + if (iter != handles.end()) { + iter->second.set_cold(&cold_header(), &entry); + } + }); + // Do the same for hot + if (_hot_depth) { + _hot_depth->all_entries([&](const HotDepthEntry& entry) { + auto iter = handles.find(entry.key); + if (iter != handles.end()) { + iter->second.set_hot(&entry); + } + }); + } + else if (_hot){ + _hot->all_entries([&](const HotEntry& entry) { + auto iter = handles.find(entry.key); + if (iter != handles.end()) { + iter->second.set_hot(&entry); + } + }); + } + } + + template + void scan_new_securities(CB cb) { + // If the ver matches no securities have been added + if (_security_count == cold_header().security_count) return; + _security_count = cold_header().security_count; + + _cold.all_entries([&](const ColdEntry& entry) { + if (entry.security.key != invalid_security_key && _found_keys.count(entry.security.key) == 0) { + _found_keys.insert(entry.security.key); + cb(entry); + } + }); + } + + void forget_security(security_key key) { + _security_count -= _found_keys.erase(key); + } + + template + void walk_old_securities(CB cb) { + // Fire callback only for securities which *have* been seen already + if (_found_keys.empty()) return; + _cold.all_entries([&](const ColdEntry& entry) { + if (entry.security.key != invalid_security_key && _found_keys.count(entry.security.key) != 0) { + cb(entry); + } + }); + } + + uint32_t security_count() { + return cold_header().security_count; + } + + const ColdHeader& cold_header() { return _cold.header(); } + const HotHeader& hot_header() { return (has_depth() ? _hot_depth->header() : _hot->header());} + bool has_depth() { return cold_header().has_depth; } + const string& symbol() const { return _symbol; } + SecurityType::type type() const { return _type; } + Exchange::type exchange() const { return _exchange; } + + // + // Queued data + + int64_t read_trades(google::protobuf::RepeatedPtrField& trades, int64_t excursor) { + if (init_queues()) { + bts_assert(_queues); + utils::WaitFreeQueue::Cursor cursor(_queues->_trade_queue, excursor); + bool overrun; + cursor.read(trades, &overrun); + // Preserve the overrun flag in the message + if (overrun && !trades.empty()) { + trades.begin()->set_gap(true); + } + return cursor.current_position(); + } + return 0; + } + + int64_t read_rfqs(google::protobuf::RepeatedPtrField& rfqs, int64_t extcursor) { + if (init_queues()) { + bts_assert(_queues); + bool overrun; + vector temp_result; + utils::WaitFreeQueue::Cursor cursor(_queues->_rfq_queue, extcursor); + cursor.read(temp_result, &overrun); + // deserialize the rfqs + for (const auto& entry : temp_result) { + rfqs.Add()->ParsePartialFromArray(&entry[0], entry.size()); + } + if (overrun && rfqs.size() > 0) { + rfqs.begin()->set_gap(true); + } + return cursor.current_position(); + } + return 0; + } + + bool is_heart_beating() { + // Channel Heartbeat - not in a tight loop + if (!_cold.is_initialized()) return true; + return is_heart_beat_ok(cold_header()); + } + +private: + + bool init_queues() { + if (_queues) return true; + if (!_cold.is_initialized()) return false; + _queues.reset(new QueueStruct(_cold.header())); + return true; + } + + string _symbol; + SecurityType::type _type; + Exchange::type _exchange; + size_t _security_count; + set _found_keys; + unique_ptr> _hot; + unique_ptr> _hot_depth; + utils::SharedListReader _cold; + unique_ptr _queues; +}; + + +/////////////////////////////////////////////////////////////////////////////////////////////////// +// Pimple for MDConsumer +/////////////////////////////////////////////////////////////////////////////////////////////////// +class MDConsumerImpl { +public: + MDConsumerImpl(set const& symbols, bool shared) { + for (auto& s : symbols) { + if (s.type() == SecurityType::FUTURE) { + _instances.push_back(make_shared(s.symbol(), s.type(), s.exchange(), shared)); + } + } + for (auto& s : symbols) { + if (s.type() != SecurityType::FUTURE) { + _instances.push_back(make_shared(s.symbol(), s.type(), s.exchange(), shared)); + } + } + } + ~MDConsumerImpl() { } + + size_t walk_sec_list(SecurityCallback cb, const string& symbol, SecurityType::type type) { + size_t result = 0; + vector securities; + for (auto& inst : _instances) { + if (match_symbol(inst->symbol(), symbol) && match_type(inst->type(), type)) { + result += inst->walk_sec_list([&](PSecurity security) { securities.push_back(security); }, symbol, type); + } + } + auto complete_securities = fill_leg_details(securities); + auto handles = get_handles(complete_securities); + for (const auto& security : complete_securities) { + auto find_result = handles.find(security->security_key()); + if (find_result != handles.end()) { + cb(*security, find_result->second); + } + } + return result; + } + + MDConsumerHandle get_handle(security_key key) { + const void* hot = nullptr; + const ColdEntry* cold = nullptr; + // Search through the instances for one that has this handle + for (auto& inst : _instances) { + if (inst->find_handle(key, &cold, &hot)) { + MDConsumerHandle handle; + handle.init(inst->cold_header(), *cold, hot); + return handle; + } + } + // If we can't find it return an empty handle. This handle could be later initialized when the security is available. + MDConsumerHandle empty_handle; + return empty_handle; + } + + map get_handles(const vector& securities) { + set keys; + for (const auto& sec : securities) { + keys.insert(sec->security_key()); + } + return get_handles(keys); + } + + map get_handles(const set& keys) { + map result; + if (keys.empty()) return result; + // Initialize the result to match the input + for (security_key key : keys) { + result.insert(std::make_pair(key, MDConsumerHandle())); + } + + for (auto& inst : _instances) { + inst->search_handles(result); + } + + // Erase anything that wasn't found from the resulting map + for (auto it = result.begin(); it != result.end(); ) { + if (!it->second.is_initialized()) it = result.erase(it); + else ++it; + } + + return result; + } + + shared_ptr get_managed_handle(security_key key) { + shared_ptr handle = make_shared(get_handle(key)); + if (!handle->is_initialized()) { + Mutex::scoped_lock lock(_workmutex); + _deferred_handles[key] = handle; + } + return handle; + } + + vector fill_leg_details(const vector& securities, int depth=0) { + vector result; + if (securities.empty() || depth > 64) return result; + // + // Gather ids for legs + set keys; + for (const auto& sec : securities) { + if (sec->security_type() == SecurityType::SPREAD) { + for (const auto& leg : sec->spread().legs()) { + keys.insert(leg.security().security_key()); + } + } + } + // If there are no spreads everything is (by definition) resolved already + if (keys.empty()) return securities; + // + // gather handles for direct legs + auto handles = get_handles(keys); + + // + // Some of the legs will themselves be spreads + vector legs; + legs.reserve(handles.size()); + for (const auto& pair : handles) { + if (pair.second.is_initialized()) { + legs.push_back(make_shared(pair.second.get_normalized_security())); + } + } + // recurse to find details + legs = fill_leg_details(legs, depth+1); + map leg_map; + for (auto& leg : legs) { + leg_map.insert(make_pair(leg->security_key(), move(leg))); + } + + // + // update legs + for (auto& sec : securities) { + if (sec->security_type() == SecurityType::SPREAD) { + for (auto& leg : *sec->mutable_spread()->mutable_legs()) { + auto key = leg.security().security_key(); + auto leg_iter = leg_map.find(key); + if (leg_iter == leg_map.end()) goto next; + bts_assert(leg.security().security_key() == leg_iter->first); + leg.mutable_security()->CopyFrom(*leg_iter->second); + } + sec->mutable_spread()->set_normalized(false); + result.push_back(sec); + } + else { + result.push_back(sec); + } + next: continue; + } + return result; + } + + size_t scan_securities(bool resolve_legs) { + Mutex::scoped_lock lock(_workmutex); + size_t count = 0; + if (_callbacks.empty() && _deferred_handles.empty()) return 0; + // Call each _instance and pass it a callback + for (auto& inst : _instances) { + vector found_securities; + inst->scan_new_securities([&](const ColdEntry& entry) { + // Check if any deferred handles need this security (could be sped up with get_handles() but it isn't hot + auto iter = _deferred_handles.find(entry.security.key); + if (iter != _deferred_handles.end()) { + (*iter->second) = get_handle(entry.security.key); + _deferred_handles.erase(iter); + } + found_securities.push_back(convert_psecurity(entry.security)); + }); + + if (resolve_legs && !found_securities.empty()) { + sort(found_securities.begin(), found_securities.end(), SecurityLess()); + + vector resolved_securities = fill_leg_details(found_securities); + sort(resolved_securities.begin(), resolved_securities.end(), SecurityLess()); + // Extract the list of unresolved securities + vector unresolved_securities; + set_difference(found_securities.begin(), found_securities.end(), + resolved_securities.begin(), resolved_securities.end(), + inserter(unresolved_securities, unresolved_securities.begin()), SecurityLess()); + + // Return any unresolved spreads to the source so they can be 'discovered' again + // We can safely send a security key to every _instance because by definition + // a security cannot appear in 2 instances + if (unresolved_securities.size() > 250) { + LOG(MarketData,Warn,"Failed to resolve legs for {}" + " spreads. This can happen when a spread is subscribed without " + "it's underliers and may lead to higher than expected CPU use.", + unresolved_securities.size() ); + } + for (auto& security : unresolved_securities) { + bts_assert(security->security_type() == SecurityType::SPREAD); + bts_assert(security->security_key() != invalid_security_key); + inst->forget_security(security->security_key()); + } + + auto handles = get_handles(resolved_securities); + for (const auto& security : resolved_securities) { + auto find_result = handles.find(security->security_key()); + if (find_result != handles.end()) { + // Pass new securities to each callback + for (auto& cb : _callbacks) { + cb(*security, find_result->second); + } + } + else { + // "should never happen" + inst->forget_security(security->security_key()); + } + } + count += resolved_securities.size(); + } + else /*!resolve_legs*/ { + auto handles = get_handles(found_securities); + for (const auto& security : found_securities) { + auto find_result = handles.find(security->security_key()); + if (find_result != handles.end()) { + for (auto& cb : _callbacks) { + cb(*security, find_result->second); + } + } + else { + inst->forget_security(security->security_key()); + } + } + count += found_securities.size(); + } + } + + return count; + } + + void on_security(SecurityCallback cb, bool resolve_legs) { + Mutex::scoped_lock lock(_workmutex); + // Walk the current security list and fire the callbacks for existing securities + // Note: We have to lock over the initial walk because if scan_securities() finds a new security concurrently with this walk + // it might mark it seen without this callback getting called. + for (auto& inst : _instances) { + vector securities; + inst->walk_old_securities([&](const ColdEntry& entry) { + securities.push_back(convert_psecurity(entry.security)); + }); + auto filled_securities = resolve_legs ? fill_leg_details(securities) : securities; + auto handles = get_handles(filled_securities); + for (const auto& security : filled_securities) { + auto find_result = handles.find(security->security_key()); + bts_assert_d(find_result != handles.end()); + if (find_result != handles.end()) { + cb(*security, find_result->second); + } + } + } + // store the callback + _callbacks.push_back(cb); + lock.release(); + // Scan for new securities too + scan_securities(resolve_legs); + } + + MDConsumerChannel get_channel(const std::string& symbol, SecurityType::type type, Exchange::type exchange) const { + for (auto& inst : _instances) { + if (inst->symbol() == symbol && inst->type() == type && inst->exchange() == exchange) { + return MDConsumerChannel(inst); + } + } + bts_assert_msg(false, "Invalid symbol/type/exchange in MDConsumer::get_channel"); + } + + bool all_heartbeats_ok() { + for (auto& inst : _instances) { + if (!inst->is_heart_beating()) return false; + } + return true; + } + + size_t security_count() { + size_t result = 0; + for (auto& inst : _instances) { + result += inst->security_count(); + } + return result; + } + +private: + typedef tbb::queuing_mutex Mutex; + Mutex _workmutex; + vector> _instances; + map> _deferred_handles; + vector _callbacks; +}; + + +/////////////////////////////////////////////////////////////////////////////////////////////////// +// MDConsumer public interface, forwards to pimple +/////////////////////////////////////////////////////////////////////////////////////////////////// + +MDConsumer::MDConsumer(set const& symbols, bool shared) : _pimpl(new MDConsumerImpl(symbols, shared)) {} +MDConsumer::MDConsumer(const UserSymbol& s, bool shared) : + _pimpl() +{ + set symbols; + symbols.insert(s); + _pimpl.reset(new MDConsumerImpl(symbols, shared)); +} + +MDConsumer::MDConsumer(bool shared) : _pimpl() { + set all_symbols; + get_system_md_symbols(Exchange::TESTEXCHANGE, SecurityType::UNSET, all_symbols); + std::ostringstream msg; + for (const auto& symbol : all_symbols) { + if (msg.tellp() > 0) { msg << ", "; } + msg << symbol.symbol() << "-" << SecurityType::type_Name(symbol.type()) << " " << Exchange::type_Name(symbol.exchange()); + } + LOG(MarketData,Info,"Created MDConsumer for securities: {}" ,msg.str()); + + _pimpl.reset(new MDConsumerImpl(all_symbols, shared)); +} + +MDConsumer::MDConsumer(Exchange::type exchange, bool shared) : _pimpl() { + set all_symbols; + get_system_md_symbols(exchange, SecurityType::UNSET, all_symbols); + std::ostringstream msg; + for (const auto& symbol : all_symbols) { + if (msg.tellp() > 0) { msg << ", "; } + msg << symbol.symbol() << "-" << SecurityType::type_Name(symbol.type()) << " " << Exchange::type_Name(symbol.exchange()); + } + LOG(MarketData,Info, "Created MDConsumer for securities: {}" ,msg.str()); + + _pimpl.reset(new MDConsumerImpl(all_symbols, shared)); +} + +MDConsumer::~MDConsumer() {} + +bool MDConsumer::all_heartbeats_ok() { return _pimpl->all_heartbeats_ok(); } +size_t MDConsumer::security_count() { return _pimpl->security_count();} + +MDConsumerHandle MDConsumer::get_handle(security_key key) { return _pimpl->get_handle(key); } +std::map MDConsumer::get_handles(const std::set& keys) { return _pimpl->get_handles(keys); } +shared_ptr MDConsumer::get_managed_handle(security_key key) { return _pimpl->get_managed_handle(key); } + +size_t MDConsumer::walk_security_list(SecurityCallback cb, const string& symbol, SecurityType::type type) { + return _pimpl->walk_sec_list(cb, symbol, type); +} + +size_t MDConsumer::scan_securities(bool resolve_legs) { return _pimpl->scan_securities(resolve_legs); } +void MDConsumer::on_security(SecurityCallback cb, bool resolve_legs) { _pimpl->on_security(cb, resolve_legs); } +MDConsumerChannel MDConsumer::get_channel(const std::string& symbol, SecurityType::type type, Exchange::type exchange) const { + return _pimpl->get_channel(symbol, type, exchange); +} +MDConsumerChannel MDConsumer::get_channel(const UserSymbol& u) const { + return _pimpl->get_channel(u.symbol(), u.type(), u.exchange()); +} + +MDConsumerChannel::MDConsumerChannel(const shared_ptr& inst) : _pimpl(inst) {} +int64_t MDConsumerChannel::read_trades(google::protobuf::RepeatedPtrField& trades, int64_t cursor) { return _pimpl->read_trades(trades, cursor); } +int64_t MDConsumerChannel::read_rfqs(google::protobuf::RepeatedPtrField& quotes, int64_t cursor) { return _pimpl->read_rfqs(quotes, cursor); } +const std::string& MDConsumerChannel::symbol() const { return _pimpl->symbol(); } +SecurityType::type MDConsumerChannel::type() const { return _pimpl->type(); } +Exchange::type MDConsumerChannel::exchange() const { return _pimpl->exchange(); } +Platform::type MDConsumerChannel::platform() const { return platform_for_exchange(_pimpl->exchange()); } + +bool MDConsumerChannel::is_heart_beating() const { return _pimpl->is_heart_beating(); } +bool MDConsumerChannel::is_initialized() const { return _pimpl && security_count() > 0; } +TimeStamp MDConsumerChannel::create_time() const { return _pimpl ? _pimpl->cold_header().create_time : TimeStamp::invalid(); } + +size_t MDConsumerChannel::security_count() const { + return _pimpl->cold_header().security_count; +} +size_t MDConsumerChannel::bbo_count() const { + return is_initialized() ? + _pimpl->hot_header().bbo_count : 0; +} +size_t MDConsumerChannel::depth_count() const { + return is_initialized() ? + _pimpl->hot_header().depth_count : 0; +} +size_t MDConsumerChannel::trade_count() const { + return _pimpl->cold_header().trade_count; +} +size_t MDConsumerChannel::rfq_count() const { + return _pimpl->cold_header().rfq_count; +} +size_t MDConsumerChannel::customer_bbo_count() const { + return _pimpl->cold_header().customer_bbo_count; +} + +bool is_heart_beat_ok(const ColdHeader &header ) { + uint32_t s_heartbeat_ms = header.heartbeat_ms; + bool s_disable_heartbeat = header.disable_heartbeat; + uint32_t s_grace_period_ms = header.heartbeat_boot_seconds * 1000; + TimeStamp hb = header.heartbeat; + TimeStamp now = TimeStamp::now(); + return s_disable_heartbeat || (now - hb).milliseconds() < s_heartbeat_ms || (now - header.create_time).milliseconds() < s_grace_period_ms; +} + +}}//namespace + +#endif diff --git a/src/server/sparkd/md/MDConsumer.h b/src/server/sparkd/md/MDConsumer.h new file mode 100644 index 000000000..6033abfae --- /dev/null +++ b/src/server/sparkd/md/MDConsumer.h @@ -0,0 +1,98 @@ +#pragma once + +#include "lib/utils/bts_no_copy.h" +#include "lib/utils/bts_export.h" + +#include "common.h" +#include "MDConsumerHandle.h" +#include "UserSymbol.h" + +#include +#include +#include +#include +#include +#include + +namespace bts { +namespace md2 { + +/* typedef void (security_callback)(const Security& security); */ +typedef std::function SecurityCallback; + +class MDConsumerImpl; +class MDConsumerInst; +class MDConsumerChannel; +class BTS_EXPORT MDConsumer +{ +public: + // Consumer for all symbols + MDConsumer(bool shared=true); + MDConsumer(Exchange::type exchange, bool shared = true); + MDConsumer(const UserSymbol&, bool shared=true); + // Consumer for a given set of symbols + MDConsumer(std::set const& symbols, bool shared=true); + ~MDConsumer(); + + bool all_heartbeats_ok(); + size_t security_count(); + + MDConsumerHandle get_handle(security_key key); + std::map get_handles(const std::set& keys); + + std::shared_ptr get_managed_handle(security_key key); + + // Walk the entire available security list + size_t walk_security_list(SecurityCallback callback, const std::string& symbol="", SecurityType::type type = SecurityType::UNSET); + + // scan the security list for changes + size_t scan_securities(bool resolve_legs = true); + void on_security(SecurityCallback cb, bool resolve_legs = true); + + MDConsumerChannel get_channel(const std::string& symbol, SecurityType::type type, Exchange::type exchange) const; + MDConsumerChannel get_channel(const UserSymbol&) const; + + void swap(MDConsumer& other) { + _pimpl.swap(other._pimpl); + } + uintptr_t id() { return reinterpret_cast(_pimpl.get()); } + +private: + //BTS_NO_COPY(MDConsumer); COPY OK + std::shared_ptr _pimpl; +}; + +class BTS_EXPORT MDConsumerChannel { + std::shared_ptr _pimpl; +public: + MDConsumerChannel(const std::shared_ptr& inst); + + const std::string& symbol() const; + SecurityType::type type() const; + Exchange::type exchange() const; + Platform::type platform() const; + bool is_initialized() const; + utils::TimeStamp create_time() const; + + // Scan for new trades. If overrun is true some trades were lost + int64_t read_trades(google::protobuf::RepeatedPtrField& trades, int64_t cursor); + // Scan for new rfqs. If overrun=true some entries were lost + int64_t read_rfqs(google::protobuf::RepeatedPtrField& quotes, int64_t cursor); + + // Check the heartbeat has not passed the threshhold (PublisherOptions.heartbeat_ms) + bool is_heart_beating() const; + + // + // Statistic counters + size_t security_count() const; + size_t bbo_count() const; + size_t depth_count() const; + size_t trade_count() const; + size_t rfq_count() const; + size_t customer_bbo_count() const; + +}; + + +}}//namespace + diff --git a/src/server/sparkd/md/MDConsumerHandle.cpp b/src/server/sparkd/md/MDConsumerHandle.cpp new file mode 100644 index 000000000..5f7d19f5b --- /dev/null +++ b/src/server/sparkd/md/MDConsumerHandle.cpp @@ -0,0 +1,107 @@ +#ifndef NO_CPP0X +#include "MDConsumerHandle.h" + +namespace bts { +namespace md2 { + +MDConsumerHandle::MDConsumerHandle() : + _hot(nullptr), + _cdata(nullptr), + _cheader(nullptr) +{ + static_assert(sizeof(MDConsumerHandle) == sizeof(uintptr_t) * 3, ""); +} + +MDConsumerHandle::MDConsumerHandle(const MDConsumerHandle& other) : + _hot(other._hot), + _cdata(other._cdata), + _cheader(other._cheader) +{} + +MDConsumerHandle::~MDConsumerHandle() {} + +void MDConsumerHandle::init(const ColdHeader& cheader, const ColdEntry& centry, const void* pvhot) { + _hot = pvhot; + _cdata = ¢ry; + _cheader = &cheader; +} + +BTS_EXPORT TimeStamp copy_bbo(const TimeStamp& time, const SingleWriterPriceSizeStore& in_bid, const SingleWriterPriceSizeStore& in_ask, PriceSize& out_bid, PriceSize& out_ask) { + // We expect the time, in_bid and in_ask to be changing (rapidly) + // + TimeStamp t; + int attempts = 0; + // If the publisher crashed and left TimeStamp::invalid() in memory we need to be able to escape the loop + // This max_attempts allows get_bbo() to return 'false' in this case so the calling thread can continue and + // hopefully check the heartbeat. + const int max_attempts = 1619200; + do { + t = time; + out_bid = in_bid.load(); + out_ask = in_ask.load(); + ++attempts; + } while((t != time || t == TimeStamp::invalid()) && attempts < max_attempts); + if (attempts >= max_attempts) { + return TimeStamp::invalid(); + } + return t; +} + +TimeStamp MDConsumerHandle::last_market_update() const { + if (has_depth()) { + return hotd()->timestamp; + } + return hot()->timestamp; +} + +PriceSize MDConsumerHandle::get_bid(size_t depth) const { + bts_assert_d(has_depth()); + bts_assert_d(depth < COCONUT_MAX_DEPTH); + return hotd()->bid(depth).load(); +} +PriceSize MDConsumerHandle::get_ask(size_t depth) const { + bts_assert_d(has_depth()); + bts_assert_d(depth < COCONUT_MAX_DEPTH); + return hotd()->ask(depth).load(); +} + +bool MDConsumerHandle::get_customer_bbo(TimeStamp& time, PriceSize& out_bid, PriceSize& out_ask) const { + if (!is_initialized()) return false; + + bool result = false; + while(_cdata->customer_bbo_updated > time || time == TimeStamp::invalid()) { + time = _cdata->customer_bbo_updated; + out_bid = _cdata->customer_bid.load(); + out_ask = _cdata->customer_ask.load(); + result = true; + } + return result; +} + +bool MDConsumerHandle::get_last_trade(TimeStamp& time, PriceSize& trade, datetime::LLDateTime& trade_timestamp, size_t* total_volume) const { + if (!is_initialized()) return false; + bool result = false; + while (_cdata->last_trade_updated > time) { + time = _cdata->last_trade_updated; + trade = _cdata->last_trade.load(); + trade_timestamp = _cdata->last_trade_time; + if (total_volume) { *total_volume = _cdata->total_volume; } + result = true; + } + return result; +} + +bool is_heart_beat_ok(const ColdHeader &); +bool MDConsumerHandle::is_heart_beating() const { + return _cheader && is_heart_beat_ok(*_cheader); +} +TimeStamp MDConsumerHandle::heartbeat() const { + return _cheader->heartbeat; +} +CoconutSecurityStatus::type MDConsumerHandle::status() const { return (CoconutSecurityStatus::type)(has_depth() ? hotd()->status : hot()->status); } +uint16_t MDConsumerHandle::sequence() const { return has_depth() ? hotd()->sequence : hot()->sequence; } + +}}//namespace + + +#endif diff --git a/src/server/sparkd/md/MDConsumerHandle.h b/src/server/sparkd/md/MDConsumerHandle.h new file mode 100644 index 000000000..c92baefec --- /dev/null +++ b/src/server/sparkd/md/MDConsumerHandle.h @@ -0,0 +1,96 @@ +#pragma once + +#include "lib/utils/bts_no_copy.h" +#include "lib/utils/bts_export.h" +#include "Structure/structure.h" +#include + +namespace bts { +namespace md2 { + +BTS_EXPORT TimeStamp copy_bbo(const TimeStamp& , const SingleWriterPriceSizeStore& , + const SingleWriterPriceSizeStore& , PriceSize& , PriceSize& ); + +// Can be an invalid handle if you request a security which has not been discovered yet. +class BTS_EXPORT MDConsumerHandle +{ +public: + MDConsumerHandle(); + ~MDConsumerHandle(); + MDConsumerHandle(const MDConsumerHandle& other); + + void init(const ColdHeader&, const ColdEntry&, const void* hot); + + bool is_initialized() const { return _hot != nullptr && _cdata != nullptr && _cheader != nullptr; } + bool is_heart_beating() const; + TimeStamp heartbeat() const; + CoconutSecurityStatus::type status() const; + uint16_t sequence() const; + bool has_depth() const { return _cheader && _cheader->has_depth; } + security_key key() const { return _cdata ? _cdata->security.key : invalid_security_key; } + Security get_normalized_security() const { bts_assert(is_initialized()); return convert_security(_cdata->security); } + + + TimeStamp last_market_update() const; + + bool get_bbo(TimeStamp& time, PriceSize& bid, PriceSize& ask) const + { + if (!is_initialized()) return false; + + if (_cheader->has_depth) { + auto* pdepth = hotd(); + if (pdepth->timestamp > time || pdepth->timestamp == TimeStamp::invalid()) { + TimeStamp temp = copy_bbo(pdepth->timestamp, pdepth->bid(0), pdepth->ask(0), bid, ask); + if (temp == TimeStamp::invalid()) { return false; } + time = temp; + return true; + } + } + else { + auto* pdepth = hot(); + if (pdepth->timestamp > time || pdepth->timestamp == TimeStamp::invalid()) { + TimeStamp temp = copy_bbo(pdepth->timestamp, pdepth->bid, pdepth->ask, bid, ask); + if (temp == TimeStamp::invalid()) { return false; } + time = temp; + return true; + } + } + return false; + } + + // Last bbo or depth update + PriceSize get_bid(size_t depth) const; + PriceSize get_ask(size_t depth) const; + + bool get_customer_bbo(TimeStamp& time, PriceSize& out_bid, PriceSize& out_ask) const; + bool get_last_trade(TimeStamp& time, PriceSize& out_trade, datetime::LLDateTime& out_trade_timestamp, size_t* total_volume=nullptr) const; + + TimeStamp misc_timestamp() const { return _cdata->misc.misc_timestamp; } + MiscData misc() const { return _cdata->misc; } + + MDConsumerHandle& operator=(const MDConsumerHandle& other) { + _hot = other._hot; + _cdata = other._cdata; + // Ensure that the data pointers are stored before the header. + // This is primarily a compiler barrier - I don't want to change all the members to std::atomics for the cheaper compiler barrier + // since this is not called often + std::atomic_thread_fence(std::memory_order_release); + _cheader = other._cheader; + return *this; + } + +private: + friend class MDConsumerInst; + void set_hot(const void* hot) { _hot = hot; } + void set_cold(const ColdHeader* header, const ColdEntry* cold) { _cheader = header; _cdata = cold; } + + const HotDepthEntry* hotd() const { bts_assert_d(has_depth()); return static_cast(_hot); } + const HotEntry* hot() const { return static_cast(_hot); } + // COPY OK -- BTS_NO_COPY(MDConsumerHandle); + const void* _hot; + const ColdEntry* _cdata; + const ColdHeader* _cheader; +}; + +}}//namespace + diff --git a/src/server/sparkd/md/MDConsumerInterop.cpp b/src/server/sparkd/md/MDConsumerInterop.cpp new file mode 100644 index 000000000..3367f05c1 --- /dev/null +++ b/src/server/sparkd/md/MDConsumerInterop.cpp @@ -0,0 +1,189 @@ +#ifdef _WIN32 +#include "../bts_conn_exports.h" +#include "common.h" +#include "UserSymbol.h" +#include "MDConsumer.h" +#include "get_symbols.h" +#include "relay/RelayClient.h" +#include "core/bts_core/utils/Dispatcher.h" +#include +#include "conn/bts_conn/exchange/co2.pb.h" +#include +#include +#include + +namespace bts { +namespace md2 { + +#define CAPI extern "C" BTS_EXPORT + +// +// MDConsumer + +struct BlitUserSymbol { + const char* symbol; + Exchange::type exchange; + SecurityType::type type; +}; + +CAPI MDConsumer* create_md_consumer(BlitUserSymbol* usersymbols, uint32_t count, bool shared) { + if (usersymbols == nullptr || count == 0) { + return new MDConsumer(shared); + } + else { + std::set symbols; + for (size_t x = 0; x < count; ++x) { + symbols.insert(UserSymbol(usersymbols[x].symbol, usersymbols[x].exchange, usersymbols[x].type)); + } + return new MDConsumer(symbols, shared); + } +} + +CAPI void destroy_md_consumer(MDConsumer* consumer) { + if (consumer != nullptr) { + delete consumer; + } +} + +CAPI bool md_consumer_all_heartbeats_ok(MDConsumer* consumer) { + if (consumer) { + return consumer->all_heartbeats_ok(); + } + return false; +} + +CAPI bool md_consumer_get_handle(MDConsumer* consumer, security_key key, MDConsumerHandle& handle) { + bts_assert(consumer); + bts_assert(key != invalid_security_key); + handle = consumer->get_handle(key); + return handle.is_initialized(); +} + +typedef void(*security_cb)(security_key, const char*, size_t, MDConsumerHandle& handle); + +CAPI size_t md_consumer_walk_security_list(MDConsumer* consumer, security_cb cb) { + bts_assert(consumer); + std::string buffer; + return consumer->walk_security_list([&](const Security& security, MDConsumerHandle handle) { + buffer.clear(); + security.SerializePartialToString(&buffer); + cb(security.security_key(), buffer.c_str(), buffer.size(), handle); + }); +} + + +CAPI void md_consumer_scan_securities(MDConsumer* consumer) { + bts_assert(consumer); + static std::atomic working = 0; + if (working <= 0) { + ++working; + md::scan_securities_thread().post([=]() { + DEFER(--working);//exceptionsafe + consumer->scan_securities(); + }); + } +} + +#ifdef _WIN32 +void safe_call_clr_callback(security_cb cb, security_key key, const char* buffer, size_t buffer_size, MDConsumerHandle& handle) { + // Protect against clr shutting down + __try { + cb(key, buffer, buffer_size, handle); + } __except (EXCEPTION_CONTINUE_EXECUTION) {} +} + +CAPI void md_consumer_set_on_security(MDConsumer* consumer, security_cb cb) { + bts_assert(consumer); + consumer->on_security([=](const Security& security, MDConsumerHandle handle) { + std::string buffer; + security.SerializePartialToString(&buffer); + safe_call_clr_callback(cb, security.security_key(), buffer.c_str(), buffer.size(), handle); + }); +} +#endif + +typedef void(*just_security_cb)(security_key, const char*, size_t); +CAPI void md_consumer_get_single_security(const ColdEntry* entry, just_security_cb cb) { + bts_assert(entry); + auto security = convert_security(entry->security); + std::string buffer; + security.SerializePartialToString(&buffer); + cb(security.security_key(), buffer.c_str(), buffer.size()); +} + +bool is_heart_beat_ok(const ColdHeader &header); +CAPI bool md_consumer_is_heart_beating(const ColdHeader* header) { + if (header == nullptr) return false; + return is_heart_beat_ok(*header); +} + +typedef void(*raw_receive_pb)(const char*, size_t); +CAPI int md_consumer_read_trades(MDConsumer* consumer, BlitUserSymbol& symbol, int64_t& cursor, raw_receive_pb cb) { + bts_assert(consumer); + google::protobuf::Arena arena; + co2::TradeList* ptrades = google::protobuf::Arena::CreateMessage(&arena); + auto channel = consumer->get_channel(symbol.symbol, symbol.type, symbol.exchange); + if (channel.is_initialized()) { + cursor = channel.read_trades(*ptrades->mutable_items(), cursor); + std::string buffer; + ptrades->SerializeToString(&buffer); + cb(buffer.c_str(), buffer.size()); + return ptrades->items_size(); + } + return 0; +} +CAPI int md_consumer_read_rfqs(MDConsumer* consumer, BlitUserSymbol& symbol, int64_t& cursor, raw_receive_pb cb) { + bts_assert(consumer); + google::protobuf::Arena arena; + co2::RFQList* plist = google::protobuf::Arena::CreateMessage(&arena); + auto channel = consumer->get_channel(symbol.symbol, symbol.type, symbol.exchange); + if (channel.is_initialized()) { + cursor = channel.read_rfqs(*plist->mutable_items(), cursor); + std::string buffer; + plist->SerializeToString(&buffer); + cb(buffer.c_str(), buffer.size()); + return plist->items_size(); + } + return 0; +} + +// +// MDConsumerChannel + +// +// Relay control +CAPI void add_relay_symbol(md::RelayControl* prelay, BlitUserSymbol& symbol, uint32_t flags) { + co2::SecurityChannel channel; + channel.set_symbol(symbol.symbol); + channel.set_exchange(symbol.exchange); + channel.set_type(symbol.type); + channel.set_flags(flags); + if (channel.symbol() == "*") { + std::set system_symbols; + get_system_md_symbols(channel.exchange(), channel.type(), system_symbols); + for (const auto& sym : system_symbols) { + channel.set_symbol(sym.symbol()); + md::add_relay_symbol(prelay, channel); + } + } + else { + md::add_relay_symbol(prelay, channel); + } +} + +typedef void(*usersymbol_cb)(BlitUserSymbol& symbol); +CAPI void md_consumer_get_system_md_symbols(usersymbol_cb cb) { + std::set system_symbols; + get_system_md_symbols(Exchange::TESTEXCHANGE, SecurityType::UNSET, system_symbols); + for (auto& s : system_symbols) { + BlitUserSymbol blit_sym = { s.symbol().c_str(), s.exchange(), s.type() }; + cb(blit_sym); + } +} +typedef void(*just_action)(); +CAPI void post_on_scan_security_thread(just_action a) { + md::scan_securities_thread().post(a); +} + +}} //namespace +#endif diff --git a/src/server/sparkd/md/MDProducer.cpp b/src/server/sparkd/md/MDProducer.cpp new file mode 100644 index 000000000..784bb5fb3 --- /dev/null +++ b/src/server/sparkd/md/MDProducer.cpp @@ -0,0 +1,157 @@ +#ifndef NO_CPP0X +#include "MDProducer.h" + +#include "lib/utils/SharedList.h" +#include "lib/utils/WaitFreeQueue.h" +#include "Structure/structure.h" +#include "MemorySegmentUtils.h" + +using namespace std; + +namespace bts { +namespace md2 { + + +// Owns shared memory for a single symbol+type +class MDProducerImpl { +public: + MDProducerImpl(const string& symbol, SecurityType::type type, Exchange::type exchange, Platform::type platform, bool shared) : + _symbol(symbol), + _type(type), + _exchange(exchange), + _hot(hot_segment_name(symbol, type, exchange, platform), 1024, 1024, shared), + _cold(cold_segment_name(symbol, type, exchange, platform), 6144, 2048, shared), + _trades(_cold.header().trade_buffer, BTS_ARRAYSIZE(_cold.header().trade_buffer), true), + _rfqs(_cold.header().rfq_buffer, BTS_ARRAYSIZE(_cold.header().rfq_buffer), true) + { + // NOTE: Block sizes have been tuned based on sim. + // We should monitor and adjust based on production use for optimal performance + _cold.header().has_depth = enable_depth(symbol, type); + _cold.header().platform = platform; + + static_assert(sizeof(HotEntry) <= 32,"HotEntry should be less than 32 bytes"); + } + + MDProducerHandle add_security(const FixedSecurity& security) { + void* h_entry = nullptr; + if (has_depth()) { + h_entry = new (_hot.add(sizeof(HotDepthEntry))) HotDepthEntry(); + } + else { + h_entry = new (_hot.add(sizeof(HotEntry))) HotEntry(); + } + + // Calculating the size for the variable sized entries is tricky + ColdEntry* c_entry = new (_cold.add(ColdEntry::fixed_size() + security.size)) ColdEntry(security.size); + memcpy(&c_entry->security, &security, security.size); + + MDProducerHandle handle(_cold.header(), _hot.header(), *c_entry, h_entry, MDProducerChannel(*this)); + + if (security.key != invalid_security_key) { + // The only safe way to set the security key is through the set_key() function + // If the security had a key it was copied to cold by the memcpy but not the hot + // So we clear it from the cold and let set_key() set it in hot and cold and make the security public. + c_entry->security.key = invalid_security_key; + handle.set_key(security.key); + } + + return handle; + } + + void heartbeat() { + _cold.header().heartbeat = utils::TimeStamp::now(); + } + + void add_trade(const co2::TradeMessage& trade) { + bts_assert(trade.key() != invalid_security_key); + _trades.push(trade); + _cold.header().trade_count++; + } + + void add_rfq(const co2::QuoteRequestMessage& rfq) { + ColdHeader::RFQBuffer buffer = {{0}}; + if (rfq.SerializePartialToArray(&buffer[0], buffer.size())) { + _rfqs.push(buffer); + _cold.header().rfq_count++; + } + else { + LOG(MarketData,Warn, "Unable to store RFQ due to size limitation {}", + rfq.DebugString()); + } + } + + const string& symbol() const { return _symbol; } + SecurityType::type type() const { return _type; } + Exchange::type exchange() const { return _exchange; } + bool has_depth() { return _cold.header().has_depth; } + +private: + BTS_NO_COPY(MDProducerImpl); + string _symbol; + SecurityType::type _type; + Exchange::type _exchange; + utils::SharedList _hot; + utils::SharedList _cold; + utils::WaitFreeQueue _trades; + utils::WaitFreeQueue _rfqs; +}; + +// +// MDProducer +// + +MDProducer::MDProducer(Platform::type platform, bool shared) : + _mutex(new Mutex()), + _producers(), + _platform(platform), + _shared(shared) +{ +} +MDProducer::MDProducer(MDProducer&& other) : + _mutex(std::move(other._mutex)), + _producers(std::move(other._producers)), + _platform(other._platform), + _shared(other._shared) +{} + +MDProducer::~MDProducer() {} + +MDProducerImpl& MDProducer::get_producer(const char* symbol, SecurityType::type type, Exchange::type exchange) { + for (auto& impl : _producers) { + if (impl->symbol() == symbol && impl->type() == type && impl->exchange() == exchange) { + return *impl; + } + } + + // Not found add a new one + unique_ptr newp(new MDProducerImpl(symbol, type, exchange, _platform, _shared)); + _producers.push_back(move(newp)); + return *_producers.back(); +} + +MDProducerHandle MDProducer::add_security(const FixedSecurity& security) { + bts_assert_d(strlen(security.symbol()) > 0); + Mutex::scoped_lock lock(*_mutex); + auto& impl = get_producer(security.symbol(), security.type, security.exchange); + auto result = impl.add_security(security); + if (security.key != invalid_security_key) { result.set_key(security.key); } + return result; +} + +MDProducerChannel MDProducer::get_channel(const char* symbol, SecurityType::type type, Exchange::type exchange) { + Mutex::scoped_lock lock(*_mutex); + return MDProducerChannel(get_producer(symbol, type, exchange)); +} + +// +// MDProducerChannel +// + +MDProducerChannel::MDProducerChannel(MDProducerImpl& impl) : _me(&impl) {} +void MDProducerChannel::heartbeat() {_me->heartbeat();} +void MDProducerChannel::add_trade(const co2::TradeMessage& trade) {_me->add_trade(trade);} +void MDProducerChannel::add_rfq(const co2::QuoteRequestMessage& rfq) {_me->add_rfq(rfq);} + +}}//namespace + +#endif diff --git a/src/server/sparkd/md/MDProducer.h b/src/server/sparkd/md/MDProducer.h new file mode 100644 index 000000000..445bf2d8b --- /dev/null +++ b/src/server/sparkd/md/MDProducer.h @@ -0,0 +1,48 @@ +#pragma once + +#include "lib/utils/bts_no_copy.h" +#include "lib/utils/bts_export.h" + +#include "common.h" +#include "MDProducerHandle.h" +#include "MDProducerChannel.h" +#include "Structure/FixedSecurity.h" +#include "tbb/queuing_mutex.h" +#include +#include + +namespace bts { +namespace md2 { + +class MDProducerImpl; + +class BTS_EXPORT MDProducer +{ +public: + MDProducer(Platform::type platform, bool shared=true); + MDProducer(MDProducer&& other); + ~MDProducer(); + + // A single MDProducer owns a given symbol+type. + // Attempting to add the same Symbol+type to a second + // producer (even in a different process) is a crash-worthy error + MDProducerHandle add_security(const FixedSecurity& security); + + // A Producer Channel holds data shared between the + MDProducerChannel get_channel(const char* symbol, SecurityType::type type, Exchange::type exchange); + +private: + BTS_NO_COPY(MDProducer); + + MDProducerImpl& get_producer(const char* symbol, SecurityType::type type, Exchange::type exchange); + + typedef tbb::queuing_mutex Mutex; + std::unique_ptr _mutex; + std::vector> _producers; + Platform::type _platform; + bool _shared; +}; + + +}}//namespace + diff --git a/src/server/sparkd/md/MDProducerChannel.h b/src/server/sparkd/md/MDProducerChannel.h new file mode 100644 index 000000000..46f49644e --- /dev/null +++ b/src/server/sparkd/md/MDProducerChannel.h @@ -0,0 +1,23 @@ +#pragma once + +#include "lib/utils/bts_export.h" +#include "server/sparkd/md/protos/coconut.pb.h" + +namespace bts { +namespace md2 { + +class MDProducerImpl; + +class BTS_EXPORT MDProducerChannel { + MDProducerImpl* _me; + MDProducerChannel(): _me(nullptr) {} + friend struct Co2TestInstrument; +public: + friend class MDProducerHandle; + MDProducerChannel(MDProducerImpl& impl); + void heartbeat(); + void add_trade(const co2::TradeMessage& trade); + void add_rfq(const co2::QuoteRequestMessage& rfq); +}; + +}} diff --git a/src/server/sparkd/md/MDProducerHandle.cpp b/src/server/sparkd/md/MDProducerHandle.cpp new file mode 100644 index 000000000..243f31fe9 --- /dev/null +++ b/src/server/sparkd/md/MDProducerHandle.cpp @@ -0,0 +1,311 @@ +#ifndef NO_CPP0X +#include "MDProducerHandle.h" +#include "common.h" + +namespace bts { +namespace md2 { +using namespace datetime; + +MDProducerHandle::MDProducerHandle(ColdHeader& cheader, HotHeader& hheader, ColdEntry& centry, void* phot, MDProducerChannel channel) : + _hdata(phot), + _hheader(&hheader), + _cdata(¢ry), + _cheader(&cheader), + _channel(channel) +{ +} + +MDProducerHandle::MDProducerHandle() : + _hdata(nullptr), + _hheader(nullptr), + _cdata(nullptr), + _cheader(nullptr), + _channel() +{} + +MDProducerHandle::~MDProducerHandle() {} + +MDProducerHandle& MDProducerHandle::operator=(MDProducerHandle const& other) { + _hdata = other._hdata; + _hheader = other._hheader; + _cdata = other._cdata; + _cheader = other._cheader; + _channel = other._channel; + return *this; +} + +void MDProducerHandle::set_key(security_key key) { + if (cdata().security.key == key) return; + bts_assert(cdata().security.key == invalid_security_key); + // ! Assignment order important ! + // Once the cold data has a key the security becomes visible so the hot data *must* be able to be found or we assert + if (has_depth()) { + hdepth().key = key; + } + else { + hdata().key = key; + } + + cdata().security.key = key; + + // Increment public security count + cheader().security_count++; + + // If this is a spread make sure the legs have valid keys + if (security().type == SecurityType::SPREAD) { + const auto& psecurity = static_cast(security()); + for (size_t x = 0; x < psecurity.leg_count(); ++x) { + bts_assert(psecurity.legs[x].key != invalid_security_key); + } + } +} + +security_key MDProducerHandle::key() const { + security_key key = has_depth() ? hdepth().key : hdata().key; + bts_assert_d(key == cdata().security.key); + return key; +} + +void MDProducerHandle::set_legs(const std::vector& legs) { + bts_assert(cdata().security.type == SecurityType::SPREAD); + FixedSpreadSecurityL* fss = (FixedSpreadSecurityL*)&cdata().security; + bts_assert(fss->leg_count() == legs.size()); + for (size_t x = 0; x < legs.size(); ++x) { + security_key key = legs[x]; + fss->legs[x].key = key; + // attempt to validate that this FixedSpreadLeg is valid + bts_assert_d(fss->legs[x].leg_quantity > 0); + } +} + +void MDProducerHandle::update_bid(TimeStamp time, const PriceSize price, uint32_t depth) { + if (!has_depth()) { + if (depth == 0 && hdata().bid.store(price)) { + hdata().timestamp = time; + hheader().bbo_count++; + } + } + else { + bts_assert(depth < COCONUT_MAX_DEPTH); + if (hdepth().bid(depth).store(price)) { + hdepth().timestamp = time; + if (depth == 0) { + hheader().bbo_count++; + hdepth().status = CoconutSecurityStatus::OK; + } + else { hheader().depth_count++; } + } + } +} + +void MDProducerHandle::update_ask(TimeStamp time, const PriceSize price, uint32_t depth) { + if (!has_depth()) { + if (depth == 0 && hdata().ask.store(price)) { + hdata().timestamp = time; + hheader().bbo_count++; + } + } + else { + bts_assert(depth < COCONUT_MAX_DEPTH); + if (hdepth().ask(depth).store(price)) { + hdepth().timestamp = time; + if (depth == 0) { + hheader().bbo_count++; + hdepth().status = CoconutSecurityStatus::OK; + } + else { hheader().depth_count++; } + } + } +} +void MDProducerHandle::update_bbo(TimeStamp time, const PriceSize bid, const PriceSize ask, uint16_t seq) { + if (!has_depth()) { + auto oldtime = hdata().timestamp; + hdata().timestamp = TimeStamp::invalid(); + bool set_bid = hdata().bid.store(bid); + bool set_ask = hdata().ask.store(ask); + hdata().status = CoconutSecurityStatus::OK; + if (set_bid || set_ask) { + hdata().timestamp = time; + hdata().sequence = seq; + hheader().bbo_count++; + } + else { + hdata().timestamp = oldtime; + } + } + else { + auto oldtime = hdepth().timestamp; + hdepth().timestamp = TimeStamp::invalid(); + bool set_bid = hdepth().bid(0).store(bid); + bool set_ask = hdepth().ask(0).store(ask); + hdepth().status = CoconutSecurityStatus::OK; + if (set_bid || set_ask) { + hdepth().timestamp = time; + hdepth().sequence = seq; + hheader().bbo_count++; + } + else { + hdepth().timestamp = oldtime; + } + } +} + +void MDProducerHandle::set_status(CoconutSecurityStatus::type status) { + if (has_depth()) { + hdepth().status = status; + } + else { + hdata().status = status; + } +} + +void MDProducerHandle::update_numeric_id_hack_for_cboe(uint64_t id) { + // This assignment breaks the rules for immutability of the security after security_key is assigned + // This hack is for CBOE csm because they change their keys between sessions (*boggle*) + cdata().security.numeric_id = id; +} + +void MDProducerHandle::add_trade(TimeStamp time, const PriceSize price, LLDateTime tradetime, uint32_t volume) { + cdata().last_trade.store(price); + cdata().last_trade_time = tradetime; + cdata().total_volume = volume; + cdata().last_trade_updated = time; + + if (key() != invalid_security_key) { + co2::TradeMessage trade; + trade.set_exponent(price.exponent()); + trade.set_mantissa(price.mantissa()); + trade.set_size(price.size()); + trade.set_time(tradetime); + if (volume > 0) trade.set_total_volume(volume); + trade.set_key(key()); + _channel.add_trade(trade); + } +} +void MDProducerHandle::set_last_trade(TimeStamp time, const PriceSize ps, datetime::LLDateTime tradetime, uint32_t total_volume) { + cdata().last_trade.store(ps); + cdata().last_trade_time = tradetime; + cdata().total_volume = total_volume; + cdata().last_trade_updated = time; +} + +void MDProducerHandle::add_rfq(TimeStamp, const co2::QuoteRequestMessage& rfq) { + if (rfq.key() != invalid_security_key) { + bts_assert(security().key == rfq.key()); + _channel.add_rfq(rfq); + } +} + +void MDProducerHandle::update_customer_bbo(TimeStamp time, const PriceSize bid, const PriceSize ask) { + cdata().customer_bid.store(bid); + cdata().customer_ask.store(ask); + cdata().customer_bbo_updated = time; + cheader().customer_bbo_count++; +} + +void MDProducerHandle::update_exchange_status(TimeStamp, TradingStatus::type status) { + cdata().exchange_status = status; + cheader().exchange_status_count++; +} + +void MDProducerHandle::update_settlement(TimeStamp time, Price price, Date date) { + if (date > cdata().misc.settlement_date) { + cdata().misc.previous_settlement_price = cdata().misc.settlement_price; + cdata().misc.previous_settlement_date = cdata().misc.settlement_date; + } + cdata().misc.settlement_price = price; + cdata().misc.settlement_date = date; + cdata().misc.misc_timestamp = time; +} +void MDProducerHandle::update_previous_settlement(TimeStamp time, Price price, Date date) { + cdata().misc.previous_settlement_price = price; + cdata().misc.previous_settlement_date = date; + cdata().misc.misc_timestamp = time; +} + +void MDProducerHandle::update_opening(TimeStamp time, Price price, Date date) { + cdata().misc.open_price = price; + cdata().misc.open_date = date; + cdata().misc.misc_timestamp = time; +} + +void MDProducerHandle::update_closing(TimeStamp time, Price price, Date date) { + cdata().misc.close_price = price; + cdata().misc.close_date = date; + cdata().misc.misc_timestamp = time; +} + +void MDProducerHandle::update_session_high(TimeStamp time, Price price) { + cdata().misc.session_high = price; + cdata().misc.misc_timestamp = time; +} + +void MDProducerHandle::update_session_low(TimeStamp time, Price price) { + cdata().misc.session_low = price; + cdata().misc.misc_timestamp = time; +} + +void MDProducerHandle::update_open_interest(TimeStamp, Price ) { + //cdata().misc.open_interest = price; + //cdata().misc_timestamp = TimeStamp::now(); +} + +void MDProducerHandle::update_session_high_bid(TimeStamp, Price ) { + //cdata().misc.session_high_bid = price; + //cdata().misc_timestamp = TimeStamp::now(); +} + +void MDProducerHandle::update_session_low_offer(TimeStamp, Price ) { + //cdata().misc.session_low_offer = price; + //cdata().misc_timestamp = TimeStamp::now(); +} + +void MDProducerHandle::update_fixing_price(TimeStamp, Price ) { + //cdata().misc.fixing_price = price; + //cdata().misc_timestamp = TimeStamp::now(); +} + +void MDProducerHandle::mark_deleted() { + TimeStamp ts = TimeStamp::now(); + cdata().deleted = true; + PriceSize& no_market = PriceSize::NO_MARKET; + if (has_depth()) { + update_bbo(ts, no_market, no_market); + } + else { + for (size_t x = 0; x < COCONUT_MAX_DEPTH; ++x) { + update_bid(ts, no_market, x); + update_ask(ts, no_market, x); + } + } +} + +HotEntry& MDProducerHandle::hdata() const { + bts_assert_d(_hdata && !has_depth()); + return *static_cast(_hdata); +} + +HotDepthEntry& MDProducerHandle::hdepth() const { + bts_assert_d(_hdata && has_depth()); + return *static_cast(_hdata); +} + +HotHeader& MDProducerHandle::hheader() const { + bts_assert_d(_hheader); + return *_hheader; +} + +ColdEntry& MDProducerHandle::cdata() const { + bts_assert_d(_cdata); + return *_cdata; +} + +ColdHeader& MDProducerHandle::cheader() const { + bts_assert_d(_cheader); + return *_cheader; +} + + +}}//namespace + +#endif diff --git a/src/server/sparkd/md/MDProducerHandle.h b/src/server/sparkd/md/MDProducerHandle.h new file mode 100644 index 000000000..05ff76f27 --- /dev/null +++ b/src/server/sparkd/md/MDProducerHandle.h @@ -0,0 +1,85 @@ +#pragma once + +#include "lib/utils/bts_no_copy.h" +#include "lib/utils/bts_export.h" +#include "Structure/structure.h" +#include "MDProducerChannel.h" +#include "lib/utils/datetime.h" + +#include +#include + + +namespace bts { + namespace co2 { struct Work; } +namespace md2 { + +class BTS_EXPORT MDProducerHandle +{ +public: + MDProducerHandle(ColdHeader&, HotHeader&, ColdEntry&, void* hot, MDProducerChannel channel); + ~MDProducerHandle(); + + MDProducerHandle& operator=(MDProducerHandle const& other); + + const FixedSecurity& security() const { return cdata().security; } + FixedSecurity* mutable_security() { return key() == invalid_security_key ? &cdata().security : nullptr; } + + bool has_depth() const { return cheader().has_depth; } + void set_key(security_key); + security_key key() const; + void set_legs(const std::vector& legs); + + void update_bid(TimeStamp time, const PriceSize bid, uint32_t depth=0); // hot + void update_ask(TimeStamp time, const PriceSize ask, uint32_t depth=0); // hot + void update_bbo(TimeStamp time, const PriceSize bid_price, const PriceSize ask_price, uint16_t seq=0); // hot + void set_status(md::CoconutSecurityStatus::type status); // hot + void update_numeric_id_hack_for_cboe(uint64_t); + + // Updates trade log and last trade + void add_trade(TimeStamp time, const PriceSize pricesize, datetime::LLDateTime date, uint32_t total_volume=0); // cold + // Update last_trade without updating trade log + void set_last_trade(TimeStamp time, const PriceSize ps, datetime::LLDateTime date, uint32_t total_volume); + void add_rfq(TimeStamp time, const co2::QuoteRequestMessage& rfq); // cold + + void update_customer_bbo(TimeStamp time, const PriceSize bid, const PriceSize ask); // cold + void update_exchange_status(TimeStamp time, TradingStatus::type status); // cold + + // Misc fields + void update_settlement(TimeStamp time, Price price, datetime::Date date); // cold + void update_previous_settlement(TimeStamp time, Price price, datetime::Date date); // cold + void update_opening(TimeStamp time, Price price, datetime::Date date); // cold + void update_closing(TimeStamp time, Price price, datetime::Date date); // cold + void update_session_high(TimeStamp time, Price price); // cold + void update_session_low(TimeStamp time, Price price); // cold + void update_open_interest(TimeStamp time, Price price); // cold + void update_session_high_bid(TimeStamp time, Price price); // cold + void update_session_low_offer(TimeStamp time, Price price); // cold + void update_fixing_price(TimeStamp time, Price price); // cold + + void mark_deleted(); // cold + + +private: + // Private default constructor allows friend classes to store this class in a concurrent_queue + MDProducerHandle(); + friend class tbb::concurrent_queue < MDProducerHandle > ; + friend class SecurityIDRecorder; + friend struct bts::co2::Work;//fuckingc++ + + HotEntry& hdata() const; + HotDepthEntry& hdepth() const; + HotHeader& hheader() const; + ColdEntry& cdata() const; + ColdHeader& cheader() const; + + // COPY OK -- BTS_NO_COPY(MDConsumerHandle); + void* _hdata; + HotHeader* _hheader; + ColdEntry* _cdata; + ColdHeader* _cheader; + MDProducerChannel _channel; +}; + +}}//namespace + diff --git a/src/server/sparkd/md/MDRecoverableConsumer.cpp b/src/server/sparkd/md/MDRecoverableConsumer.cpp new file mode 100644 index 000000000..b48f0f36a --- /dev/null +++ b/src/server/sparkd/md/MDRecoverableConsumer.cpp @@ -0,0 +1,155 @@ +#ifndef NO_CPP0X +#include "MDRecoverableConsumer.h" +#include "PlatformLoader.h" +#include "lib/logging/logging.h" +#include + + +using namespace std; +namespace bts { +namespace md { +using namespace md2; + +set get_symbols(Exchange::type); + +MDRecoverableConsumer::MDRecoverableConsumer() : // (visual studio 2012) MDRecoverableConsumer(get_symbols(Exchange::TESTEXCHANGE)) + _symbols(get_symbols(Exchange::TESTEXCHANGE)), + _current(init(_symbols)), + _old(), + _callbacks(), + _seen(), + _lock () +{} + +MDRecoverableConsumer::MDRecoverableConsumer(Exchange::type exchange) : // (visual studio 2012) MDRecoverableConsumer(get_symbols(exchange)) + _symbols(get_symbols(exchange)), + _current(init(_symbols)), + _old(), + _callbacks(), + _seen(), + _lock () +{} + +MDRecoverableConsumer::MDRecoverableConsumer(std::set symbols) : + _symbols(symbols), + _current(init(symbols)), + _old(), + _callbacks(), + _seen(), + _lock () +{ +} + +MDRecoverableConsumer::~MDRecoverableConsumer() {} + +unique_ptr MDRecoverableConsumer::init(set& symbols) { + unique_ptr result(new MDConsumerGroup()); + result->consumer = MDConsumer(symbols); + return result; +} + +size_t MDRecoverableConsumer::walk_security_list(SecurityCallback callback) { + return _current->consumer.walk_security_list(callback); +} +void MDRecoverableConsumer::on_security(SecurityCallback cb) { + // Add a filter to the callback to avoid calling the on_security callback with duplicates + auto filtered_cb = [this,cb](const Security& security, MDConsumerHandle handle) { + auto key = security.security_key(); + if (_seen.find(key) == _seen.end()) { + _seen.insert(key); + cb(security, handle); + } + }; + _current->consumer.on_security(filtered_cb); + Mutex::scoped_lock l (_lock); + _callbacks.push_back(filtered_cb); +} + +size_t MDRecoverableConsumer::scan_securities() { + return _current->consumer.scan_securities(); +} + +std::shared_ptr MDRecoverableConsumer::get_managed_handle(security_key key) { + Mutex::scoped_lock l (_lock); + auto handle = _current->consumer.get_managed_handle(key); + _current->handles.push_back(handle); + return handle; +} + +bool MDRecoverableConsumer::all_heartbeats_ok() { + return _current->consumer.all_heartbeats_ok(); +} + +bool MDRecoverableConsumer::try_recover() { + Mutex::scoped_lock l (_lock); + // Replace current with a new one (if the new one is better) + auto temp = init(_symbols); + if (!temp->consumer.all_heartbeats_ok()) return false; + + std::swap(temp, _current); + // push current to old + _old.push_back(std::move(temp)); + // copy old callbacks so we can reinstate them + auto old_callbacks = _callbacks; + + // Add an on_security callback to current to check the old handles + l.release(); + + // re-instante old callbacks + for (auto &cb : old_callbacks) { + _current->consumer.on_security(cb); + } + + _current->consumer.on_security([&](const Security& security, MDConsumerHandle newhandle) { + Mutex::scoped_lock l (_lock); + for (auto& group : _old) { + for (auto& handle : group->handles) { + if (handle->key() == security.security_key()) { + // update the handle! (if this is called guarunteed success) + *handle = newhandle; + bts_assert(handle->is_initialized()); + _current->handles.push_back(move(handle)); + } + } + } + cleanup(); + }); + + return true; +} + +// cleanup should only ever be called while _lock is locked +void MDRecoverableConsumer::cleanup() { + for (auto& group : _old) { + auto& handles = group->handles; + handles.erase( + remove_if( + handles.begin(), + handles.end(), + [](shared_ptr& handle) { return handle == nullptr; }), + handles.end()); + } + auto old_sz = _old.size (); + _old.erase( + remove_if( + _old.begin(), + _old.end(), + [](unique_ptr& pgroup) -> bool { return pgroup->handles.empty(); }), + _old.end()); + if (old_sz > _old.size ()) { + LOG (MarketData,Info,"Cleaned up {} MDConsumer instances" ,old_sz - _old.size () ); + } +} + + +// +// Utility functions +set get_symbols(Exchange::type exchange) { + set all_symbols; + get_system_md_symbols(exchange, SecurityType::UNSET, all_symbols); + return all_symbols; +} + + +}}//namespace +#endif diff --git a/src/server/sparkd/md/MDRecoverableConsumer.h b/src/server/sparkd/md/MDRecoverableConsumer.h new file mode 100644 index 000000000..8076a89da --- /dev/null +++ b/src/server/sparkd/md/MDRecoverableConsumer.h @@ -0,0 +1,63 @@ +#pragma once + +#include "lib/utils/bts_no_copy.h" +#include "common.h" +#include "MDConsumer.h" +#include "MDConsumerHandle.h" +#include "UserSymbol.h" +#include +#include +#include + +namespace bts { +namespace md { + +class BTS_EXPORT MDRecoverableConsumer +{ +public: + MDRecoverableConsumer(); + // Map only a certain exchange for more granular recovery + MDRecoverableConsumer(Exchange::type); + MDRecoverableConsumer(std::set symbols); + ~MDRecoverableConsumer(); + + // + // Security functions + size_t walk_security_list(md2::SecurityCallback callback); + void on_security(md2::SecurityCallback cb); + // Scans for added securities - updates managed handles + size_t scan_securities(); + + // + // Data Access + std::shared_ptr get_managed_handle(security_key key); + + // + // Recover from publisher failure + bool all_heartbeats_ok(); + bool try_recover(); + +private: + BTS_NO_COPY(MDRecoverableConsumer); + + void cleanup(); + + struct MDConsumerGroup { + MDConsumerGroup() : consumer(), handles() {} + md2::MDConsumer consumer; + std::vector> handles; + }; + std::unique_ptr init(std::set& symbols); + + std::set _symbols; + std::unique_ptr _current; + std::vector> _old; + + std::vector _callbacks; + std::set _seen; + typedef tbb::mutex Mutex; + Mutex _lock; +}; + +}}//namespace + diff --git a/src/server/sparkd/md/MemorySegmentUtils.cpp b/src/server/sparkd/md/MemorySegmentUtils.cpp new file mode 100644 index 000000000..1c2b867d4 --- /dev/null +++ b/src/server/sparkd/md/MemorySegmentUtils.cpp @@ -0,0 +1,27 @@ +#ifndef NO_CPP0X +#include "MemorySegmentUtils.h" +#include "lib/config/config.h" +#include "lib/config/protos/MarketData.pb.h" + +using namespace std; + +namespace bts { +namespace md2 { + +bool enable_depth(const string&, SecurityType::type type) { + return type == SecurityType::FUTURE || type == SecurityType::SPREAD; +} + +string segment_name(const string& symbol, SecurityType::type type, Exchange::type exchange, Platform::type platform) { + return "bts_edge_" + config::get().shared_memory_namespace() + "_" + symbol + "_" + SecurityType::type_Name(type) + "_" + Exchange::type_Name(exchange) + "_" + Platform::type_Name(platform); +} + +std::string hot_segment_name(const std::string& symbol, SecurityType::type type, Exchange::type exchange, Platform::type platform) { + return segment_name(symbol, type, exchange, platform) + "_hot"; +} +std::string cold_segment_name(const std::string& symbol, SecurityType::type type, Exchange::type exchange, Platform::type platform) { + return segment_name(symbol, type, exchange, platform) + "_cold"; +} + +}}//namespace +#endif diff --git a/src/server/sparkd/md/MemorySegmentUtils.h b/src/server/sparkd/md/MemorySegmentUtils.h new file mode 100644 index 000000000..de75e3354 --- /dev/null +++ b/src/server/sparkd/md/MemorySegmentUtils.h @@ -0,0 +1,14 @@ +#pragma once +#include "common.h" +#include + +namespace bts { +namespace md2 { + +bool enable_depth(const std::string& symbol, SecurityType::type type); +std::string segment_name(const std::string& symbol, SecurityType::type type, Exchange::type exchange, Platform::type platform); +std::string hot_segment_name(const std::string& symbol, SecurityType::type type, Exchange::type exchange, Platform::type platform); +std::string cold_segment_name(const std::string& symbol, SecurityType::type type, Exchange::type exchange, Platform::type platform); + + +}}//namespace diff --git a/src/server/sparkd/md/PlatformLoader.cpp b/src/server/sparkd/md/PlatformLoader.cpp new file mode 100644 index 000000000..afa42917a --- /dev/null +++ b/src/server/sparkd/md/PlatformLoader.cpp @@ -0,0 +1,125 @@ +#ifndef NO_CPP0X + +#include "PlatformLoader.h" +#include "lib/config/config.h" +#include "lib/security_model/SecurityHelper.h" + + +#include "MemorySegmentUtils.h" // for cleanup_shared_memory +#include "lib/utils/SharedList.h" // for cleanup_shared_memory + +#include "lib/config/protos/MarketData.pb.h" +#include "lib/config/protos/Symbols.pb.h" +#include + +#ifdef _LINUX +#include +#endif + +using namespace std; +using namespace bts; + +namespace bts { +namespace md2 { + +PlatformLoader::PlatformLoader(const md::PlatformConfig& config) : + _config(config), + _platform(Platform::UNSET) +{ + bts_assert_msg(Platform::type_Parse(config.platform(), &_platform), "Invalid market data platform " + config.platform()); +} + +PlatformLoader::~PlatformLoader(){} + +void cleanup_shared_memory(const vector& symbols, Platform::type platform) { + for (const auto& sym : symbols) { + string name = hot_segment_name(sym.symbol(), sym.type(), sym.exchange(), platform); + utils::SharedList::delete_segments(name); + LOG(MarketData,Info, "Cleaned up /run/shm/{}", name); + name = cold_segment_name(sym.symbol(), sym.type(), sym.exchange(), platform); + utils::SharedList::delete_segments(name); + LOG(MarketData,Info, "Cleaned up /run/shm/{}", name); + } +} + +// Does not return +void PlatformLoader::run() { + // Gather symbols to use + vector symbols; + size_t count = get_system_md_symbols(_config.exchange(), _config.type(), &symbols); + if (count == 0) { + LOG(MarketData,Error, "No Symbols configured for this publisher."); + return; + } + + // Quickly try to clean the shared memory segments before anyone gets a stale one + cleanup_shared_memory(symbols, _platform); + + + switch (_platform) { + case md::Platform::MDP3ONIXS: + break; + default: + bts_assert(false); + break; + } + bts_assert_msg(false, "Market data run should not return"); +} + + +/////////////////////////////// +size_t get_system_md_symbols(set& result) { + auto& config = config::get(); + return get_system_md_symbols(config.exchange(), config.type(), result); +} + +size_t get_system_md_symbols(Exchange::type exchange, SecurityType::type type, std::set& result) { + auto const &all_symbols = config::get ().symbol(); + for (const auto &s : all_symbols) { + if (s.exchange() != Exchange::TESTEXCHANGE && + (exchange == Exchange::TESTEXCHANGE || exchange == s.exchange()) && + (type == SecurityType::UNSET || type == s.type())) { + UserSymbol us(s.symbol(),s.exchange(), s.type()); + result.insert(us); + } + } + return result.size(); +} + +size_t get_system_md_symbols(Exchange::type exchange, SecurityType::type type, std::vector* result) { + bts_assert(result); + set result_set; + size_t size = get_system_md_symbols(exchange, type, result_set); + result->reserve(size); + result->assign(result_set.begin(), result_set.end()); + return size; +} + +extern "C" BTS_EXPORT Platform::type platform_for_exchange(Exchange::type exchange) { + using namespace bts::md; + const auto& pmap = config::get(); + + const PlatformMap::PlatformProps platforms[] = { + pmap.sim(), pmap.srlabs(), pmap.spryware(), pmap.csm(), pmap.cme(), pmap.ice(), pmap.replay(), pmap.activ(), pmap.mdp3(), pmap.csm_onixs(), pmap.blue() + }; + const Platform::type types[] = { + Platform::SIM, Platform::SRLABS, Platform::SPRYWARE, Platform::CSM, Platform::CME, Platform::ICE, Platform::REPLAY, Platform::ACTIV, Platform::MDP3ONIXS, Platform::CSM_ONIXS, Platform::BLUE + }; + static_assert(BTS_ARRAYSIZE(platforms) == BTS_ARRAYSIZE(types),"Platforms should be the same size as types"); + + for (size_t x = 0; x < BTS_ARRAYSIZE(platforms); x++) + { + for (int y = 0; y < platforms[x].exchange_size(); ++y) { + if (platforms[x].exchange(y) == exchange) { + return types[x]; + } + } + } + + return Platform::UNSET; +} + + +}}//namespace + +#endif diff --git a/src/server/sparkd/md/PlatformLoader.h b/src/server/sparkd/md/PlatformLoader.h new file mode 100644 index 000000000..99b330eae --- /dev/null +++ b/src/server/sparkd/md/PlatformLoader.h @@ -0,0 +1,39 @@ +#pragma once + +#include "lib/utils/bts_no_copy.h" +#include "UserSymbol.h" +#include "common.h" +#include "get_symbols.h" + +#include "lib/config/protos/MarketData.pb.h" + +#include +#include +#include + +namespace bts { +namespace md2 { + +class BTS_EXPORT PlatformLoader +{ +public: + PlatformLoader(const md::PlatformConfig&); + ~PlatformLoader(); + + // Does not return + void run(); + +private: + // Some linux assemblies don't have a proper SONAME so we preload them by path to satisfy imports + void preload(Platform::type); + BTS_NO_COPY(PlatformLoader); + + const md::PlatformConfig& _config; + Platform::type _platform; +}; + + +BTS_EXPORT void cleanup_shared_memory(const std::vector& symbols, Platform::type platform); + +}}//namespace + diff --git a/src/server/sparkd/md/SecurityConstructor.h b/src/server/sparkd/md/SecurityConstructor.h new file mode 100644 index 000000000..05a758e5c --- /dev/null +++ b/src/server/sparkd/md/SecurityConstructor.h @@ -0,0 +1,56 @@ +#pragma once + +#include "core/bts_core/utils/bts_no_copy.h" +#include "Structure/FixedSecurity.h" +#include "../bts_conn_exports.h" +#include +#include +#include + +namespace bts { +namespace md2 { + + +class BTS_EXPORT SecurityConstructor { + // An object "pool" optimized for the case where only one instance is ever used at a time + template + class SingleObjectPool { + public: + using ptr_type = std::unique_ptr>; + ptr_type acquire() { + T* pt = _inst ? _inst.release() : new T(); + return ptr_type(pt, [this](T* ptr) { + return_inst(std::unique_ptr(ptr)); + }); + } + private: + void return_inst(std::unique_ptr t) { + if (!_inst) { + reset(t); + _inst = std::move(t); + } + } + unique_ptr _inst; + }; + +public: + SecurityConstructor() : _future(), _option(), _spread() {} + + SingleObjectPool& future() { return _future; } + SingleObjectPool& option() { return _option; } + SingleObjectPool& spread() { return _spread; } + +private: + template + void reset(T& t) { + new (&t)T(); + } + + SingleObjectPool _future; + SingleObjectPool _option; + SingleObjectPool _spread; +}; + + +}}//namespace + diff --git a/src/server/sparkd/md/SecurityIDRecorder.cpp b/src/server/sparkd/md/SecurityIDRecorder.cpp new file mode 100644 index 000000000..1009883cd --- /dev/null +++ b/src/server/sparkd/md/SecurityIDRecorder.cpp @@ -0,0 +1,198 @@ +#include "SecurityIDRecorder.h" + +#include "lib/utils/defer.h" + +#include "lib/config/config.h" +#include "lib/utils/Sqlite_utils.h" +#include "lib/logging/logging.h" +#include +#include // for reservation + +#include "lib/config/protos/MarketData.pb.h" +#include "lib/config/protos/system.pb.h" + +#include +#include + +using namespace std; +using namespace bts::utils; + +namespace bts { +namespace md2 { + +//GCC_DIAG_OFF("-Weffc++") // Plain Struct +struct SecurityIDTask { + std::string descriptor; + SecurityType::type type; + SecurityIDRecorder::callback cb; + void* arg; +}; +//GCC_DIAG_RESTORE() +struct SecurityIDRecorderState { + SecurityIDRecorderState() : + work(), + pending_handle(), + db(nullptr, sqlite3_close), + select_stmt(nullptr, sqlite3_finalize), + insert_stmt(nullptr, sqlite3_finalize) + {} + tbb::concurrent_queue work; + tbb::concurrent_queue pending_handle; + sqlite3_ptr db; + sqlite3_stmt_ptr select_stmt; + sqlite3_stmt_ptr insert_stmt; +}; + +SecurityIDRecorder::SecurityIDRecorder(): + _self(new SecurityIDRecorderState()) +{ + init(); +} + +SecurityIDRecorder::~SecurityIDRecorder() +{ +} + +void SecurityIDRecorder::init() { + string db_dir = config::get().data_path() + "/md/"; + string db_path = db_dir + "/security_keys.sqlite"; + + if (!boost::filesystem::exists(db_dir)) { + boost::filesystem::create_directories(db_dir); + } + + _self->db = open_db(db_path.c_str()); + + // Set a long busy timeout + sqlite3_busy_timeout(_self->db.get(), 9999); + sqlite3_exec_nofail(_self->db, "PRAGMA journal_mode=WAL"); + sqlite3_exec_nofail(_self->db, "PRAGMA syncronous=0"); + sqlite3_exec_nofail(_self->db, "CREATE TABLE IF NOT EXISTS security_keys (descriptor STRING NOT NULL PRIMARY KEY, type int);"); + + // Prepare statements + _self->insert_stmt = prepare_query(_self->db, "INSERT INTO security_keys (descriptor, type) values (?, ?)"); + _self->select_stmt = prepare_query(_self->db, "SELECT rowid from security_keys where descriptor=? LIMIT 1;"); +} + +void SecurityIDRecorder::get_key(const std::string& descriptor, SecurityType::type type, callback cb, void* arg) { + SecurityIDTask task; + task.descriptor = descriptor; + task.type = type; + task.cb = cb; + task.arg = arg; + _self->work.push(task); +} + +void SecurityIDRecorder::get_key(const std::string& descriptor, SecurityType::type type, Callback cb) { + Callback* pcb = new Callback(cb); + get_key(descriptor, type, [](void* data, security_key key) { + Callback* pcb = static_cast(data); + DEFER(if (pcb) delete pcb); // exception safe + (*pcb)(key); + }, pcb); +} + +void SecurityIDRecorder::ensure_key(MDProducerHandle handle) { + _self->pending_handle.push(handle); +} + +bool SecurityIDRecorder::do_work() { + // Check if any handles need work first + work_pending_handles(); + + // Drain the work queue + // Query the database for each security. If it has an ID return it + // If it does not have an ID add it to a list + // Then persist that list + SecurityIDTask temp; + vector new_securities; + size_t count = 0; + bool incomplete = false; + while (_self->work.try_pop(temp)) { + security_key key = search(temp.descriptor); + if (key != invalid_security_key) { + temp.cb(temp.arg, key); + } + else { + new_securities.push_back(temp); + ++count; + // Limit the amount of time we spend writing to sqlite during startup to avoid blocking the heartbeat + if (count > 500) { + incomplete = true; + break; + } + } + } + + + if (new_securities.size() > 0) { + sqlite3_exec_nofail(_self->db.get(), "BEGIN TRANSACTION"); + + for (SecurityIDTask& security : new_securities) { + security_key key = insert(security.descriptor, security.type); + bts_assert_d(key); + security.cb(security.arg, key); + } + + sqlite3_exec_nofail(_self->db.get(), "COMMIT TRANSACTION"); + } + return incomplete; +} + +void SecurityIDRecorder::work_pending_handles() { + MDProducerHandle handle; + vector defer; + while(_self->pending_handle.try_pop(handle)) { + // A spread will not have a descriptor until all of its leg keys are available + string descriptor = security_descriptor(handle.security()); + if (descriptor.empty()) { + defer.push_back(handle); + continue; + } + + MDProducerHandle* phandle = new MDProducerHandle(handle); + DEFER(if (phandle) delete phandle); + get_key(descriptor, handle.security().type, [](void* arg, security_key key) { + auto* phandle = static_cast(arg); + if (phandle) { + phandle->set_key(key); + delete phandle; + } + }, phandle); + phandle = nullptr; + } + + for (auto& handle : defer) { + _self->pending_handle.push(handle); + } +} + + +security_key SecurityIDRecorder::search(const std::string& descriptor) { + sqlite3_bind_text(_self->select_stmt.get(), 1, descriptor.c_str(), descriptor.size(), SQLITE_STATIC); + DEFER(sqlite3_reset(_self->select_stmt.get())); + int result = sqlite3_step(_self->select_stmt.get()); + if (result == SQLITE_DONE || result == SQLITE_ROW) { + return sqlite3_column_int(_self->select_stmt.get(), 0); + } + return invalid_security_key; +} + +security_key SecurityIDRecorder::insert(const std::string& descriptor, SecurityType::type type) { + sqlite3_bind_text(_self->insert_stmt.get(), 1, descriptor.c_str(), descriptor.size(), SQLITE_STATIC); + sqlite3_bind_int(_self->insert_stmt.get(), 2, (int)type); + DEFER(sqlite3_reset(_self->insert_stmt.get())); + int result = sqlite3_step(_self->insert_stmt.get()); + if (result == SQLITE_DONE || result == SQLITE_ROW) { + return sqlite3_last_insert_rowid(_self->db.get()); + } + // Just in case our insert failed due to duplicate row + auto key = search(descriptor); + if (key == invalid_security_key) { + LOG(MarketData,Error,"Failed to assign security key. SQLite return code: {}",result); + } + return key; +} + +}}//namespace + diff --git a/src/server/sparkd/md/SecurityIDRecorder.h b/src/server/sparkd/md/SecurityIDRecorder.h new file mode 100644 index 000000000..8430dc520 --- /dev/null +++ b/src/server/sparkd/md/SecurityIDRecorder.h @@ -0,0 +1,56 @@ +#pragma once + +#include "common.h" +#include "MDProducerHandle.h" +#include "lib/utils/bts_export.h" + +#include +#include +#include +#include +#include + +namespace bts { +namespace md2 { + +/* +* Manages security to unique 32 bit id mapping across multiple processes +* Provides a callback when ids are safely persisted. +*/ + +struct SecurityIDRecorderState; +class BTS_EXPORT SecurityIDRecorder +{ +public: + SecurityIDRecorder(); + ~SecurityIDRecorder(); + typedef void(*callback)(void*, security_key); + typedef std::function Callback; + + // Asyncronously retrieve the security_key for the given descriptor + void get_key(const std::string& descriptor, SecurityType::type, callback cb, void* arg); + + // A less efficient but higher level get_key which performs heap allocation in the call path + void get_key(const std::string& descriptor, SecurityType::type, Callback cb); + + // A higher level interface which allows you to just pass in a handle and have it generate a descriptor and set the key + void ensure_key(MDProducerHandle handle); + + // Someone must call work periodically to do lookups and flush the reserved ids to disk. + // returne true if there is more work to be done + bool do_work(); + +private: + std::unique_ptr _self; + + void init(); + security_key search(const std::string& descriptor); + security_key insert(const std::string& descriptor, SecurityType::type); + + void work_pending_handles(); + + BTS_NO_COPY(SecurityIDRecorder); +}; + +}}//namespace + diff --git a/src/server/sparkd/md/SecurityIndex.h b/src/server/sparkd/md/SecurityIndex.h new file mode 100644 index 000000000..1abc2dee5 --- /dev/null +++ b/src/server/sparkd/md/SecurityIndex.h @@ -0,0 +1,204 @@ +#pragma once + +#include "lib/utils/bts_no_copy.h" +#include "lib/logging/logging.h" +#include "server/sparkd/md/MDProducerHandle.h" +#include "common.h" +#include "SpreadResolver.h" +#include +#include +#include +#include +#include +#include + +namespace bts { +namespace md2 { + + +/* + * Maps the exchange ID to the MDProducerHandle and associated data. + * Allows fast mapping of exchange id => SecurityData template type + */ + +template +class SimpleSecurityIndex +{ +public: + SimpleSecurityIndex(): + _map(), + _spread_resolver() + {} + ~SimpleSecurityIndex() {} + + bool has_security(ExchangeKey key) const { + tbb::queuing_rw_mutex::scoped_lock lock(_mutex, false); + return _map.find(key) != _map.end(); + } + + SecurityData* get_security(ExchangeKey key) const { + tbb::queuing_rw_mutex::scoped_lock lock(_mutex, false); + auto result = _map.find(key); + if (result == _map.end()) { + return nullptr; + } + return result->second.get(); + } + + security_key get_key(ExchangeKey key) const { + auto security = get_security(key); + if (security != nullptr) { + return security->handle.key(); + } + return invalid_security_key; + } + + template + void walk_security_list(CB cb) const { + tbb::queuing_rw_mutex::scoped_lock lock(_mutex, false); + for (const auto& pair : _map) { + cb(*pair.second); + } + } + + // Transfer ownership of the data to the index + SecurityData* add_security(ExchangeKey key, std::unique_ptr entry) { + tbb::queuing_rw_mutex::scoped_lock lock(_mutex, true); + // result is pair + auto result = _map.insert(std::make_pair(key, std::unique_ptr())); + if (!result.second) { + LOG(MarketData,Error, "Found duplicate entry for platform id '{} - " ,key); + bts_assert(result.second); + } + result.first->second = std::move(entry); + return result.first->second.get(); + } + + SecurityData* add_security(ExchangeKey key, MDProducerHandle& handle) { + std::unique_ptr data(new SecurityData(key, handle)); + return add_security(key, std::move(data)); + } + + // + // Spread resolve logic + + void add_spread_legs(ExchangeKey key, const std::vector& legs) { + _spread_resolver.add(key, legs); + } + + // Resolves spreads and returns the number spreads resolved + size_t resolve_spreads() { + return _spread_resolver.resolve( + [&](ExchangeKey key) { return get_key(key); }, + [&](ExchangeKey& key, std::vector& legs) { on_spread_resolved(key, legs); }); + } + +protected: + void on_spread_resolved(ExchangeKey& key, std::vector& legs) { + auto* entry = get_security(key); + bts_assert_d(entry); + if (entry) { + entry->handle.set_legs(legs); + } + } + +private: + BTS_NO_COPY(SimpleSecurityIndex); + typedef std::unordered_map> MapType; + MapType _map; + mutable tbb::queuing_rw_mutex _mutex; + SpreadResolver _spread_resolver; +}; + + +// +// SecurityIndex using tbb concurrent_hash_map +// + +template +class ConcurrentSecurityIndex +{ +public: + ConcurrentSecurityIndex() : + _map(), + _spread_resolver() + {} + ~ConcurrentSecurityIndex() {} + + SecurityData* get_security(ExchangeKey key) const { + typename MapType::const_accessor accessor; + if (_map.find(accessor, key)) { + return accessor->second; + } + return nullptr; + } + + bool has_security(ExchangeKey key) const { + return get_security(key) != nullptr; + } + + security_key get_key(ExchangeKey key) const { + const auto* result = get_security(key); + if (result != nullptr) { + return result->handle.key(); + } + return invalid_security_key; + } + + template + void walk_security_list(CB cb) const { + for (auto iter = _map.begin(); iter != _map.end(); ++iter) { + cb(*iter->second); + } + } + + // Transfer ownership of the data to the index + SecurityData* add_security(ExchangeKey key, std::unique_ptr entry) { + typename MapType::accessor accessor; + if (_map.insert(accessor, key)) { + // There is no delete security and concurrent_has_map does not yet support std::move + accessor->second = entry.release(); + return accessor->second; + } + else { + LOG(MarketData,Error, "Found duplicate entry for platform id '{}' - ",key); + bts_assert(false); + } + } + + SecurityData* add_security(ExchangeKey key, MDProducerHandle& handle) { + std::unique_ptr data(new SecurityData(key, handle)); + return add_security(key, std::move(data)); + } + + // + // Spread resolve logic + + void add_spread_legs(ExchangeKey key, const std::vector& legs) { + _spread_resolver.add(key, legs); + } + + // Resolves spreads and returns the number spreads resolved + size_t resolve_spreads() { + return _spread_resolver.resolve( + [&](ExchangeKey key) { return get_key(key); }, + [&](ExchangeKey& key, std::vector& legs) { on_spread_resolved(key, legs); }); + } + +protected: + void on_spread_resolved(ExchangeKey& key, std::vector& legs) { + auto* entry = get_security(key); + bts_assert_d(entry); + if (entry) { + entry->handle.set_legs(legs); + } + } +private: + BTS_NO_COPY(ConcurrentSecurityIndex); + typedef tbb::concurrent_hash_map MapType; + MapType _map; + SpreadResolver _spread_resolver; +}; + +}}//namespace + diff --git a/src/server/sparkd/md/SettlementRecorder.cpp b/src/server/sparkd/md/SettlementRecorder.cpp new file mode 100644 index 000000000..59c6d0a03 --- /dev/null +++ b/src/server/sparkd/md/SettlementRecorder.cpp @@ -0,0 +1,140 @@ +#ifndef NO_CPP0X +#include "SettlementRecorder.h" +#include "lib/utils/Sqlite_utils.h" +#include "lib/config/config.h" +#include "lib/config/protos/system.pb.h" +#include +#include +#include +#include +#include +#include + +namespace bts { +namespace co2 { + +using datetime::Date; +using std::unique_ptr; +using std::string; +using namespace utils; + +enum WorkType { + SNAPSHOT, + UPDATE +}; + +struct Work { + WorkType type; + MDProducerHandle handle; + Price price; + Date date; +}; + +struct SettlementRecorderImpl { + SettlementRecorderImpl() : + work_queue(), + db(nullptr, sqlite3_close), + select_stmt(nullptr, sqlite3_finalize), + insert_stmt(nullptr, sqlite3_finalize) + {} + tbb::concurrent_queue work_queue; + sqlite3_ptr db; + sqlite3_stmt_ptr select_stmt; + sqlite3_stmt_ptr insert_stmt; +}; + +bool record_settle(sqlite3_stmt* insert, security_key key, Price price, Date date) { + sqlite3_bind_int64(insert, 1, key); + sqlite3_bind_int(insert, 2, date.to_short()); + sqlite3_bind_double(insert, 3, price); + auto result = sqlite3_step(insert); + sqlite3_reset(insert); + return (result == SQLITE_DONE || result == SQLITE_ROW); +} +void publish_settles(sqlite3_stmt* select, MDProducerHandle handle) { + // Query sqlite for the top 2 settlements + sqlite3_bind_int(select, 1, handle.key()); + auto result = sqlite3_step(select); + if (result == SQLITE_ROW) { + Price price = sqlite3_column_double(select, 0); + Date date = Date(sqlite3_column_int(select, 1)); + handle.update_settlement(TimeStamp::now(), price, date); + result = sqlite3_step(select); + if (result == SQLITE_ROW) { + price = sqlite3_column_double(select, 0); + date = Date(sqlite3_column_int(select, 1)); + handle.update_previous_settlement(TimeStamp::now(), price, date); + } + } + sqlite3_reset(select); +} + +void apply_work(SettlementRecorderImpl& me, Work work) { + switch (work.type) { + case UPDATE: + if (record_settle(me.insert_stmt.get(), work.handle.key(), work.price, work.date)) { + publish_settles(me.select_stmt.get(), work.handle); + } + break; + case SNAPSHOT: + publish_settles(me.select_stmt.get(), work.handle); + } +} + +SettlementRecorder::SettlementRecorder(Platform::type platform) : + _self(new SettlementRecorderImpl()) +{ + string db_dir = config::get().data_path() + "/md/"; + string db_path = db_dir + "/settlements_" + Platform::type_Name(platform) +".sqlite"; + if (!boost::filesystem::exists(db_dir)) { + boost::filesystem::create_directories(db_dir); + } + + // initialize database + _self->db = open_db(db_path.c_str()); + sqlite3* db = _self->db.get(); + sqlite3_busy_timeout(db, 9999); + sqlite3_exec_nofail(db, "PRAGMA journal_mode=OFF"); + sqlite3_exec_nofail(db, "PRAGMA syncronous=0"); + // key=(security_key+date) + sqlite3_exec_nofail(db, "CREATE TABLE IF NOT EXISTS settlements (key INTEGER NOT NULL, date INTEGER NOT NULL, price REAL NOT NULL, PRIMARY KEY (key,date));"); + + // Delete any records > 10 days old + auto cutoff = datetime::local_now() - boost::gregorian::days(10); + auto encoded = datetime::ll_to_date(datetime::ptime_to_ll(cutoff)).to_short(); + char query[256]; + sprintf(query, "delete from settlements where date < %i", encoded); + sqlite3_exec_nofail(db, query); + + // statements + _self->select_stmt = prepare_query(db, "select price,date from settlements where key=? order by date desc limit 2"); + _self->insert_stmt = prepare_query(db, "insert or replace into settlements(key, date, price) values(?,?,?)"); +} + +SettlementRecorder::~SettlementRecorder() { } + +void SettlementRecorder::restore_settlement(MDProducerHandle handle) { + Work work = { SNAPSHOT, handle, 0, Date() }; + _self->work_queue.push(work); +} + +void SettlementRecorder::add_settlement(MDProducerHandle handle, Price price, datetime::Date date) { + Work work = { UPDATE, handle, price, date }; + _self->work_queue.push(work); +} + +void SettlementRecorder::do_work() { + TimeStamp start = TimeStamp::now(); + Work work; + while ((TimeStamp::now() - start).milliseconds() < 100 && + _self->work_queue.try_pop(work)) { + if (work.handle.key() == invalid_security_key) { + _self->work_queue.push(work); + continue; + } + apply_work(*_self, work); + } +} + +}}//namespace +#endif diff --git a/src/server/sparkd/md/SettlementRecorder.h b/src/server/sparkd/md/SettlementRecorder.h new file mode 100644 index 000000000..1229225e5 --- /dev/null +++ b/src/server/sparkd/md/SettlementRecorder.h @@ -0,0 +1,31 @@ +#pragma once + +#include "MDProducerHandle.h" +#include "lib/utils/bts_export.h" +#include "lib/utils/datetime.h" +#include + + +namespace bts { +namespace co2 { +using namespace md2; + +struct SettlementRecorderImpl; +class BTS_EXPORT SettlementRecorder +{ +public: + SettlementRecorder(Platform::type platform); + ~SettlementRecorder(); + + // Put settlement update in a queue + // in do_work() drain queue and update settlement 'depth' + void restore_settlement(MDProducerHandle handle); + void add_settlement(MDProducerHandle handle, Price price, datetime::Date date); + void do_work(); + +private: + std::unique_ptr _self; +}; + +}}//namespace + diff --git a/src/server/sparkd/md/SpreadResolver.h b/src/server/sparkd/md/SpreadResolver.h new file mode 100644 index 000000000..20d3fa161 --- /dev/null +++ b/src/server/sparkd/md/SpreadResolver.h @@ -0,0 +1,79 @@ +#pragma once + +#include "lib/utils/bts_no_copy.h" +#include +#include + +namespace bts { +namespace md2 { + + +/* + * SpreadResolved accepts exchange spread definitions and resolves the legs + */ + +template +class SpreadResolver { + struct ResolverEntry { + ResolverEntry(ExchangeKey key) : exchange_key(key), legs() {} + ResolverEntry() : exchange_key(), legs() {} + ExchangeKey exchange_key; + std::vector legs; + }; +public: + SpreadResolver() : _queue() {} + template + void add(ExchangeKey key, LegContainer legs) { + ResolverEntry entry(key); + entry.legs.insert(end(entry.legs), begin(legs), end(legs)); + _queue.push(entry); + } + + // callback should accept const ExchangeKey& key, vector& legs) + // When the callback is called the legs will be completely resolved + template + size_t resolve(MapKey keymap, CallBack on_resolved) { + // Drain the queue and call keymap to resolve each leg + size_t result = 0; + std::vector unresolved; + std::vector legs; + while(!_queue.empty()) { + legs.clear(); + ResolverEntry entry; + if(_queue.try_pop(entry)) { + // Attempt to look up each spread leg + for(ExchangeKey key : entry.legs) { + security_key result = keymap(key); + if (result == invalid_security_key) { + unresolved.push_back(entry); + break; + } + else { + legs.push_back(result); + } + } + if (legs.size() == entry.legs.size()) { + ++result; + on_resolved(entry.exchange_key, legs); + } + } + } + + // If any legs remain unresolved push the entry back in the queue for later + for(auto& entry : unresolved) { + _queue.push(entry); + } + + return result; + } + + size_t count() const { return _queue.unsafe_size(); } +private: + BTS_NO_COPY(SpreadResolver); + tbb::concurrent_queue _queue; +}; + + + +}}//namespace + diff --git a/src/server/sparkd/md/Structure/FixedSecurity.cpp b/src/server/sparkd/md/Structure/FixedSecurity.cpp new file mode 100644 index 000000000..f3605cdef --- /dev/null +++ b/src/server/sparkd/md/Structure/FixedSecurity.cpp @@ -0,0 +1,283 @@ +#ifndef NO_CPP0X + +#include "FixedSecurity.h" +#include "PriceSize.h" +#include + +#ifdef _WIN32 +#pragma warning(disable: 4351) +#endif + +using namespace std; + +namespace bts { +namespace md2 { + +FixedSecurity::FixedSecurity() : + size(sizeof(FixedSecurity)), + key(invalid_security_key), + numeric_id(0), + type(SecurityType::STOCK), + exchange(Exchange::TESTEXCHANGE), + display_factor(0), + strike_display_factor(0), + contract_size(0), + cboe_class_key(0), + cme_vtt(0), + cme_display_decimals(0), + cme_denominator_sub_fraction(0), + cme_min_price_increment(0), + cme_market_segment_id(0), + cme_unit_of_measure_qty(0), + _symbol(), + _cfi_code(), + _security_desc(), + _alt_symbol() +{ + memset(_symbol, 0, BTS_ARRAYSIZE(_symbol)); + memset(_alt_symbol, 0, BTS_ARRAYSIZE(_alt_symbol)); + memset(_cfi_code, 0, BTS_ARRAYSIZE(_cfi_code)); + memset(_security_desc, 0, BTS_ARRAYSIZE(_security_desc)); + memset(_platform_id, 0, BTS_ARRAYSIZE(_platform_id)); +} + +FixedSecurity::~FixedSecurity(){} + +FixedFutureSecurity::FixedFutureSecurity(): + expiration_year(0), + expiration_month(0), + expiration_day(0) +{ + type = SecurityType::FUTURE; + size = sizeof(FixedFutureSecurity); +} + +FixedOptionSecurity::FixedOptionSecurity() : + expiration_year(0), + expiration_month(0), + expiration_day(0), + strike(0) +{ + type = SecurityType::OPTION; + size = sizeof(FixedOptionSecurity); +} + +FixedSpreadLeg::FixedSpreadLeg(): + key(invalid_security_key), + leg_side(Side::type_MIN), + leg_quantity(0), + leg_price(0), + leg_delta(1) +{ +} + +void write_date_string(uint32_t year, uint32_t month, uint32_t day, std::ostringstream& result) { + result << std::setw(2) << std::setfill('0') << month << "/" + << std::setw(2) << std::setfill('0') << day << "/" + << year - 2000; +} + +const std::string security_descriptor(const FixedSecurity& security) { + const char separator = ' '; + std::ostringstream result; + // If the same symbol is listed at 2 different exchanges it should have a different ID + result << security.symbol() << separator << Exchange::type_Name(security.exchange); + switch(security.type) { + case SecurityType::OPTION: { + auto* psec = (FixedOptionSecurity*)&security; + result << separator; + write_date_string(psec->expiration_year, psec->expiration_month, psec->expiration_day, result); + result << separator << psec->strike << (psec->call_put == PutOrCall::CALL ? 'C' : 'P'); + break; + } + case SecurityType::FUTURE: { + auto* psec = (FixedFutureSecurity*)&security; + result << separator; + write_date_string(psec->expiration_year, psec->expiration_month, psec->expiration_day, result); + break; + } + case SecurityType::SPREAD: { + // Since a fixed spread is normalized instead of writing out the leg details + // we write out the fixed ids of the legs. If any leg is "invalid_id" return an error + auto* psec = (FixedSpreadSecurityL*)&security; + bts_assert(psec->leg_count() > 0); + for (size_t x = 0; x < psec->leg_count(); ++x) { + auto& leg = psec->legs[x]; + if (leg.key == invalid_security_key) return ""; + result << separator << leg.key << "-q" << leg.leg_quantity << "p" << leg.leg_price << "d" << leg.leg_delta << Side::type_Name(leg.leg_side); + } + break; + } + case SecurityType::STOCK: // already have symbol+exchange + default: + break; + } + + return result.str(); +} + +Security convert_security(const FixedSecurity& security) { + Security res; + convert_security(security, &res); + return res; +} +void convert_security(const FixedSecurity& security, Security* pres) { + bts_assert(pres); + Security& res = *pres; + res.set_security_key(security.key); + res.set_security_type(security.type); + res.set_exchange(security.exchange); + + res.set_display_factor(pow10(security.display_factor)); + res.set_strike_display_factor(pow10(security.strike_display_factor)); + res.set_numeric_id(security.numeric_id); + { + std::ostringstream convert; + convert << security.numeric_id; + res.set_security_id(convert.str()); + } + res.set_contract_size(security.contract_size); + + if (security.cboe_class_key) { res.set_class_key(security.cboe_class_key); } + if (security.cboe_premium_break) { res.set_cboe_premium_break(security.cboe_premium_break); } + if (security.cboe_minimum_above_premium_fraction) { res.set_cboe_minimum_above_premium_fraction(security.cboe_minimum_above_premium_fraction); } + if (security.cboe_minimum_below_premium_fraction) { res.set_cboe_minimum_below_premium_fraction(security.cboe_minimum_below_premium_fraction); } + + if (security.cme_vtt) { res.set_cme_vtt(security.cme_vtt); } + if (security.cme_display_decimals) { res.set_cme_display_decimals(security.cme_display_decimals); } + if (security.cme_denominator_sub_fraction) { res.set_cme_denominator_sub_fraction(security.cme_denominator_sub_fraction); } + if (security.cme_min_price_increment) { res.set_cme_min_price_increment(security.cme_min_price_increment); } + if (security.cme_market_segment_id) { res.set_cme_market_segment_id(security.cme_market_segment_id); } + if (security.cme_unit_of_measure_qty) { res.set_cme_unit_of_measure_qty(security.cme_unit_of_measure_qty); } + res.set_cfi_code(security.cfi_code()); + res.set_security_desc(security.security_desc()); + res.set_platform_id(security.platform_id()); + if (strlen(security.alt_symbol()) > 0) { + res.set_cme_symbol(security.alt_symbol()); + } + + switch(security.type) { + case SecurityType::STOCK: + res.mutable_stock()->set_symbol(security.symbol()); + break; + case SecurityType::OPTION: { + bts_assert(security.size == sizeof(FixedOptionSecurity)); + const FixedOptionSecurity* psec = static_cast(&security); + res.mutable_option()->set_symbol(security.symbol()); + res.mutable_option()->set_expiration_year(psec->expiration_year); + res.mutable_option()->set_expiration_month(psec->expiration_month); + res.mutable_option()->set_expiration_day(psec->expiration_day); + res.mutable_option()->set_strike(psec->strike); + res.mutable_option()->set_call_put(psec->call_put); + break; + } + case SecurityType::FUTURE: { + bts_assert(security.size == sizeof(FixedFutureSecurity)); + const auto* psec = static_cast(&security); + res.mutable_future()->set_symbol(security.symbol()); + res.mutable_future()->set_expiration_year(psec->expiration_year); + res.mutable_future()->set_expiration_month(psec->expiration_month); + res.mutable_future()->set_expiration_day(psec->expiration_day); + break; + } + case SecurityType::SPREAD: { + auto* spread = res.mutable_spread(); + spread->set_symbol(security.symbol()); + spread->set_normalized(true); + const auto* psec = static_cast(&security); + for (size_t leg_index = 0; leg_index < psec->leg_count(); ++leg_index) { + const FixedSpreadLeg& fleg = psec->legs[leg_index]; + auto* leg = spread->add_legs(); + leg->set_leg_side(fleg.leg_side); + leg->set_leg_quantity(fleg.leg_quantity); + leg->set_leg_price(fleg.leg_price); + leg->set_leg_delta(fleg.leg_delta); + // Set the leg key but leave the rest of the security uninitialized + leg->mutable_security()->set_security_key(fleg.key); + leg->mutable_security()->set_security_type(SecurityType::UNSET); // required + leg->mutable_security()->set_exchange(security.exchange); // required + } + break; + } + default: bts_assert(false); + } +} + +size_t recursive_leg_count(const Security& security) { + if (security.security_type() != SecurityType::SPREAD) return 0; + size_t result = security.spread().legs_size(); + for (const auto& leg : security.spread().legs()) { + result += recursive_leg_count(leg.security()); + } + return result; +} + +void convert_security(const Security& src, FixedSecurity& dest, bool set_security_key) { + dest.key = set_security_key ? src.security_key() : invalid_security_key; + dest.type = src.security_type(); + dest.exchange = src.exchange(); + dest.display_factor = to_exponent(src.display_factor()); + dest.strike_display_factor = to_exponent(src.strike_display_factor()); + + dest.numeric_id = src.numeric_id(); + dest.contract_size = src.contract_size(); + dest.cboe_class_key = src.class_key(); + dest.cboe_premium_break = src.cboe_premium_break(); + dest.cboe_minimum_above_premium_fraction = src.cboe_minimum_above_premium_fraction(); + dest.cboe_minimum_below_premium_fraction = src.cboe_minimum_below_premium_fraction(); + dest.cme_vtt = src.cme_vtt(); + dest.cme_display_decimals = src.cme_display_decimals(); + dest.cme_denominator_sub_fraction = src.cme_denominator_sub_fraction(); + dest.cme_min_price_increment = src.cme_min_price_increment(); + dest.cme_market_segment_id = src.cme_market_segment_id(); + dest.cfi_code(src.cfi_code()); + dest.security_desc(src.security_desc()); + dest.alt_symbol(src.cme_symbol()); + dest.platform_id(src.platform_id()); + + switch(src.security_type()) { + case SecurityType::STOCK: + dest.symbol(src.stock().symbol()); + break; + case SecurityType::FUTURE: { + dest.symbol(src.future().symbol()); + FixedFutureSecurity& future = static_cast(dest); + future.expiration_year = src.future().expiration_year(); + future.expiration_month = src.future().expiration_month(); + future.expiration_day = src.future().expiration_day(); + break; + } + case SecurityType::OPTION: { + dest.symbol(src.option().symbol()); + FixedOptionSecurity& option = static_cast(dest); + option.expiration_year = src.option().expiration_year(); + option.expiration_month = src.option().expiration_month(); + option.expiration_day = src.option().expiration_day(); + option.strike = src.option().strike(); + option.call_put = src.option().call_put(); + break; + } + case SecurityType::SPREAD: { + // For nested spreads this preserves the structure when normalizing + dest.symbol(src.spread().symbol()); + FixedSpreadSecurityXL& spread = static_cast(dest); + // set_leg_count sets up the size of the spread and verified that the number of legs does not exceed the allocated space + spread.set_leg_count(src.spread().legs_size()); // NOT recursive as Co2 spreads are normalized + for (int x = 0; x < src.spread().legs_size(); ++x) { + const auto& src_leg = src.spread().legs(x); + spread.legs[x].key = set_security_key ? src_leg.security().security_key() : invalid_security_key; + spread.legs[x].leg_side = src_leg.leg_side(); + spread.legs[x].leg_quantity = src_leg.leg_quantity(); + spread.legs[x].leg_price = src_leg.leg_price(); + spread.legs[x].leg_delta = src_leg.leg_delta(); + } + break; + } + default: + bts_assert(false); + } +} + +}}//namespace + +#endif diff --git a/src/server/sparkd/md/Structure/FixedSecurity.h b/src/server/sparkd/md/Structure/FixedSecurity.h new file mode 100644 index 000000000..87deba316 --- /dev/null +++ b/src/server/sparkd/md/Structure/FixedSecurity.h @@ -0,0 +1,197 @@ +#pragma once + +#include "../common.h" +#include "lib/utils/bts_export.h" +#include "lib/utils/warn_ctrl.h" +#include "lib/utils/bts_array_size.h" +#include "lib/utils/bts_assert.h" +#include "lib/utils/bts_no_copy.h" + +#include +#include + +namespace bts { +namespace md2 { + +GCC_DIAG_OFF("-Weffc++") // No Virtual Destructors by design + +#define STRING_PROPERTY(NAME) const char* NAME() const { return _##NAME; } \ + void NAME(const char* val, size_t size) { strncpy(_##NAME, val, std::min(BTS_ARRAYSIZE(_##NAME)-1, size)); _##NAME[std::min(BTS_ARRAYSIZE(_##NAME)-1, size)] = '\0'; } \ + void NAME(const std::string& val) { NAME(val.c_str(), val.size()); } + +// Caution: FixedSecurity is a variable sized structure +// Use FixedSecurity::size to determine size not sizeof() +class BTS_EXPORT FixedSecurity +{ +public: + FixedSecurity(); + + ~FixedSecurity(); + + STRING_PROPERTY(symbol); + STRING_PROPERTY(alt_symbol); + STRING_PROPERTY(cfi_code); + STRING_PROPERTY(security_desc); + STRING_PROPERTY(platform_id); + + size_t size; + security_key key; + uint64_t numeric_id; + SecurityType::type type; + Exchange::type exchange; + + int8_t display_factor; + int8_t strike_display_factor; + uint32_t contract_size; + + uint32_t cboe_class_key; + + uint32_t cme_vtt; + uint32_t cme_display_decimals; + uint32_t cme_denominator_sub_fraction; + double cme_min_price_increment; + uint32_t cme_market_segment_id; + double cme_unit_of_measure_qty; + + + double cboe_premium_break; + double cboe_minimum_above_premium_fraction; + double cboe_minimum_below_premium_fraction; + +private: + // COPY OK BTS_NO_COPY(FixedSecurity); + char _symbol[16]; // CME defines this as a char[6] but I'm padding + char _cfi_code[8]; // CME defines this as char[6] + char _security_desc[32]; // cme security desc FIX=107 + char _alt_symbol[16]; // "cme symbol" or alternate symbol + char _platform_id[64]; +}; + +class BTS_EXPORT FixedFutureSecurity : public FixedSecurity +{ +public: + FixedFutureSecurity(); + uint32_t expiration_year; + uint32_t expiration_month; + uint32_t expiration_day; +}; + +class BTS_EXPORT FixedOptionSecurity : public FixedSecurity +{ +public: + FixedOptionSecurity(); + uint32_t expiration_year; + uint32_t expiration_month; + uint32_t expiration_day; + double strike; + PutOrCall::type call_put; +}; + +struct BTS_EXPORT FixedSpreadLeg +{ + FixedSpreadLeg(); + security_key key; + Side::type leg_side; + double leg_quantity; + double leg_price; + double leg_delta; +}; + +template +class FixedSpreadSecurity : public FixedSecurity +{ + char _summary[128]; +public: + FixedSpreadSecurity() : _leg_count(0), _leg_max(LEGCOUNT), legs() { + type = SecurityType::SPREAD; + size = sizeof(FixedSpreadSecurity<1>); + memset(&_summary, 0, BTS_ARRAYSIZE(_summary)); + } + void set_leg_count(size_t count) { + bts_assert(count > 0); + bts_assert(count <= LEGCOUNT); + bts_assert(count <= _leg_max); + _leg_count = count; + size = sizeof(FixedSpreadSecurity<1>) + (sizeof(FixedSpreadLeg) * (count - 1)); + } + STRING_PROPERTY(summary); + + static size_t static_count() { return LEGCOUNT; } + uint32_t leg_count() const { return _leg_count; } + +private: + // memcpy only please + uint32_t _leg_count; + uint32_t _leg_max; + BTS_NO_COPY(FixedSpreadSecurity); +public: + // NO members after legs array + FixedSpreadLeg legs[LEGCOUNT]; +}; + +// Stack allocate a FixedSpreadSecurity with a high number of legs then memcpy the right size to shared memory +typedef FixedSpreadSecurity<16> FixedSpreadSecurityS; +typedef FixedSpreadSecurity<128> FixedSpreadSecurityL; +typedef FixedSpreadSecurity<512> FixedSpreadSecurityXL; + +BTS_EXPORT const std::string security_descriptor(const FixedSecurity&); +// Convert a fixedsecurity to a normal Security - caveat: spreads will be normalized +BTS_EXPORT Security convert_security(const FixedSecurity&); +BTS_EXPORT void convert_security(const FixedSecurity&, Security*); +BTS_EXPORT size_t recursive_leg_count(const Security&); +BTS_EXPORT void convert_security(const Security& src, FixedSecurity& dest, bool set_security_key=false); + +// Stack allocates a security of the given type +template +void with_security(SecurityType::type type, size_t leg_count, C cb) { + switch(type) { + case SecurityType::STOCK: { + FixedSecurity security; + security.type = SecurityType::STOCK; + cb(security); + break; + } + case SecurityType::FUTURE: { + FixedFutureSecurity security; + bts_assert(security.type == type); + cb(security); + break; + } + case SecurityType::OPTION: { + FixedOptionSecurity security; + bts_assert(security.type == type); + cb(security); + break; + } + case SecurityType::SPREAD: { + if (leg_count <= 4) { + FixedSpreadSecurity<4> security; + cb(security); + } + else if (leg_count <= FixedSpreadSecurityS::static_count()) { + FixedSpreadSecurityS security; + cb(security); + } + else if (leg_count <= FixedSpreadSecurityL::static_count()) { + FixedSpreadSecurityL security; + cb(security); + } + else if (leg_count <= FixedSpreadSecurityXL::static_count()) { + FixedSpreadSecurityXL security; + cb(security); + } + else { + std::stringstream msg("Too many legs ("); + msg << leg_count; + msg << ")"; + bts_assert_msg(false, msg.str()); + } + break; + } + default: + bts_assert(false); + } +} + +}}//namespace + diff --git a/src/server/sparkd/md/Structure/PriceSize.cpp b/src/server/sparkd/md/Structure/PriceSize.cpp new file mode 100644 index 000000000..38a1910ef --- /dev/null +++ b/src/server/sparkd/md/Structure/PriceSize.cpp @@ -0,0 +1,21 @@ +#include "PriceSize.h" +#include "lib/os/os.h" + +namespace bts { +namespace md2 { + +BTS_EXPORT PriceSize PriceSize::NO_MARKET = PriceSize(0, -128, 0); + +int to_exponent(double value) { + if (value == 0 && os::is_nan(value)) return 0; + char dest[64]; + int len = sprintf(dest, "%.10f", value - long(value)); + --len; // skip trailing null + while (dest[len] == '0') len--; + if (len == 1) { + return ceil(log10(value)); + } + return -(len - 1); +} + +}}//namespace diff --git a/src/server/sparkd/md/Structure/PriceSize.h b/src/server/sparkd/md/Structure/PriceSize.h new file mode 100644 index 000000000..eeef48d87 --- /dev/null +++ b/src/server/sparkd/md/Structure/PriceSize.h @@ -0,0 +1,171 @@ +#pragma once + +#include "../common.h" +#include "lib/os/os.h" +#include "lib/utils/warn_ctrl.h" +#include "lib/utils/bts_array_size.h" +#include "lib/utils/bts_assert.h" +#include "lib/utils/bts_no_copy.h" + +#include +#include +#include + +namespace bts { +namespace md2 { + +typedef double Price; +typedef uint16_t Size; + +#define LOAD(x) x.load(std::memory_order_acquire) +#define STORE(x,value) x.store(value, std::memory_order_release) + +static const double powers_10[] = { + 1E-20, 1E-19, 1E-18, 1E-17, 1E-16, 1E-15, 1E-14, 1E-13, 1E-12, 1E-11, 1E-10, 1E-9, 1E-8, 1E-7, 1E-6, 1E-5, 1E-4, 1E-3, 1E-2, 1E-1, + 1E0, + 1E1, 1E2, 1E3, 1E4, 1E5, 1E6, 1E7, 1E8, 1E9, 1E10, 1E11, 1E12, 1E13, 1E14, 1E15, 1E16, 1E17, 1E18, 1E19, 1E20 +}; +static const int max_abs_power_10 = BTS_ARRAYSIZE(powers_10) / 2; +inline double pow10(int power) { + const int offset = power + max_abs_power_10; + const int powers_10_sz = BTS_ARRAYSIZE(powers_10); + bts_assert_d(offset >= 0 && offset < powers_10_sz); + if (offset >= 0 && offset < powers_10_sz) { + return powers_10[offset]; + } + return std::numeric_limits().quiet_NaN(); +} +int to_exponent(double value); +template +inline bool compact_mantissa(Mantissa& mantissa, Exponent& exponent) { + bool result = false; + // If the mantissa is larger than 39 bits will allow attempt to shrink it + while (mantissa > 0x8000000000 || mantissa < -0x8000000000) { + mantissa /= 10; + exponent += 1; + result = true; + } + return result; +} + +class PriceSize { + union { + uint64_t _val64; + // ISO C++ prohibits anonymous structs + GCC_DIAG_OFF("-Wpedantic") + struct { +#ifdef _WIN32 + // VC++ only packs bit fields when the declared type is the same size + int64_t _mantissa : 40; + int64_t _exponent : 8; + uint64_t _size : 16; +#else + int64_t _mantissa : 40; + int8_t _exponent; + uint16_t _size; +#endif + }; + GCC_DIAG_RESTORE() + }; + friend class SingleWriterPriceSizeStore; + PriceSize(uint64_t val) : _val64(val) { } + +public: + // fast path + PriceSize(int64_t mantissa, int8_t exponent, Size size) : + _mantissa(mantissa), + _exponent(exponent), + _size(size) + { + // Attempt to guard against too large mantissa + if (compact_mantissa(mantissa, exponent)) { + _mantissa = mantissa; + _exponent = exponent; + } + } + PriceSize() : _val64(PriceSize::NO_MARKET._val64) { } + + // quick copy + PriceSize(const PriceSize& other) : _val64(other._val64) { } + PriceSize& operator=(const PriceSize& other) { + _val64 = other._val64; + return *this; + } + + bool operator==(const PriceSize& other) { + return _val64 == other._val64; + } + bool operator!=(const PriceSize& other) { + return _val64 != other._val64; + } + + BTS_EXPORT static PriceSize NO_MARKET; + + template + static PriceSize from_double(double price, Size size) { + if (size == 0 || os::is_nan(price)) return NO_MARKET; + double temp; + if (fabs(modf(price, &temp)) != 0.0) { + const double rounding = 0.5 * pow10(-PRECISION); + price += (price > 0) ? rounding : -rounding; + price *= pow10(PRECISION); + return PriceSize((int64_t)price, -PRECISION, size); + } + return PriceSize((int64_t)price, 0, size); + } + static PriceSize from_double(double price, int precision, Size size) { + if (size == 0 || os::is_nan(price)) return NO_MARKET; + double temp; + if (fabs(modf(price, &temp)) != 0.0) { + const double rounding = 0.5 * pow10(-precision); + price += (price > 0) ? rounding : -rounding; + price *= pow10(precision); + return PriceSize((int64_t)price, -precision, size); + } + return PriceSize((int64_t)price, 0, size); + } + + // + // Accessors + + int64_t mantissa() const { + return _mantissa; + } + int8_t exponent() const { + return _exponent; + } + // This call is relatively slow so don't call it often + double price() const { + if (_exponent == -128) return NO_PRICE; + return mantissa() * pow10(exponent()); + } + Size size() const { + return _size; + } +}; +static_assert(sizeof(PriceSize) == sizeof(uint64_t), "PriceSize must be 64 bits for atomic assignment"); + +class SingleWriterPriceSizeStore { + std::atomic _store; + BTS_NO_COPY(SingleWriterPriceSizeStore); +public: + SingleWriterPriceSizeStore() : _store() { + store(PriceSize::NO_MARKET); + } + bool store(PriceSize ps) { + uint64_t cur = LOAD(_store); + if (cur != ps._val64) { + STORE(_store, ps._val64); + return true; + } + return false; + } + PriceSize load() const { + uint64_t val = LOAD(_store); + return PriceSize(val); + } +}; + + +}}//namespace + diff --git a/src/server/sparkd/md/Structure/readme.txt b/src/server/sparkd/md/Structure/readme.txt new file mode 100644 index 000000000..cac2d00b0 --- /dev/null +++ b/src/server/sparkd/md/Structure/readme.txt @@ -0,0 +1,8 @@ +Classes: + CacheHeader + CacheHot + bbo, status + CacheHotDepth + CacheHot + depth + CacheCold + security, open, close, settle diff --git a/src/server/sparkd/md/Structure/structure.h b/src/server/sparkd/md/Structure/structure.h new file mode 100644 index 000000000..0b4dde140 --- /dev/null +++ b/src/server/sparkd/md/Structure/structure.h @@ -0,0 +1,217 @@ +#pragma once +#include "FixedSecurity.h" +#include "PriceSize.h" +#include "lib/config/config.h" +#include "../common.h" +#include "lib/utils/TimeStamp.h" + +#include "lib/config/protos/MarketData.pb.h" + +#include +#include + +/* + * Coconut 2.0 separates data into high (hot) and low (cold) priority based on how fast the clients need to access the data + * Any data critical for the quoter/eyes goes in the Hot structure and is polled very fast. This data should be kept as small as possible + * and should be cache-line aligned + * Any other data goes in the Cold list and will be accessed more slowly +*/ + +namespace bts { +namespace md2 { + +// +// HOT Data +/////////////////////// + +using utils::TimeStamp; + +struct ALIGNED_(CACHE_LINE) HotHeader { + size_t count; + size_t segment_count; + // Just the counters. Publisher process can compute messages/second by polling these counters + size_t bbo_count; + size_t depth_count; +}; + +// These should not be cache line aligned because they are small enough for multiple to fit on one line easily +struct ALIGNED_(32) HotEntry { + HotEntry() : timestamp(), bid(), ask(), key(invalid_security_key), sequence(0), status(CoconutSecurityStatus::OK) {} + uint32_t size() const { return sizeof(HotEntry); } + TimeStamp timestamp; // 8 + SingleWriterPriceSizeStore bid; // 8 + SingleWriterPriceSizeStore ask; // 8 + security_key key; // 4 + uint16_t sequence; // 2 + uint8_t status; // 1 +}; + +const uint32_t COCONUT_MAX_DEPTH = 10; +struct ALIGNED_(CACHE_LINE) HotDepthEntry { + HotDepthEntry() : + timestamp(), sequence(0), status(CoconutSecurityStatus::OK), book(), key(invalid_security_key) + { } + uint32_t size() const { return sizeof(HotDepthEntry); } + TimeStamp timestamp; // 8 + uint16_t sequence; // 2 + //md::CoconutSecurityStatus::type status; // 4 + uint8_t status; // 1 + // Interleave bid/ask values so the top of book is always on the first line + SingleWriterPriceSizeStore book[COCONUT_MAX_DEPTH*2]; + security_key key; // NOTE: key is at a different offset than it is in HotEntry + + SingleWriterPriceSizeStore& bid(uint32_t depth) { return book[depth*2]; } + const SingleWriterPriceSizeStore& bid(uint32_t depth) const { return book[depth*2]; } + SingleWriterPriceSizeStore& ask(uint32_t depth) { return book[depth*2+1]; } + const SingleWriterPriceSizeStore& ask(uint32_t depth) const { return book[depth*2+1]; } +}; + +// +// COLD Data, brrr +/////////////////////// + +struct ALIGNED_(CACHE_LINE) ColdHeader { + ColdHeader() : + count(0), + segment_count(0), + create_time(TimeStamp::now()), + heartbeat(), + disable_heartbeat (config::get().disable_heartbeat ()), + heartbeat_ms (config::get().heartbeat_ms ()), + heartbeat_boot_seconds (config::get().heartbeat_boot_seconds ()), + security_count(0), + has_depth(false), + platform(Platform::UNSET), + trade_count(0), + coconut_status_count(0), + exchange_status_count(0), + rfq_count(0), + customer_bbo_count(0) + {} + + // uint32_t version; + size_t count; + size_t segment_count; + TimeStamp create_time; + TimeStamp heartbeat; + bool disable_heartbeat; + uint32_t heartbeat_ms; + uint32_t heartbeat_boot_seconds; + + uint32_t security_count; // incremented when a security is made public + bool has_depth; + Platform::type platform; + + std::atomic trade_count; + std::atomic coconut_status_count; + std::atomic exchange_status_count; + std::atomic rfq_count; + std::atomic customer_bbo_count; + + // Ring buffer for LastTrade for all securities in this list + uint8_t trade_buffer[1024*1024]; //1mb + // Ring buffer for RFQ + //static const int rfq_buffer_entry_size = 64; + typedef std::array RFQBuffer; + uint8_t rfq_buffer[1024*256];//256k +}; + +struct MiscData { + MiscData(): + misc_timestamp(), + settlement_price(NO_PRICE), + settlement_date(), + previous_settlement_price(NO_PRICE), + previous_settlement_date(), + open_price(NO_PRICE), + open_date(), + close_price(NO_PRICE), + close_date(), + session_high(NO_PRICE), + session_low(NO_PRICE), + open_interest(NO_PRICE), + session_high_bid(NO_PRICE), + session_low_offer(NO_PRICE), + fixing_price(NO_PRICE) + { } + TimeStamp misc_timestamp; + // Settlement + Price settlement_price; + datetime::Date settlement_date; + Price previous_settlement_price; + datetime::Date previous_settlement_date; + + // Opening/Closing + Price open_price; + datetime::Date open_date; + Price close_price; + datetime::Date close_date; + + // Misc stats for current session + Price session_high; + Price session_low; + Price open_interest; + Price session_high_bid; + Price session_low_offer; + Price fixing_price; + + bool operator==(const MiscData& other) const { + return + settlement_price == other.settlement_price && + settlement_date == other.settlement_date && + previous_settlement_price == other.previous_settlement_price && + previous_settlement_date == other.previous_settlement_date && + open_price == other.open_price && + open_date == other.open_date && + close_price == other.close_price && + close_date == other.close_date && + session_high == other.session_high && + session_low == other.session_low && + open_interest == other.open_interest && + session_high_bid == other.session_high_bid && + session_low_offer == other.session_low_offer && + fixing_price == other.fixing_price; + } + bool operator!=(const MiscData& other) const { return !(*this == other); } +}; + +struct ColdEntry { + ColdEntry(size_t security_size) : + exchange_status(TradingStatus::UNKNOWN), + last_trade_updated(), + last_trade(), + total_volume(0), + last_trade_time(), + customer_bbo_updated(), + customer_bid(), + customer_ask(), + misc(), + deleted(false) + { + this->security.size = security_size; + } + + static size_t fixed_size() { return sizeof(ColdEntry) - sizeof(FixedSecurity); } + size_t size() const { return fixed_size() + this->security.size; } + + TradingStatus::type exchange_status; + + // Last Trade + TimeStamp last_trade_updated; + SingleWriterPriceSizeStore last_trade; + size_t total_volume; + datetime::LLDateTime last_trade_time; + + // Customer BBO + TimeStamp customer_bbo_updated; + SingleWriterPriceSizeStore customer_bid; + SingleWriterPriceSizeStore customer_ask; + + MiscData misc; + + bool deleted; + FixedSecurity security; +}; + + +}} diff --git a/src/server/sparkd/md/UserSymbol.h b/src/server/sparkd/md/UserSymbol.h new file mode 100644 index 000000000..43ef4b80c --- /dev/null +++ b/src/server/sparkd/md/UserSymbol.h @@ -0,0 +1,55 @@ +#pragma once +#include "common.h" + +namespace bts { +namespace md2 { + +struct UserSymbol { + UserSymbol() : _symbol(), _exchange(), _type() {} + UserSymbol(const std::string& s, Exchange::type e, SecurityType::type t) : + _symbol(s), _exchange(e), _type(t) {} + UserSymbol(const UserSymbol& other) : + _symbol(other._symbol), _exchange(other._exchange), _type(other._type) {} + + const std::string& symbol() const { return _symbol; } + Exchange::type exchange() const { return _exchange; } + SecurityType::type type() const { return _type; } + bool operator== (const UserSymbol& other) const { + return _type == other.type() && _exchange == other.exchange() && _symbol == other.symbol(); + } + bool operator< (const UserSymbol& other) const { + if (_type < other.type()) { + return true; + } + else { + if (_type == other.type()) { + if (_exchange < other.exchange()) { + return true; + } + else { + if (_exchange == other.exchange()) { + return _symbol < other.symbol(); + } + } + return false; + } + return false; + } + } + std::string to_string() const { + return symbol() + "-" + SecurityType::type_Name(type()); + } + + // No move constructor - + // because in this case the std::string will be a stock symbol which should be + // small enough to be inlined in both msvsc and gcc/clang + +private: + std::string _symbol; + Exchange::type _exchange; + SecurityType::type _type; +}; + +std::ostream& operator<<(std::ostream& os, const UserSymbol& sym); + +}} diff --git a/src/server/sparkd/md/coconut2.h b/src/server/sparkd/md/coconut2.h new file mode 100644 index 000000000..725422cef --- /dev/null +++ b/src/server/sparkd/md/coconut2.h @@ -0,0 +1,13 @@ + +#include "common.h" +#include "PlatformLoader.h" + +#include "MDProducer.h" +#include "MDConsumer.h" +#include "Structure/FixedSecurity.h" +#include "SecurityIDRecorder.h" +#include "SettlementRecorder.h" +// TODO: separate coconut2.h into consumer and producer sides +#include "SecurityIndex.h" +#include "MDRecoverableConsumer.h" + diff --git a/src/server/sparkd/md/common.cpp b/src/server/sparkd/md/common.cpp new file mode 100644 index 000000000..462dac889 --- /dev/null +++ b/src/server/sparkd/md/common.cpp @@ -0,0 +1,43 @@ +#ifndef NO_CPP0X +#include "common.h" +#include "UserSymbol.h" +#include "lib/config/config.h" + +#include "Structure/PriceSize.h" +#include "lib/config/protos/MarketData.pb.h" + +#include +#include +#include + +namespace bts { + namespace md2 { + + const double NO_PRICE = std::numeric_limits().quiet_NaN(); + + std::ostream& operator<<(std::ostream& os, const UserSymbol& sym) { + os << sym.symbol() << "-" << SecurityType::type_Name(sym.type()); + return os; + } + } + + namespace co2 { + using namespace std; + using namespace md; + string topic(const char* data_type, const string& symbol, SecurityType::type type, Exchange::type exchange, Platform::type platform) { + std::ostringstream topic; + topic << "/md/" << config::get().channel_prefix() + << Platform::type_Name(platform) << "/" + << Exchange::type_Name(exchange) << "/" + << symbol << "/" + << SecurityType::type_Name(type) << "/" + << data_type; + return topic.str(); + } + double price(const md::BookMessage& msg) { + return msg.price(); + } + } + +}//namespace +#endif diff --git a/src/server/sparkd/md/common.h b/src/server/sparkd/md/common.h new file mode 100644 index 000000000..b656a1da8 --- /dev/null +++ b/src/server/sparkd/md/common.h @@ -0,0 +1,56 @@ +#pragma once +#include + +#include "lib/utils/bts_export.h" +#include "lib/utils/datetime.h" + +#include "server/sparkd/md/protos/coconut.pb.h" +#include "server/sparkd/md/protos/co2.pb.h" +#include "lib/security_model/security.pb.h" + +#include + +namespace bts { +namespace md2 { + BTS_EXPORT const extern double NO_PRICE; + + typedef uint32_t security_key; + const security_key invalid_security_key = 0; + + // Bring in o2 types + using security_model::Security; + using security_model::SecurityType; + using security_model::SecurityList; + using security_model::Exchange; + using security_model::PutOrCall; + using security_model::Side; + using bts::md::Platform; + using bts::md::TradingStatus; + using bts::md::CoconutSecurityStatus; + +} + +namespace md { + using md2::security_key; + // using md2::invalid_security_key; + // using md2::NO_PRICE; + + using security_model::Security; + using security_model::SecurityType; + using security_model::Security; + using security_model::Exchange; + using security_model::PutOrCall; + using security_model::Side; + using bts::md::Platform; + using bts::md::TradingStatus; +} +namespace co2 { + std::string topic(const char* data_type, const std::string& symbol, md::SecurityType::type type, md::Exchange::type exchange, md::Platform::type platform); + double price(const md::BookMessage& msg); +} + + +}//namespace + + + diff --git a/src/server/sparkd/md/get_symbols.h b/src/server/sparkd/md/get_symbols.h new file mode 100644 index 000000000..bcd1a7212 --- /dev/null +++ b/src/server/sparkd/md/get_symbols.h @@ -0,0 +1,15 @@ +#pragma once +#include "UserSymbol.h" +#include "lib/utils/bts_export.h" + +#include +#include +namespace bts { +namespace md2 { + +BTS_EXPORT size_t get_system_md_symbols(Exchange::type, SecurityType::type, std::vector* result); +BTS_EXPORT size_t get_system_md_symbols(Exchange::type, SecurityType::type, std::set& out_result); +BTS_EXPORT size_t get_system_md_symbols(std::set& result); +extern "C" BTS_EXPORT Platform::type platform_for_exchange(Exchange::type exchange); + +}} diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/CMESecurityIndex.h b/src/server/sparkd/md/platforms/mdp3_onixs/CMESecurityIndex.h new file mode 100644 index 000000000..1874f65d8 --- /dev/null +++ b/src/server/sparkd/md/platforms/mdp3_onixs/CMESecurityIndex.h @@ -0,0 +1,19 @@ +#pragma once + +#include "server/sparkd/md/SecurityIndex.h" +#include + +namespace bts { +namespace md { + +struct SecurityData { + SecurityData(uint32_t id, md2::MDProducerHandle& h) : id(id), handle(h), electronic_volume(0) {} + uint32_t id; // exchange id + md2::MDProducerHandle handle; + uint32_t electronic_volume; +}; + +typedef md2::ConcurrentSecurityIndex CMESecurityIndex; + +}}//namespace + diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp new file mode 100644 index 000000000..33f905787 --- /dev/null +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp @@ -0,0 +1,596 @@ +#ifndef NO_CPP0X + +#include "MDP3Listener.h" +#include "lib/utils/defer.h" +#include "lib/utils/CallTimer.h" +#include "lib/utils/helper_macros.h" +#include + +using namespace OnixS::CME::MDP3; +using namespace std; +using namespace bts::md2; + +namespace bts { +namespace md { + +MDP3Listener::MDP3Listener(CMESecurityIndex& index, md2::MDProducer& producer, md2::SecurityIDRecorder& recorder, co2::SettlementRecorder& settles, std::vector channels, utils::Dispatcher& disp) : + _index(index), + _producer(producer), + _packet_arrived(), + _latest_sequence(0), + _security_recorder(recorder), + _settlement_recorder(settles), + _symbol_filter_storage(), + _symbol_filter(), + _channels(channels), + _background(disp), + _security_recorder_input_timer(disp, 900, [&]() { dequeue_securities(); }), + _security_recorder_input(), + _concurrency_monitor(0), + _book_updates() +{ + _book_updates.reserve(32); + _security_recorder_input_timer.start(); +} + +MDP3Listener::~MDP3Listener() { + _security_recorder_input_timer.stop(); +} + +void MDP3Listener::add_filter_symbol(const std::string& symbol) { + _symbol_filter_storage.push_back(symbol); + _symbol_filter.clear(); + for (const auto& str : _symbol_filter_storage) { + _symbol_filter.insert(onixs::StringRef(str)); + } +} + +bool MDP3Listener::filter_security(onixs::StringRef val) { + return _symbol_filter.count(val) == 0; +} + +// +// MessageProcessingListener +void MDP3Listener::onProcessingBegin(const Message& msg, const ChannelId&) { + + // Mark the arrival of the packet + _packet_arrived = TimeStamp::now();// - TimeStamp::Interval::from_microseconds((onixs::Timestamp::utcNow() - msg.receiveTime()).microseconds()); + + // If we haven't flushed we must still be working on the same packet (and therefore the same sequence number) + if (!_book_updates.empty()) { + if (_latest_sequence != msg.sequenceNumber()) { + LOG(MarketData,Error, "expected sequence: {} got {} book updates:{}", + _latest_sequence,msg.sequenceNumber(), _book_updates.size()); + bts_assert(_latest_sequence == msg.sequenceNumber()); + } + } + _latest_sequence = msg.sequenceNumber(); +} + +void MDP3Listener::onProcessingEnd(const Message& msg, const ChannelId&) { + char msg_type; + // Market data snapshot messages do not include the 'lastInPacket' flag so we always flush when we finish processing a snapshot (35=W) message + if (msg.lastInPacket() || (msg.type().toChar(msg_type) && msg_type == 'W')) { + flush_books(); + } +} + +void MDP3Listener::onProcessingSuspended(const ChannelId&) { + // TODO: Mark this channel as out of date +} + +SecurityType::type type_from_cfi(const StringRef& cfi_code) { + + // + // Got an MRXXX which fell into spread but is actually invalid + + bts_assert(!cfi_code.empty() && cfi_code.size() > 2); + if (cfi_code[1] == 'M') return SecurityType::SPREAD; + switch (cfi_code[0]) { + case 'O': + return SecurityType::OPTION; + case 'F': + return SecurityType::FUTURE; + case 'M': + return SecurityType::SPREAD; // covered + default: + // string message = "Invalid or confusing CFI code: " + (string)cfi_code; + // bts_assert_msg(false, message.c_str()); + return SecurityType::UNSET; + } +} + +PutOrCall::type call_put(const StringRef& cfi_code) { + bts_assert_msg(cfi_code[0] == 'O', cfi_code); + return cfi_code[1] == 'C' ? PutOrCall::CALL : PutOrCall::PUT; +} + +int floorlog10(onixs::DecimalMantissa m) { + bts_assert(m >= 0); + if (m < 10) return 0; + if (m < 100) return 1; + if (m < 1000) return 2; + if (m < 10000) return 3; + if (m < 100000) return 4; + if (m < 1000000) return 5; + if (m < 10000000) return 6; + if (m < 100000000) return 7; + if (m < 1000000000) return 8; + if (m < 10000000000) return 9; + if (m < 100000000000) return 10; + if (m < 1000000000000) return 11; + if (m < 10000000000000) return 12; + if (m < 100000000000000) return 13; + if (m < 1000000000000000) return 14; + if (m < 10000000000000000) return 15; + if (m < 100000000000000000) return 16; + if (m < 1000000000000000000) return 17; + return 19; +} + +template +void set_security_common(const onixs::SecurityDefinition& def, TSecurityType& security, uint32_t exchange_id) { + security.numeric_id = exchange_id; + security.symbol(def.asset().data(), def.asset().size()); // 1151-SecurityGroup changed to 6937-Asset in MDP3 + security.cfi_code(def.CFICode().data(), def.CFICode().size()); + security.alt_symbol(def.securityGroup().data(), def.securityGroup().size()); + security.security_desc(def.symbol().data(), def.symbol().size()); + security.exchange = Exchange::GLBX; + def.tickRule(security.cme_vtt); + + onixs::FloatQuantity unit_qty; + if (def.unitOfMeasureQty(unit_qty)) { + security.cme_unit_of_measure_qty = unit_qty; + } + + def.subFraction(security.cme_denominator_sub_fraction); + def.mainFraction(security.cme_display_decimals); + def.marketSegmentId(security.cme_market_segment_id); + + onixs::PriceFactor display; + if (def.displayFactor(display)) { + auto exp = display.exponent(); + auto mant = floorlog10(display.mantissa()); + security.display_factor = exp + mant; + security.strike_display_factor = security.display_factor; + } + // mapping::display_factor((string)security.symbol(), &security.display_factor, security.type); + // bool should_multiply_strike = false; + // int8_t strike_factor = security.strike_display_factor; + // if (security.type == SecurityType::OPTION && + // mapping::strike_display_factor(security.symbol(), nullptr, &strike_factor, security.type, nullptr, nullptr, nullptr, &should_multiply_strike) && + // should_multiply_strike) { + // security.strike_display_factor = strike_factor; + // } + + onixs::Price min_price_increment; + def.minPriceIncrement(min_price_increment); + security.cme_min_price_increment = min_price_increment; +} + +datetime::Date get_expiration(const SecurityDefinition& def) { + datetime::Date result; + size_t size = def.events().size(); + for (size_t index = 0; index < size; ++index) { + if (def.events()[index].type() == EventTypes::LastEligibleTradeDate) { + auto time = def.events()[index].time(); + result.year = time.year(); + result.month = time.month(); + result.day = time.day(); + break; + } + } + return result; +} + + +// +// SecurityDefinitionListener +void MDP3Listener::onSecurityDefinition(const SecurityDefinition& def, const ChannelId&) { + LOG_IF_SLOW + // + auto exchange_id = def.securityId(); + + // Ignore duplicate security defs + if (_index.has_security(exchange_id)) return; + if (filter_security(def.asset())) return; + + // Also any securities which are not elligible for trading + if (def.tradingStatus() == SecurityTradingStatus::NotAvailableForTrading) return; + + StringRef cfi_code = def.CFICode(); + auto type = SecurityType::UNSET; + if (def.legs().size() > 0) { + type = SecurityType::SPREAD; + } + else { + type = type_from_cfi(cfi_code); + } + + switch (type) { + // + // Option + case SecurityType::OPTION: { + FixedOptionSecurity security; + set_security_common(def, security, exchange_id); + onixs::Price price; + bts_assert(def.strikePrice(price)); + security.strike = (double)price * pow10(security.strike_display_factor); + security.call_put = call_put(cfi_code); + auto date = get_expiration(def); + security.expiration_year = date.year; + security.expiration_month = date.month; + security.expiration_day = date.day; + add_security(security); + } break; + // + // Future + case SecurityType::FUTURE: { + FixedFutureSecurity security; + set_security_common(def, security, exchange_id); + auto date = get_expiration(def); + security.expiration_year = date.year; + security.expiration_month = date.month; + security.expiration_day = date.day; + add_security(security); + } break; + // + // Spread + case SecurityType::SPREAD: { + // Sometimes our CFI parsing isn't correct. There's a MRRXXX on the 6E channel for example + if (def.legs().size() == 0) return; + + // Filter out any already expired spreads + auto date = get_expiration(def); + if (date < datetime::today()) return; + + // Start simple + FixedSpreadSecurityL security; + vector leg_exchange_ids; + leg_exchange_ids.reserve(def.legs().size()); // TODOPERF: heap + set_security_common(def, security, exchange_id); + + security.set_leg_count(def.legs().size()); + for (size_t x = 0; x < security.leg_count(); ++x) { + const auto& leg = def.legs()[x]; + FloatQuantity f; + if (leg.optionDelta(f)) { + security.legs[x].leg_delta = f; + } + onixs::Price temp_price; + leg.price(temp_price); + security.legs[x].leg_price = (double)temp_price * pow10(security.display_factor); + security.legs[x].leg_quantity = leg.ratioQty(); + security.legs[x].leg_side = leg.side() == LegSides::Buy ? Side::BUY : Side::SELL; + if (leg.side() == LegSides::Undefined) return; // Do not add invalid spreads + leg_exchange_ids.push_back(leg.getUInt32(onixs::Tags::LegSecurityID)); + //leg_exchange_ids.push_back(leg.securityId()); + } + add_security(security); + _index.add_spread_legs(security.numeric_id, leg_exchange_ids); + } break; + default: + return; + } +} + +void MDP3Listener::onSecurityDefinitionDeleted(SecurityId id, const ChannelId&) { + auto* data = _index.get_security(id); + if (data != nullptr) { + data->handle.mark_deleted(); + } +} + +void MDP3Listener::onSecurityDefinitionsRecoveryStarted(const ChannelId& ch) { + LOG(MarketData,Info,"Security definition recovery started on channel {}",ch); +} +void MDP3Listener::onSecurityDefinitionsRecoveryFinished(const ChannelId& chan) { + LOG(MarketData,Info, "Security definition recovery finished on channel {}", chan); +} + +// +// SecurityStatusListener +void MDP3Listener::onSecurityStatus(const SecurityStatus&, const ChannelId&) { +} + +inline PriceSize convert(const onixs::Price price, int8_t display_factor, Size size) { + PriceSize result = PriceSize(price.mantissa(), price.exponent() + display_factor, size); + return result; +} +inline double convert(const onixs::Price price, int8_t display_factor) { + return (double)price * pow10(display_factor); +} + +inline datetime::LLDateTime convert_time(const onixs::Timestamp& timestamp) { + return datetime::ll_from_parts(timestamp.year(), timestamp.month(), timestamp.day(), timestamp.hour(), timestamp.minute(), timestamp.second(), timestamp.millisecond()); +} + +datetime::Date convert(const onixs::Timestamp& transactTime) { + datetime::Date date; + date.year = transactTime.year(); + date.month = transactTime.month(); + date.day = transactTime.day(); + return date; +} + +// +// ConsolidatedBookUpdateListener +void MDP3Listener::onConsolidatedBookUpdated(const ConsolidatedBook& book, const ChannelId&) { + LOG_IF_SLOW + + // Validate our assumptions that only one packet is processed at a time inside this class + _concurrency_monitor++; + DEFER(_concurrency_monitor--); + if (_concurrency_monitor != 1) { + LOG(MarketData,Error, + "ERROR: Concurrent calls to onConsolidatedBookUpdated may result in corrupted book data." + "Please let Eric know about this."); + } + + auto data = _index.get_security(book.securityId()); + if (data == nullptr) return; + bts_assert((uint32_t)book.securityId() == data->id); + + auto display_factor = data->handle.security().display_factor; + // Push the data into the cache - flush at the the end of the packet + // Linear search through the other updates in this packet + auto find_iter = std::find_if(_book_updates.begin(), _book_updates.end(), [&data](const BookData& entry) { + return data->handle.key() == entry.handle.key(); + }); + bool found = find_iter != _book_updates.end(); + if (!found) { + _book_updates.emplace_back(data->handle); + } + BookData& cache = found ? *find_iter : _book_updates.back(); + + // Overwrite all entries every time + size_t bids_size = data->handle.has_depth() ? book.bids().size() : min((size_t)book.bids().size(), (size_t)1); + size_t asks_size = data->handle.has_depth() ? book.asks().size() : min((size_t)book.asks().size(), (size_t)1); + for (size_t x = 0; x < bids_size; ++x) { + cache.bids[x] = convert(book.bids()[x].price(), display_factor, book.bids()[x].quantity()); + } + for (size_t x = 0; x < asks_size; ++x) { + cache.asks[x] = convert(book.asks()[x].price(), display_factor, book.asks()[x].quantity()); + } + // NaN out the remaining depth + for (size_t x = bids_size; x < COCONUT_MAX_DEPTH; ++x) { + cache.bids[x] = PriceSize::NO_MARKET; + } + for (size_t x = asks_size; x < COCONUT_MAX_DEPTH; ++x) { + cache.asks[x] = PriceSize::NO_MARKET; + } +} + +// +// HandlerStateChangeListener +void MDP3Listener::onHandlerStateChange(const HandlerStateChange& change, const ChannelId& ch) { + LOG(MarketData,Info,"Channel {} changed state from {} to {}", + ch, onixs::HandlerStates::toString(change.oldState()), + onixs::HandlerStates::toString(change.newState())); + if (change.newState() == HandlerState::BookRecoveryFinished) { + _index.walk_security_list([](SecurityData& data) { + data.handle.set_status(CoconutSecurityStatus::OK); + }); + } +} + + +// +// TradeListener +void MDP3Listener::onTrade(const Trade& trade, const ChannelId&) { + LOG_IF_SLOW + utils::TimeStamp ts = utils::TimeStamp::now(); + auto data = _index.get_security(trade.securityId()); + if (data == nullptr) return; + + onixs::SignedQuantity size = trade.quantity(); + + auto time = convert_time(trade.transactTime()); + data->handle.add_trade(ts, convert(trade.price(), data->handle.security().display_factor, size), time, data->electronic_volume + size); +} + +void MDP3Listener::onElectronicVolume(const ElectronicVolume& volume, const ChannelId&) { + auto data = _index.get_security(volume.securityId()); + if (data == nullptr) return; + data->electronic_volume = volume.quantity(); +} + +// +// RequestForQuoteListener +void MDP3Listener::onRequestForQuote(const RequestForQuote& quote, const ChannelId&) { + auto ts = utils::TimeStamp::now(); + // Note: Here we're throwing out the RFQ if we don't have a security defined + // It might make more sense to send it along and let the receiver try to match up the security key + auto data = _index.get_security(quote.securityId()); + if (!data) return; + + auto time = convert_time(quote.transactTime()); + onixs::QuoteReqId id; + if (!quote.quoteReqId(id)) return; + + co2::QuoteRequestMessage message; + message.set_time(time); + message.set_key(data->handle.key()); + id.toString(*message.mutable_request_id()); + + switch (quote.side()) { + case RequestedSides::Undefined: message.set_side(co2::QuoteRequestMessage::UNDEFINED); break; + case RequestedSides::Buy: message.set_side(co2::QuoteRequestMessage::BUY); break; + case RequestedSides::Sell: message.set_side(co2::QuoteRequestMessage::SELL); break; + case RequestedSides::Cross: message.set_side(co2::QuoteRequestMessage::CROSS); break; + } + message.set_tradable(quote.quoteType() == RequestedQuoteType::Tradable ? 1 : 0); + SignedQuantity q; + if (quote.orderQty(q)) { + message.set_size(q); + } + + data->handle.add_rfq(ts, message); +} + + +// +// StatisticsListener +void MDP3Listener::onStatisticsReset(SecurityId securityId,const ChannelId&) { + UNUSED(securityId); +} + +void MDP3Listener::onStatisticsReset(const StringRef&,const ChannelId&) { +} + +void MDP3Listener::onStatistics(const OpeningPrice& openingPrice,const ChannelId&) { + auto data = _index.get_security(openingPrice.securityId()); + if (data == nullptr) return; + data->handle.update_opening(utils::TimeStamp::now(), convert(openingPrice.value(), data->handle.security().display_factor), convert(openingPrice.transactTime())); +} + +void MDP3Listener::onStatistics(const SettlementPrice& settlementPrice,const ChannelId&) { + auto data = _index.get_security(settlementPrice.securityId()); + if (data == nullptr) return; + auto price = convert(settlementPrice.value(), data->handle.security().display_factor); + //data->handle.update_settlement(utils::TimeStamp::now(), price, convert(settlementPrice.transactTime())); + _settlement_recorder.add_settlement(data->handle, price, convert(settlementPrice.transactTime())); +} + +void MDP3Listener::onStatistics(const TradingSessionHighPrice& highPrice,const ChannelId&) { + auto data = _index.get_security(highPrice.securityId()); + if (data == nullptr) return; + data->handle.update_session_high(utils::TimeStamp::now(), convert(highPrice.value(), data->handle.security().display_factor)); +} + +void MDP3Listener::onStatistics(const TradingSessionLowPrice& lowPrice,const ChannelId&) { + auto data = _index.get_security(lowPrice.securityId()); + if (data == nullptr) return; + data->handle.update_session_low(utils::TimeStamp::now(), convert(lowPrice.value(), data->handle.security().display_factor)); +} + +void MDP3Listener::onStatistics(const SessionHighBid& highBid,const ChannelId&) { + auto data = _index.get_security(highBid.securityId()); + if (data == nullptr) return; + data->handle.update_session_high_bid(utils::TimeStamp::now(), convert(highBid.value(), data->handle.security().display_factor)); +} + +void MDP3Listener::onStatistics(const SessionLowOffer& lowOffer,const ChannelId&) { + auto data = _index.get_security(lowOffer.securityId()); + if (data == nullptr) return; + data->handle.update_session_low_offer(utils::TimeStamp::now(), convert(lowOffer.value(), data->handle.security().display_factor)); +} + +void MDP3Listener::onStatistics(const TradeVolume&,const ChannelId&) { + // This is the total volume from yesterday +} + +void MDP3Listener::onStatistics(const OpenInterest& interest,const ChannelId&) { + auto data = _index.get_security(interest.securityId()); + if (data == nullptr) return; + data->handle.update_open_interest(utils::TimeStamp::now(), interest.value()); +} + +void MDP3Listener::onStatistics(const FixingPrice& fixingPrice,const ChannelId&) { + auto data = _index.get_security(fixingPrice.securityId()); + if (data == nullptr) return; + data->handle.update_fixing_price(utils::TimeStamp::now(), convert(fixingPrice.value(), data->handle.security().display_factor)); +} + +void MDP3Listener::onStatistics(const ThresholdLimits& ,const ChannelId&) { + // noop +} + + + +// +// WarningListener +void MDP3Listener::onWarning(const Warning& warning,const ChannelId& ch) { + LOG_IF_SLOW + if (warning.code() == KnownWarnings::PacketGap) { + // Mark all securities on this channel as bad + // This is probably not fast but should be faster than creating a security I think + _index.walk_security_list([](SecurityData& data) { + data.handle.set_status(CoconutSecurityStatus::PACKET_LOSS); + }); + } + // Logging to the console can be very expensive + LOG(MarketData,Warn, "{} (Chanel {}) {}", + warning.description(),ch ,warning.source()); +} + +// +// ErrorListener +void MDP3Listener::onError(const onixs::Error& error, const ChannelId& ch) { + LOG(MarketData,Warn, "{} (Channel {}) {}", + error.description(), ch ,error.source()); +} + + +// +// Private Functions +// + +void MDP3Listener::heartbeat() { + for (auto& c : _channels) { c.heartbeat(); } +} + + +void MDP3Listener::add_security(const FixedSecurity& security) { + // Set up the shared memory for this security + auto handle = _producer.add_security(security); + bts_assert(security.numeric_id != 0); + // Add it to the index so it can receive messages + SecurityData* data = _index.add_security(security.numeric_id, handle); + + // Computing the descriptor may be expensive so defer that processing to the background worker thread (see dequeue_securities) + _security_recorder_input.push(data); +} + +void MDP3Listener::dequeue_securities() { + // already running on '_background' thread + vector defer; + + SecurityData* data = nullptr; + while (_security_recorder_input.try_pop(data)) { + bts_assert(data->handle.security().key == invalid_security_key); + + string descriptor = security_descriptor(data->handle.security()); + if (descriptor.empty()) { + defer.push_back(data); + } + else { + _security_recorder.get_key(descriptor, data->handle.security().type, [](void* pv_handle, security_key key) { + SecurityData* pdata = (SecurityData*)pv_handle; + pdata->handle.set_key(key); + }, data); + // Restore saved settlements + _settlement_recorder.restore_settlement(data->handle); + } + } + + // try again later (when spread legs have resolved) + for (auto* retry : defer) { + _security_recorder_input.push(retry); + } + +} + +void MDP3Listener::flush_books() { + // This is called at the end of the packet + // Write all book updates to shared memory and reset for the next packet + for (auto& entry : _book_updates) { + // bbo first + uint16_t sequence = (_latest_sequence & 0xFFFF); + entry.handle.update_bbo(_packet_arrived, entry.bids[0], entry.asks[0], sequence); + + // Perf Note: the remainder of the book update could be done (carefully) with a memcpy + for (size_t x = 1; x < COCONUT_MAX_DEPTH; ++x) { + entry.handle.update_bid(_packet_arrived, entry.bids[x], x); + entry.handle.update_ask(_packet_arrived, entry.asks[x], x); + } + } + // Reset pending updates + _book_updates.clear(); +} + +} +md::MDP3Listener::BookData::BookData(md2::MDProducerHandle & handle) : handle(handle), bids(), asks() +{ } +}//namespace +#endif diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h new file mode 100644 index 000000000..0aa1f209b --- /dev/null +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h @@ -0,0 +1,131 @@ +#pragma once + +#include "lib/utils/bts_no_copy.h" +#include "server/sparkd/md/coconut2.h" +#include "lib/utils/Dispatcher.h" +#include "onixs.h" +#include "CMESecurityIndex.h" + +#include +#include +#include + +namespace bts { +namespace md { + +class MDP3Listener : + public onixs::MessageProcessingListener, + public onixs::SecurityDefinitionListener, + public onixs::SecurityStatusListener, + public onixs::ConsolidatedBookUpdateListener, + public onixs::HandlerStateChangeListener, + public onixs::TradeListener, + public onixs::ElectronicVolumeListener, + public onixs::RequestForQuoteListener, + public onixs::StatisticsListener, + public onixs::WarningListener, + public onixs::ErrorListener +{ +public: + MDP3Listener(CMESecurityIndex& index, md2::MDProducer&, md2::SecurityIDRecorder&, co2::SettlementRecorder&, std::vector, utils::Dispatcher&); + ~MDP3Listener(); + + void add_filter_symbol(const std::string& symbol); + + // + // MessageProcessingListener + /* Look for heartbeat and top of book update */ + void onProcessingBegin(const onixs::Message& message, const onixs::ChannelId&); + void onProcessingEnd(const onixs::Message& Message, const onixs::ChannelId&); + void onProcessingSuspended(const onixs::ChannelId&); + + // + // SecurityDefinitionListener + void onSecurityDefinition(const onixs::SecurityDefinition& definition, const onixs::ChannelId&); + void onSecurityDefinitionDeleted(onixs::SecurityId id, const onixs::ChannelId&); + void onSecurityDefinitionsRecoveryStarted(const onixs::ChannelId&); + void onSecurityDefinitionsRecoveryFinished(const onixs::ChannelId&); + + void onSecurityStatus(const onixs::SecurityStatus& status, const onixs::ChannelId&); + + // + // ConsolidatedBookUpdateListener + void onConsolidatedBookUpdated(const onixs::ConsolidatedBook& book, const onixs::ChannelId&); + + // + // HandlerStateChangeListener + void onHandlerStateChange(const onixs::HandlerStateChange& change, const onixs::ChannelId&); + + // + // TradeListener + void onTrade(const onixs::Trade& trade, const onixs::ChannelId&); + + // + // ElectronicVolumeListener + void onElectronicVolume(const onixs::ElectronicVolume&, const onixs::ChannelId&); + + // + // RequestForQuoteListener + void onRequestForQuote(const onixs::RequestForQuote& quote, const onixs::ChannelId&); + + // + // StatisticsListener + void onStatisticsReset(onixs::SecurityId securityId,const onixs::ChannelId&); + void onStatisticsReset(const onixs::StringRef& securityGroup,const onixs::ChannelId&); + void onStatistics(const onixs::OpeningPrice& openingPrice,const onixs::ChannelId&); + void onStatistics(const onixs::SettlementPrice& settlementPrice,const onixs::ChannelId&); + void onStatistics(const onixs::TradingSessionHighPrice& highPrice,const onixs::ChannelId&); + void onStatistics(const onixs::TradingSessionLowPrice& lowPrice,const onixs::ChannelId&); + void onStatistics(const onixs::SessionHighBid& highBid,const onixs::ChannelId&); + void onStatistics(const onixs::SessionLowOffer& lowOffer,const onixs::ChannelId&); + void onStatistics(const onixs::TradeVolume& volume,const onixs::ChannelId&); + void onStatistics(const onixs::OpenInterest& interest,const onixs::ChannelId&); + void onStatistics(const onixs::FixingPrice& fixingPrice,const onixs::ChannelId&); + void onStatistics(const onixs::ThresholdLimits& limits,const onixs::ChannelId&); + + // + // WarningListener + void onWarning(const onixs::Warning& warning,const onixs::ChannelId&); + + // + // ErrorListener + void onError(const onixs::Error& error, const onixs::ChannelId&); + + void heartbeat(); + +private: + BTS_NO_COPY(MDP3Listener); + + void add_security(const md2::FixedSecurity& security); + void dequeue_securities(); + bool filter_security(onixs::StringRef); + void flush_books(); + + + CMESecurityIndex& _index; + md2::MDProducer& _producer; + utils::TimeStamp _packet_arrived; + uint64_t _latest_sequence; + md2::SecurityIDRecorder& _security_recorder; + co2::SettlementRecorder& _settlement_recorder; + std::vector _symbol_filter_storage; + std::set _symbol_filter; + std::vector _channels; + utils::Dispatcher& _background; + utils::DispatcherTimer _security_recorder_input_timer; + tbb::concurrent_queue _security_recorder_input; + std::atomic _concurrency_monitor; + + struct BookData { + BookData(md2::MDProducerHandle& handle); + md2::MDProducerHandle handle; + md2::PriceSize bids[10]; + md2::PriceSize asks[10]; + }; + std::vector _book_updates; + +}; + + +}}//namespace + diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp new file mode 100644 index 000000000..4dad247bd --- /dev/null +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp @@ -0,0 +1,318 @@ +#ifndef NO_CPP0X +#include "MDP3Main.h" + +#include "MDP3Utils.h" +#include "lib/config/config.h" +#include "server/sparkd/md/coconut2.h" +#include "lib/os/os.h" +#include "lib/utils/Dispatcher.h" +#include "lib/utils/proto_helper.h" + +#include "lib/config/protos/MarketData.pb.h" +#include "lib/config/protos/system.pb.h" + +#include +#include + +using namespace std; +using namespace bts::md2; +// using namespace OnixS::CME::MDP3; + +namespace bts { +namespace md { + +void run_onixs_mdp3_server(UserSymbol* psyms, size_t symcount) { + MDP3Main main; + vector symbols; + for (size_t x = 0; x < symcount; ++x) { + symbols.push_back(psyms[x]); + } + main.run(symbols); +} + +MDP3Main::MDP3Main() : + _producer(Platform::MDP3ONIXS), + _security_recorder(), + _settlement_recorder(Platform::MDP3ONIXS), + _engines(), + _handlers(), + _listeners() +{ +} + +MDP3Main::~MDP3Main() {} + +const CMEOptions::CMEChannel channel_settings_for(const string& id) { + const auto& config = config::get(); + for (const auto& ch : config.channel()) { + bool found = std::find_if(ch.id().begin(), ch.id().end(), [&](const string& chan_id) { return chan_id == id || chan_id == "*"; }) != ch.id().end(); + if (found) { + return ch; + } + } + return CMEOptions::CMEChannel(); +} +bool feed_engine_exists(const string& name) { + const auto& config = config::get(); + for (const auto& feed : config.feed_engine()) { + if (feed.name() == name) return true; + } + return false; +} + +const onixs::FeedEngineSettings feed_engine_settings_from_config(const CMEOptions::FeedEngineSettings& settings, int feed_count) { + onixs::FeedEngineSettings result; + if (settings.has_data_wait()) result.dataWaitTime(settings.data_wait()); + if (settings.has_spin_time()) result.redundancySpinTime(settings.spin_time()); + for (auto& core : settings.affinity()) { + result.threadsAffinity().insert(core); + } + + // num_threads() overrides anything else + if (settings.has_num_threads()) { + result.threadsCount(settings.num_threads()); + } + else { + // If num_threads is absent determine thread count based on #channels + result.threadsCount((feed_count / settings.channels_per_thread()) + 1); + } + return result; +} + +size_t count_engine_uses(const vector& channels, const string& feed_name) { + return count_if(channels.begin(), channels.end(), [&](const CMEChannel& ch) { + const auto& settings = channel_settings_for(ch.id); + return settings.has_feed_engine() && settings.feed_engine() == feed_name; + }); +} + +map create_feed_engine_settings(const std::vector& channels) { + map settings; + const auto& config = config::get(); + + // For each feed engine in feed_engine + for (const auto& feed : config.feed_engine()) { + // - see if it is in use + size_t use_count = count_engine_uses(channels, feed.name()); + // - Only create if this feed engine is in use + if (use_count > 0) { + settings[feed.name()] = feed_engine_settings_from_config(feed, use_count); + } + } + + // If a channel asks for defaults ensure it exists. + if (settings.count("default") == 0) { + size_t channels_using_default = count_engine_uses(channels, "default"); + if (channels_using_default > 0) { + settings["default"] = feed_engine_settings_from_config(CMEOptions::FeedEngineSettings(), channels_using_default); + } + } + + return settings; +} + +map> create_feed_engines(const vector& channels) { + map> result; + auto settings = create_feed_engine_settings(channels); + // Loop through settings and instantiate feed engines + for (const auto& s : settings) { + unique_ptr temp(new onixs::FeedEngine(s.second)); + result.insert(make_pair(s.first, move(temp))); + } + + return result; +} + +void MDP3Main::run(std::vector& symbols) { + await_cme_config(); + initialize(); + const auto& channels = parse_channels(symbols); + _engines = create_feed_engines(channels); + for (const auto& channel : channels) { + _handlers.push_back(prepare_handler(channel.id)); + _listeners.push_back(prepare_listener(*_handlers.back(), channel)); + std::stringstream symstr; + for (const auto& sym : channel.symbols) { symstr << sym.symbol() << "-" << SecurityType::type_Name(sym.type()) << " "; } + LOG(MarketData,Info, "CME: Initialized channel {} ({})",channel.id,symstr.str()); + } + for (auto& p_handler : _handlers) { + p_handler->start(); + } + + // + // Work timer + auto& disp = utils::Dispatcher::current(); + utils::DispatcherTimer security_timer(disp, 500, [&]() { + _security_recorder.do_work(); + }); + security_timer.start(); + + utils::DispatcherTimer spread_resolve_timer(disp, 1000, [&]() { + _index.resolve_spreads(); + }); + spread_resolve_timer.start(); + + utils::DispatcherTimer settlement_timer(disp, 100, [&]() { + _settlement_recorder.do_work(); + }); + settlement_timer.start(); + + utils::DispatcherTimer heartbeat_timer(disp, 250, [&]() { + for (const auto& listener: _listeners) { + listener->heartbeat(); + } + }); + heartbeat_timer.start(); + + LOG(MarketData,Info, "** CME MDP3 Publisher started **"); + + disp.run(); +} + + +void MDP3Main::initialize() { + const auto& options = config::get(); + + // + // License path + onixs::InitializationSettings init_settings; + auto license = options.onixs_license(); + init_settings.licenseStore(license); + if (license == "{default}") { + init_settings.licenseStore(os::get_cwd() + "/licenses/"); + } + LOG(MarketData,Info,"Loading onixs license from {}", init_settings.licenseStore()); + onixs::initialize(init_settings); +} + +// +// Initialize HandlerSettings and allocate the channel Handler +unique_ptr MDP3Main::prepare_handler(const string& channel_id) { + const auto& pub_options = config::get(); + const auto& options = channel_settings_for(channel_id); + LOG(MarketData,Info,"Loaded channel options for channel {}:\n{}",channel_id ,options.DebugString()); + onixs::HandlerSettings settings; + settings.channelId(channel_id); + settings.channelsConfigurationFile(config_path()); + settings.codingTemplatesFile(template_path()); + + // + // Tcp Replay options + + if (options.use_tcp_replay()) LOG(MarketData,Info,"CME: Enabled TCP replay"); + if (options.use_tcp_replay()) { + settings.tcpRecoveryUsername(options.tcp_replay_username()); + settings.tcpRecoveryPassword(options.tcp_replay_password()); + settings.tcpRecoveryReconnectAttempts(options.tcp_replay_connect_attempts()); + settings.tcpRecoveryReconnectInterval(options.tcp_replay_reconnect_interval_ms()); + } + + // + // Network settings + + settings.udpSocketBufferSize(options.udp_socket_buffer_size()); + if (options.has_lost_packet_wait_timeout_ms()) { + settings.lostPacketWaitTime((size_t)options.lost_packet_wait_timeout_ms() * 1000); + } + if (pub_options.has_network_interface()) { + settings.networkInterface(pub_options.network_interface()); + } + else { + if (pub_options.has_network_interface_a()) { + settings.networkInterfaceA(pub_options.network_interface_a()); + } + if (pub_options.has_network_interface_b()) { + settings.networkInterfaceB(pub_options.network_interface_b()); + } + } + + if (pub_options.has_network_interface() || + (pub_options.has_network_interface_a() && + pub_options.has_network_interface_b()) ) { + if (options.prefer_backup_feeds()) { + settings.instrumentFeedLayout(onixs::RecoveryFeedLayout::FeedBWithFailoverToFeedA); + settings.snapshotFeedLayout(onixs::RecoveryFeedLayout::FeedBWithFailoverToFeedA); + } + else { + settings.instrumentFeedLayout(onixs::RecoveryFeedLayout::FeedAWithFailoverToFeedB); + settings.snapshotFeedLayout(onixs::RecoveryFeedLayout::FeedAWithFailoverToFeedB); + } + } + else if (pub_options.has_network_interface_a()) { + settings.instrumentFeedLayout(onixs::RecoveryFeedLayout::FeedAOnly); + settings.snapshotFeedLayout(onixs::RecoveryFeedLayout::FeedAOnly); + } + else if (pub_options.has_network_interface_b()) { + settings.instrumentFeedLayout(onixs::RecoveryFeedLayout::FeedBOnly); + settings.snapshotFeedLayout(onixs::RecoveryFeedLayout::FeedBOnly); + } + + // + // Misc settings + + settings.recoverSecurityDefinitionsOnGap(false); + settings.bookUpdateNotificationStrategy(onixs::BookUpdateNotificationStrategies::OnEndOfEvent); + + // + // Logging + + settings.logDirectory(config::get().onixs_log_path()); + settings.logMode((onixs::LogModes::Enum)options.onixs_log_mode()); + settings.advancedLogOptions((onixs::AdvancedLogOptions::Enum)options.onixs_log_advanced()); + settings.logFileNamePrefix(channel_id + "_"); + + + // + // Associate with feed handler + + try { + auto temp = unique_ptr(new onixs::Handler(settings)); + if (options.has_feed_engine()) { + temp->bindToFeedEngine(_engines[options.feed_engine()].get()); + } + return temp; + } + catch (std::exception& e) { + LOG(MarketData,Error,"{}", e.what()); + throw; + } +} + +// +// Create a listener for this channel and register callbacks +unique_ptr MDP3Main::prepare_listener(onixs::Handler& handler, const CMEChannel& channel) { + // We need to get a handler to the producer channels so we can produce heartbeats + vector producer_channels; + for (auto& symbol : channel.symbols) { + producer_channels.push_back(_producer.get_channel(symbol.symbol().c_str(), symbol.type(), symbol.exchange())); + //handler.addSecurityAssetFilter(symbol.symbol()); + } + + unique_ptr listener(new MDP3Listener(_index, _producer, _security_recorder, _settlement_recorder, producer_channels, utils::Dispatcher::current())); + for (auto& symbol : channel.symbols) { + listener->add_filter_symbol(symbol.symbol()); + } + + handler.registerMessageProcessingListener(listener.get()); + handler.registerSecurityDefinitionListener(listener.get()); + handler.registerSecurityStatusListener(listener.get()); + handler.registerConsolidatedBookUpdateListener(listener.get()); + handler.registerStateChangeListener(listener.get()); + + if (!config::get().quoter_mode()) { + handler.registerTradeListener(listener.get()); + handler.registerElectronicVolumeListener(listener.get()); + handler.registerRequestForQuoteListener(listener.get()); + handler.registerStatisticsListener(listener.get()); + } + + handler.registerWarningListener(listener.get()); + handler.registerErrorListener(listener.get()); + + return listener; +} + + +}}//namespace +#endif + diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h new file mode 100644 index 000000000..e4a914654 --- /dev/null +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Main.h @@ -0,0 +1,50 @@ +#pragma once + +#include "lib/utils/bts_no_copy.h" +#include "server/sparkd/md/coconut2.h" +#include "MDP3Listener.h" +#include "CMESecurityIndex.h" +#include "onixs.h" + +#include +#include +#include + +namespace bts { +namespace md { + +void run_onixs_mdp3_server(md2::UserSymbol* sym, size_t symcount); + +struct CMEChannel; + +/* + * The MDP3 provider is very simple + * Control starts in MDP3Main which gets a list of all the symbols we need and starts the onixs handlers + * There is a single listener for all callbacks which maps the exchange id to the MDProducerHandle and writes + * the data to it. + */ +class MDP3Main +{ +public: + MDP3Main(); + ~MDP3Main(); + + void run(std::vector&); + void initialize(); + std::unique_ptr prepare_handler(const std::string& channel); + std::unique_ptr prepare_listener(onixs::Handler& handler, const CMEChannel& channel); + +private: + BTS_NO_COPY(MDP3Main); + + CMESecurityIndex _index; + md2::MDProducer _producer; + md2::SecurityIDRecorder _security_recorder; + co2::SettlementRecorder _settlement_recorder; + std::map> _engines; + std::vector> _handlers; + std::vector> _listeners; +}; + +}}//namespace + diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Utils.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Utils.cpp new file mode 100644 index 000000000..29812d353 --- /dev/null +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Utils.cpp @@ -0,0 +1,128 @@ +#ifndef NO_CPP0X +#include "MDP3Utils.h" +#include "onixs.h" + +#include "lib/config/config.h" +#include "lib/utils/datetime.h" +#include "lib/utils/bts_chrono.h" + +#include "lib/config/protos/MarketData.pb.h" +#include "lib/config/protos/system.pb.h" + +#include +#include + +using namespace std; +using namespace bts::md2; + +namespace bts { +namespace md { + +string data_path() { + auto p = config::get().data_path() + "cme_mdp3/"; +#ifdef _WIN32 + // OnixS requires win32 paths to look like Win32 paths - everyone else in the world is ok with '/' + os::normalize_path(p, '\\'); +#else + os::normalize_path(p); +#endif + return p; +} +const string config_path() { return data_path() + "config.xml"; } +const string template_path() { return data_path() + "templates.xml"; } + +bool check_file_exists_with_instructions(const string& filename, const string& url, bool log) { + if (!boost::filesystem::exists(filename)) { + if (log) { + LOG(MarketData,Error, + "File {} is not present on this system. " + "Please download it from the CME ftp site. " + "I will wait until the file is present.\n$ curl {} > {} ", + filename, url, filename); + } + return false; + } + return true; +} + +// For exponential backoff on logging this return true if x is a power of 2 or 0 +bool is_pow_two_or_0(uint32_t x) { return (x & (~x + 1)) == x; } + +void await_cme_config() { + const auto config_url = "ftp://ftp.cmegroup.com/SBEFix/Production/Configuration/config.xml"; + const auto templates_url = "ftp://ftp.cmegroup.com/SBEFix/Production/Templates/templates_FixBinary.xml"; + + boost::filesystem::create_directories(data_path()); + + // Check config is present and give instructions for downloading it if not + // If the file is not present log an error and wait a second. Continue to log the error ever power of 2 seconds until the file appears or the process is killed. + uint32_t count = 0; + while(!check_file_exists_with_instructions(config_path(), config_url, is_pow_two_or_0(count))) { + ++count; + std::this_thread::sleep_for(1s); + } + count = 0; + while(!check_file_exists_with_instructions(template_path(), templates_url, is_pow_two_or_0(count))) { + ++count; + std::this_thread::sleep_for(1s); + } + + // If we waited for someone to download the file give it a second to be finished downloading + if (count > 0) { + std::this_thread::sleep_for(1s); + } +} + +SecurityType::type infer_security_type(const string& label) { + if (label.find("Option") != std::string::npos) { + return SecurityType::OPTION; + } + + if (label.find("Future") != std::string::npos) { + return SecurityType::FUTURE; + } + + // TODO: Allow configuration to override this? + LOG(MarketData,Warn,"Unable to guess security type from label: '{}" + "' Treating as OPTION.",label); + return SecurityType::OPTION; +} + +const vector parse_channels(const vector& symbols) { + vector result; + + pugi::xml_document doc; + bts_assert_msg(doc.load_file(config_path().c_str()), "Error parsing config.xml as xml"); + + auto configuration = doc.root().child("configuration"); + bts_assert(configuration); + + LOG(MarketData,Info,"CME: Loaded '{}' config. Last updated: {}" , + configuration.attribute("environment").as_string(), + configuration.attribute("updated").as_string()); + auto channels = configuration.children("channel"); + for (const auto& chan : channels) { + CMEChannel channel; + channel.id = chan.attribute("id").as_string(); + auto type = infer_security_type(chan.attribute("label").as_string()); + if (!(channel.id.empty())) { + auto product_list = chan.child("products").children("product"); + for (const auto& product : product_list) { + auto symbol = product.attribute("code").as_string(); + for (auto s : symbols) { + if ((s.type() == type || s.type() == SecurityType::SPREAD) && s.symbol() == symbol) { + channel.symbols.insert(s); + } + } + } + if (!channel.symbols.empty()) { + result.push_back(channel); + } + } + } + + return result; +} + +}}//namespace +#endif diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Utils.h b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Utils.h new file mode 100644 index 000000000..333d144c3 --- /dev/null +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Utils.h @@ -0,0 +1,23 @@ +#pragma once + +#include "server/sparkd/md/coconut2.h" +#include +#include +#include + +namespace bts { +namespace md { + +struct CMEChannel { + std::string id; + std::set symbols; +}; + +void await_cme_config(); +const std::vector parse_channels(const std::vector& symbols); + +const std::string config_path(); +const std::string template_path(); + +}}//namespace + diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/licenses/Onixs.BTS.lic b/src/server/sparkd/md/platforms/mdp3_onixs/licenses/Onixs.BTS.lic new file mode 100644 index 000000000..b91537e9d --- /dev/null +++ b/src/server/sparkd/md/platforms/mdp3_onixs/licenses/Onixs.BTS.lic @@ -0,0 +1,21 @@ + + production + + Blue Trading Systems + + + 2017-04-24 + 2018-08-29 + + + + + jqN4XONnMeiDSJIAa6CK9m9SitquhMq3MhJW97fG26Fp+GPog9ObMLS2elPvippdZsRWmJGsjTmFxAg8d41Vr+tpNgObi3VT3DtfYjGESTmi+u4vvJ9PNi8tQWQ7HPYBJe1X6j84ijhe5e1jHEcaXZ08ExDbESebn2ndHb4Phv4= + + + + + + + F5502F3BD9CE954022ABCBF94A7E4052 + \ No newline at end of file diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/onixs.h b/src/server/sparkd/md/platforms/mdp3_onixs/onixs.h new file mode 100644 index 000000000..8a5909761 --- /dev/null +++ b/src/server/sparkd/md/platforms/mdp3_onixs/onixs.h @@ -0,0 +1,4 @@ +#pragma once +#include + +namespace onixs = OnixS::CME::MDP3; diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/tests/mdp3_tests.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/tests/mdp3_tests.cpp new file mode 100644 index 000000000..06ca686a6 --- /dev/null +++ b/src/server/sparkd/md/platforms/mdp3_onixs/tests/mdp3_tests.cpp @@ -0,0 +1,210 @@ +#include "catch/catch.h" + +#include "lib/config/config.h" +#include "lib/config/protos/MarketData.pb.h" + +#include +#include +#include +#include + +#include "../MDP3Utils.h" +#include "../onixs.h" + +using namespace bts::md; +using namespace std; + + +namespace bts { +namespace md { + +map create_feed_engine_settings(const std::vector& channels); +const CMEOptions::CMEChannel channel_settings_for(const string& id); + +namespace test { + +vector test_channels(int count, ...) { + // Doing this the C way because I still haven't updated VS + vector result; + va_list ap; + va_start(ap, count); + for (int x = 0; x < count; ++x) { + const char* val = va_arg(ap, char*); + CMEChannel temp; + temp.id = val; + result.push_back(temp); + } + va_end(ap); + return result; +} + + +TEST_CASE("mdp3_tests","[md]") { + SECTION("test_defaults") { + config::internal::initialize_for_test("CMEOptions {}"); + + vector channels = test_channels(1, "310"); + auto result = create_feed_engine_settings(channels); + REQUIRE(result.size() == 0); + + channels = test_channels(2, "310", "311"); + result = create_feed_engine_settings(channels); + REQUIRE(result.size() == 0); + + channels = test_channels(4, "310", "311", "400", "412"); + result = create_feed_engine_settings(channels); + REQUIRE(result.size() == 0); + + REQUIRE(!channel_settings_for("310").has_feed_engine()); + REQUIRE(!channel_settings_for("311").has_feed_engine()); + REQUIRE(!channel_settings_for("315").has_feed_engine()); + } + + SECTION("test_segregate_futures") { + config::internal::initialize_for_test( + "CMEOptions {\n" + " value {\n" + " feed_engine {\n" + " name: 'default'\n" + " }\n" + " feed_engine {\n" + " name: 'futures'\n" + " num_threads: 4\n" + " }\n" + " # Futures\n" + " channel {\n" + " id: '310'\n" + " id: '312'\n" + " feed_engine: 'futures'\n" + " }\n" + " # Options\n" + " channel {\n" + " id: '311'\n" + " id: '313'\n" + " id: '315'\n" + " feed_engine: 'default'\n" + " }\n" + " }\n" + "}"); + + vector channels = test_channels(1, "310"); + auto result = create_feed_engine_settings(channels); + REQUIRE(result.size() == 1); + REQUIRE(result.begin()->first == "futures"); + REQUIRE(result.begin()->second.threadsCount() == 4); + + channels = test_channels(1, "311"); + result = create_feed_engine_settings(channels); + REQUIRE(result.size() == 1); + REQUIRE(result.begin()->first == "default"); + REQUIRE(result.begin()->second.threadsCount() == 1); + + channels = test_channels(2, "311", "310"); + result = create_feed_engine_settings(channels); + REQUIRE(result.size() == 2); + REQUIRE(result["default"].threadsCount() == 1); + REQUIRE(result["futures"].threadsCount() == 4); + + // Test channels not listed in config + channels = test_channels(2, "317", "318"); + result = create_feed_engine_settings(channels); + REQUIRE(result.size() == 0); + } + + SECTION("test_no_default") { + config::internal::initialize_for_test( + "CMEOptions {\n" + " value {\n" + " feed_engine {\n" + " name: 'futures'\n" + " num_threads: 4\n" + " }\n" + " # any channel not listed will use the default feed engine\n" + " # Futures\n" + " channel {\n" + " id: '310'\n" + " id: '312'\n" + " feed_engine: 'futures'\n" + " }\n" + " # Options\n" + " channel {\n" + " id: '311'\n" + " id: '313'\n" + " id: '315'\n" + " use_tcp_replay: false\n" + " }\n" + " }\n" + "}"); + + vector channels = test_channels(1, "310"); + auto result = create_feed_engine_settings(channels); + REQUIRE(result.size() == 1); + REQUIRE(result.begin()->first == "futures"); + REQUIRE(result.begin()->second.threadsCount() == 4); + + channels = test_channels(1, "311"); + result = create_feed_engine_settings(channels); + REQUIRE(result.size() == 0); + + channels = test_channels(2, "311", "310"); + result = create_feed_engine_settings(channels); + REQUIRE(result.size() == 1); + REQUIRE(result["futures"].threadsCount() == 4); + + + REQUIRE(channel_settings_for("310").use_tcp_replay() == true); + REQUIRE(channel_settings_for("311").use_tcp_replay() == false); + REQUIRE(channel_settings_for("315").use_tcp_replay() == false); + } + + SECTION("test_one_thread_each") { + config::internal::initialize_for_test( + "CMEOptions {\n" + " value {\n" + " channel {\n" + " id: '*'\n" + " }\n" + " }\n" + "}\n"); + + vector channels = test_channels(1, "310", "311"); + auto result = create_feed_engine_settings(channels); + REQUIRE(result.size() == 0); + + const auto& settings = channel_settings_for("311"); + REQUIRE(!settings.has_feed_engine()); + } + + SECTION("test_common_futures_thread") { + config::internal::initialize_for_test( + "CMEOptions {\n" + " value {\n" + " feed_engine {\n" + " name: 'futures'\n" + " num_threads: 4\n" + " }\n" + " channel {\n" + " id: '310'\n" + " feed_engine: 'futures'\n" + " udp_socket_buffer_size: 1234\n" + " }\n" + " channel {\n" + " id: '*'\n" + " }\n" + " }\n" + "}\n"); + + vector channels = test_channels(6, "310", "311", "315", "100", "10", "42"); + auto result = create_feed_engine_settings(channels); + REQUIRE(result.size() == 1); + result = create_feed_engine_settings(test_channels(1, "315")); + REQUIRE(result.size() == 0); + + REQUIRE(!channel_settings_for("311").has_feed_engine()); + REQUIRE(channel_settings_for("310").has_feed_engine()); + REQUIRE(channel_settings_for("310").udp_socket_buffer_size() == 1234); + REQUIRE(channel_settings_for("311").has_udp_socket_buffer_size() == false); + } +} + +}}}//namespace diff --git a/src/server/sparkd/md/protos/co2.proto b/src/server/sparkd/md/protos/co2.proto new file mode 100644 index 000000000..08666874a --- /dev/null +++ b/src/server/sparkd/md/protos/co2.proto @@ -0,0 +1,126 @@ +syntax='proto2'; +package bts.co2; +import "lib/security_model/security.proto"; +import "server/sparkd/md/protos/coconut.proto"; + +option cc_enable_arenas=true; + +message SecurityChannel { + optional uint64 time = 1; + optional uint32 flags = 2; + optional bts.security_model.SecurityType.type type = 3 [default = UNSET]; + optional bts.security_model.Exchange.type exchange = 4 [default = TESTEXCHANGE]; + optional string symbol = 5; + + optional int64 trade_cursor = 6; + optional int64 rfq_cursor = 7; + optional uint64 segment_id = 8; +} + +message MDRequest { + // Separate out timestamps by symbol+type so we can easily add symbols 'lazily' + repeated SecurityChannel channels = 1; + enum Flags { + SECURITY = 0; + BBO = 0x1; + DEPTH = 0x2; + LAST_TRADE = 0x4; + TRADE_LOG = 0x8; + RFQ = 0x10; + CUSTOMER_BBO = 0x20; + MISC = 0x40; + } + optional string host = 2; +} + +message MDReply { + message Update { + required uint32 key = 1; + optional bts.security_model.Security security = 2; + optional BookMessage book = 3; + optional TopBookMessage customer_bbo = 4; + optional MiscMessage misc = 5; + optional TradeMessage last_trade = 6; + } + message ChannelReply { + required SecurityChannel channel = 1; + required md.Platform.type platform = 2; + repeated Update updates = 3; + repeated TradeMessage trade_log = 4; + repeated QuoteRequestMessage rfq_log = 5; + } + repeated ChannelReply channels = 1; +} + +message PriceSizeMessage { + required int64 mantissa = 1; + required sint32 exponent = 2; + optional uint32 size = 3; +} + +message TopBookMessage { + // required uint32 key = 1; + // optional int64 time = 2; + optional PriceSizeMessage bid = 10; + optional PriceSizeMessage ask = 11; +} + +message BookMessage { + // required uint32 key = 1; + // optional int64 time = 2; + repeated PriceSizeMessage bids = 10; + repeated PriceSizeMessage asks = 11; +} + +message MiscMessage { + // required int32 key = 1; + optional double settlement_price = 2; + optional uint32 settlement_date = 3; + optional double previous_settlement_price = 20; + optional uint32 previous_settlement_date = 21; + optional double open_price = 4; + optional uint32 open_date = 5; + optional double close_price = 6; + optional uint32 close_date = 7; + optional double session_high = 8; + optional double session_low = 9; + optional double open_interest = 10; + optional double session_high_bid = 11; + optional double session_low_offer = 12; + optional double fixing_price = 13; +} + +message TradeMessage { + required uint32 key = 1; + optional int64 time = 2; + // Flattened out the TradeMessage so it can be stored directly in shared memory + //optional PriceSizeMessage price_size = 3; + optional int64 mantissa = 4; + optional sint32 exponent = 5; + optional uint32 size = 6; + optional uint32 total_volume = 7; + optional bool gap = 8 [default = false]; +} +message QuoteRequestMessage { + enum Side { + UNDEFINED = 0; + BUY = 49; + SELL = 50; + CROSS = 99; + } + required uint32 key = 1; + optional int64 time = 2; + optional string request_id = 3; + //optional double price = 4; + optional int32 size = 5; + optional Side side = 6 [default = UNDEFINED]; + optional bool tradable = 7; + optional bool gap = 8 [default = false]; +} + +message TradeList { + repeated TradeMessage items = 1; +} +message RFQList { + repeated QuoteRequestMessage items = 1; +} \ No newline at end of file diff --git a/src/server/sparkd/md/protos/coconut.proto b/src/server/sparkd/md/protos/coconut.proto new file mode 100755 index 000000000..e933915ee --- /dev/null +++ b/src/server/sparkd/md/protos/coconut.proto @@ -0,0 +1,334 @@ +syntax='proto2'; +package bts.md; +// option cc_enable_arenas = true; + +// +// Notes +// +// When you update this file you also need to update the coconut system to handle the new message type. +// +// Use this handy checklist +// +// [ ] SubscriptionBuilder +// [ ] SubscriptionEx +// [ ] // Remember to increase the size of PublishOptions._flags +// [ ] MDRecorder +// [ ] ReplaySubscription +// [ ] Any market data provider which should produce the message (duh) + +import "lib/security_model/security.proto"; + +// +// Enums +// + +message Platform +{ + enum type + { + UNSET = 0; + SIM = 1; + SRLABS = 2; + SPRYWARE = 3; + CSM = 4; + CME = 5; + STREAM = 6; + ICE = 7; + REPLAY = 8; + ACTIV = 9; + MDP3ONIXS = 10; + CSM_ONIXS = 11; + BLUE = 12; + } +} + +message BidOrAsk +{ + enum type + { + BID = 1; + ASK = 2; + } +} + +message CBOEVolumeType +{ + enum type + { + TOTAL = 1; + CUSTOMER = 2; + ALL_OR_NONE = 3; + CUSTOMER_ALL_OR_NONE = 4; + } +} + +message TradingStatus +{ + enum type + { + UNKNOWN = 1; + NORMAL = 2; + CLOSED = 3; + PRE_OPEN = 4; + OPENING_ROTATION = 5; + FAST_MARKET = 6; // CBOE + SUSPENDED = 7; + HALTED = 8; + PRICE_INDICATION = 9; // CME + PRE_CROSS = 10; // CME + CROSS = 11; // CME + NO_CANCEL = 12; // CME + } +} + +message CoconutSecurityStatus +{ + enum type + { + OK = 0; + PACKET_LOSS = 2; + } +} + +// Instrument definition (Security) from exchange_messages.proto + +message BookMessage +{ + optional int64 time = 1; + required BidOrAsk.type side = 2; + optional double price = 3; + optional uint32 size = 4; + + // No longer needed since 1.111 as customer bbo has a dedicated channel + //optional CBOEVolumeType.type volume_type = 5 [default = TOTAL]; + + // Performance + optional uint64 server_arrival_time = 6; + optional uint64 client_arrival_time = 7; + + // Price now integer 3.0 + // optional int64 mantissa = 8; + // optional int32 exponent = 9; +} + +message TradeMessage +{ + optional int64 time = 1; + optional double price = 2; + optional uint32 size = 3; + optional uint32 total_volume = 4; + //optional string trade_condition = 5; + + // Added for Co2 to avoid heap + optional uint32 security_key = 6; +} + +message MiscMessage { + optional double settlement_price = 1; + optional uint64 settlement_timestamp = 2; + + optional double opening_price = 3; + optional uint32 opening_size = 4; + + optional double closing_price = 5; + //optional uint32 closing_size = 6; + + optional double session_high = 7; + optional double session_low = 8; + optional double open_interest = 9; + optional double prior = 10; + optional double session_high_bid = 11; + optional double session_low_offer = 12; + optional double fixing_price = 13; + + //optional double reference_price = 14; + //optional int64 reference_date = 15; +} + +message QuoteRequestMessage { + optional int64 time = 1; + optional string request_id = 2; + + optional double price = 10; + optional int32 size = 11; + enum Side { + UNDEFINED = 0; + // match o2 Side + BUY = 49; + SELL = 50; + CROSS = 99; + } + optional Side side = 12 [default = UNDEFINED]; + optional int64 transact_time = 13; + optional uint32 quote_type = 14; + + // Co2 + optional uint32 security_key = 15; + optional uint32 sequence = 16; + +} + +// +// Wire messages (Coconut 1) +// + +message SecurityRow +{ + optional uint64 security_id = 1; + required bts.security_model.Security security = 2; +} + +message ExchangeStatusRow +{ + optional uint64 security_id = 1; + required TradingStatus.type status = 2; +} + +message CoconutStatusRow +{ + optional uint64 security_id = 1; + required CoconutSecurityStatus.type status = 2; +} + + +message BBORow +{ + optional uint64 security_id = 1; + optional BookMessage bid = 2; + optional BookMessage ask = 3; +} + +message CustomerBBORow +{ + optional uint64 security_id = 1; + optional BookMessage bid = 2; + optional BookMessage ask = 3; +} + +message BookRow +{ + optional uint64 security_id = 1; + optional uint32 depth = 2; + required BookMessage book = 3; +} + +message TradeRow +{ + optional uint64 security_id = 1; + optional TradeMessage trade = 2; +} + +message MiscRow { + optional uint64 security_id = 1; + optional MiscMessage misc = 2; + optional bts.security_model.Security security = 3; +} + +message QuoteRequestRow { + // Optional for backwards compat with the replayer + optional uint64 security_id = 1; + required QuoteRequestMessage request = 2; +} + +// +// Status / Metadata messages +// + +message CoconutStatus +{ + optional string symbol = 1; + optional bts.security_model.SecurityType.type type = 2 [default = UNSET]; + optional bts.security_model.Exchange.type exchange = 3 [default = TESTEXCHANGE]; + //optional string status = 4; + + optional uint32 security_count = 10; + optional uint32 bbo_count = 11; + optional uint32 trade_count = 12; + optional uint32 estatus_count = 13; + optional uint32 rfq_count = 14; + optional uint32 depth_count = 15; + optional uint32 cstatus_count = 16; + optional uint32 custbbo_count = 17; + + //optional string last_error = 20; + //optional string last_warning = 21; + + optional uint32 bbo1 = 30; + optional uint32 bbo5 = 31; + optional uint32 bbo15 = 32; + optional uint32 bbo_burst = 33; + optional string bbo_burst_when = 34; + optional uint32 trade1 = 35; + optional uint32 trade5 = 36; + optional uint32 trade15 = 37; + optional uint32 trade_burst = 38; + optional string trade_burst_when = 39; + optional uint32 depth1 = 40; + optional uint32 depth5 = 41; + optional uint32 depth15 = 42; + optional uint32 depth_burst = 43; + optional string depth_burst_when = 44; + optional uint32 custbbo1 = 45; + optional uint32 custbbo5 = 46; + optional uint32 custbbo15 = 47; +} + +message Co2Status { + optional string symbol = 1; + optional bts.security_model.SecurityType.type type = 2 [default = UNSET]; + optional bts.security_model.Exchange.type exchange = 3 [default = TESTEXCHANGE]; + optional Platform.type platform = 4 [default = UNSET]; + + optional uint32 security_count_in = 10; + optional uint32 security_count_out = 11; + + optional uint32 bbo_count_in = 20; + optional uint32 bbo_count_out = 21; + + optional uint32 trade_count_in = 30; + optional uint32 trade_count_out = 31; + + optional uint32 rfq_count_in = 40; + optional uint32 rfq_count_out = 41; + + optional uint32 depth_count_in = 50; + optional uint32 depth_count_out = 51; + + optional uint32 custbbo_count_in = 60; + optional uint32 custbbo_count_out = 61; + +} + + +// +// Replayer messages +// + + +// Replay file format +// [uin16_t length] [byte HEADER] [uint16_t unused] CoconutReplayHeader - [uint16_t length] [byte CoconutRowType] [uint16_t delta seconds] [Row type] + +message CoconutRowType { + enum type { + HEADER = 1; + SECURITY = 2; + EXCHANGE_STATUS = 3; + BBO = 4; + BOOK = 5; + TRADE = 6; + MISC = 7; + RFQ = 8; + CUSTOMER_BBO = 9; + } +} + +message CoconutReplayHeader { + required string magic = 1; + required uint32 version = 2; + required int64 timestamp = 3; + optional string host = 4; + + optional int32 naggle_ms = 5; +} + + diff --git a/src/server/sparkd/md/readme.txt b/src/server/sparkd/md/readme.txt new file mode 100644 index 000000000..7c8c37380 --- /dev/null +++ b/src/server/sparkd/md/readme.txt @@ -0,0 +1,96 @@ + + +******************** +classes +******************** + +btscmd md2 => PlatformLoader + +PlatformLoader: + * Loads the platform specific dll + * Calls ->start_marketdata() with a full vector + +SharedList / SharedListWriter + * Shared memory list object which spans across shared memory segments as necessary + * O(N) walk + * atomic add() + * Single adder / multiple readers + * no delete + * Custom header + +SecurityIDRecorder + * add_security(security, lambda callback) -> int returns security id instantly + * security_persisted callback when security id is safely persisted + +SecurityIndex + * Platform Implementation helper + * Maps exhange_id to (ProducerHandle + Data) + * + +CacheEntry / CacheEntryDepth (CacheStructure.h) + * CacheEntry(symbol, depth, # legs) -- size determined in constructor + * Simple inheritance tree for optional depth + * Has FixedSecurity + * Fits in SharedList + * CRITICAL fields are at the front cache aligned. bbo/depth. Security goes at the end + + * size, is_public(), is_persisted, is_resolved (spread) + * statistics like bbo/trade counters + * is_status_ok + * bbo, depth, trade, status, open, close + +FixedSecurity / FixsdSpreadSecurity + * Normalized version of our protobuf security + * Has methods to convert to pb Security + * Size known at construction w/ # legs - spread legs of type security_key + * Can use a union for option/future fields + +MDProducer(symbol, type) + * manages lifetime of SharedList instances + * Hands out MDProducerHandle - entry for one security + +MDProducerHandle + * pointers to shared memory + * update methods + +MDConsumer + * list of securities + * find_security_id(symbol, type, etc) + * get_cache_handle(security id#) + * Hands out MDCacheHandle + - if security is null add MDCacheHandle to a list to be updated + next time securities are polled + +MDCacheHandle + * If security is not present add this is "null" + * has pointers to relevant shared memory + + + +******************** +* Cache Structure +******************** + +2 Shared memory segment for each (SYMBOL, TYPE) - md_hot (bbo,depth,status), md_cold (others) + +When a new security is discovered + 1. id is allocated to the security + 2. New CacheEntry is allocated in appropriate arena (is_public=false) + 3. md_cache entry count is incremented + 4. When id is persisted is_public=true, increment version on cache arena + +/* OUTDATED ** +When a new security is discovered: + 1. md_cache is extended and a block is reserved (is_public: false) + 2. When securitys id has been persisted to disk it is added to security_list + 3. Clients can watch the security_list for changes which indicate new securities + 4. When a security shows up in the security_list you can go get it from the md_cache. +*/ + +*** TODO *** + ** No way around it - we need a fixed size, normalized Security + * Number of spread legs is always known before resolve + * Each leg can be invalid_security until resolved + + + diff --git a/src/server/sparkd/md/relay/QueryServer.cpp b/src/server/sparkd/md/relay/QueryServer.cpp new file mode 100644 index 000000000..618ad5a23 --- /dev/null +++ b/src/server/sparkd/md/relay/QueryServer.cpp @@ -0,0 +1,382 @@ +#ifndef NO_CPP0X +#include "QueryServer.h" +#include "lib/config/config.h" +#include "lib/utils/Dispatcher.h" +#include "lib/utils/TimeStamp.h" +#include "lib/utils/helper_macros.h" + +#include "server/sparkd/md/MDConsumer.h" +#include "server/sparkd/md/PlatformLoader.h" + +#include "lib/logging/logging.h" + +#include "lib/config/protos/system.pb.h" +#include "server/sparkd/md/protos/co2.pb.h" + +#include +#include +#include +#include +#include +#include + +using namespace bts::md2; +using namespace bts::co2; +using std::unique_ptr; +using std::shared_ptr; + +namespace bts { +namespace md { + +/* + * NORMAL OPERATION: + * Client sends time=0 to server + * Server sends complete snapshot back (time=502) + * client sends time=502 to server + * Server sends any changes that have happened after time=502 and any securities discovered since time=502 + * + * ERROR RECOVERY: + * Client sends time=0 to server + * Server sends complete snapshot back (time=502) + * Server detects heartbeat failure and recovers. All timestamps > 502 + * Client sends time=502 to server + * Server sends complete snapshot back (time=605) + * - Since the server shared memory has been recreated the client will automatically overwrite all stale data. + * - If a security has been deleted the client will leave it frozen (OK) + * + * We scan for bad heartbeats on a different time + * If a bad heartbeat is detected we try to construct a new Consumer + * When a good consumer is created we std::move the old consumer and handles to delayed queue for deletion + * This way we don't have to lock anything during the recovery process because all pointers will remain valid + * for the duration of existing calls. + * +*/ +struct DatedHandle { + TimeStamp discovered; + MDConsumerHandle handle; +}; + +struct MDConsumerGroup { + MDConsumerGroup(const UserSymbol& symbol) : + symbol(symbol), + platform(platform_for_exchange(symbol.exchange())), + consumer(symbol), + channel(consumer.get_channel(symbol)), + handles(new tbb::concurrent_vector()) + { + initialize_handles(); + } + // non-reentrant + void try_recover(utils::Dispatcher& d) { + // if hearbeat is ok just return + if (consumer.all_heartbeats_ok()) return; + // otherwise construct a new MDConsumer + MDConsumer temp(symbol); + if (temp.all_heartbeats_ok()) { + consumer.swap(temp); + auto temphandles = std::make_unique>(); + handles.swap(temphandles); + initialize_handles(); + // Switch to shared pointer for lambda. + shared_ptr> oldhandles(std::move(temphandles)); + d.post([oldhandles,temp]() mutable { + // After some time has passed all outstanding requests are guarunteed to be complete + // Explicit reset of oldhandles ensures they are deleted before the shared memory is unmapped + // (lambda capture object data members order is not specified) + oldhandles.reset(); + }, 60000); + } + } + + UserSymbol symbol; + Platform::type platform; + MDConsumer consumer; + MDConsumerChannel channel; + + // Can add to handles without invalidating readers (no lock needed) + // When we recover we swap a new vector into this unique_ptr and delayed-delete + unique_ptr> handles; + +private: + BTS_NO_COPY(MDConsumerGroup); + void add_security(MDConsumerHandle handle) { + bts_assert(handles); + bts_assert_d(handle.is_initialized()); + DatedHandle h = { TimeStamp::now(), handle }; + handles->push_back(h); + } + // We can't copy or move this because initialize_handles implicitly captures 'this' + void initialize_handles() { + bts_assert(handles->empty()); + auto id = consumer.id(); + channel = consumer.get_channel(symbol.symbol(), symbol.type(), symbol.exchange()); + consumer.on_security( + [this,id](const Security&, MDConsumerHandle handle) { + if (id == consumer.id()) { + add_security(handle); + } + }, false); + } +}; + +// Wrap the flat_map to ensure immutability +class Map { + // Can't use unique_ptr because flat_map requires assignment. + typedef boost::container::flat_map> InternalMap; + InternalMap _map; +public: + Map(InternalMap map) : _map(move(map)) { } + Map(Map&& other) : _map(move(other._map)) {} + + InternalMap::iterator begin() { return _map.begin(); } + InternalMap::iterator end() { return _map.end(); } + InternalMap::const_iterator begin() const { return _map.begin(); } + InternalMap::const_iterator end() const { return _map.end(); } + InternalMap::const_iterator find(const UserSymbol& symbol) const { return _map.find(symbol); } +}; + +Map form_groups(const std::set& symbols) { + boost::container::flat_map> groups; + for (const auto& sym : symbols) { + groups.emplace(std::make_pair(sym, std::make_unique(sym))); + } + groups.shrink_to_fit(); + return Map(groups); +} + +void convert(PriceSize ps, PriceSizeMessage* pps) { + pps->set_mantissa(ps.mantissa()); + pps->set_exponent(ps.exponent()); + pps->set_size(ps.size()); +} +void set_price(PriceSize ps, co2::TradeMessage* ptrade) { + ptrade->set_mantissa(ps.mantissa()); + ptrade->set_exponent(ps.exponent()); + ptrade->set_size(ps.size()); +} +void copy_misc(MiscData misc, co2::MiscMessage* pmisc) { + pmisc->set_settlement_price(misc.settlement_price); + pmisc->set_settlement_date(misc.settlement_date.to_short()); + pmisc->set_previous_settlement_price(misc.previous_settlement_price); + pmisc->set_previous_settlement_date(misc.previous_settlement_date.to_short()); + pmisc->set_open_price(misc.open_price); + pmisc->set_open_date(misc.open_date.to_short()); + pmisc->set_close_price(misc.close_price); + pmisc->set_close_date(misc.close_date.to_short()); + //pmisc->set_fixing_price(misc.fixing_price); + pmisc->set_session_high(misc.session_high); + pmisc->set_session_low(misc.session_low); + //pmisc->set_open_interest(misc.open_interest); + //pmisc->set_session_high_bid(misc.session_high_bid); + //pmisc->set_session_low_offer(misc.session_low_offer); +} + +// Handles market data queries and responds +struct QueryServer : public IQueryServer { + QueryServer(const std::set& symbols) : + groups(form_groups(symbols)) + { } + + // Scan for new securities and get their handles+times + // not-reentrant + void scan_securities() const { + for (auto& group : groups) { + if (group.second->consumer.all_heartbeats_ok()) { + TimeStamp start = TimeStamp::now(); + size_t count = group.second->consumer.scan_securities(false); + auto delta = (TimeStamp::now() - start); + if (delta.milliseconds() > 10) { + LOG(MarketData,Diag,"Scanned handles: {}ms found: {} securities. " , + delta.milliseconds() ,count ,group.second->symbol ); + } + } + } + } + + // not-reentrant + void check_heartbeats(utils::Dispatcher& dispatcher) { + for (auto& group : groups) { + if (!group.second->consumer.all_heartbeats_ok()) { + group.second->try_recover(dispatcher); + } + } + } + + + // the hot part + void fill(const SecurityChannel& chan, MDConsumerGroup& group, MDReply::ChannelReply* reply) const { + bts_assert(reply); + reply->mutable_channel()->CopyFrom(chan); + reply->set_platform(group.platform); + reply->mutable_channel()->set_time(TimeStamp::now().to_uint64()); + auto flags = (MDRequest::Flags)chan.flags(); + + // Get a copy of handles. If recovery happens during this call + // we can safely continue using this pointer for 60 seconds + auto phandles = group.handles.get(); + if (!phandles) return; + + MDReply::Update temp_update; + for (const auto& instrument : *phandles) { + bool snapshot = false; + MDConsumerHandle handle = instrument.handle; + temp_update.Clear(); + MDReply::Update* pupd = &temp_update; + pupd->set_key(instrument.handle.key()); + bts_assert_d(handle.is_initialized()); + if (!handle.is_initialized()) continue; + // If the security was just discovered try to ensure all data is sent + if (instrument.discovered > TimeStamp(chan.time())) { + pupd->mutable_security()->CopyFrom(handle.get_normalized_security()); + snapshot = true; + } + if (HASFLAG(flags, MDRequest::BBO) || HASFLAG(flags, MDRequest::DEPTH)) { + PriceSize bid, ask; + TimeStamp time(snapshot ? 0 : chan.time()); + if (handle.get_bbo(time, bid, ask)) { + auto* p = pupd->mutable_book(); + convert(bid, p->add_bids()); + convert(ask, p->add_asks()); + if (HASFLAG(flags, MDRequest::DEPTH) && handle.has_depth()) { + for (size_t x = 1; x < COCONUT_MAX_DEPTH; ++x) { + PriceSize bid = handle.get_bid(x); + if (bid != PriceSize::NO_MARKET) { + convert(bid, p->add_bids()); + } + PriceSize ask = handle.get_ask(x); + if (ask != PriceSize::NO_MARKET) { + convert(ask, p->add_asks()); + } + } + } + } + } + if (HASFLAG(flags, MDRequest::LAST_TRADE)) { + TimeStamp time(snapshot ? 0 : chan.time()); + PriceSize price; + datetime::LLDateTime trade_time; + size_t volume; + if (handle.get_last_trade(time, price, trade_time, &volume)) { + auto* trade = pupd->mutable_last_trade(); + trade->set_key(handle.key()); + trade->set_time(trade_time); + trade->set_total_volume(volume); + set_price(price, trade); + } + } + if (HASFLAG(flags, MDRequest::CUSTOMER_BBO)) { + TimeStamp time(snapshot ? 0 : chan.time()); + PriceSize bid, ask; + if (handle.get_customer_bbo(time, bid, ask)) { + auto* p = pupd->mutable_customer_bbo(); + convert(bid, p->mutable_bid()); + convert(ask, p->mutable_ask()); + } + } + if (HASFLAG(flags, MDRequest::MISC)) { + TimeStamp time(snapshot ? 0 : chan.time()); + if (handle.misc_timestamp() > time) { + copy_misc(handle.misc(), pupd->mutable_misc()); + } + } + if (pupd->has_book() || + pupd->has_last_trade() || + pupd->has_customer_bbo() || + pupd->has_misc() || + pupd->has_security()) { + reply->add_updates()->CopyFrom(*pupd); + } + } + + reply->mutable_channel()->set_segment_id(group.channel.create_time().to_uint64()); + bool is_same_segment = (chan.segment_id() == group.channel.create_time().to_uint64()); + + if (HASFLAG(flags, MDRequest::TRADE_LOG)) { + int64_t request_cursor = is_same_segment ? chan.trade_cursor() : 0; + auto cursor = group.channel.read_trades(*reply->mutable_trade_log(), request_cursor); + if (cursor != chan.trade_cursor()) { + reply->mutable_channel()->set_trade_cursor(cursor); + } + } + if (HASFLAG(flags, MDRequest::RFQ)) { + int64_t request_cursor = is_same_segment ? chan.rfq_cursor() : 0; + auto cursor = group.channel.read_rfqs(*reply->mutable_rfq_log(), request_cursor); + if (cursor != chan.rfq_cursor()) { + reply->mutable_channel()->set_rfq_cursor(cursor); + } + } + } + + void query(const MDRequest& request, MDReply* reply) const { + bts_assert(reply != nullptr); + for (auto& ch : request.channels()) { + auto groupiter = groups.find(UserSymbol(ch.symbol(), ch.exchange(), ch.type())); + if (groupiter != groups.end()) { + auto group = groupiter->second; + fill(ch, *group, reply->add_channels()); + } + } + } + +private: + Map groups; +}; + +/// In the future this might be replaced by a Service +// struct PubSubRpcServer : public pubsub::RPCHandler +// { +// const QueryServer& _db; +// utils::Dispatcher& _threadpool; +// PubSubRpcServer(const QueryServer& db, pubsub::PubsubHub& hub, utils::Dispatcher& threadpool) : +// pubsub::RPCHandler(hub, "/md/rpc"), +// _db(db), +// _threadpool(threadpool) +// { } +// +// // Inherited via RPCHandler +// virtual void handle(pubsub::msgs::Request const & request_id, MDRequest const & request) override { +// _threadpool.post([=]() { +// google::protobuf::Arena arena; +// MDReply* preply = google::protobuf::Arena::CreateMessage(&arena); +// TimeStamp start = TimeStamp::now(); +// _db.query(request, preply); +// auto elapsed = (TimeStamp::now() - start); +// LOG(Diag, MarketData) << "Request for: " << request.host() << " took " << (elapsed.microseconds()) << "us and returned " << preply->ByteSize() << " bytes before compression"; +// reply(request_id, *preply); +// }); +// } +// }; + +// void run_query_server() { +// pubsub::PubsubHub hub( +// config::get().host(), +// config::get().edge_md_pubsub_port()); +// std::set all_symbols; +// get_system_md_symbols(config::get().exchange(), config::get().type(), all_symbols); +// QueryServer db(all_symbols); +// utils::Dispatcher& dispatcher = utils::Dispatcher::current(); +// PubSubRpcServer server(db, hub, dispatcher); +// server.start(); +// +// // Scan for new securities every half second +// utils::DispatcherTimer security_timer(dispatcher, config::get().security_scan_interval(), [&db]() { +// db.scan_securities(); +// }); +// security_timer.start(); +// +// // Scan for dead producers every 30 seconds +// utils::DispatcherTimer health_timer(dispatcher, config::get().recovery_interval(), [&db]() { +// db.check_heartbeats(utils::Dispatcher::current()); +// }); +// health_timer.start(); +// +// utils::Dispatcher::current().run(std::min(4,os::num_cores())); +// } + + +// std::unique_ptr create_query_server(const std::set& symbols) { +// return utils::make_unique(symbols); +// } + +}}//namespace +#endif diff --git a/src/server/sparkd/md/relay/QueryServer.h b/src/server/sparkd/md/relay/QueryServer.h new file mode 100644 index 000000000..20356fd95 --- /dev/null +++ b/src/server/sparkd/md/relay/QueryServer.h @@ -0,0 +1,24 @@ +#pragma once + +#include "lib/utils/bts_no_copy.h" +#include "lib/utils/bts_export.h" +#include "../UserSymbol.h" +#include +#include + +#include "server/sparkd/md/protos/co2.pb.h" + +namespace bts { +namespace md { + +struct IQueryServer { + virtual ~IQueryServer() {} + virtual void scan_securities() const = 0; + virtual void query(const co2::MDRequest& request, co2::MDReply* reply) const = 0; +}; +std::unique_ptr create_query_server(const std::set& symbols); + +BTS_EXPORT void run_query_server(); + +}}//namespace + diff --git a/src/server/sparkd/md/relay/RelayClient.cpp b/src/server/sparkd/md/relay/RelayClient.cpp new file mode 100644 index 000000000..7eb2663fb --- /dev/null +++ b/src/server/sparkd/md/relay/RelayClient.cpp @@ -0,0 +1,417 @@ +#ifndef NO_CPP0X +#include "RelayClient.h" +#include "RelayRecorder.h" + +#include "lib/config/config.h" +#include "lib/utils/Dispatcher.h" +#include "lib/utils/TimeStamp.h" +#include "server/sparkd/md/MDProducer.h" +#include "server/sparkd/md/get_symbols.h" +#include "lib/logging/logging.h" + +#include "lib/config/protos/system.pb.h" +#include "lib/config/protos/MarketData.pb.h" + +#include "server/sparkd/md/protos/co2.pb.h" + +#include +#include +#include +#include +#include +#include +#include + +using namespace std; + +namespace bts { +namespace md { +using namespace md2; +using namespace co2; + +PriceSize convert(const PriceSizeMessage& psm) { + return psm.size() > 0 ? PriceSize(psm.mantissa(), psm.exponent(), psm.size()) : PriceSize::NO_MARKET; +} +PriceSize trade_pricesize(const co2::TradeMessage& trade) { + return trade.size() > 0 ? + PriceSize(trade.mantissa(), trade.exponent(), trade.size()) : PriceSize::NO_MARKET; +} +PriceSize book_at(const google::protobuf::RepeatedPtrField& side, int index) { + if (side.size() > index) { + return convert(side.Get(index)); + } + return PriceSize::NO_MARKET; +} + +class RelayProducer : public IRelayWriter { + bool _shared; + std::map _producers; + unordered_map _handles; + + void record_update(MDProducerHandle& handle, const MDReply::Update& update) { + TimeStamp updated = TimeStamp::now(); + bool snapshot = update.has_security(); + // If this is a snapshot an empty book() means "no market" rather than "no change" + if (update.has_book() || snapshot) { + // Write bbo first + handle.update_bbo(updated, book_at(update.book().bids(), 0), book_at(update.book().asks(), 0)); + // Then update depth remembering to always overwrite the whole book (for shinking markets) + for (size_t x = 1; x < COCONUT_MAX_DEPTH; ++x) { + handle.update_bid(updated, book_at(update.book().bids(), x), x); + handle.update_ask(updated, book_at(update.book().asks(), x), x); + } + } + if (update.has_customer_bbo()) { + handle.update_customer_bbo(updated, convert(update.customer_bbo().bid()), convert(update.customer_bbo().ask())); + } + if (update.has_misc()) { + auto& misc = update.misc(); + if (misc.has_settlement_price()) handle.update_settlement(updated, misc.settlement_price(), datetime::Date(misc.settlement_date())); + if (misc.has_previous_settlement_price()) handle.update_previous_settlement(updated, misc.previous_settlement_price(), datetime::Date(misc.previous_settlement_date())); + if (misc.has_open_price()) handle.update_opening(updated, misc.open_price(), datetime::Date(misc.open_date())); + if (misc.has_close_price()) handle.update_closing(updated, misc.close_price(), datetime::Date(misc.close_date())); + if (misc.has_session_high()) handle.update_session_high(updated, misc.session_high()); + if (misc.has_session_low()) handle.update_session_low(updated, misc.session_low()); + if (misc.has_open_interest()) handle.update_open_interest(updated, misc.open_interest()); + if (misc.has_session_high_bid()) handle.update_session_high_bid(updated, misc.session_high_bid()); + if (misc.has_session_low_offer()) handle.update_session_low_offer(updated, misc.session_low_offer()); + if (misc.has_fixing_price()) handle.update_fixing_price(updated, misc.fixing_price()); + } + if (update.has_last_trade()) { + handle.set_last_trade(updated, trade_pricesize(update.last_trade()), update.last_trade().time(), update.last_trade().total_volume()); + } + } + +public: + RelayProducer(bool shared) : + _shared(shared), + _producers(), + _handles() + {} + + MDProducer& ensure_producer(Platform::type platform) { + auto find_result = _producers.find(platform); + if (find_result == _producers.end()) { + auto result = _producers.insert(make_pair(platform, MDProducer(platform, _shared))); + if (result.second) { + return result.first->second; + } + } + return find_result->second; + } + + void write(const co2::MDReply& data, bool* phas_new_securities) override { + for (const auto& ch : data.channels()) { + // Per-security data + int added_security_count = 0; + for (const auto& update : ch.updates()) { + auto find_result = _handles.find(update.key()); + if (update.has_security() && find_result == _handles.end()) { + if (phas_new_securities) *phas_new_securities = true; + added_security_count++; + MDProducer& producer = ensure_producer(ch.platform()); + with_security(update.security().security_type(), update.security().spread().legs_size(), [&](FixedSecurity& fs) { + convert_security(update.security(), fs, true); + MDProducerHandle handle = producer.add_security(fs); + _handles.insert(make_pair(handle.key(), handle)); + record_update(handle, update); + }); + } + else if (find_result != _handles.end()) { + MDProducerHandle& handle = find_result->second; + // This hack is for CBOE csm because they change their keys between sessions + if (update.has_security() && ch.platform() == Platform::CSM) { + handle.update_numeric_id_hack_for_cboe(update.security().numeric_id()); + } + record_update(handle, update); + } + } + // Heartbeat + auto iter = _producers.find(ch.platform()); + if (iter != _producers.end()) { + auto& producer = iter->second; + auto channel = producer.get_channel(ch.channel().symbol().c_str(), ch.channel().type(), ch.channel().exchange()); + channel.heartbeat(); + + // RFQ and TRADE LOG + if (ch.trade_log_size() > 0 || ch.rfq_log_size() > 0) { + if (ch.trade_log_size() > 0) { + LOG(MarketData,Diag, "Added {} trades", ch.trade_log_size()); + } + for (const auto& trade : ch.trade_log()) { + channel.add_trade(trade); + } + for (const auto& rfq : ch.rfq_log()) { + channel.add_rfq(rfq); + } + } + } + if (added_security_count > 0) { + LOG(MarketData,Diag,"Added {} {} - {}" , + added_security_count , ch.channel().symbol() ,SecurityType::type_Name(ch.channel().type())); + } + } + } +}; + +shared_ptr mkrelayproducer(bool shared) { + return make_shared(shared); +} + +// Single Threaded +// class RelayClient { +// MDRequest _request; +// shared_ptr _reply; +// pubsub::RPC _rpc; +// IRelayWriter& _writer; +// utils::Dispatcher& _thread; +// utils::TimeStamp _request_log; +// +// public: +// RelayClient(pubsub::PubsubHub& hub, const std::set& symbols, utils::Dispatcher& d, IRelayWriter& writer) : +// _request(), +// _reply(make_shared()), +// _rpc(hub, "/md/rpc"), +// _writer(writer), +// _thread(d), +// _request_log(utils::TimeStamp::now()) +// { +// ostringstream fmt; +// fmt << config::get().host_name() << "(" << os::process_id() << ")"; +// _request.set_host(fmt.str()); +// for (const auto& s : symbols) { +// SecurityChannel ch; +// ch.set_symbol(s.symbol()); +// ch.set_type(s.type()); +// ch.set_exchange(s.exchange()); +// ch.set_flags(0xff); +// add_symbol(ch); +// } +// } +// +// // Can be called safely with the same symbol +// void add_symbol(const SecurityChannel& ch) { +// bts_assert(_thread.check_access()); +// auto find_result = +// std::find_if(_request.mutable_channels()->begin(), _request.mutable_channels()->end(), +// [&](const SecurityChannel& c) { +// return c.symbol() == ch.symbol() && +// c.type() == ch.type() && +// c.exchange() == ch.exchange(); }); +// if (find_result == _request.mutable_channels()->end()) { +// auto* new_channel = _request.add_channels(); +// new_channel->CopyFrom(ch); +// new_channel->set_time(0); +// } +// else { +// // update flags +// auto flags = find_result->flags() | ch.flags(); +// if (flags != find_result->flags()) { +// find_result->set_flags(flags); +// // After changing flags request a new snapshot +// find_result->set_time(0); +// } +// } +// } +// +// void request_md() { +// auto now = TimeStamp::now(); +// if ((now - _request_log).seconds() > 60) { +// _request_log = now; +// LOG(Info, MarketData) << "Requesting MD: " << _request.DebugString(); +// } +// _rpc.send(_request); +// } +// pubsub::RPCBase::ReplyType receive_md(uint32_t timeout, MDConsumer* pconsumer) { +// bts_assert(_thread.check_access()); +// bts_assert(_reply); +// auto result = _rpc.receive(timeout , _reply.get()); +// if (result == pubsub::RPCBase::REPLY_OK) { +// // Write data to shared mem +// bool has_new_security = false; +// _writer.write(*_reply, &has_new_security); +// // If the reply included and we have a consumer have it scan them +// if (has_new_security && pconsumer) { +// scan_securities_thread().post([pconsumer]() { +// auto count = pconsumer->scan_securities(true); +// LOG(Info, MarketData) << "Discovered " << count << " securities."; +// }); +// } +// // Update reply timetamps +// for (auto& req : *_request.mutable_channels()) { +// for (auto& reply : _reply->channels()) { +// if (req.type() == reply.channel().type() && +// req.exchange() == reply.channel().exchange() && +// req.symbol() == reply.channel().symbol()) { +// +// req.set_time(reply.channel().time()); +// req.set_trade_cursor(reply.channel().trade_cursor()); +// req.set_rfq_cursor(reply.channel().rfq_cursor()); +// if (req.segment_id() != reply.channel().segment_id()) { +// LOG(Diag, MarketData) << "New segment_id = " << reply.channel().segment_id(); +// req.set_segment_id(reply.channel().segment_id()); +// } +// } +// } +// } +// +// if (has_new_security) { +// // release memory and start fresh +// _reply = make_shared(); +// } +// } +// return result; +// } +// }; + +// struct RelayScheduler { +// utils::Dispatcher& _dispatcher; +// MDConsumer* _pconsumer; +// uint32_t _interval; +// RelayClient& _client; +// pubsub::RPCBase::ReplyType _last_result; +// bool running; +// double speed_factor; +// +// RelayScheduler(utils::Dispatcher& dispatcher, MDConsumer* pconsumer, uint32_t interval, RelayClient& client) : +// _dispatcher(dispatcher), +// _pconsumer(pconsumer), +// _interval(interval), +// _client(client), +// _last_result(pubsub::RPCBase::REPLY_OK), +// running(true), +// speed_factor(config::get().speed()) +// { } +// +// void run_once() { +// if (_last_result != pubsub::RPCBase::REPLY_TIMEOUT) { +// _client.request_md(); +// } +// _last_result = _client.receive_md(_interval, _pconsumer); +// } +// void run() { +// if (running) { +// int64_t next_update = _interval; +// TimeStamp start = TimeStamp::now(); +// run_once(); +// int64_t elapsed_ms = (TimeStamp::now() - start).milliseconds(); +// int64_t delay = (next_update - elapsed_ms) / speed_factor; +// _dispatcher.post([=]() { run(); }, std::max(delay, (int64_t)0)); +// } +// } +// }; +// +// +// void relay_everything() { +// pubsub::PubsubHub hub( +// config::get().host(), +// config::get().edge_md_pubsub_port()); +// std::set system_symbols; +// get_system_md_symbols(system_symbols); +// utils::Dispatcher& thread = utils::Dispatcher::current(); +// RelayProducer producer(true); +// RelayClient client(hub, system_symbols, thread, producer); +// RelayScheduler scheduler( +// thread, nullptr, +// config::get().naggle_ms(), +// client); +// thread.post([&]() { scheduler.run(); }); +// thread.run(); +// } +// +// +// void record_everything(std::string path, std::string filename) { +// auto options = config::get(); +// if (filename.empty()) { +// ostringstream file; +// file << "marketdata" << "-"; +// int year, month, day, hour, minute, unused; +// datetime::ll_to_parts(datetime::ll_local_now(), year, month, day, hour, minute, unused, unused); +// file +// << year +// << setw(2) << setfill('0') << month +// << setw(2) << setfill('0') << day +// << "-" +// << setw(2) << setfill('0') << hour +// << setw(2) << setfill('0') << minute +// << ".coconut"; +// filename = file.str(); +// } +// +// if (path.empty()) { +// path = options.output_path(); +// } +// +// auto naggle_ms = config::get().naggle_ms(); +// RelayRecorder recorder(path, filename, naggle_ms); +// +// pubsub::PubsubHub hub( +// config::get().host(), +// config::get().edge_md_pubsub_port()); +// std::set system_symbols; +// get_system_md_symbols(system_symbols); +// utils::Dispatcher& thread = utils::Dispatcher::current(); +// +// RelayClient client(hub, system_symbols, thread, recorder); +// RelayScheduler scheduler( +// thread, nullptr, naggle_ms, client); +// thread.post([&]() { scheduler.run(); }); +// +// utils::DispatcherTimer flush_timer(thread, 30 * 1000, [&]() { +// recorder.flush(); +// }); +// flush_timer.start(); +// thread.run(); +// } + +/* + * extern api for c# +*/ + +// struct RelayControl { +// RelayControl(utils::Dispatcher& d, MDConsumer* pconsumer, uint32_t interval) : +// dispatcher(d), +// hub(config::get().host(), +// config::get().edge_md_pubsub_port()), +// producer(false), +// client(hub, std::set(), d, producer), +// scheduler(d, pconsumer, interval, client) +// { +// dispatcher.post([=]() { scheduler.run(); }); +// } +// utils::Dispatcher& dispatcher; +// pubsub::PubsubHub hub; +// RelayProducer producer; +// RelayClient client; +// RelayScheduler scheduler; +// }; + +// RelayControl* start_private_relay(MDConsumer* pconsumer, uint32_t interval) { +// bts_assert(pconsumer); +// return new RelayControl(utils::Dispatcher::get_named("mdrelay"), pconsumer, interval); +// } +// void stop_relay(RelayControl* relay) { +// if (relay) { +// relay->scheduler.running = false; +// bool terminated = false; +// relay->dispatcher.post([&]() { +// terminated = true; +// }, 500); +// while (!terminated) utils::wait_ms(100); +// relay->dispatcher.terminate(); +// } +// } +// void add_relay_symbol(RelayControl* relay, const SecurityChannel& chan) { +// bts_assert(relay); +// relay->dispatcher.post([=]() { +// relay->client.add_symbol(chan); +// }); +// } + +utils::Dispatcher& scan_securities_thread() { + return utils::Dispatcher::background(); +} + + +} }//namespace +#endif diff --git a/src/server/sparkd/md/relay/RelayClient.h b/src/server/sparkd/md/relay/RelayClient.h new file mode 100644 index 000000000..c3312753d --- /dev/null +++ b/src/server/sparkd/md/relay/RelayClient.h @@ -0,0 +1,37 @@ +#pragma once + +#include "lib/utils/bts_no_copy.h" +#include "lib/utils/bts_export.h" +#include "server/sparkd/md/UserSymbol.h" +#include "../MDConsumer.h" +#include "lib/utils/Dispatcher.h" +#include +#include + +#include "server/sparkd/md/protos/co2.pb.h" + +namespace bts { +namespace pubsub { + class PubsubHub; +} +namespace md { + +struct RelayControl; +void add_relay_symbol(RelayControl* relay, const co2::SecurityChannel& chan); + +struct IRelayWriter { + virtual ~IRelayWriter() {}; + virtual void write(const co2::MDReply& data, bool* has_new_securities) = 0; +}; + +utils::Dispatcher& scan_securities_thread(); + +extern "C" { +BTS_EXPORT void relay_everything(); + +BTS_EXPORT RelayControl* start_private_relay(md2::MDConsumer* consumer, uint32_t interval); +BTS_EXPORT void stop_relay(RelayControl* relay); +} + +}}//namespace + diff --git a/src/server/sparkd/md/relay/RelayRecorder.cpp b/src/server/sparkd/md/relay/RelayRecorder.cpp new file mode 100644 index 000000000..bab53fe27 --- /dev/null +++ b/src/server/sparkd/md/relay/RelayRecorder.cpp @@ -0,0 +1,236 @@ +#ifndef NO_CPP0X + +#include "RelayRecorder.h" +#include "lib/config/config.h" +#include "lib/utils/HTTPStream.h" +#include "lib/utils/datetime.h" +#include "lib/logging/logging.h" +#include "lib/utils/bts_chrono.h" + +#include "server/sparkd/md/protos/coconut.pb.h" +#include "server/sparkd/md/protos/co2.pb.h" +#include "lib/config/protos/system.pb.h" + +#include +#include +#include +#include +#include + +using namespace std; +namespace bts { +namespace md { + +const char* magic = "Co2"; + +template +void writebuf(ofstream& out, const T& pb) { + uint32_t size = pb.ByteSize(); + out.write(reinterpret_cast(&size), sizeof(size)); + bts_assert(pb.SerializeToOstream(&out)); +} + +template +bool readbuf(istream& in, T* result) { + if (result) { + uint32_t size; + in.read(reinterpret_cast(&size), sizeof(size)); + vector buffer(size); + in.read(&buffer[0], size); + if (!in.fail()) { + result->ParsePartialFromArray(&buffer[0], size); + return true; + } + } + return false; +} + +RelayRecorder::RelayRecorder(const std::string& base_path, const std::string& filename, uint32_t naggle): + _filename(base_path+"/"+filename), + _file() +{ + LOG(MarketData,Info,"Recording market data to: {}" , _filename); + boost::system::error_code ec; + boost::filesystem::create_directories(base_path, ec); + _file.open(_filename, ios::out | ios::binary | ios::trunc); + bts_assert(_file.is_open()); + + CoconutReplayHeader header; + header.set_magic(magic); + header.set_version(2); + header.set_timestamp(datetime::ll_now()); + header.set_host(config::get().config_host()); + header.set_naggle_ms(naggle); + writebuf(_file, header); +} + +RelayRecorder::~RelayRecorder() { } + +void RelayRecorder::write(const co2::MDReply& data, bool*) { + writebuf(_file, data); + flush(); +} + +void RelayRecorder::flush() { + _file.flush(); +} + +struct ConsoleReplayWriter : IRelayWriter +{ + ConsoleReplayWriter() : count(0) { } + int count; + virtual void write(const co2::MDReply& data, bool*) override + { + ++count; + cout << count << " MDReply {" << endl; + data.PrintDebugString(); + cout << "}" << endl; + } +}; + +struct ConsoleTraceWriter : IRelayWriter +{ + int count; + security_key key; + ConsoleTraceWriter(security_key k): count(0), key(k) { } + virtual void write(const co2::MDReply & data, bool*) override { + ++count; + for (const auto& chan : data.channels()) { + for (const auto& update : chan.updates()) { + if (update.key() == key) { + cout << count << " Update {" << endl; + update.PrintDebugString(); + cout << "}" << endl; + } + } + } + } +}; + +void replay_recording(istream& in, IRelayWriter& writer, int32_t naggle_ms) { + bts_assert(!in.bad()); + CoconutReplayHeader header; + size_t count = 0; + if (readbuf(in, &header) && header.magic() == magic && header.version() <= 2) { + co2::MDReply message; + while (readbuf(in, &message)) { + writer.write(message, nullptr); + message.Clear(); + if (naggle_ms >0) std::this_thread::sleep_for(std::chrono::duration(naggle_ms)); + else if (naggle_ms == -1/*interactive*/) { + ++count; + cout << count << endl; + cin.get(); + } + } + } + else { + cout << "Invalid file format." << endl; + } +} + +void replay_file(const std::string& filename, IRelayWriter& writer, int32_t naggle_ms) { + ifstream in; + in.open(filename, ios::binary | ios::in); + if (in.is_open()) { + replay_recording(in, writer, naggle_ms); + } + else { + cout << "Cannot read file: " << filename; + } +} + +void replay_url(const std::string& url, IRelayWriter& writer, int32_t naggle_ms) { + // split up url parts + size_t http_len = BTS_ARRAYSIZE("http://") - 1; + auto after_host = url.find('/', http_len); + auto host = url.substr(http_len, after_host-http_len); + auto path = url.substr(after_host); + auto last_slash = url.rfind('/'); + auto filename = url.substr(last_slash+1); + + utils::download_file_if_newer(host, path, filename); + replay_file(filename, writer, naggle_ms); +} + +shared_ptr mkrelayproducer(bool shared); +/*this is the public one*/ +void replay_recording(const string& filename, int32_t naggle_ms) { + auto writer = mkrelayproducer(true); + if (config::get().interactive()) { + naggle_ms = -1; + } + + if (filename.find("http://") == 0) { + replay_url(filename, *writer, naggle_ms); + } + else { + replay_file(filename, *writer, naggle_ms); + } +} +void print_recording(const string& filename) { + ConsoleReplayWriter writer; + replay_file(filename, writer, 0); +} + +void trace_recording(const string& filename, security_key key) { + ConsoleTraceWriter writer(key); + replay_file(filename, writer, 0); +} + +void unprint_recording(const std::string& src, const std::string& dest) { + // quick and dirty minimal error checking + ifstream in; + in.open(src, ios::binary | ios::in); + ofstream out; + out.open(dest, ios::binary | ios::out); + + // error handling + if (!(in.good() && out.good())) { + cout << "Error opening files." << endl; + return; + } + + // Write out the header + CoconutReplayHeader header; + header.set_magic(magic); + header.set_version(2); + header.set_timestamp(datetime::ll_now()); + header.set_host(config::get().host_name()); + writebuf(out, header); + + int index = 0; + string message_buffer; + co2::MDReply reply_buffer; + while (in) { + // '1 MDReply {' + in >> index; + if (in.eof()) break; + cout << index << " "; + char buffer[128]; + in.getline(buffer, BTS_ARRAYSIZE(buffer), '{'); + bts_assert(string(buffer) == " MDReply "); + // getline consumes the { + message_buffer.clear(); + char bcount = 1; + while (bcount > 0 && !in.eof()) { + char ch; + in.get(ch); + if (ch == '{') ++bcount; + if (ch == '}') --bcount; + if (bcount > 0) message_buffer += ch; + } + cout << "buffered "; + reply_buffer.Clear(); + if (google::protobuf::TextFormat::ParseFromString(message_buffer, &reply_buffer)) { + cout << "parsed "; + writebuf(out, reply_buffer); + cout << "written"; + } + cout << endl; + } + out.flush(); +} + +}}//namespace +#endif diff --git a/src/server/sparkd/md/relay/RelayRecorder.h b/src/server/sparkd/md/relay/RelayRecorder.h new file mode 100644 index 000000000..c107844f1 --- /dev/null +++ b/src/server/sparkd/md/relay/RelayRecorder.h @@ -0,0 +1,34 @@ +#pragma once + +#include "lib/utils/bts_no_copy.h" +#include "RelayClient.h" +#include +#include +#include + +namespace bts { +namespace md { + +class RelayRecorder : public IRelayWriter { +public: + RelayRecorder(const std::string& base_path, const std::string& filename, uint32_t naggle); + ~RelayRecorder(); + virtual void write(const co2::MDReply&, bool*) override; + + //void flush_and_copy(); + void flush(); + +private: + std::string _filename; + std::ofstream _file; + BTS_NO_COPY(RelayRecorder); +}; + +BTS_EXPORT void record_everything(std::string path, std::string filename); +BTS_EXPORT void print_recording(const std::string&); +BTS_EXPORT void unprint_recording(const std::string& src, const std::string& dest); +BTS_EXPORT void trace_recording(const std::string& file, security_key key); +BTS_EXPORT void replay_recording(const std::string&, int32_t delay_ms); + +}}//namespace + diff --git a/src/server/sparkd/md/test/Co2TestInstrument.cpp b/src/server/sparkd/md/test/Co2TestInstrument.cpp new file mode 100644 index 000000000..3dadddec2 --- /dev/null +++ b/src/server/sparkd/md/test/Co2TestInstrument.cpp @@ -0,0 +1,60 @@ +#ifndef NO_CPP0X +#include "Co2TestInstrument.h" +#include "lib/security_model/SecurityHelper.h" +#include "server/sparkd/md/MemorySegmentUtils.h" + +using namespace bts::md2; +namespace bts { +namespace md2 { + +Co2TestInstrument::Co2TestInstrument(const std::string& symbol, SecurityType::type type): + _cold(nullptr), + _hot_buffer(), + _hot(new (&_hot_buffer[0]) HotDepthEntry()), + _cheader(std::make_shared()), + _hheader(std::make_shared()) +{ + _cheader->has_depth = enable_depth(symbol, type); + Security sec; + sec.set_security_type(type); + md::SecurityHelper::set_symbol(sec, symbol); + with_security(type, 4, [&](FixedSecurity& fsec) { + void* temp = new uint8_t[ColdEntry::fixed_size() + fsec.size]; + _cold = new (temp) ColdEntry(fsec.size); + memcpy(&_cold->security, &fsec, fsec.size); + }); +} + +Co2TestInstrument::Co2TestInstrument(const Security & security,std::shared_ptr cheader, std::shared_ptr hheader): + _cold(nullptr), + _hot_buffer(), + _hot(new (&_hot_buffer[0]) HotDepthEntry()), + _cheader(cheader), + _hheader(hheader) +{ + bts_assert_d(_cheader->has_depth == false); + //_cheader->has_depth = enable_depth(md::SecurityHelper::symbol(security), security.security_type()); + with_security(security.security_type(), security.spread().legs_size(), [&](FixedSecurity& fsec) { + convert_security(security, fsec, false); + void* temp = new uint8_t[ColdEntry::fixed_size() + fsec.size]; + _cold = new (temp) ColdEntry(fsec.size); + memcpy(&_cold->security, &fsec, fsec.size); + }); +} + +MDProducerHandle Co2TestInstrument::producer() { + return MDProducerHandle(*_cheader, *_hheader, *_cold, _hot, MDProducerChannel()); +} + +MDConsumerHandle Co2TestInstrument::consumer() { + MDConsumerHandle handle; + handle.init(*_cheader, *_cold, _hot); + return handle; +} + +Co2TestInstrument::~Co2TestInstrument() { + if (_cold != nullptr) { delete _cold; } +} + +}}//namespace +#endif diff --git a/src/server/sparkd/md/test/Co2TestInstrument.h b/src/server/sparkd/md/test/Co2TestInstrument.h new file mode 100644 index 000000000..1081ebc93 --- /dev/null +++ b/src/server/sparkd/md/test/Co2TestInstrument.h @@ -0,0 +1,29 @@ +#pragma once + +#include "lib/utils/bts_no_copy.h" +#include "lib/security_model/SecurityHelper.h" +#include "server/sparkd/md/coconut2.h" +#include + +namespace bts { +namespace md2 { + +struct Co2TestInstrument +{ + Co2TestInstrument(const std::string& symbol, SecurityType::type type); + Co2TestInstrument(const Security& security, std::shared_ptr, std::shared_ptr); + ~Co2TestInstrument(); + + MDProducerHandle producer(); + MDConsumerHandle consumer(); +private: + ColdEntry* _cold; + std::array _hot_buffer; + HotDepthEntry *_hot; + std::shared_ptr _cheader; + std::shared_ptr _hheader; + BTS_NO_COPY(Co2TestInstrument); +}; + +}}//namespace + diff --git a/src/server/sparkd/md/test/md2_tests.cpp b/src/server/sparkd/md/test/md2_tests.cpp new file mode 100644 index 000000000..6cfdb404d --- /dev/null +++ b/src/server/sparkd/md/test/md2_tests.cpp @@ -0,0 +1,1533 @@ +#ifndef NO_CPP0X + +#include "catch/catch.h" +#include "../SettlementRecorder.h" +#include "../coconut2.h" +#include "../relay/QueryServer.h" +#include "Co2TestInstrument.h" +#include "lib/config/config.h" +#include "lib/os/os.h" +#include "lib/security_model/SecurityHelper.h" +#include "lib/utils/Dispatcher.h" +#include "lib/utils/defer.h" +#include "lib/utils/warn_ctrl.h" +#include "lib/utils/bts_chrono.h" + +#include "lib/config/protos/MarketData.pb.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +using namespace bts::md2; +using namespace bts::md; +using namespace std; + +namespace bts { +namespace md2 { +namespace test { + +// +// Helper functions for Producer/Consumer tests +// +vector create_test_securities(MDProducer& producer, const char* postfix ="") { + FixedOptionSecurity fos; + FixedFutureSecurity ffs; + ffs.symbol("FOO"+string(postfix)); + fos.symbol("FOOX"+string(postfix)); + ffs.key = invalid_security_key; + fos.key = invalid_security_key; + ffs.expiration_year = fos.expiration_year = 2015; + ffs.expiration_day = fos.expiration_day = 27; + fos.strike = 1000; + fos.call_put = PutOrCall::CALL; + // Add 12 expirations to 2 different shared memory regions + vector result; + for (uint32_t x = 1; x <= 12; ++x) { + ffs.expiration_month = fos.expiration_month = x; + result.push_back(producer.add_security(ffs)); + result.push_back(producer.add_security(fos)); + // In a real app these securities would now be passed to the SecurityIDRecorder + } + return result; +} + +set get_test_symbols(const char* postfix="") { + set symbols; + UserSymbol future_symbol(string("FOO")+postfix, Exchange::TESTEXCHANGE, SecurityType::FUTURE); + symbols.insert(future_symbol); + UserSymbol option_symbol(string("FOOX")+postfix, Exchange::TESTEXCHANGE, SecurityType::OPTION); + symbols.insert(option_symbol); + return symbols; +} + +Security mkfuture(string symbol, security_key key, int year, int month, int day) { + Security result; + result.set_security_type(SecurityType::FUTURE); + SecurityHelper::set_symbol(result, symbol); + result.set_exchange(Exchange::TESTEXCHANGE); + result.set_security_key(key); + result.mutable_future()->set_expiration_day(day); + result.mutable_future()->set_expiration_month(month); + result.mutable_future()->set_expiration_year(year); + return result; +} +Security mkoption(string symbol, security_key key, int year, int month, int day, double strike) { + Security result; + result.set_security_type(SecurityType::OPTION); + SecurityHelper::set_symbol(result, symbol); + result.set_exchange(Exchange::TESTEXCHANGE); + result.set_security_key(key); + result.mutable_option()->set_expiration_day(day); + result.mutable_option()->set_expiration_month(month); + result.mutable_option()->set_expiration_year(year); + result.mutable_option()->set_strike(strike); + result.mutable_option()->set_call_put(PutOrCall::PUT); + return result; +} +FixedOptionSecurity mkfixedoption(string symbol, security_key key, int year, int month, int day, double strike, PutOrCall::type callput) { + FixedOptionSecurity fos; + fos.type = SecurityType::OPTION; + fos.exchange = Exchange::TESTEXCHANGE; + fos.key = key; + fos.strike = strike; + fos.expiration_day = day; + fos.expiration_month = month; + fos.expiration_year = year; + fos.call_put = callput; + fos.symbol(symbol); + return fos; +} +Security mkspread(string symbol, security_key key, const vector& legs) { + Security result; + result.set_security_type(SecurityType::SPREAD); + SecurityHelper::set_symbol(result, symbol); + result.set_exchange(Exchange::TESTEXCHANGE); + result.set_security_key(key); + for (const auto& leg : legs) { + auto* pleg = result.mutable_spread()->add_legs(); + pleg->mutable_security()->CopyFrom(leg); + } + return result; +} +Security mkspread(string symbol, security_key key, Security leg1, Security leg2) { + Security legs[] = { leg1, leg2 }; + vector vlegs(begin(legs), end(legs)); + return mkspread(symbol, key, vlegs); +} +Security mkspread(string symbol, security_key key, Security leg1, Security leg2, Security leg3) { + Security legs[] = { leg1, leg2, leg3 }; + vector vlegs(begin(legs), end(legs)); + return mkspread(symbol, key, vlegs); +} + +void assign_ids(vector::iterator begin, vector::iterator end, security_key start_key = 1000) { + for (; begin != end; ++begin) { + (*begin).set_key(++start_key); + } +} + +class Co2Tests +{ +public: + void setup() { + config::internal::initialize_for_test( + "PlatformMap {" + " value {" + " sim {" + " exchange: TESTEXCHANGE" + " }" + " }" + "}" + "PublisherOptions {" + " value {" + " heartbeat_ms: 100" + " heartbeat_boot_seconds: 1" + " }" + "}" + ); + } + + + void test_security_id_recorder() { + // Verify callbacks happen after work complete + SecurityIDRecorder idsvc, idsvc2; + security_key result = invalid_security_key; + std::string desc = "FOOX 1"; + idsvc.get_key(desc, SecurityType::FUTURE, [](void* arg, security_key key) { + security_key* presult = (security_key*)arg; + *presult = key; + }, &result); + idsvc.do_work(); + REQUIRE(result != invalid_security_key); + security_key result2 = invalid_security_key; + idsvc2.get_key(desc, SecurityType::OPTION, [](void* arg, security_key key) { + *((security_key*)arg) = key; + }, &result2); + idsvc2.do_work(); + REQUIRE(result == result2); + } + + void test_price_size() { + // PriceSize violates some type punning rules + // So it is important to have this test work correctly on the target platform + PriceSize p, p2; + p = PriceSize::from_double<2>(4.5, 14); + SingleWriterPriceSizeStore memory; + memory.store(p); + p2 = memory.load(); + REQUIRE(p.price() == 4.5); + REQUIRE(p.size() == 14); + REQUIRE(p2.price() == 4.5); + REQUIRE(p2.size() == 14); + + p = PriceSize::from_double<2>(4.0, 5); + memory.store(p); + p2 = memory.load(); + REQUIRE(p2.price() == 4.0); + REQUIRE(p2.size() == 5); + + p = PriceSize::from_double<2>(-4.2, 6); + memory.store(p); + p2 = memory.load(); + REQUIRE(p2.price() == -4.2); + REQUIRE(p2.size() == 6); + + p = PriceSize(4, -1, 2); + memory.store(p); + p2 = memory.load(); + REQUIRE(p2.price() == 0.4); + REQUIRE(p2.size() == 2); + + // Test requirement for more than 32 bit mantissa + double bigval = -9999.9921875; + p = PriceSize::from_double<7>(bigval, 42); + memory.store(p); + p2 = memory.load(); + REQUIRE(p2.price() == bigval); + REQUIRE(p2.size() == 42); + } + + void test_slow_double_conversion() { + double values[] = { 1.5, 0.5, 0.2155, 0.216, 42.1 }; + int expected[] = { -1, -1, -4, -3, -1 }; + static_assert(BTS_ARRAYSIZE(values) == BTS_ARRAYSIZE(expected), ""); + for (size_t x = 0; x < BTS_ARRAYSIZE(values); ++x) { + REQUIRE(to_exponent(values[x]) == expected[x]); + } + + // Tests powers of 10. Important for display_factor conversion + for (int x = 8; x > -8; --x) { + double d = pow10(x); + REQUIRE(to_exponent(d) == x); + } + } + + struct ExchangeDecimal { + int64_t mantissa; + int32_t exponent; + }; + bool close_enough(double a, double b) { + // We need at least 7 places but we're testing with 10 just in case + const double acceptable_error = 1E-12; + double delta = abs(a - b); + // Equal if the delta is less than error + return delta < acceptable_error; + } + void test_compact_exponent() { + ExchangeDecimal test_values[] = { + { 2030000000000, -9 }, // mantissa > pow(2,40) + { -2030000000000, -9 }, // mantissa > pow(2,40) + { 2022750000000, -9 },// mantissa > pow(2,40) + { -2022750000000, -9 },// mantissa > pow(2,40) + { 20227500000000, -10 },// mantissa > pow(2,40) + { 202275000000000, -11 },// mantissa > pow(2,40) + { 2022750000000000, -12 },// mantissa > pow(2,40) + { 20227500000000000, -13 },// mantissa > pow(2,40) + { -20227500000000000, -13 },// mantissa > pow(2,40) + { 203, 1}, // control + { -14000000000, -8 }, // control + { -14, 1 }, // control + { 20227500, -4 }, // control + }; + for (auto p : test_values) { + double expected = p.mantissa * pow10(p.exponent); + compact_mantissa(p.mantissa, p.exponent); + double compacted = p.mantissa * pow10(p.exponent); + REQUIRE(close_enough(expected, compacted)); + } + } + + void test_producer_consumer() { + MDProducer producer(Platform::SIM); + vector producers = create_test_securities(producer); + + // Add bbo data for those expirations + auto write_time = TimeStamp::now(); + for (auto& p : producers) { + p.update_bid(write_time, PriceSize::from_double<2>(5.0, 5)); + p.update_ask(write_time, PriceSize::from_double<2>(5.0, 5)); + } + + // Symbol list for the consumer + set symbols; + UserSymbol future_symbol( "FOO", Exchange::TESTEXCHANGE, SecurityType::FUTURE ); + symbols.insert(future_symbol); + UserSymbol option_symbol( "FOOX", Exchange::TESTEXCHANGE, SecurityType::OPTION ); + symbols.insert(option_symbol); + MDConsumer consumer(symbols); + + vector consumer_securities; + // Poll for security list - none visible until they have keys + size_t count = consumer.walk_security_list([&](const Security& security, MDConsumerHandle) {consumer_securities.push_back(security);}); + // Since no security has an id none are "public" + REQUIRE(consumer_securities.size() == count); + REQUIRE(count == 0); + // Assign a unique ID to each security (from the producer) + assign_ids(producers.begin(), producers.end()); + // Now securities can be enumerated by the client + count = consumer.walk_security_list([&](const Security& security, MDConsumerHandle) {consumer_securities.push_back(security);}); + REQUIRE(consumer_securities.size() == count); + REQUIRE(count == 24); + + vector consumers; + for(const auto& security : consumer_securities) { + consumers.push_back(consumer.get_handle(security.security_key())); + } + + for (const auto& c : consumers) { + PriceSize bid, ask; + TimeStamp read_time; + // get_boo returns true becuase this time is after the change + REQUIRE(c.get_bbo(read_time, bid, ask)); + REQUIRE((read_time - TimeStamp()).seconds() > 0); + REQUIRE(bid.price() == 5.0); + REQUIRE(ask.price() == 5.0); + REQUIRE(bid.size() == 5); + REQUIRE(ask.size() == 5); + } + + // Second query before a write has happend finds no updates + auto read_time = TimeStamp::now(); + for (const auto& c : consumers) { + PriceSize bid, ask; + REQUIRE(!c.get_bbo(read_time, bid, ask)); + } + + // update everything once more + write_time = TimeStamp::now(); + for (auto& p : producers) { + // Update bid/ask with one call + p.update_bbo(write_time, PriceSize::from_double<2>(6.55, 6), PriceSize::from_double<2>(6.55, 6)); + } + // Now read_time is < write_time + for (const auto& c : consumers) { + TimeStamp modified_time = read_time; + PriceSize bid, ask; + // get_bbo returns true becuase this time is after the change + REQUIRE(c.get_bbo(modified_time, bid, ask)); + REQUIRE( (modified_time - read_time).seconds() > 0); + REQUIRE(bid.price() == 6.55); + REQUIRE(ask.price() == 6.55); + REQUIRE(bid.size() == 6); + REQUIRE(ask.size() == 6); + } + } + + // When a security is added the consumer should provide callback + void test_consumer_callback_on_security_added() { + MDProducer producer(Platform::SIM); + vector producers = create_test_securities(producer); + + // Symbol list for the consumer + set symbols; + UserSymbol future_symbol( "FOO", Exchange::TESTEXCHANGE, SecurityType::FUTURE ); + symbols.insert(future_symbol); + UserSymbol option_symbol( "FOOX", Exchange::TESTEXCHANGE, SecurityType::OPTION ); + symbols.insert(option_symbol); + MDConsumer consumer(symbols); + + // Add the on_security callback + vector securities; + consumer.on_security([&](const Security& security, MDConsumerHandle handle) { + REQUIRE(security.security_key() != invalid_security_key); + REQUIRE(security.security_key() == handle.key()); + securities.push_back(security); + }); + // No securities have ids yet + REQUIRE(securities.size() == 0); + // Check for new securities + consumer.scan_securities(); + // Still none + REQUIRE(securities.size() == 0); + producers.front().set_key(1001); + consumer.scan_securities(); + REQUIRE(securities.size() == 1); + + // Assign the rest of the ids + assign_ids(producers.begin() + 1, producers.end(), producers.front().security().key); + consumer.scan_securities(); + REQUIRE(securities.size() == producers.size()); + } + + // If an MDConsumer is asked for a handle to a security which does not exist in the system + // The consumer should return a handle which always returns false on all calls but wakes up + // when the security is added. + // * Case 1 - the shared memory has been created but the security has not been discovered + // * Case 2 - The producer has never been created + void test_consumer_defer_handle1() { + MDProducer producer(Platform::SIM); + vector producers = create_test_securities(producer); + + set symbols; + UserSymbol future_symbol( "FOO", Exchange::TESTEXCHANGE, SecurityType::FUTURE ); + symbols.insert(future_symbol); + UserSymbol option_symbol( "FOOX", Exchange::TESTEXCHANGE, SecurityType::OPTION ); + symbols.insert(option_symbol); + MDConsumer consumer(symbols); + + // Get a handle to 24 consumers which haven't been created yet + vector consumers; + vector> managed_consumers; + security_key key = 1; + for(; key < 24; ++key) { + consumers.push_back(consumer.get_handle(key+1000)); + managed_consumers.push_back(consumer.get_managed_handle(key+1000)); + } + + // Set bbo on the producers - still not public + TimeStamp write_time = TimeStamp::now(); + for (auto& p : producers) { + p.update_bbo(write_time, PriceSize::from_double<3>(4.5, 5), PriceSize::from_double<3>(6.75, 7)); + } + // Even though bbo is available it is not given out because the producing security does not have an id + PriceSize bid, ask; + for(auto& c : consumers) { + TimeStamp zerotime; + REQUIRE(!c.get_bbo(zerotime, bid, ask)); + } + for(auto& mc : managed_consumers) { + TimeStamp zerotime; + REQUIRE(!mc->get_bbo(zerotime, bid, ask)); + } + + assign_ids(producers.begin(), producers.end()); + consumer.scan_securities(); + size_t bbo_count = 0; + // Existing old handles remain invalid + for(auto& c : consumers) { + TimeStamp zerotime; + if (c.get_bbo(zerotime, bid, ask)) { + bbo_count++; + } + } + REQUIRE(bbo_count == 0); + // Now, magically, the existing (managed) handles are valid + for(auto& mc : managed_consumers) { + TimeStamp zerotime; + if (mc->get_bbo(zerotime, bid, ask)) { + bbo_count++; + } + } + REQUIRE(bbo_count > 0); + } + + + // It is legal and valid to create a consumer before the producer process has started + // Handles given out by that consumer will become valid once the securities are discovered + void test_consumer_defer_handle2() { + set symbols; + UserSymbol future_symbol( "FOO", Exchange::TESTEXCHANGE, SecurityType::FUTURE ); + symbols.insert(future_symbol); + UserSymbol option_symbol( "FOOX", Exchange::TESTEXCHANGE, SecurityType::OPTION ); + symbols.insert(option_symbol); + MDConsumer consumer(symbols); + + // Get a handle to 24 consumers which haven't been created yet + vector consumers; + vector> managed_consumers; + security_key key = 1; + for(; key < 24; ++key) { + consumers.push_back(consumer.get_handle(key+1000)); + managed_consumers.push_back(consumer.get_managed_handle(key+1000)); + } + + // Even though bbo is available it is not given out because the producing security does not have an id + PriceSize bid, ask; + for(auto& c : consumers) { + TimeStamp zerotime; + REQUIRE(!c.get_bbo(zerotime, bid, ask)); + } + for(auto& mc : managed_consumers) { + TimeStamp zerotime; + REQUIRE(!mc->get_bbo(zerotime, bid, ask)); + } + + // Only NOW create the producers (and the shared memory) + MDProducer producer(Platform::SIM); + vector producers = create_test_securities(producer); + // Set bbo on the producers - still not public + TimeStamp write_time = TimeStamp::now(); + for (auto& p : producers) { + p.update_bbo(write_time, PriceSize::from_double<3>( 4.5, 5 ), PriceSize::from_double<3>(6.75, 7)); + } + + assign_ids(producers.begin(), producers.end()); + consumer.scan_securities(); + size_t bbo_count = 0; + // Existing old handles remain invalid + for(auto& c : consumers) { + TimeStamp zerotime; + if (c.get_bbo(zerotime, bid, ask)) { + bbo_count++; + } + } + REQUIRE(bbo_count == 0); + // Now, magically, the existing (managed) handles are valid + for(auto& mc : managed_consumers) { + TimeStamp zerotime; + if (mc->get_bbo(zerotime, bid, ask)) { + bbo_count++; + } + } + REQUIRE(bbo_count > 0); + } + + void test_spread_resolver() { + SpreadResolver resolver; + vector legs; + legs.push_back("1"); + legs.push_back("2"); + legs.push_back("3"); + legs.push_back("4"); + resolver.add("1234", legs); + legs.clear(); + legs.push_back("4"); + legs.push_back("5"); + resolver.add("45", legs); + auto key_map = [](string& key) -> security_key { + if (key == "1") return 1; + if (key == "2") return 2; + if (key == "3") return 3; + if (key == "4") return 4; + if (key == "5") return invalid_security_key; + REQUIRE(false); + return 0; + }; + auto key_map2 = [](string& key) -> security_key { + if (key == "4") return 4; + if (key == "5") return 5; + REQUIRE(false); + return 0; + }; + + // 2 unresolved spreads + REQUIRE(resolver.count() == 2); + resolver.resolve(key_map, [&](const string& key, const vector& legs) { + REQUIRE(key == "1234"); + REQUIRE(legs.size() == 4); + }); + // 1 spread remains + REQUIRE(resolver.count() == 1); + resolver.resolve(key_map2, [&](const string& key, const vector& legs) { + REQUIRE(key == "45"); + REQUIRE(legs.size() == 2); + }); + REQUIRE(resolver.count() == 0); + } + + + void test_no_double_security_callback() { + MDProducer producer(Platform::SIM); + vector producers = create_test_securities(producer); + // Assign ids so securities are already public in the producer + assign_ids(producers.begin(), producers.end()); + + // Symbol list for the consumer + auto syms = get_test_symbols(); + MDConsumer consumer = MDConsumer(syms); + + // Add the on_security callback - callback fires for existing securities + vector securities; + consumer.on_security([&](const Security& security, MDConsumerHandle handle) { + REQUIRE(security.security_key() != invalid_security_key); + REQUIRE(security.security_key() == handle.key()); + securities.push_back(security); + }); + // Found all securities + REQUIRE(securities.size() == producers.size()); + // Check for new securities + consumer.scan_securities(); + // Expect no duplicates + REQUIRE(securities.size() == producers.size()); + } + + void test_recovery() { + auto producer1 = make_shared(Platform::SIM); + vector producers = create_test_securities(*producer1); + assign_ids(producers.begin(), producers.end()); + producer1->get_channel("FOO", SecurityType::FUTURE, Exchange::TESTEXCHANGE).heartbeat(); + producer1->get_channel("FOOX", SecurityType::OPTION, Exchange::TESTEXCHANGE).heartbeat(); + for (auto& pro : producers) { pro.update_bbo(TimeStamp::now(), PriceSize(1, 0, 1), PriceSize(2, 0, 2)); } + + // Symbol list for the consumer + MDRecoverableConsumer consumer(get_test_symbols()); + map> products; + consumer.on_security([&](const Security& security, MDConsumerHandle handle) { + REQUIRE(security.security_key() == handle.key()); + products.insert(make_pair(security.security_key(), consumer.get_managed_handle(security.security_key()))); + }); + consumer.scan_securities(); + REQUIRE(products.size() == producers.size()); + + // Choose a handle + for (auto& pair : products) { + auto& handle = pair.second; + REQUIRE(handle->is_initialized()); + REQUIRE(handle->is_heart_beating()); + } + + // Delete the producer + try { + producers.clear(); + producer1.reset(); + } catch(std::exception&) { + LOG(MarketData,Error,"Failed test trying to destroy producer."); + REQUIRE(false); + } + + // Wait out the heartbeat + auto options = config::get(); + auto ms = std::max(options.heartbeat_ms(), options.heartbeat_boot_seconds()*1000)+10; + + std::this_thread::sleep_for(std::chrono::duration(ms)); + + // Make sure heartbeat is done + try { + for (auto& pair : products) { + auto& handle = pair.second; + REQUIRE(handle->is_initialized()); + REQUIRE(handle->is_heart_beating() == false); + } + } catch(std::exception&) { + LOG(MarketData,Error,"Failed to validate heartbeat while producer is down."); + REQUIRE(false); + } + + // Create a new publisher and attempt to recover + shared_ptr producer2; + try { + producer2 = make_shared(Platform::SIM); + } catch(std::exception&) { + LOG(MarketData,Error,"Failed to create new producer"); + REQUIRE(false); + } + producers = create_test_securities(*producer2); + REQUIRE(products.size() == producers.size()); + assign_ids(producers.begin(), producers.end()); + producer2->get_channel("FOO", SecurityType::FUTURE, Exchange::TESTEXCHANGE).heartbeat(); + producer2->get_channel("FOOX", SecurityType::OPTION, Exchange::TESTEXCHANGE).heartbeat(); + + // set bbo to a different value + for (auto& pro : producers) { + pro.update_bbo(TimeStamp::now(), PriceSize(4, 0, 5), PriceSize(6, 0, 7)); + } + + // Attempt to recover the managed handles + REQUIRE(consumer.try_recover()); + + // Since we know that both securities exist they should have been updated to the new section + for (auto& pair : products) { + auto& handle = pair.second; + REQUIRE(handle->is_initialized()); + REQUIRE(handle->is_heart_beating()); + TimeStamp tzero; + PriceSize bid, ask; + REQUIRE(handle->get_bbo(tzero, bid, ask)); + REQUIRE(bid.price() == 4.0); + REQUIRE(bid.size() == 5); + REQUIRE(ask.price() == 6.0); + REQUIRE(ask.size() == 7); + } + } + + + void test_no_block_forever_on_publisher_crash() { +#ifdef _WIN32 +#pragma warning(push, 0) +#endif + // OK if this isn't aligned for testing + auto src = std::make_unique ("Foo", SecurityType::FUTURE); +#ifdef _WIN32 +#pragma warning(pop) +#endif + TimeStamp read = TimeStamp::now(); + src->producer().update_bbo(TimeStamp::invalid(), PriceSize::from_double<3>(100.0, 100), PriceSize::from_double<3>(101.0, 101)); + PriceSize bid, ask; + TimeStamp start = TimeStamp::now(); + bool result = src->consumer().get_bbo(read, bid, ask); + TimeStamp end = TimeStamp::now(); + REQUIRE(result == false); + REQUIRE((end - start).seconds() < 5); + } + + void test_get_handles() { + MDProducer producer(Platform::SIM); + vector producers = create_test_securities(producer); + // Add entries to the shared memory that we aren't looking for + vector p2_no_id = create_test_securities(producer, "2"); + vector p3 = create_test_securities(producer, "3"); + + for (auto& p : producers) { p.update_bbo(TimeStamp::now(), PriceSize::from_double<3>(5.0, 5), PriceSize::from_double<3>(5.0, 5)); } + assign_ids(producers.begin(), producers.end()); + assign_ids(p3.begin(), p3.end()); + + // Symbol list for the consumer - only search half of the symbols + set symbols; + UserSymbol future_symbol( "FOO", Exchange::TESTEXCHANGE, SecurityType::FUTURE ); + symbols.insert(future_symbol); + UserSymbol option_symbol( "FOOX", Exchange::TESTEXCHANGE, SecurityType::OPTION ); + symbols.insert(option_symbol); + MDConsumer consumer(symbols); + + set keys; + consumer.walk_security_list([&](const Security& security, MDConsumerHandle) { keys.insert(security.security_key()); }); + REQUIRE(keys.size() == producers.size()); + auto map = consumer.get_handles(keys); + REQUIRE(map.size() == keys.size()); + for (const auto& entry : map) { + REQUIRE(entry.second.is_initialized()); + } + } + + void test_denormalization() { + // By using a different symbol for each security + // we put them all on separate shared mem sections (co2 channels) + security_key key = 999; + Security f1 = mkfuture("F1", ++key, 2016, 5, 1); + Security o1 = mkoption("O1", ++key, 2016, 5, 1, 100); + Security o2 = mkoption("O2", ++key, 2016, 5, 2, 200); + Security o3 = mkoption("O3", ++key, 2016, 5, 3, 300); + Security s4 = mkspread("S3", ++key, o2, o3); + Security s3 = mkspread("S4", ++key, o1, s4); + Security s2 = mkspread("S2", ++key, o1, s4, s3); + Security s1 = mkspread("S1", ++key, f1, s2); + Security all_secs_array[] = { f1, o1, o2, o3, s1, s2, s3, s4 }; + + MDProducer producer(Platform::SIM); + vector producers; + for (auto& sec : all_secs_array) { + with_security(sec.security_type(), sec.spread().legs_size(), [&](FixedSecurity& fsec) { + convert_security(sec, fsec, true); + producers.push_back(producer.add_security(fsec)); + }); + } + + set csymbols; + csymbols.insert(UserSymbol("F1", Exchange::TESTEXCHANGE, SecurityType::FUTURE)); + csymbols.insert(UserSymbol("O1", Exchange::TESTEXCHANGE, SecurityType::OPTION)); + csymbols.insert(UserSymbol("O2", Exchange::TESTEXCHANGE, SecurityType::OPTION)); + csymbols.insert(UserSymbol("O3", Exchange::TESTEXCHANGE, SecurityType::OPTION)); + csymbols.insert(UserSymbol("S1", Exchange::TESTEXCHANGE, SecurityType::SPREAD)); + csymbols.insert(UserSymbol("S2", Exchange::TESTEXCHANGE, SecurityType::SPREAD)); + csymbols.insert(UserSymbol("S3", Exchange::TESTEXCHANGE, SecurityType::SPREAD)); + csymbols.insert(UserSymbol("S4", Exchange::TESTEXCHANGE, SecurityType::SPREAD)); + MDConsumer consumer(csymbols); + + auto check_sec = [&](const Security& security, MDConsumerHandle handle) { + REQUIRE(security.security_key() == handle.key()); + auto iter = find_if(begin(all_secs_array), end(all_secs_array), [&](const Security& sec) { return sec.security_key() == security.security_key(); }); + REQUIRE(iter != end(all_secs_array)); + REQUIRE(SecurityHelper::symbol(*iter) == SecurityHelper::symbol(security)); + if (security.security_type() == SecurityType::SPREAD) { + REQUIRE(security.spread().normalized() == false); + REQUIRE(security.spread().has_symbol()); + REQUIRE(security.spread().legs_size() == iter->spread().legs_size()); + for (int x = 0; x < security.spread().legs_size(); ++x) { + REQUIRE(SecurityHelper::symbol(security.spread().legs(x).security()) == SecurityHelper::symbol(iter->spread().legs(x).security())); + } + } + }; + + consumer.walk_security_list(check_sec); + consumer.on_security(check_sec); + consumer.scan_securities(); + } + void test_denormalization2() { + // Just like test_denormailzation but with a different memory layout + security_key key = 999; + Security f1 = mkfuture("FF", ++key, 2016, 5, 1); + Security o1 = mkoption("OO", ++key, 2016, 5, 1, 100); + Security o2 = mkoption("OO", ++key, 2016, 5, 2, 200); + Security o3 = mkoption("OO", ++key, 2016, 5, 3, 300); + Security s4 = mkspread("SS", ++key, o2, o3); + Security s3 = mkspread("SS", ++key, o1, s4); + Security s2 = mkspread("SS", ++key, o1, s4, s3); + Security s1 = mkspread("SS", ++key, f1, s2); + Security all_secs_array[] = { f1, o1, o2, o3, s1, s2, s3, s4 }; + + MDProducer producer(Platform::SIM); + vector producers; + for (auto& sec : all_secs_array) { + with_security(sec.security_type(), sec.spread().legs_size(), [&](FixedSecurity& fsec) { + convert_security(sec, fsec, true); + producers.push_back(producer.add_security(fsec)); + }); + } + + set csymbols; + csymbols.insert(UserSymbol("FF", Exchange::TESTEXCHANGE, SecurityType::FUTURE)); + csymbols.insert(UserSymbol("OO", Exchange::TESTEXCHANGE, SecurityType::OPTION)); + csymbols.insert(UserSymbol("SS", Exchange::TESTEXCHANGE, SecurityType::SPREAD)); + MDConsumer consumer(csymbols); + + auto check_sec = [&](const Security& security, MDConsumerHandle handle) { + REQUIRE(security.security_key() == handle.key()); + auto iter = find_if(begin(all_secs_array), end(all_secs_array), [&](const Security& sec) { return sec.security_key() == security.security_key(); }); + REQUIRE(iter != end(all_secs_array)); + REQUIRE(SecurityHelper::symbol(*iter) == SecurityHelper::symbol(security)); + if (security.security_type() == SecurityType::SPREAD) { + REQUIRE(security.spread().normalized() == false); + REQUIRE(security.spread().has_symbol()); + REQUIRE(security.spread().legs_size() == iter->spread().legs_size()); + for (int x = 0; x < security.spread().legs_size(); ++x) { + REQUIRE(SecurityHelper::symbol(security.spread().legs(x).security()) == SecurityHelper::symbol(iter->spread().legs(x).security())); + } + } + }; + + consumer.walk_security_list(check_sec); + consumer.on_security(check_sec); + consumer.scan_securities(); + } + + void test_retry_unresolved_spread() { + // Make 2 spreads with option legs + // Leave out one leg for hte other spread + // The other spread only shows up once its option leg is produced + + Security o1 = mkoption("OO", 400, 2016, 5, 1, 100); + Security o2 = mkoption("OO", 401, 2016, 5, 2, 200); + Security o3 = mkoption("OO", 402, 2016, 5, 3, 300); + Security s1 = mkspread("SS", 500, o1, o2); + Security s2 = mkspread("SS", 501, o2, o3); + + MDProducer producer(Platform::SIM); + map handles; + auto add_security = [&](const Security& sec) { + with_security(sec.security_type(), sec.spread().legs_size(), [&](FixedSecurity& fsec) { + convert_security(sec, fsec, true); + handles.insert(make_pair(fsec.key, producer.add_security(fsec))); + }); + }; + // "options publisher has not started yet" + //add_security(o1); + //add_security(o2); + //add_security(o3); + add_security(s1); + add_security(s2); + + // Create consumer + set consumer_symbols; + consumer_symbols.insert(UserSymbol("OO", Exchange::TESTEXCHANGE, SecurityType::OPTION)); + consumer_symbols.insert(UserSymbol("SS", Exchange::TESTEXCHANGE, SecurityType::SPREAD)); + MDConsumer consumer(consumer_symbols); + + size_t security_count = 0; + // Indirection for check function + function check_func; + check_func = [&](const Security&) { + REQUIRE(false); + }; + consumer.on_security([&](const Security& sec, MDConsumerHandle) { ++security_count; check_func(sec); }); + consumer.scan_securities(); + REQUIRE(security_count == 0); + + // OK, now add enough options to resolve s1 but not s2 + add_security(o1); + add_security(o2); + check_func = [&](const Security& security) { + if (security.security_type() == SecurityType::SPREAD) { + set legs; + for (const auto& leg : security.spread().legs()) { + legs.insert(leg.security().security_key()); + } + REQUIRE(security.security_key() == 500); + REQUIRE(legs.size() == 2); + REQUIRE(legs.count(400) == 1); + REQUIRE(legs.count(401) == 1); + } + }; + consumer.scan_securities(); // finds o1,o2,s1 + consumer.scan_securities(); // finds nothing + REQUIRE(security_count == 3); + + // Then add the last security and we should see s2 + add_security(o3); + check_func = [&](const Security& security) { + if (security.security_type() == SecurityType::SPREAD) { + set legs; + for (const auto& leg : security.spread().legs()) { + legs.insert(leg.security().security_key()); + } + REQUIRE(security.security_key() == 501); + REQUIRE(legs.size() == 2); + REQUIRE(legs.count(401) == 1); + REQUIRE(legs.count(402) == 1); + } + }; + consumer.scan_securities(); + REQUIRE(security_count == 5); + } + + void test_settlement_recorder() { + using namespace datetime; + security_key k1 = 401, k2 = 402; + Security o1 = mkoption("OO", k1, 2016, 5, 1, 100); + Security o2 = mkoption("OO", k2, 2016, 5, 2, 200); + MDProducer producer(Platform::SIM); + auto add_security = [&](const Security& sec) { + unique_ptr phandle; + with_security(sec.security_type(), sec.spread().legs_size(), [&](FixedSecurity& fsec) { + convert_security(sec, fsec, true); + phandle.reset(new MDProducerHandle(producer.add_security(fsec))); + }); + return phandle; + }; + auto ph1 = *add_security(o1); + auto ph2 = *add_security(o2); + set consumer_symbols; + consumer_symbols.insert(UserSymbol("OO", Exchange::TESTEXCHANGE, SecurityType::OPTION)); + MDConsumer consumer(consumer_symbols); + auto h1 = consumer.get_handle(k1); + REQUIRE(h1.is_initialized()); + auto h2 = consumer.get_handle(k2); + + // Pick 2 valid dates in the future + auto today = datetime::today(); + today.year += 1; + auto yesterday = datetime::add_days(today, -1); + + { + co2::SettlementRecorder recorder(Platform::UNSET); + recorder.add_settlement(ph1, 101, yesterday); + recorder.add_settlement(ph1, 102, today); + recorder.add_settlement(ph2, 199, yesterday); + recorder.add_settlement(ph2, 188, today); + recorder.do_work(); + recorder.do_work(); + } + REQUIRE(h1.misc().settlement_price == 102); + REQUIRE(h1.misc().settlement_date == today); + REQUIRE(h1.misc().previous_settlement_price == 101); + REQUIRE(h1.misc().previous_settlement_date == yesterday); + REQUIRE(h2.misc().settlement_price == 188); + REQUIRE(h2.misc().settlement_date == today); + REQUIRE(h2.misc().previous_settlement_price == 199); + REQUIRE(h2.misc().previous_settlement_date == yesterday); + + // Update but skip persist + auto tomorrow = datetime::add_days(today, 1); + ph1.update_settlement(TimeStamp::now(), 103, tomorrow); + REQUIRE(h1.misc().settlement_price == 103); + REQUIRE(h1.misc().settlement_date == tomorrow); + REQUIRE(h1.misc().previous_settlement_price == 102); + REQUIRE(h1.misc().previous_settlement_date == today); + ph1.update_previous_settlement(TimeStamp::now(), 0, Date()); + REQUIRE(h1.misc().settlement_price == 103); + REQUIRE(h1.misc().settlement_date == tomorrow); + ph1.update_settlement(TimeStamp::now(), 0, Date()); + REQUIRE(h1.misc().settlement_price == 0); + REQUIRE(h1.misc().settlement_date.day == 0); + + // pushing a new settlement for today doesn't invalidate yesterday + ph2.update_settlement(TimeStamp::now(), 201, today); + REQUIRE(h2.misc().settlement_price == 201); + REQUIRE(h2.misc().settlement_date == today); + REQUIRE(h2.misc().previous_settlement_price == 199); + REQUIRE(h2.misc().previous_settlement_date == yesterday); + + // Restore persisted state + { + co2::SettlementRecorder recorder(Platform::UNSET); + recorder.restore_settlement(ph1); + recorder.restore_settlement(ph2); + recorder.do_work(); + } + REQUIRE(h1.misc().settlement_price == 102); + REQUIRE(h1.misc().settlement_date == today); + REQUIRE(h1.misc().previous_settlement_price == 101); + REQUIRE(h1.misc().previous_settlement_date == yesterday); + REQUIRE(h2.misc().settlement_price == 188); + REQUIRE(h2.misc().settlement_date == today); + REQUIRE(h2.misc().previous_settlement_price == 199); + REQUIRE(h2.misc().previous_settlement_date == yesterday); + } + + void teardown() {} +} ; +std::unique_ptr co2_tests; + +TEST_CASE("CO2Tests","[md]") { + + if (!co2_tests) + co2_tests = std::make_unique(); + + co2_tests->setup(); + + SECTION("test_security_id_recorder") { + co2_tests->test_security_id_recorder(); + } + SECTION("test_price_size") { + co2_tests->test_price_size(); + } + SECTION("test_slow_double_conversion") { + co2_tests->test_slow_double_conversion(); + } + SECTION("test_compact_exponent") { + co2_tests->test_compact_exponent(); + } + SECTION("test_producer_consumer") { + co2_tests->test_producer_consumer(); + } + SECTION("test_consumer_callback_on_security_added") { + co2_tests->test_consumer_callback_on_security_added(); + } + SECTION("test_consumer_defer_handle1") { + co2_tests->test_consumer_defer_handle1(); + } + SECTION("test_consumer_defer_handle2") { + co2_tests->test_consumer_defer_handle2(); + } + SECTION("test_spread_resolver") { + co2_tests->test_spread_resolver(); + } + SECTION("test_no_double_security_callback") { + co2_tests->test_no_double_security_callback(); + } + SECTION("test_recovery") { + co2_tests->test_recovery(); + } + SECTION("test_no_block_forever_on_publisher_crash") { + co2_tests->test_no_block_forever_on_publisher_crash(); + } + SECTION("test_get_handles") { + co2_tests->test_get_handles(); + } + SECTION("test_denormalization") { + co2_tests->test_denormalization(); + } + SECTION("test_denormalization2") { + co2_tests->test_denormalization2(); + } + SECTION("test_retry_unresolved_spread") { + co2_tests->test_retry_unresolved_spread(); + } + SECTION("test_settlement_recorder") { + co2_tests->test_settlement_recorder(); + } +} + +struct Co2IndexTests { + vector random_ids; + set used_ids; + const int iterations; + tbb::task_scheduler_init _tbb; + + Co2IndexTests() : + random_ids(), + used_ids(), + iterations(50), + _tbb(tbb::task_scheduler_init::automatic) + {} + + void setup() { + used_ids.clear(); + random_ids.clear(); + + std::uniform_int_distribution rand_distribution(1, 0x7FFFFFFF); + std::mt19937 random_number_engine; + auto genrandom = std::bind(rand_distribution, random_number_engine); + set temp_ids; + for (int x = 0; x < 9999; ++x) { + auto r = genrandom(); + if (x < 1000) { used_ids.insert(r); } + temp_ids.insert(r); + } + for (uint64_t id : temp_ids) { + random_ids.push_back(id); + } + REQUIRE(random_ids.size() > 1000); + } + + struct Handle { + Handle() : _key(0) {} + uint64_t _key; + security_key key() const { return _key; } + void key(security_key key) { _key = key; } + }; + + struct SecurityData { + uint64_t exchange_id; + Security security; + Handle handle; + }; + + template + void gen_securities(Index& index) { + unique_ptr temp; + uint32_t count = 0; + for (auto& id : used_ids) { + ++count; + temp.reset(new SecurityData()); + temp->security.Clear(); + temp->security.set_security_type(SecurityType::OPTION); + temp->security.mutable_option()->set_symbol("FOOX"); + temp->security.set_numeric_id(id); + temp->exchange_id = id; + temp->handle.key(count); + index.add_security(id, move(temp)); + } + } + + + template + void test_random_search(Index& index) { + vector temp(random_ids); + std::random_shuffle(temp.begin(), temp.end()); + for (auto& id : temp) { + if (used_ids.find(id) != used_ids.end()) { + REQUIRE(index.has_security(id)); + } + else { + REQUIRE(index.get_security(id) == nullptr); + } + } + } + + template + void test_walk(Index& index) { + int count = 0; + index.walk_security_list([&count](const SecurityData&) { ++count; }); + REQUIRE(count == (int) used_ids.size()); + } + + typedef SimpleSecurityIndex MySimpleIndex; + void test_simple_index_search() { + MySimpleIndex index; + gen_securities(index); + for (int x = 0; x < iterations; ++x) { + test_random_search(index); + } + } + + void test_simple_index_parallel_search() { + MySimpleIndex index; + gen_securities(index); + tbb::parallel_for(0, iterations, [&](int) { + test_random_search(index); + }); + } + + void test_simple_index_iterate() { + MySimpleIndex index; + gen_securities(index); + test_walk(index); + } + + typedef ConcurrentSecurityIndex MyConcurrentIndex; + void test_concurrent_index_search() { + MyConcurrentIndex index; + gen_securities(index); + for (int x = 0; x < iterations; ++x) { + test_random_search(index); + } + } + void test_concurrent_index_parallel_search() { + MyConcurrentIndex index; + gen_securities(index); + tbb::parallel_for(0, iterations, [&](int) { + test_random_search(index); + }); + } + void test_concurrent_index_iterate() { + MyConcurrentIndex index; + gen_securities(index); + test_walk(index); + } +} ; + +std::unique_ptr co2_index_tests; + +TEST_CASE ("Co2IndexTests","[md]") +{ + if (!co2_index_tests) + co2_index_tests = std::make_unique (); + + co2_index_tests->setup(); + + SECTION("test_simple_index_search") { + co2_index_tests->test_simple_index_search(); + } + + SECTION("test_simple_index_parallel_search") { + co2_index_tests->test_simple_index_parallel_search(); + } + + SECTION("test_simple_index_iterate") { + co2_index_tests->test_simple_index_iterate(); + } + SECTION("test_concurrent_index_search") { + co2_index_tests->test_concurrent_index_search(); + } + SECTION("test_concurrent_index_iterate") { + co2_index_tests->test_concurrent_index_iterate(); + } + SECTION("test_concurrent_index_parallel_search") { + co2_index_tests->test_concurrent_index_parallel_search(); + } +} + +struct Co2PerfTests { + static const size_t instrument_count = 2048; + vector> test_instruments; + vector producers; + vector consumers; + + Co2PerfTests() : + test_instruments(), + producers(), + consumers() + { +#ifdef _WIN32 +#pragma warning(push, 0) +#endif + // OK if this isn't aligned for testing + for (size_t x = 0; x < instrument_count; ++x) { + test_instruments.push_back( + std::unique_ptr( + new Co2TestInstrument("Foo", SecurityType::OPTION))); + } +#ifdef _WIN32 +#pragma warning(pop) +#endif + for (size_t x = 0; x < test_instruments.size(); ++x) { + auto producer = test_instruments[x]->producer(); + producer.set_key(x + 1); + producers.push_back(producer); + auto consumer = test_instruments[x]->consumer(); + REQUIRE(consumer.key() == x + 1); + consumers.push_back(consumer); + } + + std::srand(std::time(0)); + //std::random_shuffle(producers.begin(), producers.end(), [](int i) { return std::rand() % i; }); + std::random_shuffle(consumers.begin(), consumers.end(), [](int i) { return std::rand() % i; }); + } + void setup() { } + void teardown() {}; + void test_write_bbo_perf() { + REQUIRE(producers.size() == instrument_count); + + // Warmup + for (size_t x = 0; x < producers.size(); ++x) { + auto& p = producers[x]; + // p.update_bbo(TimeStamp::now(), PriceSize::from_double<3>(100.0 + x, x), PriceSize::from_double<3>(200 + x, 2 * x)); + p.update_bbo(TimeStamp::now(), PriceSize(100+x,0,x), PriceSize(200+x,0,2*x)); + } + + // Test + TimeStamp start = TimeStamp::now(); + size_t count = 0; + const size_t iterations = 1000; + for (size_t loop = 0; loop < iterations; ++loop) { + for (size_t x = 0; x < producers.size(); ++x) { + ++count; + auto& p = producers[x]; + TimeStamp t = TimeStamp::now(); + // p.update_bbo(t, PriceSize::from_double<3>(100.0 + count, count), PriceSize::from_double<3>(200 + count, 2 * count)); + p.update_bbo(t, PriceSize(100+x,0,x), PriceSize(200+x,0,2*x)); + } + } + TimeStamp end = TimeStamp::now(); + std::stringstream ss; + // Of which approximately 1/3 is generating the timestamp on lab-lx4 + ss << "\t" << (end - start).nanoseconds() / iterations / instrument_count << " ns per update *\n"; + std::cout << ss.str(); + } + + void test_read_bbo_perf() { + // Warmup + const auto& write_all = [](vector& producers) { + // Offset each price by a random amount each iteration to ensure that writes actually happen + size_t rand_factor = std::rand() % 1024; + for (size_t x = 0; x < producers.size(); ++x) { + auto& p = producers[x]; + // p.update_bbo(TimeStamp::now(), PriceSize::from_double<3>(100.0 + x + rand_factor, x+rand_factor), PriceSize::from_double<3>(200 + x+rand_factor, 2 * x + rand_factor)); + p.update_bbo(TimeStamp::now(), PriceSize(100+x+rand_factor,0,x+rand_factor), PriceSize(200+x+rand_factor,0,2*x+rand_factor)); + } + }; + write_all(producers); + + for (auto& c : consumers) { + PriceSize bid, ask; + TimeStamp ts; + REQUIRE(c.get_bbo(ts, bid, ask)); + } + + // Start a thread which loops and writes to the producers until terminated + utils::Dispatcher& writer = utils::Dispatcher::create_new("bbo_writer"); + DEFER(writer.terminate()); + tbb::queuing_mutex producer_mutex; + auto* pproducers = &producers; + function write_all_repeat = [&]() { + tbb::queuing_mutex::scoped_lock lock(producer_mutex); + if (pproducers != nullptr) { + write_all(*pproducers); + writer.post(write_all_repeat); + } + }; + write_all_repeat(); + + // Perform read test + TimeStamp start = TimeStamp::now(); + size_t read_count = 0; + const size_t iterations = 1000; + for (size_t loop = 0; loop < iterations; ++loop) { + TimeStamp ts = TimeStamp::now(); + for (size_t x = 0; x < consumers.size(); ++x) { + PriceSize bid, ask; + TimeStamp ts_temp = ts; + if (consumers[x].get_bbo(ts_temp, bid, ask)) { + ++read_count; + } + } + } + REQUIRE(read_count > 0); + + TimeStamp end = TimeStamp::now(); + std::stringstream ss; + ss << "\t" << (end - start).nanoseconds() / iterations / instrument_count << " ns per update. Actual reads = " << read_count << " / " << instrument_count*iterations << " * \n"; + std::cout << ss.str(); + + writer.terminate(); + + tbb::queuing_mutex::scoped_lock lock(producer_mutex); + pproducers = nullptr; + std::this_thread::sleep_for(2s); + + } + +} ; + +std::unique_ptr co2_perf_tests; + +const size_t Co2PerfTests::instrument_count ; +TEST_CASE ("Co2PerfTests","[md]") +{ + + if (!co2_perf_tests) + co2_perf_tests = std::make_unique(); + + co2_perf_tests->setup(); + + SECTION("test_write_bbo_perf") { + // co2_perf_tests->test_write_bbo_perf(); + } + SECTION("test_read_bbo_perf") { + // co2_perf_tests->test_read_bbo_perf(); + } +} + +// struct Co2RelayTests { +// void setup() { +// config::internal::initialize_for_test( +// "PlatformMap {" +// " value {" +// " sim {" +// " exchange: TESTEXCHANGE" +// " }" +// " }" +// "}"); +// } +// void teardown() {} + +// MDConsumer create_consumer() { +// set symbols; +// UserSymbol future_symbol("FOO", Exchange::TESTEXCHANGE, SecurityType::FUTURE); +// symbols.insert(future_symbol); +// UserSymbol option_symbol("FOOX", Exchange::TESTEXCHANGE, SecurityType::OPTION); +// symbols.insert(option_symbol); +// MDConsumer consumer(symbols); +// return consumer; +// } + + +// template +// T simulate_wire_transfer(const T& request) { +// T temp; +// std::string buffer; +// request.SerializeToString(&buffer); +// temp.ParseFromString(buffer); +// return temp; +// } + +// PriceSize convert(const co2::PriceSizeMessage& psm) { +// return PriceSize(psm.mantissa(), psm.exponent(), psm.size()); +// } + +// void relay_nans() { +// MDProducer producer(Platform::SIM); +// UserSymbol symbol("FOO", Exchange::TESTEXCHANGE, SecurityType::OPTION); + +// // initialize data +// vector producers; +// for (size_t x = 0; x < 100; ++x) { +// auto strike = 1000 + (100 * x); +// producers.push_back( +// producer.add_security( +// mkfixedoption("FOO", invalid_security_key, 2017, 1, 28, strike, PutOrCall::PUT))); +// producers.push_back( +// producer.add_security( +// mkfixedoption("FOO", invalid_security_key, 2017, 1, 28, strike, PutOrCall::CALL))); +// } +// assign_ids(producers.begin(), producers.end(), 1000); +// PriceSize price35 = PriceSize::from_double<2>(3.5, 5); +// PriceSize price5 = PriceSize::from_double<2>(5, 5); +// for(auto p : producers) { +// p.update_bbo(TimeStamp::now(), price35, price35); +// } + +// // Read using QueryServer +// set symbols; +// symbols.insert(symbol); +// auto server = create_query_server(symbols); +// REQUIRE(server); +// server->scan_securities(); + +// co2::MDRequest request; +// auto* chan = request.add_channels(); +// chan->set_type(symbol.type()); +// chan->set_symbol(symbol.symbol()); +// chan->set_exchange(symbol.exchange()); +// chan->set_flags(co2::MDRequest::BBO | co2::MDRequest::SECURITY); + +// // This request should include everything +// co2::MDReply reply; +// server->query(request, &reply); +// reply = simulate_wire_transfer(reply); +// REQUIRE((size_t)reply.channels().size() == symbols.size()); +// REQUIRE((size_t)reply.channels(0).updates().size() == producers.size()); +// // update the request to match the server time +// for (auto& update : reply.channels(0).updates()) { +// REQUIRE(update.has_security()); +// REQUIRE(update.has_book()); +// REQUIRE(update.book().asks_size() > 0); +// REQUIRE(update.book().bids_size() > 0); +// REQUIRE(convert(update.book().bids(0)) == price35); +// REQUIRE(convert(update.book().asks(0)) == price35); +// } + +// // Second request with updated time should include nothing +// request.mutable_channels(0)->set_time(reply.channels(0).channel().time()); +// reply.Clear(); +// server->query(request, &reply); +// reply = simulate_wire_transfer(reply); +// REQUIRE(reply.channels().size() == 1); +// REQUIRE(reply.channels(0).updates().size() == 0); + +// // Update a single price and make sure we get that back +// producers[0].update_bbo(TimeStamp::now(), price5, price5, 1); +// request.mutable_channels(0)->set_time(reply.channels(0).channel().time()); +// reply.Clear(); +// server->query(request, &reply); +// reply = simulate_wire_transfer(reply); +// REQUIRE(reply.channels().size() == 1); +// REQUIRE(reply.channels(0).updates().size() == 1); +// REQUIRE(reply.channels(0).updates(0).has_book()); +// REQUIRE(convert(reply.channels(0).updates(0).book().bids(0)) == price5); +// REQUIRE(convert(reply.channels(0).updates(0).book().asks(0)) == price5); + +// // NaN out all the prices - expect to get a full set of updates +// for(auto p : producers) { +// p.update_bbo(TimeStamp::now(), PriceSize::NO_MARKET, PriceSize::NO_MARKET, 2); +// } +// request.mutable_channels(0)->set_time(reply.channels(0).channel().time()); +// reply.Clear(); +// server->query(request, &reply); +// reply = simulate_wire_transfer(reply); +// REQUIRE(reply.channels().size() == 1); +// REQUIRE((size_t)reply.channels(0).updates().size() == producers.size()); +// for (auto& update : reply.channels(0).updates()) { +// REQUIRE(update.has_book()); +// REQUIRE(update.book().bids_size() > 0); +// REQUIRE(update.book().asks_size() > 0); +// REQUIRE(convert(update.book().bids(0)) == PriceSize::NO_MARKET); +// REQUIRE(convert(update.book().asks(0)) == PriceSize::NO_MARKET); +// } + +// // Fill up the markets again +// for(auto p : producers) { +// p.update_bbo(TimeStamp::now(), price35, price35, 3); +// } +// request.mutable_channels(0)->set_time(reply.channels(0).channel().time()); +// reply.Clear(); +// server->query(request, &reply); +// reply = simulate_wire_transfer(reply); +// REQUIRE(reply.channels().size() == 1); +// REQUIRE((size_t)reply.channels(0).updates().size() == producers.size()); +// for (auto& update : reply.channels(0).updates()) { +// REQUIRE(update.has_book()); +// REQUIRE(update.book().bids_size() > 0); +// REQUIRE(update.book().asks_size() > 0); +// REQUIRE(convert(update.book().bids(0)) == price35); +// REQUIRE(convert(update.book().asks(0)) == price35); +// } + +// // NaN out the bids +// for(auto p : producers) { +// p.update_bbo(TimeStamp::now(), PriceSize::NO_MARKET, price35, 4); +// } +// request.mutable_channels(0)->set_time(reply.channels(0).channel().time()); +// reply.Clear(); +// server->query(request, &reply); +// reply = simulate_wire_transfer(reply); +// REQUIRE(reply.channels().size() == 1); +// REQUIRE((size_t)reply.channels(0).updates().size() == producers.size()); +// for (auto& update : reply.channels(0).updates()) { +// REQUIRE(update.has_book()); +// REQUIRE(update.book().bids_size() > 0); +// REQUIRE(update.book().asks_size() > 0); +// REQUIRE(convert(update.book().bids(0)) == PriceSize::NO_MARKET); +// REQUIRE(convert(update.book().asks(0)) == price35); +// } +// } +// } ; + +//std::unique_ptr co2_relay_tests; + + +// TEST_CASE ("Co2RelayTests","[md]") +// { +// if (!co2_relay_tests) +// co2_relay_tests = std::make_unique(); +// co2_relay_tests->setup(); + +// SECTION("relay_nans") { +// co2_relay_tests->relay_nans(); +// } +// } + + +}}} + +#endif diff --git a/tools/Protobuf.cmake b/tools/Protobuf.cmake new file mode 100644 index 000000000..974f1a940 --- /dev/null +++ b/tools/Protobuf.cmake @@ -0,0 +1,119 @@ + +find_package(Protobuf REQUIRED) + +if(CMAKE_BUILD_TYPE MATCHES "^Debug$" OR NOT DEFINED CMAKE_BUILD_TYPE) + find_library (Protobuf_Static_Lib NAMES libprotobuf.a libprotobufd.lib ) +else() + find_library (Protobuf_Static_Lib NAMES libprotobuf.a libprotobuf.lib ) +endif() + +function(Service_Compiler_Generate_Cpp SRCS HDRS) + + set(${SRCS}) + set(${HDRS}) + + foreach(FIL ${ARGN}) + get_filename_component(ABS_FIL ${FIL} ABSOLUTE) + get_filename_component(FIL_WE ${FIL} NAME_WE) + get_filename_component(FIL_DIR ${FIL} DIRECTORY) + if(FIL_DIR) + set(FIL_WE "${FIL_DIR}/${FIL_WE}") + endif() + + list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc") + list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h") + + list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc") + list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h") + + list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc") + list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h") + + if (WIN32) + set(plugin_cmd plugin.cmd) + else() + set(plugin_cmd plugin.py) + endif() + + add_custom_command( + + OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc" + "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h" + "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.cc" + "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.client.pb.h" + "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.cc" + "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.server.pb.h" + + COMMAND ${Protobuf_PROTOC_EXECUTABLE} + + ARGS + --cpp_out=${CMAKE_BINARY_DIR}/src + --plugin=protoc-gen-service=${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/${plugin_cmd} + --service_out=lang=cpp,make_file_name=${ABS_FIL}:${CMAKE_BINARY_DIR}/src + -I ${PROTOBUF_INCLUDE_DIRS} + -I ${CMAKE_SOURCE_DIR}/src + ${ABS_FIL} + + DEPENDS ${ABS_FIL} + + COMMENT "Running C++ service compiler on ${FIL}" + VERBATIM ) + endforeach() + + set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE) + + # remove some warnings from protobuf generated files + if (WIN32) + set_source_files_properties(${${SRCS}} PROPERTIES COMPILE_FLAGS "/wd4309 /wd4838") + else () + set_source_files_properties(${${SRCS}} PROPERTIES COMPILE_FLAGS + "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized") + endif() + + set(${SRCS} ${${SRCS}} PARENT_SCOPE) + set(${HDRS} ${${HDRS}} PARENT_SCOPE) + +endfunction() + +function(Protobuf_Generate_Cpp SRCS HDRS) + + set(${SRCS}) + set(${HDRS}) + + foreach(FIL ${ARGN}) + get_filename_component(ABS_FIL ${FIL} ABSOLUTE) + get_filename_component(FIL_WE ${FIL} NAME_WE) + get_filename_component(FIL_DIR ${FIL} DIRECTORY) + if(FIL_DIR) + set(FIL_WE "${FIL_DIR}/${FIL_WE}") + endif() + + list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc") + list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h") + + add_custom_command( + OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc" + "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h" + COMMAND ${Protobuf_PROTOC_EXECUTABLE} + ARGS + --cpp_out=${CMAKE_BINARY_DIR}/src + -I ${PROTOBUF_INCLUDE_DIRS} + -I ${CMAKE_SOURCE_DIR}/src ${ABS_FIL} + DEPENDS ${ABS_FIL} + COMMENT "Running C++ protocol buffer compiler on ${FIL}" + VERBATIM ) + endforeach() + + set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE) + + # remove some warnings from protobuf generated files + if (WIN32) + set_source_files_properties(${${SRCS}} PROPERTIES COMPILE_FLAGS "/wd4309 /wd4838") + else () + set_source_files_properties(${${SRCS}} PROPERTIES COMPILE_FLAGS + "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized") + endif() + + set(${SRCS} ${${SRCS}} PARENT_SCOPE) + set(${HDRS} ${${HDRS}} PARENT_SCOPE) +endfunction() diff --git a/tools/setup_builds.sh b/tools/setup_builds.sh new file mode 100755 index 000000000..eb9e328ca --- /dev/null +++ b/tools/setup_builds.sh @@ -0,0 +1,33 @@ +#/bin/bash +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd $DIR/.. +rm -rf build + +mkdir build +cd build + +mkdir gcc_debug +cd gcc_debug +cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_COMPILER=g++ ../.. +cd .. + +mkdir gcc_release +cd gcc_release +cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=g++ ../.. +cd .. + +mkdir clang_debug +cd clang_debug +cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_COMPILER=clang++ ../.. +cd .. + +mkdir clang_release +cd clang_release +cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=clang++ ../.. +cd .. + +mkdir emacs_pp +cd emacs_pp +cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_COMPILER=rtags ../.. +cd .. +