From c69604b7fe19826829dc39dd677edd33acbeafce Mon Sep 17 00:00:00 2001 From: Kevin Darby Date: Fri, 6 Oct 2017 11:25:34 -0400 Subject: [PATCH 1/2] move externals up --- externals | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals b/externals index a15b17370..1884a761e 160000 --- a/externals +++ b/externals @@ -1 +1 @@ -Subproject commit a15b17370e680224500fe6258e3ebea5ce1a0fba +Subproject commit 1884a761e4a0296b59c6801ec7e62acca4d22997 From be0794a305ec392a4651b3066ec2e20d80da735d Mon Sep 17 00:00:00 2001 From: Kevin Darby Date: Fri, 6 Oct 2017 11:26:45 -0400 Subject: [PATCH 2/2] celoxica boilerplate --- src/lib/md/protos/co2.proto | 1 + src/server/sparkd/CMakeDeps.cmake | 2 +- src/server/sparkd/CMakeLists.txt | 7 +- src/server/sparkd/cmds/TestCeloxica.cpp | 26 +++ src/server/sparkd/cmds/TestCeloxica.h | 20 ++ src/server/sparkd/main.cpp | 2 + .../md/platforms/celoxica/CXListener.cpp | 89 ++++++++ .../sparkd/md/platforms/celoxica/CXListener.h | 61 +++++ .../sparkd/md/platforms/celoxica/CXMain.cpp | 83 +++++++ .../sparkd/md/platforms/celoxica/CXMain.h | 49 ++++ .../md/platforms/celoxica/CXSecurityIndex.h | 21 ++ .../sparkd/md/platforms/celoxica/celoxica.h | 15 ++ .../md/platforms/celoxica/tests/cx_tests.cpp | 210 +++++++++++++++++ .../md/platforms/mdp3_onixs/MDP3Utils.cpp | 215 +++++++++--------- src/server/sparkd/sparkd.vcxproj | 20 ++ src/server/sparkd/sparkd.vcxproj.filters | 78 ++++++- tools/celoxica.cmake | 19 ++ 17 files changed, 804 insertions(+), 114 deletions(-) create mode 100644 src/server/sparkd/cmds/TestCeloxica.cpp create mode 100644 src/server/sparkd/cmds/TestCeloxica.h create mode 100644 src/server/sparkd/md/platforms/celoxica/CXListener.cpp create mode 100644 src/server/sparkd/md/platforms/celoxica/CXListener.h create mode 100644 src/server/sparkd/md/platforms/celoxica/CXMain.cpp create mode 100644 src/server/sparkd/md/platforms/celoxica/CXMain.h create mode 100644 src/server/sparkd/md/platforms/celoxica/CXSecurityIndex.h create mode 100644 src/server/sparkd/md/platforms/celoxica/celoxica.h create mode 100644 src/server/sparkd/md/platforms/celoxica/tests/cx_tests.cpp create mode 100644 tools/celoxica.cmake diff --git a/src/lib/md/protos/co2.proto b/src/lib/md/protos/co2.proto index a6d2271d3..6a5c5acae 100644 --- a/src/lib/md/protos/co2.proto +++ b/src/lib/md/protos/co2.proto @@ -20,6 +20,7 @@ message Platform { MDP3ONIXS = 10; CSM_ONIXS = 11; BLUE = 12; + CELOXICA = 13; } } diff --git a/src/server/sparkd/CMakeDeps.cmake b/src/server/sparkd/CMakeDeps.cmake index 820eee3d6..09bf293c4 100644 --- a/src/server/sparkd/CMakeDeps.cmake +++ b/src/server/sparkd/CMakeDeps.cmake @@ -1 +1 @@ -set(sources ;cmds/CmdRegistry.cpp;cmds/Help.cpp;cmds/Run.cpp;cmds/Test.cpp;cmds/TestOrderman.cpp;event_grid/EventNode.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/store_tests.cpp;main.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/MDP3Utils.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;orderman/components/FillRecorder.cpp;orderman/components/Gateway.cpp;orderman/components/MDSource.cpp;orderman/components/NativeOrderMan.cpp;orderman/components/OrderRecorder.cpp;orderman/components/SpreaderOrderMan.cpp;orderman/events/ExecutionReport.cpp;orderman/events/NativeOrderStateChange.cpp;orderman/events/SpreaderOrderStateChange.cpp;orderman/events/TimeChange.cpp;orderman/native_orders/NativeOrder.cpp;orderman/native_orders/NativeOrderBuilder.cpp;orderman/native_orders/NativeOrderFactory.cpp;orderman/persistence/FillDB.cpp;orderman/persistence/OrderDB.cpp;orderman/persistence/gen_order_id.cpp;orderman/service/OrderServiceServer.cpp;orderman/spreader_orders/SpreaderOrder.cpp;orderman/spreader_orders/SpreaderOrderBase.cpp;orderman/utils/fix_order_codec.cpp;orderman/utils/orderman_utils.cpp;persistence/IStore.cpp;persistence/PersistenceServiceServer.cpp;persistence/SqliteStore.cpp;persistence/tests/PersistenceServiceTests.cpp;risk_limits/IRiskLimitObserver.cpp;risk_limits/RiskLimitManager.cpp;risk_limits/RiskLimiter.cpp;risk_limits/SharedRiskLimiter.cpp;risk_limits/SymbolKey.cpp;risk_limits/tests/RiskLimitsTests.cpp;securities/SecurityDB.cpp;spreader/QuoteLegPriceQuantityState.cpp;spreader/SimpleSpreader.cpp;spreader/SingleFillHedger.cpp;spreader/SingleLegQuoter.cpp;spreader/SpreaderBase.cpp;spreader/SpreaderTests.cpp;testing/catch_test_runner.cpp;user/UserServiceServer.cpp) \ No newline at end of file +set(sources ;cmds/CmdRegistry.cpp;cmds/Help.cpp;cmds/Run.cpp;cmds/Test.cpp;cmds/TestCeloxica.cpp;cmds/TestOrderman.cpp;event_grid/EventNode.cpp;jetfix/codec/FIXArray.cpp;jetfix/common/OutFIX.cpp;jetfix/connection/AsyncReader.cpp;jetfix/connection/IConnection.cpp;jetfix/connection/SocketConnection.cpp;jetfix/gateway/CoreGateway.cpp;jetfix/gateway/JetFIXGateway.cpp;jetfix/gateway/Session.cpp;jetfix/store/FIXFileStore.cpp;jetfix/store/session_name.cpp;jetfix/tests/async_reader_tests.cpp;jetfix/tests/decoding_tests.cpp;jetfix/tests/fix_encoding_stream_tests.cpp;jetfix/tests/fix_fragment_tests.cpp;jetfix/tests/jetfix_performance_tests.cpp;jetfix/tests/store_tests.cpp;main.cpp;md/platforms/celoxica/CXListener.cpp;md/platforms/celoxica/CXMain.cpp;md/platforms/celoxica/tests/cx_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/MDP3Utils.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;orderman/components/FillRecorder.cpp;orderman/components/Gateway.cpp;orderman/components/MDSource.cpp;orderman/components/NativeOrderMan.cpp;orderman/components/OrderRecorder.cpp;orderman/components/SpreaderOrderMan.cpp;orderman/events/ExecutionReport.cpp;orderman/events/NativeOrderStateChange.cpp;orderman/events/SpreaderOrderStateChange.cpp;orderman/events/TimeChange.cpp;orderman/native_orders/NativeOrder.cpp;orderman/native_orders/NativeOrderBuilder.cpp;orderman/native_orders/NativeOrderFactory.cpp;orderman/persistence/FillDB.cpp;orderman/persistence/OrderDB.cpp;orderman/persistence/gen_order_id.cpp;orderman/service/OrderServiceServer.cpp;orderman/spreader_orders/SpreaderOrder.cpp;orderman/spreader_orders/SpreaderOrderBase.cpp;orderman/utils/fix_order_codec.cpp;orderman/utils/orderman_utils.cpp;persistence/IStore.cpp;persistence/PersistenceServiceServer.cpp;persistence/SqliteStore.cpp;persistence/tests/PersistenceServiceTests.cpp;risk_limits/IRiskLimitObserver.cpp;risk_limits/RiskLimitManager.cpp;risk_limits/RiskLimiter.cpp;risk_limits/SharedRiskLimiter.cpp;risk_limits/SymbolKey.cpp;risk_limits/tests/RiskLimitsTests.cpp;securities/SecurityDB.cpp;spreader/QuoteLegPriceQuantityState.cpp;spreader/SimpleSpreader.cpp;spreader/SingleFillHedger.cpp;spreader/SingleLegQuoter.cpp;spreader/SpreaderBase.cpp;spreader/SpreaderTests.cpp;testing/catch_test_runner.cpp;user/UserServiceServer.cpp) \ No newline at end of file diff --git a/src/server/sparkd/CMakeLists.txt b/src/server/sparkd/CMakeLists.txt index c8271ab6c..85e59db6a 100644 --- a/src/server/sparkd/CMakeLists.txt +++ b/src/server/sparkd/CMakeLists.txt @@ -26,16 +26,21 @@ add_executable(sparkd ${Sparkd_Sources} ${Proto_Srcs}) #include paths target_include_directories( sparkd PUBLIC ${CMAKE_SOURCE_DIR}/src ${CMAKE_BINARY_DIR}/src) +#celoxica stuff +include(${CMAKE_SOURCE_DIR}/tools/celoxica.cmake) + # linking target_link_libraries( sparkd spark_lib tbb double-conversion pugixml + ${CELOXICA_LIB} + -static-libgcc -static-libstdc++ -pthread) # copy test data file(COPY jetfix/tests/sample_fix.dat DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/test_data) - + diff --git a/src/server/sparkd/cmds/TestCeloxica.cpp b/src/server/sparkd/cmds/TestCeloxica.cpp new file mode 100644 index 000000000..251acde6d --- /dev/null +++ b/src/server/sparkd/cmds/TestCeloxica.cpp @@ -0,0 +1,26 @@ +#include "TestCeloxica.h" + +#include "server/sparkd/md/platforms/celoxica/CXMain.h" +#include "server/sparkd/md/platforms/celoxica/CXListener.h" + +namespace bts +{ +namespace cmd +{ + +int TestCeloxica::execute(const std::vector&) +{ + bts::md::CXMain main; + main.initialize(); + return 0; +} + +std::string TestCeloxica::help() +{ + return "\t test BTS <> Celoxica integration \n"; +} + +} +} + + diff --git a/src/server/sparkd/cmds/TestCeloxica.h b/src/server/sparkd/cmds/TestCeloxica.h new file mode 100644 index 000000000..8e2b2bd61 --- /dev/null +++ b/src/server/sparkd/cmds/TestCeloxica.h @@ -0,0 +1,20 @@ +#pragma once + +#include "ICmd.h" + +namespace bts +{ +namespace cmd +{ + +class TestCeloxica : public ICmd +{ +public: + int execute(std::vector const &args); + std::string help(); +}; + +} +} + + diff --git a/src/server/sparkd/main.cpp b/src/server/sparkd/main.cpp index 64947d577..ffd957edb 100644 --- a/src/server/sparkd/main.cpp +++ b/src/server/sparkd/main.cpp @@ -7,6 +7,7 @@ #include "server/sparkd/cmds/Run.h" #include "server/sparkd/cmds/Help.h" #include "server/sparkd/cmds/TestOrderman.h" +#include "server/sparkd/cmds/TestCeloxica.h" #define BEGIN_CMD #define REGISTER(CMD) bts::cmd::CmdRegistry::register_cmd (#CMD) @@ -22,6 +23,7 @@ int main( int argc, char*argv [] ) REGISTER (Help); REGISTER (Run); REGISTER (TestOrderman); + REGISTER (TestCeloxica); END_CMD; bts::utils::finit_spark_lib(); diff --git a/src/server/sparkd/md/platforms/celoxica/CXListener.cpp b/src/server/sparkd/md/platforms/celoxica/CXListener.cpp new file mode 100644 index 000000000..91a490fe4 --- /dev/null +++ b/src/server/sparkd/md/platforms/celoxica/CXListener.cpp @@ -0,0 +1,89 @@ +#include "CXListener.h" + +#include "lib/utils/defer.h" +#include "lib/utils/CallTimer.h" +#include "lib/utils/helper_macros.h" +#include + +namespace celoxica { namespace gmac {} } +#include "celoxica.h" +using namespace celoxica::gmac; +using namespace std; +using namespace bts::md2; + +namespace bts +{ +namespace md +{ + +CXListener::CXListener(CXSecurityIndex& index, md2::MDProducer& producer, md2::SecurityIDRecorder& recorder, + co2::SettlementRecorder& settles, std::vector channels, + utils::Dispatcher& disp) : + _index(index), + _producer(producer), + _packet_arrived(), + _latest_sequence(0), + _security_recorder(recorder), + _settlement_recorder(settles), + _symbol_filter_storage(), + _channels(channels), + _background(disp), + _security_recorder_input_timer(disp, 900, [&]() { dequeue_securities(); }), + _security_recorder_input(), + _concurrency_monitor(0), + _book_updates() + { + _book_updates.reserve(32); + _security_recorder_input_timer.start(); + } + +CXListener::~CXListener() +{ + _security_recorder_input_timer.stop(); +} + +void CXListener::add_software_filter(const std::string& symbol) +{ + _symbol_filter_storage.push_back(symbol); + //_symbol_filter.clear(); + //for (const auto& str : _symbol_filter_storage) { + // _symbol_filter.insert(onixs::StringRef(str)); + //} +} + +void CXListener::on_GMACMessageOrderDepthV3(const GMACMessageOrderDepthV3*) +{ +} + + +void CXListener::dequeue_securities() +{ +} + +void CXListener::flush_books() +{ + // This is called at the end of the packet + // Write all book updates to shared memory and reset for the next packet + //for (auto& entry : _book_updates) { + // // bbo first + // uint16_t sequence = (_latest_sequence & 0xFFFF); + // entry.handle.update_bbo(_packet_arrived, entry.bids[0], entry.asks[0], sequence); + + // // Perf Note: the remainder of the book update could be done (carefully) with a memcpy + // for (size_t x = 1; x < COCONUT_MAX_DEPTH; ++x) { + // entry.handle.update_bid(_packet_arrived, entry.bids[x], x); + // entry.handle.update_ask(_packet_arrived, entry.asks[x], x); + // } + //} + //// Reset pending updates + //_book_updates.clear(); +} + +} + +md::CXListener::BookData::BookData(md2::MDProducerHandle & handle) : handle(handle), bids(), asks() +{ +} + +}//namespace + diff --git a/src/server/sparkd/md/platforms/celoxica/CXListener.h b/src/server/sparkd/md/platforms/celoxica/CXListener.h new file mode 100644 index 000000000..1e24823db --- /dev/null +++ b/src/server/sparkd/md/platforms/celoxica/CXListener.h @@ -0,0 +1,61 @@ +#pragma once + +#include "lib/utils/bts_no_copy.h" +#include "lib/md/coconut2.h" +#include "lib/utils/Dispatcher.h" +#include "server/sparkd/md/platforms/celoxica/CXSecurityIndex.h" + +#include +#include +#include + +namespace bts +{ +namespace md +{ +struct GMACMessageOrderDepthV3; + +class CXListener +{ +public: + CXListener(CXSecurityIndex&, md2::MDProducer&, md2::SecurityIDRecorder&, + co2::SettlementRecorder&, std::vector, utils::Dispatcher&); + ~CXListener(); + + void add_software_filter(const std::string& symbol); + + void on_GMACMessageOrderDepthV3(const GMACMessageOrderDepthV3*); + +private: + void flush_books(); + void dequeue_securities(); + +private: + BTS_NO_COPY(CXListener); + + CXSecurityIndex& _index; + md2::MDProducer& _producer; + utils::TimeStamp _packet_arrived; + uint64_t _latest_sequence; + md2::SecurityIDRecorder& _security_recorder; + co2::SettlementRecorder& _settlement_recorder; + std::vector _symbol_filter_storage; + std::vector _channels; + utils::Dispatcher& _background; + utils::DispatcherTimer _security_recorder_input_timer; + tbb::concurrent_queue _security_recorder_input; + std::atomic _concurrency_monitor; + + struct BookData { + BookData(md2::MDProducerHandle& handle); + md2::MDProducerHandle handle; + md2::PriceSize bids[10]; + md2::PriceSize asks[10]; + }; + + std::vector _book_updates; +}; + + +}}//namespace + diff --git a/src/server/sparkd/md/platforms/celoxica/CXMain.cpp b/src/server/sparkd/md/platforms/celoxica/CXMain.cpp new file mode 100644 index 000000000..3c4c9491a --- /dev/null +++ b/src/server/sparkd/md/platforms/celoxica/CXMain.cpp @@ -0,0 +1,83 @@ +#include "CXMain.h" + +#include "../mdp3_onixs/MDP3Utils.h" +#include "lib/config/config.h" +#include "lib/md/coconut2.h" +#include "lib/os/os.h" +#include "lib/utils/Dispatcher.h" +#include "lib/utils/proto_helper.h" +#include "lib/utils/helper_macros.h" + +#include "lib/config/protos/MarketData.pb.h" +#include "lib/config/protos/system.pb.h" + +namespace celoxica { namespace gmac {} } +#include "celoxica.h" + +extern "C" +{ +#define __retcode_start__ +#define __retcode_end__ +} + +#include +#include + +using namespace std; +using namespace bts::md2; +using namespace celoxica::gmac; + +namespace bts +{ +namespace md +{ + +void run_cx_server(UserSymbol* psyms, size_t symcount) +{ + CXMain main; + vector symbols; + for(size_t x = 0; x < symcount; ++x) { + symbols.push_back(psyms[x]); + } + main.run(symbols); +} + +CXMain::CXMain() : + _producer(Platform::CELOXICA), + _security_recorder(), + _settlement_recorder(Platform::CELOXICA), + _listeners() +{ +} + +CXMain::~CXMain() {} + +void CXMain::initialize() +{ + celoxica::gmac::GMACIdentify(stderr); + //const auto& options = config::get(); + +} + + +void CXMain::run(std::vector& symbols) +{ + bts::md::await_cme_config(); + initialize(); + + const auto& channels = parse_channels(symbols); +} + + + +std::unique_ptr CXMain::prepare_listener(const std::string& channel, const CMEChannel&) +{ + UNUSED(channel); + return std::unique_ptr(); +} + + +} +} + + diff --git a/src/server/sparkd/md/platforms/celoxica/CXMain.h b/src/server/sparkd/md/platforms/celoxica/CXMain.h new file mode 100644 index 000000000..83e56df9a --- /dev/null +++ b/src/server/sparkd/md/platforms/celoxica/CXMain.h @@ -0,0 +1,49 @@ +#pragma once + +#include "lib/utils/bts_no_copy.h" +#include "lib/md/coconut2.h" +#include "server/sparkd/md/platforms/celoxica/CXListener.h" +#include "server/sparkd/md/platforms/celoxica/CXSecurityIndex.h" + +#include +#include +#include + +namespace bts +{ +namespace md +{ +void run_cx_server(md2::UserSymbol* sym, size_t symcount); + +struct CMEChannel; +class CXListener; + +/* + * The CX provider is very simple. 1 CXMain maps to 1 Celoxica Queue, which maps to 1 thread and one bound CPU. + * Control starts in CXMain which starts the thread, binds to a cpu, provides a (currently unused) list of all the symbols we need + * prepare_handler configures an exchange channel to be owned by this queue, and prepare_listener sets up thing that sees + * all callbacks which maps the exchange id to the MDProducerHandle and writes to shared mem + */ +class CXMain +{ +public: + CXMain(); + ~CXMain(); + + void run(std::vector&); //call initialize, start the pump + void initialize(); //set up channels, etc + + std::unique_ptr prepare_listener(const std::string& channel, const CMEChannel&); + +private: + BTS_NO_COPY(CXMain); + + CXSecurityIndex _index; + md2::MDProducer _producer; + md2::SecurityIDRecorder _security_recorder; + co2::SettlementRecorder _settlement_recorder; + std::vector> _listeners; +}; + +}}//namespace + diff --git a/src/server/sparkd/md/platforms/celoxica/CXSecurityIndex.h b/src/server/sparkd/md/platforms/celoxica/CXSecurityIndex.h new file mode 100644 index 000000000..0c9f201b5 --- /dev/null +++ b/src/server/sparkd/md/platforms/celoxica/CXSecurityIndex.h @@ -0,0 +1,21 @@ +#pragma once + +#include "lib/md/SecurityIndex.h" +#include + +namespace bts +{ +namespace md +{ + +struct CXSecurityData +{ + CXSecurityData(uint32_t id, md2::MDProducerHandle& h) : id(id), handle(h) {} + uint32_t id; // exchange id + md2::MDProducerHandle handle; +}; + +using CXSecurityIndex = md2::ConcurrentSecurityIndex; + +}}//namespace + diff --git a/src/server/sparkd/md/platforms/celoxica/celoxica.h b/src/server/sparkd/md/platforms/celoxica/celoxica.h new file mode 100644 index 000000000..209383fa1 --- /dev/null +++ b/src/server/sparkd/md/platforms/celoxica/celoxica.h @@ -0,0 +1,15 @@ +#pragma once + +namespace celoxica +{ +namespace gmac +{ + +#include "celoxica/gmac.h" +#include "celoxica/gmac_template_v3.h" +#include "celoxica/gmac_info.h" + +} +} + + diff --git a/src/server/sparkd/md/platforms/celoxica/tests/cx_tests.cpp b/src/server/sparkd/md/platforms/celoxica/tests/cx_tests.cpp new file mode 100644 index 000000000..0358d1741 --- /dev/null +++ b/src/server/sparkd/md/platforms/celoxica/tests/cx_tests.cpp @@ -0,0 +1,210 @@ +// #include "catch/catch.hpp" + +// #include "lib/config/config.h" +// #include "lib/config/protos/MarketData.pb.h" + +// #include +// #include +// #include +// #include + +// #include "../MDP3Utils.h" +// #include "../onixs.h" + +// using namespace bts::md; +// using namespace std; + + +// namespace bts { +// namespace md { + +// map create_feed_engine_settings(const std::vector& channels); +// const CMEOptions::CMEChannel channel_settings_for(const string& id); + +// namespace test { + +// vector test_channels(int count, ...) { +// // Doing this the C way because I still haven't updated VS +// vector result; +// va_list ap; +// va_start(ap, count); +// for (int x = 0; x < count; ++x) { +// const char* val = va_arg(ap, char*); +// CMEChannel temp; +// temp.id = val; +// result.push_back(temp); +// } +// va_end(ap); +// return result; +// } + + +// TEST_CASE("mdp3_tests","[md]") { +// SECTION("test_defaults") { +// config::internal::initialize_for_test("CMEOptions {}"); + +// vector channels = test_channels(1, "310"); +// auto result = create_feed_engine_settings(channels); +// REQUIRE(result.size() == 0); + +// channels = test_channels(2, "310", "311"); +// result = create_feed_engine_settings(channels); +// REQUIRE(result.size() == 0); + +// channels = test_channels(4, "310", "311", "400", "412"); +// result = create_feed_engine_settings(channels); +// REQUIRE(result.size() == 0); + +// REQUIRE(!channel_settings_for("310").has_feed_engine()); +// REQUIRE(!channel_settings_for("311").has_feed_engine()); +// REQUIRE(!channel_settings_for("315").has_feed_engine()); +// } + +// SECTION("test_segregate_futures") { +// config::internal::initialize_for_test( +// "CMEOptions {\n" +// " value {\n" +// " feed_engine {\n" +// " name: 'default'\n" +// " }\n" +// " feed_engine {\n" +// " name: 'futures'\n" +// " num_threads: 4\n" +// " }\n" +// " # Futures\n" +// " channel {\n" +// " id: '310'\n" +// " id: '312'\n" +// " feed_engine: 'futures'\n" +// " }\n" +// " # Options\n" +// " channel {\n" +// " id: '311'\n" +// " id: '313'\n" +// " id: '315'\n" +// " feed_engine: 'default'\n" +// " }\n" +// " }\n" +// "}"); + +// vector channels = test_channels(1, "310"); +// auto result = create_feed_engine_settings(channels); +// REQUIRE(result.size() == 1); +// REQUIRE(result.begin()->first == "futures"); +// REQUIRE(result.begin()->second.threadsCount() == 4); + +// channels = test_channels(1, "311"); +// result = create_feed_engine_settings(channels); +// REQUIRE(result.size() == 1); +// REQUIRE(result.begin()->first == "default"); +// REQUIRE(result.begin()->second.threadsCount() == 1); + +// channels = test_channels(2, "311", "310"); +// result = create_feed_engine_settings(channels); +// REQUIRE(result.size() == 2); +// REQUIRE(result["default"].threadsCount() == 1); +// REQUIRE(result["futures"].threadsCount() == 4); + +// // Test channels not listed in config +// channels = test_channels(2, "317", "318"); +// result = create_feed_engine_settings(channels); +// REQUIRE(result.size() == 0); +// } + +// SECTION("test_no_default") { +// config::internal::initialize_for_test( +// "CMEOptions {\n" +// " value {\n" +// " feed_engine {\n" +// " name: 'futures'\n" +// " num_threads: 4\n" +// " }\n" +// " # any channel not listed will use the default feed engine\n" +// " # Futures\n" +// " channel {\n" +// " id: '310'\n" +// " id: '312'\n" +// " feed_engine: 'futures'\n" +// " }\n" +// " # Options\n" +// " channel {\n" +// " id: '311'\n" +// " id: '313'\n" +// " id: '315'\n" +// " use_tcp_replay: false\n" +// " }\n" +// " }\n" +// "}"); + +// vector channels = test_channels(1, "310"); +// auto result = create_feed_engine_settings(channels); +// REQUIRE(result.size() == 1); +// REQUIRE(result.begin()->first == "futures"); +// REQUIRE(result.begin()->second.threadsCount() == 4); + +// channels = test_channels(1, "311"); +// result = create_feed_engine_settings(channels); +// REQUIRE(result.size() == 0); + +// channels = test_channels(2, "311", "310"); +// result = create_feed_engine_settings(channels); +// REQUIRE(result.size() == 1); +// REQUIRE(result["futures"].threadsCount() == 4); + + +// REQUIRE(channel_settings_for("310").use_tcp_replay() == true); +// REQUIRE(channel_settings_for("311").use_tcp_replay() == false); +// REQUIRE(channel_settings_for("315").use_tcp_replay() == false); +// } + +// SECTION("test_one_thread_each") { +// config::internal::initialize_for_test( +// "CMEOptions {\n" +// " value {\n" +// " channel {\n" +// " id: '*'\n" +// " }\n" +// " }\n" +// "}\n"); + +// vector channels = test_channels(1, "310", "311"); +// auto result = create_feed_engine_settings(channels); +// REQUIRE(result.size() == 0); + +// const auto& settings = channel_settings_for("311"); +// REQUIRE(!settings.has_feed_engine()); +// } + +// SECTION("test_common_futures_thread") { +// config::internal::initialize_for_test( +// "CMEOptions {\n" +// " value {\n" +// " feed_engine {\n" +// " name: 'futures'\n" +// " num_threads: 4\n" +// " }\n" +// " channel {\n" +// " id: '310'\n" +// " feed_engine: 'futures'\n" +// " udp_socket_buffer_size: 1234\n" +// " }\n" +// " channel {\n" +// " id: '*'\n" +// " }\n" +// " }\n" +// "}\n"); + +// vector channels = test_channels(6, "310", "311", "315", "100", "10", "42"); +// auto result = create_feed_engine_settings(channels); +// REQUIRE(result.size() == 1); +// result = create_feed_engine_settings(test_channels(1, "315")); +// REQUIRE(result.size() == 0); + +// REQUIRE(!channel_settings_for("311").has_feed_engine()); +// REQUIRE(channel_settings_for("310").has_feed_engine()); +// REQUIRE(channel_settings_for("310").udp_socket_buffer_size() == 1234); +// REQUIRE(channel_settings_for("311").has_udp_socket_buffer_size() == false); +// } +// } + +// }}}//namespace diff --git a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Utils.cpp b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Utils.cpp index 90221c25d..3b3fc12d3 100644 --- a/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Utils.cpp +++ b/src/server/sparkd/md/platforms/mdp3_onixs/MDP3Utils.cpp @@ -1,128 +1,127 @@ -// #ifndef NO_CPP0X -// #include "MDP3Utils.h" -// #include "onixs.h" + #ifndef NO_CPP0X + #include "MDP3Utils.h" + + #include "lib/config/config.h" + #include "lib/utils/datetime.h" + #include "lib/utils/bts_chrono.h" -// #include "lib/config/config.h" -// #include "lib/utils/datetime.h" -// #include "lib/utils/bts_chrono.h" + #include "lib/config/protos/MarketData.pb.h" + #include "lib/config/protos/system.pb.h" -// #include "lib/config/protos/MarketData.pb.h" -// #include "lib/config/protos/system.pb.h" + #include + #include -// #include -// #include + using namespace std; + using namespace bts::md2; -// using namespace std; -// using namespace bts::md2; + namespace bts { + namespace md { -// namespace bts { -// namespace md { + string data_path() { + auto p = config::get().data_path() + "cme_mdp3/"; + #ifdef _WIN32 + // OnixS requires win32 paths to look like Win32 paths - everyone else in the world is ok with '/' + os::normalize_path(p, '\\'); + #else + os::normalize_path(p); + #endif + return p; + } + const string config_path() { return data_path() + "config.xml"; } + const string template_path() { return data_path() + "templates.xml"; } -// string data_path() { -// auto p = config::get().data_path() + "cme_mdp3/"; -// #ifdef _WIN32 -// // OnixS requires win32 paths to look like Win32 paths - everyone else in the world is ok with '/' -// os::normalize_path(p, '\\'); -// #else -// os::normalize_path(p); -// #endif -// return p; -// } -// const string config_path() { return data_path() + "config.xml"; } -// const string template_path() { return data_path() + "templates.xml"; } + bool check_file_exists_with_instructions(const string& filename, const string& url, bool log) { + if (!boost::filesystem::exists(filename)) { + if (log) { + LOG(MarketData,Error, + "File {} is not present on this system. " + "Please download it from the CME ftp site. " + "I will wait until the file is present.\n$ curl {} > {} ", + filename, url, filename); + } + return false; + } + return true; + } -// bool check_file_exists_with_instructions(const string& filename, const string& url, bool log) { -// if (!boost::filesystem::exists(filename)) { -// if (log) { -// LOG(MarketData,Error, -// "File {} is not present on this system. " -// "Please download it from the CME ftp site. " -// "I will wait until the file is present.\n$ curl {} > {} ", -// filename, url, filename); -// } -// return false; -// } -// return true; -// } + // For exponential backoff on logging this return true if x is a power of 2 or 0 + bool is_pow_two_or_0(uint32_t x) { return (x & (~x + 1)) == x; } -// // For exponential backoff on logging this return true if x is a power of 2 or 0 -// bool is_pow_two_or_0(uint32_t x) { return (x & (~x + 1)) == x; } + void await_cme_config() { + const auto config_url = "ftp://ftp.cmegroup.com/SBEFix/Production/Configuration/config.xml"; + const auto templates_url = "ftp://ftp.cmegroup.com/SBEFix/Production/Templates/templates_FixBinary.xml"; -// void await_cme_config() { -// const auto config_url = "ftp://ftp.cmegroup.com/SBEFix/Production/Configuration/config.xml"; -// const auto templates_url = "ftp://ftp.cmegroup.com/SBEFix/Production/Templates/templates_FixBinary.xml"; + boost::filesystem::create_directories(data_path()); -// boost::filesystem::create_directories(data_path()); + // Check config is present and give instructions for downloading it if not + // If the file is not present log an error and wait a second. Continue to log the error ever power of 2 seconds until the file appears or the process is killed. + uint32_t count = 0; + while(!check_file_exists_with_instructions(config_path(), config_url, is_pow_two_or_0(count))) { + ++count; + std::this_thread::sleep_for(1s); + } + count = 0; + while(!check_file_exists_with_instructions(template_path(), templates_url, is_pow_two_or_0(count))) { + ++count; + std::this_thread::sleep_for(1s); + } -// // Check config is present and give instructions for downloading it if not -// // If the file is not present log an error and wait a second. Continue to log the error ever power of 2 seconds until the file appears or the process is killed. -// uint32_t count = 0; -// while(!check_file_exists_with_instructions(config_path(), config_url, is_pow_two_or_0(count))) { -// ++count; -// std::this_thread::sleep_for(1s); -// } -// count = 0; -// while(!check_file_exists_with_instructions(template_path(), templates_url, is_pow_two_or_0(count))) { -// ++count; -// std::this_thread::sleep_for(1s); -// } + // If we waited for someone to download the file give it a second to be finished downloading + if (count > 0) { + std::this_thread::sleep_for(1s); + } + } -// // If we waited for someone to download the file give it a second to be finished downloading -// if (count > 0) { -// std::this_thread::sleep_for(1s); -// } -// } - -// SecurityType::type infer_security_type(const string& label) { -// if (label.find("Option") != std::string::npos) { -// return SecurityType::OPTION; -// } + SecurityType::type infer_security_type(const string& label) { + if (label.find("Option") != std::string::npos) { + return SecurityType::OPTION; + } -// if (label.find("Future") != std::string::npos) { -// return SecurityType::FUTURE; -// } + if (label.find("Future") != std::string::npos) { + return SecurityType::FUTURE; + } -// // TODO: Allow configuration to override this? -// LOG(MarketData,Warn,"Unable to guess security type from label: '{}" -// "' Treating as OPTION.",label); -// return SecurityType::OPTION; -// } + // TODO: Allow configuration to override this? + LOG(MarketData,Warn,"Unable to guess security type from label: '{}" + "' Treating as OPTION.",label); + return SecurityType::OPTION; + } -// const vector parse_channels(const vector& symbols) { -// vector result; + const vector parse_channels(const vector& symbols) { + vector result; -// pugi::xml_document doc; -// bts_assert_msg(doc.load_file(config_path().c_str()), "Error parsing config.xml as xml"); + pugi::xml_document doc; + bts_assert_msg(doc.load_file(config_path().c_str()), "Error parsing config.xml as xml"); -// auto configuration = doc.root().child("configuration"); -// bts_assert(configuration); + auto configuration = doc.root().child("configuration"); + bts_assert(configuration); -// LOG(MarketData,Info,"CME: Loaded '{}' config. Last updated: {}" , -// configuration.attribute("environment").as_string(), -// configuration.attribute("updated").as_string()); -// auto channels = configuration.children("channel"); -// for (const auto& chan : channels) { -// CMEChannel channel; -// channel.id = chan.attribute("id").as_string(); -// auto type = infer_security_type(chan.attribute("label").as_string()); -// if (!(channel.id.empty())) { -// auto product_list = chan.child("products").children("product"); -// for (const auto& product : product_list) { -// auto symbol = product.attribute("code").as_string(); -// for (auto s : symbols) { -// if ((s.type() == type || s.type() == SecurityType::SPREAD) && s.symbol() == symbol) { -// channel.symbols.insert(s); -// } -// } -// } -// if (!channel.symbols.empty()) { -// result.push_back(channel); -// } -// } -// } + LOG(MarketData,Info,"CME: Loaded '{}' config. Last updated: {}" , + configuration.attribute("environment").as_string(), + configuration.attribute("updated").as_string()); + auto channels = configuration.children("channel"); + for (const auto& chan : channels) { + CMEChannel channel; + channel.id = chan.attribute("id").as_string(); + auto type = infer_security_type(chan.attribute("label").as_string()); + if (!(channel.id.empty())) { + auto product_list = chan.child("products").children("product"); + for (const auto& product : product_list) { + auto symbol = product.attribute("code").as_string(); + for (auto s : symbols) { + if ((s.type() == type || s.type() == SecurityType::SPREAD) && s.symbol() == symbol) { + channel.symbols.insert(s); + } + } + } + if (!channel.symbols.empty()) { + result.push_back(channel); + } + } + } -// return result; -// } + return result; + } -// }}//namespace -// #endif + }}//namespace + #endif diff --git a/src/server/sparkd/sparkd.vcxproj b/src/server/sparkd/sparkd.vcxproj index a6aa86fe4..9cc0c9b1d 100644 --- a/src/server/sparkd/sparkd.vcxproj +++ b/src/server/sparkd/sparkd.vcxproj @@ -12,6 +12,12 @@ + + + + + + @@ -37,6 +43,9 @@ + + + @@ -88,6 +97,13 @@ + + + + + + + @@ -136,6 +152,10 @@ + + + + diff --git a/src/server/sparkd/sparkd.vcxproj.filters b/src/server/sparkd/sparkd.vcxproj.filters index 8a0f07775..715b0193b 100644 --- a/src/server/sparkd/sparkd.vcxproj.filters +++ b/src/server/sparkd/sparkd.vcxproj.filters @@ -103,6 +103,15 @@ {831b8693-c426-4cdc-8f3e-53d080313920} + + {e133c7a8-a291-4beb-b420-80ba155dccac} + + + {92796b35-dd63-400a-af56-620ce1d59d7a} + + + {163d7d32-ac34-492f-8c7d-251d8f998e75} + @@ -324,6 +333,34 @@ securities + + + md\platforms\celoxica + + + md\platforms\celoxica + + + md\platforms\celoxica\tests + + + cmds + + + cmds + + + cmds + + + cmds + + + cmds + + + cmds + @@ -575,9 +612,6 @@ user - - orderman\utils - md\platforms\mdp3_onixs @@ -695,6 +729,40 @@ securities + + + md\platforms\celoxica + + + md\platforms\celoxica + + + md\platforms\celoxica + + + md\platforms\celoxica + + + cmds + + + cmds + + + cmds + + + cmds + + + cmds + + + cmds + + + cmds + @@ -710,5 +778,7 @@ orderman\doc - + + + \ No newline at end of file diff --git a/tools/celoxica.cmake b/tools/celoxica.cmake new file mode 100644 index 000000000..7f612b51b --- /dev/null +++ b/tools/celoxica.cmake @@ -0,0 +1,19 @@ +cmake_minimum_required(VERSION 3.5) + +LIST(APPEND CELOXICA_LIB "${SPARK_LIB}/celoxica/libbase.so") +LIST(APPEND CELOXICA_LIB "${SPARK_LIB}/celoxica/libutils.so") +LIST(APPEND CELOXICA_LIB "${SPARK_LIB}/celoxica/libfix_lib.so") +LIST(APPEND CELOXICA_LIB "${SPARK_LIB}/celoxica/libclxlog.so") +LIST(APPEND CELOXICA_LIB "${SPARK_LIB}/celoxica/libade_version.so") +LIST(APPEND CELOXICA_LIB "${SPARK_LIB}/celoxica/libade.so") +LIST(APPEND CELOXICA_LIB "${SPARK_LIB}/celoxica/libclx.so") +LIST(APPEND CELOXICA_LIB "${SPARK_LIB}/celoxica/libgmac_version.so") +LIST(APPEND CELOXICA_LIB "${SPARK_LIB}/celoxica/libgmac.so") +LIST(APPEND CELOXICA_LIB "${SPARK_LIB}/celoxica/libcputimestamp.so") +LIST(APPEND CELOXICA_LIB "${SPARK_LIB}/celoxica/libclxconfig.so") +LIST(APPEND CELOXICA_LIB "${SPARK_LIB}/celoxica/libconfigfile.so") +LIST(APPEND CELOXICA_LIB "${SPARK_LIB}/celoxica/libac.so") +LIST(APPEND CELOXICA_LIB "${SPARK_LIB}/celoxica/librc.so") +LIST(APPEND CELOXICA_LIB "${SPARK_LIB}/celoxica/liblldt.so") +LIST(APPEND CELOXICA_LIB "${SPARK_LIB}/celoxica/libregister.so") +