From 501cd93fd67a096a17d6e7d8d0cb58857e6ed466 Mon Sep 17 00:00:00 2001 From: Eric Harding Date: Mon, 7 Jan 2019 15:52:37 -0500 Subject: [PATCH] SPARK-1950 Integrate SecurityCache into MDConsumer --- .vscode/launch.json | 2 +- src/lib/md/MDConsumer.cpp | 43 ++++++++++++++----- src/lib/md/MDConsumer.h | 5 ++- src/lib/md/MDConsumerHandle.cpp | 20 +++++---- src/lib/md/SecurityCache.cpp | 35 +++++++-------- src/lib/md/SecurityCache.h | 4 +- .../md/test/synthetic_book_merge_tests.cpp | 3 ++ 7 files changed, 71 insertions(+), 41 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index d3f66bbaf..cf754974d 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -209,7 +209,7 @@ "type": "cppdbg", "request": "launch", "program": "/home/bts/spark/build/ninja_debug/src/server/sparkd/sparkd", - "args": ["test","AuthToken"], + "args": ["test", "[synthetic]"], "stopAtEntry": false, "cwd": "/home/bts/spark/build/ninja_debug/src/server/sparkd/", "environment": [], diff --git a/src/lib/md/MDConsumer.cpp b/src/lib/md/MDConsumer.cpp index 5cc1d2abf..3a3eb8ab6 100644 --- a/src/lib/md/MDConsumer.cpp +++ b/src/lib/md/MDConsumer.cpp @@ -2,6 +2,7 @@ #include "MDConsumer.h" #include "MemorySegmentUtils.h" #include "PlatformLoader.h" +#include "SecurityCache.h" #include "lib/config/config.h" #include "lib/utils/SharedList.h" #include "lib/utils/WaitFreeQueue.h" @@ -530,8 +531,27 @@ private: // Pimple for MDConsumer /////////////////////////////////////////////////////////////////////////////////////////////////// class MDConsumerImpl { + typedef tbb::queuing_mutex Mutex; + Mutex _workmutex; + vector> _instances; + std::shared_ptr _sci; + map> _deferred_handles; + vector _callbacks; + SecurityCache _cache; + // co2::Platform _platform; + public: - MDConsumerImpl(const set& symbols, std::shared_ptr& ss, bool shared, co2::Platform platform = co2::PLATFORM_UNSET ) { + MDConsumerImpl( + const set& symbols, + std::shared_ptr& ss, + bool shared, co2::Platform platform = co2::PLATFORM_UNSET) : + _workmutex(), + _instances(), + _sci(), + _deferred_handles(), + _callbacks(), + _cache() + { std::ostringstream msg; for (const auto& symbol : symbols) { if (msg.tellp() > 0) { msg << ", "; } @@ -563,6 +583,11 @@ public: if (ss != nullptr) { _sci = std::make_shared(ss, [=](const set& keys) { return get_handles(keys); }); } + + // Add the SecurityCache to on_security so it can cache discovered securities + _callbacks.push_back([this](const SecurityPtr& security, MDConsumerHandle) { + _cache.add_live_security(security); + }); } ~MDConsumerImpl() { } @@ -766,7 +791,7 @@ public: } count += resolved_securities.size(); } - else /*!resolve_legs*/ { + else /* do NOT resolve_legs*/ { auto handles = get_handles(found_securities); for (const auto& security : found_securities) { auto find_result = handles.find(security->id()); @@ -801,6 +826,7 @@ public: } } + _cache.flush_to_disk(); return count; } @@ -843,6 +869,10 @@ public: scan_securities(resolve_legs); } + SecurityPtr get_security(SecurityKey key) { + return _cache.get_security(key); + } + MDConsumerChannel get_channel(const std::string& symbol, SecurityType type, Venue exchange) const { for (auto& inst : _instances) { if ((inst->symbol() == symbol && inst->type() == type && inst->exchange() == exchange) || @@ -869,14 +899,6 @@ public: return result; } -private: - typedef tbb::queuing_mutex Mutex; - Mutex _workmutex; - vector> _instances; - std::shared_ptr _sci; - map> _deferred_handles; - vector _callbacks; - // co2::Platform _platform; }; @@ -922,6 +944,7 @@ size_t MDConsumer::walk_security_list(SecurityCallback cb, const string& symbol, size_t MDConsumer::scan_securities(bool resolve_legs) { return _pimpl->scan_securities(resolve_legs); } void MDConsumer::on_security(SecurityCallback cb, bool resolve_legs) { _pimpl->on_security(cb, resolve_legs); } +SecurityPtr MDConsumer::get_security(SecurityKey key) { return _pimpl->get_security(key); } MDConsumerChannel MDConsumer::get_channel(const std::string& symbol, SecurityType type, Venue exchange) const { return _pimpl->get_channel(symbol, type, exchange); diff --git a/src/lib/md/MDConsumer.h b/src/lib/md/MDConsumer.h index 85e74f4b9..b1ea4f304 100644 --- a/src/lib/md/MDConsumer.h +++ b/src/lib/md/MDConsumer.h @@ -19,7 +19,6 @@ namespace bts { namespace md { -/* typedef void (security_callback)(const Security& security); */ typedef std::function SecurityCallback; using std::size_t; @@ -52,7 +51,9 @@ public: size_t scan_securities(bool resolve_legs = true); void on_security(SecurityCallback cb, bool resolve_legs = true); - // SecurityPtr get_security(SecurityKey key); + // Retrieve any security that has been seen in the past + // You still need to call scan_securities() periodically to discover new securities + SecurityPtr get_security(SecurityKey key); MDConsumerChannel get_channel(const std::string& symbol, SecurityType type, Venue exchange) const; MDConsumerChannel get_channel(const UserSymbol&) const; diff --git a/src/lib/md/MDConsumerHandle.cpp b/src/lib/md/MDConsumerHandle.cpp index 890a599c4..891f2ee90 100644 --- a/src/lib/md/MDConsumerHandle.cpp +++ b/src/lib/md/MDConsumerHandle.cpp @@ -223,6 +223,17 @@ void copy_synthetic_side(size_t count, api::Side side, const api::ContractSpecif } } +const spark::LegSpreaderParams& get_spreader_params(const SyntheticStrategy& synth, security_key key) { + auto strategy = synth.strategy.get(); + if (strategy) { + auto iter = strategy->leg_params_map().find(key); + if (iter != strategy->leg_params_map().end()) { + return iter->second; + } + } + return spark::LegSpreaderParams::default_instance(); +} + TimeStamp copy_synthetic_depth(const SyntheticStrategy& synth, size_t count, PriceSize* bid, PriceSize* ask) { small_vector bid_leg_travs; small_vector ask_leg_travs; @@ -232,14 +243,7 @@ TimeStamp copy_synthetic_depth(const SyntheticStrategy& synth, size_t count, Pri for (size_t x = 0; x < synth.legs.size(); ++x) { std::array bids, asks; TimeStamp t; - auto key = synth.legs[x].key(); - auto strategy = synth.strategy.get(); - auto iter = strategy->leg_params_map().find(key); - const spark::LegSpreaderParams& leg_params = - iter == strategy->leg_params_map().end() ? - spark::LegSpreaderParams::default_instance() : - iter->second; - + const spark::LegSpreaderParams& leg_params = get_spreader_params(synth, synth.legs[x].key()); auto mdtype = leg_params.market_data_type(); HandleReadResult r; if (mdtype == spark::LegSpreaderParams::DIRECT) { diff --git a/src/lib/md/SecurityCache.cpp b/src/lib/md/SecurityCache.cpp index 2675128bb..57a370221 100644 --- a/src/lib/md/SecurityCache.cpp +++ b/src/lib/md/SecurityCache.cpp @@ -13,28 +13,23 @@ struct SecurityCacheImpl { std::mutex cache_mutex; std::unordered_map cache; tbb::concurrent_queue update_queue; - void update_db(const SecurityPtr&); + void enqueue_update_db(const SecurityPtr&); + void flush(); }; -void SecurityCacheImpl::update_db(const SecurityPtr& security) { - // Batch updates to the database for performance - // algo: - // push item into queue - // post work to dispatcher with delay - // When dispatcher services request drain update_queue. - // This will cause one update per security but most will be no-op - // This is a little simpler than running a timer and trying to stop it after security discovery is done +void SecurityCacheImpl::enqueue_update_db(const SecurityPtr& security) { update_queue.push(security); - utils::Dispatcher::background().post([this]() { - if (!update_queue.empty()) { - std::vector work; - SecurityPtr temp; - while(update_queue.try_pop(temp)) { work.push_back(temp); } - if (!work.empty()) { - db::insert_or_replace(work); - } +} + +void SecurityCacheImpl::flush() { + if (!update_queue.empty()) { + std::vector work; + SecurityPtr temp; + while(update_queue.try_pop(temp)) { work.push_back(temp); } + if (!work.empty()) { + db::insert_or_replace(work); } - }, 1000); + } } ////////////////// @@ -83,8 +78,10 @@ bool SecurityCache::add_live_security(const SecurityPtr& security) { SecurityKey key(security->id()); // Insert but don't overwrite existing auto result = _me->cache.insert(std::make_pair(SecurityKey(security->id()), security)); - _me->update_db(security); + _me->enqueue_update_db(security); return result.second; } +void SecurityCache::flush_to_disk() { _me->flush(); } + }}//ns \ No newline at end of file diff --git a/src/lib/md/SecurityCache.h b/src/lib/md/SecurityCache.h index 388d66fd8..85de41b70 100644 --- a/src/lib/md/SecurityCache.h +++ b/src/lib/md/SecurityCache.h @@ -22,13 +22,15 @@ public: //Load security from cache with fallback to database SecurityPtr get_security(SecurityKey key); - // Get the security if it's in the cache otherwise return nullptr + // Get the security if it's in the cache otherwise return nullptr. Does not query security database. SecurityPtr try_get_security(SecurityKey key) const; // returns true if the cache was updated bool add_live_security(const SecurityPtr& security); SecurityPtr add_db_security(const SecurityPtr& security); + void flush_to_disk(); + }; }}//ns \ No newline at end of file diff --git a/src/lib/md/test/synthetic_book_merge_tests.cpp b/src/lib/md/test/synthetic_book_merge_tests.cpp index 822ebb907..87e8ce79a 100644 --- a/src/lib/md/test/synthetic_book_merge_tests.cpp +++ b/src/lib/md/test/synthetic_book_merge_tests.cpp @@ -135,6 +135,9 @@ tuple,vector> make_securities(c void set_leg_params(SyntheticStrategy& strat, uint32_t security_key, const spark::LegSpreaderParams& params) { + if (!strat.strategy) { + strat.strategy = std::make_unique(); + } auto* map = strat.strategy->mutable_leg_params_map(); map->insert({ security_key, params }); }