SPARK-1950 Integrate SecurityCache into MDConsumer

This commit is contained in:
Eric Harding committed 2019-01-07 15:52:37 -05:00
1 parent 7a6739c156
commit 501cd93fd6
7 files changed
+71 -41

No files matched your search

+16 -19
View File
@@ -13,28 +13,23 @@ struct SecurityCacheImpl {
std::mutex cache_mutex;
std::unordered_map<SecurityKey, SecurityPtr> cache;
tbb::concurrent_queue<SecurityPtr> update_queue;
void update_db(const SecurityPtr&);
void enqueue_update_db(const SecurityPtr&);
void flush();
};
void SecurityCacheImpl::update_db(const SecurityPtr& security) {
// Batch updates to the database for performance
// algo:
// push item into queue
// post work to dispatcher with delay
// When dispatcher services request drain update_queue.
// This will cause one update per security but most will be no-op
// This is a little simpler than running a timer and trying to stop it after security discovery is done
void SecurityCacheImpl::enqueue_update_db(const SecurityPtr& security) {
update_queue.push(security);
utils::Dispatcher::background().post([this]() {
if (!update_queue.empty()) {
std::vector<SecurityPtr> work;
SecurityPtr temp;
while(update_queue.try_pop(temp)) { work.push_back(temp); }
if (!work.empty()) {
db::insert_or_replace(work);
}
}
void SecurityCacheImpl::flush() {
if (!update_queue.empty()) {
std::vector<SecurityPtr> work;
SecurityPtr temp;
while(update_queue.try_pop(temp)) { work.push_back(temp); }
if (!work.empty()) {
db::insert_or_replace(work);
}
}, 1000);
}
}
//////////////////
@@ -83,8 +78,10 @@ bool SecurityCache::add_live_security(const SecurityPtr& security) {
SecurityKey key(security->id());
// Insert but don't overwrite existing
auto result = _me->cache.insert(std::make_pair(SecurityKey(security->id()), security));
_me->update_db(security);
_me->enqueue_update_db(security);
return result.second;
}
void SecurityCache::flush_to_disk() { _me->flush(); }
}}//ns