improvement: Perform sec lookup in const time

This commit is contained in:
John Colvin committed 2024-12-21 20:08:19 +00:00
1 parent f927ef2c5e
commit 25439b094a
6 files changed
+338 -74

No files matched your search

+172 -73
View File
@@ -3,6 +3,8 @@
#include "lib/domain_model/Security.h"
#include "lib/domain_model/Security.pb.h"
#include "lib/logging/logging.h"
#include "lib/md/security_type_index.h"
#include "lib/md/venue_index.h"
#include "lib/utils/Dispatcher.h"
#include "lib/utils/LockBox.h"
@@ -14,9 +16,152 @@
namespace bts {
namespace md {
using std::string;
using std::unordered_map;
struct SecurityCacheIndex {
btree::map<SecurityKey, SecurityPtr> cache;
btree::map<ExchangeSecurityKey, SecurityPtr> ex_cache;
VenueIndex<SecurityTypeIndex<unordered_map<string, uint32_t>>>
venue_type_ids;
const bts::api::Security *get_security(
api::Venue venue, const std::string &desc) {
auto id = get_id_(venue, desc);
auto sec_it = cache.find(SecurityKey{id});
if (sec_it == cache.end()) {
return nullptr;
}
return sec_it->second.get();
}
const bts::api::Security *get_security(
api::Venue venue, api::SecurityType type, const std::string &desc) {
auto id = get_id_(venue, type, desc);
auto sec_it = cache.find(SecurityKey{id});
if (sec_it == cache.end()) {
return nullptr;
}
return sec_it->second.get();
}
const Security *try_get_security(SecurityKey key) const {
auto iter = cache.find(key);
if (iter != cache.end()) {
return iter->second.get();
}
return (const api::Security *)nullptr;
}
const Security *try_get_security(ExchangeSecurityKey key) const {
auto iter = ex_cache.find(key);
if (iter != ex_cache.end()) {
return iter->second.get();
}
return (const api::Security *)nullptr;
}
std::vector<const Security *> get_securities(
api::Venue venue, api::SecurityType type, const std::string &name) {
std::vector<api::Security const *> result;
const bool all_types = type == api::SECURITYTYPE_UNSET;
const bool all_venues = venue == api::VENUE_UNSET;
const bool all_names = name.empty();
for (auto const &p : cache) {
const auto &s = *p.second;
if ((all_types ||
type == securities::security_type(s.Contract_case())) &&
(all_venues || venue == s.exchange()) &&
(all_names || name == securities::descriptor(s))) {
result.push_back(p.second.get());
}
}
return result;
}
std::vector<const Security *> get_securities(
std::function<bool(const Security &)> include) {
std::vector<api::Security const *> result;
for (auto const &p : cache) {
auto &s = *p.second;
if (include(s)) {
result.push_back(&s);
}
}
return result;
}
bool add_live_security(SecurityPtr security) {
bts_assert(security);
if (security->has_glbx()) {
ExchangeSecurityKey key(security->id());
ex_cache.insert(std::make_pair(key, security));
}
SecurityKey key(security->id());
// Insert but don't overwrite existing
auto result =
cache.insert(std::make_pair(SecurityKey(security->id()), security));
return result.second;
}
const Security *add_db_security(SecurityPtr security) {
bts_assert(security);
if (security->has_glbx()) {
ExchangeSecurityKey key(security->id());
ex_cache.insert(std::make_pair(key, security));
}
SecurityKey key(security->id());
auto it = cache.insert(std::make_pair(key, security));
const Security *active_sec = it.first->second.get();
// If the security was just added, active_sec is equal to security.
// If it was already present, it must be a recently inserted 'live'
// security and we should use it (instead of the db security).
venue_type_ids.get_or_add(security->exchange())
.get_or_add(bts::securities::security_type(*active_sec))
.try_emplace(
bts::securities::descriptor(*active_sec), active_sec->id());
return active_sec;
}
private:
std::uint32_t get_id_(api::Venue venue, const std::string &desc) {
auto id = get_id_(venue, api::SECURITYTYPE_SPOT, desc);
if (id != 0) {
return id;
}
id = get_id_(venue, api::SECURITYTYPE_FUTURE, desc);
if (id != 0) {
return id;
}
id = get_id_(venue, api::SECURITYTYPE_OPTION, desc);
if (id != 0) {
return id;
}
id = get_id_(venue, api::SECURITYTYPE_SPREAD, desc);
if (id != 0) {
return id;
}
id = get_id_(venue, api::SECURITYTYPE_SYNTHETIC, desc);
if (id != 0) {
return id;
}
return 0;
}
std::uint32_t get_id_(
api::Venue venue, api::SecurityType type, const std::string &desc) {
auto &type_store = venue_type_ids.get_or_add(venue);
auto &ids = type_store.get_or_add(type);
auto it = ids.find(desc);
if (it != ids.end()) {
return it->second;
}
return 0;
}
};
struct SecurityCacheImpl {
utils::LockBox<btree::map<SecurityKey, SecurityPtr>> cache;
utils::LockBox<btree::map<ExchangeSecurityKey, SecurityPtr>> ex_cache;
utils::LockBox<SecurityCacheIndex> cache;
tbb::concurrent_queue<SecurityPtr> update_queue;
void enqueue_update_db(const SecurityPtr &);
void flush();
@@ -53,23 +198,13 @@ SecurityCache::SecurityCache() : _me(std::make_shared<SecurityCacheImpl>()) {}
SecurityCache::~SecurityCache() {}
const Security *SecurityCache::try_get_security(SecurityKey key) const {
return _me->cache.lock([key](btree::map<SecurityKey, SecurityPtr> &t) {
auto iter = t.find(key);
if (iter != t.end()) {
return iter->second.get();
}
return (const api::Security *)nullptr;
});
return _me->cache.lock(
[key](SecurityCacheIndex &t) { return t.try_get_security(key); });
}
const Security *SecurityCache::try_get_security(ExchangeSecurityKey key) const {
return _me->ex_cache.lock(
[key](btree::map<ExchangeSecurityKey, SecurityPtr> &t) {
auto iter = t.find(key);
if (iter != t.end()) {
return iter->second.get();
}
return (const api::Security *)nullptr;
});
return _me->cache.lock(
[key](SecurityCacheIndex &t) { return t.try_get_security(key); });
}
const Security *SecurityCache::get_security(SecurityKey key) {
@@ -88,47 +223,31 @@ const Security *SecurityCache::get_security(SecurityKey key) {
}
const Security *SecurityCache::get_security(
api::Venue venue, const std::string &name) {
return _me->cache.lock([venue, name](SecurityCacheIndex &cache) {
return cache.get_security(venue, name);
});
}
const Security *SecurityCache::get_security(
api::Venue venue, api::SecurityType type, const std::string &name) {
const Security *result = nullptr;
_me->cache.lock([&result, venue, type, name](auto &cache) {
for (auto const &p : cache) {
if ((type == api::SECURITYTYPE_UNSET ||
type == securities::security_type(p.second->Contract_case())) &&
(venue == api::VENUE_UNSET && venue == p.second->exchange()) &&
(name.empty() || name == securities::descriptor(*p.second))) {
result = p.second.get();
break;
}
}
return _me->cache.lock([venue, type, name](SecurityCacheIndex &cache) {
return cache.get_security(venue, type, name);
});
return result;
}
std::vector<const Security *> SecurityCache::get_securities(
api::Venue venue, api::SecurityType type, const std::string &name) {
std::vector<api::Security const *> result;
_me->cache.lock([&result, venue, type, name](auto &cache) {
const bool all_types = type == api::SECURITYTYPE_UNSET;
const bool all_venues = venue == api::VENUE_UNSET;
const bool all_names = name.empty();
for (auto const &p : cache) {
const auto &s = *p.second;
if ((all_types ||
type == securities::security_type(s.Contract_case())) &&
(all_venues || venue == s.exchange()) &&
(all_names || name == securities::descriptor(s))) {
result.push_back(p.second.get());
}
}
return _me->cache.lock([venue, type, &name](SecurityCacheIndex &cache) {
return cache.get_securities(venue, type, name);
});
return result;
}
std::vector<const Security *> SecurityCache::get_securities(
std::function<bool(const Security &)> include) {
std::vector<api::Security const *> result;
_me->cache.lock([&result, include](auto &cache) {
for (auto const &p : cache) {
_me->cache.lock([&result, include](SecurityCacheIndex &cache) {
for (auto const &p : cache.cache) {
auto &s = *p.second;
if (include(s)) {
result.push_back(&s);
@@ -142,8 +261,8 @@ std::vector<api::Security const *> SecurityCache::get_securities() {
std::vector<api::Security const *> result;
result.reserve(100000);
_me->cache.lock([&result](auto &cache) {
for (auto const &p : cache) {
_me->cache.lock([&result](SecurityCacheIndex &cache) {
for (const auto &p : cache.cache) {
result.push_back(p.second.get());
}
});
@@ -154,19 +273,8 @@ const Security *SecurityCache::add_db_security(const SecurityPtr &security) {
bts_assert(security);
LOG(md, Info, "SecurityCache::add_db_security security: {}",
bts::securities::descriptor(*security));
if (security->has_glbx()) {
_me->ex_cache.lock([&security](auto &cache) {
ExchangeSecurityKey key(security->id());
cache.insert(std::make_pair(key, security));
});
}
return _me->cache.lock([&security](auto &cache) {
SecurityKey key(security->id());
auto result = cache.insert(std::make_pair(key, security));
// If result was inserted result.first->second is equal to security
// If the value was already present it must be a recently inserted 'live'
// security and we should use it
return result.first->second.get();
return _me->cache.lock([&security](SecurityCacheIndex &cache) {
return cache.add_db_security(security);
});
}
@@ -174,19 +282,10 @@ bool SecurityCache::add_live_security(const SecurityPtr &security) {
bts_assert(security);
LOG(md, Diag, "SecurityCache::add_live_security security: {}",
bts::securities::descriptor(*security));
if (security->has_glbx()) {
_me->ex_cache.lock([&security](auto &cache) {
ExchangeSecurityKey key(security->id());
cache.insert(std::make_pair(key, security));
});
}
return _me->cache.lock([&](auto &cache) {
SecurityKey key(security->id());
// Insert but don't overwrite existing
auto result =
cache.insert(std::make_pair(SecurityKey(security->id()), security));
return _me->cache.lock([&](SecurityCacheIndex &cache) {
const bool is_new_sec = cache.add_live_security(security);
_me->enqueue_update_db(security);
return result.second;
return is_new_sec;
});
}