Makd SecurityCache external to MDConsumer so it live beyond md recovery

This commit is contained in:
Eric Harding committed 2019-01-31 09:18:03 -05:00
1 parent 40a115de7e
commit 791b0fdbc1
13 files changed
+68 -40

No files matched your search

+3 -1
View File
@@ -1,5 +1,6 @@
#include "MDPeriodicDumper.h"
#include "lib/md/MDConsumer.h"
#include "lib/md/SecurityCache.h"
#include "lib/utils/TimeStamp.h"
#include "lib/utils/Dispatcher.h"
#include "lib/utils/datetime.h"
@@ -158,7 +159,8 @@ void run_periodic_dumper(const std::set<UserSymbol>& symbols, uint32_t seq_mod,
auto thread_name = "tail-" + qualifier;
threading::configure_current_thread(thread_name);
MDConsumer consumer(symbols, nullptr);
SecurityCache cache;
MDConsumer consumer(symbols, nullptr, cache);
// read all securities into a vector
std::vector<Entry> entries;
+4 -2
View File
@@ -1,5 +1,6 @@
#include "MDClient.h"
#include "get_symbols.h"
#include "SecurityCache.h"
#include <set>
namespace bts {
@@ -8,6 +9,7 @@ namespace md {
struct MDClientImpl{
utils::Dispatcher& _dispatcher;
std::shared_ptr<api::ISyntheticSecurityService> _ss;
SecurityCache _cache;
utils::DispatcherTimer _security_timer;
utils::DispatcherTimer _recovery_timer;
MDConsumer _consumer;
@@ -18,7 +20,7 @@ struct MDClientImpl{
_ss(s),
_security_timer(d, config::get<RelayOptions>().security_scan_interval(),[this]() { scan_securities(); }),
_recovery_timer(d, config::get<RelayOptions>().recovery_interval(), [this]() { check_heartbeats(); }),
_consumer(s,true),
_consumer(s, _cache),
_event()
{
_security_timer.start();
@@ -37,7 +39,7 @@ struct MDClientImpl{
void check_heartbeats() {
if (!_consumer.all_heartbeats_ok()) {
MDConsumer temp(symbols(), _ss);
MDConsumer temp(_ss,_cache);
if (temp.all_heartbeats_ok()) {
_consumer.swap(temp);
LOG(md,Info, "Recovering from bad MD. Old consumer {} new consumer {}", _consumer.id(), temp.id());
+12 -11
View File
@@ -537,20 +537,21 @@ class MDConsumerImpl {
std::shared_ptr<SyntheticConsumerInst> _sci;
map<security_key, shared_ptr<MDConsumerHandle>> _deferred_handles;
vector<SecurityCallback> _callbacks;
mutable SecurityCache _cache;
SecurityCache& _cache;
// co2::Platform _platform;
public:
MDConsumerImpl(
const set<UserSymbol>& symbols,
std::shared_ptr<api::ISyntheticSecurityService>& ss,
SecurityCache& cache,
bool shared, co2::Platform platform = co2::PLATFORM_UNSET) :
_workmutex(),
_instances(),
_sci(),
_deferred_handles(),
_callbacks(),
_cache()
_cache(cache)
{
std::ostringstream msg;
for (const auto& symbol : symbols) {
@@ -906,27 +907,27 @@ public:
// MDConsumer public interface, forwards to pimple
///////////////////////////////////////////////////////////////////////////////////////////////////
MDConsumer::MDConsumer(std::shared_ptr<api::ISyntheticSecurityService>& synt, bool shared) : _pimpl() {
MDConsumer::MDConsumer(std::shared_ptr<api::ISyntheticSecurityService>& synt, SecurityCache& cache, bool shared) : _pimpl() {
set<UserSymbol> all_symbols;
get_system_md_symbols(VENUE_UNSET, SECURITYTYPE_UNSET, all_symbols);
_pimpl.reset(new MDConsumerImpl(all_symbols, synt, shared, co2::PLATFORM_UNSET));
_pimpl.reset(new MDConsumerImpl(all_symbols, synt, cache, shared, co2::PLATFORM_UNSET));
}
MDConsumer::MDConsumer(const UserSymbol& s, std::shared_ptr<api::ISyntheticSecurityService> synt, bool shared) :
MDConsumer::MDConsumer(const UserSymbol& s, std::shared_ptr<api::ISyntheticSecurityService> synt, SecurityCache& cache, bool shared) :
_pimpl()
{
set<UserSymbol> symbols;
symbols.insert(s);
_pimpl.reset(new MDConsumerImpl(symbols, synt, shared, co2::PLATFORM_UNSET));
_pimpl.reset(new MDConsumerImpl(symbols, synt, cache, shared, co2::PLATFORM_UNSET));
}
MDConsumer::MDConsumer(set<UserSymbol> const& symbols, std::shared_ptr<api::ISyntheticSecurityService> s, bool shared, co2::Platform platform) :
_pimpl(new MDConsumerImpl(symbols, s, shared, platform)) {}
MDConsumer::MDConsumer(set<UserSymbol> const& symbols, std::shared_ptr<api::ISyntheticSecurityService> s, SecurityCache& cache, bool shared, co2::Platform platform) :
_pimpl(new MDConsumerImpl(symbols, s, cache, shared, platform)) {}
MDConsumer::MDConsumer(api::Venue ven, co2::Platform platform) : _pimpl() {
MDConsumer::MDConsumer(SecurityCache& cache, api::Venue ven, co2::Platform platform) : _pimpl() {
set<UserSymbol> all_symbols;
get_system_md_symbols(ven, SECURITYTYPE_UNSET, all_symbols);
std::shared_ptr<api::ISyntheticSecurityService> null = nullptr;
_pimpl.reset(new MDConsumerImpl(all_symbols, null, true, platform));
_pimpl.reset(new MDConsumerImpl(all_symbols, null, cache, true, platform));
}
MDConsumer::~MDConsumer() {}
@@ -944,7 +945,7 @@ size_t MDConsumer::walk_security_list(SecurityCallback cb, const string& symbol,
size_t MDConsumer::scan_securities(bool resolve_legs) { return _pimpl->scan_securities(resolve_legs); }
void MDConsumer::on_security(SecurityCallback cb, bool resolve_legs) { _pimpl->on_security(cb, resolve_legs); }
SecurityPtr MDConsumer::get_security(SecurityKey key) const { return _pimpl->get_security(key); }
const api::Security* MDConsumer::get_security(SecurityKey key) const { return _pimpl->get_security(key).get(); }
MDConsumerChannel MDConsumer::get_channel(const std::string& symbol, SecurityType type, Venue exchange) const {
return _pimpl->get_channel(symbol, type, exchange);
+7 -5
View File
@@ -25,15 +25,17 @@ using std::size_t;
class MDConsumerImpl;
class MDConsumerInst;
class MDConsumerChannel;
class SecurityCache;
class MDConsumer
{
public:
// Consumer for all symbols
MDConsumer(std::shared_ptr<api::ISyntheticSecurityService>&, bool shared=true);
MDConsumer(const UserSymbol&, std::shared_ptr<api::ISyntheticSecurityService>, bool shared=true);
MDConsumer(std::shared_ptr<api::ISyntheticSecurityService>&, SecurityCache&, bool shared=true);
MDConsumer(const UserSymbol&, std::shared_ptr<api::ISyntheticSecurityService>, SecurityCache&, bool shared=true);
// Consumer for a given set of symbols
MDConsumer(std::set<UserSymbol> const& symbols, std::shared_ptr<api::ISyntheticSecurityService>, bool shared=true, co2::Platform platform=co2::PLATFORM_UNSET);
MDConsumer(api::Venue, co2::Platform);
MDConsumer(std::set<UserSymbol> const& symbols, std::shared_ptr<api::ISyntheticSecurityService>, SecurityCache&, bool shared=true, co2::Platform platform=co2::PLATFORM_UNSET);
MDConsumer(SecurityCache&, api::Venue, co2::Platform);
~MDConsumer();
bool all_heartbeats_ok() const;
@@ -53,7 +55,7 @@ public:
// Retrieve any security that has been seen in the past
// You still need to call scan_securities() periodically to discover new securities
SecurityPtr get_security(SecurityKey key) const;
const api::Security* get_security(SecurityKey key) const;
MDConsumerChannel get_channel(const std::string& symbol, SecurityType type, Venue exchange) const;
MDConsumerChannel get_channel(const UserSymbol&) const;
+1 -1
View File
@@ -45,7 +45,7 @@ MDRecoverableConsumer::MDRecoverableConsumer(std::set<UserSymbol> symbols, std::
MDRecoverableConsumer::~MDRecoverableConsumer() {}
unique_ptr<MDRecoverableConsumer::MDConsumerGroup> MDRecoverableConsumer::init(std::set<UserSymbol>& symbols, std::shared_ptr<api::ISyntheticSecurityService>& ss) {
return std::make_unique<MDConsumerGroup>(symbols, ss);
return std::make_unique<MDConsumerGroup>(symbols, ss, _cache);
}
size_t MDRecoverableConsumer::walk_security_list(SecurityCallback callback) {
+4 -1
View File
@@ -5,6 +5,7 @@
#include "MDConsumer.h"
#include "MDConsumerHandle.h"
#include "UserSymbol.h"
#include "SecurityCache.h"
#include <set>
#include <memory>
#include <tbb/mutex.h>
@@ -45,7 +46,8 @@ private:
struct MDConsumerGroup {
MDConsumerGroup(
std::set<UserSymbol>& symbols,
std::shared_ptr<api::ISyntheticSecurityService>& ss) : consumer(symbols, ss), handles() {}
std::shared_ptr<api::ISyntheticSecurityService>& ss,
SecurityCache& cache) : consumer(symbols, ss, cache), handles() {}
MDConsumer consumer;
std::vector<std::shared_ptr<MDConsumerHandle>> handles;
};
@@ -53,6 +55,7 @@ private:
std::set<UserSymbol> _symbols;
std::shared_ptr<api::ISyntheticSecurityService> _ss;
SecurityCache _cache;
std::unique_ptr<MDConsumerGroup> _current;
std::vector<std::unique_ptr<MDConsumerGroup>> _old;
+1 -1
View File
@@ -37,7 +37,7 @@ void SecurityCacheImpl::flush() {
//////////////////
SecurityCache::SecurityCache() : _me(new SecurityCacheImpl()) {}
SecurityCache::SecurityCache() : _me(std::make_shared<SecurityCacheImpl>()) {}
SecurityCache::~SecurityCache() {}
SecurityPtr SecurityCache::try_get_security(SecurityKey key) const {
+1 -1
View File
@@ -13,7 +13,7 @@ namespace md {
// Store security+handle
struct SecurityCacheImpl;
class SecurityCache {
std::unique_ptr<SecurityCacheImpl> _me;
std::shared_ptr<SecurityCacheImpl> _me;
public:
SecurityCache();
+1 -1
View File
@@ -24,7 +24,7 @@ struct SecurityOracleImpl {
SecurityOracle::SecurityOracle(MDClient& client): _self(new SecurityOracleImpl(client)) { }
SecurityOracle::~SecurityOracle() { }
SecurityPtr SecurityOracle::get_security(SecurityKey key) const {
const api::Security* SecurityOracle::get_security(SecurityKey key) const {
return _self->consumer.get_security(key);
}
+1 -1
View File
@@ -13,7 +13,7 @@ class SecurityOracle {
public:
SecurityOracle(MDClient&);
~SecurityOracle();
SecurityPtr get_security(SecurityKey key) const;
const api::Security* get_security(SecurityKey key) const;
};
}}//namespace
+6 -2
View File
@@ -13,6 +13,7 @@
#include "lib/config/protos/system.pb.h"
#include "lib/md/protos/co2.pb.h"
#include "lib/md/SecurityCache.h"
#include "lib/api/apicore.pb.h"
#include "lib/api/services/MarketDataRelay.server.pb.h"
#include "lib/api/public/ServiceContext.h"
@@ -162,6 +163,7 @@ void copy_misc(IntPrice settlement, MiscData misc, co2::MiscMessage* pmisc) {
// Handles market data queries and responds
class QueryServerImpl : public IQueryServer {
SecurityCache _cache;
MDConsumer _consumer;
const std::set<UserSymbol> _symbols;
const std::shared_ptr<api::ISyntheticSecurityService> _ss;
@@ -169,7 +171,8 @@ class QueryServerImpl : public IQueryServer {
public:
QueryServerImpl(const std::set<UserSymbol>& symbols, std::shared_ptr<api::ISyntheticSecurityService>& ss, metrics::MetricsContainer& stats) :
_consumer(symbols, ss),
_cache(),
_consumer(symbols, ss, _cache),
_symbols(symbols),
_ss(ss),
_groups()
@@ -211,7 +214,8 @@ public:
// not-reentrant
void check_heartbeats(utils::Dispatcher& dispatcher) {
if (!_consumer.all_heartbeats_ok()) {
MDConsumer temp(_symbols, _ss);
SecurityCache cache;
MDConsumer temp(_symbols, _ss, cache);
if (temp.all_heartbeats_ok()) {
_consumer.swap(temp);
LOG(md,Info, "Recovering from bad MD. Old consumer {} new consumer {}", _consumer.id(), temp.id());
+25 -12
View File
@@ -3,6 +3,7 @@
#include "../SettlementRecorder.h"
#include "../coconut2.h"
#include "../relay/QueryServer.h"
#include "../SecurityCache.h"
#include "Co2TestInstrument.h"
#include "lib/config/config.h"
#include "lib/os/os.h"
@@ -368,7 +369,8 @@ public:
symbols.insert(future_symbol);
UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION );
symbols.insert(option_symbol);
MDConsumer consumer(symbols, nullptr);
SecurityCache cache;
MDConsumer consumer(symbols, nullptr, cache);
vector<Security> consumer_securities;
// Poll for security list - none visible until they have keys
@@ -438,7 +440,8 @@ public:
symbols.insert(future_symbol);
UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION );
symbols.insert(option_symbol);
MDConsumer consumer(symbols, nullptr);
SecurityCache cache;
MDConsumer consumer(symbols, nullptr, cache);
// Add the on_security callback
vector<Security> securities;
@@ -477,7 +480,8 @@ public:
symbols.insert(future_symbol);
UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION );
symbols.insert(option_symbol);
MDConsumer consumer(symbols, nullptr);
SecurityCache cache;
MDConsumer consumer(symbols, nullptr, cache);
// Get a handle to 24 consumers which haven't been created yet
vector<MDConsumerHandle> consumers;
@@ -534,7 +538,8 @@ public:
symbols.insert(future_symbol);
UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION );
symbols.insert(option_symbol);
MDConsumer consumer(symbols, nullptr);
SecurityCache cache;
MDConsumer consumer(symbols, nullptr, cache);
// Get a handle to 24 consumers which haven't been created yet
vector<MDConsumerHandle> consumers;
@@ -594,7 +599,8 @@ public:
// Symbol list for the consumer
auto syms = get_test_symbols();
MDConsumer consumer = MDConsumer(syms, nullptr);
SecurityCache cache;
MDConsumer consumer = MDConsumer(syms, nullptr, cache);
// Add the on_security callback - callback fires for existing securities
vector<Security> securities;
@@ -737,7 +743,8 @@ public:
symbols.insert(future_symbol);
UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION );
symbols.insert(option_symbol);
MDConsumer consumer(symbols, nullptr);
SecurityCache cache;
MDConsumer consumer(symbols, nullptr, cache);
set<security_key> keys;
consumer.walk_security_list([&](const SecurityPtr& security, MDConsumerHandle) { keys.insert(security->id()); });
@@ -781,7 +788,8 @@ public:
csymbols.insert(UserSymbol("S2", VENUE_BTS, SECURITYTYPE_SPREAD));
csymbols.insert(UserSymbol("S3", VENUE_BTS, SECURITYTYPE_SPREAD));
csymbols.insert(UserSymbol("S4", VENUE_BTS, SECURITYTYPE_SPREAD));
MDConsumer consumer(csymbols, nullptr);
SecurityCache cache;
MDConsumer consumer(csymbols, nullptr, cache);
auto check_sec = [&](const SecurityPtr& security, MDConsumerHandle handle) {
REQUIRE(security->id() == handle.key());
@@ -827,7 +835,8 @@ public:
csymbols.insert(UserSymbol("FF", VENUE_BTS, SECURITYTYPE_FUTURE));
csymbols.insert(UserSymbol("OO", VENUE_BTS, SECURITYTYPE_OPTION));
csymbols.insert(UserSymbol("SS", VENUE_BTS, SECURITYTYPE_SPREAD));
MDConsumer consumer(csymbols, nullptr);
SecurityCache cache;
MDConsumer consumer(csymbols, nullptr, cache);
auto check_sec = [&](const SecurityPtr& security, MDConsumerHandle handle) {
REQUIRE(security->id() == handle.key());
@@ -878,7 +887,8 @@ public:
set<UserSymbol> consumer_symbols;
consumer_symbols.insert(UserSymbol("OO", VENUE_BTS, SECURITYTYPE_OPTION));
consumer_symbols.insert(UserSymbol("SS", VENUE_BTS, SECURITYTYPE_SPREAD));
MDConsumer consumer(consumer_symbols, nullptr);
SecurityCache cache;
MDConsumer consumer(consumer_symbols, nullptr, cache);
size_t security_count = 0;
// Indirection for check function
@@ -946,7 +956,8 @@ public:
auto ph2 = *add_security(o2);
set<UserSymbol> consumer_symbols;
consumer_symbols.insert(UserSymbol("OO", VENUE_BTS, SECURITYTYPE_OPTION));
MDConsumer consumer(consumer_symbols, nullptr);
SecurityCache cache;
MDConsumer consumer(consumer_symbols, nullptr, cache);
auto h1 = consumer.get_handle(k1);
REQUIRE(h1.is_initialized());
auto h2 = consumer.get_handle(k2);
@@ -1212,6 +1223,7 @@ struct Co2IndexTests {
} ;
struct Co2RelayTests {
SecurityCache cache;
Co2RelayTests() : _stats() {}
void setup() {
config::internal::initialize_for_test(
@@ -1230,7 +1242,7 @@ struct Co2RelayTests {
symbols.insert(future_symbol);
UserSymbol option_symbol("FOOX", VENUE_BTS, SECURITYTYPE_OPTION);
symbols.insert(option_symbol);
MDConsumer consumer(symbols, nullptr);
MDConsumer consumer(symbols, nullptr, cache);
return consumer;
}
@@ -1587,7 +1599,8 @@ namespace ImpliedTests {
symbols.insert(future_symbol);
UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION );
symbols.insert(option_symbol);
MDConsumer consumer(symbols, nullptr);
SecurityCache cache;
MDConsumer consumer(symbols, nullptr, cache);
vector<Security> consumer_securities;
// Poll for security list - none visible until they have keys
size_t count = consumer.walk_security_list([&](const SecurityPtr& security, MDConsumerHandle) {consumer_securities.push_back(*security);});
@@ -92,7 +92,7 @@ struct OrderServiceServer::PImpl {
DynamicOrderRegistry const &d, metrics::MetricsContainer &stats, trace::PerfLog &log)
: service_bridge(), service_ready(false), known_orders(), known_fills(),
publish_orders(), publish_fills(),
admin_service(ctx.locate<api::admin::AdminService>()), md_consumer(ss),
admin_service(ctx.locate<api::admin::AdminService>()), cache(), md_consumer(ss, cache),
stats(stats), dyn_order_registry(d),log(log) {}
ServiceBridge *service_bridge;
@@ -106,6 +106,7 @@ struct OrderServiceServer::PImpl {
OrderServiceServer::OrdersPublisher publish_orders;
OrderServiceServer::FillsPublisher publish_fills;
api::admin::AdminService admin_service;
md::SecurityCache cache;
bts::md::MDConsumer md_consumer;
metrics::MetricsContainer &stats;
DynamicOrderRegistry const &dyn_order_registry;