Spark-681 Synthetics can be lost on startup due to race.

This commit is contained in:
Eric Harding committed 2018-03-19 16:06:51 -04:00
1 parent 6079d7f02d
commit 247365d762
2 files changed
+56 -25

No files matched your search

+55 -24
View File
@@ -293,14 +293,20 @@ private:
// Instance for synthetic securities (no shm)
///////////////////////////////////////////////////////////////////////////////////////////////////
class SyntheticConsumerInst {
typedef tbb::queuing_mutex Mutex;
typedef function<map<security_key,MDConsumerHandle>(const set<security_key>&)> IGetHandle;
typedef shared_ptr<SyntheticStrategy> ItemType;
struct IncompleteStrategy {
shared_ptr<api::Security> security;
spark::ExecutionStrategy strategy;
};
Mutex _workmutex;
shared_ptr<api::ISyntheticSecurityService> _ss;
map<security_key, ItemType> _data;
vector<shared_ptr<Security>> _pending;
set<security_key> _seen;
IGetHandle _get_handle;
bool _initialized;
@@ -312,6 +318,7 @@ class SyntheticConsumerInst {
_workmutex(),
_ss(ss),
_data(),
_pending(),
_seen(),
_get_handle(gh),
_initialized(false),
@@ -347,6 +354,11 @@ class SyntheticConsumerInst {
const vector<ItemType> scan_new_synthetics() {
Mutex::scoped_lock lock(_workmutex);
init();
if (!_pending.empty()) {
vector<shared_ptr<Security>> temp;
temp.swap(_pending);
add_snapshot(temp);
}
vector<ItemType> result;
for (const auto& i : _data) {
if (_seen.find(i.first) == _seen.end()) {
@@ -361,7 +373,11 @@ private:
void init() {
if (!_initialized) {
_initialized = true;
add_snapshot(_ss->securities());
vector<shared_ptr<Security>> snapshot;
for (const auto& pair : _ss->securities()) {
snapshot.push_back(make_shared<api::Security>(pair.second));
}
add_snapshot(snapshot);
_security_sub = _ss->on_securities().subscribe([=](const api::Security& sec) { add_security(sec); });
_meta_sub = _ss->on_metadata().subscribe([=](const synthetic::Metadata& m) { on_metadata(m); });
}
@@ -370,11 +386,16 @@ private:
void add_security(const api::Security& sec) {
Mutex::scoped_lock lock(_workmutex);
if (_data.find(sec.id()) == _data.end()) {
auto ptr = to_strategy(sec);
vector<shared_ptr<SyntheticStrategy>> vec;
auto ptr = to_strategy(make_shared<api::Security>(sec));
vector<IncompleteStrategy> vec;
vec.push_back(ptr);
auto handles = get_synthetic_handles(vec, _data, _get_handle);
_data.insert(handles.begin(), handles.end());
for (const auto& s : vec) {
if (handles.find(s.security->id()) == handles.end()) {
_pending.push_back(s.security);
}
}
}
}
@@ -389,14 +410,14 @@ private:
private:
static map<security_key, ItemType> get_synthetic_handles(
vector<shared_ptr<SyntheticStrategy>> synthetics,
vector<IncompleteStrategy> synthetics,
const map<security_key, ItemType>& existing,
const std::function<map<security_key,MDConsumerHandle>(const set<security_key>&)>& get_handles) {
// gather leg keys for batch processing
std::set<security_key> leg_keys;
for (const auto& p : synthetics) {
for (const auto& leg : p->security.synthetic().legs()) {
for (const auto& leg : p.security->synthetic().legs()) {
leg_keys.insert(leg.security().id());
}
}
@@ -410,34 +431,39 @@ private:
result_count = result.size();
for (const auto& strategy : synthetics) {
// skip any that have already been found
if (result.find(strategy->security.id()) != result.end()) continue;
strategy->legs.reserve(strategy->security.synthetic().legs_size());
for (const auto& leg : strategy->security.synthetic().legs()) {
if (result.find(strategy.security->id()) != result.end()) continue;
vector<MDConsumerHandle> legs;
legs.reserve(strategy.security->synthetic().legs_size());
for (const auto& leg : strategy.security->synthetic().legs()) {
auto leg_key = leg.security().id();
auto iter = real_handles.find(leg_key);
if (iter != real_handles.end()) {
// real handle
strategy->legs.push_back(iter->second);
legs.push_back(iter->second);
}
else {
auto iter2 = existing.find(leg_key);
if (iter2 != existing.end()) {
// existing synthetic handle
strategy->legs.push_back(MDConsumerHandle(iter2->second));
legs.push_back(MDConsumerHandle(iter2->second));
}
else {
auto iter3 = result.find(leg_key);
if (iter3 != result.end()) {
// new synthetic handle?
strategy->legs.push_back(MDConsumerHandle(iter3->second));
legs.push_back(MDConsumerHandle(iter3->second));
}
}
}
}
// If we found all the legs
if (strategy->legs.size() == (size_t)strategy->security.synthetic().legs_size()) {
if (legs.size() == (size_t)strategy.security->synthetic().legs_size()) {
auto s = make_shared<SyntheticStrategy>();
s->security.CopyFrom(*strategy.security);
s->strategy.CopyFrom(strategy.strategy);
s->legs = legs;
result.insert(
make_pair(strategy->security.id(),strategy));
make_pair(strategy.security->id(),s));
}
}
//iterate as long as progress is made
@@ -445,27 +471,32 @@ private:
return result;
}
shared_ptr<SyntheticStrategy> to_strategy(const api::Security& security) {
auto ptr = make_shared<SyntheticStrategy>();
ptr->security.CopyFrom(security);
auto meta = _ss->get_metadata(SecurityKey(security.id()));
IncompleteStrategy to_strategy(const shared_ptr<api::Security>& security) {
IncompleteStrategy res;
res.security = security;
auto meta = _ss->get_metadata(SecurityKey(security->id()));
// If metadata is set copy it otherwise we use the default proto
if (meta) {
ptr->strategy.CopyFrom(meta->strategy());
res.strategy.CopyFrom(meta->strategy());
}
return ptr;
return res;
}
void add_snapshot(const std::unordered_map<SecurityKey, Security>& securities) {
std::vector<shared_ptr<SyntheticStrategy>> strategies;
for (const auto& pair : securities) {
const auto& sec = pair.second;
if (_data.find(sec.id()) == _data.end()) {
void add_snapshot(const vector<shared_ptr<Security>>& securities) {
std::vector<IncompleteStrategy> strategies;
for (const auto& sec : securities) {
if (_data.find(sec->id()) == _data.end()) {
strategies.push_back(to_strategy(sec));
}
}
if (strategies.size() > 0) {
auto handles = get_synthetic_handles(strategies, _data, _get_handle);
// put strategies that are incomplete into a backlog
for (const auto& s : strategies) {
if (handles.find(s.security->id()) == handles.end()) {
_pending.push_back(s.security);
}
}
_data.insert(handles.begin(), handles.end());
}
}
+1 -1
View File
@@ -73,10 +73,10 @@ void run_cleanup() {
std::thread* _md_thread;
void run_market_datapublishers(bool should_return) {
bts_assert(_md_thread == nullptr);
run_cleanup();
// start a thread for the md
_md_thread = new std::thread([]() {
os::set_thread_name("md publisher");
run_cleanup();
if (config::get<MarketDataConfig>().use_replay()) {
run_replay();