Compare commits

..
Author SHA1 Message Date
Eric Harding bbca69667e DO_NOT_MERGE: Add diagnostic logging 2019-05-30 10:53:26 -04:00
Eric Harding 0c1ff61af5 Add test to reproduce MBO read problem 2019-05-28 12:48:35 -04:00
12 changed files with 166 additions and 107 deletions

No files matched your search

-8
View File
@@ -4,14 +4,6 @@ weight = 3
sort_by = "weight"
insert_anchor_links = "right"
+++
## 2.0.4
### User Visible Changes
- Fixed performance regression in order manager
### Internal changes
- Improved market data diagnostics for bad configurations
- Changed market data default buffer size
## 2.0.3
### Internal changes
- Improved backwards compatibility in UI settings
+1
View File
@@ -24,6 +24,7 @@ namespace bts {
}
static void dump_mbo(uint key, api.Side side) {
// todo: fix me
using (var ctx = new ServiceContext()) {
var mbo = ctx.locate<MBOService>();
var data = mbo.get_orders_for_security(new MBOSecurityQuery { SecurityId = key });
+1 -1
View File
@@ -98,7 +98,7 @@ message CMEOptions
// Defines amount of time Feed Engine keeps cycling before going to sleep when no useful activity can be done
optional uint32 spin_time = 4 [default = 1];
optional uint32 channels_per_thread = 5 [default = 3];
optional uint32 udp_socket_buffer_size = 11 [default = 268435456]; //256mb
optional uint32 udp_socket_buffer_size = 11 [default = 33554432]; //32mb
}
message CMEChannel {
repeated string id = 2;
+30 -1
View File
@@ -3,6 +3,7 @@
#include "lib/config/config.h"
#include "lib/config/protos/toggles.pb.h"
#include "lib/utils/thread_manager.h"
#include "lib/utils/LockBox.h"
#include <map>
#include <functional>
@@ -63,6 +64,7 @@ void print_pool_stats(const utils::ConcurrentPool<MBOSnapshot>& pool) {
}
typedef tbb::concurrent_hash_map<SecurityKey, MBOSnapshot*, SecurityKeyHashCompare> MapType;
// typedef std::unordered_map<SecurityKey, MBOSnapshot*> MapType;
struct Incremental {
MBOWorkingState* state;
inc::Update update;
@@ -71,6 +73,7 @@ struct MBOProcessorImpl {
utils::ConcurrentPool<MBOSnapshot> pool;
tbb::concurrent_queue<Incremental> incremental_input;
MapType snapshots;
// utils::LockBox<MapType> snapshots;
bool running;
utils::TimeStamp last_pool_print;
std::thread thread;
@@ -85,7 +88,6 @@ struct MBOProcessorImpl {
if (config::get<config::toggles::MBOStats>().enabled()) {
last_pool_print = utils::TimeStamp::now();
}
}
~MBOProcessorImpl() {
@@ -94,14 +96,25 @@ struct MBOProcessorImpl {
}
void publish_snapshot(SecurityKey key, MBOSnapshot** snapshot) {
LOG(md,Diag, "publish_snapshot/begin/{}", key.value());
bts_assert(*snapshot != nullptr);
MapType::accessor accessor;
snapshots.insert(accessor, key);
std::swap(accessor->second, *snapshot);
if (*snapshot != nullptr) {
// Note: relase() must happen inside accessor lifetime to ensure correctness
LOG(md,Diag, "publish_snapshot/release/{}", key.value());
pool.release(snapshot);
}
LOG(md,Diag, "publish_snapshot/end/{}", key.value());
// snapshots.lock([&](MapType& map) {
// std::swap(map[key], *snapshot);
// // Note: relase() must happen inside accessor lifetime to ensure correctness
// if (*snapshot != nullptr) {
// pool.release(snapshot);
// }
// });
}
void publish_incremental(MBOWorkingState& state) {
@@ -139,6 +152,12 @@ struct MBOProcessorImpl {
}
// publish a small batch of snapshots (then check queue again)
auto sz = dirty.size();
if (sz > 0) {
LOG(md,Diag, "process_incremental/publish/sz={}", sz);
}
for (auto iter = dirty.begin(); iter != dirty.end(); ++iter) {
publish_incremental(**iter);
clean.push_back(*iter);
@@ -276,6 +295,7 @@ void apply_incremental(uint32_t security_key, C& collection, std::unordered_map<
}
}
void apply_incremental(MBOWorkingState& state, const inc::Update& update) {
LOG(md,Diag, "apply_incremental/begin/{}", state.handle.key());
// apply the update to the working state
if (update.type == inc::UpdateType::Bid) {
apply_incremental(state.handle.key(), state.bid, state.orderid_index, update);
@@ -300,6 +320,15 @@ MBOSnapshot* MBOReader::acquire_snapshot(SecurityKey key) const {
else {
release_snapshot(&result);
}
// _me->snapshots.lock([&](const MapType& map) {
// auto iter = map.find(key);
// if (iter != map.end()) {
// result->copy_from(*iter->second);
// }
// else {
// release_snapshot(&result);
// }
// });
return result;
}
void MBOReader::release_snapshot(MBOSnapshot**s) const {
+2 -2
View File
@@ -136,9 +136,9 @@ class MBOWriter {
std::shared_ptr<MBOProcessorImpl> _me;
friend class MBOProcessor;
MBOWriter(std::shared_ptr<MBOProcessorImpl>&);
~MBOWriter();
public:
~MBOWriter();
//
// Snapshot api
MBOSnapshot* allocate_snapshot(size_t bid_count=0, size_t ask_count=0);
@@ -158,8 +158,8 @@ class MBOReader {
friend class MBOProcessor;
friend class MBOWriter;
MBOReader(std::shared_ptr<MBOProcessorImpl>&);
~MBOReader();
public:
~MBOReader();
MBOSnapshot* acquire_snapshot(SecurityKey key) const;
void release_snapshot(MBOSnapshot**) const;
};
+9 -1
View File
@@ -309,8 +309,8 @@ void MBOServiceServer::get_orders_for_security(const MBOSecurityQuery& request,
}
size_t count = 0;
auto tick = handle.get_minimum_tick();
if(handle.is_initialized()) {
auto tick = handle.get_minimum_tick();
auto mbo = handle.get_mbo();
if (mbo) {
for (const auto& o : mbo->bids) {
@@ -346,7 +346,15 @@ void MBOServiceServer::get_orders_for_security(const MBOSecurityQuery& request,
}
}
}
else {
LOG(md,Diag,"get_orders_for_security/fail/null/{}", handle.key());
}
}
else {
LOG(md,Diag,"get_orders_for_security/fail/uninitialized/{}", handle.key());
}
LOG(md,Diag,"get_orders_for_security/{}/{}b/{}a/{}", handle.key(), response.bid_size(), response.ask_size(), request.price_level());
reply(response);
}
+3
View File
@@ -379,6 +379,9 @@ std::unique_ptr<MBOSnapshot,std::function<void(MBOSnapshot*)>> MDConsumerHandle:
if (snapshot != nullptr) {
return {snapshot,[mbo=book.mbo](auto* s) { mbo->release_snapshot(&s); }};
}
else {
LOG(md,Diag,"get_mbo/fail/{}", key());
}
}
return nullptr;
}
+89
View File
@@ -2,10 +2,12 @@
#include "catch/catch.hpp"
#include "../IntPrice.h"
#include "../MBOProcessor.h"
#include "../MBOServiceServer.h"
#include "Co2TestInstrument.h"
#include <random>
#include <algorithm>
#include <thread>
namespace bts {
namespace md {
@@ -316,5 +318,92 @@ TEST_CASE("MBO change price and size", "[md][mbo]") {
REQUIRE(!not_ask);
}
TEST_CASE("MBO stress", "[md][mbo]") {
/*
This test attempts to shake out concurrency issues with MBO processing
If it fails DO NOT ignore it
*/
MBOProcessor p;
bool running = true;
std::vector<SecurityKey> keys;
for (auto x = 0; x < 157; ++x) {
keys.push_back(SecurityKey(x));
}
auto writer = p.writer();
size_t write_count = 0;
size_t wtime = 0;
size_t wcount = 0;
std::thread twriter([&]() {
size_t count = 0;
while(running) {
auto start = utils::TimeStamp::now();
for (const auto& k : keys) {
++count;
auto s = writer.allocate_snapshot();
REQUIRE(s);
for (int t = 0; t < 15000; ++t) {
s->bids.push_back({ ExchangeOrderKey(count%1024 + t), { 101, -2 }, 42 });
s->asks.push_back({ ExchangeOrderKey(count%1024 + 10000 + t), { 111, -2 }, 32 });
++write_count;
}
writer.publish_snapshot(k, &s);
}
auto end = utils::TimeStamp::now();
++wcount;
wtime += (end - start).milliseconds();
}
});
// Ensure writer has written each once
while(wcount < 1)
std::this_thread::sleep_for(std::chrono::milliseconds(10));
std::cout << "wcount: " << wcount << " - time: " << (wtime/wcount) << std::endl;
auto reader = p.reader();
std::vector<std::thread> readers;
size_t time = 0;
size_t count = 0;
size_t nullcount = 0;
for (int x = 0; x<1; ++x) {
readers.push_back(std::thread([&]() {
while(running) {
auto start = utils::TimeStamp::now();
for (const auto& k : keys) {
auto s = reader.acquire_snapshot(k);
if (s == nullptr) nullcount++;
(void)s;
reader.release_snapshot(&s);
// REQUIRE(s);
}
auto end = utils::TimeStamp::now();
++count;
time += (end - start).microseconds();
}
}));
}
std::this_thread::sleep_for(std::chrono::seconds(10));
running = false;
twriter.join();
for (auto& t : readers) t.join();
std::cout << "Write time: " << wtime/wcount << std::endl;
std::cout << "Read time: " << time/count << std::endl;
std::cout << "nullcount: " << nullcount << std::endl;
REQUIRE(nullcount == 0);
/*
Next step:
- Replace concurrent_hash with Lockbox<Unordered> and see if we can still make it happen
*/
}
}}}
#endif
@@ -358,7 +358,7 @@ OrderManager::free_buffers(std::string const &session_name) {
}
void OrderManager::init_order_services() {
auto &log = me->services_data.log;
auto& log = me->services_data.log;
for (auto &s : me->services_data.sessions) {
auto gw_name = GatewayName(s->session);
@@ -410,14 +410,13 @@ void OrderManager::init_order_services() {
};
s->exe_report_subs =
std::make_unique<ContentSubscription<NewExecutionReportEvent>>(
*this, gw_name, on_exe_r, log);
*this, gw_name, on_exe_r,log);
s->cancel_reject_subs =
std::make_unique<ContentSubscription<CancelRejectEvent>>(
*this, gw_name, log);
std::make_unique<ContentSubscription<CancelRejectEvent>>(*this,
gw_name,log);
s->gateway_status_subs = std::make_unique<
ContentSubscription<GatewayStatusChangeEvent, NoKey>>(*this, gw_name,
log);
ContentSubscription<GatewayStatusChangeEvent, NoKey>>(*this, gw_name,log);
// publishers for execution reports, gateway status and cancel
// reject evets
@@ -495,18 +494,18 @@ void OrderManager::init_order_services() {
me->services_data.cor_id = ev.trace;
};
me->services_data.md_subs =
std::make_unique<ContentSubscription<BookUpdateEvent>>(
*this, on_md_sub, on_book_update, log);
std::make_unique<ContentSubscription<BookUpdateEvent>>(*this, on_md_sub,
on_book_update,log);
me->services_data.ticker_subs =
std::make_unique<ContentSubscription<TickerUpdateEvent>>(
*this, on_ticker_sub, log);
std::make_unique<ContentSubscription<TickerUpdateEvent>>(*this,
on_ticker_sub,log);
me->services_data.settlement_subs =
std::make_unique<ContentSubscription<SettlementUpdateEvent>>(
*this, on_settlement_sub, log);
*this, on_settlement_sub,log);
me->services_data.time_subs =
std::make_unique<ContentSubscription<TimerEvent, NoKey>>(*this, log);
std::make_unique<ContentSubscription<TimerEvent, NoKey>>(*this,log);
me->services_data.order_creation_subs =
std::make_unique<ContentSubscription<CreateOrderReplyEvent>>(*this, log);
std::make_unique<ContentSubscription<CreateOrderReplyEvent>>(*this,log);
}
void OrderManager::after_event() {
@@ -515,8 +514,7 @@ void OrderManager::after_event() {
for (auto &osd : me->services_data.sessions) {
auto &fix_stream = osd->current_buffer->first_stream();
if (fix_stream.n_msgs()) {
auto scoped_do_work = trace::log_scope(me->services_data.log, {},
"OrderManager::after_event");
auto scoped_do_work = trace::log_scope(me->services_data.log, {}, "OrderManager::after_event");
fix_stream.close();
osd->gw.send_fix(osd->current_buffer, osd->current_trace);
reload_fix_buffer(*osd);
@@ -538,10 +536,8 @@ void OrderManager::reload_fix_buffer(SessionServicesData &osd) {
}
}
void OrderManager::on_poll(trace::PerfLog &log) {
auto scoped_do_work =
trace::log_scope(log, {}, "OrderManager::dispatch_incoming_fix_messages",
utils::TimeStamp::Interval::from_microseconds(2));
void OrderManager::on_poll(trace::PerfLog&log) {
auto scoped_do_work = trace::log_scope(log, {}, "OrderManager::dispatch_incoming_fix_messages");
for (auto &osd : me->services_data.sessions)
osd->gw.dispatch_incoming_fix_messages();
}
+15
View File
@@ -24,6 +24,18 @@ class ConcurrentPool {
delete [] b;
}
#if 0
T* allocate() {
return new T();
}
void reserve(size_t ) {
}
void release(T** item) {
if (*item) {
delete *item;
}
}
#else
T* allocate() {
T* result = nullptr;
++_outstanding;
@@ -46,6 +58,7 @@ class ConcurrentPool {
_all.push_back(buffer);
}
void release(T** item) {
if (*item) {
(*item)->reset();
@@ -55,6 +68,8 @@ class ConcurrentPool {
}
}
#endif
template <class CB>
void unsafe_iterate(CB cb) const {
for (auto iter = _all.begin(); iter != _all.end(); ++iter) {
@@ -316,12 +316,7 @@ unique_ptr<onixs::Handler> MDP3Main::prepare_handler(const string& channel_id, M
try {
auto handler = unique_ptr<onixs::Handler>(new onixs::Handler(settings));
auto iter = _engines.find(options.feed_engine());
if (iter == _engines.end()) {
LOG(md,Fatal, "Missing feed engine definition '{}' for channel id '{}'. Add a feed_engine entry with name:'{}' to the CMEOptions config key.", options.feed_engine(), channel_id, options.feed_engine());
bts_assert_msg(iter != _engines.end(), "Missing feed engine definition.");
}
handler->bindFeedEngine(*(iter->second));
handler->bindFeedEngine(*_engines.at(options.feed_engine()));
if (recovery) {
handler->bindTcpRecovery(*recovery);
_recovery.push_back(std::move(recovery));
@@ -1,69 +0,0 @@
Test Author: Pedro
If it breaks: Pedro
Spreader Requote Performance test
Note: This test is fairly cumbersome to setup, because performance is
very sensitive to a number of factors including compilation switches,
hardware used, thread allocation and load on the machine. Pedro has
the best handle on how do all this and should probably be the one
doing this test.
1) Make sure you are testing a release server build
2) Make sure you use real hardware, with hyper threading turned off
and cstates disabled, for the server side
3) Make sure you isolate the CPU on which you are running the
server. This is best done on Linux by editing /etc/default/grub like
this:
# this isolates CPU's 0 to 5 for better performance testing
GRUB_CMDLINE_LINUX="isolcpus=0-5"
4) Now make sure your configuration is set for the lowest latency,
including dedicating a thread to the order manager on the isolated
cores:
OrdermanCfg {
value {
executor {
thread_name: "oman_orders"
poll: true
hosted_actor: "MDSource"
hosted_actor: "OrderManager"
}
executor {
thread_name: "oman_slow"
poll: false
hosted_actor: "*"
}
}
}
Threads {
value {
thread_group {name_rx: "mdp3feed" core: 0}
thread_group {name_rx: "oman_orders" core: 1}
thread_group {name_rx: "jf_ar_sim" core: 2}
thread_group {name_rx: ".*" core:6 core:7 core:8 core:9 core:10 }
}
}
5) Now you are ready to start testing. Start a shell and create a new
synthetic say with 2 CL futures. Use the near month (fast ticking)
future as the non quoting leg, so that the other leg gets quoted
often. Make sure to disable throtling in the synthetic definition
dialog.
6) Place an order for this synthetic, away from the market.
7) Now run for say 3 minutes. Visit http://youserver:8000/perf_trace
You should see a graph indicating the median requote times for the
order you created in 5). Make sure you have at least 100 data points
and data the median time is around 12us. If you get anything greater
than 15us, the test has failed.