Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bbca69667e | ||
|
|
0c1ff61af5 |
No files matched your search
@@ -4,14 +4,6 @@ weight = 3
|
||||
sort_by = "weight"
|
||||
insert_anchor_links = "right"
|
||||
+++
|
||||
## 2.0.4
|
||||
### User Visible Changes
|
||||
- Fixed performance regression in order manager
|
||||
|
||||
### Internal changes
|
||||
- Improved market data diagnostics for bad configurations
|
||||
- Changed market data default buffer size
|
||||
|
||||
## 2.0.3
|
||||
### Internal changes
|
||||
- Improved backwards compatibility in UI settings
|
||||
|
||||
@@ -24,6 +24,7 @@ namespace bts {
|
||||
}
|
||||
|
||||
static void dump_mbo(uint key, api.Side side) {
|
||||
// todo: fix me
|
||||
using (var ctx = new ServiceContext()) {
|
||||
var mbo = ctx.locate<MBOService>();
|
||||
var data = mbo.get_orders_for_security(new MBOSecurityQuery { SecurityId = key });
|
||||
|
||||
@@ -98,7 +98,7 @@ message CMEOptions
|
||||
// Defines amount of time Feed Engine keeps cycling before going to sleep when no useful activity can be done
|
||||
optional uint32 spin_time = 4 [default = 1];
|
||||
optional uint32 channels_per_thread = 5 [default = 3];
|
||||
optional uint32 udp_socket_buffer_size = 11 [default = 268435456]; //256mb
|
||||
optional uint32 udp_socket_buffer_size = 11 [default = 33554432]; //32mb
|
||||
}
|
||||
message CMEChannel {
|
||||
repeated string id = 2;
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
#include "lib/config/config.h"
|
||||
#include "lib/config/protos/toggles.pb.h"
|
||||
#include "lib/utils/thread_manager.h"
|
||||
#include "lib/utils/LockBox.h"
|
||||
|
||||
#include <map>
|
||||
#include <functional>
|
||||
@@ -63,6 +64,7 @@ void print_pool_stats(const utils::ConcurrentPool<MBOSnapshot>& pool) {
|
||||
}
|
||||
|
||||
typedef tbb::concurrent_hash_map<SecurityKey, MBOSnapshot*, SecurityKeyHashCompare> MapType;
|
||||
// typedef std::unordered_map<SecurityKey, MBOSnapshot*> MapType;
|
||||
struct Incremental {
|
||||
MBOWorkingState* state;
|
||||
inc::Update update;
|
||||
@@ -71,6 +73,7 @@ struct MBOProcessorImpl {
|
||||
utils::ConcurrentPool<MBOSnapshot> pool;
|
||||
tbb::concurrent_queue<Incremental> incremental_input;
|
||||
MapType snapshots;
|
||||
// utils::LockBox<MapType> snapshots;
|
||||
bool running;
|
||||
utils::TimeStamp last_pool_print;
|
||||
std::thread thread;
|
||||
@@ -85,7 +88,6 @@ struct MBOProcessorImpl {
|
||||
if (config::get<config::toggles::MBOStats>().enabled()) {
|
||||
last_pool_print = utils::TimeStamp::now();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
~MBOProcessorImpl() {
|
||||
@@ -94,14 +96,25 @@ struct MBOProcessorImpl {
|
||||
}
|
||||
|
||||
void publish_snapshot(SecurityKey key, MBOSnapshot** snapshot) {
|
||||
LOG(md,Diag, "publish_snapshot/begin/{}", key.value());
|
||||
bts_assert(*snapshot != nullptr);
|
||||
MapType::accessor accessor;
|
||||
snapshots.insert(accessor, key);
|
||||
std::swap(accessor->second, *snapshot);
|
||||
if (*snapshot != nullptr) {
|
||||
// Note: relase() must happen inside accessor lifetime to ensure correctness
|
||||
LOG(md,Diag, "publish_snapshot/release/{}", key.value());
|
||||
pool.release(snapshot);
|
||||
}
|
||||
LOG(md,Diag, "publish_snapshot/end/{}", key.value());
|
||||
|
||||
// snapshots.lock([&](MapType& map) {
|
||||
// std::swap(map[key], *snapshot);
|
||||
// // Note: relase() must happen inside accessor lifetime to ensure correctness
|
||||
// if (*snapshot != nullptr) {
|
||||
// pool.release(snapshot);
|
||||
// }
|
||||
// });
|
||||
}
|
||||
|
||||
void publish_incremental(MBOWorkingState& state) {
|
||||
@@ -139,6 +152,12 @@ struct MBOProcessorImpl {
|
||||
}
|
||||
|
||||
// publish a small batch of snapshots (then check queue again)
|
||||
|
||||
auto sz = dirty.size();
|
||||
if (sz > 0) {
|
||||
LOG(md,Diag, "process_incremental/publish/sz={}", sz);
|
||||
}
|
||||
|
||||
for (auto iter = dirty.begin(); iter != dirty.end(); ++iter) {
|
||||
publish_incremental(**iter);
|
||||
clean.push_back(*iter);
|
||||
@@ -276,6 +295,7 @@ void apply_incremental(uint32_t security_key, C& collection, std::unordered_map<
|
||||
}
|
||||
}
|
||||
void apply_incremental(MBOWorkingState& state, const inc::Update& update) {
|
||||
LOG(md,Diag, "apply_incremental/begin/{}", state.handle.key());
|
||||
// apply the update to the working state
|
||||
if (update.type == inc::UpdateType::Bid) {
|
||||
apply_incremental(state.handle.key(), state.bid, state.orderid_index, update);
|
||||
@@ -300,6 +320,15 @@ MBOSnapshot* MBOReader::acquire_snapshot(SecurityKey key) const {
|
||||
else {
|
||||
release_snapshot(&result);
|
||||
}
|
||||
// _me->snapshots.lock([&](const MapType& map) {
|
||||
// auto iter = map.find(key);
|
||||
// if (iter != map.end()) {
|
||||
// result->copy_from(*iter->second);
|
||||
// }
|
||||
// else {
|
||||
// release_snapshot(&result);
|
||||
// }
|
||||
// });
|
||||
return result;
|
||||
}
|
||||
void MBOReader::release_snapshot(MBOSnapshot**s) const {
|
||||
|
||||
@@ -136,9 +136,9 @@ class MBOWriter {
|
||||
std::shared_ptr<MBOProcessorImpl> _me;
|
||||
friend class MBOProcessor;
|
||||
MBOWriter(std::shared_ptr<MBOProcessorImpl>&);
|
||||
~MBOWriter();
|
||||
|
||||
public:
|
||||
~MBOWriter();
|
||||
//
|
||||
// Snapshot api
|
||||
MBOSnapshot* allocate_snapshot(size_t bid_count=0, size_t ask_count=0);
|
||||
@@ -158,8 +158,8 @@ class MBOReader {
|
||||
friend class MBOProcessor;
|
||||
friend class MBOWriter;
|
||||
MBOReader(std::shared_ptr<MBOProcessorImpl>&);
|
||||
~MBOReader();
|
||||
public:
|
||||
~MBOReader();
|
||||
MBOSnapshot* acquire_snapshot(SecurityKey key) const;
|
||||
void release_snapshot(MBOSnapshot**) const;
|
||||
};
|
||||
|
||||
@@ -309,8 +309,8 @@ void MBOServiceServer::get_orders_for_security(const MBOSecurityQuery& request,
|
||||
}
|
||||
|
||||
size_t count = 0;
|
||||
auto tick = handle.get_minimum_tick();
|
||||
if(handle.is_initialized()) {
|
||||
auto tick = handle.get_minimum_tick();
|
||||
auto mbo = handle.get_mbo();
|
||||
if (mbo) {
|
||||
for (const auto& o : mbo->bids) {
|
||||
@@ -346,7 +346,15 @@ void MBOServiceServer::get_orders_for_security(const MBOSecurityQuery& request,
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
LOG(md,Diag,"get_orders_for_security/fail/null/{}", handle.key());
|
||||
}
|
||||
}
|
||||
else {
|
||||
LOG(md,Diag,"get_orders_for_security/fail/uninitialized/{}", handle.key());
|
||||
}
|
||||
|
||||
LOG(md,Diag,"get_orders_for_security/{}/{}b/{}a/{}", handle.key(), response.bid_size(), response.ask_size(), request.price_level());
|
||||
|
||||
reply(response);
|
||||
}
|
||||
|
||||
@@ -379,6 +379,9 @@ std::unique_ptr<MBOSnapshot,std::function<void(MBOSnapshot*)>> MDConsumerHandle:
|
||||
if (snapshot != nullptr) {
|
||||
return {snapshot,[mbo=book.mbo](auto* s) { mbo->release_snapshot(&s); }};
|
||||
}
|
||||
else {
|
||||
LOG(md,Diag,"get_mbo/fail/{}", key());
|
||||
}
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
@@ -2,10 +2,12 @@
|
||||
#include "catch/catch.hpp"
|
||||
#include "../IntPrice.h"
|
||||
#include "../MBOProcessor.h"
|
||||
#include "../MBOServiceServer.h"
|
||||
#include "Co2TestInstrument.h"
|
||||
|
||||
#include <random>
|
||||
#include <algorithm>
|
||||
#include <thread>
|
||||
|
||||
namespace bts {
|
||||
namespace md {
|
||||
@@ -316,5 +318,92 @@ TEST_CASE("MBO change price and size", "[md][mbo]") {
|
||||
REQUIRE(!not_ask);
|
||||
}
|
||||
|
||||
TEST_CASE("MBO stress", "[md][mbo]") {
|
||||
/*
|
||||
This test attempts to shake out concurrency issues with MBO processing
|
||||
If it fails DO NOT ignore it
|
||||
*/
|
||||
|
||||
MBOProcessor p;
|
||||
bool running = true;
|
||||
|
||||
std::vector<SecurityKey> keys;
|
||||
for (auto x = 0; x < 157; ++x) {
|
||||
keys.push_back(SecurityKey(x));
|
||||
}
|
||||
|
||||
auto writer = p.writer();
|
||||
size_t write_count = 0;
|
||||
size_t wtime = 0;
|
||||
size_t wcount = 0;
|
||||
std::thread twriter([&]() {
|
||||
size_t count = 0;
|
||||
while(running) {
|
||||
auto start = utils::TimeStamp::now();
|
||||
for (const auto& k : keys) {
|
||||
++count;
|
||||
auto s = writer.allocate_snapshot();
|
||||
REQUIRE(s);
|
||||
for (int t = 0; t < 15000; ++t) {
|
||||
s->bids.push_back({ ExchangeOrderKey(count%1024 + t), { 101, -2 }, 42 });
|
||||
s->asks.push_back({ ExchangeOrderKey(count%1024 + 10000 + t), { 111, -2 }, 32 });
|
||||
++write_count;
|
||||
}
|
||||
writer.publish_snapshot(k, &s);
|
||||
}
|
||||
auto end = utils::TimeStamp::now();
|
||||
++wcount;
|
||||
wtime += (end - start).milliseconds();
|
||||
}
|
||||
});
|
||||
// Ensure writer has written each once
|
||||
while(wcount < 1)
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(10));
|
||||
std::cout << "wcount: " << wcount << " - time: " << (wtime/wcount) << std::endl;
|
||||
|
||||
auto reader = p.reader();
|
||||
std::vector<std::thread> readers;
|
||||
size_t time = 0;
|
||||
size_t count = 0;
|
||||
size_t nullcount = 0;
|
||||
|
||||
for (int x = 0; x<1; ++x) {
|
||||
readers.push_back(std::thread([&]() {
|
||||
while(running) {
|
||||
auto start = utils::TimeStamp::now();
|
||||
for (const auto& k : keys) {
|
||||
auto s = reader.acquire_snapshot(k);
|
||||
if (s == nullptr) nullcount++;
|
||||
(void)s;
|
||||
reader.release_snapshot(&s);
|
||||
// REQUIRE(s);
|
||||
}
|
||||
auto end = utils::TimeStamp::now();
|
||||
++count;
|
||||
time += (end - start).microseconds();
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
std::this_thread::sleep_for(std::chrono::seconds(10));
|
||||
running = false;
|
||||
twriter.join();
|
||||
for (auto& t : readers) t.join();
|
||||
|
||||
std::cout << "Write time: " << wtime/wcount << std::endl;
|
||||
std::cout << "Read time: " << time/count << std::endl;
|
||||
std::cout << "nullcount: " << nullcount << std::endl;
|
||||
REQUIRE(nullcount == 0);
|
||||
/*
|
||||
|
||||
Next step:
|
||||
|
||||
- Replace concurrent_hash with Lockbox<Unordered> and see if we can still make it happen
|
||||
|
||||
*/
|
||||
}
|
||||
|
||||
|
||||
|
||||
}}}
|
||||
#endif
|
||||
@@ -358,7 +358,7 @@ OrderManager::free_buffers(std::string const &session_name) {
|
||||
}
|
||||
|
||||
void OrderManager::init_order_services() {
|
||||
auto &log = me->services_data.log;
|
||||
auto& log = me->services_data.log;
|
||||
|
||||
for (auto &s : me->services_data.sessions) {
|
||||
auto gw_name = GatewayName(s->session);
|
||||
@@ -410,14 +410,13 @@ void OrderManager::init_order_services() {
|
||||
};
|
||||
s->exe_report_subs =
|
||||
std::make_unique<ContentSubscription<NewExecutionReportEvent>>(
|
||||
*this, gw_name, on_exe_r, log);
|
||||
*this, gw_name, on_exe_r,log);
|
||||
|
||||
s->cancel_reject_subs =
|
||||
std::make_unique<ContentSubscription<CancelRejectEvent>>(
|
||||
*this, gw_name, log);
|
||||
std::make_unique<ContentSubscription<CancelRejectEvent>>(*this,
|
||||
gw_name,log);
|
||||
s->gateway_status_subs = std::make_unique<
|
||||
ContentSubscription<GatewayStatusChangeEvent, NoKey>>(*this, gw_name,
|
||||
log);
|
||||
ContentSubscription<GatewayStatusChangeEvent, NoKey>>(*this, gw_name,log);
|
||||
|
||||
// publishers for execution reports, gateway status and cancel
|
||||
// reject evets
|
||||
@@ -495,18 +494,18 @@ void OrderManager::init_order_services() {
|
||||
me->services_data.cor_id = ev.trace;
|
||||
};
|
||||
me->services_data.md_subs =
|
||||
std::make_unique<ContentSubscription<BookUpdateEvent>>(
|
||||
*this, on_md_sub, on_book_update, log);
|
||||
std::make_unique<ContentSubscription<BookUpdateEvent>>(*this, on_md_sub,
|
||||
on_book_update,log);
|
||||
me->services_data.ticker_subs =
|
||||
std::make_unique<ContentSubscription<TickerUpdateEvent>>(
|
||||
*this, on_ticker_sub, log);
|
||||
std::make_unique<ContentSubscription<TickerUpdateEvent>>(*this,
|
||||
on_ticker_sub,log);
|
||||
me->services_data.settlement_subs =
|
||||
std::make_unique<ContentSubscription<SettlementUpdateEvent>>(
|
||||
*this, on_settlement_sub, log);
|
||||
*this, on_settlement_sub,log);
|
||||
me->services_data.time_subs =
|
||||
std::make_unique<ContentSubscription<TimerEvent, NoKey>>(*this, log);
|
||||
std::make_unique<ContentSubscription<TimerEvent, NoKey>>(*this,log);
|
||||
me->services_data.order_creation_subs =
|
||||
std::make_unique<ContentSubscription<CreateOrderReplyEvent>>(*this, log);
|
||||
std::make_unique<ContentSubscription<CreateOrderReplyEvent>>(*this,log);
|
||||
}
|
||||
|
||||
void OrderManager::after_event() {
|
||||
@@ -515,8 +514,7 @@ void OrderManager::after_event() {
|
||||
for (auto &osd : me->services_data.sessions) {
|
||||
auto &fix_stream = osd->current_buffer->first_stream();
|
||||
if (fix_stream.n_msgs()) {
|
||||
auto scoped_do_work = trace::log_scope(me->services_data.log, {},
|
||||
"OrderManager::after_event");
|
||||
auto scoped_do_work = trace::log_scope(me->services_data.log, {}, "OrderManager::after_event");
|
||||
fix_stream.close();
|
||||
osd->gw.send_fix(osd->current_buffer, osd->current_trace);
|
||||
reload_fix_buffer(*osd);
|
||||
@@ -538,10 +536,8 @@ void OrderManager::reload_fix_buffer(SessionServicesData &osd) {
|
||||
}
|
||||
}
|
||||
|
||||
void OrderManager::on_poll(trace::PerfLog &log) {
|
||||
auto scoped_do_work =
|
||||
trace::log_scope(log, {}, "OrderManager::dispatch_incoming_fix_messages",
|
||||
utils::TimeStamp::Interval::from_microseconds(2));
|
||||
void OrderManager::on_poll(trace::PerfLog&log) {
|
||||
auto scoped_do_work = trace::log_scope(log, {}, "OrderManager::dispatch_incoming_fix_messages");
|
||||
for (auto &osd : me->services_data.sessions)
|
||||
osd->gw.dispatch_incoming_fix_messages();
|
||||
}
|
||||
|
||||
@@ -24,6 +24,18 @@ class ConcurrentPool {
|
||||
delete [] b;
|
||||
}
|
||||
|
||||
#if 0
|
||||
T* allocate() {
|
||||
return new T();
|
||||
}
|
||||
void reserve(size_t ) {
|
||||
}
|
||||
void release(T** item) {
|
||||
if (*item) {
|
||||
delete *item;
|
||||
}
|
||||
}
|
||||
#else
|
||||
T* allocate() {
|
||||
T* result = nullptr;
|
||||
++_outstanding;
|
||||
@@ -46,6 +58,7 @@ class ConcurrentPool {
|
||||
_all.push_back(buffer);
|
||||
}
|
||||
|
||||
|
||||
void release(T** item) {
|
||||
if (*item) {
|
||||
(*item)->reset();
|
||||
@@ -55,6 +68,8 @@ class ConcurrentPool {
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
template <class CB>
|
||||
void unsafe_iterate(CB cb) const {
|
||||
for (auto iter = _all.begin(); iter != _all.end(); ++iter) {
|
||||
|
||||
@@ -316,12 +316,7 @@ unique_ptr<onixs::Handler> MDP3Main::prepare_handler(const string& channel_id, M
|
||||
|
||||
try {
|
||||
auto handler = unique_ptr<onixs::Handler>(new onixs::Handler(settings));
|
||||
auto iter = _engines.find(options.feed_engine());
|
||||
if (iter == _engines.end()) {
|
||||
LOG(md,Fatal, "Missing feed engine definition '{}' for channel id '{}'. Add a feed_engine entry with name:'{}' to the CMEOptions config key.", options.feed_engine(), channel_id, options.feed_engine());
|
||||
bts_assert_msg(iter != _engines.end(), "Missing feed engine definition.");
|
||||
}
|
||||
handler->bindFeedEngine(*(iter->second));
|
||||
handler->bindFeedEngine(*_engines.at(options.feed_engine()));
|
||||
if (recovery) {
|
||||
handler->bindTcpRecovery(*recovery);
|
||||
_recovery.push_back(std::move(recovery));
|
||||
|
||||
@@ -1,69 +0,0 @@
|
||||
Test Author: Pedro
|
||||
If it breaks: Pedro
|
||||
|
||||
Spreader Requote Performance test
|
||||
|
||||
Note: This test is fairly cumbersome to setup, because performance is
|
||||
very sensitive to a number of factors including compilation switches,
|
||||
hardware used, thread allocation and load on the machine. Pedro has
|
||||
the best handle on how do all this and should probably be the one
|
||||
doing this test.
|
||||
|
||||
|
||||
1) Make sure you are testing a release server build
|
||||
|
||||
2) Make sure you use real hardware, with hyper threading turned off
|
||||
and cstates disabled, for the server side
|
||||
|
||||
3) Make sure you isolate the CPU on which you are running the
|
||||
server. This is best done on Linux by editing /etc/default/grub like
|
||||
this:
|
||||
|
||||
# this isolates CPU's 0 to 5 for better performance testing
|
||||
GRUB_CMDLINE_LINUX="isolcpus=0-5"
|
||||
|
||||
4) Now make sure your configuration is set for the lowest latency,
|
||||
including dedicating a thread to the order manager on the isolated
|
||||
cores:
|
||||
|
||||
OrdermanCfg {
|
||||
value {
|
||||
executor {
|
||||
thread_name: "oman_orders"
|
||||
poll: true
|
||||
hosted_actor: "MDSource"
|
||||
hosted_actor: "OrderManager"
|
||||
}
|
||||
executor {
|
||||
thread_name: "oman_slow"
|
||||
poll: false
|
||||
hosted_actor: "*"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Threads {
|
||||
value {
|
||||
thread_group {name_rx: "mdp3feed" core: 0}
|
||||
thread_group {name_rx: "oman_orders" core: 1}
|
||||
thread_group {name_rx: "jf_ar_sim" core: 2}
|
||||
thread_group {name_rx: ".*" core:6 core:7 core:8 core:9 core:10 }
|
||||
}
|
||||
}
|
||||
|
||||
5) Now you are ready to start testing. Start a shell and create a new
|
||||
synthetic say with 2 CL futures. Use the near month (fast ticking)
|
||||
future as the non quoting leg, so that the other leg gets quoted
|
||||
often. Make sure to disable throtling in the synthetic definition
|
||||
dialog.
|
||||
|
||||
6) Place an order for this synthetic, away from the market.
|
||||
|
||||
7) Now run for say 3 minutes. Visit http://youserver:8000/perf_trace
|
||||
You should see a graph indicating the median requote times for the
|
||||
order you created in 5). Make sure you have at least 100 data points
|
||||
and data the median time is around 12us. If you get anything greater
|
||||
than 15us, the test has failed.
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user