Merge branch 'prod' of bitbucket.org:bluetradesys/spark into nlegs
This commit is contained in:
commit
8c68a66ef1
9 files changed
+122
-72
No files matched your search
@@ -76,6 +76,10 @@ message CMEOptions
|
||||
repeated uint32 affinity = 10;
|
||||
optional uint32 udp_socket_buffer_size = 11 [default = 33554432]; //32mb
|
||||
}
|
||||
enum BookType {
|
||||
BOOKTYPE_CONSOLIDATED = 0;
|
||||
BOOKTYPE_DIRECT = 1;
|
||||
}
|
||||
message CMEChannel {
|
||||
repeated string id = 2;
|
||||
optional string feed_engine = 3 [default = "default"];
|
||||
@@ -98,6 +102,7 @@ message CMEOptions
|
||||
optional bool join_snapshot_on_gap = 32 [default = false];
|
||||
|
||||
optional bool load_security_db = 33 [default = true];
|
||||
optional BookType book_type = 34 [default = BOOKTYPE_CONSOLIDATED];
|
||||
|
||||
}
|
||||
optional string onixs_license = 1 [default = "{default}"];
|
||||
|
||||
@@ -199,9 +199,18 @@ void SecurityIDRecorder::bulk_insert(const std::vector<const api::Security*>& da
|
||||
if (data.size() == 0) return;
|
||||
sqlite3_exec_nofail(_self->db.get(), "BEGIN TRANSACTION");
|
||||
std::string descriptor;
|
||||
for (const api::Security* sec : data) {
|
||||
descriptor = security_descriptor(*sec);
|
||||
insert(prefix + descriptor, securitytype_of(*sec));
|
||||
{
|
||||
auto insert_stmt = prepare_query(_self->db, "INSERT INTO security_keys (rowid, descriptor, type) values (?, ?, ?)");
|
||||
for (const api::Security* sec : data) {
|
||||
descriptor = prefix + security_descriptor(*sec);
|
||||
auto type = securitytype_of(*sec);
|
||||
|
||||
sqlite3_bind_int(insert_stmt.get(), 1, sec->id());
|
||||
sqlite3_bind_text(insert_stmt.get(), 2, descriptor.c_str(), descriptor.size(), SQLITE_STATIC);
|
||||
sqlite3_bind_int(insert_stmt.get(), 3, (int)type);
|
||||
sqlite3_step(insert_stmt.get());
|
||||
sqlite3_reset(insert_stmt.get());
|
||||
}
|
||||
}
|
||||
sqlite3_exec_nofail(_self->db.get(), "COMMIT TRANSACTION");
|
||||
}
|
||||
|
||||
@@ -121,9 +121,6 @@ void record_security_keys(md::SecurityIDRecorder& idrec, const co2::MDReply& rep
|
||||
}
|
||||
}
|
||||
}
|
||||
// Sort by id only actually works if the recording was done on a fresh machine
|
||||
std::sort(data.begin(), data.end(), [](const Security* a, const Security* b) { return a->id() < b->id(); });
|
||||
// We could insert garbage until data[0]->id()
|
||||
idrec.bulk_insert(data, "REPLAY: ");
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
#include "lib/logging/logging.h"
|
||||
#include "lib/os/os.h"
|
||||
#include "lib/quickbus/quickbus.h"
|
||||
#include "lib/utils/Dispatcher.h"
|
||||
|
||||
#include "lib/md/SyntheticSecurityService.h"
|
||||
#include "server/sparkd/exchange_sim/ExchangeSim.h"
|
||||
@@ -53,6 +54,11 @@ int Run::execute(std::vector<std::string> const &) {
|
||||
orderman::OrderServiceServer order_service_server(sc, synthetic_service);
|
||||
support::ErrorReportServer error_report_service(sc);
|
||||
admin::AdminServiceServer admin_service(sc);
|
||||
|
||||
auto bg_health = std::make_shared<metrics::IntCounterMetric>("spark_background_thread");
|
||||
utils::DispatcherTimer timer(utils::Dispatcher::background(), 5000, [=]() { bg_health->increment(); });
|
||||
timer.start();
|
||||
stats.publish(bg_health);
|
||||
|
||||
std::thread([]() {
|
||||
os::set_thread_name("ExchangeSim");
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
#include "MDP3Listener.h"
|
||||
#include "lib/utils/datetime.h"
|
||||
#include <vector>
|
||||
#include "server/sparkd/tracing/bts_trace.h"
|
||||
#include "lib/utils/helper_macros.h"
|
||||
#include "lib/config/protos/MarketData.pb.h"
|
||||
|
||||
using namespace std;
|
||||
using namespace bts::co2;
|
||||
@@ -84,7 +84,8 @@ MDP3Listener::MDP3Listener(
|
||||
MDProducer& producer,
|
||||
SecurityIDRecorder& recorder,
|
||||
co2::SettlementRecorder& ,
|
||||
std::vector<MDProducerChannel> channels) :
|
||||
const std::vector<MDProducerChannel> channels,
|
||||
CMEOptions::BookType bt) :
|
||||
_index(index),
|
||||
_producer(producer),
|
||||
_packet_arrived(),
|
||||
@@ -94,6 +95,7 @@ MDP3Listener::MDP3Listener(
|
||||
_symbol_filter_storage(),
|
||||
_symbol_filter(),
|
||||
_channels(channels),
|
||||
_booktype(bt),
|
||||
_concurrency_monitor(0),
|
||||
_book_updates()
|
||||
{
|
||||
@@ -188,11 +190,6 @@ double double_of_price(const Price& price, int8_t display_factor) {
|
||||
return (price.mantissa() * pow10(price.exponent())) * pow10(display_factor);
|
||||
}
|
||||
|
||||
inline PriceSize pricesize_of_price_size(const onixs::PRICE price, int8_t display_factor, Size size) {
|
||||
PriceSize result(price.mantissa(), price.exponent() + display_factor, size);
|
||||
return result;
|
||||
}
|
||||
|
||||
inline datetime::LLDateTime ll_of_timestamp(const onixs::Timestamp& ts) {
|
||||
return datetime::ll_from_parts(ts.year(), (int)ts.month(), ts.day(), ts.hour(), ts.minute(), ts.second(), ts.millisecond());
|
||||
}
|
||||
@@ -335,49 +332,18 @@ void MDP3Listener::onDefinition(onixs::Handler&, const onixs::Security&, const o
|
||||
_index.add_spread_legs(fsec.numeric_id, leg_exchange_ids);
|
||||
}
|
||||
|
||||
|
||||
//
|
||||
// Book
|
||||
void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, const onixs::ConsolidatedBook& book) {
|
||||
tracepoint (bts_trace,trace_order,(char*) "onBookUpdate", 0,_latest_sequence);
|
||||
|
||||
auto data = _index.get_security(sec.id());
|
||||
if (data == nullptr) return;
|
||||
bts_assert(data->id == (uint32_t)sec.id());
|
||||
auto display_factor = data->handle.security().display_factor;
|
||||
|
||||
// Put the data into the cache. Flush at the end of the packet.
|
||||
auto key = data->handle.key();
|
||||
auto find_iter = std::find_if(_book_updates.begin(), _book_updates.end(), [key](const BookData& entry) {
|
||||
return key == entry.handle.key();
|
||||
});
|
||||
bool found = find_iter != _book_updates.end();
|
||||
if (!found) {
|
||||
_book_updates.emplace_back(data->handle);
|
||||
}
|
||||
BookData& cache = found ? *find_iter : _book_updates.back();
|
||||
|
||||
// Overwrite all entries every time
|
||||
size_t bids_size = book.bids().size();
|
||||
size_t asks_size = book.offers().size();
|
||||
if (!data->handle.has_depth()) {
|
||||
bids_size = min(bids_size, size_t { 1 });
|
||||
asks_size = min(asks_size, size_t { 1 });
|
||||
}
|
||||
for (size_t x = 0; x < bids_size; ++x) {
|
||||
cache.bids[x] = pricesize_of_price_size(book.bids()[x].price(), display_factor, book.bids()[x].quantity());
|
||||
}
|
||||
for (size_t x = 0; x < asks_size; ++x) {
|
||||
cache.asks[x] = pricesize_of_price_size(book.offers()[x].price(), display_factor, book.offers()[x].quantity());
|
||||
}
|
||||
// NaN out the remaining depth
|
||||
for (size_t x = bids_size; x < COCONUT_MAX_DEPTH; ++x) {
|
||||
cache.bids[x] = PriceSize::NO_MARKET;
|
||||
}
|
||||
for (size_t x = asks_size; x < COCONUT_MAX_DEPTH; ++x) {
|
||||
cache.asks[x] = PriceSize::NO_MARKET;
|
||||
void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, const onixs::DirectBook& book) {
|
||||
if (_booktype == CMEOptions::BOOKTYPE_DIRECT) {
|
||||
onBookUpdate(sec, book);
|
||||
}
|
||||
}
|
||||
|
||||
void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, const onixs::ConsolidatedBook& book) {
|
||||
if (_booktype == CMEOptions::BOOKTYPE_CONSOLIDATED) {
|
||||
onBookUpdate(sec, book);
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// Trades
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
#include "lib/utils/bts_no_copy.h"
|
||||
#include "lib/md/coconut2.h"
|
||||
#include "CMESecurityIndex.h"
|
||||
#include "server/sparkd/tracing/bts_trace.h"
|
||||
|
||||
#include "onixs.h"
|
||||
#include <tbb/concurrent_queue.h>
|
||||
@@ -13,13 +14,23 @@
|
||||
namespace bts {
|
||||
namespace md {
|
||||
|
||||
inline PriceSize pricesize_of_price_size(const onixs::PRICE price, int8_t display_factor, Size size) {
|
||||
PriceSize result(price.mantissa(), price.exponent() + display_factor, size);
|
||||
return result;
|
||||
}
|
||||
|
||||
class MDP3Listener :
|
||||
public onixs::HandlerListener,
|
||||
public onixs::MarketDataListener,
|
||||
public onixs::SecurityListener
|
||||
{
|
||||
public:
|
||||
MDP3Listener(CMESecurityIndex& index, MDProducer&, SecurityIDRecorder&, co2::SettlementRecorder&, std::vector<MDProducerChannel>);
|
||||
MDP3Listener(CMESecurityIndex& index,
|
||||
MDProducer&,
|
||||
SecurityIDRecorder&,
|
||||
co2::SettlementRecorder&,
|
||||
const std::vector<MDProducerChannel>,
|
||||
CMEOptions::BookType);
|
||||
virtual ~MDP3Listener();
|
||||
|
||||
//
|
||||
@@ -51,10 +62,52 @@ public:
|
||||
//
|
||||
// Book
|
||||
// void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::MboBook&) override;
|
||||
// void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::DirectBook&) override;
|
||||
void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::DirectBook&) override;
|
||||
// void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::ImpliedBook&) override;
|
||||
void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::ConsolidatedBook&) override;
|
||||
|
||||
template <class TBook>
|
||||
void onBookUpdate(const onixs::Security& sec, const TBook& book) {
|
||||
tracepoint (bts_trace,trace_order,(char*) "onBookUpdate", 0,_latest_sequence);
|
||||
|
||||
auto data = _index.get_security(sec.id());
|
||||
if (data == nullptr) return;
|
||||
bts_assert(data->id == (uint32_t)sec.id());
|
||||
auto display_factor = data->handle.security().display_factor;
|
||||
|
||||
// Put the data into the cache. Flush at the end of the packet.
|
||||
auto key = data->handle.key();
|
||||
auto find_iter = std::find_if(_book_updates.begin(), _book_updates.end(), [key](const BookData& entry) {
|
||||
return key == entry.handle.key();
|
||||
});
|
||||
bool found = find_iter != _book_updates.end();
|
||||
if (!found) {
|
||||
_book_updates.emplace_back(data->handle);
|
||||
}
|
||||
BookData& cache = found ? *find_iter : _book_updates.back();
|
||||
|
||||
// Overwrite all entries every time
|
||||
size_t bids_size = book.bids().size();
|
||||
size_t asks_size = book.offers().size();
|
||||
if (!data->handle.has_depth()) {
|
||||
bids_size = std::min(bids_size, size_t { 1 });
|
||||
asks_size = std::min(asks_size, size_t { 1 });
|
||||
}
|
||||
for (size_t x = 0; x < bids_size; ++x) {
|
||||
cache.bids[x] = pricesize_of_price_size(book.bids()[x].price(), display_factor, book.bids()[x].quantity());
|
||||
}
|
||||
for (size_t x = 0; x < asks_size; ++x) {
|
||||
cache.asks[x] = pricesize_of_price_size(book.offers()[x].price(), display_factor, book.offers()[x].quantity());
|
||||
}
|
||||
// NaN out the remaining depth
|
||||
for (size_t x = bids_size; x < COCONUT_MAX_DEPTH; ++x) {
|
||||
cache.bids[x] = PriceSize::NO_MARKET;
|
||||
}
|
||||
for (size_t x = asks_size; x < COCONUT_MAX_DEPTH; ++x) {
|
||||
cache.asks[x] = PriceSize::NO_MARKET;
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// Trades
|
||||
void onTrade(onixs::Handler&, const onixs::Security&, const onixs::RecoveryEntryArgs&) override;
|
||||
@@ -125,6 +178,7 @@ private:
|
||||
std::vector<std::string> _symbol_filter_storage;
|
||||
std::set<onixs::StrRef> _symbol_filter;
|
||||
std::vector<MDProducerChannel> _channels;
|
||||
CMEOptions::BookType _booktype;
|
||||
std::atomic<int> _concurrency_monitor;
|
||||
|
||||
struct BookData {
|
||||
|
||||
@@ -213,7 +213,9 @@ unique_ptr<onixs::Handler> MDP3Main::prepare_handler(const string& channel_id) {
|
||||
settings.feeds().connectivityConfigurationFile(config_path());
|
||||
|
||||
// Market data options
|
||||
settings.bookManagement().consolidatedBooks().maintain(true);
|
||||
if (options.book_type() == CMEOptions::BOOKTYPE_CONSOLIDATED) {
|
||||
settings.bookManagement().consolidatedBooks().maintain(true);
|
||||
}
|
||||
|
||||
// 10k default is *far* too low
|
||||
settings.recoveryQueueMaxSize(options.recovery_queue_size());
|
||||
@@ -334,12 +336,13 @@ unique_ptr<MDP3Listener> MDP3Main::prepare_listener(onixs::Handler& handler, con
|
||||
producer_channels.push_back(_producer.get_channel(symbol.symbol().c_str(), symbol.type(), symbol.exchange()));
|
||||
}
|
||||
|
||||
unique_ptr<MDP3Listener> listener(new MDP3Listener(_index, _producer, _security_recorder, _settlement_recorder, producer_channels));
|
||||
const auto& options = channel_settings_for(channel.id);
|
||||
|
||||
unique_ptr<MDP3Listener> listener(new MDP3Listener(_index, _producer, _security_recorder, _settlement_recorder, producer_channels, options.book_type()));
|
||||
for (auto& symbol : channel.symbols) {
|
||||
listener->add_filter_symbol(symbol.symbol());
|
||||
}
|
||||
|
||||
const auto& options = channel_settings_for(channel.id);
|
||||
if (options.load_security_db()) {
|
||||
// Load the list of known securities and add them to the index for fast startup
|
||||
db::load_securities(channel.symbols, [&](const api::Security& security) {
|
||||
|
||||
@@ -42,6 +42,7 @@ struct CounterMetric : public Metric {
|
||||
void decrement(T val) { _val -= val; }
|
||||
void set(T val) { _val = val; }
|
||||
};
|
||||
typedef CounterMetric<int> IntCounterMetric;
|
||||
|
||||
template <class T>
|
||||
class AtomicCounterMetric : public Metric {
|
||||
|
||||
@@ -88,7 +88,8 @@ void IcebergOrder::on_init_recovered(api::orderman::OrderRecord const &rec,
|
||||
}
|
||||
}
|
||||
|
||||
void IcebergOrder::on_activate(Price p, Quantity q, OrderServices &services, TickToTradeCorID) {
|
||||
void IcebergOrder::on_activate(Price p, Quantity q, OrderServices &services,
|
||||
TickToTradeCorID) {
|
||||
bts_assert(!me->services);
|
||||
me->services = &services;
|
||||
me->child_builder.activate(services);
|
||||
@@ -184,6 +185,7 @@ void IcebergOrder::transition(api::orderman::OrderStatus new_status,
|
||||
auto const &to = OrderStatus_Name(new_status);
|
||||
LOG(OrderMan, Info, "Transitioning iceberg order {} from {} to {}. Msg: {}",
|
||||
key().value(), from, to, msg);
|
||||
me->state.set_last_msg(msg);
|
||||
me->state.set_status(new_status);
|
||||
}
|
||||
|
||||
@@ -229,8 +231,9 @@ void IcebergOrder::send_next_child_order() {
|
||||
auto remaining = me->state.quantity() - me->state.filled_quantity();
|
||||
auto to_send = std::min(remaining, me->show_quantity.value());
|
||||
me->child.order = me->child_builder.build();
|
||||
LOG(OrderMan, Info, "Iceberg order {} creating new child order {} with size "
|
||||
"{} price {}. Filled {} so far out of {}",
|
||||
LOG(OrderMan, Info,
|
||||
"Iceberg order {} creating new child order {} with size "
|
||||
"{} price {}. Filled {} so far out of {}",
|
||||
key().value(), me->child.order.key().value(), to_send, me->state.price(),
|
||||
me->state.filled_quantity(), me->state.quantity());
|
||||
|
||||
@@ -251,14 +254,20 @@ void IcebergOrder::on_child_order_update(OrderStateUpdateEvent const &ev) {
|
||||
break;
|
||||
|
||||
case ORDER_STATUS_COMPLETED: {
|
||||
api::orderman::OrderState merged_state;
|
||||
merge_states(me->state, me->child.state, &merged_state);
|
||||
me->state = merged_state;
|
||||
me->child.clear();
|
||||
if (me->state.quantity() > me->state.filled_quantity())
|
||||
send_next_child_order();
|
||||
else
|
||||
transition(ORDER_STATUS_COMPLETED);
|
||||
if (me->child.state.filled_quantity() > 0) {
|
||||
api::orderman::OrderState merged_state;
|
||||
merge_states(me->state, me->child.state, &merged_state);
|
||||
me->state = merged_state;
|
||||
me->child.clear();
|
||||
if (me->state.quantity() > me->state.filled_quantity())
|
||||
send_next_child_order();
|
||||
else
|
||||
transition(ORDER_STATUS_COMPLETED);
|
||||
} else {
|
||||
me->child.clear();
|
||||
transition(ORDER_STATUS_ERROR,
|
||||
"Child order completed without fills");
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -302,5 +311,5 @@ void IcebergOrder::on_child_order_update(OrderStateUpdateEvent const &ev) {
|
||||
}
|
||||
report_state();
|
||||
}
|
||||
}
|
||||
}
|
||||
} // namespace orderman
|
||||
} // namespace bts
|
||||
Reference in new issue
Block a user