Merge branch 'prod' of bitbucket.org:bluetradesys/spark into nlegs

This commit is contained in:
Taha Afzal committed 2018-06-29 19:14:56 -04:00
commit 8c68a66ef1
9 files changed
+122 -72

No files matched your search

+5
View File
@@ -76,6 +76,10 @@ message CMEOptions
repeated uint32 affinity = 10;
optional uint32 udp_socket_buffer_size = 11 [default = 33554432]; //32mb
}
enum BookType {
BOOKTYPE_CONSOLIDATED = 0;
BOOKTYPE_DIRECT = 1;
}
message CMEChannel {
repeated string id = 2;
optional string feed_engine = 3 [default = "default"];
@@ -98,6 +102,7 @@ message CMEOptions
optional bool join_snapshot_on_gap = 32 [default = false];
optional bool load_security_db = 33 [default = true];
optional BookType book_type = 34 [default = BOOKTYPE_CONSOLIDATED];
}
optional string onixs_license = 1 [default = "{default}"];
+12 -3
View File
@@ -199,9 +199,18 @@ void SecurityIDRecorder::bulk_insert(const std::vector<const api::Security*>& da
if (data.size() == 0) return;
sqlite3_exec_nofail(_self->db.get(), "BEGIN TRANSACTION");
std::string descriptor;
for (const api::Security* sec : data) {
descriptor = security_descriptor(*sec);
insert(prefix + descriptor, securitytype_of(*sec));
{
auto insert_stmt = prepare_query(_self->db, "INSERT INTO security_keys (rowid, descriptor, type) values (?, ?, ?)");
for (const api::Security* sec : data) {
descriptor = prefix + security_descriptor(*sec);
auto type = securitytype_of(*sec);
sqlite3_bind_int(insert_stmt.get(), 1, sec->id());
sqlite3_bind_text(insert_stmt.get(), 2, descriptor.c_str(), descriptor.size(), SQLITE_STATIC);
sqlite3_bind_int(insert_stmt.get(), 3, (int)type);
sqlite3_step(insert_stmt.get());
sqlite3_reset(insert_stmt.get());
}
}
sqlite3_exec_nofail(_self->db.get(), "COMMIT TRANSACTION");
}
-3
View File
@@ -121,9 +121,6 @@ void record_security_keys(md::SecurityIDRecorder& idrec, const co2::MDReply& rep
}
}
}
// Sort by id only actually works if the recording was done on a fresh machine
std::sort(data.begin(), data.end(), [](const Security* a, const Security* b) { return a->id() < b->id(); });
// We could insert garbage until data[0]->id()
idrec.bulk_insert(data, "REPLAY: ");
}
#endif
+6
View File
@@ -8,6 +8,7 @@
#include "lib/logging/logging.h"
#include "lib/os/os.h"
#include "lib/quickbus/quickbus.h"
#include "lib/utils/Dispatcher.h"
#include "lib/md/SyntheticSecurityService.h"
#include "server/sparkd/exchange_sim/ExchangeSim.h"
@@ -53,6 +54,11 @@ int Run::execute(std::vector<std::string> const &) {
orderman::OrderServiceServer order_service_server(sc, synthetic_service);
support::ErrorReportServer error_report_service(sc);
admin::AdminServiceServer admin_service(sc);
auto bg_health = std::make_shared<metrics::IntCounterMetric>("spark_background_thread");
utils::DispatcherTimer timer(utils::Dispatcher::background(), 5000, [=]() { bg_health->increment(); });
timer.start();
stats.publish(bg_health);
std::thread([]() {
os::set_thread_name("ExchangeSim");
@@ -1,8 +1,8 @@
#include "MDP3Listener.h"
#include "lib/utils/datetime.h"
#include <vector>
#include "server/sparkd/tracing/bts_trace.h"
#include "lib/utils/helper_macros.h"
#include "lib/config/protos/MarketData.pb.h"
using namespace std;
using namespace bts::co2;
@@ -84,7 +84,8 @@ MDP3Listener::MDP3Listener(
MDProducer& producer,
SecurityIDRecorder& recorder,
co2::SettlementRecorder& ,
std::vector<MDProducerChannel> channels) :
const std::vector<MDProducerChannel> channels,
CMEOptions::BookType bt) :
_index(index),
_producer(producer),
_packet_arrived(),
@@ -94,6 +95,7 @@ MDP3Listener::MDP3Listener(
_symbol_filter_storage(),
_symbol_filter(),
_channels(channels),
_booktype(bt),
_concurrency_monitor(0),
_book_updates()
{
@@ -188,11 +190,6 @@ double double_of_price(const Price& price, int8_t display_factor) {
return (price.mantissa() * pow10(price.exponent())) * pow10(display_factor);
}
inline PriceSize pricesize_of_price_size(const onixs::PRICE price, int8_t display_factor, Size size) {
PriceSize result(price.mantissa(), price.exponent() + display_factor, size);
return result;
}
inline datetime::LLDateTime ll_of_timestamp(const onixs::Timestamp& ts) {
return datetime::ll_from_parts(ts.year(), (int)ts.month(), ts.day(), ts.hour(), ts.minute(), ts.second(), ts.millisecond());
}
@@ -335,49 +332,18 @@ void MDP3Listener::onDefinition(onixs::Handler&, const onixs::Security&, const o
_index.add_spread_legs(fsec.numeric_id, leg_exchange_ids);
}
//
// Book
void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, const onixs::ConsolidatedBook& book) {
tracepoint (bts_trace,trace_order,(char*) "onBookUpdate", 0,_latest_sequence);
auto data = _index.get_security(sec.id());
if (data == nullptr) return;
bts_assert(data->id == (uint32_t)sec.id());
auto display_factor = data->handle.security().display_factor;
// Put the data into the cache. Flush at the end of the packet.
auto key = data->handle.key();
auto find_iter = std::find_if(_book_updates.begin(), _book_updates.end(), [key](const BookData& entry) {
return key == entry.handle.key();
});
bool found = find_iter != _book_updates.end();
if (!found) {
_book_updates.emplace_back(data->handle);
}
BookData& cache = found ? *find_iter : _book_updates.back();
// Overwrite all entries every time
size_t bids_size = book.bids().size();
size_t asks_size = book.offers().size();
if (!data->handle.has_depth()) {
bids_size = min(bids_size, size_t { 1 });
asks_size = min(asks_size, size_t { 1 });
}
for (size_t x = 0; x < bids_size; ++x) {
cache.bids[x] = pricesize_of_price_size(book.bids()[x].price(), display_factor, book.bids()[x].quantity());
}
for (size_t x = 0; x < asks_size; ++x) {
cache.asks[x] = pricesize_of_price_size(book.offers()[x].price(), display_factor, book.offers()[x].quantity());
}
// NaN out the remaining depth
for (size_t x = bids_size; x < COCONUT_MAX_DEPTH; ++x) {
cache.bids[x] = PriceSize::NO_MARKET;
}
for (size_t x = asks_size; x < COCONUT_MAX_DEPTH; ++x) {
cache.asks[x] = PriceSize::NO_MARKET;
void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, const onixs::DirectBook& book) {
if (_booktype == CMEOptions::BOOKTYPE_DIRECT) {
onBookUpdate(sec, book);
}
}
void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, const onixs::ConsolidatedBook& book) {
if (_booktype == CMEOptions::BOOKTYPE_CONSOLIDATED) {
onBookUpdate(sec, book);
}
}
//
// Trades
@@ -3,6 +3,7 @@
#include "lib/utils/bts_no_copy.h"
#include "lib/md/coconut2.h"
#include "CMESecurityIndex.h"
#include "server/sparkd/tracing/bts_trace.h"
#include "onixs.h"
#include <tbb/concurrent_queue.h>
@@ -13,13 +14,23 @@
namespace bts {
namespace md {
inline PriceSize pricesize_of_price_size(const onixs::PRICE price, int8_t display_factor, Size size) {
PriceSize result(price.mantissa(), price.exponent() + display_factor, size);
return result;
}
class MDP3Listener :
public onixs::HandlerListener,
public onixs::MarketDataListener,
public onixs::SecurityListener
{
public:
MDP3Listener(CMESecurityIndex& index, MDProducer&, SecurityIDRecorder&, co2::SettlementRecorder&, std::vector<MDProducerChannel>);
MDP3Listener(CMESecurityIndex& index,
MDProducer&,
SecurityIDRecorder&,
co2::SettlementRecorder&,
const std::vector<MDProducerChannel>,
CMEOptions::BookType);
virtual ~MDP3Listener();
//
@@ -51,10 +62,52 @@ public:
//
// Book
// void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::MboBook&) override;
// void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::DirectBook&) override;
void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::DirectBook&) override;
// void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::ImpliedBook&) override;
void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::ConsolidatedBook&) override;
template <class TBook>
void onBookUpdate(const onixs::Security& sec, const TBook& book) {
tracepoint (bts_trace,trace_order,(char*) "onBookUpdate", 0,_latest_sequence);
auto data = _index.get_security(sec.id());
if (data == nullptr) return;
bts_assert(data->id == (uint32_t)sec.id());
auto display_factor = data->handle.security().display_factor;
// Put the data into the cache. Flush at the end of the packet.
auto key = data->handle.key();
auto find_iter = std::find_if(_book_updates.begin(), _book_updates.end(), [key](const BookData& entry) {
return key == entry.handle.key();
});
bool found = find_iter != _book_updates.end();
if (!found) {
_book_updates.emplace_back(data->handle);
}
BookData& cache = found ? *find_iter : _book_updates.back();
// Overwrite all entries every time
size_t bids_size = book.bids().size();
size_t asks_size = book.offers().size();
if (!data->handle.has_depth()) {
bids_size = std::min(bids_size, size_t { 1 });
asks_size = std::min(asks_size, size_t { 1 });
}
for (size_t x = 0; x < bids_size; ++x) {
cache.bids[x] = pricesize_of_price_size(book.bids()[x].price(), display_factor, book.bids()[x].quantity());
}
for (size_t x = 0; x < asks_size; ++x) {
cache.asks[x] = pricesize_of_price_size(book.offers()[x].price(), display_factor, book.offers()[x].quantity());
}
// NaN out the remaining depth
for (size_t x = bids_size; x < COCONUT_MAX_DEPTH; ++x) {
cache.bids[x] = PriceSize::NO_MARKET;
}
for (size_t x = asks_size; x < COCONUT_MAX_DEPTH; ++x) {
cache.asks[x] = PriceSize::NO_MARKET;
}
}
//
// Trades
void onTrade(onixs::Handler&, const onixs::Security&, const onixs::RecoveryEntryArgs&) override;
@@ -125,6 +178,7 @@ private:
std::vector<std::string> _symbol_filter_storage;
std::set<onixs::StrRef> _symbol_filter;
std::vector<MDProducerChannel> _channels;
CMEOptions::BookType _booktype;
std::atomic<int> _concurrency_monitor;
struct BookData {
@@ -213,7 +213,9 @@ unique_ptr<onixs::Handler> MDP3Main::prepare_handler(const string& channel_id) {
settings.feeds().connectivityConfigurationFile(config_path());
// Market data options
settings.bookManagement().consolidatedBooks().maintain(true);
if (options.book_type() == CMEOptions::BOOKTYPE_CONSOLIDATED) {
settings.bookManagement().consolidatedBooks().maintain(true);
}
// 10k default is *far* too low
settings.recoveryQueueMaxSize(options.recovery_queue_size());
@@ -334,12 +336,13 @@ unique_ptr<MDP3Listener> MDP3Main::prepare_listener(onixs::Handler& handler, con
producer_channels.push_back(_producer.get_channel(symbol.symbol().c_str(), symbol.type(), symbol.exchange()));
}
unique_ptr<MDP3Listener> listener(new MDP3Listener(_index, _producer, _security_recorder, _settlement_recorder, producer_channels));
const auto& options = channel_settings_for(channel.id);
unique_ptr<MDP3Listener> listener(new MDP3Listener(_index, _producer, _security_recorder, _settlement_recorder, producer_channels, options.book_type()));
for (auto& symbol : channel.symbols) {
listener->add_filter_symbol(symbol.symbol());
}
const auto& options = channel_settings_for(channel.id);
if (options.load_security_db()) {
// Load the list of known securities and add them to the index for fast startup
db::load_securities(channel.symbols, [&](const api::Security& security) {
+1
View File
@@ -42,6 +42,7 @@ struct CounterMetric : public Metric {
void decrement(T val) { _val -= val; }
void set(T val) { _val = val; }
};
typedef CounterMetric<int> IntCounterMetric;
template <class T>
class AtomicCounterMetric : public Metric {
@@ -88,7 +88,8 @@ void IcebergOrder::on_init_recovered(api::orderman::OrderRecord const &rec,
}
}
void IcebergOrder::on_activate(Price p, Quantity q, OrderServices &services, TickToTradeCorID) {
void IcebergOrder::on_activate(Price p, Quantity q, OrderServices &services,
TickToTradeCorID) {
bts_assert(!me->services);
me->services = &services;
me->child_builder.activate(services);
@@ -184,6 +185,7 @@ void IcebergOrder::transition(api::orderman::OrderStatus new_status,
auto const &to = OrderStatus_Name(new_status);
LOG(OrderMan, Info, "Transitioning iceberg order {} from {} to {}. Msg: {}",
key().value(), from, to, msg);
me->state.set_last_msg(msg);
me->state.set_status(new_status);
}
@@ -229,8 +231,9 @@ void IcebergOrder::send_next_child_order() {
auto remaining = me->state.quantity() - me->state.filled_quantity();
auto to_send = std::min(remaining, me->show_quantity.value());
me->child.order = me->child_builder.build();
LOG(OrderMan, Info, "Iceberg order {} creating new child order {} with size "
"{} price {}. Filled {} so far out of {}",
LOG(OrderMan, Info,
"Iceberg order {} creating new child order {} with size "
"{} price {}. Filled {} so far out of {}",
key().value(), me->child.order.key().value(), to_send, me->state.price(),
me->state.filled_quantity(), me->state.quantity());
@@ -251,14 +254,20 @@ void IcebergOrder::on_child_order_update(OrderStateUpdateEvent const &ev) {
break;
case ORDER_STATUS_COMPLETED: {
api::orderman::OrderState merged_state;
merge_states(me->state, me->child.state, &merged_state);
me->state = merged_state;
me->child.clear();
if (me->state.quantity() > me->state.filled_quantity())
send_next_child_order();
else
transition(ORDER_STATUS_COMPLETED);
if (me->child.state.filled_quantity() > 0) {
api::orderman::OrderState merged_state;
merge_states(me->state, me->child.state, &merged_state);
me->state = merged_state;
me->child.clear();
if (me->state.quantity() > me->state.filled_quantity())
send_next_child_order();
else
transition(ORDER_STATUS_COMPLETED);
} else {
me->child.clear();
transition(ORDER_STATUS_ERROR,
"Child order completed without fills");
}
break;
}
@@ -302,5 +311,5 @@ void IcebergOrder::on_child_order_update(OrderStateUpdateEvent const &ev) {
}
report_state();
}
}
}
} // namespace orderman
} // namespace bts