This commit is contained in:
Pedro Pinto committed 2018-10-24 13:46:41 -04:00
commit dc2879e2ff
256 files changed
+11092 -3935

No files matched your search

+1 -1
View File
@@ -1 +1 @@
set(sources bts_trace.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/Help.cpp;cmds/Md.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/RunSim.cpp;cmds/Span.cpp;cmds/Test.cpp;cmds/TestCeloxica.cpp;cmds/Web.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/WebServer.cpp;main.cpp;md/platforms/celoxica/CXListener.cpp;md/platforms/celoxica/CXMain.cpp;md/platforms/celoxica/tests/cx_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IMarginObserver.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/PositionServiceCore.cpp;position/PositionServiceServer.cpp;position/tests/PositionTests.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceServer.cpp;user/UserServiceServer.cpp;user/tests/admin_service_tests.cpp)
set(sources bts_trace.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/Help.cpp;cmds/Md.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/RunSim.cpp;cmds/Span.cpp;cmds/Test.cpp;cmds/TestCeloxica.cpp;cmds/Web.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/WebServer.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/celoxica/CXListener.cpp;md/platforms/celoxica/CXMain.cpp;md/platforms/celoxica/tests/cx_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IMarginObserver.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/PositionServiceCore.cpp;position/PositionServiceServer.cpp;position/tests/PositionTests.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceServer.cpp;user/UserServiceServer.cpp;user/tests/admin_service_tests.cpp)
+26 -12
View File
@@ -28,7 +28,9 @@
#include <cstdlib>
#include <iostream>
#include <moodycamel/concurrentqueue.h>
#include <moodycamel/readerwriterqueue.h>
#include <mutex>
#include <tbb/concurrent_queue.h>
namespace bts {
@@ -444,18 +446,6 @@ static void tbb_concurrent_queue(benchmark::State &state) {
}
BENCHMARK(tbb_concurrent_queue);
static void moody_camel_queue(benchmark::State &state) {
using Q = moodycamel::ConcurrentQueue<void *>;
Q queue(1000000);
for (auto _ : state) {
void *result;
void *arg = &queue;
queue.enqueue(arg); // NOLINT
benchmark::DoNotOptimize(queue.try_dequeue(result));
}
}
BENCHMARK(moody_camel_queue);
static void boost_spsc(benchmark::State &state) {
boost::lockfree::spsc_queue<int> q(10000);
@@ -658,6 +648,30 @@ static void same_thread_mpmc_spdlog(benchmark::State &state) {
}
BENCHMARK(same_thread_mpmc_spdlog);
static void moodycamel_concurrent_queue(benchmark::State &state) {
using Q = moodycamel::ConcurrentQueue<int>;
Q queue(8);
int i = 0;
for (auto _ : state) {
queue.enqueue(++i); // NOLINT
benchmark::DoNotOptimize(queue.try_dequeue(i));
}
}
BENCHMARK(moodycamel_concurrent_queue);
static void moodycamel_spsc_queue(benchmark::State &state) {
using Queue = moodycamel::ReaderWriterQueue<int>;
Queue q1(8);
int i = 0;
for (auto _ : state) {
q1.enqueue(++i);
benchmark::DoNotOptimize(q1.try_dequeue(i));
}
}
BENCHMARK(moodycamel_spsc_queue);
std::string Benchmark::help() {
return "Run micro-benchmarks. Use --benchmark_filter=regex to run only "
"specific benchmarks.";
@@ -0,0 +1,33 @@
#include "server/sparkd/cmds/PrintLeggedHedges.h"
#include "lib/persistence/SqliteStore.h"
#include "lib/api/services/OrderService.pb.h"
#include "lib/utils/PBUtils.h"
namespace bts {
namespace cmd {
int PrintLeggedHedges::execute(std::vector<std::string> const &) {
persistence::SqliteStore store("orders");
auto all = store.retrieve_all()["orders"];
std::cout << "order_key" << "\n";
api::orderman::OrderRecord rec;
for (auto const &kvp : all)
if (utils::pb::from_string<api::orderman::OrderRecord>(kvp.second, &rec)) {
auto &def = rec.def();
auto &state = rec.state();
if (rec.parent_key() &&
def.custom_def().has_limit() &&
def.sub_type() == api::orderman::OrderSubType::ORDER_SUB_TYPE_HEDGING_LEG) {
if (state.last_change().seconds()-state.created().seconds())
std::cout << kvp.first << "\n";
}
}
return 0;
}
std::string PrintLeggedHedges::help() {
return "Print out order ids for all hedge orders which were completed at least one second being created";
}
}}
@@ -0,0 +1,17 @@
#pragma once
#include "ICmd.h"
namespace bts {
namespace cmd {
class PrintLeggedHedges : public ICmd
{
public:
int execute(std::vector<std::string> const &args);
std::string help ();
};
}}
+3 -6
View File
@@ -18,15 +18,14 @@
#include "lib/orderman/server/OrderServiceServer.h"
#include "lib/persistence/PersistenceServiceServer.h"
#include "lib/utils/thread_manager.h"
#include "server/sparkd/exchange_sim/ExchangeSim.h"
#include "server/sparkd/http/WebServer.h"
#include "server/sparkd/md/run_md.h"
#include "server/sparkd/position/PositionServiceServer.h"
#include "server/sparkd/support/ErrorReportService.h"
#include "server/sparkd/support/ShellLatencyServer.h"
#include "server/sparkd/user/AdminServiceServer.h"
#include "server/sparkd/md/MarketDataAuxiliaryServiceServer.h"
#include "server/sparkd/user/UserServiceServer.h"
#include <thread>
namespace bts {
@@ -60,7 +59,8 @@ int Run::execute(std::vector<std::string> const &) {
orderman::OrderServiceServer order_service_server(sc, synthetic_service,
stats, perf_log);
support::ErrorReportServer error_report_service(sc);
admin::AdminServiceServer admin_service(sc);
admin::AdminServiceServer admin_service(sc);
md::auxiliary::MarketDataAuxiliaryServiceServer md_aux_server(sc);
position::PositionServiceServer position_service(sc);
metrics::ShellLatencyServer shell_latency_service(
sc, stats, utils::Dispatcher::background());
@@ -77,9 +77,6 @@ int Run::execute(std::vector<std::string> const &) {
timer2.start();
stats.publish(bg_health);
threading::make_thread("ExchangeSim", []() { sim::start_exchange_sim(); })
.detach();
std::this_thread::sleep_until(
std::chrono::time_point<std::chrono::system_clock>::max());
return 0;
+48 -10
View File
@@ -11,6 +11,7 @@
#include "lib/os/os.h"
#include "lib/config/protos/Ports.pb.h"
#include "lib/config/protos/ExchangeSimCfg.pb.h"
#include "lib/config/config.h"
#include <quickfix/FileLog.h>
@@ -18,6 +19,7 @@
#include "lib/md/relay/RelayClient.h"
#include "lib/domain_model/Security.pb.h"
#include "lib/md/relay/RelayRecorder.h"
#include "lib/md/IntPrice.h"
#include <set>
@@ -41,12 +43,18 @@ auto cfg =
"ValidateUserDefinedFields=N\n"
"ValidateFieldsOutOfOrder=N\n";
md::PriceSize coerce_precision(const co2::PriceSizeMessage& p) {
md::IntPrice raw = { p.mantissa(), (int8_t)p.exponent() };
auto price = md::IntPrice_from_double<ex_sim::precision>(raw.price());
return { price, p.size() };
}
struct MDPlayer : md::IRelayWriter
{
MDPlayer (IMDPlayerObserver &obs,std::map<uint32_t,md::Security> &known_securities) :
obs (obs),
known_securities(known_securities)
known_securities(known_securities),
forward_implied(config::get<config::ExchangeSim>().forward_implied())
{}
@@ -65,21 +73,51 @@ struct MDPlayer : md::IRelayWriter
else
sec_ptr = &it->second;
auto const &book = cr.book();
auto const &book = cr.direct();
md::PriceSize bids[md::COCONUT_MAX_DEPTH], asks[md::COCONUT_MAX_DEPTH];
size_t i = 0;
for (auto &b : book.bids())
bids[i++] = md::PriceSize(b.mantissa(),b.exponent(),b.size());
bids[i++] = md::PriceSize({b.mantissa(),(int8_t)b.exponent()},b.size());
size_t n = 0;
for (auto &a : book.asks())
asks[n++] = md::PriceSize(a.mantissa(),a.exponent(),a.size());
obs.on_depth(*sec_ptr, bids, asks);
asks[n++] = md::PriceSize({a.mantissa(),(int8_t)a.exponent()},a.size());
std::array<md::PriceSize, md::COCONUT_MAX_IMPLIED_DEPTH> implied_bids, implied_asks;
bool has_implied = false;
{size_t x = 0;
for (const auto& b : cr.implied().bids()) {
implied_bids[x] = coerce_precision(b);
x += 1;
has_implied = true;
}}
{size_t x = 0;
for (const auto& b : cr.implied().asks()) {
implied_asks[x] = coerce_precision(b);
x += 1;
has_implied = true;
}}
if (has_implied && forward_implied) {
obs.on_depth(*sec_ptr, bids, asks, &implied_bids, &implied_asks);
}
else {
obs.on_depth(*sec_ptr, bids, asks, nullptr, nullptr);
}
if (cr.has_misc()) {
if (cr.misc().has_settlement()) {
const auto& s = cr.misc().settlement();
obs.on_settlement(*sec_ptr, { s.mantissa(), (int8_t)s.exponent() });
}
}
}
}
IMDPlayerObserver &obs;
std::map<uint32_t, md::Security> &known_securities;
bool forward_implied;
};
@@ -104,22 +142,22 @@ int start_exchange_sim()
acceptor.start();
std::map<uint32_t,md::Security> known_securities;
const auto& conf = config::get<md::SimSource>();
const auto& conf = config::get<config::ExchangeSim>();
while ( true ) {
MDPlayer player(application,known_securities);
switch(conf.source_case()) {
case md::SimSource::SOURCE_NOT_SET:
case md::SimSource::kReplayUrl:
case config::ExchangeSim::SOURCE_NOT_SET:
case config::ExchangeSim::kReplayUrl:
md::replay_url(conf.replay_url(),
player,
config::get<md::RelayOptions>().naggle_ms());
break;
case md::SimSource::kReplayFile:
case config::ExchangeSim::kReplayFile:
md::replay_file(conf.replay_file(),
player,
config::get<md::RelayOptions>().naggle_ms(), true);
break;
case md::SimSource::kLocalRelay: {
case config::ExchangeSim::kLocalRelay: {
md::local_relay(player);
}
break;
@@ -22,6 +22,7 @@
#include <mutex>
#include <quickfix/fix42/ExecutionReport.h>
#include <quickfix/fix42/OrderCancelReplaceRequest.h>
#include <sstream>
namespace bts {
namespace ex_sim {
@@ -454,7 +455,11 @@ void ExchangeSimApplication::update_md(std::string const &sec_desc) {
persist_book(order_book);
auto book = matcher.get_book();
// if (sec_desc == "SIM:SIZ8") {
// std::stringstream s;
// s << book;
// LOG(ExSim,Info,"{}", s.str());
// }
auto price_factor = pow(10, handle.security().display_factor);
const auto cmax = md::COCONUT_MAX_DEPTH;
@@ -464,7 +469,7 @@ void ExchangeSimApplication::update_md(std::string const &sec_desc) {
: book.bids.end();
std::transform(book.bids.begin(), bids_end, bids_temp.begin(),
[&](const auto &bs) {
return md::PriceSize::from_double<7>(
return md::PriceSize_from_double<precision>(
bs.first.value() * price_factor, bs.second.value());
});
auto asks_end = book.asks.size() > cmax
@@ -472,16 +477,16 @@ void ExchangeSimApplication::update_md(std::string const &sec_desc) {
: book.asks.end();
std::transform(book.asks.begin(), asks_end, asks_temp.begin(),
[&](const auto &bs) {
return md::PriceSize::from_double<7>(
return md::PriceSize_from_double<precision>(
bs.first.value() * price_factor, bs.second.value());
});
handle.update_book(utils::TimeStamp::now(), bids_temp, asks_temp,me->md_sn++);
handle.update_direct(utils::TimeStamp::now(), bids_temp, asks_temp, me->md_sn++);
auto new_trades = matcher.get_new_trades();
for (auto &t : new_trades)
handle.add_trade(
utils::TimeStamp::now(),
md::PriceSize::from_double<7>(t.price.value() * price_factor,
md::PriceSize_from_double<7>(t.price.value() * price_factor,
t.quantity.value()),
datetime::ll_now(),
co2::AGGRESSOR_SIDE_NONE);
@@ -529,9 +534,21 @@ void ExchangeSimApplication::on_new_security(api::Security const &s) {
}
}
void ExchangeSimApplication::on_settlement(md::Security const& sec, md::IntPrice price) {
auto sec_desc = get_sim_sec_desc(sec);
auto iter = me->md_producer_handles.find(sec_desc);
if (iter != me->md_producer_handles.end()) {
auto &handle = iter->second;
handle.update_settlement(utils::TimeStamp::now(), price, datetime::today());
}
}
void ExchangeSimApplication::on_depth(
md::Security const &sec, md::PriceSize bids[md::COCONUT_MAX_DEPTH],
md::PriceSize asks[md::COCONUT_MAX_DEPTH]) {
md::Security const &sec,
md::PriceSize bids[md::COCONUT_MAX_DEPTH],
md::PriceSize asks[md::COCONUT_MAX_DEPTH],
std::array<md::PriceSize, md::COCONUT_MAX_IMPLIED_DEPTH>* implied_bids,
std::array<md::PriceSize, md::COCONUT_MAX_IMPLIED_DEPTH>* implied_asks) {
std::lock_guard<std::mutex> l(me->lock);
if (sec.has_synthetic())
@@ -566,6 +583,15 @@ void ExchangeSimApplication::on_depth(
// we generate a set of orders that we apply to our order book
merge_with_order_book(it->second.security.glbx().security_desc(),
previous_md_book, current_md_book);
// pass on implied market unchanged
if (implied_bids || implied_asks) {
auto iter = me->md_producer_handles.find(get_sim_sec_desc(sec));
if (iter != me->md_producer_handles.end()) {
auto &handle = iter->second;
handle.update_implied(utils::TimeStamp::now(), *implied_bids, *implied_asks);
}
}
}
void ExchangeSimApplication::merge_with_order_book(std::string const &sec_desc,
@@ -45,6 +45,7 @@ namespace bts {
namespace api {class Security;}
namespace md {class MDConsumerHandle;}
namespace ex_sim {
static const int precision = 9;
class Order;
struct Book;
class OrderBook;
@@ -99,9 +100,12 @@ public:
void persist_book(OrderBook const &b);
void on_new_security(md::Security const&) override;
void on_settlement(md::Security const&, md::IntPrice) override;
void on_depth(md::Security const&s,
md::PriceSize bids[md::COCONUT_MAX_DEPTH],
md::PriceSize asks[md::COCONUT_MAX_DEPTH]) override;
md::PriceSize asks[md::COCONUT_MAX_DEPTH],
std::array<md::PriceSize, md::COCONUT_MAX_IMPLIED_DEPTH>* implied_bids,
std::array<md::PriceSize, md::COCONUT_MAX_IMPLIED_DEPTH>* implied_asks) override;
void merge_with_order_book(std::string const &sec_desc, Book const&previous,Book const&current);
};
}}
@@ -9,9 +9,12 @@ namespace sim {
struct IMDPlayerObserver {
virtual void on_new_security(md::Security const&) = 0;
virtual void on_settlement(const md::Security&, md::IntPrice) = 0;
virtual void on_depth(md::Security const&s,
md::PriceSize bids[md::COCONUT_MAX_DEPTH],
md::PriceSize asks[md::COCONUT_MAX_DEPTH]) = 0;
md::PriceSize asks[md::COCONUT_MAX_DEPTH],
std::array<md::PriceSize, md::COCONUT_MAX_IMPLIED_DEPTH>* implied_bids,
std::array<md::PriceSize, md::COCONUT_MAX_IMPLIED_DEPTH>* implied_asks) = 0;
};
}}
+16 -10
View File
@@ -1,7 +1,7 @@
#pragma once
#include "lib/domain_model/Security.pb.h"
#include "lib/orderman/orderman_utils/orderman_common.h"
#include "server/sparkd/exchange_sim/book/Order.h"
#include <memory>
namespace bts {
@@ -24,6 +24,9 @@ struct PriceCmp {
PriceCmp(bool ascending) : ascending(ascending) {}
bool operator()(Price p1, Price p2) const {
if (price_equals(p1.value(), p2.value()))
return false;
if (ascending)
return p1 < p2;
else
@@ -44,7 +47,8 @@ struct Book {
Quantity result(0);
for (auto &kvp : bids)
if (level.value() <= kvp.first.value())
if (price_equals(level.value(), kvp.first.value()) ||
level.value() < kvp.first.value())
result.value() += kvp.second.value();
else
break;
@@ -53,7 +57,8 @@ struct Book {
return BookOp{BookOp::ORDER, api::SIDE_BUY, level, result};
for (auto &kvp : asks)
if (level.value() >= kvp.first.value())
if (price_equals(level.value(), kvp.first.value()) ||
level.value() > kvp.first.value())
result.value() += kvp.second.value();
return BookOp{BookOp::ORDER, api::SIDE_SELL, level, result};
@@ -81,11 +86,11 @@ struct Book {
return;
BookSide *side;
if (op.side == api::SIDE_BUY)
if (op.side == api::SIDE_BUY)
side = &bids;
else
side = &asks;
Quantity *current_qty;
current_qty = &(*side)[op.price];
@@ -93,9 +98,9 @@ struct Book {
case BookOp::ORDER:
current_qty->value() += op.quantity.value();
break;
case BookOp::CANCEL:
case BookOp::CANCEL:
bts_assert(current_qty->value() >= op.quantity.value());
current_qty->value()-=op.quantity.value();
current_qty->value() -= op.quantity.value();
if (current_qty->value() == 0)
side->erase(op.price);
break;
@@ -105,7 +110,8 @@ struct Book {
void match() {
if ((bids.size() && asks.size()) &&
(bids.begin()->first.value() >= asks.begin()->first.value())) {
(price_equals(bids.begin()->first.value(),asks.begin()->first.value()) ||
bids.begin()->first.value() > asks.begin()->first.value())) {
auto bsz = bids.begin()->second.value();
auto asz = asks.begin()->second.value();
@@ -132,5 +138,5 @@ inline std::ostream &operator<<(std::ostream &out, const Book &book) {
out << it->first.value() << "\t\t" << it->second.value() << "\n";
return out;
}
}
}
} // namespace ex_sim
} // namespace bts
+6 -2
View File
@@ -14,18 +14,22 @@ namespace ex_sim {
void execute(Order *order, Price price, Quantity quantity);
inline bool price_equals(double p1, double p2) {
return std::abs(p1-p2) <= 1e-9;
}
struct OrderCmp {
OrderCmp(bool ascending) : ascending(ascending) {}
bool operator()(Order const &lhs, Order const &rhs) const {
if (ascending) {
if (lhs.price() != rhs.price())
if (!price_equals(lhs.price(),rhs.price()))
return lhs.price() < rhs.price();
else
return utils::pb::smaller(lhs.create_time(),rhs.create_time());
} else {
if (lhs.price() != rhs.price())
if (!price_equals(lhs.price(),rhs.price()))
return lhs.price() > rhs.price();
else
return utils::pb::smaller(lhs.create_time(),rhs.create_time());
@@ -60,10 +60,10 @@ OrderMatcher::find_level(OrdersSide &side, double level)
for (auto it = side.begin() ; it != side.end(); it++) {
if (it->price() == level) {
if (price_equals(it->price(),level)) {
result.first = it;
for (;it != side.end(); it++) {
if (it->price() != level)
if (!price_equals(it->price(),level))
break;
}
result.second = it;
+5 -1
View File
@@ -117,7 +117,11 @@ namespace http {
GatewayRequest request;
if(request.ParseFromString(msg->get_payload())) {
iter->second.handle(request, [this,hdl](const GatewayResponse& response) {
_server.send(hdl, response.SerializeAsString(), websocketpp::frame::opcode::binary);
websocketpp::lib::error_code ec;
_server.send(hdl, response.SerializeAsString(), websocketpp::frame::opcode::binary, ec);
if (ec) {
LOG(Api,Warn, "Error sending from websocket: '{}'", ec.message());
}
});
}
else {
+56
View File
@@ -0,0 +1,56 @@
#include "ChannelVolume.h"
#include "lib/api/services/MarketDataRelay.pb.h"
#include "lib/md/PriceSize.h"
namespace bts{
namespace md{
namespace auxiliary{
using namespace bts::co2;
using namespace bts::api::md::auxiliary;
ChannelVolume::ChannelVolume(SecurityChannel* channel)
:
_channel(channel){
}
const VolumeAtPriceLevel& ChannelVolume::add_trade(const TradeMessage& m,
int& record_id){
auto price_map_it=_sec_map.find(m.key());
if(price_map_it==_sec_map.end()){
_sec_map[m.key()]=std::unordered_map<uint64_t,VolumeAtPriceLevel>();
price_map_it=_sec_map.find(m.key());
}
auto& price_map=price_map_it->second;
auto price_it=price_map.find(m.mantissa());
if(price_it==price_map.end()){
VolumeAtPriceLevel vol;
vol.set_record_id(record_id++);
vol.set_security_id(m.key());
IntPrice p = md::IntPrice (m.mantissa(),(int8_t)m.exponent());
vol.set_price(p.price());
price_map[m.mantissa()] = vol;
price_it=price_map.find(m.mantissa());
}
VolumeAtPriceLevel& vol=price_it->second;
vol.set_volume(vol.volume()+m.size());
if(m.aggressor_side()==AggressorSide::AGGRESSOR_SIDE_BUY)
vol.set_buy_volume(vol.buy_volume()+m.size());
else if(m.aggressor_side()==AggressorSide::AGGRESSOR_SIDE_SELL)
vol.set_sell_volume(vol.sell_volume()+m.size());
else if(m.aggressor_side()==AggressorSide::AGGRESSOR_SIDE_NONE)
vol.set_passive_volume(vol.passive_volume()+m.size());
return vol;
}
bool ChannelVolume::match_update(const SecurityChannel& channel){
if(_channel->type()==channel.type() &&
_channel->exchange()==channel.exchange() &&
_channel->symbol()==channel.symbol()){
_channel->CopyFrom(channel);
return true;
}
return false;
}
}}}
+28
View File
@@ -0,0 +1,28 @@
#pragma once
#include "lib/api/services/MarketDataAuxiliaryService.pb.h"
#include <unordered_map>
namespace bts{
namespace co2 {class TradeMessage;class SecurityChannel;}
namespace md{
namespace auxiliary{
// volume at the various price levels for all
// securities belonging to a single channel.
class ChannelVolume{
private:
bts::co2::SecurityChannel* const _channel;
std::unordered_map<uint32_t,std::unordered_map<uint64_t,bts::api::md::auxiliary::VolumeAtPriceLevel>> _sec_map;
public:
ChannelVolume(bts::co2::SecurityChannel* channel);
// finds the VolumeAtPriceLevel for the <sec, price_level>
// in the trade and updates the volume. if <sec,price_level>
// is not found, it increases the record_id before assigning
// it to VolumeAtPriceLevel's record_id
const bts::api::md::auxiliary::VolumeAtPriceLevel&
add_trade(const bts::co2::TradeMessage& m, int& record_id);
// if the channel is matched, it updated with it with
// the data in the arg, and returns true
bool match_update(const bts::co2::SecurityChannel& channel);
};
}}}
@@ -0,0 +1,86 @@
#include "MarketDataAuxiliaryServiceServer.h"
#include "lib/api/services/MarketDataRelay.pb.h"
#include "lib/api/services/MarketDataRelay.client.pb.h"
#include "lib/api/public/ServiceContext.h"
#include "lib/logging/logging.h"
#include "lib/utils/PBUtils.h"
#include "lib/utils/Dispatcher.h"
#include "lib/md/get_symbols.h"
#include "VolumeAtPriceState.h"
#include "lib/utils/bts_chrono.h"
#include "lib/config/config.h"
#include "lib/config/protos/toggles.pb.h"
namespace bts{
namespace md{
namespace auxiliary{
using namespace bts::co2;
using namespace bts::api::md::auxiliary;
struct MarketDataAuxiliaryServiceServer::PImpl{
bool vap_enabled;
bts::api::ServiceContext& context;
MarketDataRelay relay_service;
MarketDataAuxiliaryServiceBase::Volume_at_priceStatefulPublisher publisher;
std::unique_ptr<utils::DispatcherTimer> trade_timer;
int64_t poll_period_ms;
VolumeAtPriceState state;
uint16_t log_counter, log_trunc;
PImpl(bts::api::ServiceContext& context,
const MarketDataAuxiliaryServiceBase::Volume_at_priceStatefulPublisher& publisher)
:
vap_enabled(config::get<config::toggles::VolumeAtPrice>().enabled()),
context(context),
relay_service(context.locate<MarketDataRelay>()),
publisher(publisher),
trade_timer(),
poll_period_ms(100),
state(),
log_counter(0),
log_trunc(300){
}
};
MarketDataAuxiliaryServiceServer::MarketDataAuxiliaryServiceServer(
bts::api::ServiceContext& context)
:
MarketDataAuxiliaryServiceBase(context),
me(new PImpl(context,
make_volume_at_price_stateful_publisher())){
if(me->vap_enabled)
me->context.post([this](){_poll_md();},me->poll_period_ms);
}
void MarketDataAuxiliaryServiceServer::_poll_md(){
me->relay_service.requestmd(me->state.md_req(),
[this](const bts::api::Result<MDReply>& md_reply)
{
if(md_reply.result()!=nullptr)
_scan_trades(*md_reply.result());
me->context.post([this](){_poll_md();},me->poll_period_ms);
});
}
void MarketDataAuxiliaryServiceServer::_scan_trades(const MDReply& md_reply){
uint32_t n_trades=0, total_unique_levels=0;
bts::TimeStamp t0=bts::Timer::now();
for(const auto& cr : md_reply.channels())
me->state.on_new_trades(cr,
me->publisher,
n_trades,
total_unique_levels);
bts::TimeStamp t1=bts::Timer::now();
double usecs=static_cast<double>(us_diff (t1,t0));
if(me->log_counter++ >= me->log_trunc){
LOG(OrderMan,Info,"MarketDataAuxiliaryServiceServer::_scan_trades "
"trades:{} tot_unique:{} usecs:{}",
n_trades,total_unique_levels,usecs);
me->log_counter=0;
}
me->publisher.flush();
}
MarketDataAuxiliaryServiceServer::~MarketDataAuxiliaryServiceServer(){
}
}}}
@@ -0,0 +1,27 @@
#pragma once
#include <memory>
#include "lib/api/services/MarketDataAuxiliaryService.server.pb.h"
#include "lib/utils/bts_no_copy.h"
namespace bts{
namespace api {class ServiceContext;}
namespace co2 {class MDReply;}
namespace md{
namespace auxiliary{
class MarketDataAuxiliaryServiceServer :
public bts::api::md::auxiliary::MarketDataAuxiliaryServiceBase
{
private:
struct PImpl;
std::unique_ptr<PImpl> me;
private:
void _poll_md();
void _scan_trades(const bts::co2::MDReply&);
public:
MarketDataAuxiliaryServiceServer(bts::api::ServiceContext& context);
~MarketDataAuxiliaryServiceServer();
};
}}}
@@ -0,0 +1,60 @@
#include "VolumeAtPriceState.h"
#include "lib/md/get_symbols.h"
#include "lib/api/services/MarketDataRelay.pb.h"
//#include "lib/api/services/MarketDataRelay.client.pb.h"
namespace bts{
namespace md{
namespace auxiliary{
using namespace bts::api::md::auxiliary;
using namespace bts::co2;
using namespace std;
VolumeAtPriceState::VolumeAtPriceState()
:
_channels(),
_md_req(),
_record_id(0){
std::set<bts::md::UserSymbol> syms;
bts::md::get_system_md_symbols(syms);
for(const auto& s : syms){
SecurityChannel c;
c.set_flags(MDRequest::TRADE_LOG);
c.set_type(s.type());
c.set_exchange(s.exchange());
c.set_symbol(s.symbol());
c.set_trade_cursor(0);
c.set_segment_id(0);
_update_or_add(c);
}
}
ChannelVolume& VolumeAtPriceState::_update_or_add(const SecurityChannel& channel){
for(auto& v : _channels)
if(v.match_update(channel))
return v;
SecurityChannel* new_channel_ptr = _md_req.add_channels();
new_channel_ptr->CopyFrom(channel);
_channels.emplace_back(new_channel_ptr);
return _channels.back();
}
void VolumeAtPriceState::on_new_trades(const ChannelReply& reply,
MarketDataAuxiliaryServiceBase::Volume_at_priceStatefulPublisher& publisher,
uint32_t& n_trades,
uint32_t& total_unique_levels){
ChannelVolume& channel_vol=_update_or_add(reply.channel());
for(const TradeMessage& t : reply.trade_log()){
const VolumeAtPriceLevel& sec_level_vol=channel_vol.add_trade(t,
_record_id);
publisher.add_or_update(sec_level_vol);
n_trades++;
}
total_unique_levels = _record_id;
}
const bts::co2::MDRequest& VolumeAtPriceState::md_req() const{
return _md_req;
}
}}}
+32
View File
@@ -0,0 +1,32 @@
#pragma once
#include "lib/api/services/MarketDataAuxiliaryService.server.pb.h"
#include "lib/api/services/MarketDataRelay.pb.h"
#include "ChannelVolume.h"
namespace bts{
namespace co2 {class SecurityChannel;class ChannelReply;}
namespace md{
namespace auxiliary{
// this is the entire state for the volume at
// price levels for all channels.
class VolumeAtPriceState{
private:
std::vector<ChannelVolume> _channels;
bts::co2::MDRequest _md_req;
int _record_id;
private:
// if the channel is present, its updated (trade_cursor etc)
// else its created from the channel
ChannelVolume& _update_or_add(const bts::co2::SecurityChannel& channel);
public:
VolumeAtPriceState ();
//encorporates the reply, and publishes it
void on_new_trades(const bts::co2::ChannelReply& channel,
bts::api::md::auxiliary::MarketDataAuxiliaryServiceBase::Volume_at_priceStatefulPublisher& publisher,
uint32_t& n_trades,
uint32_t& total_unique_levels);
const bts::co2::MDRequest& md_req() const;
};
}}}
@@ -16,9 +16,8 @@ MDP3Listener::MDP3Listener(
CMESecurityIndex& index,
MDProducer& producer,
SecurityIDRecorder& recorder,
co2::SettlementRecorder& ,
const std::vector<MDProducerChannel> channels,
CMEOptions::BookType bt) :
// co2::SettlementRecorder& ,
const std::vector<MDProducerChannel> channels) :
_index(index),
_producer(producer),
_packet_arrived(),
@@ -28,7 +27,6 @@ MDP3Listener::MDP3Listener(
_symbol_filter_storage(),
_symbol_filter(),
_channels(channels),
_booktype(bt),
_concurrency_monitor(0),
_book_updates()
{
@@ -115,15 +113,11 @@ void MDP3Listener::onEndOfPacket(onixs::Handler&, const onixs::PacketArgs&) {
// Security discovery
void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, const onixs::DirectBook& book) {
if (_booktype == CMEOptions::BOOKTYPE_DIRECT) {
onBookUpdate(sec, book);
}
onBookUpdate<COCONUT_MAX_DEPTH>(sec, book, book_type_direct);
}
void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, const onixs::ConsolidatedBook& book) {
if (_booktype == CMEOptions::BOOKTYPE_CONSOLIDATED) {
onBookUpdate(sec, book);
}
void MDP3Listener::onBookUpdate(onixs::Handler&, const onixs::Security& sec, const onixs::ImpliedBook& book) {
onBookUpdate<COCONUT_MAX_IMPLIED_DEPTH>(sec, book, book_type_implied);
}
//
@@ -245,8 +239,21 @@ void MDP3Listener::flush_books() {
// This is called at the end of the packet
// Write all book updates to shared memory and reset for the next packet
for (auto& entry : _book_updates) {
uint16_t sequence = (_latest_sequence & 0xFFFF);
entry.handle.update_book(_packet_arrived, entry.bids, entry.asks, sequence);
uint32_t sequence = _latest_sequence;
if (entry.type_flags == book_type_direct) {
entry.handle.update_direct(_packet_arrived,
entry.bids.first, entry.asks.first, sequence);
}
else if (entry.type_flags == book_type_implied) {
entry.handle.update_implied(_packet_arrived,
entry.bids.second, entry.asks.second, sequence);
}
else if (entry.type_flags == book_type_both) {
entry.handle.update_books(_packet_arrived,
entry.bids.first, entry.asks.first,
entry.bids.second, entry.asks.second,
sequence);
}
}
// Reset pending updates
_book_updates.clear();
@@ -258,7 +265,7 @@ void MDP3Listener::update_all_status(CoconutSecurityStatus status) {
});
}
md::MDP3Listener::BookData::BookData(MDProducerHandle & handle) : handle(handle), bids(), asks() { }
md::MDP3Listener::BookData::BookData(MDProducerHandle & handle) : handle(handle), type_flags(0), bids(), asks() { }
}}//namespace
@@ -12,22 +12,28 @@
#include <vector>
#include <atomic>
#include <array>
#include <utility>
namespace bts {
namespace md {
class MDP3Listener :
public onixs::HandlerListener,
public onixs::MarketDataListener,
public onixs::SecurityListener
{
typedef uint32_t BookType;
static constexpr BookType book_type_direct=1;
static constexpr BookType book_type_implied=2;
static constexpr BookType book_type_both=(book_type_direct|book_type_implied);
public:
MDP3Listener(CMESecurityIndex& index,
MDProducer&,
SecurityIDRecorder&,
co2::SettlementRecorder&,
const std::vector<MDProducerChannel>,
CMEOptions::BookType);
// co2::SettlementRecorder&,
const std::vector<MDProducerChannel>);
virtual ~MDP3Listener();
//
@@ -71,11 +77,11 @@ public:
// Book
// void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::MboBook&) override;
void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::DirectBook&) override;
// void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::ImpliedBook&) override;
void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::ConsolidatedBook&) override;
void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::ImpliedBook&) override;
// void onBookUpdate(onixs::Handler&, const onixs::Security&, const onixs::ConsolidatedBook&) override;
template <class TBook>
void onBookUpdate(const onixs::Security& sec, const TBook& book);
template <size_t MAX_DEPTH,class TBook>
void onBookUpdate(const onixs::Security& sec, const TBook& book, BookType booktype);
//
// Trades
@@ -105,8 +111,18 @@ public:
// Misc
template <typename StatisticsArgs>
void onSettlementPrice(const onixs::Security& sec, const StatisticsArgs& args);
void onSettlementPrice(onixs::Handler&, const onixs::Security& s, const onixs::DailyStatistics49Args& arg) override { onSettlementPrice(s, arg); }
void onSettlementPrice(onixs::Handler&, const onixs::Security& s, const onixs::DailyStatistics33Args& arg) override { onSettlementPrice(s, arg); }
void onSettlementPrice(onixs::Handler&, const onixs::Security& s, const onixs::DailyStatistics49Args& arg) override {
auto type = arg.entry().settlPriceType();
if (type.finalDaily() && type.actual()) {
onSettlementPrice(s, arg);
}
}
void onSettlementPrice(onixs::Handler&, const onixs::Security& s, const onixs::DailyStatistics33Args& arg) override {
auto type = arg.entry().settlPriceType();
if (type.finalDaily() && type.actual()) {
onSettlementPrice(s, arg);
}
}
void onSettlementPrice(onixs::Handler&, const onixs::Security& s, const onixs::Recovery52EntryArgs& arg) override { onSettlementPrice(s, arg); }
void onSettlementPrice(onixs::Handler&, const onixs::Security& s, const onixs::Recovery38EntryArgs& arg) override { onSettlementPrice(s, arg); }
@@ -181,17 +197,16 @@ private:
std::vector<std::string> _symbol_filter_storage;
std::set<onixs::StrRef> _symbol_filter;
std::vector<MDProducerChannel> _channels;
CMEOptions::BookType _booktype;
std::atomic<int> _concurrency_monitor;
struct BookData {
BookData(MDProducerHandle& handle);
MDProducerHandle handle;
std::array<PriceSize, COCONUT_MAX_DEPTH> bids;
std::array<PriceSize, COCONUT_MAX_DEPTH> asks;
BookType type_flags;
std::pair<std::array<PriceSize, COCONUT_MAX_DEPTH>, std::array<PriceSize,COCONUT_MAX_IMPLIED_DEPTH>> bids;
std::pair<std::array<PriceSize, COCONUT_MAX_DEPTH>, std::array<PriceSize,COCONUT_MAX_IMPLIED_DEPTH>> asks;
};
std::vector<BookData> _book_updates;
};
//////////////////////////////////////////
@@ -341,9 +356,8 @@ void MDP3Listener::onSpread(onixs::Handler&, const onixs::Security&, const Instr
_index.add_spread_legs(fsec.numeric_id, leg_exchange_ids);
}
template <class TBook>
void MDP3Listener::onBookUpdate(const onixs::Security& sec, const TBook& book) {
template <size_t MAX_DEPTH,class TBook>
void MDP3Listener::onBookUpdate(const onixs::Security& sec, const TBook& onixs_book, BookType booktype) {
tracepoint (bts_trace,trace_order,(char*) "onBookUpdate", _latest_sequence, (uint16_t) _latest_sequence, 0);
auto data = _index.get_security(sec.id());
@@ -361,26 +375,31 @@ void MDP3Listener::onBookUpdate(const onixs::Security& sec, const TBook& book) {
_book_updates.emplace_back(data->handle);
}
BookData& cache = found ? *find_iter : _book_updates.back();
cache.type_flags |= booktype;
auto& cache_bids = std::get<std::array<PriceSize,MAX_DEPTH>>(cache.bids);
auto& cache_asks = std::get<std::array<PriceSize,MAX_DEPTH>>(cache.asks);
// Overwrite all entries every time
size_t bids_size = book.bids().size();
size_t asks_size = book.offers().size();
size_t bids_size = onixs_book.bids().size();
size_t asks_size = onixs_book.offers().size();
if (!data->handle.has_depth()) {
bids_size = std::min(bids_size, size_t { 1 });
asks_size = std::min(asks_size, size_t { 1 });
}
for (size_t x = 0; x < bids_size; ++x) {
cache.bids[x] = pricesize_of_price_size(book.bids()[x].price(), display_factor, book.bids()[x].quantity());
cache_bids[x] = pricesize_of_price_size(onixs_book.bids()[x].price(), display_factor, onixs_book.bids()[x].quantity());
}
for (size_t x = 0; x < asks_size; ++x) {
cache.asks[x] = pricesize_of_price_size(book.offers()[x].price(), display_factor, book.offers()[x].quantity());
cache_asks[x] = pricesize_of_price_size(onixs_book.offers()[x].price(), display_factor, onixs_book.offers()[x].quantity());
}
if (!found) {
// NaN out the remaining depth
for (size_t x = bids_size; x < COCONUT_MAX_DEPTH; ++x) {
cache.bids[x] = PriceSize::NO_MARKET;
}
for (size_t x = asks_size; x < COCONUT_MAX_DEPTH; ++x) {
cache.asks[x] = PriceSize::NO_MARKET;
for (size_t x = bids_size; x < MAX_DEPTH; ++x) {
cache_bids[x] = NO_MARKET;
}
for (size_t x = asks_size; x < MAX_DEPTH; ++x) {
cache_asks[x] = NO_MARKET;
}
}
}
@@ -434,10 +453,11 @@ void MDP3Listener::onSettlementPrice(const onixs::Security& sec, const Statistic
onixs::Timestamp time;
args.entry().entryPx(price);
args.entry().tradingReferenceDate(time);
// args.entry().
auto display_factor = data->handle.security().display_factor;
data->handle.update_settlement(
utils::TimeStamp::now(),
double_of_price(price, display_factor),
convert_price(price, display_factor),
date_of_timestamp(time));
}
}
@@ -457,16 +477,19 @@ void MDP3Listener::onOpenInterest(const onixs::Security& sec, const InterestArgs
template <typename SessionArgs>
void MDP3Listener::onSessionOpeningPrice(const onixs::Security& sec, const SessionArgs& args) {
tracepoint (bts_trace,trace_order,(char*) "onOpeningPrice.session", _latest_sequence, (uint16_t) _latest_sequence, 0);
auto* data = _index.get_security(sec.id());
if (data != nullptr) {
onixs::Decimal price = args.entry().entryPx();
onixs::Timestamp time = args.message().transactTime();
auto display_factor = data->handle.security().display_factor;
data->handle.update_opening(
utils::TimeStamp::now(),
double_of_price(price, display_factor),
date_of_timestamp(time));
}
// auto type = args.entry().settlPriceType();
// if (type.final() && type.actual()) {
auto* data = _index.get_security(sec.id());
if (data != nullptr) {
onixs::Decimal price = args.entry().entryPx();
onixs::Timestamp time = args.message().transactTime();
auto display_factor = data->handle.security().display_factor;
data->handle.update_opening(
utils::TimeStamp::now(),
double_of_price(price, display_factor),
date_of_timestamp(time));
}
//
}
template <typename RecoveryArgs>
void MDP3Listener::onRecoveryOpeningPrice(const onixs::Security& sec, const RecoveryArgs& args) {
@@ -12,16 +12,22 @@ namespace bts {
namespace md {
inline PriceSize pricesize_of_price_size(const onixs::Decimal price, int8_t display_factor, Size size) {
PriceSize result(price.mantissa(), price.exponent() + display_factor, size);
PriceSize result({price.mantissa(), (int8_t)(price.exponent() + display_factor)}, size);
return result;
}
template <class Price>
double double_of_price(const Price& price, int8_t display_factor) {
template <class OnixPrice>
double double_of_price(const OnixPrice& price, int8_t display_factor) {
// order of multiplication is important because of floating point error
return (price.mantissa() * pow10(price.exponent())) * pow10(display_factor);
}
template <class OnixsPrice>
IntPrice convert_price(const OnixsPrice& price, int8_t display_factor) {
return IntPrice(price.mantissa(), price.exponent() + display_factor);
}
inline int floorlog10(int64_t m) {
bts_assert(m >= 0);
if (m < 10) return 0;
@@ -28,7 +28,7 @@ namespace md {
MDP3Main::MDP3Main() :
_producer(PLATFORM_MDP3ONIXS),
_security_recorder(),
_settlement_recorder(PLATFORM_MDP3ONIXS),
// _settlement_recorder(PLATFORM_MDP3ONIXS),
_engines(),
_handlers(),
_listeners(),
@@ -171,10 +171,10 @@ void MDP3Main::run(std::vector<UserSymbol>& symbols) {
});
spread_resolve_timer.start();
utils::DispatcherTimer settlement_timer(disp, 100, [&]() {
_settlement_recorder.do_work();
});
settlement_timer.start();
// utils::DispatcherTimer settlement_timer(disp, 100, [&]() {
// _settlement_recorder.do_work();
// });
// settlement_timer.start();
utils::DispatcherTimer heartbeat_timer(disp, 250, [&]() {
for (const auto& listener: _listeners) {
@@ -223,12 +223,9 @@ unique_ptr<onixs::Handler> MDP3Main::prepare_handler(const string& channel_id) {
settings.feeds().connectivityConfigurationFile(config_path());
// Market data options
if (options.book_type() == CMEOptions::BOOKTYPE_CONSOLIDATED) {
settings.bookManagement().consolidatedBooks().maintain(true);
}
else {
settings.bookManagement().directBooks().maintain(true);
}
settings.bookManagement().directBooks().maintain(true);
settings.bookManagement().impliedBooks().maintain(true);
settings.bookManagement().consolidatedBooks().maintain(false);
// 10k default is *far* too low
settings.recoveryQueueMaxSize(options.recovery_queue_size());
@@ -357,7 +354,7 @@ unique_ptr<MDP3Listener> MDP3Main::prepare_listener(onixs::Handler& handler, con
const auto& options = channel_settings_for(channel.id);
unique_ptr<MDP3Listener> listener(new MDP3Listener(_index, _producer, _security_recorder, _settlement_recorder, producer_channels, options.book_type()));
unique_ptr<MDP3Listener> listener(new MDP3Listener(_index, _producer, _security_recorder, /*_settlement_recorder,*/ producer_channels));
for (auto& symbol : channel.symbols) {
listener->add_filter_symbol(symbol.symbol());
}
@@ -40,7 +40,7 @@ private:
CMESecurityIndex _index;
MDProducer _producer;
SecurityIDRecorder _security_recorder;
co2::SettlementRecorder _settlement_recorder;
// co2::SettlementRecorder _settlement_recorder;
std::map<std::string, std::unique_ptr<onixs::FeedEngine>> _engines;
std::map<std::string, std::unique_ptr<onixs::Handler>> _handlers;
std::vector<std::unique_ptr<MDP3Listener>> _listeners;
+8 -2
View File
@@ -10,6 +10,8 @@
#include "lib/md/Diag/ShmDumper.h"
#include "platforms/mdp3_onixs/MDP3Main.h"
#include "lib/md/SecurityDB.h"
#include "server/sparkd/exchange_sim/ExchangeSim.h"
#include "lib/utils/thread_manager.h"
#include <boost/filesystem.hpp>
#include <iostream>
@@ -74,7 +76,6 @@ void run_cleanup() {
std::thread* _md_thread;
void run_market_data_publishers_forever() {
os::set_thread_name("md publisher");
run_cleanup();
if (config::get<MarketDataConfig>().use_replay()) {
run_replay();
}
@@ -85,13 +86,18 @@ void run_market_data_publishers_forever() {
void run_market_datapublishers() {
bts_assert(_md_thread == nullptr);
if (!config::get<MarketDataConfig>().use_md()) return;
run_cleanup();
if (!config::get<MarketDataConfig>().use_md()) return;
// start a thread for the md
_md_thread = new std::thread([]() {
run_market_data_publishers_forever();
});
if (config::get<MarketDataConfig>().use_sim()) {
threading::make_thread("ExchangeSim", []() { sim::start_exchange_sim(); })
.detach();
}
std::this_thread::sleep_for(2s);
}
@@ -15,6 +15,9 @@
namespace bts {
namespace position{
using namespace bts::api::position;
using namespace bts::api::orderman;
struct PositionServiceServer::PImpl {
PositionServiceBase::Position_dataStatefulPublisher publisher;
bts::api::orderman::OrderService order_service;
@@ -10,10 +10,7 @@ namespace api {class ServiceContext;}
namespace api {namespace orderman {class Fill;}}
namespace position{
using namespace bts::api::position;
using namespace bts::api::orderman;
class PositionServiceServer : public PositionServiceBase
class PositionServiceServer : public bts::api::position::PositionServiceBase
{
private:
struct PImpl;
@@ -22,11 +19,13 @@ private:
// this is the function that aggrgates the
// position. is_add is to indicate if a row
// is added or removed
const PositionUnit& _on_fill (const Fill& fill, bool is_add);
const bts::api::position::PositionUnit& _on_fill (
const bts::api::orderman::Fill& fill,
bool is_add);
// given an account and key, finds (or adds if absent)
// the PositionUnit entry
PositionUnit& _get_or_add (const std::string& acct,
uint32_t sec_id);
bts::api::position::PositionUnit& _get_or_add (const std::string& acct,
uint32_t sec_id);
static void _avg_price_qty(double base_p,
double base_q,
double fill_p,
@@ -35,7 +34,7 @@ private:
const std::string& fill_id,
double& avg_price,
double& tot_qty);
static std::string _fill_account(const Fill& fill);
static std::string _fill_account(const bts::api::orderman::Fill& fill);
public:
PositionServiceServer(bts::api::ServiceContext& context);
~PositionServiceServer();
+162 -25
View File
@@ -2,8 +2,11 @@
#include "lib/persistence/SqliteStore.h"
#include "lib/logging/logging.h"
#include "lib/utils/bts_assert.h"
#include "lib/utils/PBUtils.h"
#include "lib/api/service_helpers/AdminServiceUtils.h"
#include "lib/orderman/orderman_utils/orderman_utils.h"
#include "lib/utils/TimeStamp.h"
#include <boost/tokenizer.hpp>
#include <fmt/format.h>
@@ -22,10 +25,14 @@ AdminServiceServer::AdminServiceServer(bts::api::ServiceContext& context)
_unique_key("0"),
_state(),
_store(new bts::persistence::SqliteStore(_db_name)),
_publisher(make_admin_data_stateful_publisher())
_publisher(make_admin_data_stateful_publisher()),
_md_entit_publisher(make_md_entitlement_data_stateful_publisher()),
_md_perm_publisher(make_md_permissions_data_stateful_publisher()),
_order_service(context.locate<bts::api::orderman::OrderService>()),
_order_map()
{
LOG(Gen, Info, "AdminServiceServer: service started");
std::map<std::string,std::map<std::string,std::string>> all_data = _store->retrieve_all();
map<string,map<string,string>> all_data = _store->retrieve_all();
auto table = all_data.find(_table_name);
bool data_found = false;
if (table != all_data.end())
@@ -48,58 +55,88 @@ AdminServiceServer::AdminServiceServer(bts::api::ServiceContext& context)
AdminServiceUtils::fill_connections_from_config(_state);
AdminServiceUtils::bootstrap_data(_state);
_update_and_publish(true);
}
}
_order_service.subscribe_orders() |
rxcpp::operators::subscribe<api::orderman::OrdersUpdate>(
[this](auto const& update) {
for (const auto& o : update.removed()){
const auto& it = _order_map.find(o->key());
if(it != _order_map.end())
_order_map.erase(it);
}
for (const auto& o : update.added()){
_order_map[o->key()]=o;
}
});
}
AdminServiceServer::~AdminServiceServer() {}
void AdminServiceServer::create_account(const api::admin::AccountCreateAPIArgument& request,
const api::Response<::bts::api::admin::AdminServiceCreateAccountResult>& reply){
void AdminServiceServer::create_account(
const AccountCreateAPIArgument& request,
const api::Response<AdminServiceCreateAccountResult>& reply){
bts::api::admin::AdminServiceCreateAccountResult result;
AdminServiceUtils::create_account(_state, request.parent_account_id(), request.account_name(), request.description(), false, result);
AdminServiceUtils::create_account(_state,
request.parent_account_id(),
request.account_name(),
request.description(),
false,
result);
_update_and_publish(false);
reply(result);
}
void AdminServiceServer::update_account(const api::admin::AccountUpdateAPIArgument& request,
const api::Response<::bts::api::admin::AdminServiceOperationResult>& reply){
void AdminServiceServer::update_account(
const AccountUpdateAPIArgument& request,
const api::Response<AdminServiceOperationResult>& reply){
bts::api::admin::AdminServiceOperationResult result;
AdminServiceUtils::update_account(_state, request.complete_data(), result);
_update_and_publish(false);
reply(result);
}
void AdminServiceServer::create_user(const api::admin::UserCreateAPIArgument& request,
const api::Response<::bts::api::admin::AdminServiceCreateEntityResult>& reply){
bts::api::admin::AdminServiceCreateEntityResult result;
AdminServiceUtils::create_user(_state, request.parent_group_id(), request.name(), request.password(), false,
request.license_data(), request.license_entitlements(), result);
void AdminServiceServer::create_user(
const UserCreateAPIArgument& request,
const api::Response<AdminServiceCreateEntityResult>& reply){
AdminServiceCreateEntityResult result;
AdminServiceUtils::create_user(_state,
request.parent_group_id(),
request.name(),
request.password(),
false,
request.license_data(),
request.license_entitlements(),
result);
_update_and_publish(false);
reply(result);
}
void AdminServiceServer::update_user(const api::admin::UserUpdateAPIArgument& request,
const api::Response<::bts::api::admin::AdminServiceOperationResult>& reply){
bts::api::admin::AdminServiceOperationResult result;
void AdminServiceServer::update_user(
const UserUpdateAPIArgument& request,
const api::Response<AdminServiceOperationResult>& reply){
AdminServiceOperationResult result;
AdminServiceUtils::update_user(_state, request.complete_data(), result);
_update_and_publish(false);
reply(result);
}
void AdminServiceServer::get_admin_data(const bts::api::VoidMessage&,
const api::Response<::bts::api::admin::AdminData>& reply){
reply(_state);
const api::Response<AdminData>& reply){
reply(_state);
}
void AdminServiceServer::delete_user(const bts::api::admin::IDAPIArgument& id,
const api::Response<bts::api::admin::AdminServiceOperationResult>& reply){
bts::api::admin::AdminServiceOperationResult result; result.set_is_success(true);
bts::admin::AdminServiceUtils::delete_user(_state, id, result);
void AdminServiceServer::delete_user(
const IDAPIArgument& id,
const api::Response<AdminServiceOperationResult>& reply){
AdminServiceOperationResult result; result.set_is_success(true);
AdminServiceUtils::delete_user(_state, id, result);
_update_and_publish(false);
reply(result);
}
void AdminServiceServer::delete_account(const bts::api::admin::IDAPIArgument& id,
const api::Response<bts::api::admin::AdminServiceOperationResult>& reply){
bts::api::admin::AdminServiceOperationResult result; result.set_is_success(true);
void AdminServiceServer::delete_account(
const IDAPIArgument& id,
const api::Response<AdminServiceOperationResult>& reply){
AdminServiceOperationResult result; result.set_is_success(true);
bts::admin::AdminServiceUtils::delete_account(_state, id, result);
_update_and_publish(false);
reply(result);
}
void AdminServiceServer::_update_and_publish(bool is_insert){
std::string ser_state;
auto start = utils::TimeStamp::now();
if (_state.SerializeToString(&ser_state))
{
LOG(Gen, Info, "Saving admin data");
@@ -109,8 +146,108 @@ void AdminServiceServer::_update_and_publish(bool is_insert){
_store->update(_table_name, _unique_key, ser_state);
_publisher.add_or_update(_state);
_publisher.flush();
_publish_md_entitlements();
// _publish_md_permissions();
auto elapsed = (utils::TimeStamp::now() - start).milliseconds();
LOG(Gen, Info, "Saved admin data ({}ms)", elapsed);
}
else
LOG(Gen, Error, "Could not serialize admin data, saving failed");
}
void AdminServiceServer::get_order_summary(
const bts::api::VoidMessage&,
const api::Response<OrdersSummary>& reply){
std::map<std::string,std::map<std::string,OrdersSummaryUnit>> agg;
for(auto it=_order_map.begin();it!=_order_map.end();it++){
const OrderRecord& ord=*(it->second);
if(bts::orderman::order_in_working_state(ord.state().status()) &&
bts::orderman::is_parent_order(ord))
_add_live_order(agg,ord);
}
OrdersSummary rv;
for (auto acct_it=agg.begin();acct_it!=agg.end();acct_it++){
const std::map<std::string,OrdersSummaryUnit>& user_map=acct_it->second;
for(auto user_it=user_map.begin();user_it!=user_map.end();user_it++){
const OrdersSummaryUnit& u=user_it->second;
OrdersSummaryUnit* nv=rv.add_units();
nv->CopyFrom(u);
}
}
reply(rv);
}
void AdminServiceServer::cancel_orders(
const OrderCancelRequest& req,
const api::Response<bts::api::VoidMessage>&){
vector<string> accts, users;
for(auto it=req.accounts().begin();it!=req.accounts().end();it++)
accts.push_back(*it);
for(auto it=req.users().begin();it!=req.users().end();it++)
users.push_back(*it);
_cancel_orders(accts,users);
}
void AdminServiceServer::_add_live_order(
map<string,map<string,OrdersSummaryUnit>>& agg,
const OrderRecord& ord){
const std::string& acct=ord.def().account();
const std::string& user=ord.def().user();
auto acct_it=agg.find(acct);
if(acct_it==agg.end())
agg[acct]=std::map<std::string,OrdersSummaryUnit>();
acct_it=agg.find(acct);
bts_assert(acct_it!=agg.end());
std::map<std::string,OrdersSummaryUnit>& user_map=acct_it->second;
auto user_it=user_map.find(user);
if(user_it==user_map.end()){
OrdersSummaryUnit nu;
nu.set_account(acct);
nu.set_user(user);
user_map[user]=nu;
}
user_it=user_map.find(user);
bts_assert(user_it!=user_map.end());
OrdersSummaryUnit& u=user_it->second;
u.set_total_orders(u.total_orders()+1);
u.set_total_contracts(u.total_contracts()+ord.state().quantity());
}
void AdminServiceServer::_cancel_orders(const vector<string>& accounts,
const vector<string>& users){
map<string, bool> acct_map, user_map;
vector<uint32_t> keys;
for(const std::string& a : accounts)
acct_map[a]=true;
for(const std::string& u : users)
user_map[u]=true;
for(auto it=_order_map.begin();it!=_order_map.end();it++){
const OrderRecord& ord=*(it->second);
if(bts::orderman::order_in_working_state(ord.state().status()) &&
bts::orderman::is_parent_order(ord) &&
acct_map.find(ord.def().account()) != acct_map.end() &&
user_map.find(ord.def().user()) != user_map.end())
keys.push_back(ord.key());
}
auto cb = [&](bts::api::Result<::bts::api::orderman::Response>) -> void {};
for(uint32_t k : keys){
Cancel c;
c.set_order_key(k);
_order_service.cancel_order(c, cb);
}
}
void AdminServiceServer::_publish_md_entitlements(){
// TODO, need to delete all state
std::vector<MDEntitlementReportUnit> entitlements;
AdminServiceUtils::get_md_entitlement_report(_state,entitlements);
for (const auto& e : entitlements)
_md_entit_publisher.add_or_update(e);
_md_entit_publisher.flush();
}
void AdminServiceServer::_publish_md_permissions(){
std::vector<UserSymbolMDPermissionUnit> perms;
for (const BTSUser* user : AdminServiceUtils::users(_state,false))
AdminServiceUtils::get_user_symbol_permissions(_state,*user,perms);
for (const auto& p : perms)
_md_perm_publisher.add_or_update(p);
_md_perm_publisher.flush();
}
}}
+44 -14
View File
@@ -5,10 +5,16 @@
#include "lib/utils/bts_no_copy.h"
#include "lib/persistence/IStore.h"
#include "lib/api/public/ServiceContext.h"
#include "lib/api/services/OrderService.pb.h"
#include "lib/api/services/OrderService.client.pb.h"
namespace bts {
namespace admin{
using namespace bts::api::orderman;
using namespace bts::api::admin;
using namespace std;
class AdminServiceServer : public api::admin::AdminServiceBase
{
private:
@@ -17,33 +23,57 @@ private:
const std::string _unique_key;
bts::api::admin::AdminData _state;
std::unique_ptr<bts::persistence::IStore> _store;
bts::api::admin::AdminServiceBase::Admin_dataStatefulPublisher _publisher;
AdminServiceBase::Admin_dataStatefulPublisher _publisher;
AdminServiceBase::Md_entitlement_dataStatefulPublisher _md_entit_publisher;
AdminServiceBase::Md_permissions_dataStatefulPublisher _md_perm_publisher;
OrderService _order_service;
map<uint32_t,shared_ptr<const OrderRecord>> _order_map;
public:
//BTS_NO_COPY(UserModelServiceServer);
AdminServiceServer(bts::api::ServiceContext& context);
~AdminServiceServer();
void create_account(const api::admin::AccountCreateAPIArgument& request,
const api::Response<::bts::api::admin::AdminServiceCreateAccountResult>& reply) override;
void create_account(
const AccountCreateAPIArgument& request,
const api::Response<AdminServiceCreateAccountResult>& reply) override;
void update_account(const api::admin::AccountUpdateAPIArgument& request,
const api::Response<::bts::api::admin::AdminServiceOperationResult>& reply) override;
void update_account(
const AccountUpdateAPIArgument& request,
const api::Response<AdminServiceOperationResult>& reply) override;
void create_user(const api::admin::UserCreateAPIArgument& request,
const api::Response<::bts::api::admin::AdminServiceCreateEntityResult>& reply) override;
void create_user(
const UserCreateAPIArgument& request,
const api::Response<AdminServiceCreateEntityResult>& reply) override;
void update_user(const api::admin::UserUpdateAPIArgument& request,
const api::Response<::bts::api::admin::AdminServiceOperationResult>& reply) override;
void update_user(
const UserUpdateAPIArgument& request,
const api::Response<AdminServiceOperationResult>& reply) override;
void get_admin_data(const bts::api::VoidMessage& request,
const api::Response<::bts::api::admin::AdminData>& reply) override;
const api::Response<AdminData>& reply) override;
void delete_user(const bts::api::admin::IDAPIArgument& request,
const api::Response<::bts::api::admin::AdminServiceOperationResult>& reply) override;
void delete_user(
const IDAPIArgument& request,
const api::Response<AdminServiceOperationResult>& reply) override;
void delete_account(
const IDAPIArgument& request,
const api::Response<AdminServiceOperationResult>& reply) override;
void get_order_summary(const bts::api::VoidMessage& request,
const api::Response<OrdersSummary>& reply) override;
void cancel_orders(
const OrderCancelRequest& request,
const api::Response<bts::api::VoidMessage>& reply) override;
void delete_account(const bts::api::admin::IDAPIArgument& request,
const api::Response<::bts::api::admin::AdminServiceOperationResult>& reply) override;
private:
void _update_and_publish (bool is_insert);
void _publish_md_entitlements();
void _publish_md_permissions();
void _add_live_order(map<string,map<string,OrdersSummaryUnit>>& agg,
const OrderRecord& order);
void _cancel_orders(const std::vector<string>& accounts,
const std::vector<string>& users);
};
}}