Merge branch 'master' of bitbucket.org:bluetradesys/spark into simplify_md_sdk

This commit is contained in:
Eric Harding committed 2018-11-16 11:23:51 -05:00
commit e22649c19d
69 files changed
+2026 -560

No files matched your search

+1 -1
View File
@@ -1 +1 @@
set(sources bts_trace.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/Help.cpp;cmds/Md.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/RunSim.cpp;cmds/Span.cpp;cmds/Test.cpp;cmds/TestCeloxica.cpp;cmds/Web.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/WebServer.cpp;main.cpp;md/platforms/celoxica/CXListener.cpp;md/platforms/celoxica/CXMain.cpp;md/platforms/celoxica/tests/cx_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IMarginObserver.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/PositionServiceCore.cpp;position/PositionServiceServer.cpp;position/tests/PositionTests.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceServer.cpp;user/UserServiceServer.cpp;user/tests/admin_service_tests.cpp)
set(sources bts_trace.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/Help.cpp;cmds/Md.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/RunSim.cpp;cmds/Span.cpp;cmds/Test.cpp;cmds/TestCeloxica.cpp;cmds/Web.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/WebServer.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/celoxica/CXListener.cpp;md/platforms/celoxica/CXMain.cpp;md/platforms/celoxica/tests/cx_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IMarginObserver.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/PositionServiceCore.cpp;position/PositionServiceServer.cpp;position/tests/PositionTests.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceServer.cpp;user/UserServiceServer.cpp;user/tests/admin_service_tests.cpp)
+188 -25
View File
@@ -6,6 +6,7 @@
#include "lib/mercury/actors/Actor.h"
#include "lib/mercury/common/FatWrapper.h"
#include "lib/mercury/executor/Executor.h"
#include "lib/orderman/events/OrderStateUpdateEvent.h"
#include "lib/os/os.h"
#include "lib/utils/PBUtils.h"
#include "lib/utils/Puid.h"
@@ -30,12 +31,87 @@
#include <moodycamel/concurrentqueue.h>
#include <moodycamel/readerwriterqueue.h>
#include <mutex>
#include <sys/time.h>
#include <tbb/concurrent_queue.h>
namespace bts {
namespace cmd {
using namespace orderman;
api::orderman::NativeOrderState const *
get_native_state(api::orderman::OrderState const &st) {
switch (st.custom_case()) {
case api::orderman::OrderState::kMarket:
return &st.market().native_state();
case api::orderman::OrderState::kLimit:
return &st.limit().native_state();
case api::orderman::OrderState::kStop:
return &st.stop().native_state();
case api::orderman::OrderState::kStopLimit:
return &st.stop_limit().native_state();
default:
return nullptr;
};
}
class QuickOrderStateUpdateEvent {
public:
QuickOrderStateUpdateEvent(OrderKey key, OrderKey parent_key,
api::orderman::OrderDefinition const &def,
api::orderman::OrderState const &state,
PerfTrace const &trace = PerfTrace())
: key(key), parent_key(parent_key), trace(trace), definition(&def) {
status = state.status();
price = state.price();
quantity = state.quantity();
filled_quantity = state.quantity();
last_price = state.last_price();
avg_price = state.avg_price();
last_msg = state.last_msg();
custom = state.custom_case();
auto native_state = get_native_state(state);
if (native_state) {
exchange_order_id = native_state->exchange_order_id();
last_msg_seq_num = native_state->last_msg_seq_num();
sub_error_code = native_state->sub_error_code();
native_status = native_state->native_status();
routing_record = native_state->routing_record();
native_type = native_state->native_type();
orderman::str_to(native_state->last_transact_time(),&last_transact_time);
}
}
OrderKey key;
OrderKey parent_key;
// common state
api::orderman::OrderStatus status;
double price;
uint32_t quantity;
uint32_t filled_quantity;
double last_price;
double avg_price;
std::string last_msg;
PerfTrace trace;
api::orderman::OrderState::CustomCase custom;
// native order state
std::string exchange_order_id;
uint64_t last_msg_seq_num;
api::orderman::NativeOrderState::SubError sub_error_code;
api::orderman::NativeOrderStatus native_status;
api::orderman::RoutingRecord routing_record;
api::orderman::NativeOrderType native_type;
mercury::TimeStampStr last_transact_time;
api::orderman::OrderDefinition const *definition;
};
auto sec_str = " id: 3\n"
" symbol: \"ES\"\n"
" exchange: VENUE_GLBX\n"
@@ -77,37 +153,59 @@ auto order_str = " order_key: 189285\n"
" cti_code: CTI_CTI1\n"
" order_status: OS_UNACKNOWLEDGED\n";
auto order_state_str = " status: ORDER_STATUS_COMPLETED\n"
" price: 65.56\n"
" quantity: 5\n"
" filled_quantity: 5\n"
" last_price: 65.56\n"
" avg_price: 65.56\n"
auto order_state_str = " status: ORDER_STATUS_WORKING\n"
" price: 73.42\n"
" quantity: 25\n"
" last_msg: \"\"\n"
" last_change {\n"
" seconds: 1532982945\n"
" nanos: 378466000\n"
" seconds: 1541621138\n"
" nanos: 899901000\n"
" }\n"
" limit {\n"
" native_state {\n"
" exchange_order_id: \"12345678901234567\"\n"
" native_status: NATIVE_ORDER_STATUS_FILLED\n"
" last_msg_seq_num: 868\n"
" native_status: NATIVE_ORDER_STATUS_REPLACED\n"
" last_msg_seq_num: 31869\n"
" routing_record {\n"
" session: \"sim\"\n"
" account: \"ca_ACC A\"\n"
" account: \"ACC A\"\n"
" operator_id: \"op_id_bts\"\n"
" cti_code: CTI_CODE_CTI_CODE1\n"
" customer_or_firm: CUSTOMER_OR_FIRM_FIRM\n"
" location: \"us,nc\"\n"
" }\n"
" native_type: NATIVE_ORDER_TYPE_LIMIT\n"
" last_transact_time: \"20181107-20:00:46.000\"\n"
" }\n"
" }\n"
" created {\n"
" seconds: 1532962902\n"
" nanos: 989007000\n"
" seconds: 1541619812\n"
" nanos: 895210000\n"
" }\n";
auto clean_os =
" status: ORDER_STATUS_WORKING\n"
" price: 73.42\n"
" quantity: 25\n"
" last_msg: \"\"\n"
" limit {\n"
" native_state {\n"
//" exchange_order_id: \"12345678901234567\"\n"
" native_status: NATIVE_ORDER_STATUS_REPLACED\n"
" last_msg_seq_num: 31869\n"
// " routing_record {\n"
// " session: \"sim\"\n"
// " account: \"ACC A\"\n"
// " operator_id: \"op_id_bts\"\n"
// " cti_code: CTI_CODE_CTI_CODE1\n"
// " customer_or_firm: CUSTOMER_OR_FIRM_FIRM\n"
// " location: \"us,nc\"\n"
// " }\n"
" native_type: NATIVE_ORDER_TYPE_LIMIT\n"
" last_transact_time: \"20181107-20:00:46.000\"\n"
" }\n"
" }\n";
static void gen_id(benchmark::State &state) {
std::string scope = "benchmark";
@@ -160,16 +258,29 @@ static void copy_perf_trace(benchmark::State &state) {
}
BENCHMARK(copy_perf_trace);
// static void serialize_order(benchmark::State &state) {
// auto order_state =
// utils::pb::from_string<api::orderman::OrderState>(order_state_str);
// auto
// auto order_state_cpy = order_state;
// for (auto _ : state) {
// order_state_cpy = order_state;
// }
// }
// BENCHMARK(copy_order);
static void init_with_clean_os(benchmark::State &state) {
api::orderman::OrderDefinition def;
auto order_state =
utils::pb::from_string<api::orderman::OrderState>(clean_os);
PerfTrace trace;
OrderKey k(1);
for (auto _ : state)
benchmark::DoNotOptimize(
QuickOrderStateUpdateEvent(k, k, def, order_state, trace));
}
BENCHMARK(init_with_clean_os);
static void init_with_old_order_update(benchmark::State &state) {
api::orderman::OrderDefinition def;
auto order_state =
utils::pb::from_string<api::orderman::OrderState>(order_state_str);
PerfTrace trace;
OrderKey k(1);
for (auto _ : state)
benchmark::DoNotOptimize(
orderman::OrderStateUpdateEvent(k, k, def, order_state, trace));
}
BENCHMARK(init_with_old_order_update);
static void serialize_update(benchmark::State &state) {
api::orderman::Update update;
@@ -446,7 +557,6 @@ static void tbb_concurrent_queue(benchmark::State &state) {
}
BENCHMARK(tbb_concurrent_queue);
static void boost_spsc(benchmark::State &state) {
boost::lockfree::spsc_queue<int> q(10000);
@@ -652,7 +762,7 @@ static void moodycamel_concurrent_queue(benchmark::State &state) {
using Q = moodycamel::ConcurrentQueue<int>;
Q queue(8);
int i = 0;
for (auto _ : state) {
queue.enqueue(++i); // NOLINT
benchmark::DoNotOptimize(queue.try_dequeue(i));
@@ -672,6 +782,59 @@ static void moodycamel_spsc_queue(benchmark::State &state) {
}
BENCHMARK(moodycamel_spsc_queue);
struct MyBase {
virtual int inc(int &x) = 0;
};
struct MyDerived : MyBase {
int inc(int &x) override { return x += 1; };
};
struct MyDirect {
int inc(int &x) { return x += 1; };
};
static void derived_call(benchmark::State &state) {
int x = 0;
MyDerived d;
MyBase &b = d;
for (auto _ : state) {
benchmark::DoNotOptimize(b.inc(x));
}
}
BENCHMARK(derived_call);
static void direct_call(benchmark::State &state) {
int x = 0;
MyDerived b;
for (auto _ : state) {
benchmark::DoNotOptimize(b.inc(x));
}
}
BENCHMARK(direct_call);
static void get_time_of_day(benchmark::State &state) {
std::string result(strlen("20121204-14:44:02.999"), ' ');
struct timeval tv;
for (auto _ : state) {
gettimeofday(&tv, nullptr);
auto tm = localtime(&tv.tv_sec);
sprintf(&result[0], "%d%02d%02d-%02d:%02d:%02d.%03d", tm->tm_year,
tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec, 0);
}
}
BENCHMARK(get_time_of_day);
static void fix_time_stamp(benchmark::State &state) {
std::string r(128, 32);
auto now = datetime::now();
for (auto _ : state) {
datetime::FIX_timestamp(now, &r);
}
}
BENCHMARK(fix_time_stamp);
std::string Benchmark::help() {
return "Run micro-benchmarks. Use --benchmark_filter=regex to run only "
"specific benchmarks.";
+42 -20
View File
@@ -1,36 +1,58 @@
#include "server/sparkd/cmds/PrintHedgeOrders.h"
#include "lib/persistence/SqliteStore.h"
#include "lib/api/services/OrderService.pb.h"
#include "lib/persistence/SqliteStore.h"
#include "lib/utils/PBUtils.h"
namespace bts {
namespace cmd {
int PrintHedgeOrders::execute(std::vector<std::string> const &) {
persistence::SqliteStore store("orders");
auto all = store.retrieve_all()["orders"];
persistence::SqliteStore order_store("orders");
auto all_orders = order_store.retrieve_all()["orders"];
std::cout << "order_key,time_to_completed" << "\n";
api::orderman::OrderRecord rec;
for (auto const &kvp : all)
if (utils::pb::from_string<api::orderman::OrderRecord>(kvp.second, &rec)) {
auto &def = rec.def();
auto &state = rec.state();
if (rec.parent_key() &&
def.custom_def().has_limit() &&
def.sub_type() == api::orderman::OrderSubType::ORDER_SUB_TYPE_HEDGING_LEG) {
auto created = state.created().seconds()* 1e9 + state.created().nanos();
auto updated = state.last_change().seconds()* 1e9 + state.last_change().nanos();
std::cout << kvp.first << "," << (updated-created) / 1000 / 1000 / 1000 << "\n";
std::map<uint32_t, std::pair<std::string, double>> fill_times;
persistence::SqliteStore fill_store("fills");
auto all_fills = fill_store.retrieve_all()["fills"];
for (auto const &kvp : all_fills) {
api::orderman::Fill fill_rec;
if (utils::pb::from_string<api::orderman::Fill>(kvp.second, &fill_rec)) {
auto ts = utils::pb::to_double_time(fill_rec.timestamp());
auto &p = fill_times[fill_rec.order_key()];
auto &min_time = p.second;
if (min_time < ts) {
p.first = kvp.first;
min_time = ts;
}
}
}
std::cout << "order_key,time_to_first_fill,fill_id"
<< "\n";
api::orderman::OrderRecord rec;
for (auto const &kvp : all_orders)
if (utils::pb::from_string<api::orderman::OrderRecord>(kvp.second,
&rec)) {
auto &def = rec.def();
auto &state = rec.state();
if (rec.parent_key() && def.custom_def().has_limit() &&
def.sub_type() ==
api::orderman::OrderSubType::ORDER_SUB_TYPE_HEDGING_LEG) {
auto create_ts = utils::pb::to_double_time(state.created());
auto &first_fill = fill_times[rec.key()];
auto delta = first_fill.second - create_ts;
if (first_fill.first.size())
std::cout << kvp.first << "," << delta << ","
<< first_fill.first << "\n";
}
}
return 0;
}
std::string PrintHedgeOrders::help(){
return
"Print out order ids for all hedge orders"
"which were completed at least one second being created";
std::string PrintHedgeOrders::help() {
return "Print out order ids for all hedge orders"
"which were completed at least one second being created";
}
}}
} // namespace cmd
} // namespace bts
+15 -40
View File
@@ -1,61 +1,36 @@
#include "QuickTest.h"
#include "ICmd.h"
#include "lib/api/services/OrderService.server.pb.h"
#include "lib/mercury/actors/PingPong.h"
#include "lib/mercury/broker/BrokerProxy.h"
#include "lib/mercury/common/mercury_common.h"
#include "lib/mercury/executor/Executor.h"
#include "lib/utils/PBUtils.h"
#include "lib/utils/thread_manager.h"
#include "lib/orderman/events/OrderStateUpdateEvent.h"
#include <iostream>
namespace bts {
namespace cmd {
void test_ping_pong_mercury2()
{
using namespace mercury;
Executor ex1;
Executor ex2;
ex1.init_neighbours(std::vector<Executor*> {&ex2});
ex2.init_neighbours(std::vector<Executor*> {&ex1});
ex1.spawn<Ping>();
//ex1.spawn<Pong>();
ex2.spawn<Pong>();
auto t = std::thread([&ex2]() {
cpu_set_t cpuset;
CPU_ZERO(&cpuset);
CPU_SET(1, &cpuset);
pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
while (true)
ex2.do_work();
});
bts::sleep_for(2s);
cpu_set_t cpuset;
CPU_ZERO(&cpuset);
CPU_SET(0, &cpuset);
pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset);
while (true) {
ex1.do_work();
bts::sleep_for(100ms);
}
}
using namespace bts::mercury;
using namespace bts::orderman;
int QuickTest::execute(std::vector<std::string> const &) {
test_ping_pong_mercury2();
sleep_for(10s);
for (int i = 0; i < 100; i++) {
auto t = utils::TimeStamp::now();
LOG(OrderMan,Info,"Now: {}", (utils::TimeStamp::now() - t).nanoseconds());
}
sleep_for(10s);
return 0;
}
std::string QuickTest::help() {
return "This is a place holder to make it easy to quickly write discardable "
"test code";
"test code";
}
} // namespace cmd
} // namespace bts