Fixing merge conflict

This commit is contained in:
Taha Afzal committed 2020-09-09 10:35:14 -04:00
commit 5fbd5b8804
25 files changed
+606 -236

No files matched your search

+1 -1
View File
@@ -1 +1 @@
set(sources algo/AlgoDefinitionServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CMECertCmd.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/FIXSim.cpp;cmds/Help.cpp;cmds/KeyCode.cpp;cmds/Md.cpp;cmds/ParseHistoricalFIX.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/Span.cpp;cmds/Spartan.cpp;cmds/Test.cpp;cmds/Web.cpp;cmds/spartan/algo/RunTAS.cpp;cmds/spartan/algo/TASOrderReactor.cpp;cmds/spartan/algo/TASTriggerDef.cpp;cmds/spartan/md/mdp3/back_testing/FifoPriceLevelMatcher.cpp;cmds/spartan/md/mdp3/back_testing/FifoSecurityMatcher.cpp;cmds/spartan/md/mdp3/back_testing/SimAlgo.cpp;cmds/spartan/md/mdp3/back_testing/gen_sim_input.cpp;cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp;cmds/spartan/md/mdp3/feed_control/IFeedController.cpp;cmds/spartan/md/mdp3/handler/ChannelHandler.cpp;cmds/spartan/md/mdp3/parsing/MDP3MsgBox.cpp;cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp;cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp;cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.cpp;cmds/spartan/md/mdp3/recovery/IncrementalQueue.cpp;cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp;cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp;cmds/spartan/md/mdp3/utils/mdp3_utils.cpp;cmds/spartan/util/MCastReceiver.cpp;cmds/spartan/util/PCapReader.cpp;cmds/spartan/util/SecurityDB.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/MultipartFormDecoder.cpp;http/WebServer.cpp;http/tests/MultiPartFormTests.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/IMDPlatform.cpp;md/platforms/ice_onixs/ICEListener.cpp;md/platforms/ice_onixs/ICEMain.cpp;md/platforms/ice_onixs/tests/ice_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/MarginServiceServer.cpp;position/PositionAggregator.cpp;position/PositionMap.cpp;position/PositionServicePNLHelper.cpp;position/PositionServiceServer.cpp;position/import/BasePositionServiceServer.cpp;position/import/DownloadCommonStateMachine.cpp;position/import/DownloadWindow.cpp;position/import/MoneyFileServiceServer.cpp;position/import/parsing/gmi_parser.cpp;position/import/parsing/money_parser.cpp;position/import/parsing/parser_common.cpp;position/tests/PositionTests.cpp;position/tests/position_tests_internal.cpp;subprocess/SubProcessRunner.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceDB.cpp;user/AdminServiceServer.cpp;user/AuthServiceServer.cpp;user/tests/admin_service_tests.cpp)
set(sources algo/AlgoDefinitionServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CMECertCmd.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/FIXSim.cpp;cmds/Help.cpp;cmds/KeyCode.cpp;cmds/Md.cpp;cmds/ParseHistoricalFIX.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/Span.cpp;cmds/Spartan.cpp;cmds/Test.cpp;cmds/Web.cpp;cmds/spartan/algo/RunTAS.cpp;cmds/spartan/algo/TASOrderReactor.cpp;cmds/spartan/algo/TASTriggerDef.cpp;cmds/spartan/md/mdp3/back_testing/matcher/FifoPriceLevelMatcher.cpp;cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.cpp;cmds/spartan/md/mdp3/back_testing/simulator/IAlgoModel.cpp;cmds/spartan/md/mdp3/back_testing/simulator/ISimulatorServices.cpp;cmds/spartan/md/mdp3/back_testing/simulator/Simulator.cpp;cmds/spartan/md/mdp3/back_testing/simulator/SimulatorInput.cpp;cmds/spartan/md/mdp3/common/InstrumentSnapshot.cpp;cmds/spartan/md/mdp3/feed_control/IFeedController.cpp;cmds/spartan/md/mdp3/handler/ChannelHandler.cpp;cmds/spartan/md/mdp3/parsing/MDP3MsgBox.cpp;cmds/spartan/md/mdp3/realtime/RealTimeHandler.cpp;cmds/spartan/md/mdp3/recovery/BookSnapshot.cpp;cmds/spartan/md/mdp3/recovery/DefinitionSnapshot.cpp;cmds/spartan/md/mdp3/recovery/IncrementalQueue.cpp;cmds/spartan/md/mdp3/recovery/RecoveryHandler.cpp;cmds/spartan/md/mdp3/tests/spartan_md_tests.cpp;cmds/spartan/md/mdp3/utils/mdp3_utils.cpp;cmds/spartan/util/MCastReceiver.cpp;cmds/spartan/util/PCapReader.cpp;cmds/spartan/util/SecurityDB.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/MultipartFormDecoder.cpp;http/WebServer.cpp;http/tests/MultiPartFormTests.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/IMDPlatform.cpp;md/platforms/ice_onixs/ICEListener.cpp;md/platforms/ice_onixs/ICEMain.cpp;md/platforms/ice_onixs/tests/ice_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/MarginServiceServer.cpp;position/PositionAggregator.cpp;position/PositionMap.cpp;position/PositionServicePNLHelper.cpp;position/PositionServiceServer.cpp;position/import/BasePositionServiceServer.cpp;position/import/DownloadCommonStateMachine.cpp;position/import/DownloadWindow.cpp;position/import/MoneyFileServiceServer.cpp;position/import/parsing/gmi_parser.cpp;position/import/parsing/money_parser.cpp;position/import/parsing/parser_common.cpp;position/tests/PositionTests.cpp;position/tests/position_tests_internal.cpp;subprocess/SubProcessRunner.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceDB.cpp;user/AdminServiceServer.cpp;user/AuthServiceServer.cpp;user/tests/admin_service_tests.cpp)
+131 -129
View File
@@ -1,6 +1,6 @@
#include "Spartan.h"
#include "server/sparkd/cmds/spartan/algo/RunTAS.h"
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/gen_sim_input.h"
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/simulator/SimulatorInput.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h"
#include "server/sparkd/cmds/spartan/md/mdp3/feed_control/AB_Arbitrageur.h"
#include "server/sparkd/cmds/spartan/md/mdp3/feed_control/IFeedController.h"
@@ -15,7 +15,7 @@
#include <fstream>
#include <iostream>
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/FifoSecurityMatcher.h"
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.h"
#include "server/sparkd/cmds/spartan/md/mdp3/parsing/MDP3Parser.h"
#include "server/sparkd/cmds/spartan/util/PCapReader.h"
@@ -27,149 +27,151 @@ using namespace spartan::mdp3;
std::map<std::string, std::function<int(std::vector<std::string> const &)>>
extensions = {
{"validate_fifo",
[](auto const &args) {
struct BackTesterHarness : IChannelObserver {
std::unordered_map<MDP3SecurityID, Orders> orders;
std::unordered_map<MDP3OrderKey, DeducedIceberg> icebergs;
size_t passed_tests = 0;
{"validate_fifo",
[](auto const &args) {
struct BackTesterHarness : IChannelObserver {
std::unordered_map<MDP3SecurityID, Orders> orders;
std::unordered_map<MDP3OrderKey, DeducedIceberg> icebergs;
size_t passed_tests = 0;
BackTesterHarness(std::string const &pcap)
: icebergs(find_icebergs(pcap)) {}
BackTesterHarness(std::string const &pcap)
: icebergs(find_icebergs(pcap)) {}
void
on_recovery_complete(InitialSnapshot const &snap) override {
for (auto s : snap) {
auto annotated_orders = s->orders();
add_iceberg_annotation(annotated_orders);
orders.emplace(s->security_id(), annotated_orders);
}
}
void on_recovery_complete(InitialSnapshot const &snap) override {
for (auto s : snap) {
auto annotated_orders = s->orders();
add_iceberg_annotation(annotated_orders);
orders.emplace(s->security_id(), annotated_orders);
}
}
void on_real_time_event(MDP3Event const &ev) override {
bool ignore_ibs = true;
for (auto s : ev.affected_securities) {
auto it = orders.find(s->security_id());
if (it == orders.end()) {
bts_assert(s->orders().empty());
it = orders.emplace(s->security_id(), s->orders())
.first;
}
auto &old_orders = it->second;
auto deduced = s->deduced_order_changes();
void on_real_time_event(MDP3Event const &ev) override {
for (auto s : ev.affected_securities) {
auto it = orders.find(s->security_id());
if (it == orders.end()) {
bts_assert(s->orders().empty());
it =
orders.emplace(s->security_id(), s->orders()).first;
}
auto &old_orders = it->second;
auto deduced = s->deduced_order_changes();
// We have a hard time simulating icebergs,
// for now skip them
auto found_ice = false;
for (auto &f : s->fills())
if (icebergs.count(f.key)) {
found_ice = true;
break;
}
if (found_ice)
if(ignore_ibs)
continue;
// We have a hard time simulating icebergs,
// for now skip them
auto found_ice = false;
for (auto &f : s->fills())
if (icebergs.count(f.key)) {
found_ice = true;
break;
}
if (found_ice)
continue;
if (deduced.size()) {
if (deduced.size()) {
// We have deduced orders, lets see what the matcher
// says
add_iceberg_annotation(deduced);
// We have deduced orders, lets see what the matcher
// says
add_iceberg_annotation(deduced);
FifoSecurityMatcher matcher(ev.event_id,old_orders);
auto current_orders = s->orders();
add_iceberg_annotation(current_orders);
for (auto &o : deduced)
matcher.on_order_change(o);
auto error = matcher.has_same_orders(current_orders);
if (error) {
std::cout << "Failing on event:\n " << ev.event_id
<< std::endl
<< "Error: " << error.value() << std::endl
<< "Deduced orders:\n"
<< deduced << std::endl
<< "Old Orders:\n"
<< old_orders << std::endl
<< "Order Changes:\n"
<< s->order_changes() << std::endl
<< "Fills:\n"
<< s->fills() << std::endl;
bts_assert(false);
}
old_orders = current_orders;
} else {
it->second = s->orders();
add_iceberg_annotation(it->second);
}
}
}
void add_iceberg_annotation(MDP3Order &order) {
auto it = icebergs.find(order.key);
if (it != icebergs.end())
order.iceberg_quantity = it->second.estimated_quantity;
}
void add_iceberg_annotation(Orders &orders) {
for (auto &o : orders)
add_iceberg_annotation(o.second);
}
void add_iceberg_annotation(OrderChanges &changes) {
for (auto &o : changes) {
add_iceberg_annotation(o.previous_order);
add_iceberg_annotation(o.order);
}
}
auto current_orders = s->orders();
add_iceberg_annotation(current_orders);
for (auto &o : deduced)
matcher.on_order_change(o);
auto error = matcher.has_same_orders(current_orders);
if (error) {
std::cout << "Failing on event:\n " << ev.event_id
<< std::endl
<< "Error: " << error.value() << std::endl
<< "Deduced orders:\n"
<< deduced << std::endl
<< "Old Orders:\n"
<< old_orders << std::endl
<< "Order Changes:\n"
<< s->order_changes() << std::endl
<< "Fills:\n"
<< s->fills() << std::endl;
bts_assert(false);
}
old_orders = current_orders;
} else {
it->second = s->orders();
add_iceberg_annotation(it->second);
}
}
}
void on_real_time_channel_loss() override { bts_assert(false); }
} harness(args[1]);
observe_channel(args[1], harness);
return 0;
}},
void add_iceberg_annotation(MDP3Order &order) {
auto it = icebergs.find(order.key);
if (it != icebergs.end())
order.iceberg_quantity = it->second.estimated_quantity;
}
{"deduce_orders",
[](auto const &args) {
struct OrderGen : IChannelObserver {
std::ofstream out;
OrderGen(std::string const &out_path) : out(out_path) {}
void add_iceberg_annotation(Orders &orders) {
for (auto &o : orders)
add_iceberg_annotation(o.second);
}
void on_recovery_complete(InitialSnapshot const &) override {}
void on_real_time_event(MDP3Event const &ev) override {
for (auto s : ev.affected_securities) {
auto deduced = s->deduced_order_changes();
if (deduced.size()) {
out << "sym: " << s->definition().symbol()
<< " eid: " << ev.event_id
<< " sn: " << ev.first_packet_sn << " to "
<< ev.last_packet_sn
<< " ts: " << to_time_string(ev.transact_time)
<< std::endl
<< "Order changes: " << std::endl
<< s->order_changes() << "Fills: " << std::endl
<< s->fills() << "Deduced changes: " << std::endl
<< s->deduced_order_changes() << std::endl
<< std::endl;
}
}
}
void on_real_time_channel_loss() override { bts_assert(false); }
} gen(args[2]);
void add_iceberg_annotation(OrderChanges &changes) {
for (auto &o : changes) {
add_iceberg_annotation(o.previous_order);
add_iceberg_annotation(o.order);
}
}
observe_channel(args[1], gen);
return 0;
}},
{"find_icebergs", [](auto const &args) {
std::ofstream out(args[2]);
for (auto const &[k, ice] : find_icebergs(args[1]))
out << "ok: " << k << " sz: " << ice.estimated_quantity.value()
<< std::endl;
return 0;
void on_real_time_channel_loss() override { bts_assert(false); }
} harness(args[1]);
observe_channel(args[1], harness);
return 0;
}},
{"gen_sim_input", [](auto const &args) {
auto input = gen_sim_input(args[1]);
{"deduce_orders",
[](auto const &args) {
struct OrderGen : IChannelObserver {
std::ofstream out;
OrderGen(std::string const &out_path) : out(out_path) {}
void on_recovery_complete(InitialSnapshot const &) override {}
void on_real_time_event(MDP3Event const &ev) override {
for (auto s : ev.affected_securities) {
auto deduced = s->deduced_order_changes();
if (deduced.size()) {
out << "sym: " << s->definition().symbol()
<< " eid: " << ev.event_id
<< " sn: " << ev.first_packet_sn << " to "
<< ev.last_packet_sn
<< " ts: " << to_time_string(ev.transact_time)
<< std::endl
<< "Order changes: " << std::endl
<< s->order_changes() << "Fills: " << std::endl
<< s->fills() << "Deduced changes: " << std::endl
<< s->deduced_order_changes() << std::endl
<< std::endl;
}
}
}
void on_real_time_channel_loss() override { bts_assert(false); }
} gen(args[2]);
observe_channel(args[1], gen);
return 0;
}},
{"find_icebergs",
[](auto const &args) {
std::ofstream out(args[2]);
for (auto const &[k, ice] : find_icebergs(args[1]))
out << "ok: " << k << " sz: " << ice.estimated_quantity.value()
<< std::endl;
return 0;
}},
{"gen_sim_input",
[](auto const &args) {
auto input = make_simulator_input(args[1]);
std::cout << "Secs: " << input.instruments.size()
<< " events: " << input.events.size()
<< std::endl;
<< " max_order_id: " << input.max_order_id << std::endl;
return 0;
}},
};
@@ -1 +0,0 @@
#include "SimAlgo.h"
@@ -1,34 +0,0 @@
#pragma once
#include "SimEvent.h"
#include "../common/InstrumentDefinition.h"
#include "../common/mdp3_common.h"
namespace bts::spartan::mdp3 {
class SimAlgo {
public:
// All securities
InstrumentDefinitions const &instruments() const;
// Order management
MDP3OrderKey create_order(MDP3SecurityID, api::Side, md::IntPrice,
MDP3Quantity, size_t latency_ns = 0);
void update_order(MDP3OrderKey, md::IntPrice, MDP3Quantity,
size_t latency_ns = 0);
void cancel_order(MDP3OrderKey, size_t latency_ns = 0);
// PnL as of now
int64_t pandl() const;
// Event handling
virtual void init(){};
virtual void finit(){};
virtual void on_time(MDP3TimeStamp) {}
virtual void on_event(SimEvent const &) = 0;
private:
struct PImpl;
std::unique_ptr<PImpl> me;
};
} // namespace bts::spartan::mdp3
@@ -1,21 +0,0 @@
#pragma once
#include "../common/mdp3_common.h"
namespace bts::spartan::mdp3 {
class InstrumentEvent {
public:
~InstrumentEvent();
MDP3SecurityID security_id() const;
Book const &book(api::Side) const;
Fills const &fills() const;
private:
struct PImpl;
std::unique_ptr<PImpl> me;
};
using SimEvent = std::vector<InstrumentEvent>;
}
@@ -1,24 +0,0 @@
#pragma once
#include "../common/mdp3_common.h"
#include "../common/InstrumentDefinition.h"
namespace bts::spartan::mdp3 {
struct SimExchangeEvent {
size_t event_id;
MDP3TimeStamp transact_time;
MDP3TimeStamp first_packet_send_time;
MDP3TimeStamp last_packet_send_time;
uint32_t first_packet_sn;
uint32_t last_packet_sn;
std::vector<OrderChange> order_changes;
};
struct SimInput {
std::vector<InstrumentDefinition> instruments;
std::vector<SimExchangeEvent> events;
};
SimInput gen_sim_input(std::string const&pcap);
}
@@ -1,5 +1,5 @@
#include "FifoPriceLevelMatcher.h"
#include "../common/InstrumentSnapshot.h"
#include "../../common/InstrumentSnapshot.h"
namespace bts::spartan::mdp3 {
@@ -2,8 +2,8 @@
// did not want to fwd decl everything specially
// the templated classes (will fix later)
#include "../common/InstrumentSnapshot.h"
#include "MatchingCommon.h"
#include "../MatchingCommon.h"
#include "../../common/InstrumentSnapshot.h"
namespace bts::spartan::mdp3 {
@@ -1,5 +1,5 @@
#include "FifoSecurityMatcher.h"
#include "../common/InstrumentSnapshot.h"
#include "../../common/InstrumentSnapshot.h"
namespace bts::spartan::mdp3 {
@@ -2,7 +2,7 @@
// did not want to fwd decl everything specially
// the templated classes (will fix later)
#include "../common/InstrumentSnapshot.h"
#include "../../common/InstrumentSnapshot.h"
#include "FifoPriceLevelMatcher.h"
#include <boost/container/flat_set.hpp>
@@ -0,0 +1,30 @@
#pragma once
#include "../simulator/AlgoAction.h"
#include "../simulator/AlgoEvent.h"
#include "../../common/mdp3_common.h"
namespace bts::spartan::mdp3 {
class InstrumentDefinition;
class Matcher {
public:
Matcher () {}
Matcher (std::vector<InstrumentDefinition> const&) {}
Matcher (Matcher&&) {}
Matcher &operator=(Matcher&&) {return *this;}
using MatchEffect =
std::variant<MBPUpdate,OrderUpdate>;
enum ChangeSrc {
ALGO, SIMULATOR_INPUT
};
void match(ChangeSrc,
std::vector<OrderChange> const&,
std::vector<MatchEffect> *) {}
};
}
@@ -0,0 +1,28 @@
#pragma once
#include "../../common/mdp3_common.h"
namespace bts::spartan::mdp3 {
struct CreateOrder {
MDP3OrderKey ok;
MDP3SecurityID security;
api::Side side;
MDP3OrderPriority priority;
md::IntPrice price;
MDP3Quantity quantity;
};
struct UpdateOrder {
MDP3OrderKey ok;
md::IntPrice price;
MDP3Quantity quantity;
};
struct CancelOrder {
MDP3OrderKey ok;
};
using AlgoAction = std::variant<CreateOrder,UpdateOrder,CancelOrder>;
}
@@ -0,0 +1,33 @@
#pragma once
#include "../../common/mdp3_common.h"
namespace bts::spartan::mdp3 {
struct MBPUpdate {
MDP3SecurityID security_id;
Book mbp_buy;
Book mbp_sell;
Book const &mbp_side(api::Side side) const {
if (side == api::SIDE_BUY)
return mbp_buy;
else
return mbp_sell;
}
};
struct OrderUpdate {
MDP3OrderKey key;
enum Status { FILLED, PARTIALLY_FILLED, CANCELED };
Status status;
MDP3Quantity last_fills;
md::IntPrice last_price;
};
struct AlgoEvent {
MDP3TimeStamp transact_time;
MDP3TimeStamp send_time;
std::vector<std::variant<MBPUpdate, OrderUpdate>> parts;
};
} // namespace bts::spartan::mdp3
@@ -0,0 +1,17 @@
#pragma once
#include "../../common/mdp3_common.h"
#include "AlgoEvent.h"
namespace bts::spartan::mdp3 {
class ISimulatorServices;
class IAlgo {
public:
virtual ~IAlgo();
virtual void init(ISimulatorServices &) = 0;
virtual void on_event(AlgoEvent const&) {}
};
}
@@ -0,0 +1,7 @@
#include "IAlgoModel.h"
namespace bts::spartan::mdp3 {
IAlgoModel::~IAlgoModel() {}
}
@@ -0,0 +1,18 @@
#pragma once
#include "../../common/mdp3_common.h"
#include "AlgoEvent.h"
namespace bts::spartan::mdp3 {
class ISimulatorServices;
class SimulatorEvent;
class IAlgoModel {
public:
virtual ~IAlgoModel();
virtual void init(ISimulatorServices &) = 0;
virtual void on_event(AlgoEvent const&) {}
};
}
@@ -0,0 +1,8 @@
#include "ISimulatorServices.h"
namespace bts::spartan::mdp3 {
ISimulatorServices::~ISimulatorServices() {}
}
@@ -0,0 +1,18 @@
#pragma once
#include "../../common/mdp3_common.h"
#include "AlgoAction.h"
namespace bts::spartan::mdp3 {
class InstrumentDefinition;
class ISimulatorServices {
public:
virtual ~ISimulatorServices();
virtual std::vector<InstrumentDefinition> const &instruments() const = 0;
virtual MDP3TimeStamp time() const = 0;
virtual MDP3OrderKey generate_order_key() = 0;
virtual void send(AlgoAction const &, size_t latency_ns = 0) = 0;
};
} // namespace bts::spartan::mdp3
@@ -0,0 +1,187 @@
#include "Simulator.h"
#include "../matcher/Matcher.h"
#include "AlgoEvent.h"
#include "IAlgoModel.h"
#include "ISimulatorServices.h"
#include "SimulatorInput.h"
#include "lib/utils/overloaded.h"
#include <queue>
namespace bts::spartan::mdp3 {
struct Simulator::AlgoActionRec {
MDP3TimeStamp transact_time;
AlgoRecord *algo_rec;
AlgoAction action;
bool operator<(AlgoActionRec const &rhs) const {
return rhs.transact_time.value() < transact_time.value();
}
};
struct Simulator::AlgoEventRec {
MDP3TimeStamp send_time;
AlgoEvent event;
bool operator<(AlgoEventRec const &rhs) const {
return rhs.send_time.value() < send_time.value();
}
};
struct Simulator::PImpl {
PImpl(SimulatorInput const &i) : input(i) {}
SimulatorInput const &input;
void reset() {
next_input_event = 0;
last_order_key = input.max_order_id;
algos.clear();
matcher = Matcher{input.instruments};
pending_algo_actions = {};
order_to_algo = {};
}
size_t next_input_event;
size_t last_order_key;
std::vector<std::unique_ptr<AlgoRecord>> algos;
Matcher matcher;
std::priority_queue<AlgoActionRec> pending_algo_actions;
std::unordered_map<MDP3OrderKey, AlgoRecord *> order_to_algo;
};
struct Simulator::AlgoRecord : ISimulatorServices {
Simulator &owner;
IAlgoModel &algo;
MDP3TimeStamp current_time;
RunOptions options;
uint64_t pandl;
std::priority_queue<AlgoEventRec> pending_events;
AlgoRecord(Simulator &owner, IAlgoModel &algo, const RunOptions &options)
: owner(owner), algo(algo), current_time(0), options(options), pandl(0) {
algo.init(*this);
}
std::vector<InstrumentDefinition> const &instruments() const override {
return owner.me->input.instruments;
}
MDP3OrderKey generate_order_key() override {
return MDP3OrderKey{++owner.me->last_order_key};
}
MDP3TimeStamp time() const override { return current_time; }
void send(AlgoAction const &action, size_t latency_ns) override {
if (latency_ns == 0)
latency_ns = options.default_algo_latency_ns;
bts_assert(pending_events.size());
auto &current_event = pending_events.top();
owner.me->pending_algo_actions.push(
{MDP3TimeStamp{current_event.send_time.value() + latency_ns}, this,
action});
}
void deliver_events() {
while (pending_events.size()) {
auto &event = pending_events.top().event;
current_time = event.send_time;
algo.on_event(event);
pending_events.pop();
}
}
};
Simulator::Simulator(SimulatorInput const &input) : me(new PImpl{input}) {}
Simulator::~Simulator() {}
int64_t Simulator::run(IAlgoModel &algo, const RunOptions &options) {
return run({&algo}, options)[0];
}
std::vector<uint64_t> Simulator::run(std::vector<IAlgoModel *> const &algos,
const RunOptions &options) {
me->reset();
for (auto algo : algos)
me->algos.push_back(std::make_unique<AlgoRecord>(*this, *algo, options));
while (has_work())
do_work();
std::vector<uint64_t> pandl;
for (auto &ar : me->algos)
pandl.push_back(ar->pandl);
return pandl;
}
bool Simulator::has_work() const {
if (me->next_input_event != me->input.events.size() ||
me->pending_algo_actions.size())
return true;
else {
for (auto &a : me->algos)
if (a->pending_events.size())
return true;
return false;
}
}
void Simulator::do_work() {
if (me->next_input_event != me->input.events.size() &&
me->pending_algo_actions.size()) {
// Both input events and algo actions are available, pick the
// earliest
auto &top_input_event = me->input.events[me->next_input_event];
auto top_algo_action = me->pending_algo_actions.top();
if (top_input_event.last_packet_send_time.value() <=
top_algo_action.transact_time.value()) {
// Input queue has it
me->next_input_event++;
on_simulator_input_event(top_input_event);
} else {
me->pending_algo_actions.pop();
on_algo_action(top_algo_action);
}
} else if (me->next_input_event != me->input.events.size()) {
// Only have simulator events
auto &top_input_event = me->input.events[me->next_input_event];
me->next_input_event++;
on_simulator_input_event(top_input_event);
} else if (me->pending_algo_actions.size()) {
// Only have algo actions
auto top_algo_action = me->pending_algo_actions.top();
me->pending_algo_actions.pop();
on_algo_action(top_algo_action);
}
for (auto &algo : me->algos)
algo->deliver_events();
}
void Simulator::on_algo_action(AlgoActionRec const&) {
}
void Simulator::on_simulator_input_event(SimulatorInputEvent const &) {
// std::vector<Matcher::MatchEffect> effects;
// me->matcher.match(Matcher::SIMULATOR_INPUT, ev.order_changes, &effects);
// for (auto &e : effects)
// for (auto &a : algo) {
// std::visit(
// overloaded{
// [&](MBPUpdate const &mbp_update) {
// algo->add_to_pending_mbp_event(
// ev.transact_time,
// ev.last_packet_send_time,
// mbp_update);
// },
// [&](OrderUpdate const &) {}},
// e);
// }
}
} // namespace bts::spartan::mdp3
@@ -0,0 +1,55 @@
#pragma once
#include <memory>
#include <vector>
#include "../../common/mdp3_common.h"
namespace bts::spartan::mdp3 {
struct SimulatorInput;
class IAlgoModel;
struct SimulatorInputEvent;
class Simulator {
public:
Simulator(SimulatorInput const &);
~Simulator();
struct RunOptions {
RunOptions() : default_algo_latency_ns(0),
ilink_delta_ns(0),
estimated_exchange_latency_ns(0)
{}
// The default latency added to all algo actions
size_t default_algo_latency_ns = 0;
// The delta between the market data and the corresponding
// execution report (can be negative)
int64_t ilink_delta_ns = 0;
// The latency we use when calculating how long it takes to
// process our own algo events
size_t estimated_exchange_latency_ns = 50000;
};
int64_t run(IAlgoModel &, const RunOptions & = RunOptions());
std::vector<uint64_t> run(std::vector<IAlgoModel *> const &, const RunOptions & = RunOptions());
private:
struct AlgoRecord;
struct AlgoActionRec;
struct AlgoEventRec;
struct PImpl;
void enqueue_action(AlgoActionRec const&);
bool has_work() const;
void do_work();
void on_simulator_input_event(SimulatorInputEvent const&);
void on_algo_action(AlgoActionRec const&);
MDP3TimeStamp next_simulator_event_send_time() const;
MDP3OrderKey next_order_id();
std::unique_ptr<PImpl> me;
};
} // namespace bts::spartan::mdp3
@@ -1,17 +1,17 @@
#include "gen_sim_input.h"
#include "../common/InstrumentSnapshot.h"
#include "../handler/IChannelObserver.h"
#include "../utils/mdp3_utils.h"
#include "../../common/InstrumentSnapshot.h"
#include "../../handler/IChannelObserver.h"
#include "../../utils/mdp3_utils.h"
#include "SimulatorInput.h"
namespace bts::spartan::mdp3 {
SimInput gen_sim_input(std::string const &pcap) {
SimulatorInput make_simulator_input(std::string const &pcap) {
struct Obs : IChannelObserver {
SimInput result;
SimulatorInput result;
std::unordered_map<MDP3SecurityID, InstrumentDefinition> instruments;
std::vector<SimExchangeEvent> events;
std::vector<SimulatorInputEvent> events;
std::unordered_set<MDP3SecurityID> ready_secs;
void on_recovery_complete(
@@ -38,7 +38,7 @@ SimInput gen_sim_input(std::string const &pcap) {
if (!sec_ready && s->trading_status() ==
::mdp3::SecurityTradingStatus::ReadyToTrade) {
// Transitioned to ready, boostrap book with existing orders
// Transitioned to ready, boostrap book with existing orders
ready_secs.insert(s->security_id());
for (auto const &p : s->orders()) {
@@ -47,6 +47,8 @@ SimInput gen_sim_input(std::string const &pcap) {
change.order = p.second;
change.change = OrderChange::added;
this_event.order_changes.push_back(change);
result.max_order_id =
std::max(change.order.key.value(), result.max_order_id);
}
} else if (sec_ready &&
s->trading_status() ==
@@ -55,11 +57,14 @@ SimInput gen_sim_input(std::string const &pcap) {
auto changes = s->deduced_order_changes();
if (changes.empty())
changes = s->order_changes();
for (auto &c : changes)
for (auto &c : changes) {
this_event.order_changes.push_back(c);
}
else if (sec_ready && s->trading_status() && s->trading_status() !=
::mdp3::SecurityTradingStatus::ReadyToTrade) {
result.max_order_id =
std::max(c.order.key.value(), result.max_order_id);
}
} else if (sec_ready && s->trading_status() &&
s->trading_status() !=
::mdp3::SecurityTradingStatus::ReadyToTrade) {
ready_secs.erase(s->security_id());
}
}
@@ -69,7 +74,7 @@ SimInput gen_sim_input(std::string const &pcap) {
for (auto const &p : instruments)
result.instruments.push_back(p.second);
}
void on_real_time_channel_loss() override { bts_assert(false); }
} obs;
@@ -0,0 +1,16 @@
#pragma once
#include "../../common/InstrumentDefinition.h"
#include "SimulatorInputEvent.h"
namespace bts::spartan::mdp3 {
struct SimulatorInput {
std::vector<InstrumentDefinition> instruments;
std::vector<SimulatorInputEvent> events;
size_t max_order_id = 0;
};
SimulatorInput make_simulator_input(std::string const&pcap);
}
@@ -0,0 +1,15 @@
#include "../../common/mdp3_common.h"
namespace bts::spartan::mdp3 {
struct SimulatorInputEvent {
size_t event_id;
MDP3TimeStamp transact_time;
MDP3TimeStamp first_packet_send_time;
MDP3TimeStamp last_packet_send_time;
uint32_t first_packet_sn;
uint32_t last_packet_sn;
std::vector<OrderChange> order_changes;
};
} // namespace bts::spartan::mdp3
@@ -5,8 +5,8 @@
#include "lib/utils/TempFile.h"
#include "lib/utils/overloaded.h"
#include "lib/utils/type_name.h"
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/FifoPriceLevelMatcher.h"
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/FifoSecurityMatcher.h"
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoPriceLevelMatcher.h"
#include "server/sparkd/cmds/spartan/md/mdp3/back_testing/matcher/FifoSecurityMatcher.h"
#include "server/sparkd/cmds/spartan/md/mdp3/common/InstrumentSnapshot.h"
#include "server/sparkd/cmds/spartan/md/mdp3/feed_control/AB_Arbitrageur.h"
#include "server/sparkd/cmds/spartan/md/mdp3/feed_control/IFeedController.h"
@@ -100,12 +100,12 @@ TEST_CASE("spartan_md", "[spartan_md]") {
auto buy_ok = is->build_book_from_orders(api::SIDE_BUY) ==
is->book(api::SIDE_BUY);
if (!buy_ok)
std::cout << "shit";
debug_break();
REQUIRE(buy_ok);
auto sell_ok = is->build_book_from_orders(api::SIDE_SELL) ==
is->book(api::SIDE_SELL);
if (!sell_ok)
std::cout << "shit";
debug_break();
REQUIRE(sell_ok);
}
}
@@ -369,12 +369,12 @@ struct ObsWrapper : mdp3::IChannelObserver {
last_print = Timer::now();
std::stringstream str;
mdp3::cme_ts_to_time_str(ev.transact_time.value(), &str);
std::cout << "\rPacket time: " << str.str() << " "
<< (ev.first_packet_sn - previous_packet) << " pkt/sec "
<< "("
std::cout << "Packet time: " << str.str() << " ("
<< (ev.first_packet_sn - previous_packet) << " pkt/sec, "
<< (ev.first_packet_send_time.value() - previous_send_ts) /
1000000000
<< "x realtime)" ;
<< "x speedup) \r";
std::cout.flush();
previous_packet = ev.first_packet_sn;
previous_send_ts = ev.first_packet_send_time.value();
}
@@ -705,6 +705,17 @@ find_icebergs(std::string const &pcap) {
size_t actual_quantity = change.order.quantity.value();
if (expected_quantity != actual_quantity)
order_data[ok].is_iceberg = true;
} else {
bts_assert(change.change == OrderChange::removed);
// The order was totally filled, but was the fill size
// consistent with what we thought was the remaining
// order size? If not then the order must have been an
// iceberg.
size_t expected_quantity =
change.previous_order.quantity.value();
auto actual_quantity = fills;
if (expected_quantity != actual_quantity)
order_data[ok].is_iceberg = true;
}
}
}