spark-325 Synthetic Book Merge. Does not yet compute weighted average for partial sizes

This commit is contained in:
Eric Harding committed 2018-03-02 12:37:40 -05:00
1 parent ca3b91afb5
commit 998fb33b59
8 files changed
+209 -21

No files matched your search

+1 -1
View File
@@ -111,7 +111,7 @@
"type": "cppdbg",
"request": "launch",
"program": "/home/bts/spark/build/ninja_debug/src/server/sparkd/sparkd",
"args": ["test", "-a", "Co2RelayTests"],
"args": ["test", "-a", "SyntheticBookMerge"],
"stopAtEntry": false,
"cwd": "/home/bts/spark/build/ninja_debug/src/server/sparkd/",
"environment": [],
+5 -4
View File
@@ -46,6 +46,7 @@ namespace bts.api {
internal SecurityType security_type() {
switch (ContractCase) {
case ContractOneofCase.Future: return SecurityType.Future;
case ContractOneofCase.Option: return SecurityType.Option;
case ContractOneofCase.Spread: return SecurityType.Spread;
case ContractOneofCase.Synthetic: return SecurityType.Synthetic;
default: return SecurityType.Unset;
@@ -57,16 +58,16 @@ namespace bts.api {
/// exchange securities
/// </summary>
/// <returns></returns>
internal IEnumerable<Tuple<uint, Side, double>> get_base_exchange_securities () {
List<Tuple<uint, Side, double>> rv = new List<Tuple<uint,Side,double>> ();
internal IEnumerable<(security_key, Side, double mult)> get_base_exchange_securities () {
List<(uint, Side, double)> rv = new List<(uint,Side,double)> ();
get_base_exchange_securities (this, Side.Buy, 1.0, ref rv);
return rv;
}
static void get_base_exchange_securities (SecurityImmutable sec, Side side, double mult, ref List<Tuple<uint, Side, double>> legs) {
static void get_base_exchange_securities (SecurityImmutable sec, Side side, double mult, ref List<(security_key, Side, double)> legs) {
Func<Side, Side, Side> resolve_side = (s1, s2) => ((s1 == Side.Buy ? 1 : -1) * (s2 == Side.Buy ? 1 : -1)) == 1 ? Side.Buy : Side.Sell;
if (sec.ContractCase == ContractOneofCase.Future)
legs.Add (Tuple.Create (sec.Id, side, mult));
legs.Add ((sec.Id, side, mult));
else if (sec.ContractCase == ContractOneofCase.Spread)
foreach (ContractLegImmutable leg in sec.Spread.Legs)
get_base_exchange_securities (leg.Security, resolve_side (side, leg.Side), leg.Quantity * mult, ref legs);
+1 -1
View File
@@ -1 +1 @@
set(sources api/apicore.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/services/AccountService.proto;api/services/ApiUnitTestService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/ShellDiagnosticService.proto;api/services/SyntheticSecurityService.proto;api/services/UserService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/Executor.conf;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/edge.conf;config/defaults/expiration_display_offset.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/testkeys.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/environment.proto;config/protos/expiration_display_offset.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/quickbus_cfg.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPriceLantencyTester.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test_relay_client.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/windows_test_driver.cpp;utils/Dispatcher.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp)
set(sources api/apicore.proto;api/apioptions.proto;api/core/ApiClientInterop.cpp;api/core/ApiServerInterop.cpp;api/public/ApiServer.cpp;api/public/ClientChannel.cpp;api/public/QuickbusClientChannel.cpp;api/public/ServerChannel.cpp;api/public/ServiceContext.cpp;api/services/AccountService.proto;api/services/ApiUnitTestService.proto;api/services/ErrorReportService.proto;api/services/HelloService.proto;api/services/MarketDataRelay.proto;api/services/OrderService.proto;api/services/OutOfProcessURLListener.proto;api/services/PersistenceService.proto;api/services/ShellDiagnosticService.proto;api/services/SyntheticSecurityService.proto;api/services/UserService.proto;api/test/ApiTests.cpp;api/test/HelloServiceServer.cpp;config/ConfigCheck.cpp;config/ConfigDownloader.cpp;config/ConfigErrorCollector.cpp;config/ConfigLog.cpp;config/ConfigPropertyEvaluator.cpp;config/ConfigValueQueue.cpp;config/ConfigurationException.cpp;config/ConfigurationState.cpp;config/PreprocessInputStream.cpp;config/config.cpp;config/defaults/Executor.conf;config/defaults/MarketData.conf;config/defaults/Ports.conf;config/defaults/Symbols.conf;config/defaults/edge.conf;config/defaults/expiration_display_offset.conf;config/defaults/jetfix.conf;config/defaults/logging.conf;config/defaults/testkeys.conf;config/defaults/toggles.conf;config/extensions/extensions.proto;config/invalid/invalid.proto;config/protos/MarketData.proto;config/protos/Ports.proto;config/protos/Symbols.proto;config/protos/UI.proto;config/protos/environment.proto;config/protos/expiration_display_offset.proto;config/protos/jetfix.proto;config/protos/logging.proto;config/protos/quickbus_cfg.proto;config/protos/system.proto;config/protos/testkeys.proto;config/protos/toggles.proto;config/tests/HTTPServer.cpp;config/tests/config_ref_tests.cpp;config/tests/config_tests.cpp;config/text_format/text_format.cpp;domain_model/Security.cpp;domain_model/Security.proto;expression/BooleanGrammar.cpp;expression/ExpressionEvaluator.cpp;expression/ParserExceptions.cpp;expression/Value.cpp;expression/tests/expression_parser_tests.cpp;logging/LoggerHub.cpp;logging/Sink.cpp;logging/logging.cpp;logging/tests/logging_tests.cpp;md/Diag/MDPriceLantencyTester.cpp;md/Diag/ShmDumper.cpp;md/MDConsumer.cpp;md/MDConsumerHandle.cpp;md/MDConsumerInterop.cpp;md/MDProducer.cpp;md/MDProducerHandle.cpp;md/MDRecoverableConsumer.cpp;md/MemorySegmentUtils.cpp;md/PlatformLoader.cpp;md/PriceSize.cpp;md/SecurityDB.cpp;md/SecurityIDRecorder.cpp;md/SettlementRecorder.cpp;md/Shm/FixedSecurity.cpp;md/SyntheticSecurityService.cpp;md/common.cpp;md/protos/co2.proto;md/relay/QueryServer.cpp;md/relay/RelayClient.cpp;md/relay/RelayRecorder.cpp;md/test/Co2TestInstrument.cpp;md/test/md2_tests.cpp;md/test/synthetic_book_merge_tests.cpp;md/test_relay_client.cpp;os/os.cpp;os/tests/os_tests.cpp;quickbus/core/LocalNode.cpp;quickbus/core/Quickbus.cpp;quickbus/core/SubscriptionID.cpp;quickbus/core/TableID.cpp;quickbus/message_model/Update.cpp;quickbus/message_model/merge_updates.cpp;quickbus/network/QBConnection.cpp;quickbus/network/QBConnectionManager.cpp;quickbus/network/endpoint_description.cpp;quickbus/network/quickbus_msgs.proto;quickbus/pubsub/TablePublisher.cpp;quickbus/pubsub/TableSubscriber.cpp;quickbus/pubsub/tests/qb_pubsub_tests.cpp;quickbus/routing/INode.cpp;quickbus/routing/NodeID.cpp;quickbus/routing/Router.cpp;quickbus/rpc/RPCClient.cpp;quickbus/rpc/RPCServer.cpp;quickbus/rpc/RPCSessionID.cpp;quickbus/rpc/rpc_utils.cpp;quickbus/rpc/tests/qb_rpc_tests.cpp;quickbus/tests/quickbus_tests.cpp;sparkproto/SparkProtos.proto;testing/windows_test_driver.cpp;utils/Dispatcher.cpp;utils/HTTPStream.cpp;utils/PBUtils.cpp;utils/Puid.cpp;utils/SharedMemory.cpp;utils/Sqlite_utils.cpp;utils/Uuid.cpp;utils/datetime.cpp;utils/download.cpp;utils/file_utils.cpp;utils/init_spark_lib.cpp;utils/is_unittest_environment.cpp)
+95 -7
View File
@@ -16,6 +16,9 @@ MDConsumerHandle::MDConsumerHandle(const std::shared_ptr<SyntheticStrategy>& def
memset(this, 0, sizeof(MDConsumerHandle));
synthetic.def = def;
_type = HANDLETYPE_SYNTHETIC;
for(size_t x = 0; x < def->legs.size(); ++x) {
bts_assert(def->legs[x].key() == def->security.synthetic().legs(x).security().id());
}
}
MDConsumerHandle::MDConsumerHandle(const MDConsumerHandle& other) {
@@ -74,17 +77,102 @@ TimeStamp copy_depth(
return t;
}
/////////////////////////////
// SYNTHETIC DEPTH
/////////////////////////////
struct Level {
double price;
uint16_t size;
void down(uint16_t q) {
bts_assert(q <= size);
size -= q;
}
bool active() const { return size > 0; }
};
struct DepthTraversal {
std::array<Level, COCONUT_MAX_DEPTH> levels;
size_t head;
api::Side side;
double mult;
DepthTraversal(api::Side side, double mult, const PriceSize* depth, size_t depth_count) : levels(), head(0), side(side), mult(mult) {
// levels is zero initialized
for (size_t x = 0; x < depth_count; ++x) {
/// every level has a size of atleast one
/// The sizes are scaled, meaning adjusted to the mult. This simplifies
/// things, however at the cost of rouding off the fractions. So ex if mult is
/// 2 and there is a size of 7, it would be rounded off to 3. The remaining 1
/// is not considered. Also if there are sizes less than mult they are
/// floored to 0 and ignored. A better solution would be to count them and
/// move till their sum exceed the mult
levels[x].price = depth[x].price();
levels[x].size = scale_size(depth[x].size(), mult);
}
}
uint16_t scale_size(uint16_t size, double m) {
double dval = std::ceil( (double)size / m );
return (uint16_t)(dval);
}
bool active() const { return head < levels.size() && levels[head].size > 0; }
double down(uint16_t qty) {
bts_assert(active());
bts_assert(qty <= levels[head].size);
double rv = levels[head].price;
levels[head].down(qty);
if (!levels[head].active()) { ++head; }
return rv;
}
uint16_t normalized_head_size() const { return active()? levels[head].size : 0; }
};
TimeStamp copy_depth(const std::vector<MDConsumerHandle>& legs, size_t count, PriceSize* bid, PriceSize* ask) {
(void)legs;
//PLACEHOLDER
for (size_t x = 0; x < count; ++x) {
bid[x] = PriceSize::from_double<4>(99,99);
ask[x] = PriceSize::from_double<4>(88,88);
void copy_depth(size_t count, api::Side, std::vector<DepthTraversal>& leg_travs, PriceSize* out_price) {
size_t out_index = 0;
while(out_index < count && sane::true_for_all(leg_travs, [](const DepthTraversal& dt) { return dt.active(); })) {
double price_at_this_level = 0.0;
uint16_t min_head = sane::min<uint16_t>(leg_travs, [](const DepthTraversal& dt) { return dt.normalized_head_size(); });
for (auto& dt : leg_travs) {
double leg_price = dt.down(min_head);
price_at_this_level += leg_price * dt.mult * (dt.side == api::SIDE_BUY ? 1.0 : -1.0);
}
out_price[out_index] = PriceSize::from_double<7>(price_at_this_level, min_head);
++out_index;
}
// Zero remaining requested
for (size_t x = out_index; x < count; ++x) {
out_price[x] = PriceSize::NO_MARKET;
}
return TimeStamp::now().shift_days(-1);
}
api::Side flip_side(api::Side s) {
return s == api::SIDE_BUY ? api::SIDE_SELL : api::SIDE_BUY;
}
TimeStamp copy_depth(const SyntheticStrategy& synth, size_t count, PriceSize* bid, PriceSize* ask) {
std::vector<DepthTraversal> bid_leg_travs;
std::vector<DepthTraversal> ask_leg_travs;
bid_leg_travs.reserve(synth.legs.size());
ask_leg_travs.reserve(synth.legs.size());
TimeStamp rv = std::numeric_limits<TimeStamp>::min();
for (size_t x = 0; x < synth.legs.size(); ++x) {
std::array<PriceSize, COCONUT_MAX_DEPTH> bids, asks;
TimeStamp t;
HandleReadResult r = synth.legs[x].get_depth(t, count, &bids[0], &asks[0]);
if (r == HandleReadResult::BUSY || t == TimeStamp::invalid()) return TimeStamp::invalid();
rv = std::max(rv, t);
const auto& leg = synth.security.synthetic().legs(x);
const auto& eff_bid = leg.side() == api::SIDE_BUY ? bids : asks;
const auto& eff_ask = leg.side() == api::SIDE_BUY ? asks : bids;
bid_leg_travs.push_back(DepthTraversal(leg.side(), leg.quantity(), &eff_bid[0], eff_bid.size()));
ask_leg_travs.push_back(DepthTraversal(leg.side(), leg.quantity(), &eff_ask[0], eff_ask.size()));
}
copy_depth(count, api::SIDE_BUY, bid_leg_travs, bid);
copy_depth(count, api::SIDE_SELL, ask_leg_travs, ask);
return rv;
}
/////////////////////////////
// END SYNTHETIC DEPTH
/////////////////////////////
TimeStamp MDConsumerHandle::last_market_update() const {
switch(_type) {
case HANDLETYPE_VOID:
+9 -8
View File
@@ -11,10 +11,16 @@
namespace bts {
namespace md {
class MDConsumerHandle;
struct SyntheticStrategy {
spark::ExecutionStrategy strategy;
api::Security security;
std::vector<MDConsumerHandle> legs;
};
TimeStamp copy_bbo(const TimeStamp&, const SingleWriterPriceSizeStore&, const SingleWriterPriceSizeStore&, PriceSize&, PriceSize& );
TimeStamp copy_depth(const TimeStamp& time, const SingleWriterPriceSizeStore* source, size_t count, PriceSize* bid, PriceSize* ask);
class MDConsumerHandle;
TimeStamp copy_depth(const std::vector<MDConsumerHandle>& legs, size_t count, PriceSize* bid, PriceSize* ask);
TimeStamp copy_depth(const SyntheticStrategy&, size_t count, PriceSize* bid, PriceSize* ask);
enum class HandleReadResult {
NO_CHANGE,
@@ -22,11 +28,6 @@ enum class HandleReadResult {
SUCCESS
};
struct SyntheticStrategy {
spark::ExecutionStrategy strategy;
api::Security security;
std::vector<MDConsumerHandle> legs;
};
// Can be an invalid handle if you request a security which has not been discovered yet.
class BTS_EXPORT MDConsumerHandle
@@ -125,7 +126,7 @@ public:
case HANDLETYPE_SYNTHETIC:
auto t = last_market_update();
if (t > time || t == TimeStamp::invalid()) {
TimeStamp temp = copy_depth(synthetic.def->legs, copydepth, bids, asks);
TimeStamp temp = copy_depth(*synthetic.def, copydepth, bids, asks);
if (temp == TimeStamp::invalid()) { return HandleReadResult::BUSY; }
time = temp;
return HandleReadResult::SUCCESS;
+1
View File
@@ -15,6 +15,7 @@ Co2TestInstrument::Co2TestInstrument(const std::string& symbol, SecurityType typ
{
_cheader->has_depth = enable_depth(symbol, type);
with_security(type, 4, nullptr, [&](FixedSecurity& fsec) {
fsec.exchange = api::VENUE_BTS;
_cold_buffer = new uint8_t[ColdEntry::fixed_size() + fsec.size];
_cold = new (_cold_buffer) ColdEntry(fsec.size);
memcpy(&_cold->security, &fsec, fsec.size);
@@ -0,0 +1,96 @@
#include "catch/catch.hpp"
#include "../MDConsumerHandle.h"
#include "Co2TestInstrument.h"
#include "lib/utils/sane_algorithm.h"
#include <memory>
#include <string>
#include <vector>
#include <set>
#include <algorithm>
using namespace std;
namespace bts {
int synthetic_book_merge_tests = 0;
namespace md {
namespace test {
// std::tuple is hard to read in the debugger
struct PS {
double price;
uint16_t size;
bool operator == (const PS& other) const {
return price == other.price && size == other.size;
}
};
void setup_market(MDProducerHandle producer, const vector<PS>& bids, const vector<PS>& asks) {
auto ts = TimeStamp::now();
for (size_t x = 0; x < bids.size(); ++x) {
const auto& bid = bids[x];
producer.update_bid(ts, PriceSize::from_double<7>(bid.price, bid.size), x);
}
for (size_t x = 0; x < asks.size(); ++x) {
const auto& ask = asks[x];
producer.update_ask(ts, PriceSize::from_double<7>(ask.price, ask.size), x);
}
}
void expect_market(MDConsumerHandle handle, const vector<PS> bids, const vector<PS>& asks) {
array<PriceSize,COCONUT_MAX_DEPTH> raw_bids,raw_asks;
TimeStamp t;
handle.get_depth(t, COCONUT_MAX_DEPTH, raw_bids.data(), raw_asks.data());
vector<PS> src_bids, src_asks;
sane::map(raw_bids, src_bids, [](const PriceSize& ps) -> PS {return {ps.price(), ps.size()};});
sane::map(raw_asks, src_asks, [](const PriceSize& ps) -> PS {return {ps.price(), ps.size()}; });
for (size_t x =0; x < bids.size(); ++x) {
REQUIRE(src_bids[x] == bids[x]);
}
for (size_t x = 0; x < asks.size(); ++x) {
REQUIRE(src_asks[x] == asks[x]);
}
}
TEST_CASE("SyntheticBookMerge", "[md]") {
// Setup
Co2TestInstrument es1("ES", api::SECURITYTYPE_FUTURE), es2("ES", api::SECURITYTYPE_FUTURE);
auto synthetic = make_shared<SyntheticStrategy>();
auto& s = synthetic->security;
auto* ps = s.mutable_synthetic();
s.set_symbol("SYN");
ps->set_name("SYN es1-es2");
auto* l1 = ps->mutable_legs()->Add();{
l1->mutable_security()->CopyFrom(es1.consumer().get_normalized_security());
l1->set_quantity(1);
l1->set_side(api::SIDE_BUY);
auto* l2 = ps->mutable_legs()->Add();
l2->mutable_security()->CopyFrom(es2.consumer().get_normalized_security());
l2->set_quantity(1);
l2->set_side(api::SIDE_SELL);
}
synthetic->legs.push_back(es1.consumer());
synthetic->legs.push_back(es2.consumer());
MDConsumerHandle synthetic_handle{synthetic};
// Simplest case
SECTION("Simple") {
setup_market(es1.producer(), {{2000.0, 10}, {1999.75, 5}, {1999.5,7}}, {{2000.25, 15}, {2000.5, 5}, {2001.0,10}});
setup_market(es2.producer(), {{2003.0, 5}, {2002.75, 15}, {2002.5,12}}, {{2003.25, 12}, {2003.5, 3}, {2003.75,5}});
// Sanity test
expect_market(es1.consumer(), {{2000.0, 10}, {1999.75, 5}, {1999.5,7}}, {{2000.25, 15}, {2000.5, 5}, {2001.0,10}});
expect_market(es2.consumer(), {{2003.0, 5}, {2002.75, 15}, {2002.5,12}}, {{2003.25, 12}, {2003.5, 3}, {2003.75,5}});
// Real test
expect_market(synthetic_handle, {{-3.25,10}, {-3.5, 2}, {-3.75,3}, {-4.25,5}}, {{-2.75,5},{-2.5,10},{-2.25,5},{-1.5,10}});
}
// Test 2 legs, 3 legs
// test 1 empty leg
// Test available size=fraction of leg quantity
}
}}}//namespace
@@ -25,6 +25,7 @@ int run_tests (std::vector<std::string> const&args)
extern int qb_rpc_tests; qb_rpc_tests=0;
extern int qb_pubsub_tests; qb_rpc_tests=0;
extern int md_tests; md_tests=42;
extern int synthetic_book_merge_tests; synthetic_book_merge_tests = 0;
utils::internal::force_unittest_environment();
auto cfg = config::get<config::Logging> ();