Add tests for SecurityIDRecorder += SpreadResolver
This commit is contained in:
1 parent
2fc4c8bd60
commit
4b3cdf9ca4
8 files changed
+127
-35
No files matched your search
@@ -7,7 +7,7 @@
|
||||
#include "lib/logging/logging.h"
|
||||
#include "lib/md/SecurityDB.h"
|
||||
#include <boost/filesystem.hpp>
|
||||
#include <tbb/queuing_mutex.h> // for reservation
|
||||
#include <tbb/concurrent_queue.h>
|
||||
|
||||
#include "lib/config/protos/MarketData.pb.h"
|
||||
#include "lib/config/protos/system.pb.h"
|
||||
@@ -22,6 +22,8 @@ using std::size_t;
|
||||
namespace bts {
|
||||
namespace md {
|
||||
|
||||
uint64_t exchange_id(const api::Security& security);
|
||||
|
||||
struct SecurityIDEntry {
|
||||
api::Security security;
|
||||
SecurityIDRecorder::Callback callback;
|
||||
@@ -58,17 +60,16 @@ struct SecurityIDRecorderState {
|
||||
security_key insert(const std::string& descriptor, SecurityType);
|
||||
};
|
||||
|
||||
SecurityIDRecorder::SecurityIDRecorder():
|
||||
SecurityIDRecorder::SecurityIDRecorder(SecurityIDRecorderParams params):
|
||||
_self(new SecurityIDRecorderState())
|
||||
{
|
||||
if (params.db) {
|
||||
_self->db = std::move(params.db);
|
||||
}
|
||||
init();
|
||||
}
|
||||
|
||||
SecurityIDRecorder::SecurityIDRecorder(utils::Dispatcher& dispatcher):
|
||||
_self(new SecurityIDRecorderState())
|
||||
{
|
||||
init();
|
||||
_self->run(dispatcher);
|
||||
if (params.dispatcher) {
|
||||
_self->run(*params.dispatcher);
|
||||
}
|
||||
}
|
||||
|
||||
SecurityIDRecorder::~SecurityIDRecorder() { }
|
||||
@@ -81,13 +82,16 @@ void SecurityIDRecorder::init() {
|
||||
boost::filesystem::create_directories(db_dir);
|
||||
}
|
||||
|
||||
_self->db = open_db(db_path.c_str());
|
||||
if (!_self->db) {
|
||||
_self->db = sqlite::open_db(db_path.c_str());
|
||||
bts_assert(_self->db);
|
||||
}
|
||||
|
||||
// Set a long busy timeout
|
||||
sqlite3_busy_timeout(_self->db.get(), 9999);
|
||||
sqlite3_exec_nofail(_self->db, "PRAGMA journal_mode=WAL");
|
||||
sqlite3_exec_nofail(_self->db, "PRAGMA syncronous=0");
|
||||
sqlite3_exec_nofail(_self->db, "CREATE TABLE IF NOT EXISTS security_keys (descriptor STRING NOT NULL PRIMARY KEY, type int);");
|
||||
sqlite::exec_nofail(_self->db, "PRAGMA journal_mode=WAL");
|
||||
sqlite::exec_nofail(_self->db, "PRAGMA syncronous=0");
|
||||
sqlite::exec_nofail(_self->db, "CREATE TABLE IF NOT EXISTS security_keys (descriptor STRING NOT NULL PRIMARY KEY, type int);");
|
||||
}
|
||||
|
||||
void SecurityIDRecorder::ensure_key(MDProducerHandle handle) {
|
||||
@@ -196,7 +200,7 @@ void SecurityIDRecorderState::work_db() {
|
||||
|
||||
|
||||
if (!new_entries.empty()) {
|
||||
sqlite3_exec_nofail(db.get(), "BEGIN TRANSACTION");
|
||||
sqlite::exec_nofail(db.get(), "BEGIN TRANSACTION");
|
||||
|
||||
for (size_t x = 0; x < new_entries.size(); ++x) {
|
||||
Security& security = new_entries[x]->security;
|
||||
@@ -205,7 +209,7 @@ void SecurityIDRecorderState::work_db() {
|
||||
security.set_id(key);
|
||||
}
|
||||
|
||||
sqlite3_exec_nofail(db.get(), "COMMIT TRANSACTION");
|
||||
sqlite::exec_nofail(db.get(), "COMMIT TRANSACTION");
|
||||
}
|
||||
|
||||
// record the securities
|
||||
@@ -231,10 +235,10 @@ void SecurityIDRecorderState::work_db() {
|
||||
|
||||
void SecurityIDRecorder::bulk_insert(const std::vector<const api::Security*>& data, const char* prefix) {
|
||||
if (data.size() == 0) return;
|
||||
sqlite3_exec_nofail(_self->db.get(), "BEGIN TRANSACTION");
|
||||
sqlite::exec_nofail(_self->db.get(), "BEGIN TRANSACTION");
|
||||
std::string descriptor;
|
||||
{
|
||||
auto insert_stmt = prepare_query(_self->db, "INSERT INTO security_keys (rowid, descriptor, type) values (?, ?, ?)");
|
||||
auto insert_stmt = sqlite::prepare_query(_self->db, "INSERT INTO security_keys (rowid, descriptor, type) values (?, ?, ?)");
|
||||
for (const api::Security* sec : data) {
|
||||
descriptor = prefix + security_descriptor(*sec);
|
||||
auto type = securitytype_of(*sec);
|
||||
@@ -246,11 +250,11 @@ void SecurityIDRecorder::bulk_insert(const std::vector<const api::Security*>& da
|
||||
sqlite3_reset(insert_stmt.get());
|
||||
}
|
||||
}
|
||||
sqlite3_exec_nofail(_self->db.get(), "COMMIT TRANSACTION");
|
||||
sqlite::exec_nofail(_self->db.get(), "COMMIT TRANSACTION");
|
||||
}
|
||||
|
||||
security_key SecurityIDRecorderState::search(const std::string& descriptor) {
|
||||
auto select_stmt = prepare_query(db, "SELECT rowid from security_keys where descriptor=? LIMIT 1;");
|
||||
auto select_stmt = sqlite::prepare_query(db, "SELECT rowid from security_keys where descriptor=? LIMIT 1;");
|
||||
sqlite3_bind_text(select_stmt.get(), 1, descriptor.c_str(), descriptor.size(), SQLITE_STATIC);
|
||||
int result = sqlite3_step(select_stmt.get());
|
||||
if (result == SQLITE_DONE || result == SQLITE_ROW) {
|
||||
@@ -260,7 +264,7 @@ security_key SecurityIDRecorderState::search(const std::string& descriptor) {
|
||||
}
|
||||
|
||||
security_key SecurityIDRecorderState::insert(const std::string& descriptor, SecurityType type) {
|
||||
auto insert_stmt = prepare_query(db, "INSERT INTO security_keys (descriptor, type) values (?, ?)");
|
||||
auto insert_stmt = sqlite::prepare_query(db, "INSERT INTO security_keys (descriptor, type) values (?, ?)");
|
||||
sqlite3_bind_text(insert_stmt.get(), 1, descriptor.c_str(), descriptor.size(), SQLITE_STATIC);
|
||||
sqlite3_bind_int(insert_stmt.get(), 2, (int)type);
|
||||
int result = sqlite3_step(insert_stmt.get());
|
||||
|
||||
@@ -3,11 +3,13 @@
|
||||
#include "common.h"
|
||||
#include "MDProducerHandle.h"
|
||||
|
||||
#include "lib/utils/Sqlite_utils.h"
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
#include <atomic>
|
||||
#include <vector>
|
||||
#include <tbb/concurrent_queue.h>
|
||||
|
||||
typedef struct sqlite3 sqlite3;
|
||||
|
||||
namespace bts {
|
||||
namespace md {
|
||||
@@ -17,13 +19,19 @@ namespace md {
|
||||
* Provides a callback when ids are safely persisted.
|
||||
*/
|
||||
|
||||
struct SecurityIDRecorderParams {
|
||||
SecurityIDRecorderParams() : dispatcher(), db(sqlite::null_db()) {}
|
||||
SecurityIDRecorderParams(utils::Dispatcher& d): dispatcher(&d), db(sqlite::null_db()) {}
|
||||
SecurityIDRecorderParams(sqlite3_ptr& db): dispatcher(), db(std::move(db)) {}
|
||||
utils::Dispatcher* dispatcher;
|
||||
sqlite3_ptr db;
|
||||
};
|
||||
struct SecurityIDRecorderState;
|
||||
|
||||
class SecurityIDRecorder
|
||||
{
|
||||
public:
|
||||
SecurityIDRecorder();
|
||||
// Automatically call do_work on this dispatcher
|
||||
SecurityIDRecorder(utils::Dispatcher&);
|
||||
SecurityIDRecorder(SecurityIDRecorderParams);
|
||||
~SecurityIDRecorder();
|
||||
typedef std::function<void(const api::Security&)> Callback;
|
||||
|
||||
@@ -32,7 +40,7 @@ public:
|
||||
// Slow (copies security) - for use by services
|
||||
void assign_key(const api::Security& security, const Callback& key_assigned);
|
||||
|
||||
// Someone must call work periodically to do lookups and flush the reserved ids to disk.
|
||||
// If a dispatcher is not provided you must call work periodically to do lookups and flush the reserved ids to disk.
|
||||
void do_work();
|
||||
|
||||
void bulk_insert(const std::vector<const api::Security*>&, const char* prefix);
|
||||
|
||||
@@ -250,6 +250,16 @@ size_t num_legs(const Security& sec) {
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
uint64_t exchange_id(const Security& security) {
|
||||
switch(security.ExchangeData_case()) {
|
||||
case Security::ExchangeDataCase::kCboe: return security.cboe().exchange_id();
|
||||
case Security::ExchangeDataCase::kGlbx: return security.glbx().exchange_id();
|
||||
case Security::ExchangeDataCase::kCfe: return security.cfe().exchange_id();
|
||||
case Security::ExchangeDataCase::kIce: bts_assert(false);
|
||||
case Security::ExchangeDataCase::EXCHANGEDATA_NOT_SET: bts_assert(false);
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
void convert_security(const Security& src, FixedSecurity& dest, bool set_security_key) {
|
||||
dest.key = set_security_key ? src.id() : invalid_security_key;
|
||||
@@ -272,7 +282,9 @@ void convert_security(const Security& src, FixedSecurity& dest, bool set_securit
|
||||
case Security::ExchangeDataCase::kCfe:
|
||||
dest.cboe_class_key = src.cfe().class_key();
|
||||
break;
|
||||
default:
|
||||
case Security::ExchangeDataCase::kIce:
|
||||
bts_assert_d(false);
|
||||
case Security::ExchangeDataCase::EXCHANGEDATA_NOT_SET:
|
||||
break;
|
||||
}
|
||||
dest.price_break = src.contract_spec().price_break() / src.contract_spec().price_multiplier();
|
||||
|
||||
@@ -180,6 +180,8 @@ void convert_security(const Security& src, FixedSecurity& dest, bool set_securit
|
||||
SecurityType securitytype_of_contractcase(Security::ContractCase cc);
|
||||
SecurityType securitytype_of(const Security&);
|
||||
size_t num_legs(const Security&);
|
||||
uint64_t exchange_id(const Security& security);
|
||||
|
||||
|
||||
// Stack allocates a security of the given type
|
||||
template <class C>
|
||||
|
||||
@@ -25,7 +25,7 @@ typedef SyntheticSecurityServiceBase::MetadataStatefulPublisher MetadataPublishe
|
||||
|
||||
struct SyntheticSecurityService::Pimpl {
|
||||
Pimpl(const MetadataStatefulPublisher& fn) :
|
||||
recorder(utils::Dispatcher::background()),
|
||||
recorder({utils::Dispatcher::background()}),
|
||||
synthetics(), on_security(),
|
||||
meta(), on_metadata(), metadata_publisher(fn)
|
||||
{
|
||||
|
||||
@@ -130,7 +130,7 @@ void replay_recording(istream& in, IRelayWriter& writer, int32_t naggle_ms) {
|
||||
CoconutReplayHeader header;
|
||||
size_t count = 0;
|
||||
#ifndef _WIN32
|
||||
md::SecurityIDRecorder idrec;
|
||||
md::SecurityIDRecorder idrec {{}};
|
||||
#endif
|
||||
if (readbuf(in, &header) && header.magic() == magic && header.version() == 3) {
|
||||
co2::MDReply message;
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
#include "lib/metrics/metrics.h"
|
||||
|
||||
#include "lib/config/protos/MarketData.pb.h"
|
||||
#include "lib/utils/Sqlite_utils.h"
|
||||
|
||||
#include <vector>
|
||||
#include <string>
|
||||
@@ -27,7 +28,6 @@
|
||||
#include <thread>
|
||||
#include <iterator>
|
||||
|
||||
|
||||
using namespace std;
|
||||
|
||||
namespace bts {
|
||||
@@ -151,6 +151,7 @@ void assign_ids(vector<MDProducerHandle>::iterator begin, vector<MDProducerHandl
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class Co2Tests
|
||||
{
|
||||
public:
|
||||
@@ -171,12 +172,20 @@ public:
|
||||
}
|
||||
|
||||
#ifndef WIN32
|
||||
void test_security_id_recorder() {
|
||||
void test_security_id_recorder(const std::string& db_name) {
|
||||
// Verify callbacks happen after work complete
|
||||
SecurityIDRecorder idsvc, idsvc2;
|
||||
auto db1 = sqlite::open_db(db_name.c_str());
|
||||
REQUIRE(db1);
|
||||
auto db2 = sqlite::open_db(db_name.c_str());
|
||||
REQUIRE(db2);
|
||||
|
||||
SecurityIDRecorder idsvc{{db1}}, idsvc2{{db2}};
|
||||
security_key result = invalid_security_key;
|
||||
std::string desc = "FOOX 1";
|
||||
Security s1 = mkoption("FOOX", invalid_security_key, 2018, 1, 20, 1850);
|
||||
Security f1 = mkfuture("FOO", invalid_security_key, 2018, 1, 20);
|
||||
idsvc.assign_key(f1, [](const api::Security& s) {
|
||||
REQUIRE(s.id() != invalid_security_key);
|
||||
});
|
||||
idsvc.assign_key(s1, [&result](const api::Security& s) {
|
||||
result = s.id();
|
||||
});
|
||||
@@ -188,7 +197,60 @@ public:
|
||||
result2 = s.id();
|
||||
});
|
||||
idsvc2.do_work();
|
||||
REQUIRE(result == result2);
|
||||
|
||||
if (db_name == ":memory:") {
|
||||
// This is a meta-test to make sure we don't accidently generate the same key sequence
|
||||
// when writing to different databases.
|
||||
REQUIRE(result != result2);
|
||||
}
|
||||
else {
|
||||
REQUIRE(result == result2);
|
||||
}
|
||||
}
|
||||
|
||||
void test_security_id_recorder_spreads() {
|
||||
// Test that security id recorder can resolve spread legs
|
||||
|
||||
// Set up "shared memory"
|
||||
auto cold_header = std::make_shared<ColdHeader>();
|
||||
auto hot_header = std::make_shared<HotHeader>();
|
||||
|
||||
// Create a spread with 2 legs
|
||||
auto f1 = mkfuture("FO1", invalid_security_key, 2018, 1, 17);
|
||||
f1.mutable_glbx()->set_exchange_id(100);
|
||||
REQUIRE(f1.id() == invalid_security_key);
|
||||
auto f1handle = std::make_unique<Co2TestInstrument> (f1, cold_header, hot_header);
|
||||
|
||||
auto f2 = mkfuture("FO2", invalid_security_key, 2018, 1, 20);
|
||||
f2.mutable_glbx()->set_exchange_id(200);
|
||||
auto f2handle = std::make_unique<Co2TestInstrument> (f2, cold_header, hot_header);
|
||||
|
||||
auto s1 = mkspread("FOO", invalid_security_key, { f1, f2 });
|
||||
auto s1handle = std::make_unique<Co2TestInstrument> (s1, cold_header, hot_header);
|
||||
|
||||
// Add spread to SecurityIDRecorder and work it
|
||||
auto db = sqlite::open_db(":memory:");
|
||||
REQUIRE(db);
|
||||
SecurityIDRecorder idsvc{{db}};
|
||||
auto workwork = [&]() { idsvc.do_work(); idsvc.do_work(); };
|
||||
// auto workwork = [&]() { for(int i=5;--i;) { idsvc.do_work(); } };
|
||||
idsvc.ensure_key(s1handle->producer());
|
||||
workwork();
|
||||
|
||||
// Expect no key
|
||||
REQUIRE(s1handle->consumer().key() == invalid_security_key);
|
||||
|
||||
// Add leg spreads
|
||||
idsvc.ensure_key(f1handle->producer());
|
||||
idsvc.ensure_key(f2handle->producer());
|
||||
workwork();
|
||||
|
||||
// Expect legs to have keys
|
||||
REQUIRE(f1handle->consumer().key() != invalid_security_key);
|
||||
REQUIRE(f2handle->consumer().key() != invalid_security_key);
|
||||
|
||||
// Expect spread to have keys
|
||||
REQUIRE(s1handle->consumer().key() != invalid_security_key);
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -1002,7 +1064,11 @@ TEST_CASE("CO2Tests","[md]") {
|
||||
|
||||
#ifndef WIN32
|
||||
SECTION("test_security_id_recorder") {
|
||||
co2_tests->test_security_id_recorder();
|
||||
co2_tests->test_security_id_recorder(":memory:");
|
||||
co2_tests->test_security_id_recorder("file::memory:?cache=shared");
|
||||
}
|
||||
SECTION("test_security_id_recorder_spreads") {
|
||||
co2_tests->test_security_id_recorder_spreads();
|
||||
}
|
||||
#endif
|
||||
SECTION("test_price_size") {
|
||||
|
||||
@@ -43,7 +43,7 @@ void run_cx_server(UserSymbol* psyms, size_t symcount)
|
||||
|
||||
CXMain::CXMain() :
|
||||
_producer(co2::PLATFORM_CELOXICA),
|
||||
_security_recorder(),
|
||||
_security_recorder({}),
|
||||
_settlement_recorder(co2::PLATFORM_CELOXICA),
|
||||
_listeners()
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user