Add tests for SecurityIDRecorder += SpreadResolver

This commit is contained in:
Eric Harding committed 2018-11-14 19:58:31 -05:00
1 parent 2fc4c8bd60
commit 4b3cdf9ca4
8 files changed
+127 -35

No files matched your search

+24 -20
View File
@@ -7,7 +7,7 @@
#include "lib/logging/logging.h"
#include "lib/md/SecurityDB.h"
#include <boost/filesystem.hpp>
#include <tbb/queuing_mutex.h> // for reservation
#include <tbb/concurrent_queue.h>
#include "lib/config/protos/MarketData.pb.h"
#include "lib/config/protos/system.pb.h"
@@ -22,6 +22,8 @@ using std::size_t;
namespace bts {
namespace md {
uint64_t exchange_id(const api::Security& security);
struct SecurityIDEntry {
api::Security security;
SecurityIDRecorder::Callback callback;
@@ -58,17 +60,16 @@ struct SecurityIDRecorderState {
security_key insert(const std::string& descriptor, SecurityType);
};
SecurityIDRecorder::SecurityIDRecorder():
SecurityIDRecorder::SecurityIDRecorder(SecurityIDRecorderParams params):
_self(new SecurityIDRecorderState())
{
if (params.db) {
_self->db = std::move(params.db);
}
init();
}
SecurityIDRecorder::SecurityIDRecorder(utils::Dispatcher& dispatcher):
_self(new SecurityIDRecorderState())
{
init();
_self->run(dispatcher);
if (params.dispatcher) {
_self->run(*params.dispatcher);
}
}
SecurityIDRecorder::~SecurityIDRecorder() { }
@@ -81,13 +82,16 @@ void SecurityIDRecorder::init() {
boost::filesystem::create_directories(db_dir);
}
_self->db = open_db(db_path.c_str());
if (!_self->db) {
_self->db = sqlite::open_db(db_path.c_str());
bts_assert(_self->db);
}
// Set a long busy timeout
sqlite3_busy_timeout(_self->db.get(), 9999);
sqlite3_exec_nofail(_self->db, "PRAGMA journal_mode=WAL");
sqlite3_exec_nofail(_self->db, "PRAGMA syncronous=0");
sqlite3_exec_nofail(_self->db, "CREATE TABLE IF NOT EXISTS security_keys (descriptor STRING NOT NULL PRIMARY KEY, type int);");
sqlite::exec_nofail(_self->db, "PRAGMA journal_mode=WAL");
sqlite::exec_nofail(_self->db, "PRAGMA syncronous=0");
sqlite::exec_nofail(_self->db, "CREATE TABLE IF NOT EXISTS security_keys (descriptor STRING NOT NULL PRIMARY KEY, type int);");
}
void SecurityIDRecorder::ensure_key(MDProducerHandle handle) {
@@ -196,7 +200,7 @@ void SecurityIDRecorderState::work_db() {
if (!new_entries.empty()) {
sqlite3_exec_nofail(db.get(), "BEGIN TRANSACTION");
sqlite::exec_nofail(db.get(), "BEGIN TRANSACTION");
for (size_t x = 0; x < new_entries.size(); ++x) {
Security& security = new_entries[x]->security;
@@ -205,7 +209,7 @@ void SecurityIDRecorderState::work_db() {
security.set_id(key);
}
sqlite3_exec_nofail(db.get(), "COMMIT TRANSACTION");
sqlite::exec_nofail(db.get(), "COMMIT TRANSACTION");
}
// record the securities
@@ -231,10 +235,10 @@ void SecurityIDRecorderState::work_db() {
void SecurityIDRecorder::bulk_insert(const std::vector<const api::Security*>& data, const char* prefix) {
if (data.size() == 0) return;
sqlite3_exec_nofail(_self->db.get(), "BEGIN TRANSACTION");
sqlite::exec_nofail(_self->db.get(), "BEGIN TRANSACTION");
std::string descriptor;
{
auto insert_stmt = prepare_query(_self->db, "INSERT INTO security_keys (rowid, descriptor, type) values (?, ?, ?)");
auto insert_stmt = sqlite::prepare_query(_self->db, "INSERT INTO security_keys (rowid, descriptor, type) values (?, ?, ?)");
for (const api::Security* sec : data) {
descriptor = prefix + security_descriptor(*sec);
auto type = securitytype_of(*sec);
@@ -246,11 +250,11 @@ void SecurityIDRecorder::bulk_insert(const std::vector<const api::Security*>& da
sqlite3_reset(insert_stmt.get());
}
}
sqlite3_exec_nofail(_self->db.get(), "COMMIT TRANSACTION");
sqlite::exec_nofail(_self->db.get(), "COMMIT TRANSACTION");
}
security_key SecurityIDRecorderState::search(const std::string& descriptor) {
auto select_stmt = prepare_query(db, "SELECT rowid from security_keys where descriptor=? LIMIT 1;");
auto select_stmt = sqlite::prepare_query(db, "SELECT rowid from security_keys where descriptor=? LIMIT 1;");
sqlite3_bind_text(select_stmt.get(), 1, descriptor.c_str(), descriptor.size(), SQLITE_STATIC);
int result = sqlite3_step(select_stmt.get());
if (result == SQLITE_DONE || result == SQLITE_ROW) {
@@ -260,7 +264,7 @@ security_key SecurityIDRecorderState::search(const std::string& descriptor) {
}
security_key SecurityIDRecorderState::insert(const std::string& descriptor, SecurityType type) {
auto insert_stmt = prepare_query(db, "INSERT INTO security_keys (descriptor, type) values (?, ?)");
auto insert_stmt = sqlite::prepare_query(db, "INSERT INTO security_keys (descriptor, type) values (?, ?)");
sqlite3_bind_text(insert_stmt.get(), 1, descriptor.c_str(), descriptor.size(), SQLITE_STATIC);
sqlite3_bind_int(insert_stmt.get(), 2, (int)type);
int result = sqlite3_step(insert_stmt.get());
+13 -5
View File
@@ -3,11 +3,13 @@
#include "common.h"
#include "MDProducerHandle.h"
#include "lib/utils/Sqlite_utils.h"
#include <functional>
#include <memory>
#include <atomic>
#include <vector>
#include <tbb/concurrent_queue.h>
typedef struct sqlite3 sqlite3;
namespace bts {
namespace md {
@@ -17,13 +19,19 @@ namespace md {
* Provides a callback when ids are safely persisted.
*/
struct SecurityIDRecorderParams {
SecurityIDRecorderParams() : dispatcher(), db(sqlite::null_db()) {}
SecurityIDRecorderParams(utils::Dispatcher& d): dispatcher(&d), db(sqlite::null_db()) {}
SecurityIDRecorderParams(sqlite3_ptr& db): dispatcher(), db(std::move(db)) {}
utils::Dispatcher* dispatcher;
sqlite3_ptr db;
};
struct SecurityIDRecorderState;
class SecurityIDRecorder
{
public:
SecurityIDRecorder();
// Automatically call do_work on this dispatcher
SecurityIDRecorder(utils::Dispatcher&);
SecurityIDRecorder(SecurityIDRecorderParams);
~SecurityIDRecorder();
typedef std::function<void(const api::Security&)> Callback;
@@ -32,7 +40,7 @@ public:
// Slow (copies security) - for use by services
void assign_key(const api::Security& security, const Callback& key_assigned);
// Someone must call work periodically to do lookups and flush the reserved ids to disk.
// If a dispatcher is not provided you must call work periodically to do lookups and flush the reserved ids to disk.
void do_work();
void bulk_insert(const std::vector<const api::Security*>&, const char* prefix);
+13 -1
View File
@@ -250,6 +250,16 @@ size_t num_legs(const Security& sec) {
}
return 0;
}
uint64_t exchange_id(const Security& security) {
switch(security.ExchangeData_case()) {
case Security::ExchangeDataCase::kCboe: return security.cboe().exchange_id();
case Security::ExchangeDataCase::kGlbx: return security.glbx().exchange_id();
case Security::ExchangeDataCase::kCfe: return security.cfe().exchange_id();
case Security::ExchangeDataCase::kIce: bts_assert(false);
case Security::ExchangeDataCase::EXCHANGEDATA_NOT_SET: bts_assert(false);
}
return -1;
}
void convert_security(const Security& src, FixedSecurity& dest, bool set_security_key) {
dest.key = set_security_key ? src.id() : invalid_security_key;
@@ -272,7 +282,9 @@ void convert_security(const Security& src, FixedSecurity& dest, bool set_securit
case Security::ExchangeDataCase::kCfe:
dest.cboe_class_key = src.cfe().class_key();
break;
default:
case Security::ExchangeDataCase::kIce:
bts_assert_d(false);
case Security::ExchangeDataCase::EXCHANGEDATA_NOT_SET:
break;
}
dest.price_break = src.contract_spec().price_break() / src.contract_spec().price_multiplier();
+2
View File
@@ -180,6 +180,8 @@ void convert_security(const Security& src, FixedSecurity& dest, bool set_securit
SecurityType securitytype_of_contractcase(Security::ContractCase cc);
SecurityType securitytype_of(const Security&);
size_t num_legs(const Security&);
uint64_t exchange_id(const Security& security);
// Stack allocates a security of the given type
template <class C>
+1 -1
View File
@@ -25,7 +25,7 @@ typedef SyntheticSecurityServiceBase::MetadataStatefulPublisher MetadataPublishe
struct SyntheticSecurityService::Pimpl {
Pimpl(const MetadataStatefulPublisher& fn) :
recorder(utils::Dispatcher::background()),
recorder({utils::Dispatcher::background()}),
synthetics(), on_security(),
meta(), on_metadata(), metadata_publisher(fn)
{
+1 -1
View File
@@ -130,7 +130,7 @@ void replay_recording(istream& in, IRelayWriter& writer, int32_t naggle_ms) {
CoconutReplayHeader header;
size_t count = 0;
#ifndef _WIN32
md::SecurityIDRecorder idrec;
md::SecurityIDRecorder idrec {{}};
#endif
if (readbuf(in, &header) && header.magic() == magic && header.version() == 3) {
co2::MDReply message;
+72 -6
View File
@@ -13,6 +13,7 @@
#include "lib/metrics/metrics.h"
#include "lib/config/protos/MarketData.pb.h"
#include "lib/utils/Sqlite_utils.h"
#include <vector>
#include <string>
@@ -27,7 +28,6 @@
#include <thread>
#include <iterator>
using namespace std;
namespace bts {
@@ -151,6 +151,7 @@ void assign_ids(vector<MDProducerHandle>::iterator begin, vector<MDProducerHandl
}
}
class Co2Tests
{
public:
@@ -171,12 +172,20 @@ public:
}
#ifndef WIN32
void test_security_id_recorder() {
void test_security_id_recorder(const std::string& db_name) {
// Verify callbacks happen after work complete
SecurityIDRecorder idsvc, idsvc2;
auto db1 = sqlite::open_db(db_name.c_str());
REQUIRE(db1);
auto db2 = sqlite::open_db(db_name.c_str());
REQUIRE(db2);
SecurityIDRecorder idsvc{{db1}}, idsvc2{{db2}};
security_key result = invalid_security_key;
std::string desc = "FOOX 1";
Security s1 = mkoption("FOOX", invalid_security_key, 2018, 1, 20, 1850);
Security f1 = mkfuture("FOO", invalid_security_key, 2018, 1, 20);
idsvc.assign_key(f1, [](const api::Security& s) {
REQUIRE(s.id() != invalid_security_key);
});
idsvc.assign_key(s1, [&result](const api::Security& s) {
result = s.id();
});
@@ -188,7 +197,60 @@ public:
result2 = s.id();
});
idsvc2.do_work();
REQUIRE(result == result2);
if (db_name == ":memory:") {
// This is a meta-test to make sure we don't accidently generate the same key sequence
// when writing to different databases.
REQUIRE(result != result2);
}
else {
REQUIRE(result == result2);
}
}
void test_security_id_recorder_spreads() {
// Test that security id recorder can resolve spread legs
// Set up "shared memory"
auto cold_header = std::make_shared<ColdHeader>();
auto hot_header = std::make_shared<HotHeader>();
// Create a spread with 2 legs
auto f1 = mkfuture("FO1", invalid_security_key, 2018, 1, 17);
f1.mutable_glbx()->set_exchange_id(100);
REQUIRE(f1.id() == invalid_security_key);
auto f1handle = std::make_unique<Co2TestInstrument> (f1, cold_header, hot_header);
auto f2 = mkfuture("FO2", invalid_security_key, 2018, 1, 20);
f2.mutable_glbx()->set_exchange_id(200);
auto f2handle = std::make_unique<Co2TestInstrument> (f2, cold_header, hot_header);
auto s1 = mkspread("FOO", invalid_security_key, { f1, f2 });
auto s1handle = std::make_unique<Co2TestInstrument> (s1, cold_header, hot_header);
// Add spread to SecurityIDRecorder and work it
auto db = sqlite::open_db(":memory:");
REQUIRE(db);
SecurityIDRecorder idsvc{{db}};
auto workwork = [&]() { idsvc.do_work(); idsvc.do_work(); };
// auto workwork = [&]() { for(int i=5;--i;) { idsvc.do_work(); } };
idsvc.ensure_key(s1handle->producer());
workwork();
// Expect no key
REQUIRE(s1handle->consumer().key() == invalid_security_key);
// Add leg spreads
idsvc.ensure_key(f1handle->producer());
idsvc.ensure_key(f2handle->producer());
workwork();
// Expect legs to have keys
REQUIRE(f1handle->consumer().key() != invalid_security_key);
REQUIRE(f2handle->consumer().key() != invalid_security_key);
// Expect spread to have keys
REQUIRE(s1handle->consumer().key() != invalid_security_key);
}
#endif
@@ -1002,7 +1064,11 @@ TEST_CASE("CO2Tests","[md]") {
#ifndef WIN32
SECTION("test_security_id_recorder") {
co2_tests->test_security_id_recorder();
co2_tests->test_security_id_recorder(":memory:");
co2_tests->test_security_id_recorder("file::memory:?cache=shared");
}
SECTION("test_security_id_recorder_spreads") {
co2_tests->test_security_id_recorder_spreads();
}
#endif
SECTION("test_price_size") {
@@ -43,7 +43,7 @@ void run_cx_server(UserSymbol* psyms, size_t symcount)
CXMain::CXMain() :
_producer(co2::PLATFORM_CELOXICA),
_security_recorder(),
_security_recorder({}),
_settlement_recorder(co2::PLATFORM_CELOXICA),
_listeners()
{