#include "catch/catch.hpp" #include "../SettlementRecorder.h" #include "../coconut2.h" #include "../relay/QueryServer.h" #include "Co2TestInstrument.h" #include "lib/config/config.h" #include "lib/os/os.h" #include "lib/utils/Dispatcher.h" #include "lib/utils/defer.h" #include "lib/utils/warn_ctrl.h" #include "lib/utils/bts_chrono.h" #include "lib/metrics/metrics.h" #include "lib/config/protos/MarketData.pb.h" #include #include #include #include #include #include #include #include #include #include #include #include using namespace std; namespace bts { #ifdef _WIN32 namespace metrics { struct MetricCollection{}; MetricsContainer::MetricsContainer() {} MetricsContainer::~MetricsContainer() {} } #endif int md_tests = 0; namespace md { // import this from RelayClient.cpp PriceSize book_at(const google::protobuf::RepeatedPtrField& side, int index); namespace test { using namespace api; using namespace co2; Platform TEST_PLATFORM = PLATFORM_SIM; // // Helper functions for Producer/Consumer tests // vector create_test_securities(MDProducer& producer, const char* postfix ="") { FixedOptionSecurity fos; FixedFutureSecurity ffs; ffs.symbol("FOO"+string(postfix)); fos.symbol("FOOX"+string(postfix)); ffs.exchange = VENUE_BTS; fos.exchange = VENUE_BTS; ffs.key = invalid_security_key; fos.key = invalid_security_key; ffs.expiration_year = fos.expiration_year = 2015; ffs.expiration_day = fos.expiration_day = 27; fos.strike = 1000; fos.call_put = PUTORCALL_CALL; // Add 12 expirations to 2 different shared memory regions vector result; for (uint32_t x = 1; x <= 12; ++x) { ffs.expiration_month = fos.expiration_month = x; result.push_back(producer.add_security(ffs)); result.push_back(producer.add_security(fos)); // In a real app these securities would now be passed to the SecurityIDRecorder } return result; } set get_test_symbols(const char* postfix="") { set symbols; UserSymbol future_symbol(string("FOO")+postfix, VENUE_BTS, SECURITYTYPE_FUTURE); symbols.insert(future_symbol); UserSymbol option_symbol(string("FOOX")+postfix, VENUE_BTS, SECURITYTYPE_OPTION); symbols.insert(option_symbol); return symbols; } Security mkfuture(string symbol, security_key key, int year, int month, int day) { Security result; result.set_symbol(symbol); result.set_exchange(VENUE_BTS); result.set_id(key); result.mutable_future()->set_expiration_day(day); result.mutable_future()->set_expiration_month(month); result.mutable_future()->set_expiration_year(year); return result; } Security mkoption(string symbol, security_key key, int year, int month, int day, double strike) { Security result; result.set_symbol(symbol); result.set_exchange(VENUE_BTS); result.set_id(key); result.mutable_option()->set_expiration_day(day); result.mutable_option()->set_expiration_month(month); result.mutable_option()->set_expiration_year(year); result.mutable_option()->set_strike(strike); result.mutable_option()->set_call_put(PUTORCALL_PUT); return result; } unique_ptr mkfixedoption(string symbol, security_key key, int year, int month, int day, double strike, PutOrCall callput) { unique_ptr fos(new FixedOptionSecurity()); fos->type = SECURITYTYPE_OPTION; fos->exchange = VENUE_BTS; fos->key = key; fos->strike = strike; fos->expiration_day = day; fos->expiration_month = month; fos->expiration_year = year; fos->call_put = callput; fos->symbol(symbol); return fos; } Security mkspread(string symbol, security_key key, const vector& legs) { Security result; result.set_symbol(symbol); result.set_exchange(VENUE_BTS); result.set_id(key); for (const auto& leg : legs) { auto* pleg = result.mutable_spread()->add_legs(); pleg->mutable_security()->CopyFrom(leg); } return result; } Security mkspread(string symbol, security_key key, Security leg1, Security leg2) { vector vlegs; vlegs.push_back(leg1); vlegs.push_back(leg2); return mkspread(symbol, key, vlegs); } Security mkspread(string symbol, security_key key, Security leg1, Security leg2, Security leg3) { vector vlegs; vlegs.push_back(leg1); vlegs.push_back(leg2); vlegs.push_back(leg3); return mkspread(symbol, key, vlegs); } void assign_ids(vector::iterator begin, vector::iterator end, security_key start_key = 1000) { for (; begin != end; ++begin) { (*begin).set_key(++start_key); } } class Co2Tests { public: void setup() { config::internal::initialize_for_test( "PlatformMap {" " value {" " glbx_platform: 'PLATFORM_SIM'" " }" "}" "PublisherOptions {" " value {" " heartbeat_ms: 100" " heartbeat_boot_seconds: 1" " }" "}" ); } #ifndef WIN32 void test_security_id_recorder() { // Verify callbacks happen after work complete SecurityIDRecorder idsvc, idsvc2; security_key result = invalid_security_key; std::string desc = "FOOX 1"; Security s1 = mkoption("FOOX", invalid_security_key, 2018, 1, 20, 1850); idsvc.assign_key(s1, [&result](const api::Security& s) { result = s.id(); }); idsvc.do_work(); REQUIRE(s1.id() == invalid_security_key); REQUIRE(result != invalid_security_key); security_key result2 = invalid_security_key; idsvc2.assign_key(s1, [&result2](const api::Security& s) { result2 = s.id(); }); idsvc2.do_work(); REQUIRE(result == result2); } #endif void test_price_size() { // PriceSize violates some type punning rules // So it is important to have this test work correctly on the target platform PriceSize p2; auto bid = PriceSize::from_double<2>(4.5, 14); auto ask = PriceSize::from_double<9>(4.5, 14); PriceLevel memory; memory = PriceLevel(bid,ask); REQUIRE(bid.price() == memory.bid().price()); REQUIRE(bid.size() == memory.bid().size()); REQUIRE(ask.price() == memory.ask().price()); REQUIRE(ask.size() == memory.ask().size()); for (auto p : { memory.bid(), memory.ask() }) { REQUIRE(p.price() == 4.5); REQUIRE(p.size() == 14); } auto p = PriceSize::from_double<2>(4.0, 5); memory = PriceLevel(p,p); for (auto p2 : { memory.bid(), memory.ask() }) { REQUIRE(p2.price() == 4.0); REQUIRE(p2.size() == 5); } p = PriceSize::from_double<2>(-4.2, 6); memory = PriceLevel(p,p); for (auto p2 : { memory.bid(), memory.ask() }) { REQUIRE(p2.price() == -4.2); REQUIRE(p2.size() == 6); } p = PriceSize(4, -1, 2); memory = PriceLevel(p,p); for (auto p2 : { memory.bid(), memory.ask() }) { REQUIRE(p2.price() == 0.4); REQUIRE(p2.size() == 2); } // Test requirement for more than 32 bit mantissa double bigval = -9999.9921875; p = PriceSize::from_double<7>(bigval, 42); memory = PriceLevel(p,p); for (auto p2 : { memory.bid(), memory.ask() }) { REQUIRE(p2.price() == bigval); REQUIRE(p2.size() == 42); } } void test_price_size_required_precision() { auto min_tick_2017 = 9999.0078125; auto p17 = PriceSize::from_double<7>(min_tick_2017, 2017); PriceLevel m17(p17, p17); for (auto p : { m17.bid(), m17.ask() }) { REQUIRE(min_tick_2017 == p.price()); REQUIRE(p.size() == 2017); } auto min_tick_2018 = 1999.00390625; auto p18 = PriceSize::from_double<8>(min_tick_2018, 2018); PriceLevel m18(p18, p18); for (auto p : { m18.bid(), m18.ask() }) { REQUIRE(min_tick_2018 == p.price()); REQUIRE(p.size() == 2018); } } void test_slow_double_conversion() { double values[] = { 1.5, 0.5, 0.2155, 0.216, 42.1 }; int expected[] = { -1, -1, -4, -3, -1 }; static_assert(BTS_ARRAYSIZE(values) == BTS_ARRAYSIZE(expected), ""); for (size_t x = 0; x < BTS_ARRAYSIZE(values); ++x) { REQUIRE(to_exponent(values[x]) == expected[x]); } // Tests powers of 10. Important for display_factor conversion for (int x = 8; x > -8; --x) { double d = pow10(x); REQUIRE(to_exponent(d) == x); } } struct ExchangeDecimal { int64_t mantissa; int32_t exponent; }; bool close_enough(double a, double b) { // We need at least 7 places but we're testing with 10 just in case const double acceptable_error = 1E-12; double delta = abs(a - b); // Equal if the delta is less than error return delta < acceptable_error; } void test_compact_exponent() { ExchangeDecimal test_values[] = { { 2030000000000, -9 }, // mantissa > pow(2,40) { -2030000000000, -9 }, // mantissa > pow(2,40) { 2022750000000, -9 },// mantissa > pow(2,40) { -2022750000000, -9 },// mantissa > pow(2,40) { 20227500000000, -10 },// mantissa > pow(2,40) { 202275000000000, -11 },// mantissa > pow(2,40) { 2022750000000000, -12 },// mantissa > pow(2,40) { 20227500000000000, -13 },// mantissa > pow(2,40) { -20227500000000000, -13 },// mantissa > pow(2,40) { 203, 1}, // control { -14000000000, -8 }, // control { -14, 1 }, // control { 20227500, -4 }, // control }; for (auto p : test_values) { double expected = p.mantissa * pow10(p.exponent); compact_mantissa(p.mantissa, p.exponent); double compacted = p.mantissa * pow10(p.exponent); REQUIRE(close_enough(expected, compacted)); } } void test_producer_consumer() { MDProducer producer(TEST_PLATFORM); vector producers = create_test_securities(producer); // Add bbo data for those expirations auto write_time = TimeStamp::now(); for (auto& p : producers) { p.update_bbo(write_time, PriceSize::from_double<2>(5.0, 5), PriceSize::from_double<2>(5.0, 5)); } // Symbol list for the consumer set symbols; UserSymbol future_symbol( "FOO", VENUE_BTS, SECURITYTYPE_FUTURE ); symbols.insert(future_symbol); UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); MDConsumer consumer(symbols, nullptr); vector consumer_securities; // Poll for security list - none visible until they have keys size_t count = consumer.walk_security_list([&](const Security& security, MDConsumerHandle) {consumer_securities.push_back(security);}); // Since no security has an id none are "public" REQUIRE(consumer_securities.size() == count); REQUIRE(count == 0); // Assign a unique ID to each security (from the producer) assign_ids(producers.begin(), producers.end()); // Now securities can be enumerated by the client count = consumer.walk_security_list([&](const Security& security, MDConsumerHandle) {consumer_securities.push_back(security);}); REQUIRE(consumer_securities.size() == count); REQUIRE(count == 24); vector consumers; for(const auto& security : consumer_securities) { consumers.push_back(consumer.get_handle(security.id())); } for (const auto& c : consumers) { PriceSize bid, ask; TimeStamp read_time; // get_boo returns true becuase this time is after the change REQUIRE(c.get_bbo(read_time, bid, ask) == HandleReadResult::SUCCESS); REQUIRE((read_time - TimeStamp()).seconds() > 0); REQUIRE(bid.price() == 5.0); REQUIRE(ask.price() == 5.0); REQUIRE(bid.size() == 5); REQUIRE(ask.size() == 5); } // Second query before a write has happend finds no updates auto read_time = TimeStamp::now(); for (const auto& c : consumers) { PriceSize bid, ask; REQUIRE(c.get_bbo(read_time, bid, ask) != HandleReadResult::SUCCESS); } // update everything once more write_time = TimeStamp::now(); for (auto& p : producers) { // Update bid/ask with one call p.update_bbo(write_time, PriceSize::from_double<2>(6.55, 6), PriceSize::from_double<2>(6.55, 6)); } // Now read_time is < write_time for (const auto& c : consumers) { TimeStamp modified_time = read_time; PriceSize bid, ask; // get_bbo returns true becuase this time is after the change REQUIRE(c.get_bbo(modified_time, bid, ask) == HandleReadResult::SUCCESS); REQUIRE( (modified_time - read_time).seconds() > 0); REQUIRE(bid.price() == 6.55); REQUIRE(ask.price() == 6.55); REQUIRE(bid.size() == 6); REQUIRE(ask.size() == 6); } } // When a security is added the consumer should provide callback void test_consumer_callback_on_security_added() { MDProducer producer(TEST_PLATFORM); vector producers = create_test_securities(producer); // Symbol list for the consumer set symbols; UserSymbol future_symbol( "FOO", VENUE_BTS, SECURITYTYPE_FUTURE ); symbols.insert(future_symbol); UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); MDConsumer consumer(symbols, nullptr); // Add the on_security callback vector securities; consumer.on_security([&](const Security& security, MDConsumerHandle handle) { REQUIRE(security.id() != invalid_security_key); REQUIRE(security.id() == handle.key()); securities.push_back(security); }); // No securities have ids yet REQUIRE(securities.size() == 0); // Check for new securities consumer.scan_securities(); // Still none REQUIRE(securities.size() == 0); producers.front().set_key(1001); consumer.scan_securities(); REQUIRE(securities.size() == 1); // Assign the rest of the ids assign_ids(producers.begin() + 1, producers.end(), producers.front().security().key); consumer.scan_securities(); REQUIRE(securities.size() == producers.size()); } // If an MDConsumer is asked for a handle to a security which does not exist in the system // The consumer should return a handle which always returns false on all calls but wakes up // when the security is added. // * Case 1 - the shared memory has been created but the security has not been discovered // * Case 2 - The producer has never been created void test_consumer_defer_handle1() { MDProducer producer(TEST_PLATFORM); vector producers = create_test_securities(producer); set symbols; UserSymbol future_symbol( "FOO", VENUE_BTS, SECURITYTYPE_FUTURE ); symbols.insert(future_symbol); UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); MDConsumer consumer(symbols, nullptr); // Get a handle to 24 consumers which haven't been created yet vector consumers; vector> managed_consumers; security_key key = 1; for(; key < 24; ++key) { consumers.push_back(consumer.get_handle(key+1000)); managed_consumers.push_back(consumer.get_managed_handle(key+1000)); } // Set bbo on the producers - still not public TimeStamp write_time = TimeStamp::now(); for (auto& p : producers) { p.update_bbo(write_time, PriceSize::from_double<3>(4.5, 5), PriceSize::from_double<3>(6.75, 7)); } // Even though bbo is available it is not given out because the producing security does not have an id PriceSize bid, ask; for(auto& c : consumers) { TimeStamp zerotime; REQUIRE(c.get_bbo(zerotime, bid, ask) != HandleReadResult::SUCCESS); } for(auto& mc : managed_consumers) { TimeStamp zerotime; REQUIRE(mc->get_bbo(zerotime, bid, ask) != HandleReadResult::SUCCESS); } assign_ids(producers.begin(), producers.end()); consumer.scan_securities(); size_t bbo_count = 0; // Existing old handles remain invalid for(auto& c : consumers) { TimeStamp zerotime; if (c.get_bbo(zerotime, bid, ask) == HandleReadResult::SUCCESS) { bbo_count++; } } REQUIRE(bbo_count == 0); // Now, magically, the existing (managed) handles are valid for(auto& mc : managed_consumers) { TimeStamp zerotime; if (mc->get_bbo(zerotime, bid, ask) == HandleReadResult::SUCCESS) { bbo_count++; } } REQUIRE(bbo_count > 0); } // It is legal and valid to create a consumer before the producer process has started // Handles given out by that consumer will become valid once the securities are discovered void test_consumer_defer_handle2() { set symbols; UserSymbol future_symbol( "FOO", VENUE_BTS, SECURITYTYPE_FUTURE ); symbols.insert(future_symbol); UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); MDConsumer consumer(symbols, nullptr); // Get a handle to 24 consumers which haven't been created yet vector consumers; vector> managed_consumers; security_key key = 1; for(; key < 24; ++key) { consumers.push_back(consumer.get_handle(key+1000)); managed_consumers.push_back(consumer.get_managed_handle(key+1000)); } // Even though bbo is available it is not given out because the producing security does not have an id PriceSize bid, ask; for(auto& c : consumers) { TimeStamp zerotime; REQUIRE(c.get_bbo(zerotime, bid, ask) != HandleReadResult::SUCCESS); } for(auto& mc : managed_consumers) { TimeStamp zerotime; REQUIRE(mc->get_bbo(zerotime, bid, ask) != HandleReadResult::SUCCESS); } // Only NOW create the producers (and the shared memory) MDProducer producer(TEST_PLATFORM); vector producers = create_test_securities(producer); // Set bbo on the producers - still not public TimeStamp write_time = TimeStamp::now(); for (auto& p : producers) { p.update_bbo(write_time, PriceSize::from_double<3>( 4.5, 5 ), PriceSize::from_double<3>(6.75, 7)); } assign_ids(producers.begin(), producers.end()); consumer.scan_securities(); size_t bbo_count = 0; // Existing old handles remain invalid for(auto& c : consumers) { TimeStamp zerotime; if (c.get_bbo(zerotime, bid, ask) == HandleReadResult::SUCCESS) { bbo_count++; } } REQUIRE(bbo_count == 0); // Now, magically, the existing (managed) handles are valid for(auto& mc : managed_consumers) { TimeStamp zerotime; if (mc->get_bbo(zerotime, bid, ask) == HandleReadResult::SUCCESS) { bbo_count++; } } REQUIRE(bbo_count > 0); } void test_spread_resolver() { SpreadResolver resolver; vector legs; legs.push_back("1"); legs.push_back("2"); legs.push_back("3"); legs.push_back("4"); resolver.add("1234", legs); legs.clear(); legs.push_back("4"); legs.push_back("5"); resolver.add("45", legs); auto key_map = [](string& key) -> security_key { if (key == "1") return 1; if (key == "2") return 2; if (key == "3") return 3; if (key == "4") return 4; if (key == "5") return invalid_security_key; REQUIRE(false); return 0; }; auto key_map2 = [](string& key) -> security_key { if (key == "4") return 4; if (key == "5") return 5; REQUIRE(false); return 0; }; // 2 unresolved spreads REQUIRE(resolver.count() == 2); resolver.resolve(key_map, [&](const string& key, const vector& legs) { REQUIRE(key == "1234"); REQUIRE(legs.size() == 4); }); // 1 spread remains REQUIRE(resolver.count() == 1); resolver.resolve(key_map2, [&](const string& key, const vector& legs) { REQUIRE(key == "45"); REQUIRE(legs.size() == 2); }); REQUIRE(resolver.count() == 0); } void test_no_double_security_callback() { MDProducer producer(TEST_PLATFORM); vector producers = create_test_securities(producer); // Assign ids so securities are already public in the producer assign_ids(producers.begin(), producers.end()); // Symbol list for the consumer auto syms = get_test_symbols(); MDConsumer consumer = MDConsumer(syms, nullptr); // Add the on_security callback - callback fires for existing securities vector securities; consumer.on_security([&](const Security& security, MDConsumerHandle handle) { REQUIRE(security.id() != invalid_security_key); REQUIRE(security.id() == handle.key()); securities.push_back(security); }); // Found all securities REQUIRE(securities.size() == producers.size()); // Check for new securities consumer.scan_securities(); // Expect no duplicates REQUIRE(securities.size() == producers.size()); } void test_recovery() { auto producer1 = make_shared(TEST_PLATFORM); vector producers = create_test_securities(*producer1); assign_ids(producers.begin(), producers.end()); producer1->get_channel("FOO", SECURITYTYPE_FUTURE, VENUE_BTS).heartbeat(); producer1->get_channel("FOOX", SECURITYTYPE_OPTION, VENUE_BTS).heartbeat(); for (auto& pro : producers) { pro.update_bbo(TimeStamp::now(), PriceSize(1, 0, 1), PriceSize(2, 0, 2)); } // Symbol list for the consumer MDRecoverableConsumer consumer(get_test_symbols(), nullptr); map> products; consumer.on_security([&](const Security& security, MDConsumerHandle handle) { REQUIRE(security.id() == handle.key()); products.insert(make_pair(security.id(), consumer.get_managed_handle(security.id()))); }); consumer.scan_securities(); REQUIRE(products.size() == producers.size()); // Choose a handle for (auto& pair : products) { auto& handle = pair.second; REQUIRE(handle->is_initialized()); REQUIRE(handle->is_heart_beating()); } // Delete the producer try { producers.clear(); producer1.reset(); } catch(std::exception&) { LOG(md,Error,"Failed test trying to destroy producer."); REQUIRE(false); } // Wait out the heartbeat auto options = config::get(); auto ms = std::max(options.heartbeat_ms(), options.heartbeat_boot_seconds()*1000)+10; std::this_thread::sleep_for(std::chrono::duration(ms)); // Make sure heartbeat is done try { for (auto& pair : products) { auto& handle = pair.second; REQUIRE(handle->is_initialized()); REQUIRE(handle->is_heart_beating() == false); } } catch(std::exception&) { LOG(md,Error,"Failed to validate heartbeat while producer is down."); REQUIRE(false); } // Create a new publisher and attempt to recover shared_ptr producer2; try { producer2 = make_shared(TEST_PLATFORM); } catch(std::exception&) { LOG(md,Error,"Failed to create new producer"); REQUIRE(false); } producers = create_test_securities(*producer2); REQUIRE(products.size() == producers.size()); assign_ids(producers.begin(), producers.end()); producer2->get_channel("FOO", SECURITYTYPE_FUTURE, VENUE_BTS).heartbeat(); producer2->get_channel("FOOX", SECURITYTYPE_OPTION, VENUE_BTS).heartbeat(); // set bbo to a different value for (auto& pro : producers) { pro.update_bbo(TimeStamp::now(), PriceSize(4, 0, 5), PriceSize(6, 0, 7)); } // Attempt to recover the managed handles REQUIRE(consumer.try_recover()); // Since we know that both securities exist they should have been updated to the new section for (auto& pair : products) { auto& handle = pair.second; REQUIRE(handle->is_initialized()); REQUIRE(handle->is_heart_beating()); TimeStamp tzero; PriceSize bid, ask; REQUIRE(handle->get_bbo(tzero, bid, ask) == HandleReadResult::SUCCESS); REQUIRE(bid.price() == 4.0); REQUIRE(bid.size() == 5); REQUIRE(ask.price() == 6.0); REQUIRE(ask.size() == 7); } } void test_no_block_forever_on_publisher_crash() { #ifdef _WIN32 #pragma warning(push, 0) #endif // OK if this isn't aligned for testing auto src = std::make_unique ("Foo", SECURITYTYPE_FUTURE); #ifdef _WIN32 #pragma warning(pop) #endif TimeStamp read = TimeStamp::now(); src->producer().update_bbo(TimeStamp::invalid(), PriceSize::from_double<3>(100.0, 100), PriceSize::from_double<3>(101.0, 101)); PriceSize bid, ask; TimeStamp start = TimeStamp::now(); auto result = src->consumer().get_bbo(read, bid, ask); TimeStamp end = TimeStamp::now(); REQUIRE(result != HandleReadResult::SUCCESS); REQUIRE((end - start).seconds() < 5); } void test_get_handles() { MDProducer producer(TEST_PLATFORM); vector producers = create_test_securities(producer); // Add entries to the shared memory that we aren't looking for vector p2_no_id = create_test_securities(producer, "2"); vector p3 = create_test_securities(producer, "3"); for (auto& p : producers) { p.update_bbo(TimeStamp::now(), PriceSize::from_double<3>(5.0, 5), PriceSize::from_double<3>(5.0, 5)); } assign_ids(producers.begin(), producers.end()); assign_ids(p3.begin(), p3.end()); // Symbol list for the consumer - only search half of the symbols set symbols; UserSymbol future_symbol( "FOO", VENUE_BTS, SECURITYTYPE_FUTURE ); symbols.insert(future_symbol); UserSymbol option_symbol( "FOOX", VENUE_BTS, SECURITYTYPE_OPTION ); symbols.insert(option_symbol); MDConsumer consumer(symbols, nullptr); set keys; consumer.walk_security_list([&](const Security& security, MDConsumerHandle) { keys.insert(security.id()); }); REQUIRE(keys.size() == producers.size()); auto map = consumer.get_handles(keys); REQUIRE(map.size() == keys.size()); for (const auto& entry : map) { REQUIRE(entry.second.is_initialized()); } } void test_denormalization() { // By using a different symbol for each security // we put them all on separate shared mem sections (co2 channels) security_key key = 999; Security f1 = mkfuture("F1", ++key, 2016, 5, 1); Security o1 = mkoption("O1", ++key, 2016, 5, 1, 100); Security o2 = mkoption("O2", ++key, 2016, 5, 2, 200); Security o3 = mkoption("O3", ++key, 2016, 5, 3, 300); Security s4 = mkspread("S3", ++key, o2, o3); Security s3 = mkspread("S4", ++key, o1, s4); Security s2 = mkspread("S2", ++key, o1, s4, s3); Security s1 = mkspread("S1", ++key, f1, s2); Security all_secs_array[] = { f1, o1, o2, o3, s1, s2, s3, s4 }; MDProducer producer(TEST_PLATFORM); vector producers; for (auto& sec : all_secs_array) { with_security_like(sec, [&](FixedSecurity& fsec) { convert_security(sec, fsec, true); producers.push_back(producer.add_security(fsec)); }); } set csymbols; csymbols.insert(UserSymbol("F1", VENUE_BTS, SECURITYTYPE_FUTURE)); csymbols.insert(UserSymbol("O1", VENUE_BTS, SECURITYTYPE_OPTION)); csymbols.insert(UserSymbol("O2", VENUE_BTS, SECURITYTYPE_OPTION)); csymbols.insert(UserSymbol("O3", VENUE_BTS, SECURITYTYPE_OPTION)); csymbols.insert(UserSymbol("S1", VENUE_BTS, SECURITYTYPE_SPREAD)); csymbols.insert(UserSymbol("S2", VENUE_BTS, SECURITYTYPE_SPREAD)); csymbols.insert(UserSymbol("S3", VENUE_BTS, SECURITYTYPE_SPREAD)); csymbols.insert(UserSymbol("S4", VENUE_BTS, SECURITYTYPE_SPREAD)); MDConsumer consumer(csymbols, nullptr); auto check_sec = [&](const Security& security, MDConsumerHandle handle) { REQUIRE(security.id() == handle.key()); auto iter = find_if(std::begin(all_secs_array), std::end(all_secs_array), [&](const Security& sec) { return sec.id() == security.id(); }); REQUIRE(iter != end(all_secs_array)); REQUIRE(iter->symbol() == (security.symbol())); if (security.Contract_case() == Security::kSpread) { REQUIRE(security.spread().normalized() == false); REQUIRE(security.spread().legs_size() == iter->spread().legs_size()); for (int x = 0; x < security.spread().legs_size(); ++x) { REQUIRE((security.spread().legs(x).security().id()) == (iter->spread().legs(x).security().id())); } } }; consumer.walk_security_list(check_sec); consumer.on_security(check_sec); consumer.scan_securities(); } void test_denormalization2() { // Just like test_denormailzation but with a different memory layout security_key key = 999; Security f1 = mkfuture("FF", ++key, 2016, 5, 1); Security o1 = mkoption("OO", ++key, 2016, 5, 1, 100); Security o2 = mkoption("OO", ++key, 2016, 5, 2, 200); Security o3 = mkoption("OO", ++key, 2016, 5, 3, 300); Security s4 = mkspread("SS", ++key, o2, o3); Security s3 = mkspread("SS", ++key, o1, s4); Security s2 = mkspread("SS", ++key, o1, s4, s3); Security s1 = mkspread("SS", ++key, f1, s2); Security all_secs_array[] = { f1, o1, o2, o3, s1, s2, s3, s4 }; MDProducer producer(TEST_PLATFORM); vector producers; for (auto& sec : all_secs_array) { with_security_like(sec, [&](FixedSecurity& fsec) { convert_security(sec, fsec, true); producers.push_back(producer.add_security(fsec)); }); } set csymbols; csymbols.insert(UserSymbol("FF", VENUE_BTS, SECURITYTYPE_FUTURE)); csymbols.insert(UserSymbol("OO", VENUE_BTS, SECURITYTYPE_OPTION)); csymbols.insert(UserSymbol("SS", VENUE_BTS, SECURITYTYPE_SPREAD)); MDConsumer consumer(csymbols, nullptr); auto check_sec = [&](const Security& security, MDConsumerHandle handle) { REQUIRE(security.id() == handle.key()); auto iter = find_if(begin(all_secs_array), end(all_secs_array), [&](const Security& sec) { return sec.id() == security.id(); }); REQUIRE(iter != end(all_secs_array)); REQUIRE((iter->id()) == (security.id())); if (security.Contract_case() == Security::kSpread) { REQUIRE(security.spread().normalized() == false); REQUIRE(security.spread().legs_size() == iter->spread().legs_size()); for (int x = 0; x < security.spread().legs_size(); ++x) { REQUIRE((security.spread().legs(x).security().id()) == (iter->spread().legs(x).security().id())); } } }; consumer.walk_security_list(check_sec); consumer.on_security(check_sec); consumer.scan_securities(); } void test_retry_unresolved_spread() { // Make 2 spreads with option legs // Leave out one leg for hte other spread // The other spread only shows up once its option leg is produced Security o1 = mkoption("OO", 400, 2016, 5, 1, 100); Security o2 = mkoption("OO", 401, 2016, 5, 2, 200); Security o3 = mkoption("OO", 402, 2016, 5, 3, 300); Security s1 = mkspread("SS", 500, o1, o2); Security s2 = mkspread("SS", 501, o2, o3); MDProducer producer(TEST_PLATFORM); map handles; auto add_security = [&](const Security& sec) { with_security_like(sec, [&](FixedSecurity& fsec) { convert_security(sec, fsec, true); handles.insert(make_pair(fsec.key, producer.add_security(fsec))); }); }; // "options publisher has not started yet" //add_security(o1); //add_security(o2); //add_security(o3); add_security(s1); add_security(s2); // Create consumer set consumer_symbols; consumer_symbols.insert(UserSymbol("OO", VENUE_BTS, SECURITYTYPE_OPTION)); consumer_symbols.insert(UserSymbol("SS", VENUE_BTS, SECURITYTYPE_SPREAD)); MDConsumer consumer(consumer_symbols, nullptr); size_t security_count = 0; // Indirection for check function function check_func; check_func = [&](const Security&) { REQUIRE(false); }; consumer.on_security([&](const Security& sec, MDConsumerHandle) { ++security_count; check_func(sec); }); consumer.scan_securities(); REQUIRE(security_count == 0); // OK, now add enough options to resolve s1 but not s2 add_security(o1); add_security(o2); check_func = [&](const Security& security) { if (security.Contract_case() == Security::kSpread) { set legs; for (const auto& leg : security.spread().legs()) { legs.insert(leg.security().id()); } REQUIRE(security.id() == 500); REQUIRE(legs.size() == 2); REQUIRE(legs.count(400) == 1); REQUIRE(legs.count(401) == 1); } }; consumer.scan_securities(); // finds o1,o2,s1 consumer.scan_securities(); // finds nothing REQUIRE(security_count == 3); // Then add the last security and we should see s2 add_security(o3); check_func = [&](const Security& security) { if (security.Contract_case() == Security::kSpread) { set legs; for (const auto& leg : security.spread().legs()) { legs.insert(leg.security().id()); } REQUIRE(security.id() == 501); REQUIRE(legs.size() == 2); REQUIRE(legs.count(401) == 1); REQUIRE(legs.count(402) == 1); } }; consumer.scan_securities(); REQUIRE(security_count == 5); } #ifndef WIN32 void test_settlement_recorder() { using namespace datetime; security_key k1 = 401, k2 = 402; Security o1 = mkoption("OO", k1, 2016, 5, 1, 100); Security o2 = mkoption("OO", k2, 2016, 5, 2, 200); MDProducer producer(TEST_PLATFORM); auto add_security = [&](const Security& sec) { unique_ptr phandle; with_security_like(sec, [&](FixedSecurity& fsec) { convert_security(sec, fsec, true); phandle.reset(new MDProducerHandle(producer.add_security(fsec))); }); return phandle; }; auto ph1 = *add_security(o1); auto ph2 = *add_security(o2); set consumer_symbols; consumer_symbols.insert(UserSymbol("OO", VENUE_BTS, SECURITYTYPE_OPTION)); MDConsumer consumer(consumer_symbols, nullptr); auto h1 = consumer.get_handle(k1); REQUIRE(h1.is_initialized()); auto h2 = consumer.get_handle(k2); // Pick 2 valid dates in the future auto today = datetime::today(); today.year += 1; auto yesterday = datetime::add_days(today, -1); { co2::SettlementRecorder recorder(PLATFORM_UNSET); recorder.add_settlement(ph1, 101, yesterday); recorder.add_settlement(ph1, 102, today); recorder.add_settlement(ph2, 199, yesterday); recorder.add_settlement(ph2, 188, today); recorder.do_work(); recorder.do_work(); } REQUIRE(h1.misc().settlement_price == 102); REQUIRE(h1.misc().settlement_date == today); REQUIRE(h1.misc().previous_settlement_price == 101); REQUIRE(h1.misc().previous_settlement_date == yesterday); REQUIRE(h2.misc().settlement_price == 188); REQUIRE(h2.misc().settlement_date == today); REQUIRE(h2.misc().previous_settlement_price == 199); REQUIRE(h2.misc().previous_settlement_date == yesterday); // Update but skip persist auto tomorrow = datetime::add_days(today, 1); ph1.update_settlement(TimeStamp::now(), 103, tomorrow); REQUIRE(h1.misc().settlement_price == 103); REQUIRE(h1.misc().settlement_date == tomorrow); REQUIRE(h1.misc().previous_settlement_price == 102); REQUIRE(h1.misc().previous_settlement_date == today); ph1.update_previous_settlement(TimeStamp::now(), 0, Date()); REQUIRE(h1.misc().settlement_price == 103); REQUIRE(h1.misc().settlement_date == tomorrow); ph1.update_settlement(TimeStamp::now(), 0, Date()); REQUIRE(h1.misc().settlement_price == 0); REQUIRE(h1.misc().settlement_date.day == 0); // pushing a new settlement for today doesn't invalidate yesterday ph2.update_settlement(TimeStamp::now(), 201, today); REQUIRE(h2.misc().settlement_price == 201); REQUIRE(h2.misc().settlement_date == today); REQUIRE(h2.misc().previous_settlement_price == 199); REQUIRE(h2.misc().previous_settlement_date == yesterday); // Restore persisted state { co2::SettlementRecorder recorder(PLATFORM_UNSET); recorder.restore_settlement(ph1); recorder.restore_settlement(ph2); recorder.do_work(); } REQUIRE(h1.misc().settlement_price == 102); REQUIRE(h1.misc().settlement_date == today); REQUIRE(h1.misc().previous_settlement_price == 101); REQUIRE(h1.misc().previous_settlement_date == yesterday); REQUIRE(h2.misc().settlement_price == 188); REQUIRE(h2.misc().settlement_date == today); REQUIRE(h2.misc().previous_settlement_price == 199); REQUIRE(h2.misc().previous_settlement_date == yesterday); } #endif void teardown() {} } ; std::unique_ptr co2_tests; TEST_CASE("CO2Tests","[md]") { // Lazy created this for config if (!co2_tests) co2_tests = std::make_unique(); co2_tests->setup(); #ifndef WIN32 SECTION("test_security_id_recorder") { co2_tests->test_security_id_recorder(); } #endif SECTION("test_price_size") { co2_tests->test_price_size(); } SECTION("test_price_size_required_precision") { co2_tests->test_price_size_required_precision(); } SECTION("test_slow_double_conversion") { co2_tests->test_slow_double_conversion(); } SECTION("test_compact_exponent") { co2_tests->test_compact_exponent(); } SECTION("test_producer_consumer") { co2_tests->test_producer_consumer(); } SECTION("test_consumer_callback_on_security_added") { co2_tests->test_consumer_callback_on_security_added(); } SECTION("test_consumer_defer_handle1") { co2_tests->test_consumer_defer_handle1(); } SECTION("test_consumer_defer_handle2") { co2_tests->test_consumer_defer_handle2(); } SECTION("test_spread_resolver") { co2_tests->test_spread_resolver(); } SECTION("test_no_double_security_callback") { co2_tests->test_no_double_security_callback(); } SECTION("test_recovery") { co2_tests->test_recovery(); } SECTION("test_no_block_forever_on_publisher_crash") { co2_tests->test_no_block_forever_on_publisher_crash(); } SECTION("test_get_handles") { co2_tests->test_get_handles(); } SECTION("test_denormalization") { co2_tests->test_denormalization(); } SECTION("test_denormalization2") { co2_tests->test_denormalization2(); } SECTION("test_retry_unresolved_spread") { co2_tests->test_retry_unresolved_spread(); } #ifndef WIN32 SECTION("test_settlement_recorder") { co2_tests->test_settlement_recorder(); } #endif } struct Co2IndexTests { vector random_ids; set used_ids; const int iterations; tbb::task_scheduler_init _tbb; Co2IndexTests() : random_ids(), used_ids(), iterations(50), _tbb(tbb::task_scheduler_init::automatic) {} void setup() { used_ids.clear(); random_ids.clear(); std::uniform_int_distribution rand_distribution(1, 0x7FFFFFFF); std::mt19937 random_number_engine; auto genrandom = std::bind(rand_distribution, random_number_engine); set temp_ids; for (int x = 0; x < 9999; ++x) { auto r = genrandom(); if (x < 1000) { used_ids.insert(r); } temp_ids.insert(r); } for (uint64_t id : temp_ids) { random_ids.push_back(id); } REQUIRE(random_ids.size() > 1000); } struct Handle { Handle() : _key(0) {} uint64_t _key; security_key key() const { return _key; } void key(security_key key) { _key = key; } }; struct SecurityData { uint64_t exchange_id; Security security; Handle handle; }; template void gen_securities(Index& index) { unique_ptr temp; uint32_t count = 0; for (auto& id : used_ids) { ++count; temp.reset(new SecurityData()); temp->security.Clear(); temp->security.set_symbol("FOOX"); temp->security.mutable_option(); temp->security.mutable_glbx()->set_exchange_id(id); temp->exchange_id = id; temp->handle.key(count); index.add_security(id, move(temp)); } } template void test_random_search(Index& index) { vector temp(random_ids); std::random_shuffle(temp.begin(), temp.end()); for (auto& id : temp) { if (used_ids.find(id) != used_ids.end()) { REQUIRE(index.has_security(id)); } else { REQUIRE(index.get_security(id) == nullptr); } } } template void test_walk(Index& index) { int count = 0; index.walk_security_list([&count](const SecurityData&) { ++count; }); REQUIRE(count == (int) used_ids.size()); } typedef SimpleSecurityIndex MySimpleIndex; void test_simple_index_search() { MySimpleIndex index; gen_securities(index); for (int x = 0; x < iterations; ++x) { test_random_search(index); } } void test_simple_index_parallel_search() { MySimpleIndex index; gen_securities(index); tbb::parallel_for(0, iterations, [&](int) { test_random_search(index); }); } void test_simple_index_iterate() { MySimpleIndex index; gen_securities(index); test_walk(index); } typedef ConcurrentSecurityIndex MyConcurrentIndex; void test_concurrent_index_search() { MyConcurrentIndex index; gen_securities(index); for (int x = 0; x < iterations; ++x) { test_random_search(index); } } void test_concurrent_index_parallel_search() { MyConcurrentIndex index; gen_securities(index); tbb::parallel_for(0, iterations, [&](int) { test_random_search(index); }); } void test_concurrent_index_iterate() { MyConcurrentIndex index; gen_securities(index); test_walk(index); } } ; // TEST_CASE ("Co2IndexTests","[md]") { // Co2IndexTests co2_index_tests; // co2_index_tests.setup(); // SECTION("test_simple_index_search") { // co2_index_tests.test_simple_index_search(); // } // SECTION("test_simple_index_parallel_search") { // co2_index_tests.test_simple_index_parallel_search(); // } // SECTION("test_simple_index_iterate") { // co2_index_tests.test_simple_index_iterate(); // } // SECTION("test_concurrent_index_search") { // co2_index_tests.test_concurrent_index_search(); // } // SECTION("test_concurrent_index_iterate") { // co2_index_tests.test_concurrent_index_iterate(); // } // SECTION("test_concurrent_index_parallel_search") { // co2_index_tests.test_concurrent_index_parallel_search(); // } // } struct Co2RelayTests { void setup() { config::internal::initialize_for_test( "PlatformMap {" " value {" " glbx_platform: 'PLATFORM_SIM'" " }" "}"); } void teardown() {} MDConsumer create_consumer() { set symbols; UserSymbol future_symbol("FOO", VENUE_BTS, SECURITYTYPE_FUTURE); symbols.insert(future_symbol); UserSymbol option_symbol("FOOX", VENUE_BTS, SECURITYTYPE_OPTION); symbols.insert(option_symbol); MDConsumer consumer(symbols, nullptr); return consumer; } template T simulate_wire_transfer(const T& request) { T temp; std::string buffer; request.SerializeToString(&buffer); temp.ParseFromString(buffer); return temp; } void relay_nans() { MDProducer producer(TEST_PLATFORM); UserSymbol symbol("FOO", VENUE_BTS, SECURITYTYPE_OPTION); Entitlements entitlements; entitlements.insert(symbol.symbol()); // initialize data vector producers; for (size_t x = 0; x < 100; ++x) { auto strike = 1000 + (100 * x); producers.push_back( producer.add_security( *mkfixedoption("FOO", invalid_security_key, 2017, 1, 28, strike, PUTORCALL_PUT))); producers.push_back( producer.add_security( *mkfixedoption("FOO", invalid_security_key, 2017, 1, 28, strike, PUTORCALL_CALL))); } assign_ids(producers.begin(), producers.end(), 1000); PriceSize price35 = PriceSize::from_double<2>(3.5, 5); PriceSize price5 = PriceSize::from_double<2>(5, 5); for(auto p : producers) { p.update_bbo(TimeStamp::now(), price35, price35); } // Read using QueryServer set symbols; symbols.insert(symbol); shared_ptr empty; metrics::MetricsContainer stats; auto server = create_query_server(symbols, empty, stats); REQUIRE(server); server->scan_securities(); co2::MDRequest request; auto* chan = request.add_channels(); chan->set_type(symbol.type()); chan->set_symbol(symbol.symbol()); chan->set_exchange(symbol.exchange()); chan->set_flags(co2::MDRequest::BBO | co2::MDRequest::SECURITY); // This request should include everything co2::MDReply reply; server->query(request, &reply, &entitlements, false); reply = simulate_wire_transfer(reply); REQUIRE((size_t)reply.channels().size() == symbols.size()); REQUIRE((size_t)reply.channels(0).updates().size() == producers.size()); // update the request to match the server time for (auto& update : reply.channels(0).updates()) { REQUIRE(update.has_security()); REQUIRE(update.has_book()); REQUIRE(update.book().asks_size() > 0); REQUIRE(update.book().bids_size() > 0); REQUIRE(book_at(update.book().bids(), 0) == price35); REQUIRE(book_at(update.book().asks(), 0) == price35); } // Second request with updated time should include nothing request.mutable_channels(0)->set_time(reply.channels(0).channel().time()); reply.Clear(); server->query(request, &reply, &entitlements, false); reply = simulate_wire_transfer(reply); REQUIRE(reply.channels().size() == 1); REQUIRE(reply.channels(0).updates().size() == 0); // Update a single price and make sure we get that back producers[0].update_bbo(TimeStamp::now(), price5, price5, 1); request.mutable_channels(0)->set_time(reply.channels(0).channel().time()); reply.Clear(); server->query(request, &reply, &entitlements, false); reply = simulate_wire_transfer(reply); REQUIRE(reply.channels().size() == 1); REQUIRE(reply.channels(0).updates().size() == 1); REQUIRE(reply.channels(0).updates(0).has_book()); REQUIRE(book_at(reply.channels(0).updates(0).book().bids(), 0) == price5); REQUIRE(book_at(reply.channels(0).updates(0).book().asks(), 0) == price5); // NaN out all the prices - expect to get a full set of updates for(auto p : producers) { p.update_bbo(TimeStamp::now(), PriceSize::NO_MARKET, PriceSize::NO_MARKET, 2); } request.mutable_channels(0)->set_time(reply.channels(0).channel().time()); reply.Clear(); server->query(request, &reply, &entitlements, false); reply = simulate_wire_transfer(reply); REQUIRE(reply.channels().size() == 1); REQUIRE((size_t)reply.channels(0).updates().size() == producers.size()); for (auto& update : reply.channels(0).updates()) { REQUIRE(update.has_book()); // We need to have a book to signal that it was updated but it can have no rows. This signifies an empty book and will be written as such by the relay client. REQUIRE(book_at(update.book().bids(), 0) == PriceSize::NO_MARKET); REQUIRE(book_at(update.book().asks(), 0) == PriceSize::NO_MARKET); } // Fill up the markets again for(auto p : producers) { p.update_bbo(TimeStamp::now(), price35, price35, 3); } request.mutable_channels(0)->set_time(reply.channels(0).channel().time()); reply.Clear(); server->query(request, &reply, &entitlements, false); reply = simulate_wire_transfer(reply); REQUIRE(reply.channels().size() == 1); REQUIRE((size_t)reply.channels(0).updates().size() == producers.size()); for (auto& update : reply.channels(0).updates()) { REQUIRE(update.has_book()); REQUIRE(update.book().bids_size() > 0); REQUIRE(update.book().asks_size() > 0); REQUIRE(book_at(update.book().bids(), 0) == price35); REQUIRE(book_at(update.book().asks(), 0) == price35); } // NaN out the bids for(auto p : producers) { p.update_bbo(TimeStamp::now(), PriceSize::NO_MARKET, price35, 4); } request.mutable_channels(0)->set_time(reply.channels(0).channel().time()); reply.Clear(); server->query(request, &reply, &entitlements, false); reply = simulate_wire_transfer(reply); REQUIRE(reply.channels().size() == 1); REQUIRE((size_t)reply.channels(0).updates().size() == producers.size()); for (auto& update : reply.channels(0).updates()) { REQUIRE(update.has_book()); REQUIRE(book_at(update.book().bids(), 0) == PriceSize::NO_MARKET); REQUIRE(book_at(update.book().asks(), 0) == price35); } } } ; TEST_CASE ("Co2RelayTests","[md]") { Co2RelayTests co2_relay_tests; co2_relay_tests.setup(); SECTION("relay_nans") { co2_relay_tests.relay_nans(); } } TEST_CASE("Co2SyntheticsTests", "[md]") { // test synthetics show up with other securities // test synehtetics with legs of synthetics // test adding a synthetic with synthetic legs later in the day } }}}