fix: CLI11 integration

* Run 'sparkd tests' outside of the CLI::App framework
  (delegate to 'catch', which has a solid cli).  This
  fixes a bug which prevented callers from specifying
  test tags (in the form of '[<tag>]'), as the CLI::App's
  parse method removes square brackets from cli options.

* Bind cli options to instance members of their subcommand's
  CLI::App derived class.  This seems to fix a bug which caused
  the [static instance] binding source to be destructed twice.

 * Add help text for several subcommands
This commit is contained in:
John Colvin committed 2021-01-16 00:01:15 -06:00
1 parent 45de55a552
commit ca4e477f56
61 files changed
+841 -742

No files matched your search

+5 -4
View File
@@ -85,10 +85,11 @@ void md_report() {
}
}
void AdminReport::register_cmd(CLI::App &app) {
auto *cmd = app.add_subcommand(
"admin-report", "Write a market data entitlement report csv to stdout");
cmd->add_subcommand("md")->callback(md_report);
AdminReport::AdminReport()
: CLI::App(
"Write a market data entitlement report csv to stdout",
"admin-report") {
add_subcommand("md")->callback(md_report);
}
} // namespace bts::cmds
+2 -2
View File
@@ -3,8 +3,8 @@
namespace bts::cmds {
struct AdminReport {
static void register_cmd(CLI::App &);
struct AdminReport : public CLI::App {
AdminReport();
};
} // namespace bts::cmds
+5 -7
View File
@@ -1273,13 +1273,11 @@ void iLinkCGWMidWeekKeyRotation() {
}
}
static void execute() { CGWFailoverFromPrimaryToBackupComponents(); }
void AutoCert::register_cmd(CLI::App &app) {
app.add_subcommand(
"auto-cert",
"Provide control over JetFIX so that we can go through AutoCert")
->callback(execute);
AutoCert::AutoCert()
: CLI::App(
"Provide control over JetFIX so that we can go through AutoCert",
"auto-cert") {
callback(CGWFailoverFromPrimaryToBackupComponents);
}
} // namespace bts::cmds
+2 -2
View File
@@ -3,8 +3,8 @@
namespace bts::cmds {
struct AutoCert {
static void register_cmd(CLI::App &);
struct AutoCert : public CLI::App {
AutoCert();
};
} // namespace bts::cmds
+25 -21
View File
@@ -322,21 +322,6 @@ static void boost_pool_allocator(::benchmark::State &state) {
}
BENCHMARK(boost_pool_allocator);
static std::vector<std::string> args;
static void execute() {
bts_assert(args.size() < 99);
int argc = args.size() + 1;
char *argv[100];
argv[0] = (char *)"spark";
for (size_t i = 0; i < args.size(); i++)
argv[i + 1] = (char *)args[i].c_str();
::benchmark::Initialize(&argc, argv);
::benchmark::RunSpecifiedBenchmarks();
}
static void boost_lockless_queue(::benchmark::State &state) {
using FreeList = boost::lockfree::queue<void *>;
FreeList queue(1000000);
@@ -798,12 +783,31 @@ void concurrent_append_collection_index(::benchmark::State &state) {
}
BENCHMARK(concurrent_append_collection_index);
void Benchmark::register_cmd(CLI::App &app) {
auto *cmd = app.add_subcommand(
"benchmark", "Run micro-benchmarks. Use --benchmark_filter=regex to run "
"only specific benchmarks.");
cmd->add_option("args", args);
cmd->callback(execute);
struct Benchmark::Args {
std::vector<std::string> args;
};
Benchmark::Benchmark()
: CLI::App(
"Run micro-benchmarks. Use --benchmark_filter=regex to run only "
"specific benchmarks.",
"benchmark"),
_args(new Benchmark::Args()) {
add_option("args", _args->args);
callback([&] {
bts_assert(_args->args.size() < 99);
int argc = _args->args.size() + 1;
char *argv[100];
argv[0] = (char *)"spark";
for (size_t i = 0; i < _args->args.size(); i++)
argv[i + 1] = (char *)_args->args[i].c_str();
::benchmark::Initialize(&argc, argv);
::benchmark::RunSpecifiedBenchmarks();
});
}
Benchmark::~Benchmark() {}
} // namespace bts::cmds
+8 -2
View File
@@ -1,10 +1,16 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include <memory>
namespace bts::cmds {
struct Benchmark {
static void register_cmd(CLI::App &);
struct Benchmark : public CLI::App {
Benchmark();
~Benchmark();
private:
struct Args;
std::unique_ptr<Args> _args;
};
} // namespace bts::cmds
+34 -36
View File
@@ -565,49 +565,47 @@ struct TestRunner {
}
};
static void execute() {
// Typically a timestamp
SessionSaver ss;
ss.load();
Config config{"NGG", "004", false, [](auto, auto, auto) {}};
IL3Reactor reactor;
IL3MessageFramer framer;
auto &connection = reactor.register_connection(
{// local dev tunnel
TCPEndpoint{"127.0.0.1", 64530}, TCPEndpoint{"127.0.0.1", 64531},
// Direct connection
TCPEndpoint{"69.50.112.199", 64530},
TCPEndpoint{"69.50.12.205", 64530}});
CMECert::CMECert() : CLI::App("Tool to drive CME Cert", "cme-cert") {
callback([] {
// Typically a timestamp
SessionSaver ss;
ss.load();
Config config{"NGG", "004", false, [](auto, auto, auto) {}};
IL3Reactor reactor;
IL3MessageFramer framer;
auto &connection = reactor.register_connection(
{// local dev tunnel
TCPEndpoint{"127.0.0.1", 64530}, TCPEndpoint{"127.0.0.1", 64531},
// Direct connection
TCPEndpoint{"69.50.112.199", 64530},
TCPEndpoint{"69.50.12.205", 64530}});
// NGG primary - 69.50.112.199 64530
// NGG backup - 69.50.112.205 64530
// NGG primary - 69.50.112.199 64530
// NGG backup - 69.50.112.205 64530
IL3GW gw(connection, framer, nullptr, ss.current, config, {});
gw.subscribe_session([&](auto &s) { ss.session_changed(s); });
gw.connect();
IL3GW gw(connection, framer, nullptr, ss.current, config, {});
gw.subscribe_session([&](auto &s) { ss.session_changed(s); });
gw.connect();
TestRunner run(reactor, gw);
TestRunner run(reactor, gw);
///////////////////////////
//
// The current test
//
///////////////////////////
///////////////////////////
//
// The current test
//
///////////////////////////
run.scratch_space();
run.scratch_space();
///////////////////////////
///////////////////////////
for (auto x = 0; x < 5; ++x) {
reactor.do_work();
sleep(1);
}
gw.send_terminate();
gw.flush();
}
void CMECert::register_cmd(CLI::App &app) {
app.add_subcommand("cme-cert", "Tool to drive CME Cert")->callback(execute);
for (auto x = 0; x < 5; ++x) {
reactor.do_work();
sleep(1);
}
gw.send_terminate();
gw.flush();
});
}
} // namespace bts::cmds
+2 -2
View File
@@ -3,8 +3,8 @@
namespace bts::cmds {
struct CMECert {
void register_cmd(CLI::App &);
struct CMECert : public CLI::App {
CMECert();
};
} // namespace bts::cmds
+14 -11
View File
@@ -3,19 +3,22 @@
namespace bts::cmds {
static std::vector<std::string> args;
struct ConfigCheck::Args {
std::vector<std::string> args;
};
static void execute() {
std::vector<const char *> ch_args;
for (auto const &s : args)
ch_args.push_back(s.c_str());
configcheck(ch_args.size(), &ch_args[0]);
ConfigCheck::ConfigCheck()
: CLI::App("Print current configuration", "config-check"),
_args(new ConfigCheck::Args()) {
add_option("args", _args->args);
callback([&] {
std::vector<const char *> ch_args;
for (auto const &s : _args->args)
ch_args.push_back(s.c_str());
configcheck(ch_args.size(), &ch_args[0]);
});
}
void ConfigCheck::register_cmd(CLI::App &app) {
auto *cmd = app.add_subcommand("config-check");
cmd->add_option("args", args);
cmd->callback(execute);
}
ConfigCheck::~ConfigCheck() {}
} // namespace bts::cmds
+8 -2
View File
@@ -1,10 +1,16 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include <memory>
namespace bts::cmds {
struct ConfigCheck {
static void register_cmd(CLI::App &);
struct ConfigCheck : public CLI::App {
ConfigCheck();
~ConfigCheck();
private:
struct Args;
std::unique_ptr<Args> _args;
};
} // namespace bts::cmds
+2 -7
View File
@@ -3,13 +3,8 @@
namespace bts::cmds {
static void execute() {
bts::sim::start_exchange_sim(bts::api::Venue::VENUE_SIM);
}
void FIXSim::register_cmd(CLI::App &app) {
app.add_subcommand("fix-sim", "Starts a standalone FIX simulator")
->callback(execute);
FIXSim::FIXSim() : CLI::App("Starts a standalone FIX simulator", "fix-sim") {
callback([] { bts::sim::start_exchange_sim(bts::api::Venue::VENUE_SIM); });
}
} // namespace bts::cmds
+2 -2
View File
@@ -3,8 +3,8 @@
namespace bts::cmds {
struct FIXSim {
static void register_cmd(CLI::App &);
struct FIXSim : public CLI::App {
FIXSim();
};
} // namespace bts::cmds
+46 -47
View File
@@ -43,7 +43,7 @@ using namespace bts::api::auth;
using namespace bts::crypto;
using namespace bts::config;
struct EncodeArgs {
struct EncryptArgs {
std::string keyfile;
std::string pass;
std::string users;
@@ -69,12 +69,6 @@ struct KeygenArgs {
std::string expiration;
};
struct Args {
EncodeArgs encrypt;
DecryptArgs decrypt;
KeygenArgs keygen;
};
std::string parse_date(std::string const &date) {
auto the_date = bts::datetime::parse_iso_date_time_string(date);
return bts::datetime::iso_date_time_string(the_date);
@@ -115,8 +109,6 @@ void write_keycode(
kc.set_allow_concurrent(concurrent);
}
static Args args;
void check_ips() {
CryptoInit::initialize();
auto ipmap = bts::os::host_ips();
@@ -124,18 +116,18 @@ void check_ips() {
std::cout << x.first << " => " << x.second << std::endl;
}
void encrypt() {
void encrypt(const EncryptArgs &args) {
CryptoInit::initialize();
std::string protobuf;
std::string kc_string;
std::stringstream oss, encrypted;
CachedPassword cache(args.encrypt.pass);
PrivateKeyPtr key = PrivateKey::read_file(args.encrypt.keyfile, &cache);
CachedPassword cache(args.pass);
PrivateKeyPtr key = PrivateKey::read_file(args.keyfile, &cache);
bts::config::KeyCode kc;
write_keycode(
args.encrypt.users, args.encrypt.expiration, args.encrypt.ip,
args.encrypt.port, args.encrypt.allow_concurrent, kc);
args.users, args.expiration, args.ip, args.port, args.allow_concurrent,
kc);
if (!google::protobuf::TextFormat::PrintToString(kc, &kc_string))
return;
oss << kc_string;
@@ -149,7 +141,7 @@ void encrypt() {
std::string pb;
google::protobuf::TextFormat::PrintToString(ekc, &pb);
std::cout << std::endl << std::endl;
std::cout << "#nusers: " << args.encrypt.users << std::endl;
std::cout << "#nusers: " << args.users << std::endl;
std::cout << "#allow concurrent : "
<< (kc.allow_concurrent() ? "true" : "false") << std::endl;
std::cout << "#expires: "
@@ -168,14 +160,13 @@ void encrypt() {
std::cout << std::endl << std::endl;
}
void decrypt() {
void decrypt(const DecryptArgs &args) {
CryptoInit::initialize();
if (args.decrypt.keyfile != "") {
PublicKeyPtr pub =
bts::crypto::PublicKey::read_file(args.decrypt.keyfile);
if (pub && args.decrypt.source != "") {
if (args.keyfile != "") {
PublicKeyPtr pub = bts::crypto::PublicKey::read_file(args.keyfile);
if (pub && args.source != "") {
bts::config::EncryptedKeyCode ekc;
std::fstream fs(args.decrypt.source);
std::fstream fs(args.source);
std::string str(
(std::istreambuf_iterator<char>(fs)),
std::istreambuf_iterator<char>());
@@ -195,12 +186,12 @@ void decrypt() {
}
}
void keygen() {
void keygen(const KeygenArgs &args) {
CryptoInit::initialize();
PrivateKeyPtr priv = PrivateKey::create(2048, AlgoType::RSA);
std::string n1 = (args.keygen.keyfile + "_private.pem"),
n2 = (args.keygen.keyfile + "_public.pem");
std::string n1 = (args.keyfile + "_private.pem"),
n2 = (args.keyfile + "_public.pem");
// I can't get anything but DES to work here.
priv->write_file(n1, CipherType::DES);
priv->public_key()->write_file(n2);
@@ -208,37 +199,45 @@ void keygen() {
std::cout << "Key pair generated " << n1 << " and " << n2 << std::endl;
}
void KeyCode::register_cmd(CLI::App &app) {
auto *cmd = app.add_subcommand("key-code");
cmd->require_subcommand();
struct KeyCode::Args {
EncryptArgs encrypt;
DecryptArgs decrypt;
KeygenArgs keygen;
};
KeyCode::KeyCode() : CLI::App("key-code"), _args(new KeyCode::Args()) {
require_subcommand();
auto *cmd = this;
// check ips
cmd->add_subcommand("check-ips")->callback(check_ips);
// decrypt
auto *decrypt_cmd = cmd->add_subcommand("decrypt");
decrypt_cmd->add_option("--expiration", args.decrypt.expiration);
decrypt_cmd->add_option("--key-file", args.decrypt.keyfile)->required();
decrypt_cmd->add_option("--pass", args.decrypt.pass);
decrypt_cmd->add_option("--source", args.decrypt.source);
decrypt_cmd->add_option("--users", args.decrypt.users);
decrypt_cmd->callback(decrypt);
decrypt_cmd->add_option("--expiration", _args->decrypt.expiration);
decrypt_cmd->add_option("--key-file", _args->decrypt.keyfile)->required();
decrypt_cmd->add_option("--pass", _args->decrypt.pass);
decrypt_cmd->add_option("--source", _args->decrypt.source);
decrypt_cmd->add_option("--users", _args->decrypt.users);
decrypt_cmd->callback([&] { decrypt(_args->decrypt); });
// encrypt
auto *encrypt_cmd = cmd->add_subcommand("encrypt");
encrypt_cmd->add_flag("--allow-concurrent", args.encrypt.allow_concurrent);
encrypt_cmd->add_option("--expiration", args.encrypt.expiration);
encrypt_cmd->add_option("--ip", args.encrypt.ip);
encrypt_cmd->add_option("--key-file", args.encrypt.keyfile)->required();
encrypt_cmd->add_option("--pass", args.encrypt.pass);
encrypt_cmd->add_option("--port", args.encrypt.port);
encrypt_cmd->add_option("--source", args.encrypt.source);
encrypt_cmd->add_option("--users", args.encrypt.users);
encrypt_cmd->callback(encrypt);
encrypt_cmd->add_flag("--allow-concurrent", _args->encrypt.allow_concurrent);
encrypt_cmd->add_option("--expiration", _args->encrypt.expiration);
encrypt_cmd->add_option("--ip", _args->encrypt.ip);
encrypt_cmd->add_option("--key-file", _args->encrypt.keyfile)->required();
encrypt_cmd->add_option("--pass", _args->encrypt.pass);
encrypt_cmd->add_option("--port", _args->encrypt.port);
encrypt_cmd->add_option("--source", _args->encrypt.source);
encrypt_cmd->add_option("--users", _args->encrypt.users);
encrypt_cmd->callback([&] { encrypt(_args->encrypt); });
// key-gen
auto *keygen_cmd = cmd->add_subcommand("key-gen");
keygen_cmd->add_option("--expiration", args.keygen.expiration);
keygen_cmd->add_option("--key-file", args.keygen.keyfile)->required();
keygen_cmd->add_option("--pass", args.keygen.pass);
keygen_cmd->add_option("--users", args.keygen.users);
keygen_cmd->callback(keygen);
keygen_cmd->add_option("--expiration", _args->keygen.expiration);
keygen_cmd->add_option("--key-file", _args->keygen.keyfile)->required();
keygen_cmd->add_option("--pass", _args->keygen.pass);
keygen_cmd->add_option("--users", _args->keygen.users);
keygen_cmd->callback([&] { keygen(_args->keygen); });
}
KeyCode::~KeyCode() {}
} // namespace bts::cmds
+8 -2
View File
@@ -1,5 +1,6 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include <memory>
namespace bts::crypto {
@@ -16,8 +17,13 @@ class KeyCode;
namespace bts::cmds {
struct KeyCode {
static void register_cmd(CLI::App &);
struct KeyCode : public CLI::App {
KeyCode();
~KeyCode();
private:
struct Args;
std::unique_ptr<Args> _args;
};
} // namespace bts::cmds
+8 -7
View File
@@ -4,14 +4,15 @@
namespace bts::cmds {
static std::vector<std::string> args;
struct Md::Args {
std::vector<std::string> args;
};
static void execute() { bts::md::md_tool(args); }
void Md::register_cmd(CLI::App &app) {
auto *cmd = app.add_subcommand("md");
cmd->add_option("args", args);
cmd->callback(execute);
Md::Md() : CLI::App("", "md"), _args(new Md::Args()) {
add_option("args", _args->args);
callback([&] { md::md_tool(_args->args); });
}
Md::~Md() {}
} // namespace bts::cmds
+8 -2
View File
@@ -1,10 +1,16 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include <memory>
namespace bts::cmds {
struct Md {
static void register_cmd(CLI::App &);
struct Md : public CLI::App {
Md();
~Md();
private:
struct Args;
std::unique_ptr<Args> _args;
};
} // namespace bts::cmds
+48 -44
View File
@@ -396,58 +396,62 @@ void print_orders(
}
}
static std::vector<std::string> args;
struct ParseHistoricalFix::Args {
std::vector<std::string> args;
};
static void execute() {
std::ifstream input(args[0]);
std::ofstream books_stream(args[0] + "_books.csv");
std::ofstream msgs_stream(args[0] + "_msgs.csv");
std::ofstream trades_stream(args[0] + "_trades.csv");
std::ofstream orders_stream(args[0] + "_orders.csv");
ParseHistoricalFix::ParseHistoricalFix()
: CLI::App(
"ParseHistoricalFIX <input> will produce 2 files, "
"<input>_mdentries.csv and <input>order_entries.csv",
"parse-historical-fix"),
_args(new ParseHistoricalFix::Args()) {
add_option("args", _args->args);
callback([&] {
auto &args = _args->args;
std::ifstream input(args[0]);
std::ofstream books_stream(args[0] + "_books.csv");
std::ofstream msgs_stream(args[0] + "_msgs.csv");
std::ofstream trades_stream(args[0] + "_trades.csv");
std::ofstream orders_stream(args[0] + "_orders.csv");
std::unordered_map<std::string, Book> books;
TradeSummaryEvent current_trade_summary;
std::unordered_map<std::string, Book> books;
TradeSummaryEvent current_trade_summary;
MDIncrementalUpdate update;
FIXParser<MDIncrementalUpdate> parser;
MDIncrementalUpdate update;
FIXParser<MDIncrementalUpdate> parser;
std::string line;
size_t c = 0;
auto ok = 0;
auto start = Timer::now();
std::string line;
size_t c = 0;
auto ok = 0;
auto start = Timer::now();
print_book_header(books_stream);
print_msg_headers(msgs_stream);
print_trades_header(trades_stream);
print_orders_header(orders_stream);
print_book_header(books_stream);
print_msg_headers(msgs_stream);
print_trades_header(trades_stream);
print_orders_header(orders_stream);
while (std::getline(input, line)) {
c++;
auto parsed_ptr = parser.parse(InMsgBuffer(
line.data(), line.size(), std::shared_ptr<std::string>{}));
if (parsed_ptr) {
auto msg =
std::dynamic_pointer_cast<const MDIncrementalUpdate>(parsed_ptr);
update_and_maybe_print_book(*msg, books, books_stream);
print_msgs(c, *msg, msgs_stream);
update_and_maybe_print_trades(
*msg, current_trade_summary, trades_stream);
print_orders(*msg, orders_stream);
while (std::getline(input, line)) {
c++;
auto parsed_ptr = parser.parse(InMsgBuffer(
line.data(), line.size(), std::shared_ptr<std::string>{}));
if (parsed_ptr) {
auto msg = std::dynamic_pointer_cast<const MDIncrementalUpdate>(
parsed_ptr);
update_and_maybe_print_book(*msg, books, books_stream);
print_msgs(c, *msg, msgs_stream);
update_and_maybe_print_trades(
*msg, current_trade_summary, trades_stream);
print_orders(*msg, orders_stream);
}
ok += (bool)(parsed_ptr);
}
ok += (bool)(parsed_ptr);
}
auto sec_duration = sec_diff(Timer::now(), start);
std::cout << (int)(c / 1000 / sec_duration) << " Km/sec parsed: " << ok
<< " out of " << c << " time: " << sec_duration << " secs\n";
auto sec_duration = sec_diff(Timer::now(), start);
std::cout << (int)(c / 1000 / sec_duration) << " Km/sec parsed: " << ok
<< " out of " << c << " time: " << sec_duration << " secs\n";
});
}
void ParseHistoricalFix::register_cmd(CLI::App &app) {
auto *cmd = app.add_subcommand(
"parse-historical-fix",
"ParseHistoricalFIX <input> will produce 2 files, "
"<input>_mdentries.csv and <input>order_entries.csv");
cmd->add_option("args", args);
cmd->callback(execute);
}
ParseHistoricalFix::~ParseHistoricalFix() {}
} // namespace bts::cmds
+8 -2
View File
@@ -1,10 +1,16 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include <memory>
namespace bts::cmds {
struct ParseHistoricalFix {
static void register_cmd(CLI::App &);
struct ParseHistoricalFix : public CLI::App {
ParseHistoricalFix();
~ParseHistoricalFix();
private:
struct Args;
std::unique_ptr<Args> _args;
};
} // namespace bts::cmds
+45 -46
View File
@@ -5,53 +5,52 @@
namespace bts::cmds {
static void execute() {
persistence::SqliteStore order_store("orders");
auto all_orders = order_store.retrieve_all()["orders"];
std::map<uint32_t, std::pair<std::string, double>> fill_times;
persistence::SqliteStore fill_store("fills");
auto all_fills = fill_store.retrieve_all()["fills"];
for (auto const &kvp : all_fills) {
api::orderman::Fill fill_rec;
if (utils::pb::from_string<api::orderman::Fill>(kvp.second, &fill_rec)) {
auto ts = utils::pb::to_double_time(fill_rec.timestamp());
auto &p = fill_times[fill_rec.order_key()];
auto &min_time = p.second;
if (min_time < ts) {
p.first = kvp.first;
min_time = ts;
}
}
}
std::cout << "order_key,time_to_first_fill,fill_id"
<< "\n";
api::orderman::OrderRecord rec;
for (auto const &kvp : all_orders)
if (utils::pb::from_string<api::orderman::OrderRecord>(
kvp.second, &rec)) {
auto &def = rec.def();
auto &state = rec.state();
if (rec.parent_key() && def.custom_def().has_limit() &&
def.sub_type() ==
api::orderman::OrderSubType::ORDER_SUB_TYPE_HEDGING_LEG) {
auto create_ts = utils::pb::to_double_time(state.created());
auto &first_fill = fill_times[rec.key()];
auto delta = first_fill.second - create_ts;
if (first_fill.first.size())
std::cout << kvp.first << "," << delta << "," << first_fill.first
<< "\n";
}
}
}
void PrintHedgeOrders::register_cmd(CLI::App &app) {
app.add_subcommand(
"print-hedge-orders",
PrintHedgeOrders::PrintHedgeOrders()
: CLI::App(
"Print out order ids for all hedge orders"
"which were completed at least one second being created")
->callback(execute);
"which were completed at least one second being created",
"print-hedge-orders") {
callback([&] {
persistence::SqliteStore order_store("orders");
auto all_orders = order_store.retrieve_all()["orders"];
std::map<uint32_t, std::pair<std::string, double>> fill_times;
persistence::SqliteStore fill_store("fills");
auto all_fills = fill_store.retrieve_all()["fills"];
for (auto const &kvp : all_fills) {
api::orderman::Fill fill_rec;
if (utils::pb::from_string<api::orderman::Fill>(
kvp.second, &fill_rec)) {
auto ts = utils::pb::to_double_time(fill_rec.timestamp());
auto &p = fill_times[fill_rec.order_key()];
auto &min_time = p.second;
if (min_time < ts) {
p.first = kvp.first;
min_time = ts;
}
}
}
std::cout << "order_key,time_to_first_fill,fill_id"
<< "\n";
api::orderman::OrderRecord rec;
for (auto const &kvp : all_orders)
if (utils::pb::from_string<api::orderman::OrderRecord>(
kvp.second, &rec)) {
auto &def = rec.def();
auto &state = rec.state();
if (rec.parent_key() && def.custom_def().has_limit() &&
def.sub_type() ==
api::orderman::OrderSubType::ORDER_SUB_TYPE_HEDGING_LEG) {
auto create_ts = utils::pb::to_double_time(state.created());
auto &first_fill = fill_times[rec.key()];
auto delta = first_fill.second - create_ts;
if (first_fill.first.size())
std::cout << kvp.first << "," << delta << ","
<< first_fill.first << "\n";
}
}
});
}
} // namespace bts::cmds
+2 -2
View File
@@ -3,8 +3,8 @@
namespace bts::cmds {
struct PrintHedgeOrders {
static void register_cmd(CLI::App &);
struct PrintHedgeOrders : public CLI::App {
PrintHedgeOrders();
};
} // namespace bts::cmds
+6 -5
View File
@@ -41,11 +41,12 @@ static void execute() {
// put test code here
}
void QuickTest::register_cmd(CLI::App &app) {
app.add_subcommand(
"quick-test", "This is a place holder to make it easy to quickly "
"write discardable test code")
->callback(execute);
QuickTest::QuickTest()
: CLI::App(
"This is a place holder to make it easy to quickly "
"write discardable test code",
"quick-test") {
callback(execute);
}
} // namespace bts::cmds
+2 -2
View File
@@ -3,8 +3,8 @@
namespace bts::cmds {
struct QuickTest {
static void register_cmd(CLI::App &);
struct QuickTest : public CLI::App {
QuickTest();
};
} // namespace bts::cmds
+2 -2
View File
@@ -123,8 +123,8 @@ static void execute() {
quickbus.run();
}
void Run::register_cmd(CLI::App &app) {
app.add_subcommand("run", "Run default sparkd services")->callback(execute);
Run::Run() : CLI::App("Run default sparkd services", "run") {
callback(execute);
}
} // namespace bts::cmds
+2 -2
View File
@@ -3,8 +3,8 @@
namespace bts::cmds {
struct Run {
static void register_cmd(CLI::App &);
struct Run : public CLI::App {
Run();
};
} // namespace bts::cmds
+2 -3
View File
@@ -56,9 +56,8 @@ static void execute() {
disp.run();
}
void Span::register_cmd(CLI::App &app) {
app.add_subcommand("span", "Test CME CORE Span integration")
->callback(execute);
Span::Span() : CLI::App("Test CME CORE Span integration", "span") {
callback(execute);
}
} // namespace bts::cmds
+2 -2
View File
@@ -3,8 +3,8 @@
namespace bts::cmds {
struct Span {
static void register_cmd(CLI::App &);
struct Span : public CLI::App {
Span();
};
} // namespace bts::cmds
+18 -18
View File
@@ -16,24 +16,24 @@
namespace bts::cmds {
void Spartan::register_cmd(CLI::App &app) {
auto *spartan_app =
app.add_subcommand("spartan", "Futures trading system for devs");
// register commands
bts::spartan::cmds::BookStats::register_cmd(*spartan_app);
bts::spartan::cmds::Compress::register_cmd(*spartan_app);
bts::spartan::cmds::CSV::register_cmd(*spartan_app);
bts::spartan::cmds::FindPolys::register_cmd(*spartan_app);
bts::spartan::cmds::Icebergs::register_cmd(*spartan_app);
bts::spartan::cmds::Json::register_cmd(*spartan_app);
bts::spartan::cmds::ListSec::register_cmd(*spartan_app);
bts::spartan::cmds::PolygonBooks::register_cmd(*spartan_app);
bts::spartan::cmds::Sim::register_cmd(*spartan_app);
bts::spartan::cmds::SimBooks::register_cmd(*spartan_app);
bts::spartan::cmds::SQL::register_cmd(*spartan_app);
bts::spartan::cmds::Stats::register_cmd(*spartan_app);
bts::spartan::cmds::ValidateFifo::register_cmd(*spartan_app);
bts::spartan::cmds::ValuePolys::register_cmd(*spartan_app);
using namespace spartan::cmds;
Spartan::Spartan() : CLI::App("Futures trading system for devs", "spartan") {
// subcommands
add_subcommand(CLI::App_p(new BookStats()));
add_subcommand(CLI::App_p(new Compress()));
add_subcommand(CLI::App_p(new CSV()));
add_subcommand(CLI::App_p(new FindPolys()));
add_subcommand(CLI::App_p(new Icebergs()));
add_subcommand(CLI::App_p(new Json()));
add_subcommand(CLI::App_p(new ListSec()));
add_subcommand(CLI::App_p(new PolygonBooks()));
add_subcommand(CLI::App_p(new Sim()));
add_subcommand(CLI::App_p(new SimBooks()));
add_subcommand(CLI::App_p(new SQL()));
add_subcommand(CLI::App_p(new Stats()));
add_subcommand(CLI::App_p(new ValidateFifo()));
add_subcommand(CLI::App_p(new ValuePolys()));
}
} // namespace bts::cmds
+2 -2
View File
@@ -3,8 +3,8 @@
namespace bts::cmds {
struct Spartan {
static void register_cmd(CLI::App &);
struct Spartan : public CLI::App {
Spartan();
};
} // namespace bts::cmds
-19
View File
@@ -1,19 +0,0 @@
#include "Test.h"
#include "lib/testing/catch_test_runner.h"
namespace bts::cmds {
static std::vector<std::string> args;
static void execute() { run_tests(args); }
void Test::register_cmd(CLI::App &app) {
auto *cmd = app.add_subcommand(
"test",
"See "
"https://github.com/philsquared/Catch/blob/master/docs/command-line.md");
cmd->add_option("args", args);
cmd->callback(execute);
}
} // namespace bts::cmds
-10
View File
@@ -1,10 +0,0 @@
#pragma once
#include <CLI11/CLI11.hpp>
namespace bts::cmds {
struct Test {
static void register_cmd(CLI::App &);
};
} // namespace bts::cmds
+30 -30
View File
@@ -22,45 +22,45 @@
namespace bts::cmds {
struct Args {
struct Web::Args {
bool md;
};
static Args args;
Web::Web()
: CLI::App(
"Run only the services required for the web admin. This cannot be "
"run concurrently with 'sparkd run.'",
"web"),
_args(new Web::Args()) {
add_flag("--md", _args->md, "Start market data");
callback([&] {
config::QuickbusNodeCfg qb_cfg;
auto &ports = config::get<config::Ports>();
qb_cfg.set_my_port(ports.qbus_port());
qbus::Quickbus quickbus(qb_cfg);
api::ServiceContext sc(&quickbus);
static void execute() {
config::QuickbusNodeCfg qb_cfg;
auto &ports = config::get<config::Ports>();
qb_cfg.set_my_port(ports.qbus_port());
qbus::Quickbus quickbus(qb_cfg);
api::ServiceContext sc(&quickbus);
metrics::MetricsContainer stats;
metrics::publish_system_metrics(stats);
trace::PerfLog perf_log;
http::SparkWebServer webserver(sc, stats, perf_log, ports.config_port());
metrics::MetricsContainer stats;
metrics::publish_system_metrics(stats);
trace::PerfLog perf_log;
http::SparkWebServer webserver(sc, stats, perf_log, ports.config_port());
user::AuthServiceServer login_server(sc);
if (_args->md) {
md::run_market_datapublishers(nullptr, nullptr);
}
user::AuthServiceServer login_server(sc);
if (args.md) {
md::run_market_datapublishers(nullptr, nullptr);
}
api::synthetic::SyntheticSecurityService synthetic_service(sc);
md::MDClient mdclient(&utils::Dispatcher::current(), &synthetic_service);
md::QueryServerPtr md_query_server(
md::run_query_server(sc, mdclient, stats));
api::synthetic::SyntheticSecurityService synthetic_service(sc);
md::MDClient mdclient(&utils::Dispatcher::current(), &synthetic_service);
md::QueryServerPtr md_query_server(
md::run_query_server(sc, mdclient, stats));
admin::AdminServiceServer admin_service(sc);
admin::AdminServiceServer admin_service(sc);
utils::Dispatcher::current().run();
utils::Dispatcher::current().run();
});
}
void Web::register_cmd(CLI::App &app) {
auto *cmd = app.add_subcommand(
"web", "Run only the services required for the web admin. This cannot "
"be run concurrently with 'sparkd run.'");
cmd->add_flag("--md", args.md, "Start market data");
cmd->callback(execute);
}
Web::~Web() {}
} // namespace bts::cmds
+8 -2
View File
@@ -1,10 +1,16 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include <memory>
namespace bts::cmds {
struct Web {
static void register_cmd(CLI::App &);
struct Web : public CLI::App {
Web();
~Web();
private:
struct Args;
std::unique_ptr<Args> _args;
};
} // namespace bts::cmds
@@ -7,26 +7,28 @@
namespace bts::spartan::cmds {
struct Args {
struct BookStats::Args {
std::string pcap;
std::string out;
};
static Args args;
static void execute() {
if (args.out.empty())
args.out = file::change_file_ext(args.pcap, "book_stats.csv", "pcap.xz");
std::ofstream out(args.out);
mdp3::BookStatRecorder bsr(out);
mdp3::observe_channel(args.pcap, bsr);
BookStats::BookStats()
: CLI::App(
"Generate a csv file of various statistics about book updates",
"book-stats"),
_args(new BookStats::Args()) {
add_option("pcap,--pcap", _args->pcap)->required();
add_option("out,--out", _args->out);
callback([&] {
if (_args->out.empty())
_args->out =
file::change_file_ext(_args->pcap, "book_stats.csv", "pcap.xz");
std::ofstream out(_args->out);
mdp3::BookStatRecorder bsr(out);
mdp3::observe_channel(_args->pcap, bsr);
});
}
void BookStats::register_cmd(CLI::App &app) {
auto *subcmd = app.add_subcommand("book-stats");
subcmd->add_option("pcap,--pcap", args.pcap)->required();
subcmd->add_option("out,--out", args.out);
subcmd->callback(execute);
}
BookStats::~BookStats() {}
} // namespace bts::spartan::cmds
@@ -1,10 +1,16 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include <memory>
namespace bts::spartan::cmds {
struct BookStats {
static void register_cmd(CLI::App &);
struct BookStats : public CLI::App {
BookStats();
~BookStats();
private:
struct Args;
std::unique_ptr<Args> _args;
};
} // namespace bts::spartan::cmds
+8 -9
View File
@@ -3,18 +3,17 @@
namespace bts::spartan::cmds {
struct Args {
struct CSV::Args {
std::string pcap;
};
static Args args;
static void execute() { mdp3::to_csv(args.pcap); }
void CSV::register_cmd(CLI::App &app) {
auto *cmd = app.add_subcommand("csv");
cmd->add_option("pcap,--pcap", args.pcap)->required();
cmd->callback(execute);
CSV::CSV()
: CLI::App("Generate csv files for various pcap-derived tables", "csv"),
_args(new CSV::Args()) {
add_option("pcap,--pcap", _args->pcap)->required();
callback([&] { mdp3::to_csv(_args->pcap); });
}
CSV::~CSV() {}
} // namespace bts::spartan::cmds
+8 -2
View File
@@ -1,10 +1,16 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include <memory>
namespace bts::spartan::cmds {
struct CSV {
static void register_cmd(CLI::App &);
struct CSV : public CLI::App {
CSV();
~CSV();
private:
struct Args;
std::unique_ptr<Args> _args;
};
} // namespace bts::spartan::cmds
@@ -3,20 +3,17 @@
namespace bts::spartan::cmds {
struct Args {
struct Compress::Args {
std::string pcap;
std::string out;
};
static Args args;
static void execute() { spartan::mdp3::compress(args.pcap, args.out); }
void Compress::register_cmd(CLI::App &app) {
auto *subcmd = app.add_subcommand("compress");
subcmd->add_option("pcap,--pcap", args.pcap)->required();
subcmd->add_option("out,--out", args.out)->required();
subcmd->callback(execute);
Compress::Compress() : CLI::App("", "compress"), _args(new Compress::Args()) {
add_option("pcap,--pcap", _args->pcap)->required();
add_option("out,--out", _args->out);
callback([&] { mdp3::compress(_args->pcap, _args->out); });
}
Compress::~Compress() {}
} // namespace bts::spartan::cmds
@@ -1,10 +1,16 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include <memory>
namespace bts::spartan::cmds {
struct Compress {
static void register_cmd(CLI::App &);
struct Compress : public CLI::App {
Compress();
~Compress();
private:
struct Args;
std::unique_ptr<Args> _args;
};
} // namespace bts::spartan::cmds
@@ -9,44 +9,43 @@ namespace bts::spartan::cmds {
const int UNSET = -1;
struct Args {
struct FindPolys::Args {
std::string pcap;
int active_threshold_fills = UNSET;
int max_vertices = UNSET;
int max_securities = UNSET;
};
static Args args;
static void execute() {
// init options (from args)
mdp3::PolygonFinder::Options options;
if (args.active_threshold_fills != UNSET)
options.active_threshold_fills = args.active_threshold_fills;
if (args.max_vertices != UNSET)
options.max_vertices = args.max_vertices;
if (args.max_securities != UNSET)
options.max_securities = args.max_securities;
mdp3::DefinitionCollector def_collector;
mdp3::PolygonFinder poly_finder(options);
mdp3::observe_channel(args.pcap, {&poly_finder, &def_collector});
for (size_t id = 1; id <= poly_finder.result.size(); id++)
std::cout << id << " -> "
<< mdp3::polygon_descriptor(
poly_finder.result[id], def_collector.ptr_result)
<< std::endl;
}
void FindPolys::register_cmd(CLI::App &app) {
auto *subcmd = app.add_subcommand("find-polys");
subcmd->add_option("pcap,--pcap", args.pcap)->required();
subcmd->add_option(
FindPolys::FindPolys()
: CLI::App("Find polygons in a pcap file", "find-polys"),
_args(new FindPolys::Args()) {
add_option("pcap,--pcap", _args->pcap)->required();
add_option(
"active-threshold-fills,--active-threshold-fills",
args.active_threshold_fills);
subcmd->add_option("max-vertices,--max-vertices", args.max_vertices);
subcmd->add_option("max-securities,--max-securities", args.max_securities);
subcmd->callback(execute);
_args->active_threshold_fills);
add_option("max-vertices,--max-vertices", _args->max_vertices);
add_option("max-securities,--max-securities", _args->max_securities);
callback([&] {
// init options (from _args->
mdp3::PolygonFinder::Options options;
if (_args->active_threshold_fills != UNSET)
options.active_threshold_fills = _args->active_threshold_fills;
if (_args->max_vertices != UNSET)
options.max_vertices = _args->max_vertices;
if (_args->max_securities != UNSET)
options.max_securities = _args->max_securities;
mdp3::DefinitionCollector def_collector;
mdp3::PolygonFinder poly_finder(options);
mdp3::observe_channel(_args->pcap, {&poly_finder, &def_collector});
for (size_t id = 1; id <= poly_finder.result.size(); id++)
std::cout << id << " -> "
<< mdp3::polygon_descriptor(
poly_finder.result[id], def_collector.ptr_result)
<< std::endl;
});
}
FindPolys::~FindPolys() {}
} // namespace bts::spartan::cmds
@@ -1,10 +1,16 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include <memory>
namespace bts::spartan::cmds {
struct FindPolys {
static void register_cmd(CLI::App &);
struct FindPolys : public CLI::App {
FindPolys();
~FindPolys();
private:
struct Args;
std::unique_ptr<Args> _args;
};
} // namespace bts::spartan::cmds
@@ -10,42 +10,42 @@
namespace bts::spartan::cmds {
struct Args {
struct Icebergs::Args {
std::string pcap;
std::string out;
};
static Args args;
static void execute() {
const char *ok_col = "ok";
const char *eid_col = "eid";
const char *total_col = "total_qty";
const char *show_qty_col = "show_qty";
const char *fills_col = "fill_qty";
if (args.out.empty())
args.out = file::change_file_ext(args.pcap, "icebergs.csv", "pcap.xz");
std::ofstream out(args.out);
auto ibs = mdp3::IcebergFinder::find_all(args.pcap);
// write cols
out << ok_col << "," << eid_col << "," << total_col << "," << show_qty_col
<< "," << fills_col;
// write rows
for (auto &i : ibs) {
auto relevent_eids = i.second.all_eids();
for (auto eid : relevent_eids)
out << i.first.value() << "," << eid << ","
<< i.second.total_qty().value() << ","
<< std::get<1>(i.second.show_qty_at(eid)) << ","
<< i.second.count_fills_by(eid) << "\n";
}
Icebergs::Icebergs()
: CLI::App("Generate a csv file of iceberg orders in a pcap", "icebergs"),
_args(new Icebergs::Args()) {
add_option("pcap,--pcap", _args->pcap)->required();
add_option("out,--out", _args->out);
callback([&] {
const char *ok_col = "ok";
const char *eid_col = "eid";
const char *total_col = "total_qty";
const char *show_qty_col = "show_qty";
const char *fills_col = "fill_qty";
if (_args->out.empty())
_args->out =
file::change_file_ext(_args->pcap, "icebergs.csv", "pcap.xz");
std::ofstream out(_args->out);
auto ibs = mdp3::IcebergFinder::find_all(_args->pcap);
// write cols
out << ok_col << "," << eid_col << "," << total_col << "," << show_qty_col
<< "," << fills_col;
// write rows
for (auto &i : ibs) {
auto relevent_eids = i.second.all_eids();
for (auto eid : relevent_eids)
out << i.first.value() << "," << eid << ","
<< i.second.total_qty().value() << ","
<< std::get<1>(i.second.show_qty_at(eid)) << ","
<< i.second.count_fills_by(eid) << "\n";
}
});
}
void Icebergs::register_cmd(CLI::App &app) {
auto *subcmd = app.add_subcommand("icebergs");
subcmd->add_option("pcap,--pcap", args.pcap)->required();
subcmd->add_option("out,--out", args.out);
subcmd->callback(execute);
}
Icebergs::~Icebergs() {}
} // namespace bts::spartan::cmds
@@ -1,10 +1,16 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include <memory>
namespace bts::spartan::cmds {
struct Icebergs {
static void register_cmd(CLI::App &);
struct Icebergs : public CLI::App {
Icebergs();
~Icebergs();
private:
struct Args;
std::unique_ptr<Args> _args;
};
} // namespace bts::spartan::cmds
+11 -12
View File
@@ -3,24 +3,23 @@
namespace bts::spartan::cmds {
struct Args {
struct Json::Args {
uint32_t start_packet = 0;
uint32_t packet_count = 0;
std::string pcap{};
};
static Args args;
static void execute() {
spartan::mdp3::to_json(args.pcap, args.start_packet, args.packet_count);
Json::Json()
: CLI::App("Generate a json file for data packets in a pcap", "json"),
_args(new Json::Args()) {
add_option("pcap,--pcap", _args->pcap)->required();
add_option("start-packet,--start-packet", _args->start_packet);
add_option("packet-count,--packet-count", _args->packet_count);
callback([&] {
mdp3::to_json(_args->pcap, _args->start_packet, _args->packet_count);
});
}
void Json::register_cmd(CLI::App &spartan_app) {
auto *subcmd = spartan_app.add_subcommand("json");
subcmd->add_option("pcap,--pcap", args.pcap)->required();
subcmd->add_option("start-packet,--start-packet", args.start_packet);
subcmd->add_option("packet-count,--packet-count", args.packet_count);
subcmd->callback(execute);
}
Json::~Json() {}
} // namespace bts::spartan::cmds
+8 -2
View File
@@ -1,10 +1,16 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include <memory>
namespace bts::spartan::cmds {
struct Json {
static void register_cmd(CLI::App &);
struct Json : public CLI::App {
Json();
~Json();
private:
struct Args;
std::unique_ptr<Args> _args;
};
} // namespace bts::spartan::cmds
+21 -24
View File
@@ -6,35 +6,32 @@
namespace bts::spartan::cmds {
struct Args {
struct ListSec::Args {
std::string pcap;
};
static Args args;
static void execute() {
mdp3::DefinitionCollector def;
mdp3::FillCounter fill_counter;
mdp3::observe_channel(args.pcap, {&def, &fill_counter});
auto fill_counts = fill_counter.result_map();
for (auto const &s : def.result) {
std::cout << s.second.asset() << "|" << s.second.symbol() << "|"
<< fill_counts[s.second.security_id()];
for (auto const &l : s.second.legs()) {
std::cout << "|";
if (def.ptr_result[l.security_id])
std::cout << def.ptr_result[l.security_id]->symbol();
else
std::cout << "?" << l.security_id;
ListSec::ListSec() : CLI::App("List securities in a pcap file", "list-sec"), _args(new ListSec::Args()) {
add_option("pcap,--pcap", _args->pcap)->required();
callback([&] {
mdp3::DefinitionCollector def;
mdp3::FillCounter fill_counter;
mdp3::observe_channel(_args->pcap, {&def, &fill_counter});
auto fill_counts = fill_counter.result_map();
for (auto const &s : def.result) {
std::cout << s.second.asset() << "|" << s.second.symbol() << "|"
<< fill_counts[s.second.security_id()];
for (auto const &l : s.second.legs()) {
std::cout << "|";
if (def.ptr_result[l.security_id])
std::cout << def.ptr_result[l.security_id]->symbol();
else
std::cout << "?" << l.security_id;
}
std::cout << std::endl;
}
std::cout << std::endl;
}
});
}
void ListSec::register_cmd(CLI::App &app) {
auto *subcmd = app.add_subcommand("list-sec");
subcmd->add_option("pcap,--pcap", args.pcap)->required();
subcmd->callback(execute);
}
ListSec::~ListSec() {}
} // namespace bts::spartan::cmds
@@ -1,10 +1,16 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include <memory>
namespace bts::spartan::cmds {
struct ListSec {
static void register_cmd(CLI::App &);
struct ListSec : public CLI::App {
ListSec();
~ListSec();
private:
struct Args;
std::unique_ptr<Args> _args;
};
} // namespace bts::spartan::cmds
@@ -18,103 +18,105 @@ namespace bts::spartan::cmds {
#define WARN(...) LOG(OrderMan, Warn, __VA_ARGS__)
#define ERROR(...) LOG(OrderMan, Error, __VA_ARGS__)
struct Args {
struct PolygonBooks::Args {
std::string pcap;
std::string db;
std::vector<int> security_ids;
std::string out_csv;
};
static Args args;
PolygonBooks::PolygonBooks()
: CLI::App(
"Generate a csv file of book updates for polygons", "polygon-books"),
_args(new PolygonBooks::Args()) {
add_option("pcap,--pcap", _args->pcap)->required();
add_option("data-base,--db,--data-base", _args->db)->required();
add_option("security-ids,--security-ids", _args->security_ids)->required();
add_option("out,--out", _args->out_csv);
callback([&] {
const char *eid_col = "eid";
const char *pid_col = "pid";
const char *log_header = "PolygonBookCSV";
// the nth leg col denotes the last event in which the nth leg's book
// changed
const int num_leg_cols = 10;
static void execute() {
const char *eid_col = "eid";
const char *pid_col = "pid";
const char *log_header = "PolygonBookCSV";
// the nth leg col denotes the last event in which the nth leg's book changed
const int num_leg_cols = 10;
// validate args
if (args.out_csv.empty())
args.out_csv =
file::change_file_ext(args.pcap, "polygon_books.csv", "pcap.xz");
if (args.out_csv.empty() || args.security_ids.empty())
return;
std::unordered_set<int> ids;
for (auto sid : args.security_ids)
ids.insert(sid);
INFO(
"generating polygon books for pids: {}",
orderman::Formatter::fmt_values(ids));
mdp3::SyntheticBookRecorder sbr(
spartan::sql::adapters::polygon_defs::read_sql(args.db.c_str(), ids));
mdp3::observe_channel(args.pcap, sbr);
// don't prepend a comma before calling 'append_book' (it does that
// internally)
auto append_book = [](std::ofstream &out,
const mdp3::SmallBook &bk) -> std::ofstream & {
uint32_t i = 0;
bts_assert(bk.size() <= 10);
for (; i < bk.size(); i++)
out << "," << bk[i].price << "," << bk[i].quantity.value();
while (i++ < mdp3::BookSide::max_depth())
out << ",NaN,0";
return out;
};
auto append_leg_book_eids =
[&](std::ostream &out,
const mdp3::SyntheticMBPUpdate &upd) -> std::ostream & {
int lcount = 0;
for (const auto &p : upd.leg_book_eids) {
out << "," << p.second;
if (++lcount == num_leg_cols) {
WARN(
"{} processing book update for polygon with more than {} legs "
"({} "
"'leg-book-event-id' columns are accounted for in the csv, "
"event "
"ids for susbequent legs will be discarded).",
*log_header, num_leg_cols, num_leg_cols);
break;
// validate args
if (_args->out_csv.empty())
_args->out_csv =
file::change_file_ext(_args->pcap, "polygon_books.csv", "pcap.xz");
if (_args->out_csv.empty() || _args->security_ids.empty())
return;
std::unordered_set<int> ids;
for (auto sid : _args->security_ids)
ids.insert(sid);
INFO(
"generating polygon books for pids: {}",
orderman::Formatter::fmt_values(ids));
mdp3::SyntheticBookRecorder sbr(
spartan::sql::adapters::polygon_defs::read_sql(
_args->db.c_str(), ids));
mdp3::observe_channel(_args->pcap, sbr);
// don't prepend a comma before calling 'append_book' (it does that
// internally)
auto append_book = [](std::ofstream &out,
const mdp3::SmallBook &bk) -> std::ofstream & {
uint32_t i = 0;
bts_assert(bk.size() <= 10);
for (; i < bk.size(); i++)
out << "," << bk[i].price << "," << bk[i].quantity.value();
while (i++ < mdp3::BookSide::max_depth())
out << ",NaN,0";
return out;
};
auto append_leg_book_eids =
[&](std::ostream &out,
const mdp3::SyntheticMBPUpdate &upd) -> std::ostream & {
int lcount = 0;
for (const auto &p : upd.leg_book_eids) {
out << "," << p.second;
if (++lcount == num_leg_cols) {
WARN(
"{} processing book update for polygon with more than {} "
"legs "
"({} "
"'leg-book-event-id' columns are accounted for in the csv, "
"event "
"ids for susbequent legs will be discarded).",
*log_header, num_leg_cols, num_leg_cols);
break;
}
}
while (lcount++ < num_leg_cols)
out << ",0";
return out;
};
// write polygon books to csv
auto ofs = std::ofstream(_args->out_csv);
// add cols
ofs << eid_col << "," << pid_col;
for (int i = 0; i < num_leg_cols; i++)
ofs << ",leg_" << i << "_eid";
auto add_bk_cols = [&](const char *side_prefix) {
for (int i = 0; i < 10; i++)
ofs << "," << side_prefix << "_p_" << i << "," << side_prefix
<< "_q_" << i;
};
add_bk_cols("bid");
add_bk_cols("ask");
for (const auto &p : sbr.result()) {
for (const auto &upd : p.second) {
ofs << std::endl;
ofs << p.first << "," << upd.synthetic_id.value();
append_leg_book_eids(ofs, upd);
append_book(ofs, upd.bids);
append_book(ofs, upd.asks);
}
}
while (lcount++ < num_leg_cols)
out << ",0";
return out;
};
// write polygon books to csv
auto ofs = std::ofstream(args.out_csv);
// add cols
ofs << eid_col << "," << pid_col;
for (int i = 0; i < num_leg_cols; i++)
ofs << ",leg_" << i << "_eid";
auto add_bk_cols = [&](const char *side_prefix) {
for (int i = 0; i < 10; i++)
ofs << "," << side_prefix << "_p_" << i << "," << side_prefix << "_q_"
<< i;
};
add_bk_cols("bid");
add_bk_cols("ask");
for (const auto &p : sbr.result()) {
for (const auto &upd : p.second) {
ofs << std::endl;
ofs << p.first << "," << upd.synthetic_id.value();
append_leg_book_eids(ofs, upd);
append_book(ofs, upd.bids);
append_book(ofs, upd.asks);
}
}
ofs.flush();
ofs.flush();
});
}
void PolygonBooks::register_cmd(CLI::App &app) {
auto *cmd = app.add_subcommand("polygon-books");
cmd->add_option("pcap,--pcap", args.pcap)->required();
cmd->add_option("db,--db", args.db)->required();
cmd->add_option("security-ids,--security-ids", args.security_ids)
->required();
cmd->add_option("out,--out", args.out_csv);
cmd->callback(execute);
}
PolygonBooks::~PolygonBooks() {}
} // namespace bts::spartan::cmds
@@ -1,10 +1,16 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include <memory>
namespace bts::spartan::cmds {
struct PolygonBooks {
static void register_cmd(CLI::App &);
struct PolygonBooks : public CLI::App {
PolygonBooks();
~PolygonBooks();
private:
struct Args;
std::unique_ptr<Args> _args;
};
} // namespace bts::spartan::cmds
+6 -9
View File
@@ -3,18 +3,15 @@
namespace bts::spartan::cmds {
struct Args {
struct SQL::Args {
std::string pcap;
};
static Args args;
static void execute() { mdp3::to_sql(args.pcap); }
void SQL::register_cmd(CLI::App &app) {
auto *subcmd = app.add_subcommand("sql");
subcmd->add_option("pcap,--pcap", args.pcap)->required();
subcmd->callback(execute);
SQL::SQL() : CLI::App("Generate a sqlite database for a pcap file", "sql"), _args(new SQL::Args()) {
add_option("pcap,--pcap", _args->pcap)->required();
callback([&] { mdp3::to_sql(_args->pcap); });
}
SQL::~SQL() {}
} // namespace bts::spartan::cmds
+8 -2
View File
@@ -1,10 +1,16 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include <memory>
namespace bts::spartan::cmds {
struct SQL {
static void register_cmd(CLI::App &);
struct SQL : public CLI::App {
SQL();
~SQL();
private:
struct Args;
std::unique_ptr<Args> _args;
};
} // namespace bts::spartan::cmds
+7 -10
View File
@@ -5,19 +5,16 @@
namespace bts::spartan::cmds {
struct Args {
struct Sim::Args {
std::string config_path;
};
static Args args;
static void execute() { mdp3::run_sim(args.config_path); }
void Sim::register_cmd(CLI::App &app) {
auto *subcmd = app.add_subcommand("sim");
subcmd->add_option("config-file,--config-file", args.config_path)
->required();
subcmd->callback(execute);
Sim::Sim()
: CLI::App("Run trading simulation(s)", "sim"), _args(new Sim::Args()) {
add_option("config-file,--config-file", _args->config_path)->required();
callback([&] { mdp3::run_sim(_args->config_path); });
}
Sim::~Sim() {}
} // namespace bts::spartan::cmds
+8 -2
View File
@@ -1,10 +1,16 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include <memory>
namespace bts::spartan::cmds {
struct Sim {
static void register_cmd(CLI::App &);
struct Sim : public CLI::App {
Sim();
~Sim();
private:
struct Args;
std::unique_ptr<Args> _args;
};
} // namespace bts::spartan::cmds
@@ -1,5 +1,6 @@
#include "SimBooks.h"
#include "server/sparkd/cmds/spartan/back_testing/matcher/Matcher.h"
#include "server/sparkd/cmds/spartan/back_testing/matcher/validation/iceberg_file_cache.h"
#include "server/sparkd/cmds/spartan/domain/SecurityCriteria.h"
#include "server/sparkd/cmds/spartan/market_data/feed_observers/DefinitionCollector.h"
#include "server/sparkd/cmds/spartan/market_data/feed_observers/IcebergFinder.h"
@@ -13,7 +14,7 @@ namespace bts::spartan::cmds {
#define _LOG(level, ...) LOG(OrderMan, level, __VA_ARGS__)
#define INFO(...) _LOG(Info, __VA_ARGS__)
struct Args {
struct SimBooks::Args {
std::string pcap;
std::string out;
std::vector<std::string> assets;
@@ -21,66 +22,70 @@ struct Args {
mdp3::SecurityCriteria security_criteria;
};
static Args args;
void execute() {
// Validate args
bts_assert(args.pcap.size());
if (args.out.empty())
args.out = file::change_file_ext(args.pcap, "sim_books.csv", "pcap.xz");
bts_assert(args.out.size());
// Sweep pcap for securityies and icebergs
mdp3::DefinitionCollector dc;
mdp3::IcebergFinder ice_finder;
INFO(
"Begin observe '{}' (1/2): detect all securityies and iceberg orders",
args.pcap);
mdp3::observe_channel(args.pcap, {&dc, &ice_finder});
INFO(
"End observe '{}' (1/2): detected {} securities, and {} iceberg orders",
args.pcap, dc.result.size(), ice_finder.result.size());
// Init MergedMatcher
mdp3::MergedMatcher matcher;
for (auto p : dc.result)
matcher.add_def(p.second);
// Init MatcherBookRecorder
mdp3::SecurityCriteria security_criteria;
if (args.assets.size() || args.security_groups.size()) {
// assets
auto &assets = security_criteria.clauses.emplace_back().assets;
assets.insert(assets.end(), args.assets.begin(), args.assets.end());
// security groups
auto &security_groups = security_criteria.clauses.emplace_back().assets;
security_groups.insert(
security_groups.end(), args.security_groups.begin(),
args.security_groups.end());
}
mdp3::MatcherBookRecorder rec(
{matcher, ice_finder.result, security_criteria, true});
// Init FeedRecorderWrapper and observe pcap to generate the csv
std::ofstream out(args.out);
mdp3::FeedRecorderWrapper rec_wrap(out, &rec);
INFO("Begin observe '{}' (2/2): record matcher books", args.pcap);
mdp3::observe_channel(args.pcap, rec_wrap);
INFO("End observe '{}' (2/2): recorded matcher book", args.pcap);
out.flush();
}
void SimBooks::register_cmd(CLI::App &app) {
auto *subcmd = app.add_subcommand(
"sim-books", "Generate a csv file of book updates by feeding deduced "
"orders from a pcap file to the simulation matcher");
subcmd->add_option("pcap,--pcap", args.pcap)->required();
subcmd->add_option("out,--out", args.out);
subcmd->add_option(
"--assets", args.assets,
SimBooks::SimBooks()
: CLI::App(
"Generate a csv file of book updates by feeding deduced orders from "
"a pcap file to the simulation matcher",
"sim-books"),
_args(new SimBooks::Args()) {
add_option("pcap,--pcap", _args->pcap)->required();
add_option("out,--out", _args->out);
add_option(
"--assets", _args->assets,
"Specify asset(s) to filter the set of securities for which books are "
"generated");
subcmd->add_option(
"--security-groups", args.security_groups,
add_option(
"--security-groups", _args->security_groups,
"Specify security group(s) to filter the set of securities for which "
"books are generated");
subcmd->callback(execute);
callback([&] {
// Validate args
bts_assert(_args->pcap.size());
if (_args->out.empty())
_args->out =
file::change_file_ext(_args->pcap, "sim_books.csv", "pcap.xz");
bts_assert(_args->out.size());
// Sweep pcap for securityies and icebergs
mdp3::DefinitionCollector dc;
// mdp3::IcebergFinder ice_finder;
INFO(
"Begin observe '{}' (1/2): detect all securityies and iceberg orders",
_args->pcap);
mdp3::observe_channel(_args->pcap, dc);
auto ibs = mdp3::iceberg_file_cache::find_icebergs(_args->pcap);
INFO(
"End observe '{}' (1/2): detected {} securities, and {} iceberg "
"orders",
_args->pcap, dc.result.size(), ibs.size());
// Init MergedMatcher
mdp3::MergedMatcher matcher;
for (auto p : dc.result)
matcher.add_def(p.second);
// Init MatcherBookRecorder
mdp3::SecurityCriteria security_criteria;
if (_args->assets.size() || _args->security_groups.size()) {
// assets
auto &assets = security_criteria.clauses.emplace_back().assets;
assets.insert(
assets.end(), _args->assets.begin(), _args->assets.end());
// security groups
auto &security_groups =
security_criteria.clauses.emplace_back().assets;
security_groups.insert(
security_groups.end(), _args->security_groups.begin(),
_args->security_groups.end());
}
mdp3::MatcherBookRecorder rec({matcher, ibs, security_criteria, true});
// Init FeedRecorderWrapper and observe pcap to generate the csv
std::ofstream out(_args->out);
mdp3::FeedRecorderWrapper rec_wrap(out, &rec);
INFO("Begin observe '{}' (2/2): record matcher books", _args->pcap);
mdp3::observe_channel(_args->pcap, rec_wrap);
INFO("End observe '{}' (2/2): recorded matcher book", _args->pcap);
out.flush();
});
}
SimBooks::~SimBooks() {}
} // namespace bts::spartan::cmds
@@ -1,10 +1,16 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include <memory>
namespace bts::spartan::cmds {
struct SimBooks {
static void register_cmd(CLI::App &);
struct SimBooks : public CLI::App {
SimBooks();
~SimBooks();
private:
struct Args;
std::unique_ptr<Args> _args;
};
} // namespace bts::spartan::cmds::sim_books
} // namespace bts::spartan::cmds
+19 -17
View File
@@ -6,18 +6,8 @@
namespace bts::spartan::cmds {
struct Args {
std::string pcap;
std::string out;
};
static Args args;
size_t total_orders = 0;
static void execute() {
if (args.out.empty())
args.out = spartan::file::change_file_ext(args.pcap, "order_counts.csv");
auto out_file = std::ofstream(args.out);
static void execute(const std::string &pcap, std::ostream &out_file) {
out_file << "sid,n_orders" << std::endl;
struct StatPrinter : spartan::mdp3::IMDP3FeedObserver {
StatPrinter(std::ostream &out) : out{out} {}
@@ -28,15 +18,27 @@ static void execute() {
}
} stats{out_file};
observe_channel(args.pcap, stats);
observe_channel(pcap, stats);
std::cout << total_orders << std::endl;
}
void Stats::register_cmd(CLI::App &app) {
auto *subcmd = app.add_subcommand("stats");
subcmd->add_option("pcap,--pcap", args.pcap)->required();
subcmd->add_option("out,--out", args.out);
subcmd->callback(execute);
struct Stats::Args {
std::string pcap;
std::string out;
};
Stats::Stats() : CLI::App("", "stats"), _args(new Stats::Args()) {
add_option("pcap,--pcap", _args->pcap)->required();
add_option("out,--out", _args->out);
callback([&] {
if (_args->out.empty())
_args->out =
spartan::file::change_file_ext(_args->pcap, "order_counts.csv");
std::ofstream out(_args->out);
execute(_args->pcap, out);
});
}
Stats::~Stats() {}
} // namespace bts::spartan::cmds
+8 -2
View File
@@ -1,10 +1,16 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include <memory>
namespace bts::spartan::cmds {
struct Stats {
static void register_cmd(CLI::App &);
struct Stats : public CLI::App {
Stats();
~Stats();
private:
struct Args;
std::unique_ptr<Args> _args;
};
} // namespace bts::spartan::cmds
@@ -7,22 +7,20 @@
namespace bts::spartan::cmds {
struct Args {
struct ValidateFifo::Args {
std::string pcap;
};
static Args args;
static void execute() {
auto ibs = mdp3::IcebergFinder::find_all(args.pcap);
mdp3::MatcherValidationObserver obs(ibs);
mdp3::observe_channel(args.pcap, obs);
ValidateFifo::ValidateFifo()
: CLI::App("", "validate-fifo"), _args(new ValidateFifo::Args()) {
add_option("pcap,--pcap", _args->pcap)->required();
callback([&] {
auto ibs = mdp3::IcebergFinder::find_all(_args->pcap);
mdp3::MatcherValidationObserver obs(ibs);
mdp3::observe_channel(_args->pcap, obs);
});
}
void ValidateFifo::register_cmd(CLI::App &app) {
auto *cmd = app.add_subcommand("validate-fifo");
cmd->add_option("pcap,--pcap", args.pcap)->required();
cmd->callback(execute);
}
ValidateFifo::~ValidateFifo() {}
} // namespace bts::spartan::cmds
@@ -1,10 +1,16 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include <memory>
namespace bts::spartan::cmds {
struct ValidateFifo {
static void register_cmd(CLI::App &);
struct ValidateFifo : public CLI::App {
ValidateFifo();
~ValidateFifo();
private:
struct Args;
std::unique_ptr<Args> _args;
};
} // namespace bts::spartan::cmds
@@ -8,40 +8,43 @@
namespace bts::spartan::cmds {
struct Args {
struct ValuePolys::Args {
std::string pcap;
};
static Args args;
ValuePolys::ValuePolys()
: CLI::App(
"Generate csv files of polygon values and opportunities. Each of "
"which are calculated from the polygon book updates in a pcap file",
"value-polys"),
_args(new ValuePolys::Args()) {
add_option("pcap,--pcap", _args->pcap)->required();
callback([&] {
mdp3::DefinitionCollector def;
mdp3::PolygonFinder poly_finder;
static void execute() {
mdp3::DefinitionCollector def;
mdp3::PolygonFinder poly_finder;
// first pass, just identify the polygons
mdp3::observe_channel(_args->pcap, {&poly_finder, &def});
// first pass, just identify the polygons
mdp3::observe_channel(args.pcap, {&poly_finder, &def});
// second pass, value the polygons
mdp3::PolygonOpportunityFinder op_finder{
poly_finder.result, def.ptr_result};
mdp3::observe_channel(_args->pcap, {&op_finder});
// second pass, value the polygons
mdp3::PolygonOpportunityFinder op_finder{poly_finder.result, def.ptr_result};
mdp3::observe_channel(args.pcap, {&op_finder});
auto poly_v = std::ofstream(
file::change_file_ext(_args->pcap, "polygon_values.csv"));
auto poly_v =
std::ofstream(file::change_file_ext(args.pcap, "polygon_values.csv"));
for (auto &v : op_finder.result)
poly_v << v << std::endl;
for (auto &v : op_finder.result)
poly_v << v << std::endl;
auto poly_op = std::ofstream(
file::change_file_ext(args.pcap, "polygon_opportunities.csv"));
spartan::mdp3::poly_opportunities_to_csv(
aggregate(op_finder.result), poly_finder.result, def.ptr_result,
poly_op);
auto poly_op = std::ofstream(
file::change_file_ext(_args->pcap, "polygon_opportunities.csv"));
spartan::mdp3::poly_opportunities_to_csv(
aggregate(op_finder.result), poly_finder.result, def.ptr_result,
poly_op);
});
}
void ValuePolys::register_cmd(CLI::App &app) {
auto *subcmd = app.add_subcommand("value-polys");
subcmd->add_option("pcap,--pcap", args.pcap)->required();
subcmd->callback(execute);
}
ValuePolys::~ValuePolys() {}
} // namespace bts::spartan::cmds
@@ -1,10 +1,16 @@
#pragma once
#include <CLI11/CLI11.hpp>
#include <memory>
namespace bts::spartan::cmds {
struct ValuePolys {
static void register_cmd(CLI::App &);
struct ValuePolys : public CLI::App {
ValuePolys();
~ValuePolys();
private:
struct Args;
std::unique_ptr<Args> _args;
};
} // namespace bts::spartan::cmds
+26 -17
View File
@@ -1,3 +1,4 @@
#include "lib/testing/catch_test_runner.h"
#include "lib/utils/init_spark_lib.h"
#include "lib/utils/thread_manager.h"
#include <CLI11/CLI11.hpp>
@@ -17,7 +18,6 @@
#include "server/sparkd/cmds/Run.h"
#include "server/sparkd/cmds/Span.h"
#include "server/sparkd/cmds/Spartan.h"
#include "server/sparkd/cmds/Test.h"
#include "server/sparkd/cmds/Web.h"
using namespace bts;
@@ -25,24 +25,33 @@ using namespace bts;
int main(int argc, char **argv) {
utils::init_spark_lib();
// The 'test' cli is delegated to the catch lib/framework.
if (argc > 1 && strcmp(argv[1], "test") == 0) {
std::vector<std::string> v;
for (int i = 2; i < argc; i++)
v.emplace_back(argv[i]);
return run_tests(v);
}
CLI::App app("Futures trading system", "sparkd");
app.require_subcommand();
// register subcommands
bts::cmds::AdminReport::register_cmd(app);
bts::cmds::AutoCert::register_cmd(app);
bts::cmds::Benchmark::register_cmd(app);
bts::cmds::ConfigCheck::register_cmd(app);
bts::cmds::FIXSim::register_cmd(app);
bts::cmds::KeyCode::register_cmd(app);
bts::cmds::Md::register_cmd(app);
bts::cmds::ParseHistoricalFix::register_cmd(app);
bts::cmds::PrintHedgeOrders::register_cmd(app);
bts::cmds::QuickTest::register_cmd(app);
bts::cmds::Run::register_cmd(app);
bts::cmds::Span::register_cmd(app);
bts::cmds::Spartan::register_cmd(app);
bts::cmds::Test::register_cmd(app);
bts::cmds::Web::register_cmd(app);
// subcommands
app.add_subcommand(CLI::App_p(new cmds::AdminReport()));
app.add_subcommand(CLI::App_p(new cmds::AutoCert()));
app.add_subcommand(CLI::App_p(new cmds::Benchmark()));
app.add_subcommand(CLI::App_p(new cmds::ConfigCheck()));
app.add_subcommand(CLI::App_p(new cmds::FIXSim()));
app.add_subcommand(CLI::App_p(new cmds::Md()));
app.add_subcommand(CLI::App_p(new cmds::ParseHistoricalFix()));
app.add_subcommand(CLI::App_p(new cmds::PrintHedgeOrders()));
app.add_subcommand(CLI::App_p(new cmds::QuickTest()));
app.add_subcommand(CLI::App_p(new cmds::Run()));
app.add_subcommand(CLI::App_p(new cmds::Span()));
app.add_subcommand(CLI::App_p(new cmds::Spartan()));
app.add_subcommand(CLI::App_p(new cmds::Web()));
// Add a 'test' dummy subcommand, so it will be listed in the cli when the
// user runs 'sparkd --help'.
app.add_subcommand("test", "Run unit test(s)");
CLI11_PARSE(app, argc, argv);
utils::finit_spark_lib();