SPARK-1653, The initial crash that john reported turned out to be a monster.
Events were getting re-ordered in our event queue, possibly due to this: https://github.com/cameron314/concurrentqueue/issues/126 We abandoned the moody camel queue.
This commit is contained in:
1 parent
c19b720121
commit
8e00da5a72
6 files changed
+237
-241
No files matched your search
@@ -5,55 +5,41 @@
|
||||
#include "lib/jetfix/common/InMsgBuffer.h"
|
||||
#include "lib/jetfix/common/OutFIX.h"
|
||||
#include "lib/jetfix/store/session_name.h"
|
||||
#include "lib/os/os.h"
|
||||
#include "lib/utils/bts_chrono.h"
|
||||
#include "lib/utils/datetime.h"
|
||||
#include "lib/utils/file_utils.h"
|
||||
#include "lib/utils/bts_chrono.h"
|
||||
#include "lib/os/os.h"
|
||||
|
||||
#include <fstream>
|
||||
#include <thread>
|
||||
#include <boost/date_time.hpp>
|
||||
#include <boost/date_time/gregorian/gregorian.hpp>
|
||||
#include <boost/filesystem.hpp>
|
||||
#include <boost/date_time.hpp>
|
||||
#include <moodycamel/readerwriterqueue.h>
|
||||
#include <tbb/concurrent_queue.h>
|
||||
|
||||
#include <fmt/format.h>
|
||||
#include <fstream>
|
||||
#include <thread>
|
||||
|
||||
namespace bts {
|
||||
namespace jetfix {
|
||||
|
||||
struct FIXFileStore::Event
|
||||
{
|
||||
Event ()
|
||||
: out_msgs (), in_msg (),
|
||||
quit (false),
|
||||
has_last_msg_processed (false),
|
||||
last_msg_processed (0),
|
||||
has_seq_num (false),
|
||||
seq_num (0)
|
||||
{}
|
||||
struct FIXFileStore::Event {
|
||||
Event()
|
||||
: out_msgs(), in_msg(), quit(false), has_last_msg_processed(false),
|
||||
last_msg_processed(0), has_seq_num(false), seq_num(0) {}
|
||||
|
||||
Event (Event const &src)
|
||||
: out_msgs (src.out_msgs),
|
||||
in_msg (src.in_msg),
|
||||
quit (src.quit),
|
||||
has_last_msg_processed (src.has_last_msg_processed),
|
||||
last_msg_processed (src.last_msg_processed),
|
||||
has_seq_num (src.has_seq_num),
|
||||
seq_num (src.seq_num)
|
||||
{}
|
||||
Event(Event const &src)
|
||||
: out_msgs(src.out_msgs), in_msg(src.in_msg), quit(src.quit),
|
||||
has_last_msg_processed(src.has_last_msg_processed),
|
||||
last_msg_processed(src.last_msg_processed),
|
||||
has_seq_num(src.has_seq_num), seq_num(src.seq_num) {}
|
||||
|
||||
Event (Event &&src)
|
||||
: out_msgs (std::move(src.out_msgs)),
|
||||
in_msg (std::move(src.in_msg)),
|
||||
quit (src.quit),
|
||||
has_last_msg_processed (src.has_last_msg_processed),
|
||||
last_msg_processed (src.last_msg_processed),
|
||||
has_seq_num (src.has_seq_num),
|
||||
seq_num (src.seq_num)
|
||||
{}
|
||||
Event(Event &&src)
|
||||
: out_msgs(std::move(src.out_msgs)), in_msg(std::move(src.in_msg)),
|
||||
quit(src.quit), has_last_msg_processed(src.has_last_msg_processed),
|
||||
last_msg_processed(src.last_msg_processed),
|
||||
has_seq_num(src.has_seq_num), seq_num(src.seq_num) {}
|
||||
|
||||
Event& operator=(Event const &src)
|
||||
{
|
||||
Event &operator=(Event const &src) {
|
||||
out_msgs = src.out_msgs;
|
||||
in_msg = src.in_msg;
|
||||
quit = src.quit;
|
||||
@@ -64,8 +50,7 @@ struct FIXFileStore::Event
|
||||
return *this;
|
||||
}
|
||||
|
||||
Event& operator=(Event const &&src)
|
||||
{
|
||||
Event &operator=(Event const &&src) {
|
||||
out_msgs = std::move(src.out_msgs);
|
||||
in_msg = std::move(src.in_msg);
|
||||
quit = src.quit;
|
||||
@@ -84,21 +69,12 @@ struct FIXFileStore::Event
|
||||
bool has_seq_num;
|
||||
uint32_t seq_num;
|
||||
};
|
||||
|
||||
struct FIXFileStore::PImpl
|
||||
{
|
||||
PImpl(config::JetFIX::Connection const&cfg) :
|
||||
active (false),
|
||||
store_called (false),
|
||||
replay_messages (),
|
||||
last_seq_num (0),
|
||||
last_seq_num_processed (0),
|
||||
seq_num_path (),
|
||||
fix_store (),
|
||||
writer_thread (),
|
||||
event_queue (),
|
||||
cfg(cfg)
|
||||
{}
|
||||
|
||||
struct FIXFileStore::PImpl {
|
||||
PImpl(config::JetFIX::Connection const &cfg)
|
||||
: active(false), store_called(false), replay_messages(), last_seq_num(0),
|
||||
last_seq_num_processed(0), seq_num_path(), fix_store(),
|
||||
writer_thread(), event_queue(), cfg(cfg) {}
|
||||
|
||||
bool active;
|
||||
bool store_called;
|
||||
@@ -108,237 +84,224 @@ struct FIXFileStore::PImpl
|
||||
std::string seq_num_path;
|
||||
std::ofstream fix_store;
|
||||
std::thread writer_thread;
|
||||
moodycamel::ReaderWriterQueue<Event> event_queue;
|
||||
tbb::concurrent_queue<Event> event_queue;
|
||||
config::JetFIX::Connection const cfg;
|
||||
};
|
||||
|
||||
std::string fix_log_path (std::string const&base_path)
|
||||
{
|
||||
std::string fix_log_path(std::string const &base_path) {
|
||||
boost::filesystem::path result = base_path;
|
||||
result /= "fix";
|
||||
return result.string ();
|
||||
return result.string();
|
||||
}
|
||||
|
||||
std::string seqnum_path (std::string const&base_path)
|
||||
{
|
||||
std::string seqnum_path(std::string const &base_path) {
|
||||
boost::filesystem::path result = base_path;
|
||||
result /= "seqnum";
|
||||
return result.string ();
|
||||
return result.string();
|
||||
}
|
||||
|
||||
std::string store_path (std::string base_path,
|
||||
std::string const&connection_name,
|
||||
std::string const&session_name)
|
||||
{
|
||||
if (base_path.empty ())
|
||||
base_path = os::bts_data_path ();
|
||||
boost::filesystem::path result = utils::normalize_data_path (base_path);
|
||||
std::string store_path(std::string base_path,
|
||||
std::string const &connection_name,
|
||||
std::string const &session_name) {
|
||||
if (base_path.empty())
|
||||
base_path = os::bts_data_path();
|
||||
boost::filesystem::path result = utils::normalize_data_path(base_path);
|
||||
result /= "jetfix";
|
||||
result /= connection_name;
|
||||
result /= session_name;
|
||||
return result.string ();
|
||||
return result.string();
|
||||
}
|
||||
|
||||
std::map<uint32_t,InMsgBuffer>
|
||||
load_replay (config::JetFIX::Connection const &cfg,std::string const&path, uint32_t max_sn)
|
||||
{
|
||||
std::map<uint32_t,InMsgBuffer> result;
|
||||
if (boost::filesystem::exists (path)) {
|
||||
std::map<uint32_t, InMsgBuffer>
|
||||
load_replay(config::JetFIX::Connection const &cfg, std::string const &path,
|
||||
uint32_t max_sn) {
|
||||
std::map<uint32_t, InMsgBuffer> result;
|
||||
if (boost::filesystem::exists(path)) {
|
||||
|
||||
std::ifstream f (path);
|
||||
std::ifstream f(path);
|
||||
std::string line_buffer;
|
||||
line_buffer.reserve (10*1024);
|
||||
auto header_size = sizeof("20150427-19:05:17.032 < ")-1;
|
||||
std::string date_buffer ("20150427-19:05:17.032");
|
||||
|
||||
line_buffer.reserve(10 * 1024);
|
||||
auto header_size = sizeof("20150427-19:05:17.032 < ") - 1;
|
||||
std::string date_buffer("20150427-19:05:17.032");
|
||||
|
||||
for (;;) {
|
||||
std::getline (f,line_buffer);
|
||||
if (!f.eof ()) {
|
||||
bts_assert (line_buffer.size () > header_size);
|
||||
if (line_buffer[22]=='<') {
|
||||
auto ts = datetime::parse_FIX_timestamp (line_buffer.substr (0,21));
|
||||
if (!datetime::is_at_least_secs_old (ts,cfg.seconds_to_replay ())) {
|
||||
InMsgBuffer msg (line_buffer.substr (24));
|
||||
auto sn = msg.get_seq_num ();
|
||||
std::getline(f, line_buffer);
|
||||
if (!f.eof()) {
|
||||
bts_assert(line_buffer.size() > header_size);
|
||||
if (line_buffer[22] == '<') {
|
||||
auto ts =
|
||||
datetime::parse_FIX_timestamp(line_buffer.substr(0, 21));
|
||||
if (!datetime::is_at_least_secs_old(ts,
|
||||
cfg.seconds_to_replay())) {
|
||||
InMsgBuffer msg(line_buffer.substr(24));
|
||||
auto sn = msg.get_seq_num();
|
||||
if (sn <= max_sn)
|
||||
result[sn] = msg;
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
} else
|
||||
break;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
void save_fix (std::ostream &out, const char*data, size_t size, bool is_outgoing)
|
||||
{
|
||||
out << datetime::FIX_timestamp (datetime::now ()) << " "
|
||||
void save_fix(std::ostream &out, const char *data, size_t size,
|
||||
bool is_outgoing) {
|
||||
out << datetime::FIX_timestamp(datetime::now()) << " "
|
||||
<< (is_outgoing ? ">" : "<") << " ";
|
||||
out.write (data,size);
|
||||
out.write(data, size);
|
||||
out << std::endl;
|
||||
}
|
||||
|
||||
void save_seq_nums (std::string const&path,uint32_t last_seq_num, uint32_t last_seq_num_processed)
|
||||
{
|
||||
std::ofstream f (path);
|
||||
void save_seq_nums(std::string const &path, uint32_t last_seq_num,
|
||||
uint32_t last_seq_num_processed) {
|
||||
std::ofstream f(path);
|
||||
f << last_seq_num << " " << last_seq_num_processed << std::endl;
|
||||
}
|
||||
|
||||
void load_seq_nums (std::string const&path,uint32_t* last_seq_num, uint32_t *last_seq_num_processed)
|
||||
{
|
||||
if (boost::filesystem::exists (path)) {
|
||||
std::ifstream f (path);
|
||||
void load_seq_nums(std::string const &path, uint32_t *last_seq_num,
|
||||
uint32_t *last_seq_num_processed) {
|
||||
if (boost::filesystem::exists(path)) {
|
||||
std::ifstream f(path);
|
||||
f >> *last_seq_num >> *last_seq_num_processed;
|
||||
} else {
|
||||
*last_seq_num = 0;
|
||||
*last_seq_num_processed = 0;
|
||||
save_seq_nums (path, *last_seq_num,*last_seq_num_processed);
|
||||
save_seq_nums(path, *last_seq_num, *last_seq_num_processed);
|
||||
}
|
||||
}
|
||||
|
||||
FIXFileStore::FIXFileStore (config::JetFIX::Connection const&cfg,std::string base_path)
|
||||
: me (new PImpl(cfg)) {
|
||||
auto sn = session_name (cfg);
|
||||
me->active = sn.size () > 0;
|
||||
FIXFileStore::FIXFileStore(config::JetFIX::Connection const &cfg,
|
||||
std::string base_path)
|
||||
: me(new PImpl(cfg)) {
|
||||
auto sn = session_name(cfg);
|
||||
me->active = sn.size() > 0;
|
||||
if (me->active) {
|
||||
|
||||
auto path = store_path (base_path,cfg.name (),sn);
|
||||
if (!boost::filesystem::exists (path))
|
||||
bts_assert (boost::filesystem::create_directories (path));
|
||||
auto path = store_path(base_path, cfg.name(), sn);
|
||||
if (!boost::filesystem::exists(path))
|
||||
bts_assert(boost::filesystem::create_directories(path));
|
||||
|
||||
// load seq numbers
|
||||
auto seq_num_p = seqnum_path (path);
|
||||
load_seq_nums (seq_num_p,&me->last_seq_num,&me->last_seq_num_processed);
|
||||
auto seq_num_p = seqnum_path(path);
|
||||
load_seq_nums(seq_num_p, &me->last_seq_num, &me->last_seq_num_processed);
|
||||
me->seq_num_path = seq_num_p;
|
||||
|
||||
|
||||
// load replay
|
||||
auto fix_log_p = fix_log_path (path);
|
||||
auto replay_messages = load_replay (cfg,fix_log_p,me->last_seq_num_processed);
|
||||
auto fix_log_p = fix_log_path(path);
|
||||
auto replay_messages =
|
||||
load_replay(cfg, fix_log_p, me->last_seq_num_processed);
|
||||
uint32_t previous_sn = 0;
|
||||
for (auto &p : replay_messages) {
|
||||
auto sn = p.first;
|
||||
auto &m = p.second;
|
||||
if (!(m.has_poss_dup_flag_flag () && m.get_poss_dup_flag ())) {
|
||||
if (previous_sn != 0)
|
||||
bts_assert (sn == (previous_sn + 1));
|
||||
if (!(m.has_poss_dup_flag_flag() && m.get_poss_dup_flag())) {
|
||||
if (previous_sn != 0)
|
||||
bts_assert(sn == (previous_sn + 1));
|
||||
previous_sn = sn;
|
||||
me->replay_messages.emplace_back (m);
|
||||
me->replay_messages.emplace_back(m);
|
||||
}
|
||||
}
|
||||
me->fix_store.open (fix_log_p,std::ofstream::app);
|
||||
me->writer_thread = std::thread (std::bind (&FIXFileStore::write,this));
|
||||
me->fix_store.open(fix_log_p, std::ofstream::app);
|
||||
me->writer_thread = std::thread(std::bind(&FIXFileStore::write, this));
|
||||
}
|
||||
}
|
||||
|
||||
FIXFileStore::~FIXFileStore ()
|
||||
{
|
||||
FIXFileStore::~FIXFileStore() {
|
||||
if (me->active) {
|
||||
Event ev;
|
||||
ev.quit = true;
|
||||
me->event_queue.enqueue (ev);
|
||||
me->writer_thread.join ();
|
||||
me->event_queue.push(ev);
|
||||
me->writer_thread.join();
|
||||
}
|
||||
}
|
||||
|
||||
bool FIXFileStore::session_active () const
|
||||
{return me->active;}
|
||||
bool FIXFileStore::session_active() const { return me->active; }
|
||||
|
||||
std::vector<InMsgBuffer> FIXFileStore::replay () const
|
||||
{
|
||||
bts_assert (me->active && !me->store_called);
|
||||
std::vector<InMsgBuffer> FIXFileStore::replay() const {
|
||||
bts_assert(me->active && !me->store_called);
|
||||
return me->replay_messages;
|
||||
}
|
||||
|
||||
uint32_t FIXFileStore::last_seq_num () const
|
||||
{
|
||||
bts_assert (me->active && !me->store_called);
|
||||
uint32_t FIXFileStore::last_seq_num() const {
|
||||
bts_assert(me->active && !me->store_called);
|
||||
return me->last_seq_num;
|
||||
}
|
||||
|
||||
uint32_t FIXFileStore::last_seq_num_processed () const
|
||||
{
|
||||
bts_assert (me->active && !me->store_called);
|
||||
|
||||
uint32_t FIXFileStore::last_seq_num_processed() const {
|
||||
bts_assert(me->active && !me->store_called);
|
||||
return me->last_seq_num_processed;
|
||||
}
|
||||
|
||||
void FIXFileStore::store (OutFIXPtr const&msgs)
|
||||
{
|
||||
bts_assert (me->active);
|
||||
void FIXFileStore::store(OutFIXPtr const &msgs) {
|
||||
bts_assert(me->active);
|
||||
me->store_called = true;
|
||||
Event ev;
|
||||
ev.out_msgs = msgs;
|
||||
me->event_queue.enqueue (ev);
|
||||
me->event_queue.push(ev);
|
||||
}
|
||||
|
||||
void FIXFileStore::store (InMsgBuffer const&msg)
|
||||
{
|
||||
bts_assert (me->active);
|
||||
void FIXFileStore::store(InMsgBuffer const &msg) {
|
||||
bts_assert(me->active);
|
||||
me->store_called = true;
|
||||
Event ev;
|
||||
ev.in_msg = msg;
|
||||
me->event_queue.enqueue (ev);
|
||||
me->event_queue.push(ev);
|
||||
}
|
||||
|
||||
void FIXFileStore::set_last_msg_processed (uint32_t last_msg_processed)
|
||||
{
|
||||
bts_assert (me->active);
|
||||
void FIXFileStore::set_last_msg_processed(uint32_t last_msg_processed) {
|
||||
bts_assert(me->active);
|
||||
me->store_called = true;
|
||||
Event ev;
|
||||
ev.has_last_msg_processed = true;
|
||||
ev.last_msg_processed = last_msg_processed;
|
||||
me->event_queue.enqueue (ev);
|
||||
me->event_queue.push(ev);
|
||||
}
|
||||
|
||||
void FIXFileStore::set_last_seq_num(uint32_t last_seq_num)
|
||||
{
|
||||
bts_assert (me->active);
|
||||
void FIXFileStore::set_last_seq_num(uint32_t last_seq_num) {
|
||||
bts_assert(me->active);
|
||||
me->store_called = true;
|
||||
Event ev;
|
||||
ev.has_seq_num = true;
|
||||
ev.seq_num = last_seq_num;
|
||||
me->event_queue.enqueue (ev);
|
||||
me->event_queue.push(ev);
|
||||
}
|
||||
|
||||
void FIXFileStore::write ()
|
||||
{
|
||||
void FIXFileStore::write() {
|
||||
Event ev;
|
||||
os::set_thread_name(fmt::format("{0}_store",me->cfg.name()).c_str());
|
||||
os::set_thread_name(fmt::format("{0}_store", me->cfg.name()).c_str());
|
||||
|
||||
while (true)
|
||||
if (me->event_queue.try_dequeue (ev)) {
|
||||
if (me->event_queue.try_pop(ev)) {
|
||||
if (ev.quit) {
|
||||
return;
|
||||
}
|
||||
else if (ev.out_msgs) {
|
||||
} else if (ev.out_msgs) {
|
||||
for (auto &encoded_stream : ev.out_msgs->fix)
|
||||
for (auto &mm : encoded_stream.msg_map ()) {
|
||||
me->last_seq_num = std::max (me->last_seq_num,mm.seq_num);
|
||||
save_seq_nums (me->seq_num_path,me->last_seq_num,me->last_seq_num_processed);
|
||||
save_fix (me->fix_store,encoded_stream.data ()+mm.offset,mm.size,true);
|
||||
for (auto &mm : encoded_stream.msg_map()) {
|
||||
me->last_seq_num = std::max(me->last_seq_num, mm.seq_num);
|
||||
save_seq_nums(me->seq_num_path, me->last_seq_num,
|
||||
me->last_seq_num_processed);
|
||||
save_fix(me->fix_store, encoded_stream.data() + mm.offset,
|
||||
mm.size, true);
|
||||
}
|
||||
me->fix_store.flush ();
|
||||
}
|
||||
else if (ev.in_msg.size ())
|
||||
{
|
||||
save_fix (me->fix_store,ev.in_msg.data (),ev.in_msg.size (),false);
|
||||
me->fix_store.flush ();
|
||||
}
|
||||
else if (ev.has_last_msg_processed) {
|
||||
me->fix_store.flush();
|
||||
} else if (ev.in_msg.size()) {
|
||||
save_fix(me->fix_store, ev.in_msg.data(), ev.in_msg.size(), false);
|
||||
me->fix_store.flush();
|
||||
} else if (ev.has_last_msg_processed) {
|
||||
me->last_seq_num_processed = ev.last_msg_processed;
|
||||
save_seq_nums (me->seq_num_path,me->last_seq_num,me->last_seq_num_processed);
|
||||
}
|
||||
else if (ev.has_seq_num) {
|
||||
save_seq_nums(me->seq_num_path, me->last_seq_num,
|
||||
me->last_seq_num_processed);
|
||||
} else if (ev.has_seq_num) {
|
||||
me->last_seq_num = ev.seq_num;
|
||||
save_seq_nums (me->seq_num_path,me->last_seq_num,me->last_seq_num_processed);
|
||||
}
|
||||
else
|
||||
bts_assert (false);
|
||||
}
|
||||
else
|
||||
std::this_thread::sleep_for (100ms);
|
||||
save_seq_nums(me->seq_num_path, me->last_seq_num,
|
||||
me->last_seq_num_processed);
|
||||
} else
|
||||
bts_assert(false);
|
||||
} else
|
||||
std::this_thread::sleep_for(100ms);
|
||||
}
|
||||
|
||||
}}
|
||||
|
||||
} // namespace jetfix
|
||||
} // namespace bts
|
||||
Reference in new issue
Block a user