SPARK-1653, The initial crash that john reported turned out to be a monster.

Events were getting re-ordered in our event queue, possibly due to
this:

https://github.com/cameron314/concurrentqueue/issues/126

We abandoned the moody camel queue.
This commit is contained in:
Pedro Pinto committed 2018-08-30 21:49:01 +00:00
1 parent c19b720121
commit 8e00da5a72
6 files changed
+237 -241

No files matched your search

+143 -180
View File
@@ -5,55 +5,41 @@
#include "lib/jetfix/common/InMsgBuffer.h"
#include "lib/jetfix/common/OutFIX.h"
#include "lib/jetfix/store/session_name.h"
#include "lib/os/os.h"
#include "lib/utils/bts_chrono.h"
#include "lib/utils/datetime.h"
#include "lib/utils/file_utils.h"
#include "lib/utils/bts_chrono.h"
#include "lib/os/os.h"
#include <fstream>
#include <thread>
#include <boost/date_time.hpp>
#include <boost/date_time/gregorian/gregorian.hpp>
#include <boost/filesystem.hpp>
#include <boost/date_time.hpp>
#include <moodycamel/readerwriterqueue.h>
#include <tbb/concurrent_queue.h>
#include <fmt/format.h>
#include <fstream>
#include <thread>
namespace bts {
namespace jetfix {
struct FIXFileStore::Event
{
Event ()
: out_msgs (), in_msg (),
quit (false),
has_last_msg_processed (false),
last_msg_processed (0),
has_seq_num (false),
seq_num (0)
{}
struct FIXFileStore::Event {
Event()
: out_msgs(), in_msg(), quit(false), has_last_msg_processed(false),
last_msg_processed(0), has_seq_num(false), seq_num(0) {}
Event (Event const &src)
: out_msgs (src.out_msgs),
in_msg (src.in_msg),
quit (src.quit),
has_last_msg_processed (src.has_last_msg_processed),
last_msg_processed (src.last_msg_processed),
has_seq_num (src.has_seq_num),
seq_num (src.seq_num)
{}
Event(Event const &src)
: out_msgs(src.out_msgs), in_msg(src.in_msg), quit(src.quit),
has_last_msg_processed(src.has_last_msg_processed),
last_msg_processed(src.last_msg_processed),
has_seq_num(src.has_seq_num), seq_num(src.seq_num) {}
Event (Event &&src)
: out_msgs (std::move(src.out_msgs)),
in_msg (std::move(src.in_msg)),
quit (src.quit),
has_last_msg_processed (src.has_last_msg_processed),
last_msg_processed (src.last_msg_processed),
has_seq_num (src.has_seq_num),
seq_num (src.seq_num)
{}
Event(Event &&src)
: out_msgs(std::move(src.out_msgs)), in_msg(std::move(src.in_msg)),
quit(src.quit), has_last_msg_processed(src.has_last_msg_processed),
last_msg_processed(src.last_msg_processed),
has_seq_num(src.has_seq_num), seq_num(src.seq_num) {}
Event& operator=(Event const &src)
{
Event &operator=(Event const &src) {
out_msgs = src.out_msgs;
in_msg = src.in_msg;
quit = src.quit;
@@ -64,8 +50,7 @@ struct FIXFileStore::Event
return *this;
}
Event& operator=(Event const &&src)
{
Event &operator=(Event const &&src) {
out_msgs = std::move(src.out_msgs);
in_msg = std::move(src.in_msg);
quit = src.quit;
@@ -84,21 +69,12 @@ struct FIXFileStore::Event
bool has_seq_num;
uint32_t seq_num;
};
struct FIXFileStore::PImpl
{
PImpl(config::JetFIX::Connection const&cfg) :
active (false),
store_called (false),
replay_messages (),
last_seq_num (0),
last_seq_num_processed (0),
seq_num_path (),
fix_store (),
writer_thread (),
event_queue (),
cfg(cfg)
{}
struct FIXFileStore::PImpl {
PImpl(config::JetFIX::Connection const &cfg)
: active(false), store_called(false), replay_messages(), last_seq_num(0),
last_seq_num_processed(0), seq_num_path(), fix_store(),
writer_thread(), event_queue(), cfg(cfg) {}
bool active;
bool store_called;
@@ -108,237 +84,224 @@ struct FIXFileStore::PImpl
std::string seq_num_path;
std::ofstream fix_store;
std::thread writer_thread;
moodycamel::ReaderWriterQueue<Event> event_queue;
tbb::concurrent_queue<Event> event_queue;
config::JetFIX::Connection const cfg;
};
std::string fix_log_path (std::string const&base_path)
{
std::string fix_log_path(std::string const &base_path) {
boost::filesystem::path result = base_path;
result /= "fix";
return result.string ();
return result.string();
}
std::string seqnum_path (std::string const&base_path)
{
std::string seqnum_path(std::string const &base_path) {
boost::filesystem::path result = base_path;
result /= "seqnum";
return result.string ();
return result.string();
}
std::string store_path (std::string base_path,
std::string const&connection_name,
std::string const&session_name)
{
if (base_path.empty ())
base_path = os::bts_data_path ();
boost::filesystem::path result = utils::normalize_data_path (base_path);
std::string store_path(std::string base_path,
std::string const &connection_name,
std::string const &session_name) {
if (base_path.empty())
base_path = os::bts_data_path();
boost::filesystem::path result = utils::normalize_data_path(base_path);
result /= "jetfix";
result /= connection_name;
result /= session_name;
return result.string ();
return result.string();
}
std::map<uint32_t,InMsgBuffer>
load_replay (config::JetFIX::Connection const &cfg,std::string const&path, uint32_t max_sn)
{
std::map<uint32_t,InMsgBuffer> result;
if (boost::filesystem::exists (path)) {
std::map<uint32_t, InMsgBuffer>
load_replay(config::JetFIX::Connection const &cfg, std::string const &path,
uint32_t max_sn) {
std::map<uint32_t, InMsgBuffer> result;
if (boost::filesystem::exists(path)) {
std::ifstream f (path);
std::ifstream f(path);
std::string line_buffer;
line_buffer.reserve (10*1024);
auto header_size = sizeof("20150427-19:05:17.032 < ")-1;
std::string date_buffer ("20150427-19:05:17.032");
line_buffer.reserve(10 * 1024);
auto header_size = sizeof("20150427-19:05:17.032 < ") - 1;
std::string date_buffer("20150427-19:05:17.032");
for (;;) {
std::getline (f,line_buffer);
if (!f.eof ()) {
bts_assert (line_buffer.size () > header_size);
if (line_buffer[22]=='<') {
auto ts = datetime::parse_FIX_timestamp (line_buffer.substr (0,21));
if (!datetime::is_at_least_secs_old (ts,cfg.seconds_to_replay ())) {
InMsgBuffer msg (line_buffer.substr (24));
auto sn = msg.get_seq_num ();
std::getline(f, line_buffer);
if (!f.eof()) {
bts_assert(line_buffer.size() > header_size);
if (line_buffer[22] == '<') {
auto ts =
datetime::parse_FIX_timestamp(line_buffer.substr(0, 21));
if (!datetime::is_at_least_secs_old(ts,
cfg.seconds_to_replay())) {
InMsgBuffer msg(line_buffer.substr(24));
auto sn = msg.get_seq_num();
if (sn <= max_sn)
result[sn] = msg;
}
}
}
else
} else
break;
}
}
return result;
}
void save_fix (std::ostream &out, const char*data, size_t size, bool is_outgoing)
{
out << datetime::FIX_timestamp (datetime::now ()) << " "
void save_fix(std::ostream &out, const char *data, size_t size,
bool is_outgoing) {
out << datetime::FIX_timestamp(datetime::now()) << " "
<< (is_outgoing ? ">" : "<") << " ";
out.write (data,size);
out.write(data, size);
out << std::endl;
}
void save_seq_nums (std::string const&path,uint32_t last_seq_num, uint32_t last_seq_num_processed)
{
std::ofstream f (path);
void save_seq_nums(std::string const &path, uint32_t last_seq_num,
uint32_t last_seq_num_processed) {
std::ofstream f(path);
f << last_seq_num << " " << last_seq_num_processed << std::endl;
}
void load_seq_nums (std::string const&path,uint32_t* last_seq_num, uint32_t *last_seq_num_processed)
{
if (boost::filesystem::exists (path)) {
std::ifstream f (path);
void load_seq_nums(std::string const &path, uint32_t *last_seq_num,
uint32_t *last_seq_num_processed) {
if (boost::filesystem::exists(path)) {
std::ifstream f(path);
f >> *last_seq_num >> *last_seq_num_processed;
} else {
*last_seq_num = 0;
*last_seq_num_processed = 0;
save_seq_nums (path, *last_seq_num,*last_seq_num_processed);
save_seq_nums(path, *last_seq_num, *last_seq_num_processed);
}
}
FIXFileStore::FIXFileStore (config::JetFIX::Connection const&cfg,std::string base_path)
: me (new PImpl(cfg)) {
auto sn = session_name (cfg);
me->active = sn.size () > 0;
FIXFileStore::FIXFileStore(config::JetFIX::Connection const &cfg,
std::string base_path)
: me(new PImpl(cfg)) {
auto sn = session_name(cfg);
me->active = sn.size() > 0;
if (me->active) {
auto path = store_path (base_path,cfg.name (),sn);
if (!boost::filesystem::exists (path))
bts_assert (boost::filesystem::create_directories (path));
auto path = store_path(base_path, cfg.name(), sn);
if (!boost::filesystem::exists(path))
bts_assert(boost::filesystem::create_directories(path));
// load seq numbers
auto seq_num_p = seqnum_path (path);
load_seq_nums (seq_num_p,&me->last_seq_num,&me->last_seq_num_processed);
auto seq_num_p = seqnum_path(path);
load_seq_nums(seq_num_p, &me->last_seq_num, &me->last_seq_num_processed);
me->seq_num_path = seq_num_p;
// load replay
auto fix_log_p = fix_log_path (path);
auto replay_messages = load_replay (cfg,fix_log_p,me->last_seq_num_processed);
auto fix_log_p = fix_log_path(path);
auto replay_messages =
load_replay(cfg, fix_log_p, me->last_seq_num_processed);
uint32_t previous_sn = 0;
for (auto &p : replay_messages) {
auto sn = p.first;
auto &m = p.second;
if (!(m.has_poss_dup_flag_flag () && m.get_poss_dup_flag ())) {
if (previous_sn != 0)
bts_assert (sn == (previous_sn + 1));
if (!(m.has_poss_dup_flag_flag() && m.get_poss_dup_flag())) {
if (previous_sn != 0)
bts_assert(sn == (previous_sn + 1));
previous_sn = sn;
me->replay_messages.emplace_back (m);
me->replay_messages.emplace_back(m);
}
}
me->fix_store.open (fix_log_p,std::ofstream::app);
me->writer_thread = std::thread (std::bind (&FIXFileStore::write,this));
me->fix_store.open(fix_log_p, std::ofstream::app);
me->writer_thread = std::thread(std::bind(&FIXFileStore::write, this));
}
}
FIXFileStore::~FIXFileStore ()
{
FIXFileStore::~FIXFileStore() {
if (me->active) {
Event ev;
ev.quit = true;
me->event_queue.enqueue (ev);
me->writer_thread.join ();
me->event_queue.push(ev);
me->writer_thread.join();
}
}
bool FIXFileStore::session_active () const
{return me->active;}
bool FIXFileStore::session_active() const { return me->active; }
std::vector<InMsgBuffer> FIXFileStore::replay () const
{
bts_assert (me->active && !me->store_called);
std::vector<InMsgBuffer> FIXFileStore::replay() const {
bts_assert(me->active && !me->store_called);
return me->replay_messages;
}
uint32_t FIXFileStore::last_seq_num () const
{
bts_assert (me->active && !me->store_called);
uint32_t FIXFileStore::last_seq_num() const {
bts_assert(me->active && !me->store_called);
return me->last_seq_num;
}
uint32_t FIXFileStore::last_seq_num_processed () const
{
bts_assert (me->active && !me->store_called);
uint32_t FIXFileStore::last_seq_num_processed() const {
bts_assert(me->active && !me->store_called);
return me->last_seq_num_processed;
}
void FIXFileStore::store (OutFIXPtr const&msgs)
{
bts_assert (me->active);
void FIXFileStore::store(OutFIXPtr const &msgs) {
bts_assert(me->active);
me->store_called = true;
Event ev;
ev.out_msgs = msgs;
me->event_queue.enqueue (ev);
me->event_queue.push(ev);
}
void FIXFileStore::store (InMsgBuffer const&msg)
{
bts_assert (me->active);
void FIXFileStore::store(InMsgBuffer const &msg) {
bts_assert(me->active);
me->store_called = true;
Event ev;
ev.in_msg = msg;
me->event_queue.enqueue (ev);
me->event_queue.push(ev);
}
void FIXFileStore::set_last_msg_processed (uint32_t last_msg_processed)
{
bts_assert (me->active);
void FIXFileStore::set_last_msg_processed(uint32_t last_msg_processed) {
bts_assert(me->active);
me->store_called = true;
Event ev;
ev.has_last_msg_processed = true;
ev.last_msg_processed = last_msg_processed;
me->event_queue.enqueue (ev);
me->event_queue.push(ev);
}
void FIXFileStore::set_last_seq_num(uint32_t last_seq_num)
{
bts_assert (me->active);
void FIXFileStore::set_last_seq_num(uint32_t last_seq_num) {
bts_assert(me->active);
me->store_called = true;
Event ev;
ev.has_seq_num = true;
ev.seq_num = last_seq_num;
me->event_queue.enqueue (ev);
me->event_queue.push(ev);
}
void FIXFileStore::write ()
{
void FIXFileStore::write() {
Event ev;
os::set_thread_name(fmt::format("{0}_store",me->cfg.name()).c_str());
os::set_thread_name(fmt::format("{0}_store", me->cfg.name()).c_str());
while (true)
if (me->event_queue.try_dequeue (ev)) {
if (me->event_queue.try_pop(ev)) {
if (ev.quit) {
return;
}
else if (ev.out_msgs) {
} else if (ev.out_msgs) {
for (auto &encoded_stream : ev.out_msgs->fix)
for (auto &mm : encoded_stream.msg_map ()) {
me->last_seq_num = std::max (me->last_seq_num,mm.seq_num);
save_seq_nums (me->seq_num_path,me->last_seq_num,me->last_seq_num_processed);
save_fix (me->fix_store,encoded_stream.data ()+mm.offset,mm.size,true);
for (auto &mm : encoded_stream.msg_map()) {
me->last_seq_num = std::max(me->last_seq_num, mm.seq_num);
save_seq_nums(me->seq_num_path, me->last_seq_num,
me->last_seq_num_processed);
save_fix(me->fix_store, encoded_stream.data() + mm.offset,
mm.size, true);
}
me->fix_store.flush ();
}
else if (ev.in_msg.size ())
{
save_fix (me->fix_store,ev.in_msg.data (),ev.in_msg.size (),false);
me->fix_store.flush ();
}
else if (ev.has_last_msg_processed) {
me->fix_store.flush();
} else if (ev.in_msg.size()) {
save_fix(me->fix_store, ev.in_msg.data(), ev.in_msg.size(), false);
me->fix_store.flush();
} else if (ev.has_last_msg_processed) {
me->last_seq_num_processed = ev.last_msg_processed;
save_seq_nums (me->seq_num_path,me->last_seq_num,me->last_seq_num_processed);
}
else if (ev.has_seq_num) {
save_seq_nums(me->seq_num_path, me->last_seq_num,
me->last_seq_num_processed);
} else if (ev.has_seq_num) {
me->last_seq_num = ev.seq_num;
save_seq_nums (me->seq_num_path,me->last_seq_num,me->last_seq_num_processed);
}
else
bts_assert (false);
}
else
std::this_thread::sleep_for (100ms);
save_seq_nums(me->seq_num_path, me->last_seq_num,
me->last_seq_num_processed);
} else
bts_assert(false);
} else
std::this_thread::sleep_for(100ms);
}
}}
} // namespace jetfix
} // namespace bts