319 lines
8.8 KiB
C++
319 lines
8.8 KiB
C++
#include "FIXFileStore.h"
|
|
|
|
#include "lib/config/config.h"
|
|
#include "server/sparkd/jetfix/codec/FIXEncodingStream.h"
|
|
#include "server/sparkd/jetfix/common/InMsgBuffer.h"
|
|
#include "server/sparkd/jetfix/common/OutFIX.h"
|
|
#include "server/sparkd/jetfix/store/session_name.h"
|
|
#include "lib/utils/datetime.h"
|
|
#include "lib/utils/file_utils.h"
|
|
#include "lib/utils/bts_chrono.h"
|
|
#include "lib/os/os.h"
|
|
|
|
#include <fstream>
|
|
#include <thread>
|
|
#include <boost/date_time/gregorian/gregorian.hpp>
|
|
#include <boost/filesystem.hpp>
|
|
#include <boost/date_time.hpp>
|
|
#include <moodycamel/readerwriterqueue.h>
|
|
#include <fmt/format.h>
|
|
|
|
namespace bts {
|
|
namespace jetfix {
|
|
|
|
struct FIXFileStore::Event
|
|
{
|
|
Event ()
|
|
: out_msgs (), in_msg (),
|
|
quit (false),
|
|
has_last_msg_processed (false),
|
|
last_msg_processed (0)
|
|
{}
|
|
|
|
Event (Event const &src)
|
|
: out_msgs (src.out_msgs),
|
|
in_msg (src.in_msg),
|
|
quit (src.quit),
|
|
has_last_msg_processed (src.has_last_msg_processed),
|
|
last_msg_processed (src.last_msg_processed)
|
|
{}
|
|
|
|
Event (Event &&src)
|
|
: out_msgs (std::move(src.out_msgs)),
|
|
in_msg (std::move(src.in_msg)),
|
|
quit (src.quit),
|
|
has_last_msg_processed (src.has_last_msg_processed),
|
|
last_msg_processed (src.last_msg_processed)
|
|
{}
|
|
|
|
Event& operator=(Event const &src)
|
|
{
|
|
out_msgs = src.out_msgs;
|
|
in_msg = src.in_msg;
|
|
quit = src.quit;
|
|
has_last_msg_processed = src.has_last_msg_processed;
|
|
last_msg_processed = src.last_msg_processed;
|
|
return *this;
|
|
}
|
|
|
|
Event& operator=(Event const &&src)
|
|
{
|
|
out_msgs = std::move(src.out_msgs);
|
|
in_msg = std::move(src.in_msg);
|
|
quit = src.quit;
|
|
has_last_msg_processed = src.has_last_msg_processed;
|
|
last_msg_processed = src.last_msg_processed;
|
|
return *this;
|
|
}
|
|
|
|
OutFIXPtr out_msgs;
|
|
InMsgBuffer in_msg;
|
|
bool quit;
|
|
bool has_last_msg_processed;
|
|
uint32_t last_msg_processed;
|
|
};
|
|
|
|
struct FIXFileStore::PImpl
|
|
{
|
|
PImpl(config::JetFIX::Connection const&cfg) :
|
|
active (false),
|
|
store_called (false),
|
|
replay_messages (),
|
|
last_seq_num (0),
|
|
last_seq_num_processed (0),
|
|
seq_num_path (),
|
|
fix_store (),
|
|
writer_thread (),
|
|
event_queue (),
|
|
cfg(cfg)
|
|
{}
|
|
|
|
bool active;
|
|
bool store_called;
|
|
std::vector<InMsgBuffer> replay_messages;
|
|
uint32_t last_seq_num;
|
|
uint32_t last_seq_num_processed;
|
|
std::string seq_num_path;
|
|
std::ofstream fix_store;
|
|
std::thread writer_thread;
|
|
moodycamel::ReaderWriterQueue<Event> event_queue;
|
|
config::JetFIX::Connection const cfg;
|
|
};
|
|
|
|
std::string fix_log_path (std::string const&base_path)
|
|
{
|
|
boost::filesystem::path result = base_path;
|
|
result /= "fix";
|
|
return result.string ();
|
|
}
|
|
|
|
std::string seqnum_path (std::string const&base_path)
|
|
{
|
|
boost::filesystem::path result = base_path;
|
|
result /= "seqnum";
|
|
return result.string ();
|
|
}
|
|
|
|
std::string store_path (std::string base_path,
|
|
std::string const&connection_name,
|
|
std::string const&session_name)
|
|
{
|
|
if (base_path.empty ())
|
|
base_path = os::bts_data_path ();
|
|
boost::filesystem::path result = utils::normalize_data_path (base_path);
|
|
result /= "jetfix";
|
|
result /= connection_name;
|
|
result /= session_name;
|
|
return result.string ();
|
|
}
|
|
|
|
std::map<uint32_t,InMsgBuffer>
|
|
load_replay (config::JetFIX::Connection const &cfg,std::string const&path, uint32_t max_sn)
|
|
{
|
|
std::map<uint32_t,InMsgBuffer> result;
|
|
if (boost::filesystem::exists (path)) {
|
|
|
|
std::ifstream f (path);
|
|
std::string line_buffer;
|
|
line_buffer.reserve (10*1024);
|
|
auto header_size = sizeof("20150427-19:05:17.032 < ")-1;
|
|
std::string date_buffer ("20150427-19:05:17.032");
|
|
|
|
for (;;) {
|
|
std::getline (f,line_buffer);
|
|
if (!f.eof ()) {
|
|
bts_assert (line_buffer.size () > header_size);
|
|
if (line_buffer[22]=='<') {
|
|
auto ts = datetime::parse_FIX_timestamp (line_buffer.substr (0,21));
|
|
if (!datetime::is_at_least_secs_old (ts,cfg.seconds_to_replay ())) {
|
|
InMsgBuffer msg (line_buffer.substr (24));
|
|
auto sn = msg.get_seq_num ();
|
|
if (sn <= max_sn)
|
|
result[sn] = msg;
|
|
}
|
|
}
|
|
}
|
|
else
|
|
break;
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
|
|
void save_fix (std::ostream &out, const char*data, size_t size, bool is_outgoing)
|
|
{
|
|
out << datetime::FIX_timestamp (datetime::now ()) << " "
|
|
<< (is_outgoing ? ">" : "<") << " ";
|
|
out.write (data,size);
|
|
out << std::endl;
|
|
}
|
|
|
|
void save_seq_nums (std::string const&path,uint32_t last_seq_num, uint32_t last_seq_num_processed)
|
|
{
|
|
std::ofstream f (path);
|
|
f << last_seq_num << " " << last_seq_num_processed << std::endl;
|
|
}
|
|
|
|
void load_seq_nums (std::string const&path,uint32_t* last_seq_num, uint32_t *last_seq_num_processed)
|
|
{
|
|
if (boost::filesystem::exists (path)) {
|
|
std::ifstream f (path);
|
|
f >> *last_seq_num >> *last_seq_num_processed;
|
|
} else {
|
|
*last_seq_num = 0;
|
|
*last_seq_num_processed = 0;
|
|
save_seq_nums (path, *last_seq_num,*last_seq_num_processed);
|
|
}
|
|
}
|
|
|
|
FIXFileStore::FIXFileStore (config::JetFIX::Connection const&cfg,std::string base_path)
|
|
: me (new PImpl(cfg)) {
|
|
auto sn = session_name (cfg);
|
|
me->active = sn.size () > 0;
|
|
if (me->active) {
|
|
|
|
auto path = store_path (base_path,cfg.name (),sn);
|
|
if (!boost::filesystem::exists (path))
|
|
bts_assert (boost::filesystem::create_directories (path));
|
|
|
|
// load seq numbers
|
|
auto seq_num_p = seqnum_path (path);
|
|
load_seq_nums (seq_num_p,&me->last_seq_num,&me->last_seq_num_processed);
|
|
me->seq_num_path = seq_num_p;
|
|
|
|
// load replay
|
|
auto fix_log_p = fix_log_path (path);
|
|
auto replay_messages = load_replay (cfg,fix_log_p,me->last_seq_num_processed);
|
|
uint32_t previous_sn = 0;
|
|
for (auto &p : replay_messages) {
|
|
auto sn = p.first;
|
|
auto &m = p.second;
|
|
if (!(m.has_poss_dup_flag_flag () && m.get_poss_dup_flag ())) {
|
|
if (previous_sn != 0)
|
|
bts_assert (sn == (previous_sn + 1));
|
|
previous_sn = sn;
|
|
me->replay_messages.emplace_back (m);
|
|
}
|
|
}
|
|
me->fix_store.open (fix_log_p,std::ofstream::app);
|
|
me->writer_thread = std::thread (std::bind (&FIXFileStore::write,this));
|
|
}
|
|
}
|
|
|
|
FIXFileStore::~FIXFileStore ()
|
|
{
|
|
if (me->active) {
|
|
Event ev;
|
|
ev.quit = true;
|
|
me->event_queue.enqueue (ev);
|
|
me->writer_thread.join ();
|
|
}
|
|
}
|
|
|
|
bool FIXFileStore::session_active () const
|
|
{return me->active;}
|
|
|
|
std::vector<InMsgBuffer> FIXFileStore::replay () const
|
|
{
|
|
bts_assert (me->active && !me->store_called);
|
|
return me->replay_messages;
|
|
}
|
|
|
|
uint32_t FIXFileStore::last_seq_num () const
|
|
{
|
|
bts_assert (me->active && !me->store_called);
|
|
return me->last_seq_num;
|
|
}
|
|
|
|
uint32_t FIXFileStore::last_seq_num_processed () const
|
|
{
|
|
bts_assert (me->active && !me->store_called);
|
|
return me->last_seq_num_processed;
|
|
}
|
|
|
|
void FIXFileStore::store (OutFIXPtr const&msgs)
|
|
{
|
|
bts_assert (me->active);
|
|
me->store_called = true;
|
|
Event ev;
|
|
ev.out_msgs = msgs;
|
|
me->event_queue.enqueue (ev);
|
|
}
|
|
|
|
void FIXFileStore::store (InMsgBuffer const&msg)
|
|
{
|
|
bts_assert (me->active);
|
|
me->store_called = true;
|
|
Event ev;
|
|
ev.in_msg = msg;
|
|
me->event_queue.enqueue (ev);
|
|
}
|
|
|
|
void FIXFileStore::set_last_msg_processed (uint32_t last_msg_processed)
|
|
{
|
|
bts_assert (me->active);
|
|
me->store_called = true;
|
|
Event ev;
|
|
ev.has_last_msg_processed = true;
|
|
ev.last_msg_processed = last_msg_processed;
|
|
me->event_queue.enqueue (ev);
|
|
}
|
|
|
|
void FIXFileStore::write ()
|
|
{
|
|
Event ev;
|
|
os::set_thread_name(fmt::format("{0}_store",me->cfg.name()).c_str());
|
|
|
|
while (true)
|
|
if (me->event_queue.try_dequeue (ev)) {
|
|
if (ev.quit) {
|
|
return;
|
|
}
|
|
else if (ev.out_msgs) {
|
|
for (auto &encoded_stream : ev.out_msgs->fix)
|
|
for (auto &mm : encoded_stream.msg_map ()) {
|
|
me->last_seq_num = std::max (me->last_seq_num,mm.seq_num);
|
|
save_seq_nums (me->seq_num_path,me->last_seq_num,me->last_seq_num_processed);
|
|
save_fix (me->fix_store,encoded_stream.data ()+mm.offset,mm.size,true);
|
|
}
|
|
me->fix_store.flush ();
|
|
}
|
|
else if (ev.in_msg.size ())
|
|
{
|
|
save_fix (me->fix_store,ev.in_msg.data (),ev.in_msg.size (),false);
|
|
me->fix_store.flush ();
|
|
}
|
|
else if (ev.has_last_msg_processed) {
|
|
me->last_seq_num_processed = ev.last_msg_processed;
|
|
save_seq_nums (me->seq_num_path,me->last_seq_num,me->last_seq_num_processed);
|
|
}
|
|
else
|
|
bts_assert (false);
|
|
}
|
|
else
|
|
std::this_thread::sleep_for (100ms);
|
|
}
|
|
|
|
}}
|
|
|