Files
spark/src/server/sparkd/jetfix/gateway/Session.cpp
T
2017-08-25 15:25:00 -04:00

803 lines
23 KiB
C++

#include "Session.h"
#include "lib/config/config.h"
#include "server/sparkd/jetfix/codec/FIXEncodingStream.h"
#include "server/sparkd/jetfix/codec/FieldDefinitions.h"
#include "server/sparkd/jetfix/codec/MessageTypes.h"
#include "server/sparkd/jetfix/common/OutFIX.h"
#include "server/sparkd/jetfix/connection/IConnection.h"
#include "server/sparkd/jetfix/gateway/ICoreGatewayEventSink.h"
#include "server/sparkd/jetfix/store/IFIXStore.h"
#include "lib/utils/TimeStamp.h"
#include "lib/utils/datetime.h"
#include "lib/logging/logging.h"
#include "lib/config/protos/jetfix.pb.h"
#include <boost/format.hpp>
#include <set>
namespace bts {
namespace jetfix {
OutFIXPtr make_logon (config::JetFIX::Connection const &session_cfg);
OutFIXPtr make_logout (config::JetFIX::Connection const &session_cfg);
OutFIXPtr make_resend_request (config::JetFIX::Connection const &session_cfg, uint32_t min, uint32_t max);
OutFIXPtr make_hb (config::JetFIX::Connection const &session_cfg, std::string const&req_id);
OutFIXPtr make_hb (config::JetFIX::Connection const &session_cfg);
OutFIXPtr make_test_request (config::JetFIX::Connection const &session_cfg,std::string const&tr_id);
OutFIXPtr make_seq_reset (config::JetFIX::Connection const &session_cfg, uint32_t end);
enum class Session::State {WLOGON,RECOVER,WTR_REPLY,READY,TERMINATED,WLOGOUT};
struct Session::RecoveryState
{
RecoveryState ()
: buffered_msgs (),
resend_request (0,0)
{}
std::map<uint32_t,InMsgBuffer> buffered_msgs;
std::pair<uint32_t,uint32_t> resend_request;
};
struct Session::PImpl
{
PImpl(config::JetFIX::Connection const &cfg,
IConnection &connection,
IFIXStore &store,
ICoreGatewayEventSink &sink,
Clock const &clock)
:
cfg (cfg),
state (State::TERMINATED),
connection (connection),
store (store),
event_sink (sink),
now (clock),
begin_of_this_session (),
last_seq_num (),
last_seq_num_processed (),
last_sent_msg_timestamp (),
last_sent_sync_timestamp (),
last_rcvd_msg_timestamp (),
start_of_logon (),
start_of_logout (),
current_time (std::make_shared<std::string> (
datetime::FIX_timestamp (datetime::now ()))),
recovery_state ()
{
}
config::JetFIX::Connection cfg;
State state;
IConnection &connection;
IFIXStore &store;
ICoreGatewayEventSink &event_sink;
Clock now;
uint32_t begin_of_this_session;
uint32_t last_seq_num;
uint32_t last_seq_num_processed;
utils::TimeStamp last_sent_msg_timestamp;
utils::TimeStamp last_sent_sync_timestamp;
utils::TimeStamp last_rcvd_msg_timestamp;
utils::TimeStamp start_of_logon;
utils::TimeStamp start_of_logout;
std::shared_ptr<std::string> current_time;
std::unique_ptr<RecoveryState> recovery_state;
};
Session::Session(config::JetFIX::Connection const &cfg,
IConnection &connection,
IFIXStore &store,
ICoreGatewayEventSink &sink,
Clock const &clock)
: me (new PImpl(cfg,connection,store,sink,clock))
{}
Session::~Session ()
{
if (me->state != State::TERMINATED)
terminate_session (JetFIXError::ERROR_NONE);
}
void Session::start ()
{
bts_assert (me->state == State::TERMINATED);
me->begin_of_this_session = me->store.last_seq_num ();
me->start_of_logon = me->now ();
me->last_seq_num = me->begin_of_this_session;
me->last_seq_num_processed = me->store.last_seq_num_processed ();
transition (State::WLOGON);
internal_send (make_logon (me->cfg));
}
void Session::send (OutFIXPtr const&msgs)
{
switch (me->state)
{
case State::READY:
internal_send (msgs);
break;
case State::TERMINATED:
case State::WLOGOUT:
break;
case State::WLOGON:
case State::RECOVER:
case State::WTR_REPLY:
default:
bts_assert (false);
}
}
void Session::receive (InMsgBuffer const&msg)
{
me->store.store (msg);
me->last_rcvd_msg_timestamp = me->now ();
if (me->state != State::WLOGON &&
msg.get_seq_num () <= me->last_seq_num_processed)
{
LOG (Jetfix,Error, "Discarding repeated message: {} sn: {} state: {} conn: {}",
(char) msg.type (), msg.get_seq_num (),
state_name (me->state), me->cfg.name ());
return;
}
switch (msg.type ()) {
case MessageTypes::Logon:
on_logon (msg);
break;
case MessageTypes::Logout:
on_logout (msg);
break;
case MessageTypes::TestRequest:
on_test_request (msg);
break;
case MessageTypes::ResendRequest:
on_resend_request (msg);
break;
case MessageTypes::SeqReset:
on_seq_reset (msg);
break;
case MessageTypes::HB:
on_hb (msg);
break;
default:
on_app (msg);
break;
}
}
void Session::shutdown ()
{
switch (me->state)
{
case State::WLOGON:
case State::RECOVER:
case State::WTR_REPLY:
case State::READY:
internal_send (make_logout (me->cfg));
transition (State::WLOGOUT);
break;
case State::WLOGOUT:
break;
case State::TERMINATED:
default:
bts_assert (false);
}
}
void Session::time (std::shared_ptr<std::string> const&time_str)
{
me->current_time = time_str;
switch (me->state)
{
case State::WLOGON:
if (seconds_elapsed (me->start_of_logon) > me->cfg.seconds_to_logon ())
logout (JetFIXError::LOGIN_TIMEOUT);
break;
case State::RECOVER:
update_resend_request_progress ();
if (!resend_request_active ())
send_resend_request ();
break;
case State::WTR_REPLY:
if (seconds_elapsed (me->last_sent_sync_timestamp) > me->cfg.seconds_between_syncs ())
send_sync_test_request ();
break;
case State::READY:
if (seconds_elapsed (me->last_sent_msg_timestamp) > me->cfg.seconds_between_hbs ())
internal_send (make_hb (me->cfg));
else if (seconds_elapsed (me->last_rcvd_msg_timestamp) >
me->cfg.seconds_between_hbs () * me->cfg.max_missed_hbs ())
logout (JetFIXError::HB_TIMEOUT);
break;
case State::WLOGOUT:
if (seconds_elapsed (me->start_of_logout) > me->cfg.seconds_to_logout ())
terminate_session (JetFIXError::LOGOUT_TIMEOUT);
break;
case State::TERMINATED:
break;
default:
bts_assert (false);
}
}
void Session::on_logon (InMsgBuffer const&msg)
{
switch (me->state)
{
case State::WLOGON:
{
auto their_last = msg.get_seq_num ();
auto our_last_processed = me->last_seq_num_processed;
if (their_last <= our_last_processed)
terminate_session (JetFIXError::INCOMING_SN_TOO_SMALL,
(boost::format ("Incoming sequence number was %d "
"but the last one we processed was %d. "
"Most likely the FIX sequencer numbers file "
"(.bts/data/jetfix/connection_name/current/seqnum) "
"was deleted. Update this file with the correct "
"incoming sequence number (second column)")
% their_last % our_last_processed).str ());
else if (their_last == (our_last_processed+1))
{
transition (State::WTR_REPLY);
send_sync_test_request ();
deliver (msg);
}
else {
me->recovery_state.reset (new RecoveryState ());
transition (State::RECOVER);
buffer (msg);
send_resend_request ();
}
break;
}
case State::RECOVER:
case State::WTR_REPLY:
case State::READY:
terminate_session (JetFIXError::UNEXPECTED_LOGON);
break;
case State::WLOGOUT:
deliver (msg);
break;
case State::TERMINATED:
break;
default:
bts_assert (false);
break;
}
}
void Session::on_logout (InMsgBuffer const&msg)
{
switch (me->state)
{
case State::WLOGON:
terminate_session (JetFIXError::LOGGED_OUT_BY_EXCHANGE,
(boost::format (". %s")
% msg.get_text ()).str ());
break;
case State::RECOVER:
internal_send (make_logout (me->cfg));
buffer (msg);
terminate_session (JetFIXError::LOGGED_OUT_BY_EXCHANGE,
(boost::format (". %s")
% msg.get_text ()).str ());
break;
case State::WTR_REPLY:
case State::READY:
internal_send (make_logout (me->cfg));
deliver (msg);
terminate_session (JetFIXError::LOGGED_OUT_BY_EXCHANGE,
(boost::format (". %s")
% msg.get_text ()).str ());
break;
case State::WLOGOUT:
deliver (msg);
terminate_session (JetFIXError::ERROR_NONE);
break;
case State::TERMINATED:
break;
default:
bts_assert (false);
break;
}
}
void Session::on_test_request (InMsgBuffer const&msg)
{
switch (me->state)
{
case State::WLOGON:
terminate_session (JetFIXError::EXPECTED_LOGON,
(boost::format ("Received message type %c ")
% (char) msg.type ()).str ());
break;
case State::RECOVER:
internal_send (make_hb (me->cfg,msg.get_test_req_id ()));
buffer (msg);
break;
case State::WTR_REPLY:
case State::READY:
deliver (msg);
internal_send (make_hb (me->cfg,msg.get_test_req_id ()));
break;
case State::WLOGOUT:
deliver (msg);
break;
case State::TERMINATED:
break;
default:
bts_assert (false);
}
}
void Session::on_resend_request(InMsgBuffer const&msg)
{
switch (me->state)
{
case State::WLOGON:
terminate_session (JetFIXError::EXPECTED_LOGON,
(boost::format ("Received message type %c ")
% (int) msg.type ()).str ());
break;
case State::RECOVER:
case State::WTR_REPLY:
case State::READY:
{
if (me->state == State::RECOVER)
buffer (msg);
else
deliver (msg);
auto last_msg_in_request = msg.get_end_seq_no ();
if (last_msg_in_request <= me->begin_of_this_session)
send_seq_reset (msg.get_begin_seq_no (), msg.get_end_seq_no ());
else
logout (JetFIXError::UNEXPECTED_RESEND_REQUEST,
(boost::format ("Received resend request for messages "
"%d to %d but my initial sequence number is %d ")
% msg.get_begin_seq_no ()
% msg.get_end_seq_no ()
% me->begin_of_this_session).str ());
break;
}
case State::WLOGOUT:
deliver (msg);
break;
case State::TERMINATED:
break;
default:
bts_assert (false);
}
}
void Session::on_seq_reset (InMsgBuffer const&msg)
{
switch (me->state)
{
case State::WLOGON:
terminate_session (JetFIXError::EXPECTED_LOGON,
(boost::format ("Received message type %c ")
% (int) msg.type ()).str ());
break;
case State::RECOVER:
buffer_or_deliver (msg);
break;
case State::WTR_REPLY:
case State::READY:
deliver (msg);
logout (JetFIXError::UNEXPECTED_SEQ_RESET);
break;
case State::WLOGOUT:
deliver (msg);
break;
case State::TERMINATED:
break;
default:
bts_assert (false);
}
}
void Session::on_hb (InMsgBuffer const &msg)
{
switch (me->state)
{
case State::WLOGON:
terminate_session (JetFIXError::EXPECTED_LOGON,
(boost::format ("Received message type %c ")
% (int) msg.type ()).str ());
break;
case State::RECOVER:
buffer (msg);
break;
case State::WTR_REPLY:
deliver (msg);
if (msg.has_test_req_id () &&
msg.get_test_req_id () == "sync")
{
me->event_sink.session_ready ();
transition (State::READY);
}
break;
case State::READY:
case State::WLOGOUT:
deliver (msg);
break;
case State::TERMINATED:
break;
default:
bts_assert (false);
}
}
void Session::on_app (InMsgBuffer const &msg)
{
switch (me->state)
{
case State::WLOGON:
terminate_session (JetFIXError::EXPECTED_LOGON,
(boost::format ("Received message type %c ")
% (int) msg.type ()).str ());
break;
case State::RECOVER:
buffer_or_deliver (msg);
break;
case State::WTR_REPLY:
case State::READY:
case State::WLOGOUT:
deliver (msg);
break;
case State::TERMINATED:
break;
default:
bts_assert (false);
}
}
void Session::deliver (InMsgBuffer const&msg)
{
auto sn = msg.get_seq_num ();
bts_assert (sn == me->last_seq_num_processed+1);
me->last_seq_num_processed++;
me->store.set_last_msg_processed (me->last_seq_num_processed);
me->event_sink.in_msg_sequenced (msg);
}
void Session::buffer (InMsgBuffer const &msg)
{
bts_assert (me->state == State::RECOVER);
bts_assert (msg.type () != MessageTypes::SeqReset);
auto sn = msg.get_seq_num ();
bts_assert (sn > me->last_seq_num_processed+1);
bts_assert (me->recovery_state);
me->recovery_state->buffered_msgs[sn] = msg;
bts_assert (me->recovery_state->buffered_msgs.begin ()->first > me->last_seq_num_processed+1);
}
void Session::buffer_or_deliver (InMsgBuffer const&msg)
{
bts_assert (me->state == State::RECOVER);
auto sn = msg.get_seq_num ();
me->recovery_state->buffered_msgs[sn] = msg;
std::set<uint32_t> to_remove;
for (auto &p : me->recovery_state->buffered_msgs) {
auto sn = p.first;
auto msg = p.second;
if (sn == me->last_seq_num_processed+1) {
to_remove.insert (sn);
deliver (msg);
if (msg.type () == MessageTypes::SeqReset) {
me->last_seq_num_processed = msg.get_new_seq_no ()-1;
me->store.set_last_msg_processed (me->last_seq_num_processed);
}
}
else if (sn <= me->last_seq_num_processed) {
// a sequence reset might have advanced us past
// buffered admin messages
to_remove.insert (sn);
}
else
break;
}
for (auto sn : to_remove)
me->recovery_state->buffered_msgs.erase (sn);
if (me->recovery_state->buffered_msgs.empty ())
{
me->recovery_state.reset ();
transition (State::WTR_REPLY);
send_sync_test_request ();
}
}
void Session::internal_send (OutFIXPtr const&msgs)
{
IConnection::WriteVec data_vec;
bts_assert (msgs->fix.size () < IConnection::MaxWriteVecSz);
msgs->set_ts (OutFIX::PRE_STAMP,utils::TimeStamp::now ());
for (size_t i = 0; i < msgs->fix.size ();i++) {
auto &s = msgs->fix[i];
me->last_seq_num =
s.stamp (me->last_seq_num,
me->last_seq_num_processed,
*me->current_time);
data_vec[i].first = s.data ();
data_vec[i].second = s.size ();
}
auto correlation_id = msgs->get_ts (jetfix::OutFIX::MD_WRITE_TS).to_uint64 ();
msgs->set_ts (OutFIX::POST_STAMP,utils::TimeStamp::now ());
if (me->connection.write (data_vec,msgs->fix.size (),correlation_id) < 0) {
terminate_session (JetFIXError::CONNECTION_ERROR);
return;
}
msgs->set_ts (OutFIX::POST_SEND,utils::TimeStamp::now ());
me->last_sent_msg_timestamp = me->now ();
me->store.store (msgs);
msgs->set_ts (OutFIX::POST_STORE,utils::TimeStamp::now ());
}
void Session::logout (JetFIXError::Code error)
{
logout (error,"");
}
void Session::logout (JetFIXError::Code error,std::string const &extra_info)
{
auto level = Info;
if (error != JetFIXError::ERROR_NONE && error != JetFIXError::USER_DISCONNECTED)
level = Error;
else
level = Info;
LOG (Jetfix,level,"Logging out of jetfix session: {} {} conn: {}",
JetFIXError::code_msg (error), extra_info, me->cfg.name ());
me->start_of_logout = me->now ();
internal_send (make_logout (me->cfg));
transition (State::WLOGOUT);
}
void Session::terminate_session (JetFIXError::Code reason)
{terminate_session (reason,"");}
void Session::terminate_session (JetFIXError::Code error,std::string const&extra_info)
{
me->recovery_state.reset ();
auto level = Info;
if (error != JetFIXError::ERROR_NONE && error != JetFIXError::USER_DISCONNECTED)
level = Error;
else
level = Info;
LOG (Jetfix,level,"Jetfix session disconnected: {} {} conn: {}",
JetFIXError::code_msg (error), extra_info, me->cfg.name ());
transition (State::TERMINATED);
me->event_sink.session_terminated ();
}
int32_t Session::seconds_elapsed (utils::TimeStamp since)
{return (me->now () - since).seconds ();}
void Session::send_seq_reset (uint32_t sn, uint32_t new_sn)
{
int end_seq_n;
if (new_sn == 0)
end_seq_n = me->last_seq_num+1;
else
end_seq_n = new_sn + 1;
auto msg = make_seq_reset (me->cfg,end_seq_n);
auto &stream = msg->fix[0];
stream.stamp (sn-1,me->last_seq_num_processed,*me->current_time);
IConnection::WriteVec data_vec;
data_vec[0].first = stream.data ();
data_vec[0].second = stream.size ();
if (me->connection.write (data_vec,1,0) < 0) {
terminate_session (JetFIXError::CONNECTION_ERROR);
return;
} else {
me->last_sent_msg_timestamp = me->now ();
me->store.store (msg);
}
}
void Session::send_resend_request ()
{
bts_assert (me->state == State::RECOVER);
bts_assert (!resend_request_active ());
auto gap = find_first_incoming_msg_gap ();
bts_assert (gap.first);
bts_assert (gap.second);
// adjust to exchange limit on max messages in request
if (gap.second > (gap.first + me->cfg.max_msgs_in_resend_req ()))
gap.second = gap.first + me->cfg.max_msgs_in_resend_req ();
LOG (Jetfix,Warn,
"Issuing recovery request in connection {} "
" asking for messages with sequence numbers from {} to {}" ,
me->cfg.name (), gap.first, gap.second);
internal_send (make_resend_request (me->cfg,gap.first, gap.second));
me->recovery_state->resend_request = gap;
}
void Session::send_sync_test_request ()
{
bts_assert (me->state == State::WTR_REPLY);
me->last_sent_sync_timestamp = me->now ();
internal_send (make_test_request (me->cfg,"sync"));
}
void Session::update_resend_request_progress ()
{
bts_assert (me->state == State::RECOVER);
if (resend_request_active ()) {
auto request_last = me->recovery_state->resend_request.second;
if (me->last_seq_num_processed >= request_last)
me->recovery_state->resend_request = std::make_pair (0,0);
}
}
std::pair<uint32_t,uint32_t> Session::find_first_incoming_msg_gap ()
{
if (me->recovery_state->buffered_msgs.empty ())
return std::make_pair (0u,0u);
else
return std::make_pair (me->last_seq_num_processed + 1,
me->recovery_state->buffered_msgs.begin ()->first-1);
}
bool Session::resend_request_active ()
{return me->recovery_state->resend_request.first != 0;}
void Session::transition (State new_state)
{
LOG (Jetfix,Warn,"Jetfix connection {} transitioning from {} to {}",
me->cfg.name (), state_name (me->state), state_name (new_state));
me->state = new_state;
}
const char * Session::state_name (State state)
{
switch (state)
{
case State::WLOGON:
return "WLOGON";
case State::RECOVER:
return "RECOVER";
case State::WTR_REPLY:
return "WTR_REPLY";
case State::READY:
return "READY";
case State::TERMINATED:
return "TERMINATED";
case State::WLOGOUT:
return "WLOGOUT";
default:
bts_assert (false);
return nullptr;
}
}
OutFIXPtr make_logon (config::JetFIX::Connection const &session_cfg)
{
auto result = std::make_shared<OutFIX> (session_cfg);
auto &stream = result->fix.back ();
auto &buffer = stream.new_msg (MessageTypes::Logon);
using namespace fix;
buffer.add<RawDataLength>(session_cfg.raw_data ().size ());
buffer.add<RawData>(session_cfg.raw_data ());
buffer.add<EncryptMethod> (0);
buffer.add<HeartBtInt> (session_cfg.seconds_between_hbs ());
buffer.add<ApplicationSystemName> ("O2");
buffer.add<ApplicationSystemVersion> ("1.0");
buffer.add<ApplicationSystemVendor> ("BTS");
if (session_cfg.has_trading_session_id()) {
buffer.add<fix::NoTradingSessions> (1);
buffer.add<fix::TradingSessionID> (session_cfg.trading_session_id());
}
if (session_cfg.has_udf_support_indicator())
buffer.add<fix::UDFSupportIndicator> (session_cfg.udf_support_indicator());
stream.close ();
return result;
}
OutFIXPtr make_logout (config::JetFIX::Connection const &session_cfg)
{
auto result = std::make_shared<OutFIX> (session_cfg);
auto stream = &result->fix.back ();
using namespace fix;
stream->new_msg (MessageTypes::Logout);
stream->close ();
return result;
}
OutFIXPtr make_resend_request (config::JetFIX::Connection const &session_cfg, uint32_t min, uint32_t max)
{
auto result = std::make_shared<OutFIX> (session_cfg);
auto &stream = result->fix.back ();
auto &buffer = stream.new_msg (MessageTypes::ResendRequest);
using namespace fix;
buffer.add<BeginSeqNo> (min);
buffer.add<EndSeqNo> (max);
stream.close ();
return result;
}
OutFIXPtr make_hb (config::JetFIX::Connection const &session_cfg, std::string const&req_id)
{
auto result = std::make_shared<OutFIX> (session_cfg);
auto &stream = result->fix.back ();
auto &buffer = stream.new_msg (MessageTypes::HB);
using namespace fix;
buffer.add<TestReqId> (req_id);
stream.close ();
return result;
}
OutFIXPtr make_hb (config::JetFIX::Connection const &session_cfg)
{
auto result = std::make_shared<OutFIX> (session_cfg);
auto &stream = result->fix.back ();
stream.new_msg (MessageTypes::HB);
stream.close ();
return result;
}
OutFIXPtr make_test_request (config::JetFIX::Connection const &session_cfg,std::string const&req_id)
{
auto result = std::make_shared<OutFIX> (session_cfg);
auto &stream = result->fix.back ();
auto &buffer = stream.new_msg (MessageTypes::TestRequest);
using namespace fix;
buffer.add<TestReqId> (req_id);
stream.close ();
return result;
}
OutFIXPtr make_seq_reset (config::JetFIX::Connection const &session_cfg, uint32_t end)
{
auto result = std::make_shared<OutFIX> (session_cfg);
auto &stream = result->fix.back ();
auto &buffer = stream.new_msg (MessageTypes::SeqReset);
using namespace fix;
buffer.add<OrigSendingTime> (datetime::now ());
buffer.add<PossDupFlag>(true);
buffer.add<GapFillFlag>(true);
buffer.add<NewSeqNo>(end);
stream.close ();
return result;
}
}}