#include "Session.h" #include "lib/config/config.h" #include "server/sparkd/jetfix/codec/FIXEncodingStream.h" #include "server/sparkd/jetfix/codec/FieldDefinitions.h" #include "server/sparkd/jetfix/codec/MessageTypes.h" #include "server/sparkd/jetfix/common/OutFIX.h" #include "server/sparkd/jetfix/connection/IConnection.h" #include "server/sparkd/jetfix/gateway/ICoreGatewayEventSink.h" #include "server/sparkd/jetfix/store/IFIXStore.h" #include "lib/utils/TimeStamp.h" #include "lib/utils/datetime.h" #include "lib/logging/logging.h" #include "lib/config/protos/jetfix.pb.h" #include #include namespace bts { namespace jetfix { OutFIXPtr make_logon (config::JetFIX::Connection const &session_cfg); OutFIXPtr make_logout (config::JetFIX::Connection const &session_cfg); OutFIXPtr make_resend_request (config::JetFIX::Connection const &session_cfg, uint32_t min, uint32_t max); OutFIXPtr make_hb (config::JetFIX::Connection const &session_cfg, std::string const&req_id); OutFIXPtr make_hb (config::JetFIX::Connection const &session_cfg); OutFIXPtr make_test_request (config::JetFIX::Connection const &session_cfg,std::string const&tr_id); OutFIXPtr make_seq_reset (config::JetFIX::Connection const &session_cfg, uint32_t end); enum class Session::State {WLOGON,RECOVER,WTR_REPLY,READY,TERMINATED,WLOGOUT}; struct Session::RecoveryState { RecoveryState () : buffered_msgs (), resend_request (0,0) {} std::map buffered_msgs; std::pair resend_request; }; struct Session::PImpl { PImpl(config::JetFIX::Connection const &cfg, IConnection &connection, IFIXStore &store, ICoreGatewayEventSink &sink, Clock const &clock) : cfg (cfg), state (State::TERMINATED), connection (connection), store (store), event_sink (sink), now (clock), begin_of_this_session (), last_seq_num (), last_seq_num_processed (), last_sent_msg_timestamp (), last_sent_sync_timestamp (), last_rcvd_msg_timestamp (), start_of_logon (), start_of_logout (), current_time (std::make_shared ( datetime::FIX_timestamp (datetime::now ()))), recovery_state () { } config::JetFIX::Connection cfg; State state; IConnection &connection; IFIXStore &store; ICoreGatewayEventSink &event_sink; Clock now; uint32_t begin_of_this_session; uint32_t last_seq_num; uint32_t last_seq_num_processed; utils::TimeStamp last_sent_msg_timestamp; utils::TimeStamp last_sent_sync_timestamp; utils::TimeStamp last_rcvd_msg_timestamp; utils::TimeStamp start_of_logon; utils::TimeStamp start_of_logout; std::shared_ptr current_time; std::unique_ptr recovery_state; }; Session::Session(config::JetFIX::Connection const &cfg, IConnection &connection, IFIXStore &store, ICoreGatewayEventSink &sink, Clock const &clock) : me (new PImpl(cfg,connection,store,sink,clock)) {} Session::~Session () { if (me->state != State::TERMINATED) terminate_session (JetFIXError::ERROR_NONE); } void Session::start () { bts_assert (me->state == State::TERMINATED); me->begin_of_this_session = me->store.last_seq_num (); me->start_of_logon = me->now (); me->last_seq_num = me->begin_of_this_session; me->last_seq_num_processed = me->store.last_seq_num_processed (); transition (State::WLOGON); internal_send (make_logon (me->cfg)); } void Session::send (OutFIXPtr const&msgs) { switch (me->state) { case State::READY: internal_send (msgs); break; case State::TERMINATED: case State::WLOGOUT: break; case State::WLOGON: case State::RECOVER: case State::WTR_REPLY: default: bts_assert (false); } } void Session::receive (InMsgBuffer const&msg) { me->store.store (msg); me->last_rcvd_msg_timestamp = me->now (); if (me->state != State::WLOGON && msg.get_seq_num () <= me->last_seq_num_processed) { LOG (Jetfix,Error, "Discarding repeated message: {} sn: {} state: {} conn: {}", (char) msg.type (), msg.get_seq_num (), state_name (me->state), me->cfg.name ()); return; } switch (msg.type ()) { case MessageTypes::Logon: on_logon (msg); break; case MessageTypes::Logout: on_logout (msg); break; case MessageTypes::TestRequest: on_test_request (msg); break; case MessageTypes::ResendRequest: on_resend_request (msg); break; case MessageTypes::SeqReset: on_seq_reset (msg); break; case MessageTypes::HB: on_hb (msg); break; default: on_app (msg); break; } } void Session::shutdown () { switch (me->state) { case State::WLOGON: case State::RECOVER: case State::WTR_REPLY: case State::READY: internal_send (make_logout (me->cfg)); transition (State::WLOGOUT); break; case State::WLOGOUT: break; case State::TERMINATED: default: bts_assert (false); } } void Session::time (std::shared_ptr const&time_str) { me->current_time = time_str; switch (me->state) { case State::WLOGON: if (seconds_elapsed (me->start_of_logon) > me->cfg.seconds_to_logon ()) logout (JetFIXError::LOGIN_TIMEOUT); break; case State::RECOVER: update_resend_request_progress (); if (!resend_request_active ()) send_resend_request (); break; case State::WTR_REPLY: if (seconds_elapsed (me->last_sent_sync_timestamp) > me->cfg.seconds_between_syncs ()) send_sync_test_request (); break; case State::READY: if (seconds_elapsed (me->last_sent_msg_timestamp) > me->cfg.seconds_between_hbs ()) internal_send (make_hb (me->cfg)); else if (seconds_elapsed (me->last_rcvd_msg_timestamp) > me->cfg.seconds_between_hbs () * me->cfg.max_missed_hbs ()) logout (JetFIXError::HB_TIMEOUT); break; case State::WLOGOUT: if (seconds_elapsed (me->start_of_logout) > me->cfg.seconds_to_logout ()) terminate_session (JetFIXError::LOGOUT_TIMEOUT); break; case State::TERMINATED: break; default: bts_assert (false); } } void Session::on_logon (InMsgBuffer const&msg) { switch (me->state) { case State::WLOGON: { auto their_last = msg.get_seq_num (); auto our_last_processed = me->last_seq_num_processed; if (their_last <= our_last_processed) terminate_session (JetFIXError::INCOMING_SN_TOO_SMALL, (boost::format ("Incoming sequence number was %d " "but the last one we processed was %d. " "Most likely the FIX sequencer numbers file " "(.bts/data/jetfix/connection_name/current/seqnum) " "was deleted. Update this file with the correct " "incoming sequence number (second column)") % their_last % our_last_processed).str ()); else if (their_last == (our_last_processed+1)) { transition (State::WTR_REPLY); send_sync_test_request (); deliver (msg); } else { me->recovery_state.reset (new RecoveryState ()); transition (State::RECOVER); buffer (msg); send_resend_request (); } break; } case State::RECOVER: case State::WTR_REPLY: case State::READY: terminate_session (JetFIXError::UNEXPECTED_LOGON); break; case State::WLOGOUT: deliver (msg); break; case State::TERMINATED: break; default: bts_assert (false); break; } } void Session::on_logout (InMsgBuffer const&msg) { switch (me->state) { case State::WLOGON: terminate_session (JetFIXError::LOGGED_OUT_BY_EXCHANGE, (boost::format (". %s") % msg.get_text ()).str ()); break; case State::RECOVER: internal_send (make_logout (me->cfg)); buffer (msg); terminate_session (JetFIXError::LOGGED_OUT_BY_EXCHANGE, (boost::format (". %s") % msg.get_text ()).str ()); break; case State::WTR_REPLY: case State::READY: internal_send (make_logout (me->cfg)); deliver (msg); terminate_session (JetFIXError::LOGGED_OUT_BY_EXCHANGE, (boost::format (". %s") % msg.get_text ()).str ()); break; case State::WLOGOUT: deliver (msg); terminate_session (JetFIXError::ERROR_NONE); break; case State::TERMINATED: break; default: bts_assert (false); break; } } void Session::on_test_request (InMsgBuffer const&msg) { switch (me->state) { case State::WLOGON: terminate_session (JetFIXError::EXPECTED_LOGON, (boost::format ("Received message type %c ") % (char) msg.type ()).str ()); break; case State::RECOVER: internal_send (make_hb (me->cfg,msg.get_test_req_id ())); buffer (msg); break; case State::WTR_REPLY: case State::READY: deliver (msg); internal_send (make_hb (me->cfg,msg.get_test_req_id ())); break; case State::WLOGOUT: deliver (msg); break; case State::TERMINATED: break; default: bts_assert (false); } } void Session::on_resend_request(InMsgBuffer const&msg) { switch (me->state) { case State::WLOGON: terminate_session (JetFIXError::EXPECTED_LOGON, (boost::format ("Received message type %c ") % (int) msg.type ()).str ()); break; case State::RECOVER: case State::WTR_REPLY: case State::READY: { if (me->state == State::RECOVER) buffer (msg); else deliver (msg); auto last_msg_in_request = msg.get_end_seq_no (); if (last_msg_in_request <= me->begin_of_this_session) send_seq_reset (msg.get_begin_seq_no (), msg.get_end_seq_no ()); else logout (JetFIXError::UNEXPECTED_RESEND_REQUEST, (boost::format ("Received resend request for messages " "%d to %d but my initial sequence number is %d ") % msg.get_begin_seq_no () % msg.get_end_seq_no () % me->begin_of_this_session).str ()); break; } case State::WLOGOUT: deliver (msg); break; case State::TERMINATED: break; default: bts_assert (false); } } void Session::on_seq_reset (InMsgBuffer const&msg) { switch (me->state) { case State::WLOGON: terminate_session (JetFIXError::EXPECTED_LOGON, (boost::format ("Received message type %c ") % (int) msg.type ()).str ()); break; case State::RECOVER: buffer_or_deliver (msg); break; case State::WTR_REPLY: case State::READY: deliver (msg); logout (JetFIXError::UNEXPECTED_SEQ_RESET); break; case State::WLOGOUT: deliver (msg); break; case State::TERMINATED: break; default: bts_assert (false); } } void Session::on_hb (InMsgBuffer const &msg) { switch (me->state) { case State::WLOGON: terminate_session (JetFIXError::EXPECTED_LOGON, (boost::format ("Received message type %c ") % (int) msg.type ()).str ()); break; case State::RECOVER: buffer (msg); break; case State::WTR_REPLY: deliver (msg); if (msg.has_test_req_id () && msg.get_test_req_id () == "sync") { me->event_sink.session_ready (); transition (State::READY); } break; case State::READY: case State::WLOGOUT: deliver (msg); break; case State::TERMINATED: break; default: bts_assert (false); } } void Session::on_app (InMsgBuffer const &msg) { switch (me->state) { case State::WLOGON: terminate_session (JetFIXError::EXPECTED_LOGON, (boost::format ("Received message type %c ") % (int) msg.type ()).str ()); break; case State::RECOVER: buffer_or_deliver (msg); break; case State::WTR_REPLY: case State::READY: case State::WLOGOUT: deliver (msg); break; case State::TERMINATED: break; default: bts_assert (false); } } void Session::deliver (InMsgBuffer const&msg) { auto sn = msg.get_seq_num (); bts_assert (sn == me->last_seq_num_processed+1); me->last_seq_num_processed++; me->store.set_last_msg_processed (me->last_seq_num_processed); me->event_sink.in_msg_sequenced (msg); } void Session::buffer (InMsgBuffer const &msg) { bts_assert (me->state == State::RECOVER); bts_assert (msg.type () != MessageTypes::SeqReset); auto sn = msg.get_seq_num (); bts_assert (sn > me->last_seq_num_processed+1); bts_assert (me->recovery_state); me->recovery_state->buffered_msgs[sn] = msg; bts_assert (me->recovery_state->buffered_msgs.begin ()->first > me->last_seq_num_processed+1); } void Session::buffer_or_deliver (InMsgBuffer const&msg) { bts_assert (me->state == State::RECOVER); auto sn = msg.get_seq_num (); me->recovery_state->buffered_msgs[sn] = msg; std::set to_remove; for (auto &p : me->recovery_state->buffered_msgs) { auto sn = p.first; auto msg = p.second; if (sn == me->last_seq_num_processed+1) { to_remove.insert (sn); deliver (msg); if (msg.type () == MessageTypes::SeqReset) { me->last_seq_num_processed = msg.get_new_seq_no ()-1; me->store.set_last_msg_processed (me->last_seq_num_processed); } } else if (sn <= me->last_seq_num_processed) { // a sequence reset might have advanced us past // buffered admin messages to_remove.insert (sn); } else break; } for (auto sn : to_remove) me->recovery_state->buffered_msgs.erase (sn); if (me->recovery_state->buffered_msgs.empty ()) { me->recovery_state.reset (); transition (State::WTR_REPLY); send_sync_test_request (); } } void Session::internal_send (OutFIXPtr const&msgs) { IConnection::WriteVec data_vec; bts_assert (msgs->fix.size () < IConnection::MaxWriteVecSz); msgs->set_ts (OutFIX::PRE_STAMP,utils::TimeStamp::now ()); for (size_t i = 0; i < msgs->fix.size ();i++) { auto &s = msgs->fix[i]; me->last_seq_num = s.stamp (me->last_seq_num, me->last_seq_num_processed, *me->current_time); data_vec[i].first = s.data (); data_vec[i].second = s.size (); } auto correlation_id = msgs->get_ts (jetfix::OutFIX::MD_WRITE_TS).to_uint64 (); msgs->set_ts (OutFIX::POST_STAMP,utils::TimeStamp::now ()); if (me->connection.write (data_vec,msgs->fix.size (),correlation_id) < 0) { terminate_session (JetFIXError::CONNECTION_ERROR); return; } msgs->set_ts (OutFIX::POST_SEND,utils::TimeStamp::now ()); me->last_sent_msg_timestamp = me->now (); me->store.store (msgs); msgs->set_ts (OutFIX::POST_STORE,utils::TimeStamp::now ()); } void Session::logout (JetFIXError::Code error) { logout (error,""); } void Session::logout (JetFIXError::Code error,std::string const &extra_info) { auto level = Info; if (error != JetFIXError::ERROR_NONE && error != JetFIXError::USER_DISCONNECTED) level = Error; else level = Info; LOG (Jetfix,level,"Logging out of jetfix session: {} {} conn: {}", JetFIXError::code_msg (error), extra_info, me->cfg.name ()); me->start_of_logout = me->now (); internal_send (make_logout (me->cfg)); transition (State::WLOGOUT); } void Session::terminate_session (JetFIXError::Code reason) {terminate_session (reason,"");} void Session::terminate_session (JetFIXError::Code error,std::string const&extra_info) { me->recovery_state.reset (); auto level = Info; if (error != JetFIXError::ERROR_NONE && error != JetFIXError::USER_DISCONNECTED) level = Error; else level = Info; LOG (Jetfix,level,"Jetfix session disconnected: {} {} conn: {}", JetFIXError::code_msg (error), extra_info, me->cfg.name ()); transition (State::TERMINATED); me->event_sink.session_terminated (); } int32_t Session::seconds_elapsed (utils::TimeStamp since) {return (me->now () - since).seconds ();} void Session::send_seq_reset (uint32_t sn, uint32_t new_sn) { int end_seq_n; if (new_sn == 0) end_seq_n = me->last_seq_num+1; else end_seq_n = new_sn + 1; auto msg = make_seq_reset (me->cfg,end_seq_n); auto &stream = msg->fix[0]; stream.stamp (sn-1,me->last_seq_num_processed,*me->current_time); IConnection::WriteVec data_vec; data_vec[0].first = stream.data (); data_vec[0].second = stream.size (); if (me->connection.write (data_vec,1,0) < 0) { terminate_session (JetFIXError::CONNECTION_ERROR); return; } else { me->last_sent_msg_timestamp = me->now (); me->store.store (msg); } } void Session::send_resend_request () { bts_assert (me->state == State::RECOVER); bts_assert (!resend_request_active ()); auto gap = find_first_incoming_msg_gap (); bts_assert (gap.first); bts_assert (gap.second); // adjust to exchange limit on max messages in request if (gap.second > (gap.first + me->cfg.max_msgs_in_resend_req ())) gap.second = gap.first + me->cfg.max_msgs_in_resend_req (); LOG (Jetfix,Warn, "Issuing recovery request in connection {} " " asking for messages with sequence numbers from {} to {}" , me->cfg.name (), gap.first, gap.second); internal_send (make_resend_request (me->cfg,gap.first, gap.second)); me->recovery_state->resend_request = gap; } void Session::send_sync_test_request () { bts_assert (me->state == State::WTR_REPLY); me->last_sent_sync_timestamp = me->now (); internal_send (make_test_request (me->cfg,"sync")); } void Session::update_resend_request_progress () { bts_assert (me->state == State::RECOVER); if (resend_request_active ()) { auto request_last = me->recovery_state->resend_request.second; if (me->last_seq_num_processed >= request_last) me->recovery_state->resend_request = std::make_pair (0,0); } } std::pair Session::find_first_incoming_msg_gap () { if (me->recovery_state->buffered_msgs.empty ()) return std::make_pair (0u,0u); else return std::make_pair (me->last_seq_num_processed + 1, me->recovery_state->buffered_msgs.begin ()->first-1); } bool Session::resend_request_active () {return me->recovery_state->resend_request.first != 0;} void Session::transition (State new_state) { LOG (Jetfix,Warn,"Jetfix connection {} transitioning from {} to {}", me->cfg.name (), state_name (me->state), state_name (new_state)); me->state = new_state; } const char * Session::state_name (State state) { switch (state) { case State::WLOGON: return "WLOGON"; case State::RECOVER: return "RECOVER"; case State::WTR_REPLY: return "WTR_REPLY"; case State::READY: return "READY"; case State::TERMINATED: return "TERMINATED"; case State::WLOGOUT: return "WLOGOUT"; default: bts_assert (false); return nullptr; } } OutFIXPtr make_logon (config::JetFIX::Connection const &session_cfg) { auto result = std::make_shared (session_cfg); auto &stream = result->fix.back (); auto &buffer = stream.new_msg (MessageTypes::Logon); using namespace fix; buffer.add(session_cfg.raw_data ().size ()); buffer.add(session_cfg.raw_data ()); buffer.add (0); buffer.add (session_cfg.seconds_between_hbs ()); buffer.add ("O2"); buffer.add ("1.0"); buffer.add ("BTS"); if (session_cfg.has_trading_session_id()) { buffer.add (1); buffer.add (session_cfg.trading_session_id()); } if (session_cfg.has_udf_support_indicator()) buffer.add (session_cfg.udf_support_indicator()); stream.close (); return result; } OutFIXPtr make_logout (config::JetFIX::Connection const &session_cfg) { auto result = std::make_shared (session_cfg); auto stream = &result->fix.back (); using namespace fix; stream->new_msg (MessageTypes::Logout); stream->close (); return result; } OutFIXPtr make_resend_request (config::JetFIX::Connection const &session_cfg, uint32_t min, uint32_t max) { auto result = std::make_shared (session_cfg); auto &stream = result->fix.back (); auto &buffer = stream.new_msg (MessageTypes::ResendRequest); using namespace fix; buffer.add (min); buffer.add (max); stream.close (); return result; } OutFIXPtr make_hb (config::JetFIX::Connection const &session_cfg, std::string const&req_id) { auto result = std::make_shared (session_cfg); auto &stream = result->fix.back (); auto &buffer = stream.new_msg (MessageTypes::HB); using namespace fix; buffer.add (req_id); stream.close (); return result; } OutFIXPtr make_hb (config::JetFIX::Connection const &session_cfg) { auto result = std::make_shared (session_cfg); auto &stream = result->fix.back (); stream.new_msg (MessageTypes::HB); stream.close (); return result; } OutFIXPtr make_test_request (config::JetFIX::Connection const &session_cfg,std::string const&req_id) { auto result = std::make_shared (session_cfg); auto &stream = result->fix.back (); auto &buffer = stream.new_msg (MessageTypes::TestRequest); using namespace fix; buffer.add (req_id); stream.close (); return result; } OutFIXPtr make_seq_reset (config::JetFIX::Connection const &session_cfg, uint32_t end) { auto result = std::make_shared (session_cfg); auto &stream = result->fix.back (); auto &buffer = stream.new_msg (MessageTypes::SeqReset); using namespace fix; buffer.add (datetime::now ()); buffer.add(true); buffer.add(true); buffer.add(end); stream.close (); return result; } }}