diff --git a/externals b/externals index 67a79263b..6b8c745fb 160000 --- a/externals +++ b/externals @@ -1 +1 @@ -Subproject commit 67a79263ba116a7646beb8368cef015530bea027 +Subproject commit 6b8c745fb2725ad92fc0292b7836e0838e7e1679 diff --git a/src/lib/config/protos/position_cfg.proto b/src/lib/config/protos/position_cfg.proto index a4b9a039a..1bdab848d 100644 --- a/src/lib/config/protos/position_cfg.proto +++ b/src/lib/config/protos/position_cfg.proto @@ -4,7 +4,6 @@ package bts.config; message GMI_Import { - // required string sftp_url= 1; required string sftp_host = 10; optional uint32 sftp_port = 11 [default = 22]; required string sftp_user = 12; diff --git a/src/lib/utils/sftp.cpp b/src/lib/utils/sftp.cpp index a4dd8d42d..6e7605dd6 100644 --- a/src/lib/utils/sftp.cpp +++ b/src/lib/utils/sftp.cpp @@ -43,7 +43,7 @@ DownloadResult sftp_download( uint32_t port, const std::string& user, const std::string& password, - const FilePicker& on_choose) { + const FilePicker& picker) { // resolve host struct hostent* he; @@ -115,7 +115,7 @@ DownloadResult sftp_download( DEFER(libssh2_sftp_shutdown(sftp_session)); // get a directory listing - std::string to_download; + std::vector candidates; { auto sftp_dir_handle = libssh2_sftp_opendir(sftp_session, "."); if (!sftp_dir_handle) { @@ -129,24 +129,20 @@ DownloadResult sftp_download( rc = libssh2_sftp_readdir(sftp_dir_handle, mem, sizeof(mem), &attrs); if (rc > 0) { if (LIBSSH2_SFTP_S_ISREG(attrs.permissions)) { - std::string name { mem, (size_t)rc }; - if (on_choose(name)) { - to_download = std::move(name); - break; - } + candidates.push_back({mem,(size_t)rc}); } } else break; }while(rc > 0); } - - if (to_download.empty()) { + auto to_download = picker(candidates); + if (!to_download.has_value()) { return DownloadFailure { "No file selected" }; } - auto sftp_handle = libssh2_sftp_open(sftp_session, to_download.c_str(), LIBSSH2_FXF_READ, 0); + auto sftp_handle = libssh2_sftp_open(sftp_session, to_download.value().c_str(), LIBSSH2_FXF_READ, 0); if (!sftp_handle) { - return DownloadFailure { fmt::format("Unable to open file '{}' with sftp", to_download) }; + return DownloadFailure { fmt::format("Unable to open file '{}' with sftp", to_download.value()) }; } DEFER(libssh2_sftp_close(sftp_handle)); @@ -164,7 +160,7 @@ DownloadResult sftp_download( else break; } while(true); - return DownloadSuccess { temp.string() }; + return DownloadSuccess { to_download.value(),temp.string() }; } }} diff --git a/src/lib/utils/sftp.h b/src/lib/utils/sftp.h index cbbe0bb78..478f1c43c 100644 --- a/src/lib/utils/sftp.h +++ b/src/lib/utils/sftp.h @@ -9,13 +9,14 @@ namespace bts { namespace utils { struct DownloadSuccess { - std::string const file_path; + std::string const sftp_file_path; + std::string const local_file_path; }; struct DownloadFailure { std::string const error_msg; }; using DownloadResult = std::variant; -using FilePicker = std::function; +using FilePicker = std::function(std::vector const&)>; void begin_sftp_download( const std::string host, @@ -23,7 +24,7 @@ void begin_sftp_download( const std::string& user, const std::string& password, uint32_t time_out_secs, - const FilePicker& on_choose, + const FilePicker& picker, std::function const); DownloadResult sftp_download( diff --git a/src/server/sparkd/cmds/QuickTest.cpp b/src/server/sparkd/cmds/QuickTest.cpp index 212710835..53aefdadc 100644 --- a/src/server/sparkd/cmds/QuickTest.cpp +++ b/src/server/sparkd/cmds/QuickTest.cpp @@ -7,10 +7,12 @@ #include "lib/utils/datetime.h" #include "lib/domain_model/sod.h" +#include "lib/utils/datetime.h" +#include #include #include #include -#include +#include namespace bts { namespace cmd { @@ -18,33 +20,18 @@ using namespace bts::orderman; int QuickTest::execute(std::vector const &) { - auto src = - "P,A,,ACC A,F1,.A020044019L, ,201912, ,F,C, ,4.4,20191122, " - " , ,20190701,0.2125,2, , , ,9G01 CNH70,200, , " - " , , , , , , ,US,C, ,12345, , ,TEST ACCOUNT " - " , , , , , , , , ,0,1, " - "1-,0,0,0,0,0,0,0,1,CALL DEC 19 CORN 440, " - " , ,0, " - " , , " - ",-1581.25,0,5000,20190701,1,0,0,0,0,0.552,20191122, " - ",US,1,2,O, , , ,1A,0, .21 1/4,0,0.31625,20190708, .31 " - "5/8,4.4375, ,0.315,0,440,212,2,8,USD,US, , J, , , , " - ",H,FA,O, , F1,0,9J5L ,99HMC, , , , , ,124Q, " - " , ,C,C,4, , ,115701, , , ,2, , " - " , , , , ,0,-518.75,0,0, , , ,M, , , , , , , , " - ",-2,0,-1.95,-0.02,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,F1,F1,F1,F1,F1,F1,F1," - "F1,F1,F1,F1,F1,F1,F1,F1,F1,F1,F1,M,C,E,N,X,Z, , , , , " - ",B,T,Y,F,D,W,K,C,C,C,M,M,M,M,M,M, " - ",C,M,M,M,M,M,M,1,0,0,0,-2,0,0,0,ATO,BAU,911,162054300,&,19, , , , " - ",1,31,TESTACCT ,ATESDGMIMC,115616,B, ,0,0,1, ,0, ,0,A,A,A,USD,USD,US"; + auto sftp_file_name = std::string("2019-08-05.adatmny.csv"); + std::regex rx("(\\d\\d\\d\\d)"); + std::smatch matches; + if (!std::regex_match(sftp_file_name, matches, rx)) + return {}; + auto y = std::stoi(matches[0]); + auto m = std::stoi(matches[1]); + auto d = std::stoi(matches[2]); - std::vector values; - boost::algorithm::split(values, src ,boost::algorithm::is_any_of(",")); - - for (auto v : values) - std::cout << v << "\n"; - - return 0; + const auto &sod = config::get(); + std::cout << datetime::make_ptime(y, m, d, sod.sod_hh(), sod.has_sod_mm(), 0, 0) << "\n"; + return 1; } std::string QuickTest::help() { diff --git a/src/server/sparkd/position/BasePositionServiceServer.cpp b/src/server/sparkd/position/BasePositionServiceServer.cpp index cbc92591a..01249fa88 100644 --- a/src/server/sparkd/position/BasePositionServiceServer.cpp +++ b/src/server/sparkd/position/BasePositionServiceServer.cpp @@ -16,8 +16,12 @@ #include "lib/utils/sftp.h" #include "server/sparkd/position/gmi_parser.h" +#include #include #include +#include + +#include #include namespace bts { @@ -136,18 +140,13 @@ void BasePositionServiceServer::start_synch() { transition(State::DOWNLOADING); using namespace bts::utils; - - auto sod = sod::get_current_sod(); - const auto& conf = config::get(); + const auto &conf = config::get(); begin_sftp_download( - conf.sftp_host(), - conf.sftp_port(), - conf.sftp_user(), - conf.sftp_password(), - config::get().download_timeout_seconds() * 1000, - [](const auto&) { return true; }, - [this, sod](auto const &result) { + conf.sftp_host(), conf.sftp_port(), conf.sftp_user(), + conf.sftp_password(), conf.download_timeout_seconds() * 1000, + [this](const auto &v) { return pick_most_recent_pos_file(v); }, + [this](auto const &result) { if (std::holds_alternative(result)) { auto error_msg = fmt::format("Failed to download GMI file. Error={}", @@ -155,47 +154,62 @@ void BasePositionServiceServer::start_synch() { LOG(OrderMan, Error, "{}", error_msg); me->ctx.post([this, error_msg]() { on_error(error_msg); }); } else { - auto file_path = std::get(result).file_path; - LOG(OrderMan, Warn, "Succesfully downloaded GMI file to {}", - file_path); - me->ctx.post([this, sod, file_path]() { - on_download_success(sod, file_path); + auto sftp_file_path = + std::get(result).sftp_file_path; + auto local_file_path = + std::get(result).local_file_path; + LOG(OrderMan, Warn, + "Successfully downloaded GMI file to {} to local file {}", + sftp_file_path, local_file_path); + me->ctx.post([this, sftp_file_path, local_file_path]() { + on_download_success(sftp_file_path, local_file_path); }); } }); } void BasePositionServiceServer::on_download_success( - datetime::PTime const &as_of, std::string const &file_path) { + std::string const &sftp_file_path, std::string const &local_file_path) { bts_assert(me->state == State::DOWNLOADING); - auto parse_result = parse_gmi_file(file_path); - if (std::holds_alternative(parse_result)) { - auto base_position = - std::get(parse_result); + auto as_of = get_as_of_date(sftp_file_path); + if (!as_of.has_value()) { + auto error_msg = + fmt::format("Could not extract as of date from sftp file name {}", + sftp_file_path); + on_error(error_msg); + return; + } - LOG(OrderMan, Warn, - "Successfully parsed GMI file at {}. Extracted {} records for the " - "base position", - file_path, base_position.positions_size()); - - transition(State::SYNCHED); - *base_position.mutable_current_as_of() = utils::pb::to_timestamp(as_of); - - me->pos_service.set_base_position(base_position, - [this](auto const &rr) { - if (!rr.success()) - on_error(rr.error_message()); - else if (!rr.value()->ok()) - on_error(rr.value()->msg()); - }, - 5000); - } else { + auto parse_result = parse_gmi_file(local_file_path); + if (!std::holds_alternative( + parse_result)) { auto error_msg = fmt::format("Failed to parse GMI file at {}. Error msg: {}", - file_path, std::get(parse_result)); + local_file_path, std::get(parse_result)); on_error(error_msg); } + + auto base_position = + std::get(parse_result); + + LOG(OrderMan, Warn, + "Successfully parsed GMI file at {}. Extracted {} records for the " + "base position", + local_file_path, base_position.positions_size()); + + transition(State::SYNCHED); + *base_position.mutable_current_as_of() = + utils::pb::to_timestamp(as_of.value()); + + me->pos_service.set_base_position(base_position, + [this](auto const &rr) { + if (!rr.success()) + on_error(rr.error_message()); + else if (!rr.value()->ok()) + on_error(rr.value()->msg()); + }, + 5000); } void BasePositionServiceServer::on_error(std::string const &error_msg) { @@ -208,11 +222,38 @@ void BasePositionServiceServer::on_error(std::string const &error_msg) { start_synch(); } -std::string -BasePositionServiceServer::get_sftp_file_name(datetime::PTime const &sod_time) { - return fmt::format("{:4}{:02}{:02}.csv", sod_time.date().year(), - sod_time.date().month().as_number(), - sod_time.date().day().as_number()); +std::optional +BasePositionServiceServer::get_as_of_date(std::string const &sftp_file_name) { + std::regex rx(".*(\\d\\d\\d\\d)-(\\d\\d)-(\\d\\d).*"); + std::smatch matches; + if (!std::regex_match(sftp_file_name, matches, rx)) + return {}; + auto y = std::stoi(matches[1]); + auto m = std::stoi(matches[2]); + auto d = std::stoi(matches[3]); + + const auto &sod = config::get(); + try { + return datetime::make_ptime(y, m, d, sod.sod_hh(), sod.has_sod_mm(), 0, + 0); + } catch (std::out_of_range const &) { + return {}; + } +} + +std::optional BasePositionServiceServer::pick_most_recent_pos_file( + std::vector const &candidates) { + + std::vector pos_files; + std::copy_if(candidates.begin(), candidates.end(), + std::back_inserter(pos_files), [](auto const &file_name) { + return boost::algorithm::ends_with(file_name, + ".adatpos.csv"); + }); + if (pos_files.empty()) + return {}; + std::sort(pos_files.begin(), pos_files.end()); + return pos_files.back(); } std::variant diff --git a/src/server/sparkd/position/BasePositionServiceServer.h b/src/server/sparkd/position/BasePositionServiceServer.h index 86853bc6a..1566b89a8 100644 --- a/src/server/sparkd/position/BasePositionServiceServer.h +++ b/src/server/sparkd/position/BasePositionServiceServer.h @@ -32,11 +32,14 @@ class BasePositionServiceServer void start_synch(); enum class State; void transition(State new_state); - std::string get_sftp_file_name(datetime::PTime const&); - void on_download_success(datetime::PTime const &as_of,std::string const &file_path); + std::optional pick_most_recent_pos_file(std::vector const&) ; + std::optional get_as_of_date(std::string const&sftp_file_name); + void on_download_success(std::string const &sftp_file_path, + std::string const &local_file_path); void on_error(std::string const&msg); std::variant parse_gmi_file(std::string const &file_path); + std::string state_name(State) const; struct PImpl; std::unique_ptr me; diff --git a/src/server/sparkd/position/gmi_parser.cpp b/src/server/sparkd/position/gmi_parser.cpp index 04bfa831d..c62e0b16c 100644 --- a/src/server/sparkd/position/gmi_parser.cpp +++ b/src/server/sparkd/position/gmi_parser.cpp @@ -155,7 +155,7 @@ parse_gmi_file(api::admin::AdminData const &admin_data, for (auto c : required_columns) if (column_idx.count(c) == 0) - return fmt::format("Could not find column {} in GMI file at ", c, + return fmt::format("Could not find column {} in GMI file at {}", c, file_path); api::position::BasePositionBatch result;