From a3e4576165f4a1362c7778de81c702b93a09dc98 Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Thu, 15 Aug 2019 16:13:09 -0400 Subject: [PATCH 1/2] Allowed for multiple records for the same security in gmi file. --- .../position/BasePositionServiceServer.cpp | 19 +++-- src/server/sparkd/position/gmi_parser.cpp | 76 ++++++++++++++----- src/server/sparkd/position/gmi_parser.h | 3 + 3 files changed, 74 insertions(+), 24 deletions(-) diff --git a/src/server/sparkd/position/BasePositionServiceServer.cpp b/src/server/sparkd/position/BasePositionServiceServer.cpp index 4c8861528..fccf6a57d 100644 --- a/src/server/sparkd/position/BasePositionServiceServer.cpp +++ b/src/server/sparkd/position/BasePositionServiceServer.cpp @@ -225,8 +225,20 @@ void BasePositionServiceServer::on_download_complete( *base_position.mutable_current_as_of() = utils::pb::to_timestamp(as_of.value()); + + auto reduced_base_pos = reduce_parsed_gmi(base_position); + + LOG(OrderMan, Warn, "Reduced parsed GMI file at {} from {} to {} records", + local_file_path, base_position.positions_size(), + reduced_base_pos.positions_size()); + + for (auto const &bp : reduced_base_pos.positions()) + LOG(OrderMan, Info, "Base position for Account: {} Sec: {} {} {} @ {} ", + bp.account_id(), bp.security_id(), api::Side_Name(bp.side()), + bp.quantity(), bp.avg_price()); + me->pos_service.set_base_position( - base_position, + reduced_base_pos, [this](auto const &rr) { if (!rr.success()) on_position_set_failure(rr.error_message()); @@ -361,10 +373,7 @@ std::optional BasePositionServiceServer::pick_most_recent_pos_file( std::vector pos_files; std::copy_if(candidates.begin(), candidates.end(), std::back_inserter(pos_files), [](auto const &file_name) { - return - boost::algorithm::starts_with(file_name, - "btspos_"); - + return boost::algorithm::starts_with(file_name, "btspos_"); }); if (pos_files.empty()) return {}; diff --git a/src/server/sparkd/position/gmi_parser.cpp b/src/server/sparkd/position/gmi_parser.cpp index 295b4443f..61771d990 100644 --- a/src/server/sparkd/position/gmi_parser.cpp +++ b/src/server/sparkd/position/gmi_parser.cpp @@ -308,28 +308,25 @@ parse_gmi(api::admin::AdminData const &admin_data, std::istream &in) { this_line_position.set_quantity(qty.value()); // Quantity - if (qty.value() == 0) - this_line_position.set_avg_price(0); - else { - - auto gmi_price = values[column_idx["CPRICE"]]; - auto price = map_gmi_price(gmi_price); - if (!price.has_value()) { - auto msg = fmt::format("Aborted parsing on line {} because " - "CPRICE {} is not a number", - line_number, gmi_price); - LOG(OrderMan, Error, "{}", msg); - return msg; - } - - // AVG Price - this_line_position.set_avg_price(price.value() / qty.value()); - // AVG Price + auto gmi_price = values[column_idx["CPRICE"]]; + auto price = map_gmi_price(gmi_price); + if (!price.has_value()) { + auto msg = fmt::format("Aborted parsing on line {} because " + "CPRICE {} is not a number", + line_number, gmi_price); + LOG(OrderMan, Error, "{}", msg); + return msg; } + // Price + this_line_position.set_avg_price(price.value()); + // Price + LOG(OrderMan, Info, - "Successfully parsed line {}. Account: {} Sec: {} {} {} @ {} ", - line_number, bts_account->name(), securities::descriptor(*security), + "Successfully parsed line {}. Account: {} (id: {}) Sec: {} (id: {}) " + "{} {} @ {} ", + line_number, bts_account->name(), bts_account->id(), + securities::descriptor(*security), security->id(), api::Side_Name(side), qty.value(), this_line_position.avg_price()); *result.add_positions() = this_line_position; @@ -343,5 +340,46 @@ parse_gmi(api::admin::AdminData const &admin_data, std::istream &in) { return result; } +api::position::BasePositionBatch +reduce_parsed_gmi(api::position::BasePositionBatch const &in) { + using namespace api::position; + + BasePositionBatch out; + *out.mutable_current_as_of() = in.current_as_of(); + + using Key = std::tuple; + std::map> index; + + for (auto const &bp : in.positions()) { + auto k = std::make_tuple(bp.security_id(), bp.side(), bp.account_id()); + index[k].push_back(bp); + } + + auto reducer = [](std::vector const &in) -> BasePosition { + bts_assert(in.size()); + + uint32_t total_qty = 0; + double total_price = 0; + + for (auto const &bp : in) { + total_qty += bp.quantity(); + total_price += (bp.quantity() * bp.avg_price()); + } + BasePosition result = in[0]; + result.set_quantity(total_qty); + if (total_qty != 0) + result.set_avg_price(total_price / total_qty); + else + result.set_avg_price(0); + return result; + }; + + for (auto const &p : index) { + auto reduced_pos = out.add_positions(); + *reduced_pos = reducer(p.second); + } + return out; +} + } // namespace position } // namespace bts diff --git a/src/server/sparkd/position/gmi_parser.h b/src/server/sparkd/position/gmi_parser.h index 58ef62c18..b02fc18c5 100644 --- a/src/server/sparkd/position/gmi_parser.h +++ b/src/server/sparkd/position/gmi_parser.h @@ -28,5 +28,8 @@ parse_gmi(api::admin::AdminData const &admin_data, std::variant parse_gmi(api::admin::AdminData const &admin_data, std::istream &); +// Accumulate records by account, security, side +api::position::BasePositionBatch reduce_parsed_gmi (api::position::BasePositionBatch const &); + } // namespace position } // namespace bts From cad611104b2e1740df72ba0125c6b352ad51d069 Mon Sep 17 00:00:00 2001 From: Pedro Pinto Date: Thu, 15 Aug 2019 16:37:48 -0400 Subject: [PATCH 2/2] Moved some files around. --- src/server/sparkd/CMakeDeps.cmake | 2 +- src/server/sparkd/cmds/Run.cpp | 2 +- .../{ => import}/BasePositionServiceServer.cpp | 6 +++--- .../{ => import}/BasePositionServiceServer.h | 0 .../sparkd/position/{ => import}/DownloadWindow.cpp | 2 +- .../sparkd/position/{ => import}/DownloadWindow.h | 0 .../{ => import}/docs/download_state_machine.org | 0 .../{ => import}/docs/download_state_machine.pdf | Bin .../position/{ => import}/docs/event_flow.pdf | Bin .../docs/reference/2019-07-08.File_Layout_Money.pdf | Bin .../reference/2019-07-08.File_Layout_Positions.pdf | Bin .../docs/reference/2019-08-07.adatpos.csv | 0 .../docs/reference/2019-08-09.adatmny.csv | 0 .../docs/reference/2019-08-09.adatpos.csv | 0 .../docs/reference/GMI Mapping spreadsheet.xlsx | Bin .../reference/Gmail - Money and Position files.pdf | Bin .../{ => import}/docs/reference/adatmny.docx | Bin .../{ => import}/docs/reference/adatpos.docx | Bin .../reference/sftp_credentials_encrypted.txt.gpg | 0 .../docs/reference/test_positions_20190708.csv | 0 .../position/{ => import/parsing}/gmi_parser.cpp | 0 .../position/{ => import/parsing}/gmi_parser.h | 0 src/server/sparkd/position/tests/PositionTests.cpp | 2 +- 23 files changed, 7 insertions(+), 7 deletions(-) rename src/server/sparkd/position/{ => import}/BasePositionServiceServer.cpp (98%) rename src/server/sparkd/position/{ => import}/BasePositionServiceServer.h (100%) rename src/server/sparkd/position/{ => import}/DownloadWindow.cpp (97%) rename src/server/sparkd/position/{ => import}/DownloadWindow.h (100%) rename src/server/sparkd/position/{ => import}/docs/download_state_machine.org (100%) rename src/server/sparkd/position/{ => import}/docs/download_state_machine.pdf (100%) rename src/server/sparkd/position/{ => import}/docs/event_flow.pdf (100%) rename src/server/sparkd/position/{ => import}/docs/reference/2019-07-08.File_Layout_Money.pdf (100%) rename src/server/sparkd/position/{ => import}/docs/reference/2019-07-08.File_Layout_Positions.pdf (100%) rename src/server/sparkd/position/{ => import}/docs/reference/2019-08-07.adatpos.csv (100%) rename src/server/sparkd/position/{ => import}/docs/reference/2019-08-09.adatmny.csv (100%) rename src/server/sparkd/position/{ => import}/docs/reference/2019-08-09.adatpos.csv (100%) rename src/server/sparkd/position/{ => import}/docs/reference/GMI Mapping spreadsheet.xlsx (100%) rename src/server/sparkd/position/{ => import}/docs/reference/Gmail - Money and Position files.pdf (100%) rename src/server/sparkd/position/{ => import}/docs/reference/adatmny.docx (100%) rename src/server/sparkd/position/{ => import}/docs/reference/adatpos.docx (100%) rename src/server/sparkd/position/{ => import}/docs/reference/sftp_credentials_encrypted.txt.gpg (100%) rename src/server/sparkd/position/{ => import}/docs/reference/test_positions_20190708.csv (100%) rename src/server/sparkd/position/{ => import/parsing}/gmi_parser.cpp (100%) rename src/server/sparkd/position/{ => import/parsing}/gmi_parser.h (100%) diff --git a/src/server/sparkd/CMakeDeps.cmake b/src/server/sparkd/CMakeDeps.cmake index 5e4d0d213..d2adfb403 100644 --- a/src/server/sparkd/CMakeDeps.cmake +++ b/src/server/sparkd/CMakeDeps.cmake @@ -1 +1 @@ -set(sources algo/AlgoDefinitionServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/Help.cpp;cmds/KeyCode.cpp;cmds/Md.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/RunSim.cpp;cmds/Span.cpp;cmds/Test.cpp;cmds/TestCeloxica.cpp;cmds/Web.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/WebServer.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/IMDPlatform.cpp;md/platforms/celoxica/CXListener.cpp;md/platforms/celoxica/CXMain.cpp;md/platforms/celoxica/tests/cx_tests.cpp;md/platforms/ice_onixs/ICEListener.cpp;md/platforms/ice_onixs/ICEMain.cpp;md/platforms/ice_onixs/tests/ice_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/BasePositionServiceServer.cpp;position/DownloadWindow.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/MarginServiceServer.cpp;position/PositionAggregator.cpp;position/PositionMap.cpp;position/PositionServicePNLHelper.cpp;position/PositionServiceServer.cpp;position/gmi_parser.cpp;position/tests/PositionTests.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceServer.cpp;user/AuthServiceServer.cpp;user/tests/admin_service_tests.cpp) \ No newline at end of file +set(sources algo/AlgoDefinitionServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/Help.cpp;cmds/KeyCode.cpp;cmds/Md.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/RunSim.cpp;cmds/Span.cpp;cmds/Test.cpp;cmds/TestCeloxica.cpp;cmds/Web.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/WebServer.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/IMDPlatform.cpp;md/platforms/celoxica/CXListener.cpp;md/platforms/celoxica/CXMain.cpp;md/platforms/celoxica/tests/cx_tests.cpp;md/platforms/ice_onixs/ICEListener.cpp;md/platforms/ice_onixs/ICEMain.cpp;md/platforms/ice_onixs/tests/ice_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/MarginServiceServer.cpp;position/PositionAggregator.cpp;position/PositionMap.cpp;position/PositionServicePNLHelper.cpp;position/PositionServiceServer.cpp;position/import/BasePositionServiceServer.cpp;position/import/DownloadWindow.cpp;position/import/parsing/gmi_parser.cpp;position/tests/PositionTests.cpp;position/tests/position_tests_internal.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceServer.cpp;user/AuthServiceServer.cpp;user/tests/admin_service_tests.cpp) \ No newline at end of file diff --git a/src/server/sparkd/cmds/Run.cpp b/src/server/sparkd/cmds/Run.cpp index 3400bb9cd..d9f9c4944 100644 --- a/src/server/sparkd/cmds/Run.cpp +++ b/src/server/sparkd/cmds/Run.cpp @@ -29,7 +29,7 @@ #include "server/sparkd/http/WebServer.h" #include "server/sparkd/md/MarketDataAuxiliaryServiceServer.h" #include "server/sparkd/md/run_md.h" -#include "server/sparkd/position/BasePositionServiceServer.h" +#include "server/sparkd/position/import/BasePositionServiceServer.h" #include "server/sparkd/position/PositionServiceServer.h" #include "server/sparkd/position/MarginServiceServer.h" #include "server/sparkd/support/ErrorReportService.h" diff --git a/src/server/sparkd/position/BasePositionServiceServer.cpp b/src/server/sparkd/position/import/BasePositionServiceServer.cpp similarity index 98% rename from src/server/sparkd/position/BasePositionServiceServer.cpp rename to src/server/sparkd/position/import/BasePositionServiceServer.cpp index fccf6a57d..bef25a144 100644 --- a/src/server/sparkd/position/BasePositionServiceServer.cpp +++ b/src/server/sparkd/position/import/BasePositionServiceServer.cpp @@ -14,8 +14,8 @@ #include "lib/utils/bts_chrono.h" #include "lib/utils/datetime.h" #include "lib/utils/sftp.h" -#include "server/sparkd/position/DownloadWindow.h" -#include "server/sparkd/position/gmi_parser.h" +#include "server/sparkd/position/import/DownloadWindow.h" +#include "server/sparkd/position/import/parsing/gmi_parser.h" #include #include @@ -105,7 +105,7 @@ void BasePositionServiceServer::parse_gmi_csv( auto base_position = std::get(parse_result); - *result.mutable_success() = base_position; + *result.mutable_success() = reduce_parsed_gmi(base_position); reply(result); return; } diff --git a/src/server/sparkd/position/BasePositionServiceServer.h b/src/server/sparkd/position/import/BasePositionServiceServer.h similarity index 100% rename from src/server/sparkd/position/BasePositionServiceServer.h rename to src/server/sparkd/position/import/BasePositionServiceServer.h diff --git a/src/server/sparkd/position/DownloadWindow.cpp b/src/server/sparkd/position/import/DownloadWindow.cpp similarity index 97% rename from src/server/sparkd/position/DownloadWindow.cpp rename to src/server/sparkd/position/import/DownloadWindow.cpp index c3affb3d4..64d963fed 100644 --- a/src/server/sparkd/position/DownloadWindow.cpp +++ b/src/server/sparkd/position/import/DownloadWindow.cpp @@ -1,4 +1,4 @@ -#include "server/sparkd/position/DownloadWindow.h" +#include "server/sparkd/position/import/DownloadWindow.h" #include "lib/config/config.h" #include "lib/config/protos/position_cfg.pb.h" #include "lib/utils/bts_assert.h" diff --git a/src/server/sparkd/position/DownloadWindow.h b/src/server/sparkd/position/import/DownloadWindow.h similarity index 100% rename from src/server/sparkd/position/DownloadWindow.h rename to src/server/sparkd/position/import/DownloadWindow.h diff --git a/src/server/sparkd/position/docs/download_state_machine.org b/src/server/sparkd/position/import/docs/download_state_machine.org similarity index 100% rename from src/server/sparkd/position/docs/download_state_machine.org rename to src/server/sparkd/position/import/docs/download_state_machine.org diff --git a/src/server/sparkd/position/docs/download_state_machine.pdf b/src/server/sparkd/position/import/docs/download_state_machine.pdf similarity index 100% rename from src/server/sparkd/position/docs/download_state_machine.pdf rename to src/server/sparkd/position/import/docs/download_state_machine.pdf diff --git a/src/server/sparkd/position/docs/event_flow.pdf b/src/server/sparkd/position/import/docs/event_flow.pdf similarity index 100% rename from src/server/sparkd/position/docs/event_flow.pdf rename to src/server/sparkd/position/import/docs/event_flow.pdf diff --git a/src/server/sparkd/position/docs/reference/2019-07-08.File_Layout_Money.pdf b/src/server/sparkd/position/import/docs/reference/2019-07-08.File_Layout_Money.pdf similarity index 100% rename from src/server/sparkd/position/docs/reference/2019-07-08.File_Layout_Money.pdf rename to src/server/sparkd/position/import/docs/reference/2019-07-08.File_Layout_Money.pdf diff --git a/src/server/sparkd/position/docs/reference/2019-07-08.File_Layout_Positions.pdf b/src/server/sparkd/position/import/docs/reference/2019-07-08.File_Layout_Positions.pdf similarity index 100% rename from src/server/sparkd/position/docs/reference/2019-07-08.File_Layout_Positions.pdf rename to src/server/sparkd/position/import/docs/reference/2019-07-08.File_Layout_Positions.pdf diff --git a/src/server/sparkd/position/docs/reference/2019-08-07.adatpos.csv b/src/server/sparkd/position/import/docs/reference/2019-08-07.adatpos.csv similarity index 100% rename from src/server/sparkd/position/docs/reference/2019-08-07.adatpos.csv rename to src/server/sparkd/position/import/docs/reference/2019-08-07.adatpos.csv diff --git a/src/server/sparkd/position/docs/reference/2019-08-09.adatmny.csv b/src/server/sparkd/position/import/docs/reference/2019-08-09.adatmny.csv similarity index 100% rename from src/server/sparkd/position/docs/reference/2019-08-09.adatmny.csv rename to src/server/sparkd/position/import/docs/reference/2019-08-09.adatmny.csv diff --git a/src/server/sparkd/position/docs/reference/2019-08-09.adatpos.csv b/src/server/sparkd/position/import/docs/reference/2019-08-09.adatpos.csv similarity index 100% rename from src/server/sparkd/position/docs/reference/2019-08-09.adatpos.csv rename to src/server/sparkd/position/import/docs/reference/2019-08-09.adatpos.csv diff --git a/src/server/sparkd/position/docs/reference/GMI Mapping spreadsheet.xlsx b/src/server/sparkd/position/import/docs/reference/GMI Mapping spreadsheet.xlsx similarity index 100% rename from src/server/sparkd/position/docs/reference/GMI Mapping spreadsheet.xlsx rename to src/server/sparkd/position/import/docs/reference/GMI Mapping spreadsheet.xlsx diff --git a/src/server/sparkd/position/docs/reference/Gmail - Money and Position files.pdf b/src/server/sparkd/position/import/docs/reference/Gmail - Money and Position files.pdf similarity index 100% rename from src/server/sparkd/position/docs/reference/Gmail - Money and Position files.pdf rename to src/server/sparkd/position/import/docs/reference/Gmail - Money and Position files.pdf diff --git a/src/server/sparkd/position/docs/reference/adatmny.docx b/src/server/sparkd/position/import/docs/reference/adatmny.docx similarity index 100% rename from src/server/sparkd/position/docs/reference/adatmny.docx rename to src/server/sparkd/position/import/docs/reference/adatmny.docx diff --git a/src/server/sparkd/position/docs/reference/adatpos.docx b/src/server/sparkd/position/import/docs/reference/adatpos.docx similarity index 100% rename from src/server/sparkd/position/docs/reference/adatpos.docx rename to src/server/sparkd/position/import/docs/reference/adatpos.docx diff --git a/src/server/sparkd/position/docs/reference/sftp_credentials_encrypted.txt.gpg b/src/server/sparkd/position/import/docs/reference/sftp_credentials_encrypted.txt.gpg similarity index 100% rename from src/server/sparkd/position/docs/reference/sftp_credentials_encrypted.txt.gpg rename to src/server/sparkd/position/import/docs/reference/sftp_credentials_encrypted.txt.gpg diff --git a/src/server/sparkd/position/docs/reference/test_positions_20190708.csv b/src/server/sparkd/position/import/docs/reference/test_positions_20190708.csv similarity index 100% rename from src/server/sparkd/position/docs/reference/test_positions_20190708.csv rename to src/server/sparkd/position/import/docs/reference/test_positions_20190708.csv diff --git a/src/server/sparkd/position/gmi_parser.cpp b/src/server/sparkd/position/import/parsing/gmi_parser.cpp similarity index 100% rename from src/server/sparkd/position/gmi_parser.cpp rename to src/server/sparkd/position/import/parsing/gmi_parser.cpp diff --git a/src/server/sparkd/position/gmi_parser.h b/src/server/sparkd/position/import/parsing/gmi_parser.h similarity index 100% rename from src/server/sparkd/position/gmi_parser.h rename to src/server/sparkd/position/import/parsing/gmi_parser.h diff --git a/src/server/sparkd/position/tests/PositionTests.cpp b/src/server/sparkd/position/tests/PositionTests.cpp index 3bcdc1447..2ad6f9b4e 100644 --- a/src/server/sparkd/position/tests/PositionTests.cpp +++ b/src/server/sparkd/position/tests/PositionTests.cpp @@ -7,7 +7,7 @@ #include #include -#include "server/sparkd/position/DownloadWindow.h" +#include "server/sparkd/position/import/DownloadWindow.h" namespace bts { namespace test {