Merge branch 'SP-91-position-migration' of bitbucket.org:bluetradesys/spark into SP-91-position-migration

This commit is contained in:
Taha Afzal committed 2019-08-15 23:43:59 -04:00
commit 69208bf347
23 files changed
+81 -31

No files matched your search

+1 -1
View File
@@ -1 +1 @@
set(sources algo/AlgoDefinitionServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/Help.cpp;cmds/KeyCode.cpp;cmds/Md.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/RunSim.cpp;cmds/Span.cpp;cmds/Test.cpp;cmds/TestCeloxica.cpp;cmds/Web.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/WebServer.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/IMDPlatform.cpp;md/platforms/celoxica/CXListener.cpp;md/platforms/celoxica/CXMain.cpp;md/platforms/celoxica/tests/cx_tests.cpp;md/platforms/ice_onixs/ICEListener.cpp;md/platforms/ice_onixs/ICEMain.cpp;md/platforms/ice_onixs/tests/ice_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/BasePositionServiceServer.cpp;position/DownloadWindow.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/MarginServiceServer.cpp;position/PositionAggregator.cpp;position/PositionMap.cpp;position/PositionServicePNLHelper.cpp;position/PositionServiceServer.cpp;position/gmi_parser.cpp;position/tests/PositionTests.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceServer.cpp;user/AuthServiceServer.cpp;user/tests/admin_service_tests.cpp)
set(sources algo/AlgoDefinitionServiceServer.cpp;cmds/AdminReport.cpp;cmds/AutoCert.cpp;cmds/Benchmark.cpp;cmds/CmdRegistry.cpp;cmds/ConfigCheck.cpp;cmds/Help.cpp;cmds/KeyCode.cpp;cmds/Md.cpp;cmds/PrintHedgeOrders.cpp;cmds/QuickTest.cpp;cmds/Run.cpp;cmds/RunSim.cpp;cmds/Span.cpp;cmds/Test.cpp;cmds/TestCeloxica.cpp;cmds/Web.cpp;dspark/mirage/quick_queue/quick_queue.cpp;exchange_sim/ExchangeSim.cpp;exchange_sim/ExchangeSimApplication.cpp;exchange_sim/book/Book.cpp;exchange_sim/book/ExchangeSim.proto;exchange_sim/book/Order.cpp;exchange_sim/book/OrderMatcher.cpp;exchange_sim/book/book_transform.cpp;exchange_sim/tests/test_book_transform.cpp;http/ApiGateway.cpp;http/WebServer.cpp;main.cpp;md/ChannelVolume.cpp;md/MarketDataAuxiliaryServiceServer.cpp;md/VolumeAtPriceState.cpp;md/platforms/IMDPlatform.cpp;md/platforms/celoxica/CXListener.cpp;md/platforms/celoxica/CXMain.cpp;md/platforms/celoxica/tests/cx_tests.cpp;md/platforms/ice_onixs/ICEListener.cpp;md/platforms/ice_onixs/ICEMain.cpp;md/platforms/ice_onixs/tests/ice_tests.cpp;md/platforms/mdp3_onixs/MDP3Listener.cpp;md/platforms/mdp3_onixs/MDP3Main.cpp;md/platforms/mdp3_onixs/tests/mdp3_tests.cpp;md/run_md.cpp;position/IPositionObserver.cpp;position/MarginServiceCore.cpp;position/MarginServiceServer.cpp;position/PositionAggregator.cpp;position/PositionMap.cpp;position/PositionServicePNLHelper.cpp;position/PositionServiceServer.cpp;position/import/BasePositionServiceServer.cpp;position/import/DownloadWindow.cpp;position/import/parsing/gmi_parser.cpp;position/tests/PositionTests.cpp;position/tests/position_tests_internal.cpp;support/ErrorReportService.cpp;support/ShellLatencyServer.cpp;user/AdminServiceServer.cpp;user/AuthServiceServer.cpp;user/tests/admin_service_tests.cpp)
+1 -1
View File
@@ -29,7 +29,7 @@
#include "server/sparkd/http/WebServer.h"
#include "server/sparkd/md/MarketDataAuxiliaryServiceServer.h"
#include "server/sparkd/md/run_md.h"
#include "server/sparkd/position/BasePositionServiceServer.h"
#include "server/sparkd/position/import/BasePositionServiceServer.h"
#include "server/sparkd/position/PositionServiceServer.h"
#include "server/sparkd/position/MarginServiceServer.h"
#include "server/sparkd/support/ErrorReportService.h"
@@ -14,8 +14,8 @@
#include "lib/utils/bts_chrono.h"
#include "lib/utils/datetime.h"
#include "lib/utils/sftp.h"
#include "server/sparkd/position/DownloadWindow.h"
#include "server/sparkd/position/gmi_parser.h"
#include "server/sparkd/position/import/DownloadWindow.h"
#include "server/sparkd/position/import/parsing/gmi_parser.h"
#include <boost/algorithm/string/predicate.hpp>
#include <boost/date_time/gregorian/gregorian.hpp>
@@ -105,7 +105,7 @@ void BasePositionServiceServer::parse_gmi_csv(
auto base_position =
std::get<api::position::BasePositionBatch>(parse_result);
*result.mutable_success() = base_position;
*result.mutable_success() = reduce_parsed_gmi(base_position);
reply(result);
return;
}
@@ -225,8 +225,20 @@ void BasePositionServiceServer::on_download_complete(
*base_position.mutable_current_as_of() =
utils::pb::to_timestamp(as_of.value());
auto reduced_base_pos = reduce_parsed_gmi(base_position);
LOG(OrderMan, Warn, "Reduced parsed GMI file at {} from {} to {} records",
local_file_path, base_position.positions_size(),
reduced_base_pos.positions_size());
for (auto const &bp : reduced_base_pos.positions())
LOG(OrderMan, Info, "Base position for Account: {} Sec: {} {} {} @ {} ",
bp.account_id(), bp.security_id(), api::Side_Name(bp.side()),
bp.quantity(), bp.avg_price());
me->pos_service.set_base_position(
base_position,
reduced_base_pos,
[this](auto const &rr) {
if (!rr.success())
on_position_set_failure(rr.error_message());
@@ -361,10 +373,7 @@ std::optional<std::string> BasePositionServiceServer::pick_most_recent_pos_file(
std::vector<std::string> pos_files;
std::copy_if(candidates.begin(), candidates.end(),
std::back_inserter(pos_files), [](auto const &file_name) {
return
boost::algorithm::starts_with(file_name,
"btspos_");
return boost::algorithm::starts_with(file_name, "btspos_");
});
if (pos_files.empty())
return {};
@@ -1,4 +1,4 @@
#include "server/sparkd/position/DownloadWindow.h"
#include "server/sparkd/position/import/DownloadWindow.h"
#include "lib/config/config.h"
#include "lib/config/protos/position_cfg.pb.h"
#include "lib/utils/bts_assert.h"
File renamed without changes.
Can't render this file because it is too large.
File renamed without changes.
Can't render this file because it is too large.
@@ -308,28 +308,25 @@ parse_gmi(api::admin::AdminData const &admin_data, std::istream &in) {
this_line_position.set_quantity(qty.value());
// Quantity
if (qty.value() == 0)
this_line_position.set_avg_price(0);
else {
auto gmi_price = values[column_idx["CPRICE"]];
auto price = map_gmi_price(gmi_price);
if (!price.has_value()) {
auto msg = fmt::format("Aborted parsing on line {} because "
"CPRICE {} is not a number",
line_number, gmi_price);
LOG(OrderMan, Error, "{}", msg);
return msg;
}
// AVG Price
this_line_position.set_avg_price(price.value() / qty.value());
// AVG Price
auto gmi_price = values[column_idx["CPRICE"]];
auto price = map_gmi_price(gmi_price);
if (!price.has_value()) {
auto msg = fmt::format("Aborted parsing on line {} because "
"CPRICE {} is not a number",
line_number, gmi_price);
LOG(OrderMan, Error, "{}", msg);
return msg;
}
// Price
this_line_position.set_avg_price(price.value());
// Price
LOG(OrderMan, Info,
"Successfully parsed line {}. Account: {} Sec: {} {} {} @ {} ",
line_number, bts_account->name(), securities::descriptor(*security),
"Successfully parsed line {}. Account: {} (id: {}) Sec: {} (id: {}) "
"{} {} @ {} ",
line_number, bts_account->name(), bts_account->id(),
securities::descriptor(*security), security->id(),
api::Side_Name(side), qty.value(), this_line_position.avg_price());
*result.add_positions() = this_line_position;
@@ -343,5 +340,46 @@ parse_gmi(api::admin::AdminData const &admin_data, std::istream &in) {
return result;
}
api::position::BasePositionBatch
reduce_parsed_gmi(api::position::BasePositionBatch const &in) {
using namespace api::position;
BasePositionBatch out;
*out.mutable_current_as_of() = in.current_as_of();
using Key = std::tuple<uint32_t, api::Side, uint32_t>;
std::map<Key, std::vector<BasePosition>> index;
for (auto const &bp : in.positions()) {
auto k = std::make_tuple(bp.security_id(), bp.side(), bp.account_id());
index[k].push_back(bp);
}
auto reducer = [](std::vector<BasePosition> const &in) -> BasePosition {
bts_assert(in.size());
uint32_t total_qty = 0;
double total_price = 0;
for (auto const &bp : in) {
total_qty += bp.quantity();
total_price += (bp.quantity() * bp.avg_price());
}
BasePosition result = in[0];
result.set_quantity(total_qty);
if (total_qty != 0)
result.set_avg_price(total_price / total_qty);
else
result.set_avg_price(0);
return result;
};
for (auto const &p : index) {
auto reduced_pos = out.add_positions();
*reduced_pos = reducer(p.second);
}
return out;
}
} // namespace position
} // namespace bts
@@ -28,5 +28,8 @@ parse_gmi(api::admin::AdminData const &admin_data,
std::variant<api::position::BasePositionBatch, std::string>
parse_gmi(api::admin::AdminData const &admin_data, std::istream &);
// Accumulate records by account, security, side
api::position::BasePositionBatch reduce_parsed_gmi (api::position::BasePositionBatch const &);
} // namespace position
} // namespace bts
@@ -7,7 +7,7 @@
#include <memory>
#include <stdlib.h>
#include "server/sparkd/position/DownloadWindow.h"
#include "server/sparkd/position/import/DownloadWindow.h"
namespace bts {
namespace test {