Compare commits

...
16 changed files with 288 additions and 33 deletions

No files matched your search

+5
View File
@@ -4,6 +4,11 @@ weight = 3
sort_by = "weight"
insert_anchor_links = "right"
+++
## 2.0.3
### Internal changes
- Improved backwards compatibility in UI settings
## 2.0.2
### User Visible Changes
- Middle click the ladder price column to show the orders for that price level
@@ -20,6 +20,27 @@ namespace bts.shell {
readonly IBus _bus;
readonly OrdersModel _orders_model;
readonly UserSettingsHub _settings_hub;
/// Fills 'sorting-criterion' settings were compromised a while back (mid-late 2018).
/// The issue was that, when a column was sorted, its 'sort member path' was assigned to the SortingCriteria.col_name,
/// rather than the <see cref="Columns"/> enum/id string.
/// This dictionary serves as a reverse-lookup so the sort member path can be converted to the enum/id string.
static readonly IReadOnlyDictionary<string, Columns> _sort_member_to_id =
new Dictionary<string, Columns> {
{ nameof(FillsVM.Row.account), Columns.Account },
{ nameof(FillsVM.Row.exchange), Columns.Exchange },
{ nameof(FillsVM.Row.fill_key), Columns.FillKey },
{ nameof(FillsVM.Row.fill_kind), Columns.FillKind },
{ nameof(FillsVM.Row.fill_time), Columns.FillTime },
{ nameof(FillsVM.Row.order_key), Columns.OrderKey },
{ nameof(FillsVM.Row.order_type), Columns.OrderType },
{ nameof(FillsVM.Row.price), Columns.Price },
{ nameof(FillsVM.Row.security), Columns.Security },
{ nameof(FillsVM.Row.security_key), Columns.SecurityKey },
{ nameof(FillsVM.Row.side), Columns.Side },
{ nameof(FillsVM.Row.size), Columns.Size },
{ nameof(FillsVM.Row.time_in_force), Columns.TimeInForce },
{ nameof(FillsVM.Row.user), Columns.User }
};
public FillsWindowFactory(TradingShellModule module, IBus bus, UserSettingsHub settings_hub, OrdersModel orders_model)
: base(module, "Fills") {
@@ -38,6 +59,7 @@ namespace bts.shell {
_bus.post(SparkTags.ShowWindow, await load(state));
IShellWindow _create(StartupData state) {
GridSettingsHelper.normalize_sorting_criterion(state.sorting_criterion, _sort_member_to_id);
var settings_manager = new WindowSettingsManager<GridSettings>(_settings_hub,
state.grid,
get: us => us.fills_grid,
@@ -99,8 +99,11 @@ namespace bts.shell.orderbook {
public OrderBookGridSettings grid_settings_wrapper() {
if (use_defaults)
return null;
if (orderbook_grid == null)
orderbook_grid = new OrderBookGridSettings(grid.clone(), _predicate_animations);
if (orderbook_grid == null) {
if (grid == null)
grid = _default_general_grid;
orderbook_grid = new OrderBookGridSettings(grid.clone(), _predicate_animations);
}
return orderbook_grid;
}
#pragma warning restore CS0612 // Type or member is obsolete
@@ -0,0 +1,27 @@
using System;
using System.Collections.Generic;
using System.Linq;
namespace mbts.ui.settings {
static class GridSettingsHelper {
public static TColKey? parse_col_id<TColKey>(string id) where TColKey : struct =>
Enum.TryParse(id, true, out TColKey key) ? key : (TColKey?)null;
public static string fmt_col_id<TColKey>(TColKey key) where TColKey : struct =>
key.ToString();
public static void normalize_sorting_criterion<TColKey>(IEnumerable<SortingCriteria> sorting_criterion, IReadOnlyDictionary<string, TColKey> sort_member_to_id) where TColKey : struct =>
normalize_sorting_criterion(sorting_criterion, new Func<string, TColKey?>[] {
parse_col_id<TColKey>,
col_name => sort_member_to_id.TryGetValue(col_name, out var k) ? k : (TColKey?)null
});
public static void normalize_sorting_criterion<TColKey>(IEnumerable<SortingCriteria> sorting_criterion, IEnumerable<Func<string, TColKey?>> conversion_dels) where TColKey : struct {
foreach (var sc in sorting_criterion) {
sc.col_name = conversion_dels
.Select(del => del(sc.col_name))
.FirstOrDefault(k => k.HasValue)?.ToString() ?? sc.col_name;
}
}
}
}
@@ -1,5 +1,8 @@
using bts.utils;
using bts;
using bts.collections;
using bts.utils;
using mbts.collections;
using mbts.ui.settings;
using mbts.ui.utility;
using mbts.utils;
using System;
@@ -51,8 +54,17 @@ namespace mbts.ui.controls {
).to_comparer();
}
public void load_from(IEnumerable<SortingCriteria> sorting_criterion) =>
_on_settings(sorting_criterion.Select(sc => (EnumHelper.parse<TColKey>(sc.col_name), sc.is_ascending ? ListSortDirection.Ascending : ListSortDirection.Descending)));
public void load_from(IEnumerable<SortingCriteria> sorting_criterion) {
var sort_member_to_id = _map
.Where(kv => !string.IsNullOrEmpty(kv.Value.col.sort_member_path))
.Distinct()
.ToDictionary(kv => kv.Value.col.sort_member_path, kv => kv.Key);
GridSettingsHelper.normalize_sorting_criterion(sorting_criterion, sort_member_to_id);
_on_settings(sorting_criterion
.Select(sc => (GridSettingsHelper.parse_col_id<TColKey>(sc.col_name), sc.is_ascending ? ListSortDirection.Ascending : ListSortDirection.Descending))
.Where(t => t.Item1.HasValue)
.Select(t => (t.Item1.Value, t.Item2)));
}
public IEnumerable<SortingCriteria> create() =>
_active_cols.AsCollection().Select(t => new SortingCriteria(t.col.id, t.direction == ListSortDirection.Ascending));
@@ -22,31 +22,71 @@ namespace bts.settings {
ConsoleEx.WriteLine("Workspace Descriptions", ConsoleColor.Cyan);
foreach(var w in workspaces) {
var r = from ws in w.value.windows
group ws by ws.owner into grouping
select new { key = grouping.Key.Replace("bts.shell.", ""), window_count = grouping.Count() };
string desc = string.Empty;
foreach(var i in r) {
desc += $"{Environment.NewLine}\t{i.window_count} - {i.key}";
// if a workspace doesn't have any windows, we don't care about it
if (w.value.windows != null) {
var r = from ws in w.value.windows
group ws by ws.owner into grouping
select new { key = grouping.Key.Replace("bts.shell.", ""), window_count = grouping.Count() };
string desc = string.Empty;
foreach (var i in r) {
desc += $"{Environment.NewLine}\t{i.window_count} - {i.key}";
}
ConsoleEx.Write($"{w.key}", ConsoleColor.Yellow);
ConsoleEx.WriteLine($" - {w.value.windows?.Count} windows - {w.value.named_windows?.Count} named windows - {desc}");
}
ConsoleEx.Write($"{w.key}", ConsoleColor.Yellow);
ConsoleEx.WriteLine($" - {w.value.windows.Count} windows - {w.value.named_windows.Count} named windows - {desc}");
}
ConsoleEx.WriteLine("");
var selected_ws = ConsoleEx.prompt("Choose workspace:", workspaces.Select(w => w.key));
var to = ConsoleEx.prompt("To User", user_names);
string copy = "Copy to another user",
delete = "Delete it",
export = "Export to file";
var name = ConsoleEx.input_prompt("Rename workspace?", selected_ws);
var action = ConsoleEx.prompt($"What do you want to do with '{selected_ws}'", copy, delete, export);
var confirm = ConsoleEx.prompt($"copy workspace '{selected_ws}' from {from} to {to} and name it '{name}'. Confirm?", "Yes", "No");
if (confirm == "Yes") {
var ws_blob = workspaces.First(w => w.key == selected_ws).value;
var to_persist = new UserUISettings(user_list.Users.First(u => u.Login == to).Id, persist, null);
to_persist.set(WorkspacesModel.WORKSPACE_NAMESPACE, name, ws_blob).Wait();
Console.WriteLine("Done");
if (action == copy) {
RunCopy();
}
else if (action == delete) {
var confirm = ConsoleEx.prompt($"Delete workspace `{selected_ws}` from {from}. Confirm?", "Yes", "No");
if (confirm == "Yes") {
var ws = workspaces.First(w => w.key == selected_ws);
from_persist.clear(ws.ns, ws.key).Wait();
Console.WriteLine("Done");
}
}
else if (action == export) {
var name = ConsoleEx.input_prompt("Filename: ", selected_ws + ".workspace");
if (System.IO.File.Exists(name)) {
var confirm = ConsoleEx.prompt($"'{name}' exists. Overwrite?", "Yes", "No");
if (confirm == "No") {
return;
}
}
var ws_blob = workspaces.First(w => w.key == selected_ws).value;
using (var file = System.IO.File.OpenWrite(name)) {
ProtoBuf.Serializer.SerializeWithLengthPrefix(file, ws_blob, ProtoBuf.PrefixStyle.Fixed32);
}
Console.WriteLine($"Export '{selected_ws}' to '{name}'. Done.");
}
// End function
void RunCopy() {
var to = ConsoleEx.prompt("To User", user_names);
var name = ConsoleEx.input_prompt("Rename workspace?", selected_ws);
var confirm = ConsoleEx.prompt($"copy workspace '{selected_ws}' from {from} to {to} and name it '{name}'. Confirm?", "Yes", "No");
if (confirm == "Yes") {
var ws_blob = workspaces.First(w => w.key == selected_ws).value;
var to_persist = new UserUISettings(user_list.Users.First(u => u.Login == to).Id, persist, null);
to_persist.set(WorkspacesModel.WORKSPACE_NAMESPACE, name, ws_blob).Wait();
Console.WriteLine("Done");
}
}
}
}
}
+6 -6
View File
@@ -83,7 +83,7 @@ namespace bts.tests.utils {
public static async Task wait_for_system_ready(TimeSpan? timeout = null) {
if(timeout == null) {
timeout = TimeSpan.FromSeconds(120000);
timeout = TimeSpan.FromMinutes(2);
}
System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
@@ -104,10 +104,7 @@ namespace bts.tests.utils {
// Console.WriteLine("Admin Service Ready: " + admin_ready);
// bts.test.Assert.IsTrue(admin_ready, "System is not ready");
Console.WriteLine("System Ready after: " + sw.Elapsed.TotalSeconds + " seconds");
bts.test.Assert.IsTrue(orders_ready, "System is not ready");
}
public static async Task<bool> wait_for_shell_latency_service_ready(TimeSpan timeout) {
@@ -151,13 +148,16 @@ namespace bts.tests.utils {
});
if(service_value == null) {
Console.WriteLine("Waiting for order service.");
await Task.Delay(10000);
if(watch.Elapsed.TotalSeconds < 3 || watch.Elapsed.TotalSeconds % 20 < 1) {
Console.WriteLine("Waiting for order service...");
}
await Task.Delay(1000);
}
sub.Dispose();
context.Dispose();
}
return service_value != null;
}
+1
View File
@@ -24,6 +24,7 @@ namespace bts {
}
static void dump_mbo(uint key, api.Side side) {
// todo: fix me
using (var ctx = new ServiceContext()) {
var mbo = ctx.locate<MBOService>();
var data = mbo.get_orders_for_security(new MBOSecurityQuery { SecurityId = key });
+1 -1
View File
@@ -5,7 +5,7 @@ message ExchangeSim {
optional bool merge_with_md = 1 [default=true];
optional bool forward_implied = 2 [default = false];
oneof source {
string replay_url = 11 [default = "http://mdreplay.bluetradesys.local/spark/futures.coconut"];
string replay_url = 11 [default = "http://10.96.18.45/spark/futures.coconut"];
string replay_file = 12 [default = "futures.coconut"];
bool local_relay = 13;
}
+2 -1
View File
@@ -190,7 +190,8 @@ message RecordOptions {
// Replay options
//
message ReplayOptions {
optional string url = 1 [default = "http://mdreplay.bluetradesys.local/spark/futures.coconut"];
// Tired of fighting with dns
optional string url = 1 [default = "http://10.96.18.45/spark/futures.coconut"];
// 2.0 = double speed, 0.5 = half speed
optional double speed = 11 [default = 1.0];
+30 -1
View File
@@ -3,6 +3,7 @@
#include "lib/config/config.h"
#include "lib/config/protos/toggles.pb.h"
#include "lib/utils/thread_manager.h"
#include "lib/utils/LockBox.h"
#include <map>
#include <functional>
@@ -63,6 +64,7 @@ void print_pool_stats(const utils::ConcurrentPool<MBOSnapshot>& pool) {
}
typedef tbb::concurrent_hash_map<SecurityKey, MBOSnapshot*, SecurityKeyHashCompare> MapType;
// typedef std::unordered_map<SecurityKey, MBOSnapshot*> MapType;
struct Incremental {
MBOWorkingState* state;
inc::Update update;
@@ -71,6 +73,7 @@ struct MBOProcessorImpl {
utils::ConcurrentPool<MBOSnapshot> pool;
tbb::concurrent_queue<Incremental> incremental_input;
MapType snapshots;
// utils::LockBox<MapType> snapshots;
bool running;
utils::TimeStamp last_pool_print;
std::thread thread;
@@ -85,7 +88,6 @@ struct MBOProcessorImpl {
if (config::get<config::toggles::MBOStats>().enabled()) {
last_pool_print = utils::TimeStamp::now();
}
}
~MBOProcessorImpl() {
@@ -94,14 +96,25 @@ struct MBOProcessorImpl {
}
void publish_snapshot(SecurityKey key, MBOSnapshot** snapshot) {
LOG(md,Diag, "publish_snapshot/begin/{}", key.value());
bts_assert(*snapshot != nullptr);
MapType::accessor accessor;
snapshots.insert(accessor, key);
std::swap(accessor->second, *snapshot);
if (*snapshot != nullptr) {
// Note: relase() must happen inside accessor lifetime to ensure correctness
LOG(md,Diag, "publish_snapshot/release/{}", key.value());
pool.release(snapshot);
}
LOG(md,Diag, "publish_snapshot/end/{}", key.value());
// snapshots.lock([&](MapType& map) {
// std::swap(map[key], *snapshot);
// // Note: relase() must happen inside accessor lifetime to ensure correctness
// if (*snapshot != nullptr) {
// pool.release(snapshot);
// }
// });
}
void publish_incremental(MBOWorkingState& state) {
@@ -139,6 +152,12 @@ struct MBOProcessorImpl {
}
// publish a small batch of snapshots (then check queue again)
auto sz = dirty.size();
if (sz > 0) {
LOG(md,Diag, "process_incremental/publish/sz={}", sz);
}
for (auto iter = dirty.begin(); iter != dirty.end(); ++iter) {
publish_incremental(**iter);
clean.push_back(*iter);
@@ -276,6 +295,7 @@ void apply_incremental(uint32_t security_key, C& collection, std::unordered_map<
}
}
void apply_incremental(MBOWorkingState& state, const inc::Update& update) {
LOG(md,Diag, "apply_incremental/begin/{}", state.handle.key());
// apply the update to the working state
if (update.type == inc::UpdateType::Bid) {
apply_incremental(state.handle.key(), state.bid, state.orderid_index, update);
@@ -300,6 +320,15 @@ MBOSnapshot* MBOReader::acquire_snapshot(SecurityKey key) const {
else {
release_snapshot(&result);
}
// _me->snapshots.lock([&](const MapType& map) {
// auto iter = map.find(key);
// if (iter != map.end()) {
// result->copy_from(*iter->second);
// }
// else {
// release_snapshot(&result);
// }
// });
return result;
}
void MBOReader::release_snapshot(MBOSnapshot**s) const {
+2 -2
View File
@@ -136,9 +136,9 @@ class MBOWriter {
std::shared_ptr<MBOProcessorImpl> _me;
friend class MBOProcessor;
MBOWriter(std::shared_ptr<MBOProcessorImpl>&);
~MBOWriter();
public:
~MBOWriter();
//
// Snapshot api
MBOSnapshot* allocate_snapshot(size_t bid_count=0, size_t ask_count=0);
@@ -158,8 +158,8 @@ class MBOReader {
friend class MBOProcessor;
friend class MBOWriter;
MBOReader(std::shared_ptr<MBOProcessorImpl>&);
~MBOReader();
public:
~MBOReader();
MBOSnapshot* acquire_snapshot(SecurityKey key) const;
void release_snapshot(MBOSnapshot**) const;
};
+9 -1
View File
@@ -309,8 +309,8 @@ void MBOServiceServer::get_orders_for_security(const MBOSecurityQuery& request,
}
size_t count = 0;
auto tick = handle.get_minimum_tick();
if(handle.is_initialized()) {
auto tick = handle.get_minimum_tick();
auto mbo = handle.get_mbo();
if (mbo) {
for (const auto& o : mbo->bids) {
@@ -346,7 +346,15 @@ void MBOServiceServer::get_orders_for_security(const MBOSecurityQuery& request,
}
}
}
else {
LOG(md,Diag,"get_orders_for_security/fail/null/{}", handle.key());
}
}
else {
LOG(md,Diag,"get_orders_for_security/fail/uninitialized/{}", handle.key());
}
LOG(md,Diag,"get_orders_for_security/{}/{}b/{}a/{}", handle.key(), response.bid_size(), response.ask_size(), request.price_level());
reply(response);
}
+3
View File
@@ -379,6 +379,9 @@ std::unique_ptr<MBOSnapshot,std::function<void(MBOSnapshot*)>> MDConsumerHandle:
if (snapshot != nullptr) {
return {snapshot,[mbo=book.mbo](auto* s) { mbo->release_snapshot(&s); }};
}
else {
LOG(md,Diag,"get_mbo/fail/{}", key());
}
}
return nullptr;
}
+89
View File
@@ -2,10 +2,12 @@
#include "catch/catch.hpp"
#include "../IntPrice.h"
#include "../MBOProcessor.h"
#include "../MBOServiceServer.h"
#include "Co2TestInstrument.h"
#include <random>
#include <algorithm>
#include <thread>
namespace bts {
namespace md {
@@ -316,5 +318,92 @@ TEST_CASE("MBO change price and size", "[md][mbo]") {
REQUIRE(!not_ask);
}
TEST_CASE("MBO stress", "[md][mbo]") {
/*
This test attempts to shake out concurrency issues with MBO processing
If it fails DO NOT ignore it
*/
MBOProcessor p;
bool running = true;
std::vector<SecurityKey> keys;
for (auto x = 0; x < 157; ++x) {
keys.push_back(SecurityKey(x));
}
auto writer = p.writer();
size_t write_count = 0;
size_t wtime = 0;
size_t wcount = 0;
std::thread twriter([&]() {
size_t count = 0;
while(running) {
auto start = utils::TimeStamp::now();
for (const auto& k : keys) {
++count;
auto s = writer.allocate_snapshot();
REQUIRE(s);
for (int t = 0; t < 15000; ++t) {
s->bids.push_back({ ExchangeOrderKey(count%1024 + t), { 101, -2 }, 42 });
s->asks.push_back({ ExchangeOrderKey(count%1024 + 10000 + t), { 111, -2 }, 32 });
++write_count;
}
writer.publish_snapshot(k, &s);
}
auto end = utils::TimeStamp::now();
++wcount;
wtime += (end - start).milliseconds();
}
});
// Ensure writer has written each once
while(wcount < 1)
std::this_thread::sleep_for(std::chrono::milliseconds(10));
std::cout << "wcount: " << wcount << " - time: " << (wtime/wcount) << std::endl;
auto reader = p.reader();
std::vector<std::thread> readers;
size_t time = 0;
size_t count = 0;
size_t nullcount = 0;
for (int x = 0; x<1; ++x) {
readers.push_back(std::thread([&]() {
while(running) {
auto start = utils::TimeStamp::now();
for (const auto& k : keys) {
auto s = reader.acquire_snapshot(k);
if (s == nullptr) nullcount++;
(void)s;
reader.release_snapshot(&s);
// REQUIRE(s);
}
auto end = utils::TimeStamp::now();
++count;
time += (end - start).microseconds();
}
}));
}
std::this_thread::sleep_for(std::chrono::seconds(10));
running = false;
twriter.join();
for (auto& t : readers) t.join();
std::cout << "Write time: " << wtime/wcount << std::endl;
std::cout << "Read time: " << time/count << std::endl;
std::cout << "nullcount: " << nullcount << std::endl;
REQUIRE(nullcount == 0);
/*
Next step:
- Replace concurrent_hash with Lockbox<Unordered> and see if we can still make it happen
*/
}
}}}
#endif
+15
View File
@@ -24,6 +24,18 @@ class ConcurrentPool {
delete [] b;
}
#if 0
T* allocate() {
return new T();
}
void reserve(size_t ) {
}
void release(T** item) {
if (*item) {
delete *item;
}
}
#else
T* allocate() {
T* result = nullptr;
++_outstanding;
@@ -46,6 +58,7 @@ class ConcurrentPool {
_all.push_back(buffer);
}
void release(T** item) {
if (*item) {
(*item)->reset();
@@ -55,6 +68,8 @@ class ConcurrentPool {
}
}
#endif
template <class CB>
void unsafe_iterate(CB cb) const {
for (auto iter = _all.begin(); iter != _all.end(); ++iter) {